"""
Module that allows to work with Plesk backup XML files from Plesk backup archive of different versions:
- read Plesk backup XML as an object model
- modify Plesk backup XML files and store them in a new Plesk backup archive
"""
import gzip
import itertools
import os
import re
import shutil
import tarfile
import zipfile

try:
    import itertools.imap as map
except ImportError:
    pass

from contextlib import closing
from xml.etree import ElementTree

from parallels.core.logging import get_logger
from parallels.core import messages
from parallels.core.dump.entity.description import Description
from parallels.core.utils.common import (
    if_not_none, parse_bool, obj, group_by_id, exists as utils_exists, is_run_on_windows, open_no_inherit
)
from parallels.core.utils.common import unique_merge, find_first, default
from parallels.core.utils.common.xml import elem, text_elem, create_xml_stream
from parallels.core.utils.common.xml import multiple_findall
from parallels.core.utils.common.xml import multiple_attrib_get
from parallels.core.dump.entity.reseller import Reseller
from parallels.core.dump.data_model import (
    Client, AuxiliaryUser, AuxiliaryUserRole, DatabaseServerOverXML,
    DnsRecord, Subscription, HostingPlan, AddonDomain, Subdomain, DomainAlias,
    parse_subscription_ips, Mailbox, Mailbox8, DnsSettings,
    ResellerPlan, Subscription8, Subdomain8,
    parse_php_settings_node, parse_logrotation_node, parse_ip,
    parse_system_ip, CustomPlanItem
)
from parallels.core.dump.entity.password import get_password
from parallels.core.dump.file_adapters import (
    TarFileAdapter, ZipFileAdapter, Plesk8UnixBackupFileAdapter, PlainFileAdapter
)
from parallels.core.utils import unique_name_generator


logger = get_logger(__name__)


def load(path, migration_list_data=None, is_expand_mode=False, discard_mailsystem=False):
    """
    is_expand_node defines how resellers filtering should work:
    - While migrating from Plesk standalone, Plesk reseller is migrated as target reseller.
    Reseller is migrated only if it is in migration list.
    - While migrating from Plesk in Expand cluster, Plesk reseller is migrated as target client.
    Reseller is migrated only if it has at least one own subscription.

    Implementation detail: additionally in Expand mode we iterate over reseller if its clients have subscriptions
    no matter if it has own subscriptions.
    """
    container = PleskBackupContainer(path, discard_mailsystem)
    if isinstance(container.archive, PlainFileAdapter):
        # For single XML file, use Plesk 10 reader,
        # to read all objects from single file,
        return PleskBackupSource10(container, migration_list_data, is_expand_mode)
    elif container.dump_version >= (11, 1):
        return PleskBackupSource11(container, migration_list_data, is_expand_mode)
    elif container.dump_version[0] in (10, 11):
        return PleskBackupSource10(container, migration_list_data, is_expand_mode)
    elif container.dump_version[0] == 9:
        return PleskBackupSource9(container, migration_list_data, is_expand_mode)
    elif container.dump_version[0] == 8:
        return PleskBackupSource8(container, migration_list_data)
    else:
        raise Exception(messages.ERROR_UNSUPPORTED_DUMP_VERSION % (container.dump_version,))


class PleskBackupContainer(object):
    """
    :type archive: parallels.core.dump.file_adapters.BaseArchiveFileAdapter
    """
    def __init__(self, file_name, discard_mailsystem=False):
        self._discard_mailsystem = discard_mailsystem
        if file_name.endswith('.xml'):
            self._load_from_plain_xml_file(file_name)
        else:
            self._load_from_archive(discard_mailsystem, file_name)

        self._filename = file_name
        self.dump_version = tuple(map(int, self.backup_info.getroot().attrib['dump-version'].split('.')))
        self.agent_name = self.backup_info.getroot().attrib['agent-name']

    @property
    def filename(self):
        """Name of a file on migrator server where this backup container is located"""
        return self._filename

    def _load_from_plain_xml_file(self, file_name):
        logger.debug(messages.DEBUG_OPENING_DUMP, file_name)
        self.archive = PlainFileAdapter(file_name)
        with open_no_inherit(file_name, 'rb') as fp:
            self.backup_info = ElementTree.ElementTree(file=fp)
        self.backup_info_file = file_name
        self.suffix = None
        self.backup_id = None

    def _load_from_archive(self, discard_mailsystem, file_name):
        # Don't use zipfile.is_zipfile to detect ZIP archive, because it will
        # find ZIP archive within uncompressed TAR archive and happily report
        # that file is ZIP arhive.

        if tarfile.is_tarfile(file_name):
            logger.debug(messages.READING_DUMP_FROM_TAR_ARCHIVE_S % file_name)
            self.archive = TarFileAdapter(file_name)
            backup_info_re = re.compile(r'(?:\S*info_)?(\d*)\.xml')
        elif zipfile.is_zipfile(file_name):
            logger.debug(messages.READING_DUMP_FROM_ZIP_ARCHIVE_S % file_name)
            self.archive = ZipFileAdapter(file_name)
            backup_info_re = re.compile(r'(?:\S*info_)?(\d*)\.xml')
        else:
            logger.debug(messages.READING_DUMP_FROM_PLESK_BACKUP_FILE % file_name)
            self.archive = Plesk8UnixBackupFileAdapter(file_name)
            backup_info_re = re.compile(r'dump()().xml')

        # Find main backup info file.
        xml_files = {
            m.group(0): m.group(1)
            for m in map(backup_info_re.match, self.archive.get_names())
            if m is not None
        }
        """
        xml_files contains all xml-files from archive:

        {
            'resellers/ivanov/clients/bob/backup_bob_info_1201180356.xml':
                '1201180356',
            'clients/cat/backup_cat_info_1201180356.xml':
                '1201180356',
            'clients/tom/domains/tom.abaturin.lp.plesk.ru/backup_tom.abaturin.lp.plesk.ru_info_1201180356.xml':
                '1201180356',
            'resellers/ivanov/backup_ivanov_info_1201180356.xml':
                '1201180356',
            'clients/tom/backup_tom_info_1201180356.xml':
                '1201180356',
            'clients/cat/domains/cat.abaturin.lp.plesk.ru/backup_cat.abaturin.lp.plesk.ru_info_1201180356.xml':
                '1201180356',
            'backup_info_1201180356.xml':
                '1201180356',
            'clients/john/domains/john.tld/backup_john.tld_info_1201180356.xml':
                '1201180356',
            'resellers/ivanov/clients/bob/domains/bob.tld/backup_bob.tld_info_1201180356.xml':
                '1201180356',
            'clients/john/backup_john_info_1201180356.xml':
                '1201180356'
        }

        We need to select top level backup. It always has shortest name (e.g. backup_info_1201180356.xml) because it is
        located in archive's root but all other XML is placed in subfolders
        """
        self.backup_info_file = min(xml_files.keys(), key=lambda x: len(x))
        self.backup_id = xml_files[self.backup_info_file]
        self.backup_prefix = self.backup_info_file[:self.backup_info_file.find('_')]

        self.backup_info = self._load_xml(self.backup_info_file)
        if discard_mailsystem:
            for mailsystem_parent_node in self.backup_info.findall('.//mailsystem/..'):
                mailsystem_parent_node.remove(mailsystem_parent_node.find('mailsystem'))

    def close(self):
        self.archive.close()

    def _load_xml(self, path):
        """
        :type path: str|unicode
        :rtype: xml.etree.ElementTree.ElementTree
        """
        with closing(self.archive.extract(path)) as source:
            tree = self._cleanup_tree(ElementTree.ElementTree(file=source))
        return tree

    def _cleanup_tree(self, tree):
        if tree.getroot().tag == '{urn:envelope}Envelope':
            data = tree.getroot().find('{urn:envelope}Data/{urn:envelope}migration-dump')
            self._remove_namespace(data, '{urn:envelope}')
            tree = ElementTree.ElementTree(data)

        if self._discard_mailsystem:
            for mailsystem_parent_node in tree.findall('.//mailsystem/..'):
                mailsystem_parent_node.remove(mailsystem_parent_node.find('mailsystem'))

        return tree

    @staticmethod
    def _remove_namespace(root_node, namespace):
        for node in root_node.iter():
            if node.tag.startswith(namespace):
                node.tag = node.tag[len(namespace):]


class PleskBackupSourceBase(object):
    """
    :type container: parallels.core.dump.dump.PleskBackupContainer
    :type backup_info: xml.etree.ElementTree.ElementTree
    """
    def __init__(self, container):
        self.container = container
        self.backup_info = self.container.backup_info

    def iter_admin_reseller_plans(self):
        """Retrieve reseller plans
        """
        raise NotImplementedError()

    def get_plans(self):
        """Retrieve hosting plans owned by admin
        """
        raise NotImplementedError()

    def get_addon_plans(self):
        """Retrieve hosting plan add-ons owned by admin
        """
        raise NotImplementedError()

    def iter_reseller_plans(self, reseller_name):
        """Retrieve hosting plans owned by given reseller
        """
        raise NotImplementedError()

    def iter_reseller_addon_plans(self, reseller_name):
        """Retrieve hosting plan add-ons owned by given reseller
        """
        raise NotImplementedError()

    def iter_hosting_plans(self, reseller_username=None):
        """Retrieve hosting plans owned by admin
        """
        if reseller_username is None:
            for hosting_plan in self.get_plans():
                yield hosting_plan
        else:
            for hosting_plan in self.iter_reseller_plans(reseller_username):
                yield hosting_plan

    def iter_hosting_plan_addons(self, reseller_username=None):
        """Retrieve hosting plans owned by admin
        """
        if reseller_username is None:
            for hosting_plan_addon in self.get_addon_plans():
                yield hosting_plan_addon
        else:
            for hosting_plan_addon in self.iter_reseller_addon_plans(reseller_username):
                yield hosting_plan_addon

    def iter_all_subscriptions_with_owner_and_reseller(self):
        """Get all subscriptions - subscriptions owned by admin, resellers and customers.

        :rtype: list[(
            parallels.core.dump.data_model.Subscription,
            parallels.core.dump.data_model.Client | None,
            parallels.core.dump.data_model.Reseller | None
        )]
        """
        for subscription in self.iter_admin_subscriptions():
            yield subscription, None, None
        
        for client in self.iter_clients():
            for subscription in client.subscriptions:
                yield subscription, client, None

        for reseller in self.iter_resellers():
            for subscription in reseller.subscriptions:
                yield subscription, reseller, reseller  # reseller subscribes himself on his own plans

            for client in reseller.clients:
                for subscription in client.subscriptions:
                    yield subscription, client, reseller

    def iter_all_subscriptions(self):
        """Get all subscriptions - subscriptions owned by admin, resellers and customers.

        :rtype: list[parallels.core.dump.data_model.Subscription]
        """
        for subscription, _, _ in self.iter_all_subscriptions_with_owner_and_reseller():
            yield subscription

    def iter_all_clients_with_owner(self):
        """
        :rtype: list[(
            parallels.core.dump.data_model.Client,
            parallels.core.dump.data_model.Reseller | None
        )]
        """
        for client in self.iter_clients():
            yield client, None

        for reseller in self.iter_resellers():
            for client in reseller.clients:
                yield client, reseller

    def iter_all_clients(self):
        """
        :rtype: list[parallels.core.dump.data_model.Client]
        """
        for client, _ in self.iter_all_clients_with_owner():
            yield client

    def iter_resellers(self):
        """
        :rtype: list[parallels.core.dump.entity.reseller.Reseller]
        """
        return []

    def iter_clients(self):
        """
        :rtype: list[parallels.core.dump.data_model.Client]
        """
        return []

    def iter_admin_subscriptions(self):
        """
        :rtype: list[parallels.core.dump.data_model.Subscription]
        """
        return []

    @property
    def admin_descriptions(self):
        """
        :rtype: list[parallels.core.dump.entity.description.Description]
        """
        return []

    def get_system_ips(self):
        """
        :rtype: list[parallels.core.dump.data_model.SystemIpAddress]
        """
        return []

    def get_default_ip(self):
        """
        :rtype: str | unicode | None
        """
        return None

    def save(self, fileobj, archive_format=None, prefix=None):
        raise NotImplementedError()

    def save_to_the_same_file(self, prefix=None):
        # Save data to another temporary file
        tmp_filename = "%s.tmp" % self.container.filename
        with open_no_inherit(tmp_filename, 'wb') as backup_file:
            self.save(backup_file, 'zip' if is_run_on_windows() else 'tar', prefix)

        # Close archive file of that dump, and copy temporary file
        # Note: when archive file is closed, it will be automatically opened if
        # we need to read some additional information from it later.
        # See implementation of archive adapters for more details.
        self.container.close()
        shutil.copy2(tmp_filename, self.container.filename)
        os.remove(tmp_filename)


class PleskBackupSource(PleskBackupSourceBase):
    """
    :type resellers_info: dict[str|unicode, xml.etree.ElementTree.Element]
    """
    def __init__(self, container, migration_list_data=None, is_expand_mode=False):
        super(PleskBackupSource, self).__init__(container)
        self.is_expand_mode = is_expand_mode
        # if None is passed - nothing is filtered out
        self.migration_list_data = _extract_migration_list_data(migration_list_data)

        self.resellers_info, self.clients_info, self.domains_info = self._index_entities()
        self.resellers_to_save = self.resellers_info.keys()
        self.clients_to_save = self.clients_info.keys()
        self.domains_to_save = self.domains_info.keys()
        self.unique_plan_names = unique_name_generator.UniqueNameGenerator("%d")

    @property
    def backup_info_file(self):
        return self.container.backup_info_file

    def _index_entities(self):
        """Get XML node for all resellers, clients and domains.
        
        Returns (resellers_map, clients_map, domains_map) where each map is dictionary
        from entity name to root XML node with entity info.

        Override this method in ancestors.
        """
        raise NotImplementedError()

    def _iter_xml_files(self, pattern):
        for member in self.container.archive.get_members():
            member_name = member.name.replace('\\', '/')
            if member.is_file \
                and member.name.endswith('.xml') \
                and member.name != self.container.backup_info_file \
                and re.match(pattern, member_name):

                yield member_name, self.container._load_xml(member.name)

    def _iter_reseller_files(self):
        for path, xml in self._iter_xml_files(re.compile(r'^resellers/[^/]+/[^/]+\.xml$')):
            node = xml.find('reseller')
            name = node.attrib['name']
            yield path, xml, node, name

    def _iter_client_files(self):
        for path, xml in self._iter_xml_files(re.compile(r'^(.*/)?clients/[^/]+/[^/]+\.xml$')):
            node = xml.find('client')
            name = node.attrib['name']
            yield path, xml, node, name

    def _iter_domain_files(self):
        for path, xml in self._iter_xml_files(re.compile(r'^(.*/)?domains/[^/]+/[^/]+\.xml$')):
            node = xml.find('domain')
            name = node.attrib['name']
            yield path, xml, node, name

        for node in self.backup_info.findall('./domain'):
            name = node.attrib['name']
            yield self.backup_info_file, self.backup_info, node, name

    def close(self):
        self.container.close()

    def _is_backup_archive_member_relevant(self, member):
        """Determine, if the given file should be extracted from archive.
        
        When saving the backup back into the tar file, we need to know whether
        a file of the source backup was relevant to migration list. If it's
        not, it won't be saved into the tar file.
        """
        # having the paths like:
        # clients/john/domains/domain.tld/info.xml
        # clients/john/domains/domain.tld/.discovered/info/dump_part
        # we want to extract from them the directory of the corresponding info.xml file:
        # clients/john/domains/domain.tld/
        # by calling get_dirname with offset=1 and offset=3, respectively.

        def get_dirname(path, offset=1):
            return '/'.join(path.split('/')[:-offset]) + '/'

        def get_filename(path):
            return path.rstrip('/').split('/').pop()

        filename_blacklist = ('dump-status.xml', 'psadump.log')
        if get_filename(member.name) in filename_blacklist or member.name.lower().endswith('.zip'):
            logger.debug(u"Discarding a file '%s' (blacklisted)." % member.name)
            return False

        indexed_by_directory = group_by_id(self._indexed, lambda i: get_dirname(i.path.lstrip('./')))
        if '.discovered' in member.name.split('/'):
            member_directory = get_dirname(member.name.lstrip('./'), offset=3)
        else:
            member_directory = get_dirname(member.name.lstrip('./'))

        if member_directory == '/':
            return True  # backup's root info xml is always relevant
        else:
            indexed_entry = indexed_by_directory.get(member_directory)

            # There can be more paths - like one below - in some backups (e.g. PfW 11.x):
            # clients/customer1/domains/customer-domain.com/databases/
            # customer1_mssql_2/.discovered/backup_customer1_mssql_2_info_1404151534/GUID_
            #
            # These files were added with intent to allow partial restore of some part of subscription:
            # its databases, hosting or mailnames.
            # However, these files aren't used even in Plesk scenarios. So, we are safe to discard these
            # paths from backup.
            if indexed_entry is None:
                logger.debug(messages.DISCARD_PATH_R_AS_HAVING_UNKNOWN % member.name)
                return False

            if indexed_entry.obj_type == 'reseller':
                if indexed_entry.obj_name not in self.resellers_to_save:
                    return False
            elif indexed_entry.obj_type == 'client':
                if indexed_entry.obj_name not in self.clients_to_save:
                    return False
            elif indexed_entry.obj_type == 'domain':
                if indexed_entry.obj_name not in self.domains_to_save:
                    return False
            else:
                assert False, messages.ASSERT_BACKUP_UNKNOWN_OBJECT_TYPE % (
                    member.name, indexed_entry.obj_type
                )
            return True

    def _get_reseller_names_filtered(self):
        backup_reseller_names = set(self.resellers_info.keys())

        def subscriptions_count(reseller):
            return sum(
                [len(self._get_reseller_domain_names_filtered(reseller))] + 
                [
                    len(subscriptions)
                    for client in self._get_reseller_client_names_filtered(reseller)
                    for subscriptions in self._get_client_domain_names_filtered(client) 
                ]
            )

        if self.migration_list_data is not None:
            if self.is_expand_mode:
                return [reseller for reseller in self.resellers_info.keys() if subscriptions_count(reseller) > 0]
            else:
                return [
                    reseller for reseller in self.resellers_info.keys()
                    if subscriptions_count(reseller) > 0
                    or reseller in self.migration_list_data.resellers
                ]
        else:
            return backup_reseller_names

    def _get_admin_plan_names_filtered(self):
        backup_plan_names = set([
            plan_node.attrib['name']
            for plan_node in self.backup_info.findall('server/account-templates/domain-template')
        ])
        if self.migration_list_data is not None:
            return backup_plan_names & self.migration_list_data.plans[None]    
        else:
            return backup_plan_names

    def _get_reseller_plan_names_filtered(self, reseller_name):
        backup_plan_names = set([
            plan_node.attrib['name']
            for plan_node in self.resellers_info[reseller_name].findall('preferences/domain-template')
        ])
        if self.migration_list_data is not None:
            return backup_plan_names & self.migration_list_data.plans.get(reseller_name, set())
        else:
            return backup_plan_names

    def _get_admin_domain_names_filtered(self):
        if self.migration_list_data is not None:
            return set(self._get_admin_domain_names()) & self.migration_list_data.subscriptions
        else:
            return self._get_admin_domain_names()

    def _get_client_domain_names_filtered(self, client_name):
        if self.migration_list_data is not None:
            return set(self._get_client_domain_names(client_name)) & self.migration_list_data.subscriptions
        else:
            return self._get_client_domain_names(client_name)

    def _get_reseller_domain_names_filtered(self, reseller_name):
        if self.migration_list_data is not None:
            return set(self._get_reseller_domain_names(reseller_name)) & self.migration_list_data.subscriptions
        else:
            return self._get_reseller_domain_names(reseller_name)

    def _get_admin_client_names_filtered(self):
        if self.migration_list_data is not None:
            return [
                client_name for client_name in self._get_admin_client_names() 
                if len(self._get_client_domain_names_filtered(client_name)) > 0
                or client_name in self.migration_list_data.customers
            ]
        else:
            return self._get_admin_client_names()

    def _get_reseller_client_names_filtered(self, reseller_name):
        if self.migration_list_data is not None:
            return [
                client_name for client_name in self._get_reseller_client_names(reseller_name)
                if len(self._get_client_domain_names_filtered(client_name)) > 0
                or client_name in self.migration_list_data.customers
            ]
        else:
            return self._get_reseller_client_names(reseller_name)
        
    def _parse_hosting_plan(self, plan_node, convert_name=lambda n: n):
        unique_id = plan_node.attrib.get('guid')
        # Plesk 9 doesn't have GUIDs for plans and we generate unique GUID.
        if unique_id is None:
            unique_id = self.unique_plan_names(plan_node.attrib['name'])
        else:
            self.unique_plan_names(unique_id)
        return HostingPlan(
            backup=self,
            node=plan_node,
            id=unique_id,
            name=convert_name(plan_node.attrib['name']),
            is_addon=plan_node.attrib.get('is-addon', 'false') == 'true',
            properties=dict((n.attrib['name'], n.text) for n in plan_node.findall('template-item')),
            log_rotation=parse_logrotation_node(plan_node.find('logrotation')),
            php_settings=parse_php_settings_node(plan_node.find('php-settings'))
        )

    def get_plans(self):
        """Get hosting plans (excluding add-ons) owned by admin.
        """
        return self._get_plans(is_addon=False)

    def get_plan_by_guid(self, guid):
        plan_node = self.backup_info.find('.//domain-template[@guid="%s"]' % guid)
        if plan_node is None:
            return None
        return self._parse_hosting_plan(plan_node)

    def get_addon_plans(self):
        """Get hosting addon plans owned by admin
        """
        return self._get_plans(is_addon=True)

    def get_custom_plan_items(self):
        """Get custom plan items"""
        plan_item_nodes = self.backup_info.findall('server/account-templates/plan-item')

        plan_items = []
        for node in plan_item_nodes:
            plan_items.append(CustomPlanItem(node))

        return plan_items

    def _get_plans(self, is_addon):
        is_addon_value = 'true' if is_addon else 'false'
        filtered_plan_names = self._get_admin_plan_names_filtered()
        plan_nodes = self.backup_info.findall('server/account-templates/domain-template')
        for plan_node in plan_nodes:
            if plan_node.attrib['name'] in filtered_plan_names and plan_node.attrib.get('is-addon', 'false') == is_addon_value:
                yield self._parse_hosting_plan(plan_node)

    def iter_admin_reseller_plans(self):
        """Get reseller plans of admin user.
        """
        plan_nodes = self.backup_info.findall('server/account-templates/reseller-template')
        for plan_node in plan_nodes:
            unique_id = plan_node.attrib.get('guid')
            # Plesk 9 doesn't have GUIDs for plans and we generate unique GUID.
            if unique_id is None:
                unique_id = self.unique_plan_names(plan_node.attrib['name'])
            else:
                self.unique_plan_names(unique_id)
            yield ResellerPlan(
                node=plan_node,
                id=unique_id,
                name=plan_node.attrib['name'],
                properties=dict((n.attrib['name'], n.text) for n in plan_node.findall('template-item')),
                ip_pool=[parse_ip(n) for n in plan_node.findall('ip_pool/ip')],
            )

    def iter_admin_and_admin_customers_subscriptions(self):
        """Get subscriptions owned by admin and subscriptions of customers owned by admin.

        Returns iterable with instances of Subscription.
        """
        for subscription in self.iter_admin_subscriptions():
            yield subscription
        
        for client in self.iter_clients():
            for subscription in client.subscriptions:
                yield subscription

    def iter_admin_subscriptions(self):
        domain_names = self._get_admin_domain_names_filtered()
        return (self.get_subscription(name) for name in domain_names)

    def iter_admin_auxiliary_user_roles(self):
        return itertools.chain(
            self._parse_roles_node(self.backup_info.find('admin/roles')),
            # additionally to roles located under <admin> node,
            # <role> node may be located directly under the XML root
            self._parse_roles_node(self.backup_info)
        )

    def iter_admin_auxiliary_users(self):
        return itertools.chain(
            self._parse_users_node(self.backup_info.find('admin/users')),
            # additionally to users located under <admin> node,
            # <user> node may be located directly under the XML root
            self._parse_users_node(self.backup_info)
        )

    def iter_resellers(self):
        """
        :rtype: list[parallels.core.dump.entity.reseller.Reseller]
        """
        for reseller_name in self._get_reseller_names_filtered():
            info = self.resellers_info[reseller_name]
            domain_names = self._get_reseller_domain_names_filtered(reseller_name)
            pinfo_nodes = info.findall('preferences/pinfo')
            auxiliary_users = self.get_auxiliary_users(info)

            personal_info = dict((pinfo_node.attrib['name'], default(pinfo_node.text, '')) for pinfo_node in pinfo_nodes)
            reseller_user = find_first(auxiliary_users, lambda user: reseller_name == user.name)
            if reseller_user:
                for key in ['im', 'im-type', 'comment']:
                    if key in reseller_user.personal_info:
                        personal_info[key] = reseller_user.personal_info[key]

            yield Reseller(
                node=info,
                login=reseller_name,
                plan_id=if_not_none(info.find('preferences/subscription/plan'), lambda p: p.attrib['plan-guid']),
                limits={
                    n.attrib['name']: n.text
                    for n in info.findall('limits-and-permissions/limit')
                },
                permissions={
                    n.attrib['name']: n.attrib['value'] == 'true'
                    for n in info.findall('limits-and-permissions/permission')
                },
                contact=info.attrib.get('contact'),
                clients=[self.get_client(client_name) for client_name in self._get_reseller_client_names_filtered(reseller_name)],
                subscriptions=[
                    self.get_subscription(domain_name) for domain_name in domain_names
                ],
                personal_info=personal_info,
                auxiliary_users=auxiliary_users,
                auxiliary_user_roles=self._get_auxiliary_user_roles(info),
                is_enabled=info.find('properties/status/enabled') is not None,
                external_id=info.attrib.get('external-id'),
                creation_date=info.attrib.get('cr-date')
            )

    def iter_reseller_plans(self, reseller_name):
        return self._iter_reseller_addon_plans(reseller_name, is_addon=False)

    def iter_reseller_addon_plans(self, reseller_name):
        return self._iter_reseller_addon_plans(reseller_name, is_addon=True)

    def _iter_reseller_addon_plans(self, reseller_name, is_addon):
        is_addon_value = 'true' if is_addon else 'false'
        res_tree = self.resellers_info[reseller_name]
        plan_nodes = res_tree.findall('preferences/domain-template')
        filtered_plan_names = self._get_reseller_plan_names_filtered(reseller_name)
        for plan_node in plan_nodes:
            if plan_node.attrib['name'] in filtered_plan_names and plan_node.attrib.get('is-addon', 'false') == is_addon_value:
                yield self._parse_hosting_plan(plan_node)

    def iter_clients(self):
        for client_name in self._get_admin_client_names_filtered():
            yield self.get_client(client_name)

    def get_client(self, client_name):
        """
        :type client_name: str|unicode
        :rtype: parallels.core.dump.entity.Client
        """
        info = self.clients_info[client_name]
        domain_names = self._get_client_domain_names_filtered(client_name)
        pinfo_nodes = info.findall('preferences/pinfo')

        auxiliary_users, auxiliary_roles = self._get_auxiliary_users_and_roles(
            client_name, info
        )

        personal_info = dict((pinfo_node.attrib['name'], default(pinfo_node.text, '')) for pinfo_node in pinfo_nodes)
        client_user = find_first(auxiliary_users, lambda user: client_name == user.name)
        if client_user:
            for key in ['im', 'im-type', 'comment']:
                if key in client_user.personal_info:
                    personal_info[key] = client_user.personal_info[key]
        
        return Client(
            node=info,
            login=client_name,
            contact=info.attrib.get('contact'),
            subscriptions=[
                self.get_subscription(domain_name) for domain_name in domain_names
            ],
            personal_info=personal_info,
            auxiliary_users=auxiliary_users,
            auxiliary_user_roles=auxiliary_roles,
            is_enabled=info.find('properties/status/enabled') is not None,
            external_id=info.attrib.get('external-id'),
            creation_date=info.attrib.get('cr-date')
        )

    def _get_auxiliary_users_and_roles(self, client_name, client_info):
        # Get list of domain XML nodes
        domain_names = self._get_client_domain_names_filtered(client_name)
        domain_nodes = []
        for domain_name in domain_names:
            domain_nodes.append(self.domains_info[domain_name])

        # Read domain administrators
        domain_admin_users, domain_admin_roles = _get_client_auxiliary_users_and_roles_for_domain_admin(
            client_info, domain_nodes
        )

        # Read mail users
        mailuser_users, mailuser_roles = _get_client_auxiliary_users_and_roles_for_mailusers(
            client_info, domain_nodes
        )

        # Read regular users
        regular_users = self.get_auxiliary_users(client_info)
        regular_roles = self._get_auxiliary_user_roles(client_info)

        # Merge users and roles by their names
        users = unique_merge(
            key=lambda user: user.name,
            # Domains and mail users have more priority than regular client
            # users, to get correct permissions in case of backup
            # created by PMM agent for Plesk 8/9
            lists=[domain_admin_users, mailuser_users, regular_users]
        )
        roles = unique_merge(
            key=lambda role: role.name,
            lists=[domain_admin_roles, mailuser_roles, regular_roles]
        )

        return users, roles

    @classmethod
    def get_auxiliary_users(cls, client_info):
        """
        :type client_info: xml.etree.ElementTree.Element
        :rtype: list[parallels.core.dump.data_model.AuxiliaryUser]
        """
        return list(cls._parse_users_node(client_info.find('users')))

    @classmethod
    def _get_auxiliary_user_roles(cls, client_info):
        """
        :rtype: list[parallels.core.dump.data_model.AuxiliaryUserRole]
        """
        return list(cls._parse_roles_node(client_info.find('roles')))

    @classmethod
    def _parse_roles_node(cls, node):
        if node is None:
            return

        for role_node in node.findall('role'):
            if 'is-built-in' in role_node.attrib and role_node.attrib['is-built-in'] == 'false':
                yield AuxiliaryUserRole(
                    name=role_node.attrib['name'],
                    permissions=dict(
                        (perm_node.attrib['name'], True if perm_node.attrib['value'] == 'true' else False)
                        for perm_node in role_node.findall('limits-and-permissions/permission')
                        if perm_node.attrib['value'] in ('true', 'false')
                    ),
                    is_domain_admin=False
                )
    
    @classmethod
    def _parse_users_node(cls, node):
        if node is None:
            return

        for user_node in node.findall('user'):
            personal_info = dict(
                (pinfo_node.attrib['name'], default(pinfo_node.text, '')) for pinfo_node in user_node.findall('preferences/pinfo')
            )

            im_map = {  # in Plesk backup IM is presented as a number, while we need a name
                '1': 'ICQ',
                '2': 'Jabber',
                '3': 'AIM',
                '4': 'Skype',
                '0': 'Other',
            }
            if 'im-type' in personal_info:
                personal_info['im-type'] = im_map[personal_info['im-type']]

            yield AuxiliaryUser(
                node=user_node,
                name=user_node.attrib['name'], 
                contact=user_node.attrib.get('contact'), 
                email=user_node.attrib.get('email'),
                roles=[perm_node.text for perm_node in user_node.findall('limits-and-permissions/role-ref')],
                personal_info=personal_info,
                subscription_name=user_node.attrib.get('subscription-name', None),
                is_active=user_node.find('properties/status/enabled') is not None,
                is_domain_admin=user_node.attrib['is-domain-admin'] != 'false',
                is_built_in=user_node.attrib.get('is-built-in') != 'false',
                external_email=user_node.attrib.get('external-email')
            )

    def has_subscription(self, name):
        return name in self.domains_info

    def get_subscription(self, name):
        if name not in self.domains_info:
            raise SubscriptionNotFoundException(name)
        domain_info = self.domains_info[name]
        ips = parse_subscription_ips(domain_info.findall('properties/ip'))
        return Subscription(
            node=domain_info,
            backup=self,
            name=name,
            plan_id=self.get_subscription_plan_guid(name),
            ip=ips.v4, ip_type=ips.v4_type, ipv6=ips.v6, ipv6_type=ips.v6_type,
            limits=dict(
                (n.attrib['name'], n.text)
                for n in domain_info.findall('limits-and-permissions/limit')
            ),
            permissions=dict(
                (n.attrib['name'], n.attrib['value'] == 'true')
                for n in domain_info.findall('limits-and-permissions/permission')
            )
        )

    def get_subscription_plan_guid(self, subscription_name):
        domain_info = self.domains_info[subscription_name]
        for plan_node in domain_info.findall('preferences/subscription/plan'):
            if plan_node.attrib.get('is-addon', 'false') == 'false':
                return plan_node.attrib['plan-guid']
        return None

    def iter_db_servers(self):
        db_server_nodes = self.backup_info.findall('server/db-servers/db-server')
        for node in db_server_nodes:
            # Line for PostgreSQL server is always present in Plesk database, even if it isn't configured,
            # but administrator login and password are missing in this case.
            # Perhaps this applies to other database types too.
            # So just skip databases without administrator login.
            db_admin_node = node.find('db-admin')
            if db_admin_node is not None:
                if node.findtext('port') == '':
                    port = 0
                else:
                    port = int(node.findtext('port'))
                yield DatabaseServerOverXML(
                    node=db_admin_node,
                    host=node.findtext('host'),
                    port=port,
                    login=db_admin_node.attrib['name'],
                    dbtype=node.attrib['type'],
                    is_default=node.attrib.get('default', 'false') == 'true',
                )

    def iter_mailboxes(self):
        for subscription_name, subscription_node in self.domains_info.items():
            if self.migration_list_data is None or subscription_name in self.migration_list_data.subscriptions:
                for domain_node in itertools.chain([subscription_node], subscription_node.findall('phosting/sites/site')):
                    mailuser_nodes = domain_node.findall('mailsystem/mailusers/mailuser')
                    for mailuser_node in mailuser_nodes:
                        yield Mailbox.parse(mailuser_node, domain_node.attrib['name'])

    def iter_subscription_mailboxes(self, subscription_name):
        subscription_node = self.domains_info[subscription_name]
        for domain_node in itertools.chain(
            [subscription_node],
            subscription_node.findall('phosting/sites/site')
        ):
            mailuser_nodes = domain_node.findall('mailsystem/mailusers/mailuser')
            for mailuser_node in mailuser_nodes:
                yield Mailbox.parse(mailuser_node, domain_node.attrib['name'])

    def iter_domain_mailboxes(self, domain_name, subscription_name):
        """Return Mailbox objects of the given domain."""
        subscription_node = self.domains_info[subscription_name]
        for domain_node in itertools.chain(
            [subscription_node],
            subscription_node.findall('phosting/sites/site')
        ):
            if domain_node.get('name') == domain_name:
                mailuser_nodes = domain_node.findall(
                    'mailsystem/mailusers/mailuser'
                )
                for mailuser_node in mailuser_nodes:
                    yield Mailbox.parse(
                        mailuser_node, domain_node.attrib['name']
                    )

    def get_subscription_site_names(self, subscription_name, include_subdomains=False):
        domain_info = self.domains_info[subscription_name]
        site_nodes = domain_info.findall('phosting/sites/site')
        return set(s.attrib['name'] for s in site_nodes if 'parent-domain-name' not in s.attrib or include_subdomains)

    def get_dns_settings(self):
        own_zones = self.backup_info.findtext('server/dns-settings/subdomain-own-zones')
        return DnsSettings(
            do_subdomains_own_zones=own_zones is not None and parse_bool(own_zones),
        )

    def iter_dns_template_records(self):
        template_node = self.backup_info.find('server/dns-settings/dns-zone')
        if template_node is not None:
            for node in template_node.findall('dnsrec'):
                yield DnsRecord.parse(node)

    def iter_sites(self, subscription_name):
        """Iterate over all subscription sites: addon domains and subdomains.
        """
        # 1) Subscription's own subdomains
        for subdomain in self.iter_subdomains(
            subscription_name, subscription_name
        ):
            yield subdomain

        # 2) Addon domains
        for addon_domain in self.iter_addon_domains(subscription_name):
            yield addon_domain

            # 3) Subdomains of addon domains
            for subdomain in self.iter_subdomains(
                subscription_name, addon_domain.name
            ):
                yield subdomain

    def iter_addon_domains(self, subscription_name):
        """Get addon domains.
        """
        return map(
            AddonDomain,
            (
                i for i in self.domains_info[subscription_name].iterfind('phosting/sites/site')
                if 'parent-domain-name' not in i.attrib
            )
        )

    def iter_subdomains(self, subscription_name, domain_name):
        # Try both >= 10.4 and < 10.4 ways to get subdomains, as Plesk agent 
        # dumps subdomains in old way for old Plesks, in new way for new Plesks

        # Plesk >= 10.4 way
        for node in self.domains_info[subscription_name].iterfind('phosting/sites/site'):
            parent_name_idna = if_not_none(node.attrib.get('parent-domain-name'), lambda s: s.encode('idna'))
            if parent_name_idna == domain_name.encode("idna"):
                yield Subdomain(node)

        # Plesk < 10.4 way
        if subscription_name == domain_name:  # get subdomains owned by subscription itself
            for node in self.domains_info[subscription_name].iterfind('phosting/subdomains/subdomain'):
                yield Subdomain8(subscription_name, node, self.get_subscription(subscription_name))
        else:  # get subdomains owned by addon domain
            for node in self.domains_info[subscription_name].iterfind(
                'phosting/sites/site[@name="%s"]/phosting/subdomains/subdomain' % (domain_name,)
            ):
                yield Subdomain8(
                    domain_name, node,
                    [addon for addon in self.iter_addon_domains(subscription_name) if addon.name == domain_name][0]
                )

    def iter_aliases(self, subscription_name):
        """
        :type subscription_name: str | unicode
        :rtype: list[parallels.core.dump.data_model.DomainAlias]
        """
        domain_aliases = []
        domain_aliases.extend(
            DomainAlias(subscription_name, node)
            for node in self.domains_info[subscription_name].iterfind('preferences/domain-alias')
        )
        for addon_domain in self.iter_addon_domains(subscription_name):
            domain_aliases.extend(
                DomainAlias(addon_domain.name, node) for node in addon_domain.node.iterfind('preferences/domain-alias')
            )
        return domain_aliases

    @property
    def admin_descriptions(self):
        """
        :rtype: list[parallels.core.dump.entity.description.Description]
        """
        return [
            Description(description_node)
            for description_node in self.backup_info.findall('admin/preferences/descriptions/description')
        ]

    def get_system_ips(self):
        return [parse_system_ip(n, self.is_windows) for n in self.backup_info.findall('server/properties/system-ip')]

    def get_default_ip(self):
        return self.backup_info.findtext('server/properties/default-ip')

    @property
    def is_windows(self):
        os_desc = self.backup_info.find('dump-info/os-description')
        return os_desc is not None and os_desc.attrib['type'] == 'windows'

    def replace_applications(self, applications_str):
        def find_applications(applications, domain_name):
            return applications.findall('application[domain-name="%s"]/sapp-installed' % (domain_name,))

        applications = ElementTree.fromstring(applications_str)
        for subscr in self.iter_all_subscriptions():
            subscr.replace_applications(find_applications(applications, subscr.name))
            for subdomain in self.iter_subdomains(subscr.name, subscr.name):
                subdomain.replace_applications(find_applications(applications, subdomain.name))

            for domain in self.iter_addon_domains(subscr.name):
                domain.replace_applications(find_applications(applications, domain.name))
                for subdomain in self.iter_subdomains(subscr.name, domain.name):
                    subdomain.replace_applications(find_applications(applications, subdomain.name))

    def get_subscription_owner_client(self, subscription_name):
        """Find subscription's owner. Return 'client' object."""
        for client in self.iter_all_clients():
            for s in client.subscriptions:
                if subscription_name == s.name:
                    return client

    @staticmethod
    def _remove_empty_nodes(root, node_name):
        """Remove all empty elements with given name from specified root node
        :type root: xml.etree.ElementTree.ElementTree | xml.etree.ElementTree.Element
        :type node_name: str
        """
        empty_nodes = []
        for node in root.findall('.//%s' % node_name):
            if len(node) == 0:
                empty_nodes.append(node)
        if len(empty_nodes) == 0:
            return
        parent_map = {c: p for p in root.iter() for c in p}
        for empty_node in empty_nodes:
            if empty_node not in parent_map:
                continue
            parent_map[empty_node].remove(empty_node)

    def remove_content(self):
        self.backup_info.getroot().attrib['content-included'] = 'false'

        for tree in itertools.chain(
            self.backup_info.getroot(),
            self.resellers_info.values(),
            self.clients_info.values(),
            self.domains_info.values(),
        ):
            for parent_node in tree.findall('.//content/..'):
                parent_node.remove(parent_node.find('content'))


def replace_prefix(file_name, prefix):
    if prefix is not None:
        paths = file_name.strip('/').split('/')
        new_path = []
        for path in paths:
            starts_with_old_prefix = False
            for old_prefix in ['backup_', 'converted_', 'info_']:
                if path.startswith(old_prefix):
                    new_path.append("".join([prefix, '_', path[len(old_prefix):]]))
                    starts_with_old_prefix = True
                    break
            if not starts_with_old_prefix:
                new_path.append(path)
        return "/".join(new_path)
    return file_name


class PleskBackupSource11(PleskBackupSource):
    def _index_entities(self):
        # TODO: Remove loading of all XMLs to memory when PMM team resolve
        # file naming issue. ATM it is impossible to reliably get file name
        # where domain information is stored.
        self._indexed = []
        resellers_info = self._index_reseller_files()
        clients_info = self._index_client_files()
        domains_info = self._index_domain_files()

        if not utils_exists(self._indexed, lambda l: l.path == self.container.backup_info_file):
            self._indexed.append(
                obj(path=self.container.backup_info_file, obj_type='backup_info', obj_name=None, xml=self.backup_info)
            )

        return resellers_info, clients_info, domains_info

    def _index_reseller_files(self):
        resellers_info = {}
        for path, xml, node, name in self._iter_reseller_files():
            self._indexed.append(obj(path=path, obj_type='reseller', obj_name=name, xml=xml))
            resellers_info[name] = node
        return resellers_info

    def _index_client_files(self):
        clients_info = {}
        for path, xml, node, name in self._iter_client_files():
            self._indexed.append(obj(path=path, obj_type='client', obj_name=name, xml=xml))
            clients_info[name] = node
        return clients_info

    def _index_domain_files(self):
        domains_info = {}
        for path, xml, node, name in self._iter_domain_files():
            self._indexed.append(obj(path=path, obj_type='domain', obj_name=name, xml=xml))
            domains_info[name] = node
        return domains_info

    def remove_encryption_properties(self):
        """
        Remove 'encryption-type' and 'verification-string' attributes from each xml file of dump
        """
        indexed_by_path = group_by_id(self._indexed, lambda l: l.path)
        for member in self.container.archive.get_members():
            if member.is_file:
                if member.name not in indexed_by_path:
                    continue
                tree = indexed_by_path[member.name].xml
                root = tree.getroot()
                root.attrib.pop('encryption-type', None)
                root.attrib.pop('verification-string', None)

    def save(self, fileobj, archive_format=None, prefix=None):
        """
        Save migration dump to a given file object

        :param fileobj: fileobj
        :param archive_format: 'tar', 'zip', None
        """
        indexed_by_path = group_by_id(self._indexed, lambda l: l.path)
        new_archive = (
            ZipFileAdapter(fileobj=fileobj, mode='w')
            if 'zip' == archive_format else TarFileAdapter(fileobj=fileobj, mode='w')
        )
        with closing(new_archive):
            for member in self.container.archive.get_members():
                if member.is_file:
                    if not self._is_backup_archive_member_relevant(member):
                        continue

                    file_name = member.name

                    if member.name in indexed_by_path:
                        source = create_xml_stream(indexed_by_path[member.name].xml)
                        file_size = len(source.getvalue())
                    else:
                        source = self.container.archive.extract(member.name)
                        file_size = member.size
                    with closing(source):
                        file_name = replace_prefix(file_name, prefix)
                        new_archive.add_file(file_name, file_size, source)

    def cut_irrelevant_subcriptions(self, relevant_subscription_names):
        # cut domain-info nodes from clients' xml files
        client_xml_by_name = {i.obj_name: i.xml for i in self._indexed if i.obj_type == 'client'}
        for client_info_node in (
            self.backup_info.findall('admin/clients/client-info') +
            self.backup_info.findall('admin/resellers/reseller-info/client-info')
        ):
            for domains_node in client_xml_by_name[client_info_node.attrib['name']].findall('client/domains'):
                for domain_info_node in domains_node.findall('domain-info'):
                    if domain_info_node.attrib['name'] not in relevant_subscription_names:
                        domains_node.remove(domain_info_node)
            self._remove_empty_nodes(client_xml_by_name[client_info_node.attrib['name']], 'domains')

        # cut domain-info nodes from resellers' xml files
        reseller_xml_by_name = {i.obj_name: i.xml for i in self._indexed if i.obj_type == 'reseller'}
        for reseller_info_node in self.backup_info.findall('admin/resellers/reseller-info'):
            reseller_xml = reseller_xml_by_name[reseller_info_node.attrib['name']]

            for domains_node in reseller_xml.findall('reseller/domains'):
                for domain_info_node in domains_node.findall('domain-info'):
                    if domain_info_node.attrib['name'] not in relevant_subscription_names:
                        domains_node.remove(domain_info_node)
            self._remove_empty_nodes(reseller_xml, 'domains')

            for client_info_node in reseller_xml.findall('reseller/clients/client-info'):
                for domain_info_node in client_info_node.findall('domain-info'):
                    if domain_info_node.attrib['name'] not in relevant_subscription_names:
                        client_info_node.remove(domain_info_node)

        # cut domain-info nodes from the root xml file
        for domains_parent_node in (
            self.backup_info.findall('admin/domains') +
            self.backup_info.findall('admin/clients/client-info') +
            self.backup_info.findall('admin/resellers/reseller-info') +
            self.backup_info.findall('admin/resellers/reseller-info/client-info')
        ):
            for domain_node in domains_parent_node.findall('domain-info'):
                if domain_node.attrib['name'] not in relevant_subscription_names:
                    domains_parent_node.remove(domain_node)
        self._remove_empty_nodes(self.backup_info, 'domains')

        self.domains_to_save = relevant_subscription_names

    def cut_irrelevant_clients(self, relevant_logins):
        # cut domain-info nodes from resellers' xml files
        reseller_xml_by_name = {i.obj_name: i.xml for i in self._indexed if i.obj_type == 'reseller'}
        for reseller_info_node in self.backup_info.findall('admin/resellers/reseller-info'):
            reseller_xml = reseller_xml_by_name[reseller_info_node.attrib['name']]

            for clients_node in reseller_xml.findall('reseller/clients'):
                for client_info_node in clients_node.findall('client-info'):
                    if client_info_node.attrib['name'] not in relevant_logins:
                        clients_node.remove(client_info_node)
            self._remove_empty_nodes(reseller_xml, 'clients')

        for clients_parent_node in (
            self.backup_info.findall('admin/clients') +
            self.backup_info.findall('admin/resellers/reseller-info')
        ):
            for client_node in clients_parent_node.findall('client-info'):
                if client_node.attrib['name'] not in relevant_logins:
                    clients_parent_node.remove(client_node)
        self._remove_empty_nodes(self.backup_info, 'clients')
        self.clients_to_save = relevant_logins

    def cut_irrelevant_resellers(self, relevant_logins):
        for resellers_parent_node in self.backup_info.findall('admin/resellers'):
            for reseller_node in resellers_parent_node.findall('reseller-info'):
                if reseller_node.attrib['name'] not in relevant_logins:
                    resellers_parent_node.remove(reseller_node)
        self._remove_empty_nodes(self.backup_info, 'resellers')
        self.resellers_to_save = relevant_logins

    def _get_admin_domain_names(self):
        return [
            n.attrib['name'] for n in itertools.chain(
                self.backup_info.findall('admin/domains/domain-info'),
                self.backup_info.findall('domain-info'),
                self.backup_info.findall('domain'),
            )
        ]

    def _get_client_domain_names(self, client_name):
        return [n.attrib['name'] for n in self.clients_info[client_name].findall('domains/domain-info')]

    def _get_reseller_domain_names(self, reseller_name):
        return [n.attrib['name'] for n in self.resellers_info[reseller_name].findall('domains/domain-info')]

    def _get_admin_client_names(self):
        return [
            n.attrib['name'] for n in itertools.chain(
                self.backup_info.findall('admin/clients/client-info'),
                self.backup_info.findall('client')
            )
        ]

    def _get_reseller_client_names(self, reseller_name):
        return [n.attrib['name'] for n in self.resellers_info[reseller_name].findall('clients/client-info')]


class PleskBackupSource10(PleskBackupSource):
    def _index_entities(self):
        resellers_info = {}
        for node in self.backup_info.findall('admin/resellers/reseller'):
            resellers_info[node.attrib['name']] = node

        clients_info = {}
        for node in \
            self.backup_info.findall('admin/clients/client') + \
            self.backup_info.findall('admin/resellers/reseller/clients/client'):

            clients_info[node.attrib['name']] = node

        domains_info = {}
        for node in \
            self.backup_info.findall('admin/domains/domain') + \
            self.backup_info.findall('admin/clients/client/domains/domain') + \
            self.backup_info.findall('admin/resellers/reseller/domains/domain') + \
            self.backup_info.findall('admin/resellers/reseller/clients/client/domains/domain'):

            domains_info[node.attrib['name']] = node

        self._indexed = []  # [ obj(path, obj_type, obj_name, xml) ]
        for path, _, node, name in self._iter_reseller_files():
            self._indexed.append(obj(path=path, obj_type='reseller', obj_name=name, xml=resellers_info[name]))
        for path, _, node, name in self._iter_client_files():
            self._indexed.append(obj(path=path, obj_type='client', obj_name=name, xml=clients_info[name]))
        for path, _, node, name in self._iter_domain_files():
            self._indexed.append(obj(path=path, obj_type='domain', obj_name=name, xml=domains_info[name]))

        return resellers_info, clients_info, domains_info

    def save(self, fileobj, archive_format=None, prefix=None):
        """
        Save migration dump to a given file object

        :param fileobj: fileobj
        :param archive_format: 'tar', 'zip', None
        """
        indexed_by_path = group_by_id(self._indexed, lambda i: i.path)
        archiver = ZipFileAdapter if 'zip' == archive_format else TarFileAdapter
        new_archive = archiver(fileobj=fileobj, mode='w')
        logger.debug(messages.SAVING_FILEOBJECT_USING_S_FORMAT % archiver.__name__)
        with closing(new_archive):
            for member in self.container.archive.get_members():
                if member.is_file:
                    file_name = member.name

                    if not self._is_backup_archive_member_relevant(member):
                        continue

                    if member.name == self.container.backup_info_file:
                        logger.debug(messages.DEBUG_SAVING_BACKUP_INFO_FILE, member.name)
                        source = create_xml_stream(self.backup_info)
                        file_size = len(source.getvalue())
                    elif member.name in indexed_by_path:
                        logger.debug(messages.DEBUG_SAVING_INDEXED_BY_PATH, member.name)
                        node = indexed_by_path[member.name].xml
                        root_node = self._wrap_into_migrationdump_node(
                            node, self.container.dump_version
                        )
                        tree = ElementTree.ElementTree(root_node)
                        source = create_xml_stream(tree)
                        file_size = len(source.getvalue())
                    else:
                        logger.debug(messages.DEBUG_SAVING_FILE, member.name, member.size)
                        source = self.container.archive.extract(member.name)
                        file_size = member.size

                    with closing(source):
                        file_name = replace_prefix(file_name, prefix)
                        new_archive.add_file(file_name, file_size, source)
            self.container.archive.close()

    def _wrap_into_migrationdump_node(self, node, dump_version):
        return elem(
            'migration-dump',
            attributes={
                'content-included': "false",
                'agent-name': "PleskX",
                'dump-format': "panel",
                'dump-version': '.'.join(map(str, dump_version))},
            children=[
                elem('dump-info', [text_elem('description', "Created by migrator.")]),
                node])

    def cut_irrelevant_subcriptions(self, relevant_subscription_names):
        for domains_node in (
            self.backup_info.findall('admin/domains') +
            self.backup_info.findall('admin/clients/client/domains') +
            self.backup_info.findall('admin/resellers/reseller/domains') +
            self.backup_info.findall('admin/resellers/reseller/clients/client/domains')
        ):
            for domain_node in domains_node.findall('domain'):
                if domain_node.attrib['name'] not in relevant_subscription_names:
                    domains_node.remove(domain_node)
        self._remove_empty_nodes(self.backup_info, 'domains')
        self.domains_to_save = relevant_subscription_names

    def cut_irrelevant_clients(self, relevant_logins):
        for clients_node in (
            self.backup_info.findall('admin/clients') +
            self.backup_info.findall('admin/resellers/reseller/clients')
        ):
            for client_node in clients_node.findall('client'):
                if client_node.attrib['name'] not in relevant_logins:
                    clients_node.remove(client_node)
        self._remove_empty_nodes(self.backup_info, 'clients')
        self.clients_to_save = relevant_logins

    def cut_irrelevant_resellers(self, relevant_logins):
        for resellers_node in self.backup_info.findall('admin/resellers'):
            for reseller_node in resellers_node.findall('reseller'):
                if reseller_node.attrib['name'] not in relevant_logins:
                    resellers_node.remove(reseller_node)
        self._remove_empty_nodes(self.backup_info, 'resellers')
        self.resellers_to_save = relevant_logins

    def _get_admin_domain_names(self):
        return [n.attrib['name'] for n in self.backup_info.findall('admin/domains/domain')]

    def _get_client_domain_names(self, client_name):
        return [n.attrib['name'] for n in self.clients_info[client_name].findall('domains/domain')]

    def _get_reseller_domain_names(self, reseller_name):
        return [n.attrib['name'] for n in self.resellers_info[reseller_name].findall('domains/domain')]

    def _get_admin_client_names(self):
        return [n.attrib['name'] for n in self.backup_info.findall('admin/clients/client')]

    def _get_reseller_client_names(self, reseller_name):
        return [n.attrib['name'] for n in self.resellers_info[reseller_name].findall('clients/client')]


class PleskBackupSource9(PleskBackupSource10):
    def iter_subdomains(self, subscription_name, domain_name):
        return (
            Subdomain8(subscription_name, i, self.get_subscription(subscription_name))
            for i in self.domains_info[subscription_name].iterfind('phosting/subdomains/subdomain')
        )

    def _get_auxiliary_users_and_roles(self, client_name, client_info):
        domain_admin_users, domain_admin_roles = self._get_client_auxiliary_users_and_roles_for_domain_admin(client_info)
        mailuser_users, mailuser_roles = self._get_client_auxiliary_users_and_roles_for_mailusers(client_info)
        return (
            domain_admin_users + mailuser_users,
            domain_admin_roles + mailuser_roles
        )

    @classmethod
    def _get_client_auxiliary_users_and_roles_for_domain_admin(cls, client_node):
        """Returns generator of tuples (AuxiliaryUser, AuxiliaryUserRole)
        where AuxiliaryUserRole is a special role created for domain user"""
        return _get_client_auxiliary_users_and_roles_for_domain_admin(
            client_node
        )

    @classmethod
    def _get_client_auxiliary_users_and_roles_for_mailusers(cls, client_node):
        return _get_client_auxiliary_users_and_roles_for_mailusers(
            client_node
        )

    def iter_sites(self, subscription_name):
        """Return Subdomain8 objects, even though formally they should be Subdomain9:
        The functions of Subdomain8 should work for Plesk 9 subdomain as well,
        make a distinct Subdomain9 type when you'll need to access Plesk 9 specific subdomain's data
        """
        return (
            Subdomain8(subscription_name, n, self.get_subscription(subscription_name))
            for n in self.domains_info[subscription_name].iterfind('phosting/subdomains/subdomain')
        )


class PleskBackupSource8(PleskBackupSourceBase):
    def __init__(self, container, migration_list_data=None):
        super(PleskBackupSource8, self).__init__(container)
        # if None is passed - nothing is filtered out
        self.migration_list_data = _extract_migration_list_data(migration_list_data)

        self.clients_info, self.domains_info = self._index_entities()

    def close(self):
        self.container.close()

    def cut_irrelevant_subcriptions(self, relevant_subscription_names):
        for client_node in self.backup_info.findall('client'):
            for domain_node in client_node.findall('domain'):
                if domain_node.attrib['name'] not in relevant_subscription_names:
                    client_node.remove(domain_node)

    def cut_irrelevant_clients(self, relevant_logins):
        for client_node in self.backup_info.findall('client'):
            if client_node.attrib['name'] not in relevant_logins:
                self.backup_info.getroot().remove(client_node)

    def cut_irrelevant_resellers(self, relevant_logins):
        # There are no resellers in Plesk 8.x, nothing to cut
        pass

    @property
    def backup_info_file(self):
        return self.container.backup_info_file

    def _index_entities(self):
        clients_info = {}
        for node in self.backup_info.findall('client'):
            clients_info[node.attrib['name']] = node

        domains_info = {}
        for node in self.backup_info.findall('client/domain'):
            domains_info[node.attrib['name']] = node

        return clients_info, domains_info

    def _get_client_domain_names_filtered(self, client_name):
        if self.migration_list_data is not None:
            return set(self._get_client_domain_names(client_name)) & self.migration_list_data.subscriptions
        else:
            return self._get_client_domain_names(client_name)

    def _get_client_domain_names(self, client_name):
        return [n.attrib['name'] for n in self.clients_info[client_name].findall('domain')]

    def _get_admin_client_names_filtered(self):
        if self.migration_list_data is not None:
            return [
                client_name for client_name in self._get_admin_client_names()
                if len(self._get_client_domain_names_filtered(client_name)) > 0
                or client_name in self.migration_list_data.customers
            ]
        else:
            return self._get_admin_client_names()

    def _get_admin_client_names(self):
        return self.clients_info.keys()

    def get_plans(self):
        """Get hosting plans (excluding add-ons) owned by admin."""
        return []

    def get_addon_plans(self):
        """Get hosting addon plans owned by admin"""
        return []

    def get_custom_plan_items(self):
        """Get custom plan items"""
        return []

    def iter_admin_reseller_plans(self):
        """Get reseller plans of admin user.
        """
        return iter([])

    def iter_admin_and_admin_customers_subscriptions(self):
        """Get subscriptions owned by admin and subscriptions of customers owned by admin.

        Returns iterable with instances of Subscription.
        """
        for client in self.iter_clients():
            for subscription in client.subscriptions:
                yield subscription

    def iter_admin_subscriptions(self):
        return iter([])

    def iter_admin_auxiliary_user_roles(self):
        return iter([])

    def iter_admin_auxiliary_users(self):
        return iter([])

    def iter_resellers(self):
        return iter([])

    def iter_reseller_plans(self, reseller_name):
        return iter([])

    def iter_reseller_addon_plans(self, reseller_name):
        return iter([])

    def iter_clients(self):
        for client_name in self._get_admin_client_names_filtered():
            yield self.get_client(client_name)

    def iter_all_clients(self):
        return self.iter_clients()

    def get_client(self, client_name):
        info = self.clients_info[client_name]
        domain_names = self._get_client_domain_names_filtered(client_name)
        pinfo_nodes = info.findall('pinfo')

        domain_admin_users, domain_admin_roles = self._get_client_auxiliary_users_and_roles_for_domain_admin(info)
        mailuser_users, mailuser_roles = self._get_client_auxiliary_users_and_roles_for_mailusers(info)
        return Client(
            node=info,
            login=client_name,
            contact=info.attrib.get('contact'),
            subscriptions=[
                self.get_subscription(domain_name) for domain_name in domain_names
            ],
            personal_info=dict(
                (pinfo_node.attrib['name'], default(pinfo_node.text, '')) for pinfo_node in pinfo_nodes
            ),
            auxiliary_users=domain_admin_users + mailuser_users,
            auxiliary_user_roles=domain_admin_roles + mailuser_roles,
            is_enabled=info.find('status/enabled') is not None,
            external_id=info.attrib.get('external-id'),
        )

    @classmethod
    def _get_client_auxiliary_users_and_roles_for_domain_admin(cls, client_node):
        """Returns generator of tuples (AuxiliaryUser, AuxiliaryUserRole)
        where AuxiliaryUserRole is a special role created for domain user"""
        return _get_client_auxiliary_users_and_roles_for_domain_admin(
            client_node
        )

    @classmethod
    def _get_client_auxiliary_users_and_roles_for_mailusers(cls, client_node):
        return _get_client_auxiliary_users_and_roles_for_mailusers(
            client_node
        )

    def get_subscription(self, name):
        if name not in self.domains_info:
            raise SubscriptionNotFoundException(name)
        domain_info = self.domains_info[name]
        ips = parse_subscription_ips(domain_info.findall('ip'))
        return Subscription8(
            node=domain_info,
            backup=self,
            name=name,
            plan_id=None,
            ip=ips.v4, ip_type=ips.v4_type, ipv6=ips.v6, ipv6_type=ips.v6_type,
            limits=dict(
                (n.attrib['name'], n.text)
                for n in domain_info.findall('limit')
            ),
            permissions=dict(
                (n.attrib['name'], n.attrib['value'] == 'true')
                for n in domain_info.findall('permission')
            )
        )

    def iter_db_servers(self):
        db_server_nodes = self.backup_info.findall('server/db-server')
        for node in db_server_nodes:
            # Line for PostgreSQL server is always present in Plesk database, even if it isn't configured,
            # but administrator login and password are missing in this case.
            # Perhaps this applies to other database types too.
            # So just skip databases without administrator login.
            db_admin_node = node.find('admin')
            if db_admin_node is not None:
                assert db_admin_node.find('password').attrib['type'] == 'plain'
                yield DatabaseServerOverXML(
                    node=db_admin_node,
                    host=node.findtext('host'),
                    port=int(node.findtext('port')),
                    login=db_admin_node.attrib['name'],
                    dbtype=node.attrib['type'],
                    is_default=node.attrib.get('default', 'false') == 'true',
                )

    def iter_mailboxes(self):
        for subscription_name, subscription_node in self.domains_info.items():
            if self.migration_list_data is None or subscription_name in self.migration_list_data.subscriptions:
                mailuser_nodes = subscription_node.findall('mailsystem/mailuser')
                for mailuser_node in mailuser_nodes:
                    yield Mailbox8.parse(mailuser_node, subscription_name)

    def iter_subscription_mailboxes(self, subscription_name):
        subscription_node = self.domains_info[subscription_name]
        mailuser_nodes = subscription_node.findall('mailsystem/mailuser')
        for mailuser_node in mailuser_nodes:
            yield Mailbox8.parse(mailuser_node, subscription_name)

    def get_subscription_site_names(self, subscription_name, include_subdomains=False):
        domain_info = self.domains_info[subscription_name]
        if include_subdomains:
            return {'%s.%s' % (subscription_name, n.attrib['name']) for n in domain_info.findall('phosting/subdomain')}
        else:
            return set()

    def get_dns_settings(self):
        return DnsSettings(
            do_subdomains_own_zones=False,
        )

    def iter_dns_template_records(self):
        template_node = self.backup_info.find('server/dns-settings/dns-zone')
        for node in template_node.findall('dnsrec'):
            yield DnsRecord.parse(node)

    def iter_sites(self, subscription_name):
        """Iterate over all subscription sites: addon domains and subdomains.
        """
        return (
            Subdomain8(subscription_name, n, self.get_subscription(subscription_name))
            for n in self.domains_info[subscription_name].iterfind('phosting/subdomain')
        )

    def iter_addon_domains(self, subscription_name):
        """Get addon domains.
        """
        return []

    def iter_subdomains(self, subscription_name, domain_name):
        domain_info = self.domains_info[subscription_name]
        return (
            Subdomain8(subscription_name, i, self.get_subscription(subscription_name))
            for i in domain_info.iterfind('phosting/subdomain')
        )

    def iter_aliases(self, subscription_name):
        return (
            DomainAlias(subscription_name, node)
            for node in self.domains_info[subscription_name].iterfind('domain-alias')
        )

    def get_system_ips(self):
        return [parse_system_ip(n, self.is_windows) for n in self.backup_info.findall('server/system-ip')]

    def get_default_ip(self):
        return self.backup_info.findtext('server/default-ip')

    @property
    def is_windows(self):
        return False

    def save(self, fileobj, archive_format=None, prefix=None):
        with closing(create_xml_stream(self.backup_info)) as s:
            dump_content = s.getvalue()
        dump_msg = self.container.archive.files[self.container.backup_info_file]
        dump_msg.set_payload(dump_content)
        dump_msg.replace_header('Content-Length', len(dump_content))

        with closing(gzip.GzipFile(fileobj=fileobj)) as gzipped:
            gzipped.write(self.container.archive.msg.as_string())


def _convert_plesk8_auxiliary_user_permissions(permissions):
    """Convert Plesk 8 auxiliary user permissions to Plesk 10.x format"""

    permissions_map = {
        "manage_maillists": "mailListsManagement",  # Mailing lists management
        "manage_anonftp": "anonymousFtpManagement",  # Anonymous FTP management
        "manage_crontab": "scheduledTasksManagement",  # Scheduler management
        "manage_dns": "dnsManagement",  # DNS zone management
        "manage_log": "logRotationManagement",  # Log rotation management
        "manage_spamfilter": "spamfilterManagement",  # Spam filter management
        "manage_virusfilter": "antivirusManagement",  # Antivirus management
        "allow_ftp_backups": "backupRestoreManagement",  # Allow backing up and restoring data and us: ftp/local
        "manage_phosting": "webSitesAndDomainsManagement",  # Allow manage hosting: create/update/delete sites
        "manage_webapps": "applicationsManagement",  # Allow manage aps applications
        "manage_webstat": "browseStats",  # Allow to view statistics
        # Skipped permissions:
        # - select_db_server
        # - stdgui
        # - multiple-sessions
        # - manage_not_chroot_shell
        # - manage_quota
        # - manage_sh_access
        # - manage_subdomains
        # - manage_domain_aliases
        # - manage_ftp_password
        # - manage_dashboard
        # - dashboard
        # - allow_local_backups
    }

    result = {
        # default permissions
        'filesManagement': True,
        'mailManagement': True,
        'userManagement': False,
        'ftpAccountsManagement': False,
        'databasesManagement': True,
        'javaApplicationsManagement': False,
        'sitebuilderManagement': False,

    }
    for permission_name, permission_value in permissions.items():
        if permission_name in permissions_map:
            result[permissions_map[permission_name]] = permission_value
    return result


def _get_client_auxiliary_users_and_roles_for_domain_admin(client_node, domain_nodes=[]):
    """Return auxiliary users and roles for domain admin on Plesk 8/9
    
    Returns generator of tuples (AuxiliaryUser, AuxiliaryUserRole)
    where AuxiliaryUserRole is a special role created for domain user
    """

    roles = []
    users = []

    # Difference between Plesk 8 and Plesk 9 is location of domain node
    for domain_node in itertools.chain(
        multiple_findall(client_node, ['domain', 'domains/domain']),
        domain_nodes
    ):
        user_nodes = domain_node.findall('domainuser')
        if len(user_nodes) == 1:
            user_node = user_nodes[0]
            if user_node.attrib['status'] == 'true':
                personal_info = dict(
                    (pinfo_node.attrib['name'], default(pinfo_node.text, ''))
                    for pinfo_node in user_node.findall('pinfo') if pinfo_node.text is not None
                )
                # create special role for domain user and put all permissions in it
                role_name = u"Domain Administrator (%s)" % (domain_node.attrib['name'],)

                users.append(AuxiliaryUser(
                    node=user_node,
                    name=domain_node.attrib['name'],  # auxiliary user name is the same as domain name
                    contact=user_node.attrib.get('contact'),  # contact could be absent in PfU 8/9
                    email=personal_info.get('email'),  # email could be absend in PfU 8/9
                    roles=[role_name],
                    personal_info=personal_info,
                    subscription_name=domain_node.attrib['name'],
                    is_active=user_node.attrib['status'] == 'true',
                    is_domain_admin=True,
                    is_built_in=False,
                    external_email=user_node.attrib.get('external-email'),
                ))
                roles.append(AuxiliaryUserRole(
                    name=role_name,
                    permissions=_convert_plesk8_auxiliary_user_permissions(dict(
                        # Difference between Plesk 8 and Plesk 9 is name of attribute
                        (perm_node.attrib['name'], True if multiple_attrib_get(perm_node, ['allowed', 'value']) == 'true' else False)
                        for perm_node in user_node.findall('permission')
                        if multiple_attrib_get(perm_node, ['allowed', 'value']) in ('true', 'false')
                    )),
                    is_domain_admin=True
                ))
        elif len(user_nodes) > 1:
            # there is only one auxiliary user per domain, in PfU 8 it is called domain administrator
            raise Exception(
                messages.EXPECTED_HAVE_ONE_NO_DOMAINUSER_NODES % (
                    client_node.attrib['name'],
                )
            )

    return users, roles


def _get_client_auxiliary_users_and_roles_for_mailusers(client_node, domain_nodes=[]):
    """Return auxiliary users and roles for mail users on Plesk 8/9
    
    Returns list with 2 elements:
    - the first one is a list of AuxiliaryUser objects - users created for each mail user
    - the second one is a list of AuxiliaryUserRole objects - roles specially created for auxiliary users
    """

    roles = dict()
    users = []
    # Difference between Plesk 8 and Plesk 9 is location of domain node
    for domain_node in itertools.chain(
        multiple_findall(client_node, ['domain', 'domains/domain']),
        domain_nodes
    ):
        # Difference between Plesk 8 and Plesk 9 is location of mailuser node
        for mail_user_node in multiple_findall(domain_node, ['mailsystem/mailuser', 'mailsystem/mailusers/mailuser']):
            permissions = frozenset([
                permission_node.attrib['name'] 
                for permission_node in multiple_findall(
                    mail_user_node, ['mailuser-permission', 'limits-and-permissions/mailuser-permission']
                )
                # Difference between Plesk 8 and Plesk 9 is name of attribute
                if multiple_attrib_get(permission_node, ['allowed', 'value']) == 'true'
            ])
            
            if 'cp-access' in permissions:
                permission_names = {'manage-spamfilter': 'spamfilter', 'manage-virusfilter': 'virusfilter'}
                plesk_permission_names = {'manage-spamfilter': 'spamfilterManagement', 'manage-virusfilter': 'antivirusManagement'}

                # list is used to save order, so role naming will be the same as if role was created by Plesk restore
                role_permissions = tuple(
                    permission for permission in ['manage-spamfilter', 'manage-virusfilter']
                    if permission in permissions
                )

                role_name = u'Mail User (%s)' % (
                    "; ".join([permission_names[permission] for permission in role_permissions])
                )

                if role_permissions not in roles:  # add role according to permissions
                    roles[role_permissions] = AuxiliaryUserRole(
                        name=role_name,
                        permissions=dict((plesk_permission_names[permission], True) for permission in role_permissions),
                        is_domain_admin=False,
                    )
                name = u'%s@%s' % (mail_user_node.attrib['name'], domain_node.attrib['name'])

                # Difference between Plesk 8 and Plesk 9 is location of password node
                xml_node = mail_user_node.find('properties')
                if xml_node is None:
                    xml_node = mail_user_node
                password = get_password(xml_node)

                if password.password_text != '':
                    users.append(AuxiliaryUser(
                        node=mail_user_node,
                        name=name,
                        contact=name,
                        email=name,
                        roles=[role_name],
                        personal_info={},
                        subscription_name=domain_node.attrib['name'],
                        is_active=True,
                        is_domain_admin=False,
                        is_built_in=False,
                        external_email='',
                    ))
                else:
                    # there is no sense to create auxiliary user for mail user with
                    #  empty password (which is the same as no password) as before migration it:
                    # - was not able to login
                    # - was not presented as a separate entity
                    pass

    return users, roles.values()


def _extract_migration_list_data(migration_list_data):
    if migration_list_data is not None:
        return obj(
            subscriptions=set(migration_list_data.subscriptions_mapping.keys()),
            customers=migration_list_data.customers_mapping.keys(),
            resellers=migration_list_data.resellers,
            plans=migration_list_data.plans
        )
    else:
        return None


class SubscriptionNotFoundException(Exception):
    def __init__(self, name):
        super(Exception, self).__init__(
            messages.FAILED_FIND_SUBSCRIPTION_S_IN_BACKUP % (
                name
            )
        )
