import logging
import os
import re
import socket
import ssl
from collections import OrderedDict
from imaplib import IMAP4, IMAP4_SSL, IMAP4_SSL_PORT, IMAP4_PORT
from time import sleep

from plesk_mail_migrator.core.entities.imap_flags import IMAPFlags
from plesk_mail_migrator.core.entities.utils.imap_flags_utils import IMAPFlagsUtils
from plesk_mail_migrator.core.entities.utils.mail_message_folder_utils import MailMessageFolderUtils
from plesk_mail_migrator.core.entities.utils.mail_message_utils import MailMessageUtils
from plesk_mail_migrator.core.progress.backup.backup_progress import BackupProgress
from plesk_mail_migrator.core.progress.backup.backup_progress_empty import BackupProgressEmpty
from plesk_mail_migrator.core.provider.backup_provider import BackupProvider
from plesk_mail_migrator.core.provider.provider_boolean_parameter import ProviderBooleanParameter
from plesk_mail_migrator.core.provider.provider_enum_parameter import ProviderEnumParameter
from plesk_mail_migrator.core.provider.provider_integer_parameter import ProviderIntegerParameter
from plesk_mail_migrator.core.provider.provider_parameter import ProviderParameter
from plesk_mail_migrator.providers.imap.exceptions import ImapAutodetectFailedException, ImapOverSSLIssue, \
    ImapOverSSLInvalidCertificateIssue
from plesk_mail_migrator.providers.imap.server_detect.detect import detect_alive_imap_server, \
    IMAP_HOST_DETECTION_METHOD_MANUAL
from plesk_mail_migrator.providers.imap.server_detect.utils import GOOGLE_DNS
from plesk_mail_migrator.utils.list_utils import get_last
from plesk_mail_migrator.utils.string_utils import split_quoted_string, is_empty, safe_utf8_decode, safe_string_repr

logger = logging.getLogger(__name__)


class IMAPProvider(BackupProvider):
    IMAP_HOST_PARAMETER = "host"
    IMAP_PORT_PARAMETER = "port"
    IMAP_FOLDER_SEPARATOR = "folder-separator"
    IMAP_TIMEOUT_PARAMETER = "timeout"
    IMAP_ALLOW_INSECURE_CONNECTIONS = "allow-insecure-connections"
    IMAP_CA_BUNDLE = "ca-bundle"
    IMAP_PROTOCOL = "imap-protocol"
    IMAP_MESSAGE_ID_CACHE_FILE = "message-ids-cache-file"
    IMAP_MAX_RETRY = "max-retry"
    IMAP_RETRY_INTERVAL = "retry-interval"
    IMAP_AUTODETECT_DNS_SERVER = "autodetect-dns-server"

    DEFAULT_FOLDER_SEPARATOR = '.'
    SMARTER_MAIL_FOLDER_SEPARATOR = '/'

    SSL_VERSIONS = OrderedDict([
        ('AUTO SELECT PROTOCOL', None),
        ('TLSv1.2', ssl.PROTOCOL_TLSv1_2),
        ('TLSv1.1', ssl.PROTOCOL_TLSv1_1),
        ('TLSv1', ssl.PROTOCOL_TLSv1)
    ])

    def __init__(self):
        self._parameters = [
            ProviderParameter(self.IMAP_HOST_PARAMETER, "IMAP host", None),
            ProviderParameter(self.IMAP_PORT_PARAMETER, "IMAP port", None),
            ProviderParameter(
                self.IMAP_FOLDER_SEPARATOR,
                "IMAP folder separator, automatically detected if not specified",
                None, "separator"
            ),
            ProviderParameter(
                self.IMAP_CA_BUNDLE,
                "Path to SSL CA bundle file (in PEM format) to verify certificates of the server",
                None, "filepath"
            ),
            ProviderBooleanParameter(
                self.IMAP_ALLOW_INSECURE_CONNECTIONS,
                "Whether to allow insecure connections: plain IMAP without encryption, "
                "IMAP over SSL with expired certificate, IMAP over SSL with self-signed certificate, etc",
                False, "true|false"
            ),
            ProviderIntegerParameter(
                self.IMAP_TIMEOUT_PARAMETER,
                "TCP timeout for IMAP connections (15 seconds by default)",
                15, "timeout"
            ),
            ProviderEnumParameter(
                self.IMAP_PROTOCOL,
                "Which IMAP protocol to use: "
                "IMAP ('imap'), IMAP over SSL ('imap-over-ssl'), or detect automatically ('auto')",
                IMAPProtocol.get_all_values(),
                IMAPProtocol.AUTO, "|".join(IMAPProtocol.get_all_values())
            ),
            ProviderParameter(
                self.IMAP_MESSAGE_ID_CACHE_FILE,
                "File which stores mapping between IMAP message UIDs and messages IDs used for migration",
                None, "filepath"
            ),
            ProviderIntegerParameter(
                self.IMAP_MAX_RETRY,
                "Maximum number of attempts to perform an operation by IMAP in case of errors (e.g. broken connection)",
                3, "max-retry"
            ),
            ProviderIntegerParameter(
                self.IMAP_RETRY_INTERVAL,
                "Interval between retry attempts (in seconds) in case of errors when "
                "performing IMAP operation (e.g. broken connection)",
                5, "retry-interval"
            ),
            ProviderParameter(
                self.IMAP_AUTODETECT_DNS_SERVER,
                "DNS server to use during IMAP host autodetection",
                GOOGLE_DNS, "hostname-or-ip"
            ),
        ]
        self._imap_protocol = None
        self._imap_host = None
        self._imap_port = None
        self._account = None

    def get_parameters(self):
        return self._parameters

    def get_title(self):
        return 'IMAP'

    def get_provider_id(self):
        return 'imap'

    def get_imap_vendor(self):
        try:
            # parse string like
            # * OK [CAPABILITY IMAP4rev1 SASL-IR LOGIN-REFERRALS ID ENABLE IDLE LITERAL+ STARTTLS AUTH=PLAIN AUTH=LOGIN AUTH=DIGEST-MD5 AUTH=CRAM-MD5] Dovecot ready.
            # to extract "Dovecot" string
            matched = re.match('.*OK (\[.*\]).* (.+) ready.*', self._imap.welcome)
            if matched:
                return matched.group(2)
        except Exception as e:
            logger.debug("Can not process IMAP welcome message to get imap vendor due to exception: %s" % safe_string_repr(e))
        return "Unknown"

    def do_backup_messages(self, account, dumper, exclude_message_ids, backup_progress, stop_mark):
        """Dump (backup) messages of specified mail account.

        :param plesk_mail_migrator.core.entities.mail_account.MailAccount account:
            Mail account for which we should dump messages
        :param plesk_mail_migrator.core.dumps.dump_writer.DumpWriter dumper:
            Object capable of writing dumps
        :param set[str] exclude_message_ids:
            Messages to exclude when dumping. Check "How the tool works with message IDs" in README for details
        :param plesk_mail_migrator.core.progress.backup.backup_progress.BackupProgress backup_progress:
            Object to report backup progress to
        :param plesk_mail_migrator.utils.stop_mark.StopMark stop_mark:
            Object which indicates if user has requested to stop backup/migration
        :rtype: list[str]
        """
        errors = []
        self._account = account
        self._login(account, backup_progress)
        backup_progress.set_imap_vendor(self.get_imap_vendor())

        if stop_mark.is_set():
            logger.debug("Backup was interrupted due to user request")
            return errors

        backup_progress.set_step(BackupProgress.STEP_TRANSFER_MESSAGES)
        backup_errors = self._backup_messages(dumper, exclude_message_ids, stop_mark)
        errors.extend(backup_errors)

        self._logout(account)

        return errors

    def _backup_messages(self, dumper, exclude_message_ids, stop_mark):
        """Dump (backup) messages, considering we are logged into IMAP server.

        :type dumper: plesk_mail_migrator.core.dumps.dump_writer.DumpWriter
        :type exclude_message_ids: set[str]
        :type stop_mark: plesk_mail_migrator.utils.stop_mark.StopMark
        :rtype: list[str]
        """
        errors = []

        dumped_messages = 0
        skipped_messages = 0

        message_ids_cache = MessageIdsCacheFile(self.get_parameter_value(self.IMAP_MESSAGE_ID_CACHE_FILE))

        list_result = self._run_imap_operation(
            lambda: self._imap.list()[1],
            "Failed to list IMAP folders: {message}"
        )

        for imap_dir_info_str in list_result:
            if stop_mark.is_set():
                message_ids_cache.save()
                logger.debug("Backup was interrupted due to user request")
                return errors

            imap_dir_info = split_quoted_string(imap_dir_info_str)
            imap_dir_name = get_last(imap_dir_info)
            logger.debug("Dump IMAP folder '%s'", imap_dir_name)

            try:
                folder = MailMessageFolderUtils.pack(imap_dir_name, self._folder_separator)

                select_result = self._run_imap_operation(
                    lambda: self._imap.select(imap_dir_name, readonly=True)[1][0],
                    "Failed to select IMAP folder '%s': {message}" % imap_dir_name
                )
                try:
                    messages_count = int(select_result)
                except ValueError:
                    logger.debug("Skip dumping IMAP folder: %s", select_result)
                    continue

                logger.debug("IMAP folder contains %s messages", messages_count)

                if messages_count == 0:
                    continue

                message_uids = []
                search_all_result = self._run_imap_operation(
                    lambda: self._imap.uid('SEARCH', 'ALL'),
                    "Failed to get list of message UIDs: {message}",
                    imap_dir_name
                )
                if len(search_all_result) > 1 and len(search_all_result[1]) > 0:
                    message_uids = search_all_result[1][0].strip().split(' ')

                folder_uidvalidity = self._detect_folder_uidvalidity(imap_dir_name)

                for message_num, message_uid in enumerate(message_uids, start=1):
                    if stop_mark.is_set():
                        message_ids_cache.save()
                        logger.debug("Backup was interrupted due to user request")
                        return errors

                    try:
                        logger.debug(
                            "Dump mail message #%s (UID %s) in the IMAP folder '%s'",
                            message_num, message_uid, imap_dir_name
                        )

                        cached_message_id = message_ids_cache.get_message_id(
                            imap_dir_name, folder_uidvalidity, message_uid
                        )
                        if cached_message_id is not None:
                            logger.debug("Message ID taken from cache: %s", cached_message_id)
                            if cached_message_id in exclude_message_ids:
                                logger.debug("Message is skipped as it is in exclude message IDs list")
                                skipped_messages += 1
                            else:
                                message_content = self._fetch_message_content(message_uid, imap_dir_name)
                                flags = self._fetch_message_flags(message_uid, imap_dir_name)
                                logger.debug("Dump message")
                                dumper.dump_from_contents(cached_message_id, folder, flags, message_content)
                                dumped_messages += 1
                        else:
                            message_content = self._fetch_message_content(message_uid, imap_dir_name)

                            message_id = MailMessageUtils.get_normalized_message_id(message_content)
                            logger.debug("Message ID: %s", message_id)
                            message_ids_cache.set_message_id(
                                imap_dir_name, folder_uidvalidity, message_uid, message_id
                            )

                            if message_id in exclude_message_ids:
                                logger.debug("Message is skipped as it is in exclude message IDs list")
                                skipped_messages += 1
                            else:
                                logger.debug("Dump message")
                                flags = self._fetch_message_flags(message_uid, imap_dir_name)
                                dumper.dump_from_contents(message_id, folder, flags, message_content)
                                dumped_messages += 1
                    except FatalException:
                        raise
                    except Exception as e:
                        logger.debug("Exception: ", exc_info=True)
                        errors.append(
                            "Failed to backup message #%s (UID %s) in IMAP folder '%s': %s" % (
                                message_num, message_uid, imap_dir_name, str(e)
                            )
                        )
            except FatalException:
                raise
            except Exception as e:
                logger.debug("Exception: ", exc_info=True)
                errors.append(
                    "Failed to backup IMAP folder '%s': %s. All messages of that folder will not be migrated" % (
                        imap_dir_name, str(e)
                    )
                )

        message_ids_cache.save()

        logger.debug("Dumping of mail messages finished")
        logger.debug("Total dumped messages: %s", dumped_messages)
        logger.debug("Total skipped messages: %s", skipped_messages)

        return errors

    def _fetch_message_content(self, message_uid, imap_dir_name):
        message_content = ''
        for attempt in range(2):  # 2 attempts for Office 365, see comments below
            # Perform 2 separate requests for message contents and message flags:
            # requesting them at the same time requires more complex parsing.
            # Consider to perform one request as future performance optimization.
            message_fetch_response = self._run_imap_operation(
                lambda: self._imap.uid('FETCH', message_uid, "(BODY.PEEK[])"),
                "Failed to fetch message body: {message}",
                imap_dir_name
            )

            # For calendar and contact items which reside inside of a regular IMAP folder,
            # Office 365 sometimes (not always) returns responses like:
            # ('OK', [None])
            # On the 2nd attempt it returns a correct response which contains
            # information how to reach a specific calendar/contact item using Outlook Web Access.
            # So here, if we have None response, let's try once more
            if (
                attempt == 0 and
                message_fetch_response is None or
                message_fetch_response[1] is None or
                message_fetch_response[1][0] is None or
                message_fetch_response[1][0][1] is None
            ):
                continue

            message_content = message_fetch_response[1][0][1]

            # Do not perform the 2nd attempt, the 2nd attempt is necessary only for Office 365
            # in certain conditions, see comment above where we continue to the next attempt
            break
        return message_content

    def _fetch_message_flags(self, message_uid, imap_dir_name):
        flags_fetch_response = self._run_imap_operation(
            lambda: self._imap.uid("FETCH", message_uid, "(FLAGS)"),
            "Failed to fetch message flags: {message}",
            imap_dir_name
        )
        flags_data = flags_fetch_response[1][0]
        flags_match = re.search(r'FLAGS\s*\((.*?)\)', flags_data)
        if flags_match:
            flags = IMAPFlagsUtils.flags_from_string(flags_match.group(1))
        else:
            flags = IMAPFlags()
        return flags

    def _detect_folder_uidvalidity(self, folder_name):
        """Get UIDVALIDITY property of a folder. In case of failure just return 0.

        :type folder_name: str | unicode
        :rtype: int
        """
        try:
            uidvalidity = 0
            status = self._run_imap_operation(
                lambda: self._imap.status(folder_name, '(UIDVALIDITY)'),
                "Failed to retrieve UIDVALIDITY of folder '%s': {message}" % folder_name,
                folder_name
            )
            if len(status) > 1 and len(status[1]) > 0:
                status_data = status[1][0]
                uidvalidity_match = re.search(r'UIDVALIDITY\s*(\d+)', status_data)
                if uidvalidity_match is not None:
                    uidvalidity = int(uidvalidity_match.group(1))
                    logger.debug("Detected UIDVALIDITY of IMAP folder '%s': %s", folder_name, str(uidvalidity))
            return uidvalidity
        except FatalException:
            raise
        except Exception:
            logger.debug("Exception: ", exc_info=True)
            logger.error(
                "Failed to get UIVALIDITY of IMAP folder '%s', messages re-sync may work incorrectly",
                folder_name
            )
            return 0

    def _login(self, account, backup_progress):
        """Login to the server by IMAP.

        :type account: plesk_mail_migrator.core.entities.mail_account.MailAccount
        :param plesk_mail_migrator.core.progress.backup.backup_progress.BackupProgress backup_progress:
            Object to report backup progress to
        :rtype: None
        """
        try:
            self._connect(backup_progress)
        except ImapOverSSLInvalidCertificateIssue as e:
            backup_progress.set_error(BackupProgress.ERROR_SECURITY_INVALID_CERTIFICATE, str(e))
            raise
        except ImapOverSSLIssue as e:
            backup_progress.set_error(BackupProgress.ERROR_SECURITY, str(e))
            raise
        except ImapAutodetectFailedException as e:
            backup_progress.set_error(BackupProgress.ERROR_IMAP_HOST_AUTODETECT, str(e))
            raise
        except Exception as e:
            backup_progress.set_error(BackupProgress.ERROR_CONNECT, str(e))
            raise

        account_login = self._get_account_login(account)
        logger.debug("Login by IMAP to account %s", account_login)
        backup_progress.set_step(BackupProgress.STEP_LOGIN)
        try:
            self._imap.login(account_login, account.user_password)
            self._account_login = account_login
            self._account_password = account.user_password
        except Exception as e:
            backup_progress.set_error(BackupProgress.ERROR_AUTHENTICATION, str(e))
            raise

        logger.debug("Detect IMAP folder separator")

        self._folder_separator = self.get_parameter_value(self.IMAP_FOLDER_SEPARATOR)

        if self._folder_separator is not None:
            logger.debug("Use IMAP folder separator specified by argument: %s", self._folder_separator)
        else:
            logger.debug("Try to detect IMAP folder separator with NAMESPACE IMAP command")
            backup_progress.set_step(BackupProgress.STEP_DETECT_FOLDER_SEPARATOR)
            error_reason = ''
            try:
                namespace_info = self._imap.namespace()
                namespace_match = re.match(r'^\(\("[^)]*" "([^)]*)"\)\)', namespace_info[1][0])

                if namespace_match:
                    self._folder_separator = namespace_match.group(1)
                    logger.debug("Detected IMAP folder separator by namespace command: %s", self._folder_separator)
                else:
                    error_reason = 'have not found separator in reply to NAMESPACE command'
            except IMAP4.error as e:
                error_reason = str(e)

            if self._folder_separator is None:
                if self._imap.welcome is not None and 'smartermail' in self._imap.welcome.lower():
                    self._folder_separator = self.SMARTER_MAIL_FOLDER_SEPARATOR
                    logger.debug(
                        "Failed to detect IMAP folder separator from the server: %s. "
                        "As it looks like SmarterMail, use default separator of SmarterMail: %s",
                        error_reason,
                        self._folder_separator
                    )
                else:
                    self._folder_separator = self.DEFAULT_FOLDER_SEPARATOR
                    logger.debug(
                        "Failed to detect IMAP folder separator from the server: %s. "
                        "Use default IMAP folder separator: %s",
                        error_reason,
                        self._folder_separator
                    )

    def _connect(self, backup_progress=None):
        """
        :param plesk_mail_migrator.core.progress.backup.backup_progress.BackupProgress | None backup_progress:
            Object to report backup progress to
        :rtype: None
        """
        if backup_progress is None:
            backup_progress = BackupProgressEmpty()

        allow_insecure_connections = self.get_parameter_value(self.IMAP_ALLOW_INSECURE_CONNECTIONS)
        imap_port_param_value = self.get_parameter_value(self.IMAP_PORT_PARAMETER)
        if self._imap_protocol is not None:
            imap_protocol = self._imap_protocol
        else:
            imap_protocol = self.get_parameter_value(self.IMAP_PROTOCOL)
        imap_timeout = self.get_parameter_value(self.IMAP_TIMEOUT_PARAMETER)

        dns_server = self.get_parameter_value(self.IMAP_AUTODETECT_DNS_SERVER)
        ca_bundle = self.get_parameter_value(self.IMAP_CA_BUNDLE)

        if imap_port_param_value is not None:
            imap_plain_port = imap_port_param_value
            imap_ssl_port = imap_port_param_value
        else:
            imap_plain_port = 143
            imap_ssl_port = 993

        user_defined_imap_host = self._get_imap_host()

        account_domain = self._account.domain
        if is_empty(account_domain) and '@' in self._account.user_name:
            _, account_domain = self._account.user_name.split('@', 1)

        sock = None
        detect_method_id = None

        if imap_protocol == IMAPProtocol.AUTO:
            if self._imap_host is not None and self._imap_port is not None:
                hostname, port = self._imap_host, self._imap_port
            elif not is_empty(user_defined_imap_host):
                hostname, port, detect_method_id = (
                    user_defined_imap_host, imap_ssl_port, IMAP_HOST_DETECTION_METHOD_MANUAL
                )
            else:
                backup_progress.set_step(BackupProgress.STEP_DETECT_IMAP_HOST)
                hostname, port, sock, detect_method_id = detect_alive_imap_server(
                    account_domain, [imap_ssl_port, imap_plain_port], imap_timeout, dns_server, ca_bundle
                )
                if backup_progress is not None:
                    backup_progress.set_autodetected_imap_host(hostname)

            backup_progress.set_imap_host_detection_method(detect_method_id)
            backup_progress.set_step(BackupProgress.STEP_CONNECT)

            try:
                logger.debug("Try to connect by IMAP over SSL to %s:%s", hostname, port)
                self._connect_ssl(
                    hostname,
                    port,
                    ca_bundle,
                    allow_insecure_connections,
                    imap_timeout,
                    sock=sock
                )
                logger.debug("Successfully connected using IMAP over SSL")
                self._imap_protocol = IMAPProtocol.IMAP_OVER_SSL
                self._imap_host = hostname
                self._imap_port = port
            except Exception as e:
                if allow_insecure_connections:
                    logger.debug("Exception: ", exc_info=True)
                    logger.debug(
                        "Failed to connect with IMAP over SSL, try to connect without using SSL to %s:%s",
                        hostname, port
                    )
                    if not is_empty(user_defined_imap_host):
                        hostname, port, sock = user_defined_imap_host, imap_plain_port, None
                    else:
                        # that means to reconnect, we can not reuse socket to which we already tried to
                        # establish SSL connection; open another "clean" connection
                        sock = None
                    self._imap = MailMigratorImap4(
                        hostname,
                        port,
                        imap_timeout,
                        sock=sock
                    )
                    logger.debug("Successfully connected using IMAP without using SSL")
                    self._imap_protocol = IMAPProtocol.IMAP
                    self._imap_host = hostname
                    self._imap_port = port
                else:
                    if is_ssl_validation_error(e):
                        raise ImapOverSSLInvalidCertificateIssue(str(e))
                    else:
                        raise ImapOverSSLIssue(str(e))
        elif imap_protocol == IMAPProtocol.IMAP_OVER_SSL:
            logger.debug("Try to connect with IMAP over SSL")

            if self._imap_host is not None and self._imap_port is not None:
                hostname, port = self._imap_host, self._imap_port
            elif not is_empty(user_defined_imap_host):
                hostname, port, detect_method_id = (
                    user_defined_imap_host, imap_ssl_port, IMAP_HOST_DETECTION_METHOD_MANUAL
                )
            else:
                backup_progress.set_step(BackupProgress.STEP_DETECT_IMAP_HOST)
                hostname, port, sock, detect_method_id = detect_alive_imap_server(
                    account_domain, [imap_ssl_port], imap_timeout, dns_server, ca_bundle
                )
                backup_progress.set_autodetected_imap_host(hostname)

            backup_progress.set_imap_host_detection_method(detect_method_id)
            backup_progress.set_step(BackupProgress.STEP_CONNECT)

            try:
                self._connect_ssl(
                    hostname,
                    port,
                    ca_bundle,
                    allow_insecure_connections,
                    imap_timeout,
                    sock=sock
                )
            except ssl.SSLError as e:
                if not allow_insecure_connections:
                    logger.debug("Exception: ", exc_info=True)
                    if is_ssl_validation_error(e):
                        raise ImapOverSSLInvalidCertificateIssue(str(e))
                    else:
                        raise ImapOverSSLIssue(str(e))
                else:
                    raise

            logger.debug("Successfully connected using IMAP over SSL")
            self._imap_protocol = IMAPProtocol.IMAP_OVER_SSL
            self._imap_host = hostname
            self._imap_port = port
        else:
            logger.debug("Try to connect using IMAP without using SSL")

            if self._imap_host is not None and self._imap_port is not None:
                hostname, port = self._imap_host, self._imap_port
            elif not is_empty(user_defined_imap_host):
                hostname, port, detect_method_id = (
                    user_defined_imap_host, imap_plain_port, IMAP_HOST_DETECTION_METHOD_MANUAL
                )
            else:
                backup_progress.set_step(BackupProgress.STEP_DETECT_IMAP_HOST)
                hostname, port, sock, detect_method_id = detect_alive_imap_server(
                    account_domain, [imap_plain_port], imap_timeout, dns_server, ca_bundle
                )
                if backup_progress is not None:
                    backup_progress.set_autodetected_imap_host(hostname)

            backup_progress.set_imap_host_detection_method(detect_method_id)
            backup_progress.set_step(BackupProgress.STEP_CONNECT)

            self._imap = MailMigratorImap4(
                hostname,
                port,
                imap_timeout,
                sock=sock
            )
            logger.debug("Successfully connected using IMAP without using SSL")
            self._imap_protocol = IMAPProtocol.IMAP
            self._imap_host = hostname
            self._imap_port = port

    def _connect_ssl(self, hostname, port, ca_bundle, allow_insecure_connections, imap_timeout, sock):
        for ssl_protocol_name, ssl_protocol in self.SSL_VERSIONS.items():
            logger.debug("Try to connect by IMAP over SSL using %s", ssl_protocol_name)
            try:
                self._imap = MailMigratorImap4Ssl(
                    hostname,
                    port,
                    ca_bundle,
                    allow_insecure_connections,
                    imap_timeout,
                    sock=sock,
                    ssl_version=ssl_protocol
                )
                break
            except ssl.SSLError as e:
                if is_unsupported_protocol_error(e) or is_wrong_ssl_version_error(e):
                    logger.debug("Exception: ", exc_info=True)
                    logger.debug("Failed to connect by IMAP over SSL using %s", ssl_protocol_name)
                    sock = None
                    if self.SSL_VERSIONS.keys().index(ssl_protocol_name) >= len(self.SSL_VERSIONS) - 1:
                        raise
                    continue
                raise

    def _logout(self, account):
        """Logout from IMAP server.

        :type account: plesk_mail_migrator.core.entities.mail_account.MailAccount
        :rtype: None
        """
        logger.debug("Logout account %s by IMAP", self._get_account_login(account))
        self._imap.logout()

    def _run_imap_operation(self, imap_operation_function, error_message, imap_dir_name=None):
        max_retry = self.get_parameter_value(self.IMAP_MAX_RETRY)
        retry_interval = self.get_parameter_value(self.IMAP_RETRY_INTERVAL)

        error = None
        result = None

        for retry in range(max_retry):
            error = None
            try:
                if retry > 0:
                    logger.info("Try to reconnect to the source server by IMAP")

                    try:
                        self._imap.shutdown()
                    except:  # we are not interested in details, just try to close connection, and continue anyway
                        pass

                    try:
                        self._connect()
                        self._imap.login(self._account_login, self._account_password)
                        if imap_dir_name is not None:
                            self._imap.select(imap_dir_name, readonly=True)
                    except Exception as e:
                        if retry == max_retry - 1:
                            raise FatalException(error_message.format(message=str(e)))
                        else:
                            raise

                result = imap_operation_function()

                if retry > 0:
                    logger.info("Operation finished successfully after {retry} retry".format(retry=retry))
                break
            except FatalException:
                raise
            except Exception as e:
                logger.warn(
                    "Operation failed: {message}. Retrying in {interval} seconds... (attempt {retry})".format(
                        message=e, interval=retry_interval, retry=retry + 1
                    )
                )
                sleep(self.get_parameter_value(self.IMAP_RETRY_INTERVAL))

                error = e

        if error:
            raise Exception(error_message.format(message=error))
        else:
            return result

    @staticmethod
    def _get_account_login(account):
        """
        :type account: plesk_mail_migrator.core.entities.mail_account.MailAccount
        :rtype: str | unicode
        """
        if is_empty(account.domain):
            if '@' in account.user_name:
                user_name, domain = account.user_name.split('@', 1)
                return '%s@%s' % (
                    user_name,
                    # Convert to punycode, as mail servers (at least configured by Plesk) usually
                    # don't accept logins in unicode form.
                    safe_utf8_decode(domain).encode('idna')
                )
            else:
                return account.user_name
        else:
            return '%s@%s' % (
                account.user_name,
                # Convert to punycode, as mail servers (at least configured by Plesk)
                # usually don't accept logins in unicode form.
                safe_utf8_decode(account.domain).encode('idna')
            )

    def _get_imap_host(self):
        imap_host = self.get_parameter_value(self.IMAP_HOST_PARAMETER)
        if imap_host is None:
            return None
        else:
            return safe_utf8_decode(imap_host).encode('idna')


class MailMigratorImap4Ssl(IMAP4_SSL):
    """Wrapper around IMAP4_SSL which allows more strict checks for SSL certificate validity"""

    def __init__(
        self, host='', port=IMAP4_SSL_PORT,
        ca_bundle=None, allow_insecure_connections=False, timeout=None,
        sock=None, ssl_version=None
    ):
        self.host = None
        self.port = None
        self.sock = sock
        self.sslobj = None
        self.file = None

        self._allow_insecure_connections = allow_insecure_connections
        self._ca_bundle = ca_bundle
        self._timeout = timeout
        self._ssl_version = ssl_version

        IMAP4_SSL.__init__(self, host, port)

    def open(self, host='', port=IMAP4_SSL_PORT):
        """Override method of IMAP4_SSL to add check for valid certificate, and add timeout option"""
        self.host = host
        self.port = port
        if self.sock is None:
            if self._timeout is not None:
                self.sock = socket.create_connection((host, port), timeout=self._timeout)
            else:
                self.sock = socket.create_connection((host, port))

        if self._ssl_version is not None:
            self.sslobj = ssl.wrap_socket(
                self.sock, self.keyfile, self.certfile,
                ca_certs=self._ca_bundle, cert_reqs=not self._allow_insecure_connections, ssl_version=self._ssl_version
            )
        else:
            self.sslobj = ssl.wrap_socket(
                self.sock, self.keyfile, self.certfile,
                ca_certs=self._ca_bundle, cert_reqs=not self._allow_insecure_connections
            )

        self.file = self.sslobj.makefile('rb')


class MailMigratorImap4(IMAP4):
    def __init__(self, host='', port=IMAP4_PORT, timeout=None, sock=None):
        self._timeout = timeout
        self.sock = sock
        IMAP4.__init__(self, host, port)

    def open(self, host='', port=IMAP4_PORT):
        """Override method of IMAP4 to add timeout option"""
        self.host = host
        self.port = port
        if self.sock is None:
            if self._timeout is not None:
                self.sock = socket.create_connection((host, port), self._timeout)
            else:
                self.sock = socket.create_connection((host, port))
        self.file = self.sock.makefile('rb')


class IMAPProtocol(object):
    @classmethod
    def get_all_values(cls):
        return [cls.IMAP, cls.IMAP_OVER_SSL, cls.AUTO]

    IMAP = 'imap'
    IMAP_OVER_SSL = 'imap-over-ssl'
    AUTO = 'auto'


def is_ssl_validation_error(e):
    return 'CERTIFICATE_VERIFY_FAILED' in repr(e).upper()


def is_unsupported_protocol_error(e):
    return 'UNSUPPORTED_PROTOCOL' in repr(e).upper()


def is_wrong_ssl_version_error(e):
    return 'WRONG_SSL_VERSION' in repr(e).upper()


class MessageIdsCacheFile(object):
    """File which caches message ID for each dumped/transferred message.

    By message ID we mean ID used by mail migrator (both Linux and Windows)
    to identify if the message was already transferred or not.
    Usually for IMAP it is taken from "Message-ID" header,
    but when header is absent, hash of the message contents is used.

    So, to calculate message ID when using IMAP you have to get whole contents of the message,
    which is a slow operation.
    (Note: other ways to transfer mail messages could have their own ways to calculate message ID).

    To speed up detection if the message was already transferred we have cache of calculated message IDs
    for each dumped/transferred message. We identify each message by 3 parameters (and all of them are necessary):
    - IMAP folder name
    - IMAP folder UIDVALIDITY attribute
    - Mail message UID
    Check RFC for more details:
    https://tools.ietf.org/html/draft-ietf-imap-disc-00
    https://tools.ietf.org/html/rfc3501 (pay attention to https://tools.ietf.org/html/rfc3501#section-2.3.1.1)
    """

    def __init__(self, filename):
        self._filename = filename
        self._data = self._load()

    def get_message_id(self, folder_name, folder_uidvalidity, message_uid):
        """Get cached message ID

        :type folder_name: str | unicode
        :type folder_uidvalidity: int | str | unicode
        :type message_uid: int | str | unicode
        :rtype: str | unicode | None
        """
        return self._data.get((folder_name, str(folder_uidvalidity), str(message_uid)))

    def set_message_id(self, folder_name, folder_uidvalidity, message_uid, message_id):
        """Update cache - set message ID for specified message

        :type folder_name: str | unicode
        :type folder_uidvalidity: int | str | unicode
        :type message_uid: int | str | unicode
        :type message_id: str | unicode
        :rtype: None
        """
        self._data[(folder_name, str(folder_uidvalidity), str(message_uid))] = message_id

    def _load(self):
        if self._filename and os.path.exists(self._filename):
            try:
                result = {}
                with open(self._filename, 'rb') as fp:
                    for line in fp:
                        items = line.rstrip("\n").split("\t")
                        if len(items) == 4:
                            folder_name, folder_uidvalidity, message_uid, message_id = items
                            result[(folder_name, folder_uidvalidity, message_uid)] = message_id
                return result
            except Exception:
                logger.debug("Exception: ", exc_info=True)
                logger.warn("Failed to load message IDs cache from file %s", self._filename)
                return {}
        else:
            return {}

    def save(self):
        if self._filename is None:
            return

        # First, write data to a temporary file
        temp_filename = "%s.new" % self._filename
        with open(temp_filename, 'wb') as fp:
            first = True
            for (folder_name, folder_uidvalidity, message_uid), message_id in self._data.items():
                if first:
                    first = False
                else:
                    fp.write("\n")

                fp.write("\t".join([folder_name, folder_uidvalidity, message_uid, message_id]))

        # Second, rename temporary file
        if os.path.exists(self._filename):
            # Remove the file if it exists - otherwise rename will fail on Windows
            os.unlink(self._filename)
        # Rename operation is atomic, so whole contents is inside of the destination file
        os.rename(temp_filename, self._filename)


class FatalException(Exception):
    """
    Exception which means that there is no sense to continue backup/restore/migration
    For example: we completely can not connect to the source IMAP server after several retries
    """
    pass
