import re
import sys
import traceback

# Python 2/3 compatibility
try:
    from httplib import IncompleteRead
except ImportError:
    from http.client import IncompleteRead
try:
    from urlparse import urlparse, urlunparse
except ImportError:
    from urllib.parse import urlparse, urlunparse

from parallels.core.logging import get_logger
from parallels.core.utils.common import unique_list, safe_string_repr, default, safe_format
from parallels.hosting_check import DomainIssue, Severity, WebsiteAvailabilityIssueType
from parallels.hosting_check.utils import http_client
from parallels.hosting_check.utils.html_parser import HtmlParser
from parallels.hosting_check.utils.logwatch import LogWatch
from parallels.hosting_check.messages import MSG
from parallels.hosting_check.utils.text_utils import format_string

logger = get_logger(__name__)


class WebsiteAvailabilityChecker(object):
    def __init__(self, check_timeout=30, depth=2, group_directories=True, messages=None):
        """
        :type check_timeout: int
        :type depth: int
        :type group_directories: bool
        :type messages: dict
        """
        self._check_timeout = check_timeout
        self._depth = depth
        self._group_directories = group_directories
        self._messages = messages

    def check(self, domains):
        """
        :type domains: list[parallels.hosting_check.DomainWebService]
        :rtype: list[parallels.hosting_check.DomainIssue]
        """
        issues = []
        for domain in domains:
            self._check_single_domain(domain, issues)
        return issues

    def _check_single_domain(self, domain, issues):
        """
        :type domain: parallels.hosting_check.DomainWebService
        :type issues: list[parallels.hosting_check.DomainIssue]
        :rtype: None
        """
        checked_urls = CheckedUrls()

        # Check main page
        self._check_domain_relative_url(
            domain, issues, domain.relative_url, checked_urls,
            depth=self._depth
        )

        # Check APS paths
        if len(domain.aps_domain_relative_urls) > 0:
            aps_domain_relative_urls = [
                '%s/' % domain_relative_url
                for domain_relative_url in domain.aps_domain_relative_urls
            ]
            with logger.debug_level_context(
                "Checking APS application URLs: '{urls}'", urls=aps_domain_relative_urls
            ):
                for domain_relative_url in aps_domain_relative_urls:
                    self._check_domain_relative_url(
                        domain, issues, domain_relative_url, checked_urls,
                        depth=self._depth
                    )

    def _check_domain_relative_url(self, domain, issues, domain_relative_url, checked_urls, depth):
        """
        :type domain: parallels.hosting_check.DomainWebService
        :type issues: list[parallels.hosting_check.DomainIssue]
        :type domain_relative_url: str | unicode
        :type checked_urls: parallels.hosting_check.checkers.website_availability_checker.CheckedUrls
        :type depth: int
        :rtype: None
        """
        try:
            already_checked = (
                # Exactly that URL was checked
                checked_urls.already_checked(domain_relative_url) or
                # Check by group is enabled and group which includes the URL was checked
                (
                    self._group_directories and
                    checked_urls.already_checked_group(domain_relative_url)
                )
            )
            if not already_checked:
                full_url = self._get_url(domain.protocol, domain.domain_name, domain_relative_url)
                with logger.info_progress_context(u"Check URL '{url}'", url=full_url):
                    response = self._check_domain_relative_url_plain(domain, domain_relative_url, issues)

                    if depth > 1 and response is not None:
                        self._fill_links(domain, domain_relative_url, response, checked_urls)

                    checked_urls.add(domain_relative_url)

            if depth > 1:
                for link in checked_urls.get_links(domain_relative_url):
                    self._check_domain_relative_url(domain, issues, link, checked_urls, depth - 1)
        except KeyboardInterrupt:
            # for compatibility with python 2.4
            raise
        except Exception as e:
            logger.exception()
            url = self._get_url(domain.protocol, domain.domain_name, domain_relative_url)
            issues.append(DomainIssue(
                domain_name=domain.domain_name, 
                severity=Severity.WARNING, 
                category=WebsiteAvailabilityIssueType.INTERNAL_ERROR,
                problem=self._msg(
                    'web_availability_internal_error',
                    error_message=safe_string_repr(e),
                    url=url, server_description=domain.server_description
                )
            ))

    def _check_domain_relative_url_plain(self, domain, domain_relative_url, issues):
        """
        :type domain: parallels.hosting_check.DomainWebService
        :type domain_relative_url: str | unicode
        :type issues: list[parallels.hosting_check.DomainIssue]
        :rtype: parallels.hosting_check.utils.http_client.HttpResponse
        """
        # Check the page itself
        if isinstance(domain_relative_url, str) and not self._url_is_utf_encoded(domain_relative_url):
            failed_url = unicode(domain_relative_url, errors='replace')
            issues.append(DomainIssue(
                domain_name=domain.domain_name,
                severity=Severity.WARNING,
                category=WebsiteAvailabilityIssueType.CANNOT_DECODE_URL,
                problem=self._msg(
                    'cannot_decode_url',
                    url=failed_url)))
            return None
        url = self._get_url(domain.protocol, domain.domain_name, domain_relative_url)
        url_for_log = self._get_url(
            domain.protocol, domain.domain_name, domain_relative_url, for_log=True
        )

        logger.debug(
            u"Target IP is: %s, source IP is: %s",
            domain.web_server_ip, domain.source_web_server_ip
        )

        logwatch = LogWatch(domain)
        logwatch.start()

        response = self._get_page(url, domain.web_server_ip, domain.server_description, domain.domain_name, issues)

        if response is None:
            return None

        source_url_for_log = None
        if (
                domain.source_web_server_ip is not None
                or domain.source_domain_name is not None
                or domain.source_protocol is not None
                or domain.source_relative_url is not None
        ):
            source_domain_name = default(domain.source_domain_name, domain.domain_name)
            source_protocol = default(domain.source_protocol, domain.protocol)

            source_domain_relative_url = self._get_source_domain_relative_url(domain, domain_relative_url)

            source_url = self._get_url(source_protocol, source_domain_name, source_domain_relative_url)
            source_url_for_log = self._get_url(
                source_protocol, source_domain_name, source_domain_relative_url, for_log=True
            )

            source_response = self._get_page(
                source_url, domain.source_web_server_ip, domain.source_server_description, source_domain_name,
                issues, Severity.WARNING
            )
        else:
            source_response = None

        if (
            response.code == 401 and
            source_response is not None and source_response.code == 401
        ):
            # 401 response code is valid in case of password-protected
            # area. That is a common case for default Plesk site, so we
            # don't consider it as failure. And we should not perform
            # any other kind of checks for such pages.
            return None

        if (
            response.code in (301, 302)
            and source_response is not None
            and source_response.code not in (301, 302)
        ):
            logger.debug("SEO redirect introduced on target, going to use redirection URL for checks")
            url = self._seo_equal_url(source_url_for_log, response, url)
            response = self._get_page(url, domain.web_server_ip, domain.server_description, domain.domain_name, issues)
            if response is None:
                return None

        bad_status_code = self._check_code(
            url_for_log, domain, response, source_response, issues, source_url_for_log
        )
        if not bad_status_code:
            self._check_title(
                url_for_log, domain, response, source_response, issues, source_url_for_log
            )

        logger.debug(
            u"Searching page contents and error logs for error messages"
        )
        self._check_signatures(
            url_for_log, domain, response, source_response, issues, source_url_for_log
        )
        self._check_error_logs(logwatch, domain, issues)

        return response

    @staticmethod
    def _get_source_domain_relative_url(domain, domain_relative_url):
        """
        :type domain: parallels.hosting_check.DomainWebService
        :type domain_relative_url: str | unicode
        :rtype: str | unicode
        """
        source_relative_url = domain.source_relative_url
        if source_relative_url is None:
            source_relative_url = domain.relative_url

        target_relative_url = domain.relative_url
        if domain_relative_url == target_relative_url:
            return source_relative_url

        if not domain_relative_url.startswith(domain.relative_url):
            raise Exception(safe_format(
                """Internal error: relative URL link '{link}' must start with relative base URL link '{base}'.""",
                link=domain_relative_url, base=domain.relative_url
            ))

        tail_url = domain_relative_url[len(target_relative_url):]

        return (source_relative_url.rstrip('/') + '/' + tail_url.lstrip('/')).lstrip('/')

    def _fill_links(self, domain, domain_relative_url, response, checked_urls):
        """
        :type domain: parallels.hosting_check.DomainWebService
        :type domain_relative_url: str | unicode
        :type response: parallels.hosting_check.utils.http_client.HttpResponse
        :type checked_urls: parallels.hosting_check.checkers.website_availability_checker.CheckedUrls
        :rtype: None
        """
        domain_url = self._get_url(domain.protocol, domain.domain_name, '')

        # Work with child pages

        links = []

        links.extend(self._get_page_links(
            domain_url, domain_relative_url, response.body, domain.www_alias_enabled,
            domain_relative_url=domain.relative_url
        ))
        links.extend(self._get_page_frames(
            domain_url, domain_relative_url, response.body, domain.www_alias_enabled,
            domain_relative_url=domain.relative_url
        ))

        if response.code in (301, 302):
            redirect_location = response.header('location')
            if redirect_location:
                redirect_path = self._get_local_link(
                    domain_url, domain_relative_url, redirect_location, domain.www_alias_enabled,
                    domain_relative_url=domain.relative_url
                )
                if redirect_path is not None:
                    links.append(redirect_path)

        checked_urls.set_links(domain_relative_url, unique_list(links))

    @classmethod
    def _get_url(cls, scheme, host, tail, for_log=False):
        """
        :type scheme: str | unicode
        :type host: str | unicode
        :type tail: str | unicode
        :type for_log: bool
        :rtype: str | unicode
        """
        return u"%s://%s/%s" % (
            scheme,
            host if for_log else host.encode('idna'),
            cls._normalize_path(tail)
        )

    @classmethod
    def _url_is_utf_encoded(cls, url):
        """Return true, if url is a valid UTF-8, otherwise return false."""
        try:
            unicode(url, errors='strict')
            return True
        except UnicodeDecodeError:
            invalid_url = unicode(url, errors='replace')
            logger.debug("Cannot decode the URL '%s'" % invalid_url)
        return False

    @classmethod
    def _normalize_path(cls, path):
        normalized_path_sections = []
        for path_part in path.split('/'):
            if path_part == '.':
                continue
            elif path_part == '..':
                if normalized_path_sections:
                    normalized_path_sections.pop()
            else:
                normalized_path_sections.append(path_part)

        normalized_path = '/'.join(normalized_path_sections)

        return normalized_path

    @classmethod
    def _get_page_links(cls, domain_url, path, page_contents, www_alias_enabled=False, domain_relative_url=None):
        """Return a list of unique relative links found on page."""
        links = HtmlParser.get_tag_attributes('a', ['href'], page_contents)
        return cls._get_local_links(domain_url, path, unique_list(links), www_alias_enabled, domain_relative_url)

    @classmethod
    def _get_page_frames(cls, domain_url, path, page_contents, www_alias_enabled=False, domain_relative_url=None):
        """Return a list of unique relative frames references found on page."""
        frames = HtmlParser.get_tag_attributes('frame', ['src'], page_contents)
        return cls._get_local_links(domain_url, path, set(frames), www_alias_enabled, domain_relative_url)

    @classmethod
    def _get_local_links(cls, domain_url, path, url_list, www_alias_enabled=False, domain_relative_url=None):
        """Filter URL list and return a list of local URLs for domain URL.
        
        This method searches for links that point to pages in the same domain.
        Returns:
            List of local relative URLs.
        """
        local_links = []
        for url in url_list:
            link = cls._get_local_link(domain_url, path, url, www_alias_enabled, domain_relative_url)
            if link is not None:
                local_links.append(link)
        return local_links

    @classmethod
    def _get_local_link(cls, domain_url, path, url, www_alias_enabled=False, domain_relative_url=None):
        (url_scheme, url_host, url_path, url_params, url_query, url_fragments) = urlparse(url)
        is_local = url_scheme == '' and url_host == ''
        if is_local:
            if url.startswith('/'):  # relative to domains's root
                return url.lstrip('/')
            elif '/' not in path:  # we're in domains's root
                return url
            else:  # relative to current path, which is not domain's root
                path_dir = path[:path.rfind('/')]
                return '%s/%s' % (path_dir, url)

        homepage_scheme, homepage_host = urlparse(domain_url)[:2]
        url_host_for_compare = url_host
        homepage_host_for_compare = homepage_host

        if www_alias_enabled:
            if url_host_for_compare.startswith('www.'):
                url_host_for_compare = url_host_for_compare[4:]
            if homepage_host_for_compare.startswith('www.'):
                homepage_host_for_compare = homepage_host_for_compare[4:]

        same_site = (
            url_scheme == homepage_scheme
            and url_host_for_compare.lower() == homepage_host_for_compare.lower()
            and (
                domain_relative_url is None
                or (
                    domain_relative_url is not None
                    and url_path.lstrip('/').startswith(domain_relative_url)
                )
            )
        )
        if same_site:
            return urlunparse(('', '', url_path, url_params, url_query, url_fragments)).lstrip('/')

    def _get_page(self, url, ip, server_description, domain_name, issues, severity=Severity.ERROR):
        """Return HTTP response

        :rtype: parallels.hosting_check.utils.http_client.HttpResponse | None
        """
        try:
            client = http_client.HttpClient(self._check_timeout)
            response = client.get(url, ip)
        except IncompleteRead as e:
            logger.debug("Error:\n%s", traceback.format_exc())
            issues.append(DomainIssue(
                domain_name=domain_name,
                severity=severity,
                category=WebsiteAvailabilityIssueType.FAILED_HTTP_REQUEST,
                problem=self._msg(
                    'web_availability_request_error',
                    server_description=server_description, url=url, error_message=safe_string_repr(e)
                )
            ))
            return None
            
        except IOError as e:
            logger.debug("Error:\n%s", traceback.format_exc())
            issues.append(DomainIssue(
                domain_name=domain_name,
                severity=severity,
                category=WebsiteAvailabilityIssueType.CONNECTION_ISSUE,
                problem=self._msg(
                    'web_availability_failed_to_connect',
                    server_description=server_description, url=url, error_message=safe_string_repr(e)
                )
            ))
            return None

        return response

    def _check_code(
        self, url, domain,
        response, source_response,
        issues, source_url=None
    ):
        """
        Returns:
            True if there are issues with status code,
            False otherwise

        :type url: str | unicode
        :type domain: parallels.hosting_check.DomainWebService
        :type response: parallels.hosting_check.utils.http_client.HttpResponse
        :type source_response: parallels.hosting_check.utils.http_client.HttpResponse | None
        :type issues: list
        :type source_url: str | unicode | None
        :rtype: bool
        """
        status_code_errors = {
            5: WebsiteAvailabilityIssueType.STATUS_CODE_5xx,
            4: WebsiteAvailabilityIssueType.STATUS_CODE_4xx,
        }

        if source_url is None:
            source_url = url

        if (
            source_response is not None and
            not self._status_codes_equal(source_response.code, response.code)
        ):
            issues.append(DomainIssue(
                domain_name=domain.domain_name,
                severity=Severity.ERROR,
                category=WebsiteAvailabilityIssueType.DIFFERENT_STATUS_CODE,
                problem=self._msg(
                    'web_availability_different_http_code',
                    source_server_description=domain.source_server_description,
                    source_status_code=source_response.code,
                    target_server_description=domain.server_description,
                    target_status_code=response.code,
                    url=url,
                    source_url=source_url
                )
            ))
            return True

        if response.code_class in status_code_errors:
            if source_response is None:
                issues.append(DomainIssue(
                    domain_name=domain.domain_name,
                    severity=Severity.ERROR,
                    category=status_code_errors[response.code_class],
                    problem=self._msg(
                        'web_availability_webserver_returned_http_code_target',
                        server_description=domain.server_description,
                        status_code=response.code,
                        url=url
                    )
                ))
            else:
                issues.append(DomainIssue(
                    domain_name=domain.domain_name,
                    severity=Severity.INFO,
                    category=status_code_errors[response.code_class],
                    problem=self._msg(
                        'web_availability_webserver_returned_http_code_both',
                        source_server_description=domain.source_server_description,
                        target_server_description=domain.server_description,
                        status_code=response.code,
                        url=url,
                        source_url=source_url
                    )
                ))

            return True

        return False

    @staticmethod
    def _status_codes_equal(source_code, target_code):
        """Check if we consider HTTP status codes equal
        """

        # During migration 301/302 codes may be exchanged,
        # that is an often situation when migrating from old Plesks
        # so we consider the equal
        redirect_codes = (301, 302)
        if source_code in redirect_codes and target_code in redirect_codes:
            return True
        return source_code == target_code

    @staticmethod
    def _seo_equal_url(source_url, target_response, target_url):
        """Check SEO redirect is switched on for migrated domain
        @type source_url: basestring | None
        @type target_response: parallels.hosting_check.utils.http_client.HttpResponse
        @type target_url: basestring
        @rtype: basestring
        """

        # During migration 301/302 codes may be exchanged,
        # that is an often situation when migrating from old Plesks
        # so we consider the equal
        if source_url is None:
            source_url = target_url
        location = target_response.header('Location')
        if location is not None:
            tmp_location = re.sub(r'^https?://', '', location)
        else:
            return source_url
        tmp_source_url = re.sub(r'^https?://', '', source_url)
        tmp_target_url = re.sub(r'^https?://', '', target_url)
        if tmp_source_url == tmp_target_url and tmp_location == tmp_target_url:
            return location
        m = re.search("(^https?://)", location)
        return m.group(1) + tmp_source_url

    def _check_signatures(
        self, url, domain,
        response, source_response,
        issues, source_url=None
    ):
        """Look web pages for signatures - indicators of an issue.

        For example, signature of a problem is "PHP Fatal error"
        string in page contents. Function checks both target
        and source page (if available), and emits different issues
        if signature was found on source server or not.

        Arguments:
        - domain - instance of DomainWebService with info
        about particular domain to perform signatures check; list 
        of signature objects (instance of Signature class) are provided
        here too, to make possible check different sites for different 
        signatures
        - response - response from a (target) server
        - source_response - response from a source server;
        may be None, if we have single server only, or there were issues
        fetching the page
        - issues - list of issues where we should add new found issues
        """
        if source_url is None:
            source_url = url

        for signature in domain.signatures:
            page_signature_text = signature.find(response.body)
            if source_response is not None:
                source_page_signature_text = signature.find(
                    source_response.body
                )

                if source_page_signature_text is not None and page_signature_text is not None:
                    issues.append(DomainIssue(
                        domain_name=domain.domain_name, 
                        severity=Severity.INFO,
                        category=signature.problem_found_on_source, 
                        problem=self._msg(
                            'web_availability_signature_found_on_source',
                            url=url,
                            source_url=source_url,
                            source_error_message=source_page_signature_text,
                            target_error_message=page_signature_text
                        )
                    ))
                elif page_signature_text is not None:
                    issues.append(DomainIssue(
                        domain_name=domain.domain_name, 
                        severity=Severity.ERROR, 
                        category=signature.problem_found_on_target_only, 
                        problem=self._msg(
                            'web_availability_signature_found_on_target',
                            url=url, error_message=page_signature_text
                        )
                    ))
            else:
                if page_signature_text is not None:
                    issues.append(DomainIssue(
                        domain_name=domain.domain_name, 
                        severity=Severity.ERROR, 
                        category=signature.problem_found_on_target_only, 
                        problem=self._msg(
                            'web_availability_signature_found_on_target',
                            url=url, error_message=page_signature_text
                        )
                    ))

    def _check_title(
        self, url, domain,
        response, source_response,
        issues, source_url=None
    ):
        """Compare HTML titles of migrated webpage.

        Arguments:
            url: URL, which is being checked
            domain: DomainWebService object
            response: target server response for URL
            source_response: source server response for URL
            issues: a list of issues to be updated
        Returns:
            True if there are issues with title,
            False otherwise

        :type url: str | unicode
        :type domain: parallels.hosting_check.DomainWebService
        :type response: parallels.hosting_check.utils.http_client.HttpResponse
        :type source_response: parallels.hosting_check.utils.http_client.HttpResponse | None
        :type issues: list
        :type source_url: str | unicode | None
        :rtype: bool
        """
        if source_response is None: 
            # check is applicable for migration scenarios, 
            # when both source and target servers are available
            return

        if source_url is None:
            source_url = url

        titles = []
        for page in [source_response.body, response.body]:
            title = HtmlParser.get_tag_content('title', page)
            titles.append(title)

        error_text = None
        if len(titles[0]) > len(titles[1]):
            error_text = self._msg(
                'web_availability_title_missing_target', url=url
            )
        elif len(titles[0]) < len(titles[1]):
            error_text = self._msg(
                'web_availability_title_missing_source', url=url
            )
        elif not titles[0] == titles[1]:
            def is_plesk_frame_forwarding(response_body):
                return 'Your browser does not support frames' in response_body

            def equal_idn_insensitive(list1, list2):
                list1_idna = []
                list2_idna = []
                for item in list1:
                    list1_idna.append(_safe_string_decode(item).encode('idna'))
                for item in list2:
                    list2_idna.append(_safe_string_decode(item).encode('idna'))
                return list1_idna == list2_idna

            if not (
                # Check for special case - frame forwarding migration from Plesk 9.x
                # when domain has national symbols. In that case, title of page is updated -
                # on source Plesk 9.x you have IDN-encoded name, on target Plesk you have national symbols
                # in UTF-8 in page title
                is_plesk_frame_forwarding(source_response.body) and
                equal_idn_insensitive(titles[0], titles[1])
            ):
                def compose_title(title_list):
                    return " ".join([
                        _safe_string_decode(t)
                        for t in title_list
                    ])

                error_text = self._msg(
                    'web_availability_title_changed',
                    source_url=source_url,
                    url=url,
                    title_source=compose_title(titles[0]),
                    title_target=compose_title(titles[1])
                )
        if error_text:
            issue = DomainIssue(
                domain_name=domain.domain_name,
                severity=Severity.ERROR, 
                category=WebsiteAvailabilityIssueType.DIFFERENT_TITLES, 
                problem=error_text,
            )
            issues.append(issue)
            return True 

        return False

    def _check_error_logs(self, logwatch, domain, issues):
        """Search in domain logs for errors; add errors to the list of issues"""
        for filename in logwatch.get_log_files():
            text = logwatch.get_new_records(filename) 
            for signature in domain.signatures:
                found_text = signature.find(text)
                if found_text is not None:
                    issues.append(DomainIssue(
                        domain_name=domain.domain_name, 
                        severity=Severity.ERROR, 
                        category=signature.problem_found_on_target_only, 
                        problem=self._msg(
                            'web_availability_signature_found_in_target_log',
                            error_log_file=filename,
                            error_message=found_text
                        )
                    ))

    def _msg(self, message_id, **format_args):
        """Get problem/log message by its ID and format it in string.format way"""
        if self._messages is not None and message_id in self._messages:
            return format_string(
                unicode(self._messages[message_id]),
                format_args
            )
        return MSG(message_id, **format_args)


class CheckedUrls(object):
    """Store which URLs were already checked to reduce number of checks"""

    def __init__(self):
        self._checked_urls = []
        self._links = {}

    def add(self, url):
        """Mark URL as checked

        url (string) - relative URL 

        :type url: str | unicode
        :rtype: None
        """
        self._checked_urls.append(url)

    def already_checked(self, url):
        """Return whether URL was already checked
        
        If URL was checked (no difference - recursively or not) - return True.
        Otherwise return False.

        :type url: str | unicode
        :rtype: bool
        """
        return url in self._checked_urls

    def already_checked_group(self, url):
        """Return whether URL of the same group was already checked

        We perform grouping to reduce number of checks. By group
        we mean pair (directory, extension). If you check each of such
        groups at least once, you have good probability to have:
        - scripting settings of each directory checked
        - permissions of each directory checked

        So, for example if you have pages '/news/index.php' and
        'news/top-news.php', we consider that it is enough to check only one
        of them.

        :type url: str | unicode
        :rtype: bool
        """
        checked_url_groups = set(
            self._classify_url(checked_url)
            for checked_url in self._checked_urls
        )
        url_group = self._classify_url(url)

        return url_group in checked_url_groups

    def set_links(self, url, links):
        """Set links - list of relative URLs to which specified URL refers.

        :type url: str | unicode
        :type links: list[str | unicode]
        :rtype: None
        """
        self._links[url] = links

    def get_links(self, url):
        """Get links - list of relative URLs to which specified URL refers.

        :type url: str | unicode
        :rtype: list[str | unicode]
        """
        return self._links.get(url, {})

    @staticmethod
    def _classify_url(url):
        """Classify URL by group to reduce number of checked URLs.
        
        Return a tuple of (directory, extension).
        
        Example:
            '/old/news/index.php'
            ->
            ('/old/news', 'php')

        :type url: str | unicode
        :rtype: tuple
        """
        if url == '':
            return '/', '', ''
        
        # Get path excluding URL parameters and fragments
        # So for example:
        # - for URL 'test/index.php?page=news' you get 'test/index.php'
        # - for URL 'index.php#top' you get 'index.php'
        path = urlparse(url)[2]

        if '/' in path:
            dirname, _ = path.rsplit('/', 1) 
        else:
            dirname = ''
        ext_index = path.rfind('.')
        if ext_index != -1:
            extension = path[ext_index:]
        else:
            extension = ''

        return dirname, extension


def _safe_string_decode(string):
    """Decode string to unicode, fallback to Python's repr in case of failure
    
    If string can be interpreted as UTF-8, then return it as unicode string.
    Otherwise return its __repr__.  Usually you want to get just plain string,
    but if there are some issues with encoding (for example, server returned
    some binary data) - at least get its Python representation.

    Consider that website page may be encoded in another encoding, for example,
    CP1251, or KOI8-R. This function is very simple, it does not handle this
    situation at all.  Better way is to use some encoding detection library.
    """
    try:
        return u"%s" % (string.decode('utf-8'),)
    except UnicodeError:
        return u"%r" % (string,)
