path: root/g_sorcery/package_db.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
    package_db.py
    ~~~~~~~~~~~~~
    
    package database
    
    :copyright: (c) 2013 by Jauhien Piatlicki
    :license: GPL-2, see LICENSE for more details.
"""

import glob
import hashlib
import multiprocessing
import os
import shutil
import sys

import portage

from .compatibility import basestring, py2k, TemporaryDirectory

from .exceptions import DBStructureError, IntegrityError, \
     InvalidKeyError, SyncError
from .fileutils import FileJSON, hash_file, load_remote_file, copy_all, wget
from .g_collections import Package
from .logger import Logger, ProgressBar


class PackageDB(object):
    """
    Package database.
    Database is a directory and related data structure.

    Directory layout.
    ~~~~~~~~~~~~~~~~~
    db dir
        manifest.json: database manifest
        categories.json: information about categories
        category1
            packages.json: list of packages
            package1
                versions.json: list of versions
                version1.json: description of a package
                version2.json: description of a package
                ...
            package2
            ...
        category2
        ...
    
    """

    class Iterator(object):
        """
        Iterator class over the package database.
        """
        def __init__(self, package_db):
            self.pkg_iter = iter(package_db.database.items())
            try:
                self.pkgname, self.vers_dict = next(self.pkg_iter)
            except StopIteration:
                self.pkgname, self.vers_dict = None, None
            if self.vers_dict:
                self.vers_iter = iter(self.vers_dict.items())
            else:
                self.vers_iter = None

        def __iter__(self):
            return self

        if py2k:
            def next(self):
                if not self.vers_iter:
                    raise StopIteration
                ver, ebuild_data = None, None
                while not ver:
                    try:
                        ver, ebuild_data = next(self.vers_iter)
                    except StopIteration:
                        ver, ebuild_data = None, None

                    if not ver:
                        self.pkgname, self.vers_dict = next(self.pkg_iter)
                        self.vers_iter = iter(self.vers_dict.items())

                category, name = self.pkgname.split('/')
                return (Package(category, name, ver), ebuild_data)
        else:
            def __next__(self):
                if not self.vers_iter:
                    raise StopIteration
                ver, ebuild_data = None, None
                while not ver:
                    try:
                        ver, ebuild_data = next(self.vers_iter)
                    except StopIteration:
                        ver, ebuild_data = None, None

                    if not ver:
                        self.pkgname, self.vers_dict = next(self.pkg_iter)
                        self.vers_iter = iter(self.vers_dict.items())

                category, name = self.pkgname.split('/')
                return (Package(category, name, ver), ebuild_data)


    def __init__(self, directory):
        """
        Args:
            directory: database directory.
        """
        self.logger = Logger()
        self.CATEGORIES_NAME = 'categories.json'
        self.PACKAGES_NAME = 'packages.json'
        self.VERSIONS_NAME = 'versions.json'
        self.directory = os.path.abspath(directory)
        self.reset_db()

    def __iter__(self):
        return(PackageDB.Iterator(self))

    def reset_db(self):
        """
        Reset database.
        """
        self.database = {}
        self.categories = {}

    def sync(self, db_uri):
        """
        Synchronize local database with remote database.

        Args:
            db_uri: URI for synchronization with remote database.
        """
        real_db_uri = self.get_real_db_uri(db_uri)
        download_dir = TemporaryDirectory()
        if wget(real_db_uri, download_dir.name):
            raise SyncError('sync failed: ' + real_db_uri)
        
        temp_dir = TemporaryDirectory()
        for f_name in glob.iglob(os.path.join(download_dir.name, '*.tar.gz')):
            self.logger.info("unpacking " + f_name)
            os.system("tar -xvzf " + f_name + " -C " + temp_dir.name)

        tempdb_dir = os.path.join(temp_dir.name, os.listdir(temp_dir.name)[0])
        tempdb = PackageDB(tempdb_dir)

        if not tempdb.check_manifest():
            raise IntegrityError('Manifest check failed.')

        self.logger.info("copy files to an actual database")
        self.clean()
        copy_all(tempdb_dir, self.directory)
        
        if not self.check_manifest():
            raise IntegrityError('Manifest check failed, db inconsistent.')
                
        del download_dir
        del temp_dir

    def get_real_db_uri(self, db_uri):
        """
        Convert self.db_uri to URI where remote database can be
        fetched from.

        Returns:
            URI of remote database file.
        """
        return db_uri
            
    def manifest(self):
        """
        Generate database manifest.
        """
        categories = FileJSON(self.directory, self.CATEGORIES_NAME, [])
        categories = categories.read()
        manifest = {}
        names = [self.CATEGORIES_NAME]
        for name in names:
            manifest[name] = hash_file(os.path.join(self.directory, name),
                                      hashlib.md5())
        for category in categories:
            category_path = os.path.join(self.directory, category)
            if not os.path.isdir(category_path):
                raise DBStructureError('Empty category: ' + category)
            for root, dirs, files in os.walk(category_path):
                for f in files:
                    manifest[os.path.join(root[len(self.directory)+1:], f)] = \
                    hash_file(os.path.join(root, f), hashlib.md5())
        m_f = FileJSON(self.directory, 'manifest.json', [])
        m_f.write(manifest)

    def _check_manifest_process(self, i, entries, errors):
        for name, value in entries:
            if hash_file(os.path.join(self.directory, name), hashlib.md5()) != \
                value:
                errors.append(name)


    def check_manifest(self):
        """
        Check database manifest.

        Returns:
            Tuple with first element containing result of manifest check
            as boolean and second element containing list of files with errors.
        """
        self.logger.info("checking manifest")
        m_f = FileJSON(self.directory, 'manifest.json', [])
        manifest = m_f.read()

        manager = multiprocessing.Manager()

        result = True
        errors = manager.list()

        names = [self.CATEGORIES_NAME]
        for name in names:
            if not name in manifest:
                raise DBStructureError('Bad manifest: no ' + name + ' entry')

        proc_num = multiprocessing.cpu_count() + 1

        portion = len(manifest) // proc_num
        procs = []
        entries = manifest.items()
        
        for i in range(proc_num - 1):
            procs.append(multiprocessing.Process(target=self._check_manifest_process, args=(i, entries[i * portion:(i+1) * portion - 1], errors)))
        procs.append(multiprocessing.Process(target=self._check_manifest_process, args=(proc_num - 1, entries[(proc_num - 1) * portion:], errors)))

        for proc in procs:
            proc.start()

        for proc in procs:
            proc.join()

        if errors:
            result = False

        return (result, errors)

    def clean(self):
        """
        Clean database.
        """
        if os.path.exists(self.directory):
            shutil.rmtree(self.directory)
        self.reset_db()
        self.write_and_manifest()

    def write_and_manifest(self):
        self.write()
        self.manifest()

    def write(self):
        """
        Write database.
        """
        categories_f = FileJSON(self.directory, self.CATEGORIES_NAME, [])
        categories_f.write(self.categories)

        if self.database:
            self.logger.info("writing database")

        progress_bar = ProgressBar(20, len(list(self.database)))
        if self.database:
                progress_bar.begin()
        
        for pkgname, versions in self.database.items():
            category, name = pkgname.split('/')
            if not category or (not category in self.categories):
                raise DBStructureError('Non existent: ' + category)
            for version, content in versions.items():
                f = FileJSON(os.path.join(self.directory, category, name),
                    version + '.json', [])
                f.write(content)
                self.additional_write_version(category, name, version)
            f = FileJSON(os.path.join(self.directory, category, name),
                self.VERSIONS_NAME, [])
            f.write(list(versions))
            self.additional_write_package(category, name)
            f = FileJSON(os.path.join(self.directory, category),
                self.PACKAGES_NAME, [])
            pkgs = f.read()
            if not pkgs:
                pkgs = []
            pkgs.append(name)
            f.write(pkgs)
            progress_bar.increment()

        for category in self.categories:
            self.additional_write_category(category)
            
        self.additional_write()

        if self.database:
            progress_bar.end()
            print("")

    def additional_write_version(self, category, package, version):
        """
        Hook to be overrided.
        """
        pass

    def additional_write_package(self, category, package):
        """
        Hook to be overrided.
        """
        pass

    def additional_write_category(self, category):
        """
        Hook to be overrided.
        """
        pass

    def additional_write(self):
        """
        Hook to be overrided.
        """
        pass

    def read(self):
        """
        Read database.
        """
        sane, errors = self.check_manifest()
        if not sane:
            raise IntegrityError('Manifest error: ' + str(errors))
        categories_f = FileJSON(self.directory, self.CATEGORIES_NAME, [])
        self.categories = categories_f.read()
        for category in self.categories:
            category_path = os.path.join(self.directory, category)
            if not os.path.isdir(category_path):
                raise DBStructureError('Empty category: ' + category)
            
            f = FileJSON(category_path, self.PACKAGES_NAME, [])
            packages = f.read()
            if not packages:
                raise DBStructureError('Empty category: ' + category)
            
            for name in packages:
                package_path = os.path.join(category_path, name)
                if not os.path.isdir(category_path):
                    error_msg = 'Empty package: ' + category + '/' + name
                    raise DBStructureError(error_msg)
                
                f = FileJSON(package_path, self.VERSIONS_NAME, [])
                versions = f.read()
                if not versions:
                    error_msg = 'Empty package: ' + category + '/' + name
                    raise DBStructureError(error_msg)

                pkgname = category + '/' + name
                self.database[pkgname] = {}
                for version in versions:
                    f = FileJSON(package_path, version + '.json', [])
                    ebuild_data = f.read()
                    self.database[pkgname][version] = ebuild_data
                    self.additional_read_version(category, name, version)
                self.additional_read_package(category, name)
            self.additional_read_category(category)
        self.additional_read()

    def additional_read_version(self, category, package, version):
        """
        Hook to be overrided.
        """
        pass

    def additional_read_package(self, category, package):
        """
        Hook to be overrided.
        """
        pass

    def additional_read_category(self, category):
        """
        Hook to be overrided.
        """
        pass

    def additional_read(self):
        """
        Hook to be overrided.
        """
        pass
        
    def add_category(self, category, description=None):
        """
        Add a category.

        Args:
            category: Category name.
            description: Category description.
        """
        if not description:
            description = {}
        self.categories[category] = description

    def add_package(self, package, ebuild_data=None):
        """
        Add a package.

        Args:
            package: package_db.Package instance.
            ebuild_data: Dictionary with package description.
        """
        if not ebuild_data:
            ebuild_data = {}
        category = package.category
        name = package.name
        version = package.version
        pkgname = category + '/' + name
        if category and not category in self.categories:
            raise InvalidKeyError('Non-existent category: ' + category)
        if pkgname and not pkgname in self.database:
            self.database[pkgname] = {}
        self.database[pkgname][version] = ebuild_data

    def list_categories(self):
        """
        List all categories.

        Returns:
            List with category names.
        """
        return list(self.categories)

    def in_category(self, category, name):
        """
        Tests whether a package is in a given category.

        Args:
            category: Category name.
            name: Package name.

        Returns:
            Boolean value.
        """
        if not category or (not category in self.categories):
            raise InvalidKeyError('No such category: ' + category)
        return (category + '/' + name) in self.database

    def list_package_names(self, category):
        """
        List package names in a category.

        Args:
            category: Category name.

        Returns:
            List of package names.
        """
        if not category or (not category in self.categories):
            raise InvalidKeyError('No such category: ' + category)
        res = [x.split('/')[1] for x in self.database
               if x.split('/')[0] == category]
        return res

    def list_catpkg_names(self):
        """
        List category/package entries.

        Returns:
            List with category/package entries.
        """
        return list(self.database)

    def list_package_versions(self, category, name):
        """
        List package versions.

        Args:
            category: Category name.
            name: package name.

        Returns:
            List of package versions.
        """
        if not category or (not category in self.categories):
            raise InvalidKeyError('No such category: ' + category)
        pkgname = category + '/' + name
        if not pkgname in self.database:
            raise InvalidKeyError('No such package: ' + pkgname)
        return list(self.database[pkgname])

    def list_all_packages(self):
        """
        List all packages in a database.

        Returns:
            List of package_db.Package instances.
        """
        result = []
        for pkgname, versions in self.database.items():
            for version in versions:
                category, name = pkgname.split('/')
                result.append(Package(category, name, version))
        return result

    def get_package_description(self, package):
        """
        Get package ebuild data.

        Args:
            package: package_db.Package instance.

        Returns:
            Dictionary with package ebuild data.
        """
        #a possible exception should be catched in the caller
        return self.database[package.category \
                             + '/' + package.name][package.version]

    def get_max_version(self, category, name):
        """
        Get the recent available version of a package.

        Args:
            category: Category name.
            name: package name.

        Returns:
            The recent version of a package.
        """
        pkgname = category + '/' + name
        if not pkgname in self.database:
            raise InvalidKeyError('No such package: ' + pkgname)
        versions = list(self.database[pkgname])
        max_ver = versions[0]
        for version in versions[1:]:
            if portage.pkgcmp(portage.pkgsplit(pkgname + '-' + version),
                              portage.pkgsplit(pkgname + '-' + max_ver)) > 0:
                max_ver = version
        return max_ver


class DBGenerator(object):
    """
    Generator for package databases.
    Creates new databases or syncs with existing.
    """

    __slots__ = ('package_db_class')

    def __init__(self, package_db_class=PackageDB):
        self.package_db_class = package_db_class

    def __call__(self, directory, repository,
                 common_config=None, config=None, generate=True):
        """
        Get a package database.

        A directory layout looks like:
        backend directory
            config.json
            repo1 dir
                config.json
                db - directory with database
            repo2 dir
                config.json
                db - directory with database

        Args:
            directory: Directory for package databases (backend directory).
            repository: Repository name.
            common_config: Backend config.
            config: Repository config.
            generate: Whether package tree should be generated.

        Returns:
            Package database.
        """
        db_path = os.path.join(directory, repository, "db")
        pkg_db = self.package_db_class(db_path)

        config_f = FileJSON(os.path.join(directory, repository),
                            "config.json", [])
        if config:
            config_f.write(config)
        else:
            config = config_f.read()

        common_config_f = FileJSON(directory, "config.json", [])
        if common_config:
            common_config_f.write(common_config)
        else:
            common_config = common_config_f.read()

        if generate:
            pkg_db.clean()
            self.generate_tree(pkg_db, common_config, config)
            #todo: make db write on every add_package and only necessary info
            pkg_db.write()
            pkg_db.manifest()
        return pkg_db

    def generate_tree(self, pkg_db, common_config, config):
        """
        Generate package entries.

        Args:
            pkg_db: Package database.
            common_config: Backend config.
            config: Repository config.
        """
        data = self.download_data(common_config, config)
        self.process_data(pkg_db, data, common_config, config)

    def download_data(self, common_config, config):
        """
        Obtain data for database generation.

        Args:
            common_config: Backend config.
            config: Repository config.

        Returns:
            Downloaded data.
        """
        uries = self.get_download_uries(common_config, config)
        uries = self.decode_download_uries(uries)
        data = {}
        for uri in uries:
            self.process_uri(uri, data)
        return data

    def process_uri(self, uri, data):
        """
        Download and parse data from a given URI.

        Args:
            uri: URI.
            data: Data dictionary.
        """
        data.update(load_remote_file(**uri))

    def get_download_uries(self, common_config, config):
        """
        Get uries to download from.

        Args:
            common_config: Backend config.
            config: Repository config.

        Returns:
            List with URI entries.
            Each entry has one of the following formats:
            1. String with URI.
            2. A dictionary with entries:
                - uri: URI.
                - parser: Parser to be applied to downloaded data.
                - open_file: Whether parser accepts file objects.
                - open_mode: Open mode for a downloaded file.
                The only mandatory entry is uri.
        """
        return [config["repo_uri"]]

    def decode_download_uries(self, uries):
        """
        Convert URI list with incomplete and string entries
        into list with complete dictionary entries.

        Args:
            uries: List of URIes.

        Returns:
            List of URIes with dictionary entries.
        """
        decoded = []
        for uri in uries:
            decuri = {}
            if isinstance(uri, basestring):
                decuri["uri"] = uri
                decuri["parser"] = self.parse_data
                decuri["open_file"] = True
                decuri["open_mode"] = "r"
            else:
                decuri = uri
                if not "parser" in decuri:
                    decuri["parser"] = self.parse_data
                if not "open_file" in decuri:
                    decuri["open_file"] = True
                if not "open_mode" in decuri:
                    decuri["open_mode"] = "r"
            decoded.append(decuri)
        return decoded

    def parse_data(self, data_file):
        """
        Parse downloaded data.

        Args:
            data_file: File name of open file object.

        Returns:
            Parsed data.
        """
        #todo: implement reasonable default
        pass

    def process_data(self, pkg_db, data, common_config, config):
        """
        Process downloaded data and fill database.

        Args:
            pkg_db: Package database.
            data: Dictionary with data, keys are file names.
            common_config; Backend config.
            config: Repository config.
        """
        pass

    def convert(self, configs, dict_name, value):
        """
        Convert a value using configs.
        This function is aimed to be used for conversion
        of values such as license or package names.

        Args:
            configs: List of configs.
            dict_name: Name of a dictionary in config
        that should be used for conversion.
            value: Value to convert.

        Returns:
            Converted value.
        """
        result = value
        for config in configs:
            if config:
                if dict_name in config:
                    transform = config[dict_name]
                    if value in transform:
                        result = transform[value]
        return result

    def convert_dependency(self, configs, dependency, external=True):
        """
        Convert dependency.

        Args:
            configs: List of configs.
            dependency: Dependency to be converted.
            external: Whether external deps should be keeped.

        Returns:
            Right dependency name or None if dependency is external and
        external == False.
        """
        external_dep = ""
        for config in configs:
            if config:
                if "external" in config:
                    ext_deps = config["external"]
                    if dependency in ext_deps:
                        external_dep = ext_deps[dependency]
        if external_dep:
            if external:
                return self.convert_external_dependency(configs, dependency)
            else:
                return None
        else:
            return self.convert_internal_dependency(configs, dependency)

    def convert_internal_dependency(self, configs, dependency):
        """
        Hook to convert internal dependencies.
        """
        return dependency

    def convert_external_dependency(self, configs, dependency):
        """
        Hook to convert external dependencies.
        """
        return dependency
        
    def in_config(self, configs, list_name, value):
        """
        Check whether value is in config.

        Args:
            configs: List of configs.
            list_name: Name of a list in config
        that should be used for checking.
            value: Value to look for.

        Returns:
            Boolean.
        """
        result = False
        for config in configs:
            if config:
                if list_name in config:
                    if value in config[list_name]:
                        result = True
                        break
        return result