initial import

src/ahriman/application/ahriman.py

@@ -0,0 +1,76 @@
+import argparse
+import os
+
+import ahriman.version as version
+
+from ahriman.application.application import Application
+from ahriman.core.configuration import Configuration
+
+
+def _get_app(args: argparse.Namespace) -> Application:
+    config = _get_config(args.config)
+    return Application(config)
+
+
+def _get_config(config_path: str) -> Configuration:
+    config = Configuration()
+    config.load(config_path)
+    config.load_logging()
+    return config
+
+
+def _remove_lock(path: str) -> None:
+    try:
+        os.remove(path)
+    except FileNotFoundError:
+        pass
+
+
+def add(args: argparse.Namespace) -> None:
+    _get_app(args).add(args.package)
+
+
+def remove(args: argparse.Namespace) -> None:
+    _get_app(args).remove(args.package)
+
+
+def update(args: argparse.Namespace) -> None:
+    _get_app(args).update()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='ArcHlinux ReposItory MANager')
+    parser.add_argument('-c', '--config', help='configuration path', default='/etc/ahriman.ini')
+    parser.add_argument('--force', help='force run, remove file lock', action='store_true')
+    parser.add_argument('--lock', help='lock file', default='/tmp/ahriman.lock')
+    parser.add_argument('-v', '--version', action='version', version=version.__version__)
+    subparsers = parser.add_subparsers(title='commands')
+
+    add_parser = subparsers.add_parser('add', description='add package')
+    add_parser.add_argument('package', help='package name', nargs='+')
+    add_parser.set_defaults(fn=add)
+
+    remove_parser = subparsers.add_parser('remove', description='remove package')
+    remove_parser.add_argument('package', help='package name', nargs='+')
+    remove_parser.set_defaults(fn=remove)
+
+    update_parser = subparsers.add_parser('update', description='run updates')
+    update_parser.set_defaults(fn=update)
+
+    args = parser.parse_args()
+
+    if args.force:
+        _remove_lock(args.lock)
+    if os.path.exists(args.lock):
+        raise RuntimeError('Another application instance is run')
+
+    if 'fn' not in args:
+        parser.print_help()
+        exit(1)
+
+    try:
+        open(args.lock, 'w').close()
+        args.fn(args)
+    finally:
+        _remove_lock(args.lock)
+

src/ahriman/application/application.py

@@ -0,0 +1,29 @@
+import os
+
+from typing import List
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.repository import Repository
+from ahriman.core.task import Task
+from ahriman.models.package import Package
+
+
+class Application:
+
+    def __init__(self, config: Configuration) -> None:
+        self.config = config
+        self.repository = Repository(config)
+
+    def add(self, names: List[str]) -> None:
+        for name in names:
+            package = Package.load(name, self.config.get('aur', 'url'))
+            task = Task(package, self.config, self.repository.paths)
+            task.fetch(os.path.join(self.repository.paths.manual, package.name))
+
+    def remove(self, names: List[str]) -> None:
+        self.repository.process_remove(names)
+
+    def update(self) -> None:
+        updates = self.repository.updates()
+        packages = self.repository.process_build(updates)
+        self.repository.process_update(packages)

src/ahriman/core/configuration.py

@@ -0,0 +1,40 @@
+import configparser
+import os
+
+from logging.config import fileConfig
+from typing import Any, Dict, Optional
+
+from ahriman.core.exceptions import MissingConfiguration
+
+
+# built-in configparser extension
+class Configuration(configparser.RawConfigParser):
+
+    def __init__(self) -> None:
+        configparser.RawConfigParser.__init__(self, allow_no_value=True)
+        self.path = None  # type: Optional[str]
+
+    @property
+    def include(self) -> str:
+        return self.get('settings', 'include')
+
+    def get_section(self, section: str) -> Dict[str, str]:
+        if not self.has_section(section):
+            raise MissingConfiguration(section)
+        return dict(self[section])
+
+    def load(self, path: str) -> None:
+        self.path = path
+        self.read(self.path)
+        self.load_includes()
+
+    def load_includes(self) -> None:
+        try:
+            include_dir = self.include
+            for conf in filter(lambda p: p.endswith('.ini'), sorted(os.listdir(include_dir))):
+                self.read(os.path.join(self.include, conf))
+        except (FileNotFoundError, configparser.NoOptionError):
+            pass
+
+    def load_logging(self) -> None:
+        fileConfig(self.get('settings', 'logging'))

src/ahriman/core/exceptions.py

@@ -0,0 +1,21 @@
+from typing import Any
+
+
+class BuildFailed(Exception):
+    def __init__(self, package: str) -> None:
+        Exception.__init__(self, f'Package {package} build failed, check logs for details')
+
+
+class InvalidOptionException(Exception):
+    def __init__(self, value: Any) -> None:
+        Exception.__init__(self, f'Invalid or unknown option value `{value}`')
+
+
+class InvalidPackageInfo(Exception):
+    def __init__(self, details: Any) -> None:
+        Exception.__init__(self, f'There are errors during reading package information: `{details}`')
+
+
+class MissingConfiguration(Exception):
+    def __init__(self, name: str) -> None:
+        Exception.__init__(self, f'No section `{name}` found')

src/ahriman/core/repo_wrapper.py

@@ -0,0 +1,40 @@
+import logging
+import os
+
+from ahriman.core.exceptions import BuildFailed
+from ahriman.core.util import check_output
+from ahriman.models.repository_paths import RepositoryPaths
+
+
+class RepoWrapper:
+
+    def __init__(self, name: str, paths: RepositoryPaths) -> None:
+        self.logger = logging.getLogger('build_details')
+        self.name = name
+        self.paths = paths
+
+    @property
+    def repo_path(self) -> str:
+        return os.path.join(self.paths.repository, f'{self.name}.db.tar.gz')
+
+    def add(self, path: str) -> None:
+        check_output(
+            'repo-add', '-R', self.repo_path, path,
+            exception=BuildFailed(path),
+            cwd=self.paths.repository,
+            logger=self.logger)
+
+    def remove(self, path: str, package: str) -> None:
+        try:
+            os.remove(path)
+        except FileNotFoundError:
+            pass
+        try:
+            os.remove(f'{path}.sig')  # sign if any
+        except FileNotFoundError:
+            pass
+        check_output(
+            'repo-remove', self.repo_path, package,
+            exception=BuildFailed(path),
+            cwd=self.paths.repository,
+            logger=self.logger)

src/ahriman/core/repository.py

@@ -0,0 +1,123 @@
+import logging
+import os
+import shutil
+
+from typing import List
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.repo_wrapper import RepoWrapper
+from ahriman.core.sign import Sign
+from ahriman.core.task import Task
+from ahriman.models.package import Package
+from ahriman.models.repository_paths import RepositoryPaths
+
+
+class Repository:
+
+    def __init__(self, config: Configuration) -> None:
+        self.logger = logging.getLogger('builder')
+        self.config = config
+
+        self.aur_url = config.get('aur', 'url')
+        self.name = config.get('repository', 'name')
+
+        self.paths = RepositoryPaths(config.get('repository', 'root'))
+        self.paths.create_tree()
+
+        self.sign = Sign(config)
+        self.wrapper = RepoWrapper(self.name, self.paths)
+
+    def _clear_build(self) -> None:
+        for package in os.listdir(self.paths.sources):
+            shutil.rmtree(os.path.join(self.paths.sources, package), ignore_errors=True)
+
+    def _clear_manual(self) -> None:
+        for package in os.listdir(self.paths.manual):
+            shutil.rmtree(os.path.join(self.paths.manual, package), ignore_errors=True)
+
+    def _clear_packages(self) -> None:
+        for package in os.listdir(self.paths.packages):
+            shutil.rmtree(os.path.join(self.paths.packages, package), ignore_errors=True)
+
+    def process_build(self, updates: List[Package]) -> List[str]:
+        for package in updates:
+            try:
+                task = Task(package, self.config, self.paths)
+                task.fetch()
+                built = task.build()
+                for src in built:
+                    dst = os.path.join(self.paths.packages, os.path.basename(src))
+                    shutil.move(src, dst)
+            except Exception:
+                self.logger.exception(f'{package.name} build exception', exc_info=True)
+                continue
+        self._clear_build()
+
+        return [
+            os.path.join(self.paths.packages, fn)
+            for fn in os.listdir(self.paths.packages)
+        ]
+
+    def process_remove(self, packages: List[str]) -> str:
+        for fn in os.listdir(self.paths.repository):
+            if '.pkg.' not in fn:
+                continue
+
+            full_path = os.path.join(self.paths.repository, fn)
+            try:
+                local = Package.load(full_path, self.aur_url)
+                if local.name not in packages:
+                    continue
+                self.wrapper.remove(full_path, local.name)
+            except Exception:
+                self.logger.exception(f'could not load package from {fn}', exc_info=True)
+                continue
+
+        self.sign.sign_repository(self.wrapper.repo_path)
+        return self.wrapper.repo_path
+
+    def process_update(self, packages: List[str]) -> str:
+        for package in packages:
+            files = self.sign.sign_package(package)
+            for src in files:
+                dst = os.path.join(self.paths.repository, os.path.basename(src))
+                shutil.move(src, dst)
+            package_fn = os.path.join(self.paths.repository, os.path.basename(package))
+            self.wrapper.add(package_fn)
+        self._clear_packages()
+
+        self.sign.sign_repository(self.wrapper.repo_path)
+        return self.wrapper.repo_path
+
+    def updates(self) -> List[Package]:
+        result: List[Package] = []
+        checked_base: List[str] = []
+
+        # repository updates
+        for fn in os.listdir(self.paths.repository):
+            if '.pkg.' not in fn:
+                continue
+
+            try:
+                local = Package.load(os.path.join(self.paths.repository, fn), self.aur_url)
+                remote = Package.load(local.name, self.aur_url)
+            except Exception:
+                self.logger.exception(f'could not load package from {fn}', exc_info=True)
+                continue
+            if local.name in checked_base:
+                continue
+
+            if local.is_outdated(remote):
+                result.append(remote)
+            checked_base.append(local.name)
+
+        # manual updates
+        for fn in os.listdir(self.paths.manual):
+            local = Package.load(os.path.join(self.paths.manual, fn), self.aur_url)
+            if local.name in checked_base:
+                continue
+            result.append(local)
+            checked_base.append(local.name)
+        self._clear_manual()
+
+        return result

src/ahriman/core/sign.py

@@ -0,0 +1,44 @@
+import logging
+import os
+
+from typing import List
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.exceptions import BuildFailed
+from ahriman.core.util import check_output
+from ahriman.models.sign_settings import SignSettings
+
+
+class Sign:
+
+    def __init__(self, config: Configuration) -> None:
+        self.logger = logging.getLogger('build_details')
+
+        self.key = config.get('sign', 'key', fallback=None)
+        self.sign = SignSettings.from_option(config.get('sign', 'enabled'))
+
+    def process(self, path: str) -> List[str]:
+        cwd = os.path.dirname(path)
+        check_output(
+            *self.sign_cmd(path),
+            exception=BuildFailed(path),
+            cwd=os.path.dirname(path),
+            logger=self.logger)
+        return [path, f'{path}.sig']
+
+    def sign_cmd(self, path: str) -> List[str]:
+        cmd = ['gpg']
+        if self.key is not None:
+            cmd.extend(['-u', self.key])
+        cmd.extend(['-b', path])
+        return cmd
+
+    def sign_package(self, path: str) -> List[str]:
+        if self.sign != SignSettings.SignPackages:
+            return [path]
+        return self.process(path)
+
+    def sign_repository(self, path: str) -> List[str]:
+        if self.sign != SignSettings.SignRepository:
+            return [path]
+        return self.process(path)

src/ahriman/core/task.py

@@ -0,0 +1,54 @@
+import os
+import logging
+import shutil
+
+from typing import List, Optional
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.exceptions import BuildFailed
+from ahriman.core.util import check_output
+from ahriman.models.package import Package
+from ahriman.models.repository_paths import RepositoryPaths
+
+
+class Task:
+
+    def __init__(self, package: Package, config: Configuration, paths: RepositoryPaths) -> None:
+        self.logger = logging.getLogger('builder')
+        self.build_logger = logging.getLogger('build_details')
+        self.package = package
+        self.paths = paths
+
+        self.archbuild_flags = config.get('build', 'archbuild_flags').split()
+        self.extra_build = config.get('build', 'extra_build')
+        self.makepkg_flags = config.get('build', 'makepkg_flags').split()
+        self.multilib_build = config.get('build', 'multilib_build')
+
+    @property
+    def git_path(self) -> str:
+        return os.path.join(self.paths.sources, self.package.name)
+
+    def build(self) -> List[str]:
+        build_tool = self.multilib_build if self.package.is_multilib else self.extra_build
+
+        cmd = [build_tool, '-r', self.paths.chroot]
+        cmd.extend(self.archbuild_flags)
+        if self.makepkg_flags:
+            cmd.extend(['--', '--'] + self.makepkg_flags)
+        self.logger.info(f'using {cmd} for {self.package.name}')
+
+        check_output(
+            *cmd,
+            exception=BuildFailed(self.package.name),
+            cwd=self.git_path,
+            logger=self.build_logger)
+
+        # well it is not actually correct, but we can deal with it
+        return check_output('makepkg', '--packagelist',
+                            exception=BuildFailed(self.package.name),
+                            cwd=self.git_path).splitlines()
+
+    def fetch(self, path: Optional[str] = None) -> None:
+        git_path = path or self.git_path
+        shutil.rmtree(git_path, ignore_errors=True)
+        check_output('git', 'clone', self.package.url, git_path, exception=None)

src/ahriman/core/util.py

@@ -0,0 +1,20 @@
+import subprocess
+
+from logging import Logger
+from typing import Optional
+
+
+def check_output(*args: str, exception: Optional[Exception],
+                 cwd = None, stderr: int = subprocess.STDOUT,
+                 logger: Optional[Logger] = None) -> str:
+    try:
+        result = subprocess.check_output(args, cwd=cwd, stderr=stderr).decode('utf8').strip()
+        if logger is not None:
+            for line in result.splitlines():
+                logger.debug(line)
+    except subprocess.CalledProcessError as e:
+        if e.output is not None and logger is not None:
+            for line in e.output.decode('utf8').splitlines():
+                logger.debug(line)
+        raise exception or e
+    return result

src/ahriman/models/package.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import aur
+import os
+
+from configparser import RawConfigParser
+from dataclasses import dataclass
+from srcinfo.parse import parse_srcinfo
+from typing import Type
+
+from ahriman.core.exceptions import InvalidPackageInfo
+from ahriman.core.util import check_output
+
+
+@dataclass
+class Package:
+    name: str
+    version: str
+    url: str
+
+    @property
+    def is_multilib(self) -> bool:
+        return self.name.startswith('lib32-')
+
+    @classmethod
+    def from_archive(cls: Type[Package], path: str, aur_url: str) -> Package:
+        name, version = check_output('expac', '-p', '%n %v', path, exception=None).split()
+        return cls(name, version, f'{aur_url}/{name}.git')
+
+    @classmethod
+    def from_aur(cls: Type[Package], name: str, aur_url: str)-> Package:
+        package = aur.info(name)
+        return cls(package.name, package.version, f'{aur_url}/{name}.git')
+
+    @classmethod
+    def from_build(cls: Type[Package], path: str) -> Package:
+        git_config = RawConfigParser()
+        git_config.read(os.path.join(path, '.git', 'config'))
+
+        with open(os.path.join(path, '.SRCINFO')) as fn:
+            src_info, errors = parse_srcinfo(fn.read())
+        if errors:
+            raise InvalidPackageInfo(errors)
+
+        return cls(src_info['pkgbase'], f'{src_info["pkgver"]}-{src_info["pkgrel"]}',
+                   git_config.get('remote "origin"', 'url'))
+
+    @classmethod
+    def load(cls: Type[Package], path: str, aur_url: str) -> Package:
+        try:
+            if os.path.isdir(path):
+                package: Package = cls.from_build(path)
+            elif os.path.exists(path):
+                package = cls.from_archive(path, aur_url)
+            else:
+                package = cls.from_aur(path, aur_url)
+            return package
+        except InvalidPackageInfo:
+            raise
+        except Exception as e:
+            raise InvalidPackageInfo(str(e))
+
+    def is_outdated(self, remote: Package) -> bool:
+        result = check_output('vercmp', self.version, remote.version, exception=None)
+        return True if int(result) < 0 else False

src/ahriman/models/repository_paths.py

@@ -0,0 +1,35 @@
+import os
+
+from dataclasses import dataclass
+
+
+@dataclass
+class RepositoryPaths:
+    root: str
+
+    @property
+    def chroot(self) -> str:
+        return os.path.join(self.root, 'chroot')
+
+    @property
+    def manual(self) -> str:
+        return os.path.join(self.root, 'manual')
+
+    @property
+    def packages(self) -> str:
+        return os.path.join(self.root, 'packages')
+
+    @property
+    def repository(self) -> str:
+        return os.path.join(self.root, 'repository')
+
+    @property
+    def sources(self) -> str:
+        return os.path.join(self.root, 'sources')
+
+    def create_tree(self) -> None:
+        os.makedirs(self.chroot, mode=0o755, exist_ok=True)
+        os.makedirs(self.manual, mode=0o755, exist_ok=True)
+        os.makedirs(self.packages, mode=0o755, exist_ok=True)
+        os.makedirs(self.repository, mode=0o755, exist_ok=True)
+        os.makedirs(self.sources, mode=0o755, exist_ok=True)

src/ahriman/models/sign_settings.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from enum import Enum, auto
+from typing import Type
+
+from ahriman.core.exceptions import InvalidOptionException
+
+
+class SignSettings(Enum):
+    Disabled = auto()
+    SignPackages = auto()
+    SignRepository = auto()
+
+    @classmethod
+    def from_option(cls: Type[SignSettings], value: str) -> SignSettings:
+        if value.lower() in ('no', 'disabled'):
+            return cls.Disabled
+        elif value.lower() in ('package', 'packages', 'sign-package'):
+            return cls.SignPackages
+        elif value.lower() in ('repository', 'sign-repository'):
+            return cls.SignRepository
+        raise InvalidOptionException(value)

src/ahriman/version.py

@@ -0,0 +1 @@
+__version__ = '0.1.0'