add depdendency manager and switch to pyalpm instead of expac

CONFIGURING.md

@@ -9,11 +9,14 @@ Base configuration settings.
 * `include` - path to directory with configuration files overrides, string, required.
 * `logging` - path to logging configuration, string, required. Check `logging.ini` for reference.
 
-## `aur` group
+## `alpm` group
 
-AUR related configuration.
+libalpm and AUR related configuration.
 
-* `url` - base url for AUR, string, required.
+* `aur_url` - base url for AUR, string, required.
+* `database` - path to pacman local database cache, string, required.
+* `repositories` - list of pacman repositories, space separated list of strings, required.
+* `root` - root for alpm library, string, required.
 
 ## `build_*` groups
 

README.md

@@ -8,7 +8,8 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github
 * Multi-architecture support
 * VCS packages support
 * Sign support with gpg (repository, package, per package settings)
-* Synchronization to remote services (rsync, s3) and report generation (html) 
+* Synchronization to remote services (rsync, s3) and report generation (html)
+* Dependency manager
 * Repository status interface
 
 ## Installation and run
@@ -24,7 +25,3 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github
     * configure `/etc/sudoers.d/ahriman` to allow running command without password.
 * Start and enable `ahriman.timer` via `systemctl`.
 * Add packages by using `ahriman add {package}` command.
-
-## Limitations
-
-* It does not manage dependencies, so you have to add them before main package.

package/archlinux/PKGBUILD

@@ -7,7 +7,7 @@ pkgdesc="ArcHlinux ReposItory MANager"
 arch=('any')
 url="https://github.com/arcan1s/ahriman"
 license=('GPL3')
-depends=('devtools' 'expac' 'git' 'python-aur' 'python-srcinfo')
+depends=('devtools' 'git' 'pyalpm' 'python-aur' 'python-srcinfo')
 makedepends=('python-pip')
 optdepends=('aws-cli: sync to s3'
             'breezy: -bzr packages support'
@@ -23,7 +23,7 @@ optdepends=('aws-cli: sync to s3'
 source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz"
         'ahriman.sysusers'
         'ahriman.tmpfiles')
-sha512sums=('941821639fe4410152a21251d9b0fe5f96ee3a60b88e2067ea4a83ef04b5d1393828152ef4843575449bdef8d44ad6a69f9e41e82516d4d1850bd14f17822785'
+sha512sums=('ddbae1368359c93e4a00a196b06e003895fe5bd36f763f2313c2cdc1cf37db96f258807be087a8aceebb8d608c332cd88695600fd28a728af17187f1568711b4'
             '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
             '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')
 backup=('etc/ahriman.ini'

package/etc/ahriman.ini

@@ -2,8 +2,11 @@
 include = /etc/ahriman.ini.d
 logging = /etc/ahriman.ini.d/logging.ini
 
-[aur]
+[alpm]
-url = https://aur.archlinux.org
+aur_url = https://aur.archlinux.org
+database = /var/lib/pacman
+repositories = core extra community multilib
+root = /
 
 [build]
 archbuild_flags =

setup.py

@@ -28,6 +28,7 @@ setup(
     ],
     install_requires=[
         'aur',
+        'pyalpm',
         'srcinfo',
     ],
     setup_requires=[

src/ahriman/application/ahriman.py

@@ -27,7 +27,7 @@ from ahriman.core.configuration import Configuration
 
 
 def add(args: argparse.Namespace) -> None:
-    Application.from_args(args).add(args.package)
+    Application.from_args(args).add(args.package, args.without_dependencies)
 
 
 def rebuild(args: argparse.Namespace) -> None:
@@ -75,6 +75,7 @@ if __name__ == '__main__':
 
     add_parser = subparsers.add_parser('add', description='add package')
     add_parser.add_argument('package', help='package name or archive path', nargs='+')
+    add_parser.add_argument('--without-dependencies', help='do not add dependencies', action='store_true')
     add_parser.set_defaults(fn=add)
 
     check_parser = subparsers.add_parser('check', description='check for updates')

src/ahriman/application/application.py

@@ -24,11 +24,12 @@ import logging
 import os
 import shutil
 
-from typing import Callable, List, Optional, Type
+from typing import Callable, Iterable, List, Optional, Set, Type
 
 from ahriman.core.build_tools.task import Task
 from ahriman.core.configuration import Configuration
 from ahriman.core.repository import Repository
+from ahriman.core.tree import Tree
 from ahriman.models.package import Package
 
 
@@ -45,6 +46,14 @@ class Application:
         config = Configuration.from_path(args.config)
         return cls(args.architecture, config)
 
+    def _known_packages(self) -> Set[str]:
+        known_packages = set()
+        # local set
+        for package in self.repository.packages():
+            known_packages.update(package.packages)
+        known_packages.update(self.repository.pacman.all_packages())
+        return known_packages
+
     def _finalize(self) -> None:
         self.report()
         self.sync()
@@ -63,35 +72,55 @@ class Application:
 
         return updates
 
-    def add(self, names: List[str]) -> None:
+    def add(self, names: Iterable[str], without_dependencies: bool) -> None:
-        def add_manual(name: str) -> None:
+        known_packages = self._known_packages()
-            package = Package.load(name, self.config.get('aur', 'url'))
+
-            Task.fetch(os.path.join(self.repository.paths.manual, package.base), package.git_url)
+        def add_manual(name: str) -> str:
+            package = Package.load(name, self.repository.pacman, self.config.get('alpm', 'aur_url'))
+            path = os.path.join(self.repository.paths.manual, package.base)
+            Task.fetch(path, package.git_url)
+            return path
 
         def add_archive(src: str) -> None:
             dst = os.path.join(self.repository.paths.packages, os.path.basename(src))
             shutil.move(src, dst)
 
-        for name in names:
+        def process_dependencies(path: str) -> None:
-            if os.path.isfile(name):
+            if without_dependencies:
-                add_archive(name)
+                return
-            else:
+            dependencies = Package.dependencies(path)
-                add_manual(name)
+            self.add(dependencies.difference(known_packages), without_dependencies)
 
-    def remove(self, names: List[str]) -> None:
+        def process_single(name: str) -> None:
+            if not os.path.isfile(name):
+                path = add_manual(name)
+                process_dependencies(path)
+            else:
+                add_archive(name)
+
+        for name in names:
+            process_single(name)
+
+    def remove(self, names: Iterable[str]) -> None:
         self.repository.process_remove(names)
         self._finalize()
 
-    def report(self, target: Optional[List[str]] = None) -> None:
+    def report(self, target: Optional[Iterable[str]] = None) -> None:
         targets = target or None
         self.repository.process_report(targets)
 
-    def sync(self, target: Optional[List[str]] = None) -> None:
+    def sync(self, target: Optional[Iterable[str]] = None) -> None:
         targets = target or None
         self.repository.process_sync(targets)
 
-    def update(self, updates: List[Package]) -> None:
+    def update(self, updates: Iterable[Package]) -> None:
-        packages = self.repository.process_build(updates)
+        def process_single(portion: Iterable[Package]):
-        self.repository.process_update(packages)
+            packages = self.repository.process_build(portion)
-        self._finalize()
+            self.repository.process_update(packages)
+            self._finalize()
 
+        tree = Tree()
+        tree.load(updates)
+        for num, level in enumerate(tree.levels()):
+            self.logger.info(f'processing level #{num} {[package.base for package in level]}')
+            process_single(level)

src/ahriman/application/lock.py

@@ -39,7 +39,6 @@ class Lock:
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.remove()
-        return True
 
     def check(self) -> None:
         if self.path is None:

src/ahriman/core/alpm/pacman.py

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman 
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from pyalpm import Handle
+from typing import List, Set
+
+from ahriman.core.configuration import Configuration
+
+
+class Pacman:
+
+    def __init__(self, config: Configuration) -> None:
+        root = config.get('alpm', 'root')
+        pacman_root = config.get('alpm', 'database')
+        self.handle = Handle(root, pacman_root)
+        for repository in config.getlist('alpm', 'repositories'):
+            self.handle.register_syncdb(repository, 0)  # 0 is pgp_level
+
+    def all_packages(self) -> List[str]:
+        result: Set[str] = set()
+        for database in self.handle.get_syncdbs():
+            result.update({package.name for package in database.pkgcache})
+
+        return list(result)

src/ahriman/core/alpm/repo.py

@@ -27,7 +27,7 @@ from ahriman.core.util import check_output
 from ahriman.models.repository_paths import RepositoryPaths
 
 
-class RepoWrapper:
+class Repo:
 
     def __init__(self, name: str, paths: RepositoryPaths, sign_args: List[str]) -> None:
         self.logger = logging.getLogger('build_details')

src/ahriman/core/exceptions.py

@@ -35,7 +35,7 @@ class InitializeException(Exception):
         Exception.__init__(self, 'Could not load service')
 
 
-class InvalidOptionException(Exception):
+class InvalidOption(Exception):
     def __init__(self, value: Any) -> None:
         Exception.__init__(self, f'Invalid or unknown option value `{value}`')
 

src/ahriman/core/repository.py

@@ -21,13 +21,14 @@ import logging
 import os
 import shutil
 
-from typing import Dict, List, Optional
+from typing import Dict, Iterable, List, Optional
 
+from ahriman.core.alpm.pacman import Pacman
+from ahriman.core.alpm.repo import Repo
 from ahriman.core.build_tools.task import Task
 from ahriman.core.configuration import Configuration
-from ahriman.core.repo.repo_wrapper import RepoWrapper
 from ahriman.core.report.report import Report
-from ahriman.core.sign.gpg_wrapper import GPGWrapper
+from ahriman.core.sign.gpg import GPG
 from ahriman.core.upload.uploader import Uploader
 from ahriman.core.util import package_like
 from ahriman.core.watcher.client import Client
@@ -42,15 +43,16 @@ class Repository:
         self.architecture = architecture
         self.config = config
 
-        self.aur_url = config.get('aur', 'url')
+        self.aur_url = config.get('alpm', 'aur_url')
         self.name = config.get('repository', 'name')
 
         self.paths = RepositoryPaths(config.get('repository', 'root'), architecture)
         self.paths.create_tree()
 
-        self.sign = GPGWrapper(architecture, config)
+        self.pacman = Pacman(config)
-        self.wrapper = RepoWrapper(self.name, self.paths, self.sign.repository_sign_args)
+        self.sign = GPG(architecture, config)
-        self.web_report = Client.load(architecture, config)
+        self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
+        self.web = Client.load(architecture, config)
 
     def _clear_build(self) -> None:
         for package in os.listdir(self.paths.sources):
@@ -71,16 +73,16 @@ class Repository:
                 continue
             full_path = os.path.join(self.paths.repository, fn)
             try:
-                local = Package.load(full_path, self.aur_url)
+                local = Package.load(full_path, self.pacman, self.aur_url)
                 result.setdefault(local.base, local).packages.update(local.packages)
             except Exception:
                 self.logger.exception(f'could not load package from {fn}', exc_info=True)
                 continue
         return list(result.values())
 
-    def process_build(self, updates: List[Package]) -> List[str]:
+    def process_build(self, updates: Iterable[Package]) -> List[str]:
         def build_single(package: Package) -> None:
-            self.web_report.set_building(package.base)
+            self.web.set_building(package.base)
             task = Task(package, self.architecture, self.config, self.paths)
             task.clone()
             built = task.build()
@@ -92,7 +94,7 @@ class Repository:
             try:
                 build_single(package)
             except Exception:
-                self.web_report.set_failed(package.base)
+                self.web.set_failed(package.base)
                 self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True)
                 continue
         self._clear_build()
@@ -102,10 +104,10 @@ class Repository:
             for fn in os.listdir(self.paths.packages)
         ]
 
-    def process_remove(self, packages: List[str]) -> str:
+    def process_remove(self, packages: Iterable[str]) -> str:
         def remove_single(package: str) -> None:
             try:
-                self.wrapper.remove(package, package)
+                self.repo.remove(package, package)
             except Exception:
                 self.logger.exception(f'could not remove {package}', exc_info=True)
 
@@ -116,41 +118,41 @@ class Repository:
                 to_remove = local.packages.intersection(packages)
             else:
                 to_remove = set()
-            self.web_report.remove(local.base, to_remove)
+            self.web.remove(local.base, to_remove)
             for package in to_remove:
                 remove_single(package)
 
-        return self.wrapper.repo_path
+        return self.repo.repo_path
 
-    def process_report(self, targets: Optional[List[str]]) -> None:
+    def process_report(self, targets: Optional[Iterable[str]]) -> None:
         if targets is None:
             targets = self.config.getlist('report', 'target')
         for target in targets:
             Report.run(self.architecture, self.config, target, self.paths.repository)
 
-    def process_sync(self, targets: Optional[List[str]]) -> None:
+    def process_sync(self, targets: Optional[Iterable[str]]) -> None:
         if targets is None:
             targets = self.config.getlist('upload', 'target')
         for target in targets:
             Uploader.run(self.architecture, self.config, target, self.paths.repository)
 
-    def process_update(self, packages: List[str]) -> str:
+    def process_update(self, packages: Iterable[str]) -> str:
         for package in packages:
-            local = Package.load(package, self.aur_url)  # we will use it for status reports
+            local = Package.load(package, self.pacman, self.aur_url)  # we will use it for status reports
             try:
                 files = self.sign.sign_package(package, local.base)
                 for src in files:
                     dst = os.path.join(self.paths.repository, os.path.basename(src))
                     shutil.move(src, dst)
                 package_fn = os.path.join(self.paths.repository, os.path.basename(package))
-                self.wrapper.add(package_fn)
+                self.repo.add(package_fn)
-                self.web_report.set_success(local)
+                self.web.set_success(local)
             except Exception:
                 self.logger.exception(f'could not process {package}', exc_info=True)
-                self.web_report.set_failed(local.base)
+                self.web.set_failed(local.base)
         self._clear_packages()
 
-        return self.wrapper.repo_path
+        return self.repo.repo_path
 
     def updates_aur(self, no_vcs: bool) -> List[Package]:
         result: List[Package] = []
@@ -165,12 +167,12 @@ class Repository:
                 continue
 
             try:
-                remote = Package.load(local.base, self.aur_url)
+                remote = Package.load(local.base, self.pacman, self.aur_url)
                 if local.is_outdated(remote):
                     result.append(remote)
-                    self.web_report.set_pending(local.base)
+                    self.web.set_pending(local.base)
             except Exception:
-                self.web_report.set_failed(local.base)
+                self.web.set_failed(local.base)
                 self.logger.exception(f'could not load remote package {local.base}', exc_info=True)
                 continue
 
@@ -181,9 +183,9 @@ class Repository:
 
         for fn in os.listdir(self.paths.manual):
             try:
-                local = Package.load(os.path.join(self.paths.manual, fn), self.aur_url)
+                local = Package.load(os.path.join(self.paths.manual, fn), self.pacman, self.aur_url)
                 result.append(local)
-                self.web_report.set_unknown(local)
+                self.web.set_unknown(local)
             except Exception:
                 self.logger.exception(f'could not add package from {fn}', exc_info=True)
         self._clear_manual()

src/ahriman/core/sign/gpg.py

@@ -28,7 +28,7 @@ from ahriman.core.util import check_output
 from ahriman.models.sign_settings import SignSettings
 
 
-class GPGWrapper:
+class GPG:
 
     def __init__(self, architecture: str, config: Configuration) -> None:
         self.logger = logging.getLogger('build_details')

src/ahriman/core/tree.py

@@ -0,0 +1,75 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman 
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+import shutil
+import tempfile
+
+from typing import Iterable, List, Set
+
+from ahriman.core.build_tools.task import Task
+from ahriman.models.package import Package
+
+
+class Tree:
+
+    def __init__(self) -> None:
+        self.packages: List[Leaf] = []
+
+    def levels(self) -> List[List[Package]]:
+        result: List[List[Package]] = []
+
+        unprocessed = [leaf for leaf in self.packages]
+        while unprocessed:
+            result.append([leaf.package for leaf in unprocessed if leaf.is_root(unprocessed)])
+            unprocessed = [leaf for leaf in unprocessed if not leaf.is_root(unprocessed)]
+
+        return result
+
+    def load(self, packages: Iterable[Package]) -> None:
+        for package in packages:
+            leaf = Leaf(package)
+            leaf.load_dependencies()
+            self.packages.append(leaf)
+
+
+class Leaf:
+
+    def __init__(self, package: Package) -> None:
+        self.package = package
+        self.dependencies: Set[str] = set()
+
+    def is_root(self, packages: Iterable[Leaf]) -> bool:
+        '''
+        :param packages:
+        :return: true if any of packages is dependency of the leaf, false otherwise
+        '''
+        for package in packages:
+            if package.package.packages.intersection(self.dependencies):
+                return False
+        return True
+
+    def load_dependencies(self) -> None:
+        clone_dir = tempfile.mkdtemp()
+        try:
+            Task.fetch(clone_dir, self.package.git_url)
+            self.dependencies = Package.dependencies(clone_dir)
+        finally:
+            shutil.rmtree(clone_dir, ignore_errors=True)

src/ahriman/models/package.py

@@ -25,9 +25,11 @@ import shutil
 import tempfile
 
 from dataclasses import dataclass, field
+from pyalpm import Handle
 from srcinfo.parse import parse_srcinfo
-from typing import Set, Type
+from typing import List, Set, Type
 
+from ahriman.core.alpm.pacman import Pacman
 from ahriman.core.exceptions import InvalidPackageInfo
 from ahriman.core.util import check_output
 
@@ -79,9 +81,9 @@ class Package:
             shutil.rmtree(clone_dir, ignore_errors=True)
 
     @classmethod
-    def from_archive(cls: Type[Package], path: str, aur_url: str) -> Package:
+    def from_archive(cls: Type[Package], path: str, pacman: Pacman, aur_url: str) -> Package:
-        package, base, version = check_output('expac', '-p', '%n %e %v', path, exception=None).split()
+        package = pacman.handle.load_pkg(path)
-        return cls(base, version, aur_url, {package})
+        return cls(package.base, package.version, aur_url, {package.name})
 
     @classmethod
     def from_aur(cls: Type[Package], name: str, aur_url: str)-> Package:
@@ -99,12 +101,27 @@ class Package:
         return cls(src_info['pkgbase'], f'{src_info["pkgver"]}-{src_info["pkgrel"]}', aur_url, packages)
 
     @staticmethod
-    def load(path: str, aur_url: str) -> Package:
+    def dependencies(path: str) -> Set[str]:
+        with open(os.path.join(path, '.SRCINFO')) as fn:
+            src_info, errors = parse_srcinfo(fn.read())
+        if errors:
+            raise InvalidPackageInfo(errors)
+        makedepends = src_info['makedepends']
+        # sum over each package
+        depends: List[str] = src_info.get('depends', [])
+        for package in src_info['packages'].values():
+            depends.extend(package.get('depends', []))
+        # we are not interested in dependencies inside pkgbase
+        packages = set(src_info['packages'].keys())
+        return set(depends + makedepends) - packages
+
+    @staticmethod
+    def load(path: str, pacman: Pacman, aur_url: str) -> Package:
         try:
             if os.path.isdir(path):
                 package: Package = Package.from_build(path, aur_url)
             elif os.path.exists(path):
-                package = Package.from_archive(path, aur_url)
+                package = Package.from_archive(path, pacman, aur_url)
             else:
                 package = Package.from_aur(path, aur_url)
             return package

src/ahriman/models/report_settings.py

@@ -21,7 +21,7 @@ from __future__ import annotations
 
 from enum import Enum, auto
 
-from ahriman.core.exceptions import InvalidOptionException
+from ahriman.core.exceptions import InvalidOption
 
 
 class ReportSettings(Enum):
@@ -31,4 +31,4 @@ class ReportSettings(Enum):
     def from_option(value: str) -> ReportSettings:
         if value.lower() in ('html',):
             return ReportSettings.HTML
-        raise InvalidOptionException(value)
+        raise InvalidOption(value)

src/ahriman/models/sign_settings.py

@@ -21,7 +21,7 @@ from __future__ import annotations
 
 from enum import Enum, auto
 
-from ahriman.core.exceptions import InvalidOptionException
+from ahriman.core.exceptions import InvalidOption
 
 
 class SignSettings(Enum):
@@ -34,4 +34,4 @@ class SignSettings(Enum):
             return SignSettings.SignPackages
         elif value.lower() in ('repository', 'sign-repository'):
             return SignSettings.SignRepository
-        raise InvalidOptionException(value)
+        raise InvalidOption(value)

src/ahriman/models/upload_settings.py

@@ -21,7 +21,7 @@ from __future__ import annotations
 
 from enum import Enum, auto
 
-from ahriman.core.exceptions import InvalidOptionException
+from ahriman.core.exceptions import InvalidOption
 
 
 class UploadSettings(Enum):
@@ -34,4 +34,4 @@ class UploadSettings(Enum):
             return UploadSettings.Rsync
         elif value.lower() in ('s3',):
             return UploadSettings.S3
-        raise InvalidOptionException(value)
+        raise InvalidOption(value)