From 2d6d42f969790e9d9917c03d0125a69b5e8c2d11 Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Mon, 16 Feb 2026 00:12:51 +0200 Subject: [PATCH] feat: archive package tree implementation (#153) * store built packages in archive tree instead of repository * write tests to support new changes * implement atomic_move method, move files only with lock * use generic packages tree for all repos * lookup through archive packages before build * add archive trigger * add archive trigger * regenerate docs * gpg loader fix * support requires repostory flag * drop excess REQUIRES_REPOSITORY * simplify symlionk creation * remove generators * fix sttyle * add separate function for symlinks creation * fix rebase * add note about slicing * smol refactoring of archive_tree class * remove duplicate code * fix typos * few review fixes * monor fixes and typos * clean empty directories * remove side effect from getter * drop recursive remove * ensure_exists now accepts only argument * add package like guard to symlinks fix * speedup archive_lookup processing by iterrupting cycle * remove custom filelock * fix naming * remove remove flag from repo * review fixes * restore wrapper around filelock * extract repository explorer to separate class * docs update * fix ide findings --- .github/workflows/setup.sh | 2 +- CONTRIBUTING.md | 5 + docker/Dockerfile | 3 +- docs/ahriman.core.archive.rst | 29 ++ docs/ahriman.core.database.migrations.rst | 8 + docs/ahriman.core.housekeeping.rst | 8 + docs/ahriman.core.repository.rst | 8 + docs/ahriman.core.rst | 1 + docs/configuration.rst | 7 + docs/requirements.txt | 2 + package/archlinux/PKGBUILD | 2 +- package/share/ahriman/settings/ahriman.ini | 2 + .../ahriman.ini.d/00-housekeeping.ini | 4 + .../settings/ahriman.ini.d/00-triggers.ini | 1 + pyproject.toml | 1 + src/ahriman/application/handlers/handler.py | 36 +- src/ahriman/application/handlers/status.py | 2 +- .../application/handlers/tree_migrate.py | 18 + src/ahriman/core/alpm/repo.py | 33 +- src/ahriman/core/archive/__init__.py | 20 ++ src/ahriman/core/archive/archive_tree.py | 185 ++++++++++ src/ahriman/core/archive/archive_trigger.py | 70 ++++ .../core/build_tools/package_version.py | 5 +- .../core/database/migrations/m016_archive.py | 81 +++++ src/ahriman/core/database/sqlite.py | 12 +- src/ahriman/core/housekeeping/__init__.py | 1 + .../housekeeping/archive_rotation_trigger.py | 116 +++++++ .../housekeeping/logs_rotation_trigger.py | 1 - src/ahriman/core/report/report_trigger.py | 1 - src/ahriman/core/repository/__init__.py | 1 + src/ahriman/core/repository/executor.py | 206 ++++++++--- src/ahriman/core/repository/explorer.py | 70 ++++ src/ahriman/core/support/keyring_trigger.py | 1 - .../core/support/mirrorlist_trigger.py | 1 - src/ahriman/core/upload/upload_trigger.py | 1 - src/ahriman/core/utils.py | 57 ++- src/ahriman/models/package.py | 15 +- src/ahriman/models/repository_paths.py | 62 +++- .../web/middlewares/exception_handler.py | 14 +- src/ahriman/web/views/v1/packages/package.py | 8 +- src/ahriman/web/views/v1/service/upload.py | 5 +- subpackages.py | 1 + .../application/handlers/test_handler.py | 83 +---- .../handlers/test_handler_tree_migrate.py | 32 ++ .../handlers/test_handler_validate.py | 2 +- tests/ahriman/conftest.py | 17 +- tests/ahriman/core/alpm/test_repo.py | 21 +- tests/ahriman/core/archive/conftest.py | 34 ++ .../ahriman/core/archive/test_archive_tree.py | 176 ++++++++++ .../core/archive/test_archive_trigger.py | 37 ++ .../database/migrations/test_m016_archive.py | 78 +++++ tests/ahriman/core/database/test_sqlite.py | 2 +- tests/ahriman/core/housekeeping/conftest.py | 19 +- .../test_archive_rotation_trigger.py | 83 +++++ .../test_logs_rotation_trigger.py | 9 +- tests/ahriman/core/log/test_lazy_logging.py | 1 - .../core/report/test_report_trigger.py | 7 - .../ahriman/core/repository/test_executor.py | 326 ++++++++++++------ .../ahriman/core/repository/test_explorer.py | 56 +++ .../ahriman/core/status/test_local_client.py | 2 +- .../core/support/test_keyring_trigger.py | 7 - .../core/support/test_mirrorlist_trigger.py | 7 - tests/ahriman/core/test_utils.py | 53 +++ tests/ahriman/core/triggers/test_trigger.py | 2 +- .../core/upload/test_upload_trigger.py | 7 - tests/ahriman/models/conftest.py | 4 - tests/ahriman/models/test_package.py | 9 + tests/ahriman/models/test_repository_paths.py | 44 ++- .../v1/service/test_view_v1_service_logs.py | 2 +- .../v1/service/test_view_v1_service_upload.py | 10 +- tests/testresources/core/ahriman.ini | 3 + 71 files changed, 1876 insertions(+), 363 deletions(-) create mode 100644 docs/ahriman.core.archive.rst create mode 100644 src/ahriman/core/archive/__init__.py create mode 100644 src/ahriman/core/archive/archive_tree.py create mode 100644 src/ahriman/core/archive/archive_trigger.py create mode 100644 src/ahriman/core/database/migrations/m016_archive.py create mode 100644 src/ahriman/core/housekeeping/archive_rotation_trigger.py create mode 100644 src/ahriman/core/repository/explorer.py create mode 100644 tests/ahriman/core/archive/conftest.py create mode 100644 tests/ahriman/core/archive/test_archive_tree.py create mode 100644 tests/ahriman/core/archive/test_archive_trigger.py create mode 100644 tests/ahriman/core/database/migrations/test_m016_archive.py create mode 100644 tests/ahriman/core/housekeeping/test_archive_rotation_trigger.py create mode 100644 tests/ahriman/core/repository/test_explorer.py diff --git a/.github/workflows/setup.sh b/.github/workflows/setup.sh index 2a52bfc2..6731e824 100755 --- a/.github/workflows/setup.sh +++ b/.github/workflows/setup.sh @@ -10,7 +10,7 @@ echo -e '[arcanisrepo]\nServer = https://repo.arcanis.me/$arch\nSigLevel = Never # refresh the image pacman -Syyu --noconfirm # main dependencies -pacman -S --noconfirm devtools git pyalpm python-bcrypt python-inflection python-pyelftools python-requests python-systemd sudo +pacman -S --noconfirm devtools git pyalpm python-bcrypt python-filelock python-inflection python-pyelftools python-requests python-systemd sudo # make dependencies pacman -S --noconfirm --asdeps base-devel python-build python-flit python-installer python-tox python-wheel # optional dependencies diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ecac0386..74981a72 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -165,6 +165,11 @@ Again, the most checks can be performed by `tox` command, though some additional # Blank line again and package imports from ahriman.core.configuration import Configuration + # Multiline import example + from ahriman.core.database.operations import ( + AuthOperations, + BuildOperations, + ) ``` * One file should define only one class, exception is class satellites in case if file length remains less than 400 lines. diff --git a/docker/Dockerfile b/docker/Dockerfile index b7606a6f..fa5eed5c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,7 +24,8 @@ RUN pacman -S --noconfirm --asdeps \ devtools \ git \ pyalpm \ - python-bcrypt \ + python-bcrypt \ + python-filelock \ python-inflection \ python-pyelftools \ python-requests \ diff --git a/docs/ahriman.core.archive.rst b/docs/ahriman.core.archive.rst new file mode 100644 index 00000000..33dbb4d6 --- /dev/null +++ b/docs/ahriman.core.archive.rst @@ -0,0 +1,29 @@ +ahriman.core.archive package +============================ + +Submodules +---------- + +ahriman.core.archive.archive\_tree module +----------------------------------------- + +.. automodule:: ahriman.core.archive.archive_tree + :members: + :no-undoc-members: + :show-inheritance: + +ahriman.core.archive.archive\_trigger module +-------------------------------------------- + +.. automodule:: ahriman.core.archive.archive_trigger + :members: + :no-undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: ahriman.core.archive + :members: + :no-undoc-members: + :show-inheritance: diff --git a/docs/ahriman.core.database.migrations.rst b/docs/ahriman.core.database.migrations.rst index 08054327..4b90cc56 100644 --- a/docs/ahriman.core.database.migrations.rst +++ b/docs/ahriman.core.database.migrations.rst @@ -132,6 +132,14 @@ ahriman.core.database.migrations.m015\_logs\_process\_id module :no-undoc-members: :show-inheritance: +ahriman.core.database.migrations.m016\_archive module +----------------------------------------------------- + +.. automodule:: ahriman.core.database.migrations.m016_archive + :members: + :no-undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/ahriman.core.housekeeping.rst b/docs/ahriman.core.housekeeping.rst index 7a80d7c6..5b69d920 100644 --- a/docs/ahriman.core.housekeeping.rst +++ b/docs/ahriman.core.housekeeping.rst @@ -4,6 +4,14 @@ ahriman.core.housekeeping package Submodules ---------- +ahriman.core.housekeeping.archive\_rotation\_trigger module +----------------------------------------------------------- + +.. automodule:: ahriman.core.housekeeping.archive_rotation_trigger + :members: + :no-undoc-members: + :show-inheritance: + ahriman.core.housekeeping.logs\_rotation\_trigger module -------------------------------------------------------- diff --git a/docs/ahriman.core.repository.rst b/docs/ahriman.core.repository.rst index 5b1b465f..c2d5dcb3 100644 --- a/docs/ahriman.core.repository.rst +++ b/docs/ahriman.core.repository.rst @@ -28,6 +28,14 @@ ahriman.core.repository.executor module :no-undoc-members: :show-inheritance: +ahriman.core.repository.explorer module +--------------------------------------- + +.. automodule:: ahriman.core.repository.explorer + :members: + :no-undoc-members: + :show-inheritance: + ahriman.core.repository.package\_info module -------------------------------------------- diff --git a/docs/ahriman.core.rst b/docs/ahriman.core.rst index e302d8b1..5543482f 100644 --- a/docs/ahriman.core.rst +++ b/docs/ahriman.core.rst @@ -8,6 +8,7 @@ Subpackages :maxdepth: 4 ahriman.core.alpm + ahriman.core.archive ahriman.core.auth ahriman.core.build_tools ahriman.core.configuration diff --git a/docs/configuration.rst b/docs/configuration.rst index cf5600fd..9a6dc20e 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -97,6 +97,13 @@ libalpm and AUR related configuration. Group name can refer to architecture, e.g * ``sync_files_database`` - download files database from mirror, boolean, required. * ``use_ahriman_cache`` - use local pacman package cache instead of system one, boolean, required. With this option enabled you might want to refresh database periodically (available as additional flag for some subcommands). If set to ``no``, databases must be synchronized manually. +``archive`` group +----------------- + +Describes settings for packages archives management extensions. + +* ``keep_built_packages`` - keep this amount of built packages with different versions, integer, required. ``0`` (or negative number) will effectively disable archives removal. + ``auth`` group -------------- diff --git a/docs/requirements.txt b/docs/requirements.txt index e5a585f1..54fe9240 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -40,6 +40,8 @@ docutils==0.21.2 # sphinx # sphinx-argparse # sphinx-rtd-theme +filelock==3.24.0 + # via ahriman (pyproject.toml) frozenlist==1.6.0 # via # aiohttp diff --git a/package/archlinux/PKGBUILD b/package/archlinux/PKGBUILD index 722f5c16..ad44670d 100644 --- a/package/archlinux/PKGBUILD +++ b/package/archlinux/PKGBUILD @@ -8,7 +8,7 @@ pkgdesc="ArcH linux ReposItory MANager" arch=('any') url="https://ahriman.readthedocs.io/" license=('GPL-3.0-or-later') -depends=('devtools>=1:1.0.0' 'git' 'pyalpm' 'python-bcrypt' 'python-inflection' 'python-pyelftools' 'python-requests') +depends=('devtools>=1:1.0.0' 'git' 'pyalpm' 'python-bcrypt' 'python-filelock' 'python-inflection' 'python-pyelftools' 'python-requests') makedepends=('python-build' 'python-flit' 'python-installer' 'python-wheel') source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgbase-$pkgver.tar.gz" "$pkgbase.sysusers" diff --git a/package/share/ahriman/settings/ahriman.ini b/package/share/ahriman/settings/ahriman.ini index 40b8acb0..d983b3dd 100644 --- a/package/share/ahriman/settings/ahriman.ini +++ b/package/share/ahriman/settings/ahriman.ini @@ -44,9 +44,11 @@ triggers[] = ahriman.core.report.ReportTrigger triggers[] = ahriman.core.upload.UploadTrigger triggers[] = ahriman.core.gitremote.RemotePushTrigger triggers[] = ahriman.core.housekeeping.LogsRotationTrigger +triggers[] = ahriman.core.housekeeping.ArchiveRotationTrigger ; List of well-known triggers. Used only for configuration purposes. triggers_known[] = ahriman.core.gitremote.RemotePullTrigger triggers_known[] = ahriman.core.gitremote.RemotePushTrigger +triggers_known[] = ahriman.core.housekeeping.ArchiveRotationTrigger triggers_known[] = ahriman.core.housekeeping.LogsRotationTrigger triggers_known[] = ahriman.core.report.ReportTrigger triggers_known[] = ahriman.core.upload.UploadTrigger diff --git a/package/share/ahriman/settings/ahriman.ini.d/00-housekeeping.ini b/package/share/ahriman/settings/ahriman.ini.d/00-housekeeping.ini index 2fdc2990..a677f3fa 100644 --- a/package/share/ahriman/settings/ahriman.ini.d/00-housekeeping.ini +++ b/package/share/ahriman/settings/ahriman.ini.d/00-housekeeping.ini @@ -1,3 +1,7 @@ +[archive] +; Keep amount of last built packages in archive. 0 means keep all packages +keep_built_packages = 1 + [logs-rotation] ; Keep last build logs for each package keep_last_logs = 5 diff --git a/package/share/ahriman/settings/ahriman.ini.d/00-triggers.ini b/package/share/ahriman/settings/ahriman.ini.d/00-triggers.ini index ae29f15c..b097cc3a 100644 --- a/package/share/ahriman/settings/ahriman.ini.d/00-triggers.ini +++ b/package/share/ahriman/settings/ahriman.ini.d/00-triggers.ini @@ -1,5 +1,6 @@ [build] ; List of well-known triggers. Used only for configuration purposes. +triggers_known[] = ahriman.core.archive.ArchiveTrigger triggers_known[] = ahriman.core.distributed.WorkerLoaderTrigger triggers_known[] = ahriman.core.distributed.WorkerTrigger triggers_known[] = ahriman.core.support.KeyringTrigger diff --git a/pyproject.toml b/pyproject.toml index ae839788..dc82e6ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ authors = [ dependencies = [ "bcrypt", + "filelock", "inflection", "pyelftools", "requests", diff --git a/src/ahriman/application/handlers/handler.py b/src/ahriman/application/handlers/handler.py index e14fab43..75056e52 100644 --- a/src/ahriman/application/handlers/handler.py +++ b/src/ahriman/application/handlers/handler.py @@ -20,7 +20,7 @@ import argparse import logging -from collections.abc import Callable, Iterable +from collections.abc import Callable from multiprocessing import Pool from typing import ClassVar, TypeVar @@ -28,9 +28,9 @@ from ahriman.application.lock import Lock from ahriman.core.configuration import Configuration from ahriman.core.exceptions import ExitCode, MissingArchitectureError, MultipleArchitecturesError from ahriman.core.log.log_loader import LogLoader +from ahriman.core.repository import Explorer from ahriman.core.types import ExplicitBool from ahriman.models.repository_id import RepositoryId -from ahriman.models.repository_paths import RepositoryPaths # this workaround is for several things @@ -169,11 +169,6 @@ class Handler: Raises: MissingArchitectureError: if no architecture set and automatic detection is not allowed or failed """ - configuration = Configuration() - configuration.load(args.configuration) - # pylint, wtf??? - root = configuration.getpath("repository", "root") # pylint: disable=assignment-from-no-return - # preparse systemd repository-id argument # we are using unescaped values, so / is not allowed here, because it is impossible to separate if from dashes if args.repository_id is not None: @@ -184,27 +179,10 @@ class Handler: if repository_parts: args.repository = "-".join(repository_parts) # replace slash with dash - # extract repository names first - if (from_args := args.repository) is not None: - repositories: Iterable[str] = [from_args] - elif from_filesystem := RepositoryPaths.known_repositories(root): - repositories = from_filesystem - else: # try to read configuration now - repositories = [configuration.get("repository", "name")] + configuration = Configuration() + configuration.load(args.configuration) + repositories = Explorer.repositories_extract(configuration, args.repository, args.architecture) - # extract architecture names - if (architecture := args.architecture) is not None: - parsed = set( - RepositoryId(architecture, repository) - for repository in repositories - ) - else: # try to read from file system - parsed = set( - RepositoryId(architecture, repository) - for repository in repositories - for architecture in RepositoryPaths.known_architectures(root, repository) - ) - - if not parsed: + if not repositories: raise MissingArchitectureError(args.command) - return sorted(parsed) + return sorted(repositories) diff --git a/src/ahriman/application/handlers/status.py b/src/ahriman/application/handlers/status.py index b1b92f0f..0f045b0d 100644 --- a/src/ahriman/application/handlers/status.py +++ b/src/ahriman/application/handlers/status.py @@ -66,7 +66,7 @@ class Status(Handler): Status.check_status(args.exit_code, packages) comparator: Callable[[tuple[Package, BuildStatus]], Comparable] = lambda item: item[0].base - filter_fn: Callable[[tuple[Package, BuildStatus]], bool] =\ + filter_fn: Callable[[tuple[Package, BuildStatus]], bool] = \ lambda item: args.status is None or item[1].status == args.status for package, package_status in sorted(filter(filter_fn, packages), key=comparator): PackagePrinter(package, package_status)(verbose=args.info) diff --git a/src/ahriman/application/handlers/tree_migrate.py b/src/ahriman/application/handlers/tree_migrate.py index 2100e164..dbc78498 100644 --- a/src/ahriman/application/handlers/tree_migrate.py +++ b/src/ahriman/application/handlers/tree_migrate.py @@ -21,6 +21,7 @@ import argparse from ahriman.application.handlers.handler import Handler, SubParserAction from ahriman.core.configuration import Configuration +from ahriman.core.utils import symlink_relative, walk from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_paths import RepositoryPaths @@ -49,6 +50,7 @@ class TreeMigrate(Handler): target_tree.tree_create() # perform migration TreeMigrate.tree_move(current_tree, target_tree) + TreeMigrate.symlinks_fix(target_tree) @staticmethod def _set_service_tree_migrate_parser(root: SubParserAction) -> argparse.ArgumentParser: @@ -66,6 +68,22 @@ class TreeMigrate(Handler): parser.set_defaults(lock=None, quiet=True, report=False) return parser + @staticmethod + def symlinks_fix(paths: RepositoryPaths) -> None: + """ + fix package archive symlinks + + Args: + paths(RepositoryPaths): new repository paths + """ + archives = {path.name: path for path in walk(paths.archive)} + for symlink in walk(paths.repository): + if symlink.exists(): # no need to check for symlinks as we have just walked through the tree + continue + if (source_archive := archives.get(symlink.name)) is not None: + symlink.unlink() + symlink_relative(symlink, source_archive) + @staticmethod def tree_move(from_tree: RepositoryPaths, to_tree: RepositoryPaths) -> None: """ diff --git a/src/ahriman/core/alpm/repo.py b/src/ahriman/core/alpm/repo.py index 6021e50c..db8a79e0 100644 --- a/src/ahriman/core/alpm/repo.py +++ b/src/ahriman/core/alpm/repo.py @@ -31,20 +31,21 @@ class Repo(LazyLogging): Attributes: name(str): repository name - paths(RepositoryPaths): repository paths instance + root(Path): repository root sign_args(list[str]): additional args which have to be used to sign repository archive uid(int): uid of the repository owner user """ - def __init__(self, name: str, paths: RepositoryPaths, sign_args: list[str]) -> None: + def __init__(self, name: str, paths: RepositoryPaths, sign_args: list[str], root: Path | None = None) -> None: """ Args: name(str): repository name paths(RepositoryPaths): repository paths instance sign_args(list[str]): additional args which have to be used to sign repository archive + root(Path | None, optional): repository root. If none set, the default will be used (Default value = None) """ self.name = name - self.paths = paths + self.root = root or paths.repository self.uid, _ = paths.root_owner self.sign_args = sign_args @@ -56,7 +57,7 @@ class Repo(LazyLogging): Returns: Path: path to repository database """ - return self.paths.repository / f"{self.name}.db.tar.gz" + return self.root / f"{self.name}.db.tar.gz" def add(self, path: Path) -> None: """ @@ -66,35 +67,37 @@ class Repo(LazyLogging): path(Path): path to archive to add """ check_output( - "repo-add", *self.sign_args, "-R", str(self.repo_path), str(path), + "repo-add", *self.sign_args, "--remove", str(self.repo_path), str(path), exception=BuildError.from_process(path.name), - cwd=self.paths.repository, + cwd=self.root, logger=self.logger, - user=self.uid) + user=self.uid, + ) def init(self) -> None: """ create empty repository database. It just calls add with empty arguments """ check_output("repo-add", *self.sign_args, str(self.repo_path), - cwd=self.paths.repository, logger=self.logger, user=self.uid) + cwd=self.root, logger=self.logger, user=self.uid) - def remove(self, package: str, filename: Path) -> None: + def remove(self, package_name: str, filename: Path) -> None: """ remove package from repository Args: - package(str): package name to remove + package_name(str): package name to remove filename(Path): package filename to remove """ # remove package and signature (if any) from filesystem - for full_path in self.paths.repository.glob(f"{filename}*"): + for full_path in self.root.glob(f"{filename.name}*"): full_path.unlink() # remove package from registry check_output( - "repo-remove", *self.sign_args, str(self.repo_path), package, - exception=BuildError.from_process(package), - cwd=self.paths.repository, + "repo-remove", *self.sign_args, str(self.repo_path), package_name, + exception=BuildError.from_process(package_name), + cwd=self.root, logger=self.logger, - user=self.uid) + user=self.uid, + ) diff --git a/src/ahriman/core/archive/__init__.py b/src/ahriman/core/archive/__init__.py new file mode 100644 index 00000000..3862fe71 --- /dev/null +++ b/src/ahriman/core/archive/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2021-2025 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from ahriman.core.archive.archive_trigger import ArchiveTrigger diff --git a/src/ahriman/core/archive/archive_tree.py b/src/ahriman/core/archive/archive_tree.py new file mode 100644 index 00000000..5d929bd4 --- /dev/null +++ b/src/ahriman/core/archive/archive_tree.py @@ -0,0 +1,185 @@ +# +# Copyright (c) 2021-2025 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import datetime + +from collections.abc import Iterator +from pathlib import Path + +from ahriman.core.alpm.repo import Repo +from ahriman.core.log import LazyLogging +from ahriman.core.utils import package_like, symlink_relative, utcnow, walk +from ahriman.models.package import Package +from ahriman.models.package_description import PackageDescription +from ahriman.models.repository_paths import RepositoryPaths + + +class ArchiveTree(LazyLogging): + """ + wrapper around archive tree + + Attributes: + paths(RepositoryPaths): repository paths instance + repository_id(RepositoryId): repository unique identifier + sign_args(list[str]): additional args which have to be used to sign repository archive + """ + + def __init__(self, repository_path: RepositoryPaths, sign_args: list[str]) -> None: + """ + Args: + repository_path(RepositoryPaths): repository paths instance + sign_args(list[str]): additional args which have to be used to sign repository archive + """ + self.paths = repository_path + self.repository_id = repository_path.repository_id + self.sign_args = sign_args + + @staticmethod + def _package_symlinks_create(package_description: PackageDescription, root: Path, archive: Path) -> bool: + """ + process symlinks creation for single package + + Args: + package_description(PackageDescription): archive descriptor + root(Path): path to the archive repository root + archive(Path): path to directory with archives + + Returns: + bool: ``True`` if symlinks were created and ``False`` otherwise + """ + symlinks_created = False + # here we glob for archive itself and signature if any + for file in archive.glob(f"{package_description.filename}*"): + try: + symlink_relative(root / file.name, file) + symlinks_created = True + except FileExistsError: + continue # symlink is already created, skip processing + + return symlinks_created + + def _repo(self, root: Path) -> Repo: + """ + constructs :class:`ahriman.core.alpm.repo.Repo` object for given path + + Args: + root(Path): root of the repository + + Returns: + Repo: constructed object with correct properties + """ + return Repo(self.repository_id.name, self.paths, self.sign_args, root) + + def directories_fix(self, paths: set[Path]) -> None: + """ + remove empty repository directories recursively + + Args: + paths(set[Path]): repositories to check + """ + root = self.paths.archive / "repos" + for repository in paths: + parents = [repository] + list(repository.parents[:-1]) + for parent in parents: + path = root / parent + if list(path.iterdir()): + continue # directory is not empty + path.rmdir() + + def repository_for(self, date: datetime.date | None = None) -> Path: + """ + get full path to repository at the specified date + + Args: + date(datetime.date | None, optional): date to generate path. If none supplied then today will be used + (Default value = None) + + Returns: + Path: path to the repository root + """ + date = date or utcnow().date() + return ( + self.paths.archive + / "repos" + / date.strftime("%Y") + / date.strftime("%m") + / date.strftime("%d") + / self.repository_id.name + / self.repository_id.architecture + ) + + def symlinks_create(self, packages: list[Package]) -> None: + """ + create symlinks for the specified packages in today's repository + + Args: + packages(list[Package]): list of packages to be updated + """ + root = self.repository_for() + repo = self._repo(root) + + for package in packages: + archive = self.paths.archive_for(package.base) + + for package_name, single in package.packages.items(): + if single.filename is None: + self.logger.warning("received empty package filename for %s", package_name) + continue + + if self._package_symlinks_create(single, root, archive): + repo.add(root / single.filename) + + def symlinks_fix(self) -> Iterator[Path]: + """ + remove broken symlinks across repositories for all dates + + Yields: + Path: path of the sub-repository with removed symlinks + """ + for path in walk(self.paths.archive / "repos"): + root = path.parent + *_, name, architecture = root.parts + if self.repository_id.name != name or self.repository_id.architecture != architecture: + continue # we only process same name repositories + + if not package_like(path): + continue + if not path.is_symlink(): + continue # find symlinks only + if path.exists(): + continue # filter out not broken symlinks + + # here we don't have access to original archive, so we have to guess name based on archive name + # normally it should be fine to do so + package_name = path.name.rsplit("-", maxsplit=3)[0] + self._repo(root).remove(package_name, path) + yield path.parent.relative_to(self.paths.archive / "repos") + + def tree_create(self) -> None: + """ + create repository tree for current repository + """ + root = self.repository_for() + if root.exists(): + return + + with self.paths.preserve_owner(): + root.mkdir(0o755, parents=True) + # init empty repository here + self._repo(root).init() diff --git a/src/ahriman/core/archive/archive_trigger.py b/src/ahriman/core/archive/archive_trigger.py new file mode 100644 index 00000000..0d24b5f4 --- /dev/null +++ b/src/ahriman/core/archive/archive_trigger.py @@ -0,0 +1,70 @@ +# +# Copyright (c) 2021-2025 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from ahriman.core.archive.archive_tree import ArchiveTree +from ahriman.core.configuration import Configuration +from ahriman.core.sign.gpg import GPG +from ahriman.core.triggers import Trigger +from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId +from ahriman.models.result import Result + + +class ArchiveTrigger(Trigger): + """ + archive repository extension + + Attributes: + paths(RepositoryPaths): repository paths instance + tree(ArchiveTree): archive tree wrapper + """ + + def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: + """ + Args: + repository_id(RepositoryId): repository unique identifier + configuration(Configuration): configuration instance + """ + Trigger.__init__(self, repository_id, configuration) + + self.paths = configuration.repository_paths + self.tree = ArchiveTree(self.paths, GPG(configuration).repository_sign_args) + + def on_result(self, result: Result, packages: list[Package]) -> None: + """ + run trigger + + Args: + result(Result): build result + packages(list[Package]): list of all available packages + """ + self.tree.symlinks_create(packages) + + def on_start(self) -> None: + """ + trigger action which will be called at the start of the application + """ + self.tree.tree_create() + + def on_stop(self) -> None: + """ + trigger action which will be called before the stop of the application + """ + repositories = set(self.tree.symlinks_fix()) + self.tree.directories_fix(repositories) diff --git a/src/ahriman/core/build_tools/package_version.py b/src/ahriman/core/build_tools/package_version.py index 4637af7d..23982b19 100644 --- a/src/ahriman/core/build_tools/package_version.py +++ b/src/ahriman/core/build_tools/package_version.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from pyalpm import vercmp # type: ignore[import-not-found] - from ahriman.core.build_tools.task import Task from ahriman.core.configuration import Configuration from ahriman.core.log import LazyLogging @@ -113,5 +111,4 @@ class PackageVersion(LazyLogging): else: remote_version = remote.version - result: int = vercmp(self.package.version, remote_version) - return result < 0 + return self.package.vercmp(remote_version) < 0 diff --git a/src/ahriman/core/database/migrations/m016_archive.py b/src/ahriman/core/database/migrations/m016_archive.py new file mode 100644 index 00000000..65bf6b53 --- /dev/null +++ b/src/ahriman/core/database/migrations/m016_archive.py @@ -0,0 +1,81 @@ +# +# Copyright (c) 2021-2025 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from dataclasses import replace +from sqlite3 import Connection + +from ahriman.core.alpm.pacman import Pacman +from ahriman.core.configuration import Configuration +from ahriman.core.repository import Explorer +from ahriman.core.sign.gpg import GPG +from ahriman.core.utils import atomic_move, package_like, symlink_relative +from ahriman.models.package import Package +from ahriman.models.pacman_synchronization import PacmanSynchronization +from ahriman.models.repository_paths import RepositoryPaths + + +__all__ = ["migrate_data"] + + +def migrate_data(connection: Connection, configuration: Configuration) -> None: + """ + perform data migration + + Args: + connection(Connection): database connection + configuration(Configuration): configuration instance + """ + del connection + + for repository_id in Explorer.repositories_extract(configuration): + paths = replace(configuration.repository_paths, repository_id=repository_id) + pacman = Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled) + + # create archive directory if required + if not paths.archive.is_dir(): + with paths.preserve_owner(): + paths.archive.mkdir(mode=0o755, parents=True) + + move_packages(paths, pacman) + + +def move_packages(repository_paths: RepositoryPaths, pacman: Pacman) -> None: + """ + move packages from repository to archive and create symbolic links + + Args: + repository_paths(RepositoryPaths): repository paths instance + pacman(Pacman): alpm wrapper instance + """ + for archive in filter(package_like, repository_paths.repository.iterdir()): + if not archive.is_file(follow_symlinks=False): + continue # skip symbolic links if any + + package = Package.from_archive(archive, pacman) + artifacts = [archive] + # check if there are signatures for this package and append it here too + if (signature := GPG.signature(archive)).exists(): + artifacts.append(signature) + + for source in artifacts: + target = repository_paths.ensure_exists(repository_paths.archive_for(package.base)) / source.name + # move package to the archive directory + atomic_move(source, target) + # create symlink to the archive + symlink_relative(source, target) diff --git a/src/ahriman/core/database/sqlite.py b/src/ahriman/core/database/sqlite.py index 1ff3eb34..e649dc38 100644 --- a/src/ahriman/core/database/sqlite.py +++ b/src/ahriman/core/database/sqlite.py @@ -25,8 +25,16 @@ from typing import Self from ahriman.core.configuration import Configuration from ahriman.core.database.migrations import Migrations -from ahriman.core.database.operations import AuthOperations, BuildOperations, ChangesOperations, \ - DependenciesOperations, EventOperations, LogsOperations, PackageOperations, PatchOperations +from ahriman.core.database.operations import ( + AuthOperations, + BuildOperations, + ChangesOperations, + DependenciesOperations, + EventOperations, + LogsOperations, + PackageOperations, + PatchOperations, +) from ahriman.models.repository_id import RepositoryId diff --git a/src/ahriman/core/housekeeping/__init__.py b/src/ahriman/core/housekeeping/__init__.py index 7ae1efb8..4bc7500f 100644 --- a/src/ahriman/core/housekeeping/__init__.py +++ b/src/ahriman/core/housekeeping/__init__.py @@ -17,4 +17,5 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +from ahriman.core.housekeeping.archive_rotation_trigger import ArchiveRotationTrigger from ahriman.core.housekeeping.logs_rotation_trigger import LogsRotationTrigger diff --git a/src/ahriman/core/housekeeping/archive_rotation_trigger.py b/src/ahriman/core/housekeeping/archive_rotation_trigger.py new file mode 100644 index 00000000..11b477df --- /dev/null +++ b/src/ahriman/core/housekeeping/archive_rotation_trigger.py @@ -0,0 +1,116 @@ +# +# Copyright (c) 2021-2025 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from collections.abc import Callable +from functools import cmp_to_key + +from ahriman.core import context +from ahriman.core.alpm.pacman import Pacman +from ahriman.core.configuration import Configuration +from ahriman.core.triggers import Trigger +from ahriman.core.utils import package_like +from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId +from ahriman.models.result import Result + + +class ArchiveRotationTrigger(Trigger): + """ + remove packages from archive + + Attributes: + keep_built_packages(int): number of last packages to keep + paths(RepositoryPaths): repository paths instance + """ + + CONFIGURATION_SCHEMA = { + "archive": { + "type": "dict", + "schema": { + "keep_built_packages": { + "type": "integer", + "required": True, + "coerce": "integer", + "min": 0, + }, + }, + }, + } + + def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: + """ + Args: + repository_id(RepositoryId): repository unique identifier + configuration(Configuration): configuration instance + """ + Trigger.__init__(self, repository_id, configuration) + + section = next(iter(self.configuration_sections(configuration))) + self.keep_built_packages = max(configuration.getint(section, "keep_built_packages"), 0) + self.paths = configuration.repository_paths + + @classmethod + def configuration_sections(cls, configuration: Configuration) -> list[str]: + """ + extract configuration sections from configuration + + Args: + configuration(Configuration): configuration instance + + Returns: + list[str]: read configuration sections belong to this trigger + """ + return list(cls.CONFIGURATION_SCHEMA.keys()) + + def archives_remove(self, package: Package, pacman: Pacman) -> None: + """ + remove older versions of the specified package + + Args: + package(Package): package which has been updated to check for older versions + pacman(Pacman): alpm wrapper instance + """ + packages: dict[tuple[str, str], Package] = {} + # we can't use here load_archives, because it ignores versions + for full_path in filter(package_like, self.paths.archive_for(package.base).iterdir()): + local = Package.from_archive(full_path, pacman) + packages.setdefault((local.base, local.version), local).packages.update(local.packages) + + comparator: Callable[[Package, Package], int] = lambda left, right: left.vercmp(right.version) + to_remove = sorted(packages.values(), key=cmp_to_key(comparator)) + # 0 will implicitly be translated into [:0], meaning we keep all packages + for single in to_remove[:-self.keep_built_packages]: + self.logger.info("removing version %s of package %s", single.version, single.base) + for archive in single.packages.values(): + for path in self.paths.archive_for(single.base).glob(f"{archive.filename}*"): + path.unlink() + + def on_result(self, result: Result, packages: list[Package]) -> None: + """ + run trigger + + Args: + result(Result): build result + packages(list[Package]): list of all available packages + """ + ctx = context.get() + pacman = ctx.get(Pacman) + + for package in result.success: + self.archives_remove(package, pacman) diff --git a/src/ahriman/core/housekeeping/logs_rotation_trigger.py b/src/ahriman/core/housekeeping/logs_rotation_trigger.py index d3bed0c7..492dfc13 100644 --- a/src/ahriman/core/housekeeping/logs_rotation_trigger.py +++ b/src/ahriman/core/housekeeping/logs_rotation_trigger.py @@ -47,7 +47,6 @@ class LogsRotationTrigger(Trigger): }, }, } - REQUIRES_REPOSITORY = True def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ diff --git a/src/ahriman/core/report/report_trigger.py b/src/ahriman/core/report/report_trigger.py index 01eab29d..7347e0e9 100644 --- a/src/ahriman/core/report/report_trigger.py +++ b/src/ahriman/core/report/report_trigger.py @@ -336,7 +336,6 @@ class ReportTrigger(Trigger): }, }, } - REQUIRES_REPOSITORY = True def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ diff --git a/src/ahriman/core/repository/__init__.py b/src/ahriman/core/repository/__init__.py index 8b2c328d..d98b27b9 100644 --- a/src/ahriman/core/repository/__init__.py +++ b/src/ahriman/core/repository/__init__.py @@ -17,4 +17,5 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +from ahriman.core.repository.explorer import Explorer from ahriman.core.repository.repository import Repository diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py index 146c1d20..5265b354 100644 --- a/src/ahriman/core/repository/executor.py +++ b/src/ahriman/core/repository/executor.py @@ -27,7 +27,7 @@ from ahriman.core.build_tools.package_archive import PackageArchive from ahriman.core.build_tools.task import Task from ahriman.core.repository.cleaner import Cleaner from ahriman.core.repository.package_info import PackageInfo -from ahriman.core.utils import safe_filename +from ahriman.core.utils import atomic_move, filelock, list_flatmap, package_like, safe_filename, symlink_relative from ahriman.models.changes import Changes from ahriman.models.event import EventType from ahriman.models.package import Package @@ -41,6 +41,140 @@ class Executor(PackageInfo, Cleaner): trait for common repository update processes """ + def _archive_lookup(self, package: Package) -> list[Path]: + """ + check if there is a rebuilt package already + + Args: + package(Package): package to check + + Returns: + list[Path]: list of built packages and signatures if available, empty list otherwise + """ + archive = self.paths.archive_for(package.base) + if not archive.is_dir(): + return [] + + for path in filter(package_like, archive.iterdir()): + # check if package version is the same + built = Package.from_archive(path, self.pacman) + if built.version != package.version: + continue + + packages = built.packages.values() + # all packages must be either any or same architecture + if not all(single.architecture in ("any", self.architecture) for single in packages): + continue + + return list_flatmap(packages, lambda single: archive.glob(f"{single.filename}*")) + + return [] + + def _archive_rename(self, description: PackageDescription, package_base: str) -> None: + """ + rename package archive removing special symbols + + Args: + description(PackageDescription): package description + package_base(str): package base name + """ + if description.filename is None: + self.logger.warning("received empty package filename for base %s", package_base) + return # suppress type checking, it never can be none actually + + if (safe := safe_filename(description.filename)) != description.filename: + atomic_move(self.paths.packages / description.filename, self.paths.packages / safe) + description.filename = safe + + def _package_build(self, package: Package, path: Path, packager: str | None, + local_version: str | None) -> str | None: + """ + build single package + + Args: + package(Package): package to build + path(Path): path to directory with package files + packager(str | None): packager identifier used for this package + local_version(str | None): local version of the package + + Returns: + str | None: current commit sha if available + """ + self.reporter.set_building(package.base) + + task = Task(package, self.configuration, self.architecture, self.paths) + patches = self.reporter.package_patches_get(package.base, None) + commit_sha = task.init(path, patches, local_version) + + loaded_package = Package.from_build(path, self.architecture, None) + if prebuilt := list(self._archive_lookup(loaded_package)): + self.logger.info("using prebuilt packages for %s-%s", loaded_package.base, loaded_package.version) + built = [] + for artifact in prebuilt: + with filelock(artifact): + shutil.copy(artifact, path) + built.append(path / artifact.name) + else: + built = task.build(path, PACKAGER=packager) + + package.with_packages(built, self.pacman) + for src in built: + dst = self.paths.packages / src.name + atomic_move(src, dst) + + return commit_sha + + def _package_remove(self, package_name: str, path: Path) -> None: + """ + remove single package from repository + + Args: + package_name(str): package name + path(Path): path to package archive + """ + try: + self.repo.remove(package_name, path) + except Exception: + self.logger.exception("could not remove %s", package_name) + + def _package_remove_base(self, package_base: str) -> None: + """ + remove package base from repository + + Args: + package_base(str): package base name + """ + try: + with self.in_event(package_base, EventType.PackageRemoved): + self.reporter.package_remove(package_base) + except Exception: + self.logger.exception("could not remove base %s", package_base) + + def _package_update(self, filename: str | None, package_base: str, packager_key: str | None) -> None: + """ + update built package in repository database + + Args: + filename(str | None): archive filename + package_base(str): package base name + packager_key(str | None): packager key identifier + """ + if filename is None: + self.logger.warning("received empty package filename for base %s", package_base) + return # suppress type checking, it never can be none actually + + # in theory, it might be NOT packages directory, but we suppose it is + full_path = self.paths.packages / filename + files = self.sign.process_sign_package(full_path, packager_key) + + for src in files: + dst = self.paths.ensure_exists(self.paths.archive_for(package_base)) / src.name + atomic_move(src, dst) # move package to archive directory + if not (symlink := self.paths.repository / dst.name).exists(): + symlink_relative(symlink, dst) # create link to archive + + self.repo.add(self.paths.repository / filename) + def process_build(self, updates: Iterable[Package], packagers: Packagers | None = None, *, bump_pkgrel: bool = False) -> Result: """ @@ -55,21 +189,6 @@ class Executor(PackageInfo, Cleaner): Returns: Result: build result """ - def build_single(package: Package, local_path: Path, packager_id: str | None) -> str | None: - self.reporter.set_building(package.base) - task = Task(package, self.configuration, self.architecture, self.paths) - local_version = local_versions.get(package.base) if bump_pkgrel else None - patches = self.reporter.package_patches_get(package.base, None) - commit_sha = task.init(local_path, patches, local_version) - built = task.build(local_path, PACKAGER=packager_id) - - package.with_packages(built, self.pacman) - for src in built: - dst = self.paths.packages / src.name - shutil.move(src, dst) - - return commit_sha - packagers = packagers or Packagers() local_versions = {package.base: package.version for package in self.packages()} @@ -80,16 +199,21 @@ class Executor(PackageInfo, Cleaner): try: with self.in_event(single.base, EventType.PackageUpdated, failure=EventType.PackageUpdateFailed): packager = self.packager(packagers, single.base) - last_commit_sha = build_single(single, Path(dir_name), packager.packager_id) + local_version = local_versions.get(single.base) if bump_pkgrel else None + commit_sha = self._package_build(single, Path(dir_name), packager.packager_id, local_version) + # update commit hash for changes keeping current diff if there is any changes = self.reporter.package_changes_get(single.base) - self.reporter.package_changes_update(single.base, Changes(last_commit_sha, changes.changes)) + self.reporter.package_changes_update(single.base, Changes(commit_sha, changes.changes)) + # update dependencies list package_archive = PackageArchive(self.paths.build_root, single, self.pacman, self.scan_paths) dependencies = package_archive.depends_on() self.reporter.package_dependencies_update(single.base, dependencies) + # update result set result.add_updated(single) + except Exception: self.reporter.set_failed(single.base) result.add_failed(single) @@ -107,19 +231,6 @@ class Executor(PackageInfo, Cleaner): Returns: Result: remove result """ - def remove_base(package_base: str) -> None: - try: - with self.in_event(package_base, EventType.PackageRemoved): - self.reporter.package_remove(package_base) - except Exception: - self.logger.exception("could not remove base %s", package_base) - - def remove_package(package: str, archive_path: Path) -> None: - try: - self.repo.remove(package, archive_path) # remove the package itself - except Exception: - self.logger.exception("could not remove %s", package) - packages_to_remove: dict[str, Path] = {} bases_to_remove: list[str] = [] @@ -136,6 +247,7 @@ class Executor(PackageInfo, Cleaner): }) bases_to_remove.append(local.base) result.add_removed(local) + elif requested.intersection(local.packages.keys()): packages_to_remove.update({ package: properties.filepath @@ -152,11 +264,11 @@ class Executor(PackageInfo, Cleaner): # remove packages from repository files for package, filename in packages_to_remove.items(): - remove_package(package, filename) + self._package_remove(package, filename) # remove bases from registered for package in bases_to_remove: - remove_base(package) + self._package_remove_base(package) return result @@ -172,27 +284,6 @@ class Executor(PackageInfo, Cleaner): Returns: Result: path to repository database """ - def rename(archive: PackageDescription, package_base: str) -> None: - if archive.filename is None: - self.logger.warning("received empty package name for base %s", package_base) - return # suppress type checking, it never can be none actually - if (safe := safe_filename(archive.filename)) != archive.filename: - shutil.move(self.paths.packages / archive.filename, self.paths.packages / safe) - archive.filename = safe - - def update_single(name: str | None, package_base: str, packager_key: str | None) -> None: - if name is None: - self.logger.warning("received empty package name for base %s", package_base) - return # suppress type checking, it never can be none actually - # in theory, it might be NOT packages directory, but we suppose it is - full_path = self.paths.packages / name - files = self.sign.process_sign_package(full_path, packager_key) - for src in files: - dst = self.paths.repository / safe_filename(src.name) - shutil.move(src, dst) - package_path = self.paths.repository / safe_filename(name) - self.repo.add(package_path) - current_packages = {package.base: package for package in self.packages()} local_versions = {package_base: package.version for package_base, package in current_packages.items()} @@ -207,8 +298,8 @@ class Executor(PackageInfo, Cleaner): packager = self.packager(packagers, local.base) for description in local.packages.values(): - rename(description, local.base) - update_single(description.filename, local.base, packager.key) + self._archive_rename(description, local.base) + self._package_update(description.filename, local.base, packager.key) self.reporter.set_success(local) result.add_updated(local) @@ -216,12 +307,13 @@ class Executor(PackageInfo, Cleaner): if local.base in current_packages: current_package_archives = set(current_packages[local.base].packages.keys()) removed_packages.extend(current_package_archives.difference(local.packages)) + except Exception: self.reporter.set_failed(local.base) result.add_failed(local) self.logger.exception("could not process %s", local.base) - self.clear_packages() + self.clear_packages() self.process_remove(removed_packages) return result diff --git a/src/ahriman/core/repository/explorer.py b/src/ahriman/core/repository/explorer.py new file mode 100644 index 00000000..72de682f --- /dev/null +++ b/src/ahriman/core/repository/explorer.py @@ -0,0 +1,70 @@ +# +# Copyright (c) 2021-2026 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from collections.abc import Iterable + +from ahriman.core.configuration import Configuration +from ahriman.models.repository_id import RepositoryId +from ahriman.models.repository_paths import RepositoryPaths + + +class Explorer: + """ + helper to read filesystem and find created repositories + """ + + @staticmethod + def repositories_extract(configuration: Configuration, repository: str | None = None, + architecture: str | None = None) -> list[RepositoryId]: + """ + get known architectures + + Args: + configuration(Configuration): configuration instance + repository(str | None, optional): predefined repository name if available (Default value = None) + architecture(str | None, optional): predefined repository architecture if available (Default value = None) + + Returns: + list[RepositoryId]: list of repository names and architectures for which tree is created + """ + # pylint, wtf??? + root = configuration.getpath("repository", "root") # pylint: disable=assignment-from-no-return + + # extract repository names first + if repository is not None: + repositories: Iterable[str] = [repository] + elif from_filesystem := RepositoryPaths.known_repositories(root): + repositories = from_filesystem + else: # try to read configuration now + repositories = [configuration.get("repository", "name")] + + # extract architecture names + if architecture is not None: + parsed = set( + RepositoryId(architecture, repository) + for repository in repositories + ) + else: # try to read from file system + parsed = set( + RepositoryId(architecture, repository) + for repository in repositories + for architecture in RepositoryPaths.known_architectures(root, repository) + ) + + return sorted(parsed) diff --git a/src/ahriman/core/support/keyring_trigger.py b/src/ahriman/core/support/keyring_trigger.py index 13086485..137f4201 100644 --- a/src/ahriman/core/support/keyring_trigger.py +++ b/src/ahriman/core/support/keyring_trigger.py @@ -103,7 +103,6 @@ class KeyringTrigger(Trigger): }, }, } - REQUIRES_REPOSITORY = True def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ diff --git a/src/ahriman/core/support/mirrorlist_trigger.py b/src/ahriman/core/support/mirrorlist_trigger.py index 3f278f37..5269a3ec 100644 --- a/src/ahriman/core/support/mirrorlist_trigger.py +++ b/src/ahriman/core/support/mirrorlist_trigger.py @@ -90,7 +90,6 @@ class MirrorlistTrigger(Trigger): }, }, } - REQUIRES_REPOSITORY = True def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ diff --git a/src/ahriman/core/upload/upload_trigger.py b/src/ahriman/core/upload/upload_trigger.py index af678096..1ff1bc54 100644 --- a/src/ahriman/core/upload/upload_trigger.py +++ b/src/ahriman/core/upload/upload_trigger.py @@ -160,7 +160,6 @@ class UploadTrigger(Trigger): }, }, } - REQUIRES_REPOSITORY = True def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ diff --git a/src/ahriman/core/utils.py b/src/ahriman/core/utils.py index 8be3676d..f10932d6 100644 --- a/src/ahriman/core/utils.py +++ b/src/ahriman/core/utils.py @@ -18,6 +18,7 @@ # along with this program. If not, see . # # pylint: disable=too-many-lines +import contextlib import datetime import io import itertools @@ -25,11 +26,13 @@ import logging import os import re import selectors +import shutil import subprocess from collections.abc import Callable, Iterable, Iterator, Mapping from dataclasses import asdict from enum import Enum +from filelock import FileLock from pathlib import Path from pwd import getpwuid from typing import Any, IO, TypeVar @@ -39,11 +42,13 @@ from ahriman.core.types import Comparable __all__ = [ + "atomic_move", "check_output", "check_user", "dataclass_view", "enum_values", "extract_user", + "filelock", "filter_json", "full_version", "list_flatmap", @@ -58,6 +63,7 @@ __all__ = [ "safe_filename", "srcinfo_property", "srcinfo_property_list", + "symlink_relative", "trim_package", "utcnow", "walk", @@ -68,6 +74,25 @@ R = TypeVar("R", bound=Comparable) T = TypeVar("T") +def atomic_move(src: Path, dst: Path) -> None: + """ + move file from ``source`` location to ``destination``. This method uses lock and :func:`shutil.move` to ensure that + file will be copied (if not rename) atomically. This method blocks execution until lock is available + + Args: + src(Path): path to the source file + dst(Path): path to the destination + + Examples: + This method is a drop-in replacement for :func:`shutil.move` (except it doesn't allow to override copy method) + which first locking destination file. To use it simply call method with arguments:: + + >>> atomic_move(src, dst) + """ + with filelock(dst): + shutil.move(src, dst) + + # pylint: disable=too-many-locals def check_output(*args: str, exception: Exception | Callable[[int, list[str], str, str], Exception] | None = None, cwd: Path | None = None, input_data: str | None = None, @@ -239,6 +264,25 @@ def extract_user() -> str | None: return os.getenv("SUDO_USER") or os.getenv("DOAS_USER") or os.getenv("USER") +@contextlib.contextmanager +def filelock(path: Path) -> Iterator[FileLock]: + """ + wrapper around :class:`filelock.FileLock`, which also removes locks afterward + + Args: + path(Path): path to lock on. The lock file will be created as ``.{path.name}.lock`` + + Yields: + FileLock: acquired file lock instance + """ + lock_path = path.with_name(f".{path.name}.lock") + try: + with FileLock(lock_path) as lock: + yield lock + finally: + lock_path.unlink(missing_ok=True) + + def filter_json(source: dict[str, Any], known_fields: Iterable[str]) -> dict[str, Any]: """ filter json object by fields used for json-to-object conversion @@ -279,7 +323,7 @@ def full_version(epoch: str | int | None, pkgver: str, pkgrel: str) -> str: return f"{prefix}{pkgver}-{pkgrel}" -def list_flatmap(source: Iterable[T], extractor: Callable[[T], list[R]]) -> list[R]: +def list_flatmap(source: Iterable[T], extractor: Callable[[T], Iterable[R]]) -> list[R]: """ extract elements from list of lists, flatten them and apply ``extractor`` @@ -510,6 +554,17 @@ def srcinfo_property_list(key: str, srcinfo: Mapping[str, Any], package_srcinfo: return values +def symlink_relative(symlink: Path, source: Path) -> None: + """ + create symlink with relative path to the target directory + + Args: + symlink(Path): path to symlink to create + source(Path): source file to be symlinked + """ + symlink.symlink_to(source.relative_to(symlink.parent, walk_up=True)) + + def trim_package(package_name: str) -> str: """ remove version bound and description from package name. Pacman allows to specify version bound (=, <=, >= etc.) for diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 145b42f7..a88b955b 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -357,7 +357,7 @@ class Package(LazyLogging): if local_version is None: return None # local version not found, keep upstream pkgrel - if vercmp(self.version, local_version) > 0: + if self.vercmp(local_version) > 0: return None # upstream version is newer than local one, keep upstream pkgrel *_, local_pkgrel = parse_version(local_version) @@ -378,6 +378,19 @@ class Package(LazyLogging): details = "" if self.is_single_package else f""" ({" ".join(sorted(self.packages.keys()))})""" return f"{self.base}{details}" + def vercmp(self, version: str) -> int: + """ + typed wrapper around :func:`pyalpm.vercmp()` + + Args: + version(str): version to compare + + Returns: + int: negative if current version is less than provided, positive if greater than and zero if equals + """ + result: int = vercmp(self.version, version) + return result + def view(self) -> dict[str, Any]: """ generate json package view diff --git a/src/ahriman/models/repository_paths.py b/src/ahriman/models/repository_paths.py index d0e48ce9..f723a58c 100644 --- a/src/ahriman/models/repository_paths.py +++ b/src/ahriman/models/repository_paths.py @@ -85,6 +85,16 @@ class RepositoryPaths(LazyLogging): return Path(self.repository_id.architecture) # legacy tree suffix return Path(self.repository_id.name) / self.repository_id.architecture + @property + def archive(self) -> Path: + """ + archive directory root + + Returns: + Path: archive directory root + """ + return self.root / "archive" + @property def build_root(self) -> Path: """ @@ -208,6 +218,18 @@ class RepositoryPaths(LazyLogging): return set(walk(instance)) + def archive_for(self, package_base: str) -> Path: + """ + get path to archive specified search criteria + + Args: + package_base(str): package base name + + Returns: + Path: path to archive directory for package base + """ + return self.archive / "packages" / package_base[0] / package_base + def cache_for(self, package_base: str) -> Path: """ get path to cached PKGBUILD and package sources for the package base @@ -220,6 +242,27 @@ class RepositoryPaths(LazyLogging): """ return self.cache / package_base + def ensure_exists(self, directory: Path) -> Path: + """ + get path based on ``directory`` callable provided and ensure it exists + + Args: + directory(Path): path to directory to check + + Returns: + Path: original path based on extractor provided. Directory will always exist + + Examples: + This method calls directory accessor and then checks if there is a directory and - otherwise - creates it:: + + >>> paths.ensure_exists(paths.archive_for(package_base)) + """ + if not directory.is_dir(): + with self.preserve_owner(): + directory.mkdir(mode=0o755, parents=True) + + return directory + @contextlib.contextmanager def preserve_owner(self) -> Iterator[None]: """ @@ -265,6 +308,7 @@ class RepositoryPaths(LazyLogging): """ for directory in ( self.cache_for(package_base), + self.archive_for(package_base), ): shutil.rmtree(directory, ignore_errors=True) @@ -275,12 +319,12 @@ class RepositoryPaths(LazyLogging): if self.repository_id.is_empty: return # do not even try to create tree in case if no repository id set - with self.preserve_owner(): - for directory in ( - self.cache, - self.chroot, - self.packages, - self.pacman, - self.repository, - ): - directory.mkdir(mode=0o755, parents=True, exist_ok=True) + for directory in ( + self.archive, + self.cache, + self.chroot, + self.packages, + self.pacman, + self.repository, + ): + self.ensure_exists(directory) diff --git a/src/ahriman/web/middlewares/exception_handler.py b/src/ahriman/web/middlewares/exception_handler.py index 39599685..c95007df 100644 --- a/src/ahriman/web/middlewares/exception_handler.py +++ b/src/ahriman/web/middlewares/exception_handler.py @@ -21,8 +21,18 @@ import aiohttp_jinja2 import logging from aiohttp.typedefs import Middleware -from aiohttp.web import HTTPClientError, HTTPException, HTTPMethodNotAllowed, HTTPNoContent, HTTPServerError, \ - HTTPUnauthorized, Request, StreamResponse, json_response, middleware +from aiohttp.web import ( + HTTPClientError, + HTTPException, + HTTPMethodNotAllowed, + HTTPNoContent, + HTTPServerError, + HTTPUnauthorized, + Request, + StreamResponse, + json_response, + middleware, +) from ahriman.web.middlewares import HandlerType diff --git a/src/ahriman/web/views/v1/packages/package.py b/src/ahriman/web/views/v1/packages/package.py index 957d5fc4..d119c8cd 100644 --- a/src/ahriman/web/views/v1/packages/package.py +++ b/src/ahriman/web/views/v1/packages/package.py @@ -25,8 +25,12 @@ from ahriman.models.build_status import BuildStatusEnum from ahriman.models.package import Package from ahriman.models.user_access import UserAccess from ahriman.web.apispec.decorators import apidocs -from ahriman.web.schemas import PackageNameSchema, PackageStatusSchema, PackageStatusSimplifiedSchema, \ - RepositoryIdSchema +from ahriman.web.schemas import ( + PackageNameSchema, + PackageStatusSchema, + PackageStatusSimplifiedSchema, + RepositoryIdSchema, +) from ahriman.web.views.base import BaseView from ahriman.web.views.status_view_guard import StatusViewGuard diff --git a/src/ahriman/web/views/v1/service/upload.py b/src/ahriman/web/views/v1/service/upload.py index f0dbe224..145df917 100644 --- a/src/ahriman/web/views/v1/service/upload.py +++ b/src/ahriman/web/views/v1/service/upload.py @@ -26,6 +26,7 @@ from tempfile import NamedTemporaryFile from typing import ClassVar from ahriman.core.configuration import Configuration +from ahriman.core.utils import atomic_move from ahriman.models.repository_paths import RepositoryPaths from ahriman.models.user_access import UserAccess from ahriman.web.apispec.decorators import apidocs @@ -152,10 +153,8 @@ class UploadView(BaseView): files.append(await self.save_file(part, target, max_body_size=max_body_size)) - # and now we can rename files, which is relatively fast operation - # it is probably good way to call lock here, however for filename, current_location in files: target_location = current_location.parent / filename - current_location.rename(target_location) + atomic_move(current_location, target_location) raise HTTPCreated diff --git a/subpackages.py b/subpackages.py index 1e7659fb..ec871c39 100644 --- a/subpackages.py +++ b/subpackages.py @@ -37,6 +37,7 @@ SUBPACKAGES = { "ahriman-triggers": [ prefix / "share" / "ahriman" / "settings" / "ahriman.ini.d" / "00-triggers.ini", site_packages / "ahriman" / "application" / "handlers" / "triggers_support.py", + site_packages / "ahriman" / "core" / "archive", site_packages / "ahriman" / "core" / "distributed", site_packages / "ahriman" / "core" / "support", ], diff --git a/tests/ahriman/application/handlers/test_handler.py b/tests/ahriman/application/handlers/test_handler.py index dc8001b2..52e8ff00 100644 --- a/tests/ahriman/application/handlers/test_handler.py +++ b/tests/ahriman/application/handlers/test_handler.py @@ -145,63 +145,11 @@ def test_repositories_extract(args: argparse.Namespace, configuration: Configura args.configuration = configuration.path args.repository = "repo" mocker.patch("ahriman.core.configuration.Configuration.load", new=lambda self, _: self.copy_from(configuration)) - known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures") - known_repositories_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_repositories") + extract_mock = mocker.patch("ahriman.core.repository.Explorer.repositories_extract", + return_value=[RepositoryId("arch", "repo")]) assert Handler.repositories_extract(args) == [RepositoryId("arch", "repo")] - known_architectures_mock.assert_not_called() - known_repositories_mock.assert_not_called() - - -def test_repositories_extract_repository(args: argparse.Namespace, configuration: Configuration, - mocker: MockerFixture) -> None: - """ - must generate list of available repositories based on flags and tree - """ - args.architecture = "arch" - args.configuration = configuration.path - mocker.patch("ahriman.core.configuration.Configuration.load", new=lambda self, _: self.copy_from(configuration)) - known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures") - known_repositories_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_repositories", - return_value={"repo"}) - - assert Handler.repositories_extract(args) == [RepositoryId("arch", "repo")] - known_architectures_mock.assert_not_called() - known_repositories_mock.assert_called_once_with(configuration.repository_paths.root) - - -def test_repositories_extract_repository_legacy(args: argparse.Namespace, configuration: Configuration, - mocker: MockerFixture) -> None: - """ - must generate list of available repositories based on flags and tree (legacy mode) - """ - args.architecture = "arch" - args.configuration = configuration.path - mocker.patch("ahriman.core.configuration.Configuration.load", new=lambda self, _: self.copy_from(configuration)) - known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures") - known_repositories_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_repositories", - return_value=set()) - - assert Handler.repositories_extract(args) == [RepositoryId("arch", "aur")] - known_architectures_mock.assert_not_called() - known_repositories_mock.assert_called_once_with(configuration.repository_paths.root) - - -def test_repositories_extract_architecture(args: argparse.Namespace, configuration: Configuration, - mocker: MockerFixture) -> None: - """ - must read repository name from config - """ - args.configuration = configuration.path - args.repository = "repo" - mocker.patch("ahriman.core.configuration.Configuration.load", new=lambda self, _: self.copy_from(configuration)) - known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures", - return_value={"arch"}) - known_repositories_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_repositories") - - assert Handler.repositories_extract(args) == [RepositoryId("arch", "repo")] - known_architectures_mock.assert_called_once_with(configuration.repository_paths.root, "repo") - known_repositories_mock.assert_not_called() + extract_mock.assert_called_once_with(pytest.helpers.anyvar(Configuration, True), args.repository, args.architecture) def test_repositories_extract_empty(args: argparse.Namespace, configuration: Configuration, @@ -212,8 +160,7 @@ def test_repositories_extract_empty(args: argparse.Namespace, configuration: Con args.command = "config" args.configuration = configuration.path mocker.patch("ahriman.core.configuration.Configuration.load", new=lambda self, _: self.copy_from(configuration)) - mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures", return_value=set()) - mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_repositories", return_value=set()) + mocker.patch("ahriman.core.repository.Explorer.repositories_extract", return_value=[]) with pytest.raises(MissingArchitectureError): Handler.repositories_extract(args) @@ -227,12 +174,11 @@ def test_repositories_extract_systemd(args: argparse.Namespace, configuration: C args.configuration = configuration.path args.repository_id = "i686/some/repo/name" mocker.patch("ahriman.core.configuration.Configuration.load", new=lambda self, _: self.copy_from(configuration)) - known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures") - known_repositories_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_repositories") + extract_mock = mocker.patch("ahriman.core.repository.Explorer.repositories_extract", + return_value=[RepositoryId("i686", "some-repo-name")]) assert Handler.repositories_extract(args) == [RepositoryId("i686", "some-repo-name")] - known_architectures_mock.assert_not_called() - known_repositories_mock.assert_not_called() + extract_mock.assert_called_once_with(pytest.helpers.anyvar(Configuration, True), "some-repo-name", "i686") def test_repositories_extract_systemd_with_dash(args: argparse.Namespace, configuration: Configuration, @@ -243,12 +189,11 @@ def test_repositories_extract_systemd_with_dash(args: argparse.Namespace, config args.configuration = configuration.path args.repository_id = "i686-some-repo-name" mocker.patch("ahriman.core.configuration.Configuration.load", new=lambda self, _: self.copy_from(configuration)) - known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures") - known_repositories_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_repositories") + extract_mock = mocker.patch("ahriman.core.repository.Explorer.repositories_extract", + return_value=[RepositoryId("i686", "some-repo-name")]) assert Handler.repositories_extract(args) == [RepositoryId("i686", "some-repo-name")] - known_architectures_mock.assert_not_called() - known_repositories_mock.assert_not_called() + extract_mock.assert_called_once_with(pytest.helpers.anyvar(Configuration, True), "some-repo-name", "i686") def test_repositories_extract_systemd_legacy(args: argparse.Namespace, configuration: Configuration, @@ -259,10 +204,8 @@ def test_repositories_extract_systemd_legacy(args: argparse.Namespace, configura args.configuration = configuration.path args.repository_id = "i686" mocker.patch("ahriman.core.configuration.Configuration.load", new=lambda self, _: self.copy_from(configuration)) - known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures") - known_repositories_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_repositories", - return_value=set()) + extract_mock = mocker.patch("ahriman.core.repository.Explorer.repositories_extract", + return_value=[RepositoryId("i686", "aur")]) assert Handler.repositories_extract(args) == [RepositoryId("i686", "aur")] - known_architectures_mock.assert_not_called() - known_repositories_mock.assert_called_once_with(configuration.repository_paths.root) + extract_mock.assert_called_once_with(pytest.helpers.anyvar(Configuration, True), None, "i686") diff --git a/tests/ahriman/application/handlers/test_handler_tree_migrate.py b/tests/ahriman/application/handlers/test_handler_tree_migrate.py index af45aaba..3a94b7c3 100644 --- a/tests/ahriman/application/handlers/test_handler_tree_migrate.py +++ b/tests/ahriman/application/handlers/test_handler_tree_migrate.py @@ -6,6 +6,7 @@ from unittest.mock import call as MockCall from ahriman.application.handlers.tree_migrate import TreeMigrate from ahriman.core.configuration import Configuration +from ahriman.models.package import Package from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_paths import RepositoryPaths @@ -16,6 +17,7 @@ def test_run(args: argparse.Namespace, configuration: Configuration, mocker: Moc """ tree_create_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create") application_mock = mocker.patch("ahriman.application.handlers.tree_migrate.TreeMigrate.tree_move") + symlinks_mock = mocker.patch("ahriman.application.handlers.tree_migrate.TreeMigrate.symlinks_fix") _, repository_id = configuration.check_loaded() old_paths = configuration.repository_paths new_paths = RepositoryPaths(old_paths.root, old_paths.repository_id, _force_current_tree=True) @@ -23,6 +25,36 @@ def test_run(args: argparse.Namespace, configuration: Configuration, mocker: Moc TreeMigrate.run(args, repository_id, configuration, report=False) tree_create_mock.assert_called_once_with() application_mock.assert_called_once_with(old_paths, new_paths) + symlinks_mock.assert_called_once_with(new_paths) + + +def test_symlinks_fix(repository_paths: RepositoryPaths, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must replace symlinks during migration + """ + mocker.patch("ahriman.application.handlers.tree_migrate.walk", side_effect=[ + [ + repository_paths.archive_for(package_ahriman.base) / "file", + repository_paths.archive_for(package_ahriman.base) / "symlink", + ], + [ + repository_paths.repository / "file", + repository_paths.repository / "symlink", + ], + ]) + mocker.patch("pathlib.Path.exists", autospec=True, side_effect=lambda p: p.name == "file") + unlink_mock = mocker.patch("pathlib.Path.unlink") + symlink_mock = mocker.patch("pathlib.Path.symlink_to") + + TreeMigrate.symlinks_fix(repository_paths) + unlink_mock.assert_called_once_with() + symlink_mock.assert_called_once_with( + Path("..") / + ".." / + ".." / + repository_paths.archive_for(package_ahriman.base).relative_to(repository_paths.root) / + "symlink" + ) def test_move_tree(mocker: MockerFixture) -> None: diff --git a/tests/ahriman/application/handlers/test_handler_validate.py b/tests/ahriman/application/handlers/test_handler_validate.py index 16d5a7f4..5d4281e2 100644 --- a/tests/ahriman/application/handlers/test_handler_validate.py +++ b/tests/ahriman/application/handlers/test_handler_validate.py @@ -79,7 +79,7 @@ def test_run_repo_specific_triggers(args: argparse.Namespace, configuration: Con _, repository_id = configuration.check_loaded() # remove unused sections - for section in ("customs3", "github:x86_64", "logs-rotation", "mirrorlist"): + for section in ("archive", "customs3", "github:x86_64", "logs-rotation", "mirrorlist"): configuration.remove_section(section) configuration.set_option("report", "target", "test") diff --git a/tests/ahriman/conftest.py b/tests/ahriman/conftest.py index 7f780f3b..6f5b427a 100644 --- a/tests/ahriman/conftest.py +++ b/tests/ahriman/conftest.py @@ -1,8 +1,10 @@ import datetime import pytest +from dataclasses import replace from pathlib import Path from pytest_mock import MockerFixture +from sqlite3 import Cursor from typing import Any, TypeVar from unittest.mock import MagicMock, PropertyMock @@ -11,12 +13,14 @@ from ahriman.core.alpm.remote import AUR from ahriman.core.auth import Auth from ahriman.core.configuration import Configuration from ahriman.core.database import SQLite +from ahriman.core.database.migrations import Migrations from ahriman.core.repository import Repository from ahriman.core.spawn import Spawn from ahriman.core.status import Client from ahriman.core.status.watcher import Watcher from ahriman.models.aur_package import AURPackage from ahriman.models.build_status import BuildStatus, BuildStatusEnum +from ahriman.models.migration import Migration from ahriman.models.package import Package from ahriman.models.package_description import PackageDescription from ahriman.models.package_source import PackageSource @@ -48,7 +52,9 @@ def anyvar(cls: type[T], strict: bool = False) -> T: T: any wrapper """ class AnyVar(cls): - """any value wrapper""" + """ + any value wrapper + """ def __eq__(self, other: Any) -> bool: """ @@ -271,16 +277,23 @@ def configuration(repository_id: RepositoryId, tmp_path: Path, resource_path_roo @pytest.fixture -def database(configuration: Configuration) -> SQLite: +def database(configuration: Configuration, mocker: MockerFixture) -> SQLite: """ database fixture Args: configuration(Configuration): configuration fixture + mocker(MockerFixture): mocker object Returns: SQLite: database test instance """ + original_method = Migrations.perform_migration + + def perform_migration(self: Migrations, cursor: Cursor, migration: Migration) -> None: + original_method(self, cursor, replace(migration, migrate_data=lambda *args: None)) + + mocker.patch.object(Migrations, "perform_migration", autospec=True, side_effect=perform_migration) return SQLite.load(configuration) diff --git a/tests/ahriman/core/alpm/test_repo.py b/tests/ahriman/core/alpm/test_repo.py index 8cc87036..3543509d 100644 --- a/tests/ahriman/core/alpm/test_repo.py +++ b/tests/ahriman/core/alpm/test_repo.py @@ -4,6 +4,16 @@ from pathlib import Path from pytest_mock import MockerFixture from ahriman.core.alpm.repo import Repo +from ahriman.models.package import Package +from ahriman.models.repository_paths import RepositoryPaths + + +def test_root(repository_paths: RepositoryPaths) -> None: + """ + must correctly define repository root + """ + assert Repo(repository_paths.repository_id.name, repository_paths, []).root == repository_paths.repository + assert Repo(repository_paths.repository_id.name, repository_paths, [], Path("path")).root == Path("path") def test_repo_path(repo: Repo) -> None: @@ -22,6 +32,7 @@ def test_repo_add(repo: Repo, mocker: MockerFixture) -> None: repo.add(Path("path")) check_output_mock.assert_called_once() # it will be checked later assert check_output_mock.call_args[0][0] == "repo-add" + assert "--remove" in check_output_mock.call_args[0] def test_repo_init(repo: Repo, mocker: MockerFixture) -> None: @@ -35,21 +46,23 @@ def test_repo_init(repo: Repo, mocker: MockerFixture) -> None: assert check_output_mock.call_args[0][0] == "repo-add" -def test_repo_remove(repo: Repo, mocker: MockerFixture) -> None: +def test_repo_remove(repo: Repo, package_ahriman: Package, mocker: MockerFixture) -> None: """ - must call repo-remove on package addition + must call repo-remove on package removal """ + filepath = package_ahriman.packages[package_ahriman.base].filepath mocker.patch("pathlib.Path.glob", return_value=[]) check_output_mock = mocker.patch("ahriman.core.alpm.repo.check_output") - repo.remove("package", Path("package.pkg.tar.xz")) + repo.remove(package_ahriman.base, filepath) check_output_mock.assert_called_once() # it will be checked later assert check_output_mock.call_args[0][0] == "repo-remove" + assert package_ahriman.base in check_output_mock.call_args[0] def test_repo_remove_fail_no_file(repo: Repo, mocker: MockerFixture) -> None: """ - must fail on missing file + must fail removal on missing file """ mocker.patch("pathlib.Path.glob", return_value=[Path("package.pkg.tar.xz")]) mocker.patch("pathlib.Path.unlink", side_effect=FileNotFoundError) diff --git a/tests/ahriman/core/archive/conftest.py b/tests/ahriman/core/archive/conftest.py new file mode 100644 index 00000000..8eb910a5 --- /dev/null +++ b/tests/ahriman/core/archive/conftest.py @@ -0,0 +1,34 @@ +import pytest + +from ahriman.core.archive import ArchiveTrigger +from ahriman.core.archive.archive_tree import ArchiveTree +from ahriman.core.configuration import Configuration + + +@pytest.fixture +def archive_tree(configuration: Configuration) -> ArchiveTree: + """ + archive tree fixture + + Args: + configuration(Configuration): configuration fixture + + Returns: + ArchiveTree: archive tree test instance + """ + return ArchiveTree(configuration.repository_paths, []) + + +@pytest.fixture +def archive_trigger(configuration: Configuration) -> ArchiveTrigger: + """ + archive trigger fixture + + Args: + configuration(Configuration): configuration fixture + + Returns: + ArchiveTrigger: archive trigger test instance + """ + _, repository_id = configuration.check_loaded() + return ArchiveTrigger(repository_id, configuration) diff --git a/tests/ahriman/core/archive/test_archive_tree.py b/tests/ahriman/core/archive/test_archive_tree.py new file mode 100644 index 00000000..c6dc2381 --- /dev/null +++ b/tests/ahriman/core/archive/test_archive_tree.py @@ -0,0 +1,176 @@ +from pathlib import Path +from pytest_mock import MockerFixture +from unittest.mock import call as MockCall + +from ahriman.core.archive.archive_tree import ArchiveTree +from ahriman.core.utils import utcnow +from ahriman.models.package import Package + + +def test_repo(archive_tree: ArchiveTree) -> None: + """ + must return correct repository object + """ + local = Path("local") + repo = archive_tree._repo(local) + + assert repo.sign_args == archive_tree.sign_args + assert repo.name == archive_tree.repository_id.name + assert repo.root == local + + +def test_repository_for(archive_tree: ArchiveTree) -> None: + """ + must correctly generate path to repository + """ + path = archive_tree.repository_for() + assert path.is_relative_to(archive_tree.paths.archive / "repos") + assert (archive_tree.repository_id.name, archive_tree.repository_id.architecture) == path.parts[-2:] + assert set(map("{:02d}".format, utcnow().timetuple()[:3])).issubset(path.parts) + + +def test_directories_fix(archive_tree: ArchiveTree, mocker: MockerFixture) -> None: + """ + must remove empty directories recursively + """ + root = archive_tree.paths.archive / "repos" + (root / "a" / "b").mkdir(parents=True, exist_ok=True) + (root / "a" / "b" / "file").touch() + (root / "a" / "b" / "c" / "d").mkdir(parents=True, exist_ok=True) + + _original_rmdir = Path.rmdir + rmdir_mock = mocker.patch("pathlib.Path.rmdir", autospec=True, side_effect=_original_rmdir) + + archive_tree.directories_fix({Path("a") / "b" / "c" / "d"}) + rmdir_mock.assert_has_calls([ + MockCall(root / "a" / "b" / "c" / "d"), + MockCall(root / "a" / "b" / "c"), + ]) + + +def test_symlinks_create(archive_tree: ArchiveTree, package_ahriman: Package, package_python_schedule: Package, + mocker: MockerFixture) -> None: + """ + must create symlinks + """ + _original_exists = Path.exists + + symlinks_mock = mocker.patch("pathlib.Path.symlink_to", side_effect=(None, FileExistsError, FileExistsError)) + add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add") + mocker.patch("pathlib.Path.glob", autospec=True, side_effect=lambda path, name: [path / name[:-1]]) + + archive_tree.symlinks_create([package_ahriman, package_python_schedule]) + symlinks_mock.assert_has_calls([ + MockCall(Path("..") / + ".." / + ".." / + ".." / + ".." / + ".." / + archive_tree.paths.archive_for(package.base) + .relative_to(archive_tree.paths.root) + .relative_to("archive") / + single.filename + ) + for package in (package_ahriman, package_python_schedule) + for single in package.packages.values() + ]) + add_mock.assert_called_once_with( + archive_tree.repository_for() / package_ahriman.packages[package_ahriman.base].filename + ) + + +def test_symlinks_create_empty_filename(archive_tree: ArchiveTree, package_ahriman: Package, + mocker: MockerFixture) -> None: + """ + must skip symlinks creation if filename is not set + """ + package_ahriman.packages[package_ahriman.base].filename = None + symlinks_mock = mocker.patch("pathlib.Path.symlink_to") + + archive_tree.symlinks_create([package_ahriman]) + symlinks_mock.assert_not_called() + + +def test_symlinks_fix(archive_tree: ArchiveTree, mocker: MockerFixture) -> None: + """ + must fix broken symlinks + """ + _original_exists = Path.exists + + def exists_mock(path: Path) -> bool: + if path.name.startswith("symlink"): + return True + return _original_exists(path) + + mocker.patch("pathlib.Path.is_symlink", side_effect=[True, True, False]) + mocker.patch("pathlib.Path.exists", autospec=True, side_effect=exists_mock) + walk_mock = mocker.patch("ahriman.core.archive.archive_tree.walk", return_value=[ + archive_tree.repository_for() / filename + for filename in ( + "symlink-1.0.0-1-x86_64.pkg.tar.zst", + "symlink-1.0.0-1-x86_64.pkg.tar.zst.sig", + "broken_symlink-1.0.0-1-x86_64.pkg.tar.zst", + "file-1.0.0-1-x86_64.pkg.tar.zst", + ) + ]) + remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove") + + assert list(archive_tree.symlinks_fix()) == [ + archive_tree.repository_for().relative_to(archive_tree.paths.archive / "repos"), + ] + walk_mock.assert_called_once_with(archive_tree.paths.archive / "repos") + remove_mock.assert_called_once_with( + "broken_symlink", archive_tree.repository_for() / "broken_symlink-1.0.0-1-x86_64.pkg.tar.zst") + + +def test_symlinks_fix_foreign_repository(archive_tree: ArchiveTree, mocker: MockerFixture) -> None: + """ + must skip symlinks check if repository name or architecture doesn't match + """ + _original_exists = Path.exists + + def exists_mock(path: Path) -> bool: + if path.name.startswith("symlink"): + return True + return _original_exists(path) + + mocker.patch("pathlib.Path.is_symlink", side_effect=[True, True, False]) + mocker.patch("pathlib.Path.exists", autospec=True, side_effect=exists_mock) + mocker.patch("ahriman.core.archive.archive_tree.walk", return_value=[ + archive_tree.repository_for().with_name("i686") / filename + for filename in ( + "symlink-1.0.0-1-x86_64.pkg.tar.zst", + "broken_symlink-1.0.0-1-x86_64.pkg.tar.zst", + "file-1.0.0-1-x86_64.pkg.tar.zst", + ) + ]) + remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove") + + assert list(archive_tree.symlinks_fix()) == [] + remove_mock.assert_not_called() + + +def test_tree_create(archive_tree: ArchiveTree, mocker: MockerFixture) -> None: + """ + must create repository root if not exists + """ + owner_guard_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.preserve_owner") + mkdir_mock = mocker.patch("pathlib.Path.mkdir") + init_mock = mocker.patch("ahriman.core.alpm.repo.Repo.init") + + archive_tree.tree_create() + owner_guard_mock.assert_called_once_with() + mkdir_mock.assert_called_once_with(0o755, parents=True) + init_mock.assert_called_once_with() + + +def test_tree_create_exists(archive_tree: ArchiveTree, mocker: MockerFixture) -> None: + """ + must skip directory creation if already exists + """ + mocker.patch("pathlib.Path.exists", return_value=True) + mkdir_mock = mocker.patch("pathlib.Path.mkdir") + + archive_tree.tree_create() + mkdir_mock.assert_not_called() diff --git a/tests/ahriman/core/archive/test_archive_trigger.py b/tests/ahriman/core/archive/test_archive_trigger.py new file mode 100644 index 00000000..6c92912c --- /dev/null +++ b/tests/ahriman/core/archive/test_archive_trigger.py @@ -0,0 +1,37 @@ +from pathlib import Path +from pytest_mock import MockerFixture + +from ahriman.core.archive import ArchiveTrigger +from ahriman.models.package import Package +from ahriman.models.result import Result + + +def test_on_result(archive_trigger: ArchiveTrigger, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must create symlinks for actual repository + """ + symlinks_mock = mocker.patch("ahriman.core.archive.archive_tree.ArchiveTree.symlinks_create") + archive_trigger.on_result(Result(), [package_ahriman]) + symlinks_mock.assert_called_once_with([package_ahriman]) + + +def test_on_start(archive_trigger: ArchiveTrigger, mocker: MockerFixture) -> None: + """ + must create repository tree on load + """ + tree_mock = mocker.patch("ahriman.core.archive.archive_tree.ArchiveTree.tree_create") + archive_trigger.on_start() + tree_mock.assert_called_once_with() + + +def test_on_stop(archive_trigger: ArchiveTrigger, mocker: MockerFixture) -> None: + """ + must fix broken symlinks on stop + """ + local = Path("local") + symlinks_mock = mocker.patch("ahriman.core.archive.archive_tree.ArchiveTree.symlinks_fix", return_value=[local]) + directories_mock = mocker.patch("ahriman.core.archive.archive_tree.ArchiveTree.directories_fix") + + archive_trigger.on_stop() + symlinks_mock.assert_called_once_with() + directories_mock.assert_called_once_with({local}) diff --git a/tests/ahriman/core/database/migrations/test_m016_archive.py b/tests/ahriman/core/database/migrations/test_m016_archive.py new file mode 100644 index 00000000..962e4c7a --- /dev/null +++ b/tests/ahriman/core/database/migrations/test_m016_archive.py @@ -0,0 +1,78 @@ +import pytest + +from dataclasses import replace +from pathlib import Path +from pytest_mock import MockerFixture +from sqlite3 import Connection +from typing import Any +from unittest.mock import call as MockCall + +from ahriman.core.alpm.pacman import Pacman +from ahriman.core.configuration import Configuration +from ahriman.core.database.migrations.m016_archive import migrate_data, move_packages +from ahriman.models.package import Package +from ahriman.models.repository_paths import RepositoryPaths + + +def test_migrate_data(connection: Connection, configuration: Configuration, mocker: MockerFixture) -> None: + """ + must perform data migration + """ + _, repository_id = configuration.check_loaded() + repositories = [ + repository_id, + replace(repository_id, architecture="i686"), + ] + mocker.patch("ahriman.core.repository.Explorer.repositories_extract", return_value=repositories) + migration_mock = mocker.patch("ahriman.core.database.migrations.m016_archive.move_packages") + + migrate_data(connection, configuration) + migration_mock.assert_has_calls([ + MockCall(replace(configuration.repository_paths, repository_id=repository), pytest.helpers.anyvar(int)) + for repository in repositories + ]) + + +def test_move_packages(repository_paths: RepositoryPaths, pacman: Pacman, package_ahriman: Package, + mocker: MockerFixture) -> None: + """ + must move packages to the archive directory + """ + + def is_file(self: Path, *args: Any, **kwargs: Any) -> bool: + return "file" in self.name + + mocker.patch("pathlib.Path.iterdir", return_value=[ + repository_paths.repository / ".hidden-file.pkg.tar.xz", + repository_paths.repository / "directory", + repository_paths.repository / "file.pkg.tar.xz", + repository_paths.repository / "file.pkg.tar.xz.sig", + repository_paths.repository / "file2.pkg.tar.xz", + repository_paths.repository / "symlink.pkg.tar.xz", + ]) + mocker.patch("pathlib.Path.is_dir", return_value=True) + mocker.patch("pathlib.Path.is_file", autospec=True, side_effect=is_file) + mocker.patch("pathlib.Path.exists", return_value=True) + archive_mock = mocker.patch("ahriman.models.package.Package.from_archive", return_value=package_ahriman) + move_mock = mocker.patch("ahriman.core.database.migrations.m016_archive.atomic_move") + symlink_mock = mocker.patch("pathlib.Path.symlink_to") + + move_packages(repository_paths, pacman) + archive_mock.assert_has_calls([ + MockCall(repository_paths.repository / filename, pacman) + for filename in ("file.pkg.tar.xz", "file2.pkg.tar.xz") + ]) + move_mock.assert_has_calls([ + MockCall(repository_paths.repository / filename, repository_paths.archive_for(package_ahriman.base) / filename) + for filename in ("file.pkg.tar.xz", "file.pkg.tar.xz.sig", "file2.pkg.tar.xz") + ]) + symlink_mock.assert_has_calls([ + MockCall( + Path("..") / + ".." / + ".." / + repository_paths.archive_for(package_ahriman.base).relative_to(repository_paths.root) / + filename + ) + for filename in ("file.pkg.tar.xz", "file.pkg.tar.xz.sig", "file2.pkg.tar.xz") + ]) diff --git a/tests/ahriman/core/database/test_sqlite.py b/tests/ahriman/core/database/test_sqlite.py index 91aa1bc8..dd795d4a 100644 --- a/tests/ahriman/core/database/test_sqlite.py +++ b/tests/ahriman/core/database/test_sqlite.py @@ -16,7 +16,7 @@ def test_load(configuration: Configuration, mocker: MockerFixture) -> None: init_mock.assert_called_once_with() -def test_init(database: SQLite, configuration: Configuration, mocker: MockerFixture) -> None: +def test_init(database: SQLite, mocker: MockerFixture) -> None: """ must run migrations on init """ diff --git a/tests/ahriman/core/housekeeping/conftest.py b/tests/ahriman/core/housekeeping/conftest.py index f1f3a0cb..c809f3d4 100644 --- a/tests/ahriman/core/housekeeping/conftest.py +++ b/tests/ahriman/core/housekeeping/conftest.py @@ -1,13 +1,28 @@ import pytest from ahriman.core.configuration import Configuration -from ahriman.core.housekeeping import LogsRotationTrigger +from ahriman.core.housekeeping import ArchiveRotationTrigger, LogsRotationTrigger + + +@pytest.fixture +def archive_rotation_trigger(configuration: Configuration) -> ArchiveRotationTrigger: + """ + archive rotation trigger fixture + + Args: + configuration(Configuration): configuration fixture + + Returns: + ArchiveRotationTrigger: archive rotation trigger test instance + """ + _, repository_id = configuration.check_loaded() + return ArchiveRotationTrigger(repository_id, configuration) @pytest.fixture def logs_rotation_trigger(configuration: Configuration) -> LogsRotationTrigger: """ - logs roration trigger fixture + logs rotation trigger fixture Args: configuration(Configuration): configuration fixture diff --git a/tests/ahriman/core/housekeeping/test_archive_rotation_trigger.py b/tests/ahriman/core/housekeeping/test_archive_rotation_trigger.py new file mode 100644 index 00000000..ed2977c4 --- /dev/null +++ b/tests/ahriman/core/housekeeping/test_archive_rotation_trigger.py @@ -0,0 +1,83 @@ +import pytest + +from dataclasses import replace +from pathlib import Path +from pytest_mock import MockerFixture +from typing import Any +from unittest.mock import call as MockCall + +from ahriman.core.alpm.pacman import Pacman +from ahriman.core.configuration import Configuration +from ahriman.core.housekeeping import ArchiveRotationTrigger +from ahriman.models.package import Package +from ahriman.models.result import Result + + +def test_configuration_sections(configuration: Configuration) -> None: + """ + must correctly parse target list + """ + assert ArchiveRotationTrigger.configuration_sections(configuration) == ["archive"] + + +def test_archives_remove(archive_rotation_trigger: ArchiveRotationTrigger, package_ahriman: Package, + pacman: Pacman, mocker: MockerFixture) -> None: + """ + must remove older packages + """ + def package(version: Any, *args: Any, **kwargs: Any) -> Package: + generated = replace(package_ahriman, version=str(version)) + generated.packages = { + key: replace(value, filename=str(version)) + for key, value in generated.packages.items() + } + return generated + + mocker.patch("pathlib.Path.is_dir", return_value=True) + mocker.patch("ahriman.core.housekeeping.archive_rotation_trigger.package_like", return_value=True) + mocker.patch("pathlib.Path.glob", return_value=[Path(str(i)) for i in range(5)]) + mocker.patch("pathlib.Path.iterdir", return_value=[Path(str(i)) for i in range(5)]) + mocker.patch("ahriman.models.package.Package.from_archive", side_effect=package) + unlink_mock = mocker.patch("pathlib.Path.unlink", autospec=True) + + archive_rotation_trigger.archives_remove(package_ahriman, pacman) + unlink_mock.assert_has_calls([ + MockCall(Path("0")), + MockCall(Path("1")), + ]) + + +def test_archives_remove_keep(archive_rotation_trigger: ArchiveRotationTrigger, package_ahriman: Package, + pacman: Pacman, mocker: MockerFixture) -> None: + """ + must keep all packages if set to + """ + def package(version: Any, *args: Any, **kwargs: Any) -> Package: + generated = replace(package_ahriman, version=str(version)) + generated.packages = { + key: replace(value, filename=str(version)) + for key, value in generated.packages.items() + } + return generated + + mocker.patch("pathlib.Path.is_dir", return_value=True) + mocker.patch("ahriman.core.housekeeping.archive_rotation_trigger.package_like", return_value=True) + mocker.patch("pathlib.Path.glob", return_value=[Path(str(i)) for i in range(5)]) + mocker.patch("pathlib.Path.iterdir", return_value=[Path(str(i)) for i in range(5)]) + mocker.patch("ahriman.models.package.Package.from_archive", side_effect=package) + unlink_mock = mocker.patch("pathlib.Path.unlink", autospec=True) + + archive_rotation_trigger.keep_built_packages = 0 + archive_rotation_trigger.archives_remove(package_ahriman, pacman) + unlink_mock.assert_not_called() + + +def test_on_result(archive_rotation_trigger: ArchiveRotationTrigger, package_ahriman: Package, + package_python_schedule: Package, mocker: MockerFixture) -> None: + """ + must rotate archives + """ + mocker.patch("ahriman.core._Context.get") + remove_mock = mocker.patch("ahriman.core.housekeeping.ArchiveRotationTrigger.archives_remove") + archive_rotation_trigger.on_result(Result(added=[package_ahriman], failed=[package_python_schedule]), []) + remove_mock.assert_called_once_with(package_ahriman, pytest.helpers.anyvar(int)) diff --git a/tests/ahriman/core/housekeeping/test_logs_rotation_trigger.py b/tests/ahriman/core/housekeeping/test_logs_rotation_trigger.py index 9be90aff..52443434 100644 --- a/tests/ahriman/core/housekeeping/test_logs_rotation_trigger.py +++ b/tests/ahriman/core/housekeeping/test_logs_rotation_trigger.py @@ -7,13 +7,6 @@ from ahriman.core.status import Client from ahriman.models.result import Result -def test_requires_repository() -> None: - """ - must require repository identifier to be set to start - """ - assert LogsRotationTrigger.REQUIRES_REPOSITORY - - def test_configuration_sections(configuration: Configuration) -> None: """ must correctly parse target list @@ -21,7 +14,7 @@ def test_configuration_sections(configuration: Configuration) -> None: assert LogsRotationTrigger.configuration_sections(configuration) == ["logs-rotation"] -def test_rotate(logs_rotation_trigger: LogsRotationTrigger, mocker: MockerFixture) -> None: +def test_on_result(logs_rotation_trigger: LogsRotationTrigger, mocker: MockerFixture) -> None: """ must rotate logs """ diff --git a/tests/ahriman/core/log/test_lazy_logging.py b/tests/ahriman/core/log/test_lazy_logging.py index a3f416e2..eb2579de 100644 --- a/tests/ahriman/core/log/test_lazy_logging.py +++ b/tests/ahriman/core/log/test_lazy_logging.py @@ -2,7 +2,6 @@ import logging import pytest from pytest_mock import MockerFixture -from unittest.mock import call as MockCall from ahriman.core.alpm.repo import Repo from ahriman.core.build_tools.task import Task diff --git a/tests/ahriman/core/report/test_report_trigger.py b/tests/ahriman/core/report/test_report_trigger.py index e1f40794..9e79edc9 100644 --- a/tests/ahriman/core/report/test_report_trigger.py +++ b/tests/ahriman/core/report/test_report_trigger.py @@ -5,13 +5,6 @@ from ahriman.core.report import ReportTrigger from ahriman.models.result import Result -def test_requires_repository() -> None: - """ - must require repository identifier to be set to start - """ - assert ReportTrigger.REQUIRES_REPOSITORY - - def test_configuration_sections(configuration: Configuration) -> None: """ must correctly parse target list diff --git a/tests/ahriman/core/repository/test_executor.py b/tests/ahriman/core/repository/test_executor.py index 45e12d07..8e33b1a1 100644 --- a/tests/ahriman/core/repository/test_executor.py +++ b/tests/ahriman/core/repository/test_executor.py @@ -1,5 +1,6 @@ import pytest +from dataclasses import replace from pathlib import Path from pytest_mock import MockerFixture from typing import Any @@ -13,34 +14,223 @@ from ahriman.models.packagers import Packagers from ahriman.models.user import User +def test_archive_lookup(executor: Executor, package_ahriman: Package, package_python_schedule: Package, + mocker: MockerFixture) -> None: + """ + must existing packages which match the version + """ + mocker.patch("pathlib.Path.is_dir", return_value=True) + mocker.patch("pathlib.Path.iterdir", return_value=[ + Path("1.pkg.tar.zst"), + Path("2.pkg.tar.zst"), + Path("3.pkg.tar.zst"), + ]) + mocker.patch("ahriman.models.package.Package.from_archive", side_effect=[ + package_ahriman, + package_python_schedule, + replace(package_ahriman, version="1"), + ]) + glob_mock = mocker.patch("pathlib.Path.glob", return_value=[Path("1.pkg.tar.xz")]) + + assert list(executor._archive_lookup(package_ahriman)) == [Path("1.pkg.tar.xz")] + glob_mock.assert_called_once_with(f"{package_ahriman.packages[package_ahriman.base].filename}*") + + +def test_archive_lookup_version_mismatch(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must return nothing if no packages found with the same version + """ + mocker.patch("pathlib.Path.is_dir", return_value=True) + mocker.patch("pathlib.Path.iterdir", return_value=[ + Path("1.pkg.tar.zst"), + ]) + mocker.patch("ahriman.models.package.Package.from_archive", return_value=replace(package_ahriman, version="1")) + + assert list(executor._archive_lookup(package_ahriman)) == [] + + +def test_archive_lookup_architecture_mismatch(executor: Executor, package_ahriman: Package, + mocker: MockerFixture) -> None: + """ + must return nothing if architecture doesn't match + """ + package_ahriman.packages[package_ahriman.base].architecture = "x86_64" + mocker.patch("pathlib.Path.is_dir", return_value=True) + mocker.patch("ahriman.core.repository.executor.Executor.architecture", return_value="i686") + mocker.patch("pathlib.Path.iterdir", return_value=[ + Path("1.pkg.tar.zst"), + ]) + mocker.patch("ahriman.models.package.Package.from_archive", return_value=package_ahriman) + + assert list(executor._archive_lookup(package_ahriman)) == [] + + +def test_archive_lookup_no_archive_directory( + executor: Executor, + package_ahriman: Package, + mocker: MockerFixture) -> None: + """ + must return nothing if no archive directory found + """ + mocker.patch("pathlib.Path.is_dir", return_value=False) + assert list(executor._archive_lookup(package_ahriman)) == [] + + +def test_archive_rename(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must correctly remove package archive + """ + path = "gconf-3.2.6+11+g07808097-10-x86_64.pkg.tar.zst" + safe_path = "gconf-3.2.6-11-g07808097-10-x86_64.pkg.tar.zst" + package_ahriman.packages[package_ahriman.base].filename = path + rename_mock = mocker.patch("ahriman.core.repository.executor.atomic_move") + + executor._archive_rename(package_ahriman.packages[package_ahriman.base], package_ahriman.base) + rename_mock.assert_called_once_with(executor.paths.packages / path, executor.paths.packages / safe_path) + assert package_ahriman.packages[package_ahriman.base].filename == safe_path + + +def test_archive_rename_empty_filename(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must skip renaming if filename is not set + """ + package_ahriman.packages[package_ahriman.base].filename = None + rename_mock = mocker.patch("ahriman.core.repository.executor.atomic_move") + + executor._archive_rename(package_ahriman.packages[package_ahriman.base], package_ahriman.base) + rename_mock.assert_not_called() + + +def test_package_build(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must build single package + """ + mocker.patch("ahriman.core.build_tools.task.Task.build", return_value=[Path(package_ahriman.base)]) + status_client_mock = mocker.patch("ahriman.core.status.Client.set_building") + init_mock = mocker.patch("ahriman.core.build_tools.task.Task.init", return_value="sha") + package_mock = mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman) + lookup_mock = mocker.patch("ahriman.core.repository.executor.Executor._archive_lookup", return_value=[]) + with_packages_mock = mocker.patch("ahriman.models.package.Package.with_packages") + rename_mock = mocker.patch("ahriman.core.repository.executor.atomic_move") + + assert executor._package_build(package_ahriman, Path("local"), "packager", None) == "sha" + status_client_mock.assert_called_once_with(package_ahriman.base) + init_mock.assert_called_once_with(pytest.helpers.anyvar(int), pytest.helpers.anyvar(int), None) + package_mock.assert_called_once_with(Path("local"), executor.architecture, None) + lookup_mock.assert_called_once_with(package_ahriman) + with_packages_mock.assert_called_once_with([Path(package_ahriman.base)], executor.pacman) + rename_mock.assert_called_once_with(Path(package_ahriman.base), executor.paths.packages / package_ahriman.base) + + +def test_package_build_copy(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must copy package from archive if there are already built ones + """ + path = package_ahriman.packages[package_ahriman.base].filepath + mocker.patch("ahriman.core.build_tools.task.Task.build", return_value=[Path(package_ahriman.base)]) + mocker.patch("ahriman.core.build_tools.task.Task.init") + mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman) + mocker.patch("ahriman.core.repository.executor.Executor._archive_lookup", return_value=[path]) + mocker.patch("ahriman.core.repository.executor.atomic_move") + mocker.patch("ahriman.models.package.Package.with_packages") + copy_mock = mocker.patch("shutil.copy") + rename_mock = mocker.patch("ahriman.core.repository.executor.atomic_move") + + executor._package_build(package_ahriman, Path("local"), "packager", None) + copy_mock.assert_called_once_with(path, Path("local")) + rename_mock.assert_called_once_with(Path("local") / path, executor.paths.packages / path) + + +def test_package_remove(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must run remove for packages + """ + repo_remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove") + executor._package_remove(package_ahriman.base, package_ahriman.packages[package_ahriman.base].filepath) + repo_remove_mock.assert_called_once_with( + package_ahriman.base, package_ahriman.packages[package_ahriman.base].filepath) + + +def test_package_remove_failed(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must suppress errors during archive removal + """ + mocker.patch("ahriman.core.alpm.repo.Repo.remove", side_effect=Exception) + executor._package_remove(package_ahriman.base, package_ahriman.packages[package_ahriman.base].filepath) + + +def test_package_remove_base(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must run remove base from status client + """ + status_client_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_remove") + executor._package_remove_base(package_ahriman.base) + status_client_mock.assert_called_once_with(package_ahriman.base) + + +def test_package_remove_base_failed(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must suppress errors during base removal + """ + mocker.patch("ahriman.core.status.local_client.LocalClient.package_remove", side_effect=Exception) + executor._package_remove_base(package_ahriman.base) + + +def test_package_update(executor: Executor, package_ahriman: Package, user: User, mocker: MockerFixture) -> None: + """ + must update built package in repository + """ + rename_mock = mocker.patch("ahriman.core.repository.executor.atomic_move") + symlink_mock = mocker.patch("pathlib.Path.symlink_to") + repo_add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add") + sign_package_mock = mocker.patch("ahriman.core.sign.gpg.GPG.process_sign_package", side_effect=lambda fn, _: [fn]) + filepath = next(package.filepath for package in package_ahriman.packages.values()) + + executor._package_update(filepath, package_ahriman.base, user.key) + # must move files (once) + rename_mock.assert_called_once_with( + executor.paths.packages / filepath, executor.paths.archive_for(package_ahriman.base) / filepath) + # must sign package + sign_package_mock.assert_called_once_with(executor.paths.packages / filepath, user.key) + # symlink to the archive + symlink_mock.assert_called_once_with( + Path("..") / + ".." / + ".." / + executor.paths.archive_for(package_ahriman.base).relative_to(executor.paths.root) / + filepath) + # must add package + repo_add_mock.assert_called_once_with(executor.paths.repository / filepath) + + +def test_package_update_empty_filename(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must skip update for package which does not have path + """ + rename_mock = mocker.patch("ahriman.core.repository.executor.atomic_move") + executor._package_update(None, package_ahriman.base, None) + rename_mock.assert_not_called() + + def test_process_build(executor: Executor, package_ahriman: Package, passwd: Any, mocker: MockerFixture) -> None: """ must run build process """ mocker.patch("ahriman.models.repository_paths.getpwuid", return_value=passwd) mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) - mocker.patch("ahriman.core.build_tools.task.Task.build", return_value=[Path(package_ahriman.base)]) - init_mock = mocker.patch("ahriman.core.build_tools.task.Task.init", return_value="sha") - move_mock = mocker.patch("shutil.move") - status_client_mock = mocker.patch("ahriman.core.status.Client.set_building") changes_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_changes_get", return_value=Changes("commit", "change")) commit_sha_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_changes_update") depends_on_mock = mocker.patch("ahriman.core.build_tools.package_archive.PackageArchive.depends_on", return_value=Dependencies()) dependencies_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_dependencies_update") - with_packages_mock = mocker.patch("ahriman.models.package.Package.with_packages") + build_mock = mocker.patch("ahriman.core.repository.executor.Executor._package_build", return_value="sha") executor.process_build([package_ahriman], Packagers("packager"), bump_pkgrel=False) - init_mock.assert_called_once_with(pytest.helpers.anyvar(int), pytest.helpers.anyvar(int), None) - with_packages_mock.assert_called_once_with([Path(package_ahriman.base)], executor.pacman) changes_mock.assert_called_once_with(package_ahriman.base) + build_mock.assert_called_once_with(package_ahriman, pytest.helpers.anyvar(Path, strict=True), None, None) depends_on_mock.assert_called_once_with() dependencies_mock.assert_called_once_with(package_ahriman.base, Dependencies()) - # must move files (once) - move_mock.assert_called_once_with(Path(package_ahriman.base), executor.paths.packages / package_ahriman.base) - # must update status - status_client_mock.assert_called_once_with(package_ahriman.base) commit_sha_mock.assert_called_once_with(package_ahriman.base, Changes("sha", "change")) @@ -50,7 +240,7 @@ def test_process_build_bump_pkgrel(executor: Executor, package_ahriman: Package, """ mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) mocker.patch("ahriman.core.build_tools.task.Task.build", return_value=[Path(package_ahriman.base)]) - mocker.patch("shutil.move") + mocker.patch("ahriman.core.repository.executor.atomic_move") init_mock = mocker.patch("ahriman.core.build_tools.task.Task.init") executor.process_build([package_ahriman], Packagers("packager"), bump_pkgrel=True) @@ -67,7 +257,7 @@ def test_process_build_failure(executor: Executor, package_ahriman: Package, moc mocker.patch("ahriman.core.repository.executor.Executor.packages_built") mocker.patch("ahriman.core.build_tools.task.Task.build", return_value=[Path(package_ahriman.base)]) mocker.patch("ahriman.core.build_tools.task.Task.init") - mocker.patch("shutil.move", side_effect=Exception) + mocker.patch("ahriman.core.repository.executor.atomic_move", side_effect=Exception) status_client_mock = mocker.patch("ahriman.core.status.Client.set_failed") executor.process_build([package_ahriman]) @@ -79,15 +269,15 @@ def test_process_remove_base(executor: Executor, package_ahriman: Package, mocke must run remove process for whole base """ mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) - repo_remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove") - status_client_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_remove") + remove_mock = mocker.patch("ahriman.core.repository.executor.Executor._package_remove") + base_remove_mock = mocker.patch("ahriman.core.repository.executor.Executor._package_remove_base") executor.process_remove([package_ahriman.base]) # must remove via alpm wrapper - repo_remove_mock.assert_called_once_with( + remove_mock.assert_called_once_with( package_ahriman.base, package_ahriman.packages[package_ahriman.base].filepath) # must update status and remove package files - status_client_mock.assert_called_once_with(package_ahriman.base) + base_remove_mock.assert_called_once_with(package_ahriman.base) def test_process_remove_with_debug(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: @@ -99,12 +289,12 @@ def test_process_remove_with_debug(executor: Executor, package_ahriman: Package, f"{package_ahriman.base}-debug": package_ahriman.packages[package_ahriman.base], } mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) - mocker.patch("ahriman.core.status.local_client.LocalClient.package_remove") - repo_remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove") + mocker.patch("ahriman.core.repository.executor.Executor._package_remove_base") + remove_mock = mocker.patch("ahriman.core.repository.executor.Executor._package_remove") executor.process_remove([package_ahriman.base]) # must remove via alpm wrapper - repo_remove_mock.assert_has_calls([ + remove_mock.assert_has_calls([ MockCall(package_ahriman.base, package_ahriman.packages[package_ahriman.base].filepath), MockCall(f"{package_ahriman.base}-debug", package_ahriman.packages[package_ahriman.base].filepath), ]) @@ -116,12 +306,12 @@ def test_process_remove_base_multiple(executor: Executor, package_python_schedul must run remove process for whole base with multiple packages """ mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_python_schedule]) - repo_remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove") - status_client_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_remove") + remove_mock = mocker.patch("ahriman.core.repository.executor.Executor._package_remove") + status_client_mock = mocker.patch("ahriman.core.repository.executor.Executor._package_remove_base") executor.process_remove([package_python_schedule.base]) # must remove via alpm wrapper - repo_remove_mock.assert_has_calls([ + remove_mock.assert_has_calls([ MockCall(package, props.filepath) for package, props in package_python_schedule.packages.items() ], any_order=True) @@ -135,45 +325,27 @@ def test_process_remove_base_single(executor: Executor, package_python_schedule: must run remove process for single package in base """ mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_python_schedule]) - repo_remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove") - status_client_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_remove") + remove_mock = mocker.patch("ahriman.core.repository.executor.Executor._package_remove") + status_client_mock = mocker.patch("ahriman.core.repository.executor.Executor._package_remove_base") executor.process_remove(["python2-schedule"]) # must remove via alpm wrapper - repo_remove_mock.assert_called_once_with( + remove_mock.assert_called_once_with( "python2-schedule", package_python_schedule.packages["python2-schedule"].filepath) # must not update status status_client_mock.assert_not_called() -def test_process_remove_failed(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: - """ - must suppress tree clear errors during package base removal - """ - mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) - mocker.patch("ahriman.core.status.local_client.LocalClient.package_remove", side_effect=Exception) - executor.process_remove([package_ahriman.base]) - - -def test_process_remove_tree_clear_failed(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: - """ - must suppress remove errors - """ - mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) - mocker.patch("ahriman.core.alpm.repo.Repo.remove", side_effect=Exception) - executor.process_remove([package_ahriman.base]) - - def test_process_remove_nothing(executor: Executor, package_ahriman: Package, package_python_schedule: Package, mocker: MockerFixture) -> None: """ must not remove anything if it was not requested """ mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) - repo_remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove") + remove_mock = mocker.patch("ahriman.core.repository.executor.Executor._package_remove") executor.process_remove([package_python_schedule.base]) - repo_remove_mock.assert_not_called() + remove_mock.assert_not_called() def test_process_remove_unknown(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: @@ -181,11 +353,11 @@ def test_process_remove_unknown(executor: Executor, package_ahriman: Package, mo must remove unknown package base """ mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[]) - repo_remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove") - status_client_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_remove") + remove_mock = mocker.patch("ahriman.core.repository.executor.Executor._package_remove") + status_client_mock = mocker.patch("ahriman.core.repository.executor.Executor._package_remove_base") executor.process_remove([package_ahriman.base]) - repo_remove_mock.assert_not_called() + remove_mock.assert_not_called() status_client_mock.assert_called_once_with(package_ahriman.base) @@ -195,9 +367,8 @@ def test_process_update(executor: Executor, package_ahriman: Package, user: User """ mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_ahriman]) mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) - move_mock = mocker.patch("shutil.move") - repo_add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add") - sign_package_mock = mocker.patch("ahriman.core.sign.gpg.GPG.process_sign_package", side_effect=lambda fn, _: [fn]) + rename_mock = mocker.patch("ahriman.core.repository.executor.Executor._archive_rename") + update_mock = mocker.patch("ahriman.core.repository.executor.Executor._package_update") status_client_mock = mocker.patch("ahriman.core.status.Client.set_success") remove_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_remove") packager_mock = mocker.patch("ahriman.core.repository.executor.Executor.packager", return_value=user) @@ -206,12 +377,8 @@ def test_process_update(executor: Executor, package_ahriman: Package, user: User # must return complete assert executor.process_update([filepath], Packagers("packager")) packager_mock.assert_called_once_with(Packagers("packager"), "ahriman") - # must move files (once) - move_mock.assert_called_once_with(executor.paths.packages / filepath, executor.paths.repository / filepath) - # must sign package - sign_package_mock.assert_called_once_with(executor.paths.packages / filepath, user.key) - # must add package - repo_add_mock.assert_called_once_with(executor.paths.repository / filepath) + rename_mock.assert_called_once_with(package_ahriman.packages[package_ahriman.base], package_ahriman.base) + update_mock.assert_called_once_with(filepath.name, package_ahriman.base, user.key) # must update status status_client_mock.assert_called_once_with(package_ahriman) # must clear directory @@ -226,58 +393,26 @@ def test_process_update_group(executor: Executor, package_python_schedule: Packa """ must group single packages under one base """ - mocker.patch("shutil.move") mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_python_schedule]) mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_python_schedule]) - repo_add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add") + update_mock = mocker.patch("ahriman.core.repository.executor.Executor._package_update") status_client_mock = mocker.patch("ahriman.core.status.Client.set_success") remove_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_remove") executor.process_update([package.filepath for package in package_python_schedule.packages.values()]) - repo_add_mock.assert_has_calls([ - MockCall(executor.paths.repository / package.filepath) + update_mock.assert_has_calls([ + MockCall(package.filename, package_python_schedule.base, None) for package in package_python_schedule.packages.values() ], any_order=True) status_client_mock.assert_called_once_with(package_python_schedule) remove_mock.assert_called_once_with([]) -def test_process_update_unsafe(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: - """ - must encode file name - """ - path = "gconf-3.2.6+11+g07808097-10-x86_64.pkg.tar.zst" - safe_path = "gconf-3.2.6-11-g07808097-10-x86_64.pkg.tar.zst" - package_ahriman.packages[package_ahriman.base].filename = path - - mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_ahriman]) - mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) - move_mock = mocker.patch("shutil.move") - repo_add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add") - - executor.process_update([Path(path)]) - move_mock.assert_has_calls([ - MockCall(executor.paths.packages / path, executor.paths.packages / safe_path), - MockCall(executor.paths.packages / safe_path, executor.paths.repository / safe_path) - ]) - repo_add_mock.assert_called_once_with(executor.paths.repository / safe_path) - - -def test_process_empty_filename(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: - """ - must skip update for package which does not have path - """ - package_ahriman.packages[package_ahriman.base].filename = None - mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_ahriman]) - mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) - executor.process_update([package.filepath for package in package_ahriman.packages.values()]) - - def test_process_update_failed(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: """ must process update for failed package """ - mocker.patch("shutil.move", side_effect=Exception) + mocker.patch("ahriman.core.repository.executor.Executor._package_update", side_effect=Exception) mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_ahriman]) mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) status_client_mock = mocker.patch("ahriman.core.status.Client.set_failed") @@ -294,8 +429,7 @@ def test_process_update_removed_package(executor: Executor, package_python_sched without_python2 = Package.from_json(package_python_schedule.view()) del without_python2.packages["python2-schedule"] - mocker.patch("shutil.move") - mocker.patch("ahriman.core.alpm.repo.Repo.add") + mocker.patch("ahriman.core.repository.executor.Executor._package_update") mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[without_python2]) mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_python_schedule]) remove_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_remove") diff --git a/tests/ahriman/core/repository/test_explorer.py b/tests/ahriman/core/repository/test_explorer.py new file mode 100644 index 00000000..5e71810e --- /dev/null +++ b/tests/ahriman/core/repository/test_explorer.py @@ -0,0 +1,56 @@ +from pytest_mock import MockerFixture + +from ahriman.core.configuration import Configuration +from ahriman.core.repository import Explorer +from ahriman.models.repository_id import RepositoryId + + +def test_repositories_extract(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must generate list of available repositories based on arguments + """ + known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures") + known_repositories_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_repositories") + + assert Explorer.repositories_extract(configuration, "repo", "arch") == [RepositoryId("arch", "repo")] + known_architectures_mock.assert_not_called() + known_repositories_mock.assert_not_called() + + +def test_repositories_extract_repository(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must generate list of available repositories based on arguments and tree + """ + known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures") + known_repositories_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_repositories", + return_value={"repo"}) + + assert Explorer.repositories_extract(configuration, architecture="arch") == [RepositoryId("arch", "repo")] + known_architectures_mock.assert_not_called() + known_repositories_mock.assert_called_once_with(configuration.repository_paths.root) + + +def test_repositories_extract_repository_legacy(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must generate list of available repositories based on arguments and tree (legacy mode) + """ + known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures") + known_repositories_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_repositories", + return_value=set()) + + assert Explorer.repositories_extract(configuration, architecture="arch") == [RepositoryId("arch", "aur")] + known_architectures_mock.assert_not_called() + known_repositories_mock.assert_called_once_with(configuration.repository_paths.root) + + +def test_repositories_extract_architecture(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must read repository name from config + """ + known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures", + return_value={"arch"}) + known_repositories_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_repositories") + + assert Explorer.repositories_extract(configuration, repository="repo") == [RepositoryId("arch", "repo")] + known_architectures_mock.assert_called_once_with(configuration.repository_paths.root, "repo") + known_repositories_mock.assert_not_called() diff --git a/tests/ahriman/core/status/test_local_client.py b/tests/ahriman/core/status/test_local_client.py index 38ad7330..d698f1f6 100644 --- a/tests/ahriman/core/status/test_local_client.py +++ b/tests/ahriman/core/status/test_local_client.py @@ -35,7 +35,7 @@ def test_event_get(local_client: LocalClient, package_ahriman: Package, mocker: local_client.repository_id) -def test_logs_rotate(local_client: LocalClient, package_ahriman: Package, mocker: MockerFixture) -> None: +def test_logs_rotate(local_client: LocalClient, mocker: MockerFixture) -> None: """ must rotate logs """ diff --git a/tests/ahriman/core/support/test_keyring_trigger.py b/tests/ahriman/core/support/test_keyring_trigger.py index 29ec7b2a..13631858 100644 --- a/tests/ahriman/core/support/test_keyring_trigger.py +++ b/tests/ahriman/core/support/test_keyring_trigger.py @@ -7,13 +7,6 @@ from ahriman.core.sign.gpg import GPG from ahriman.core.support import KeyringTrigger -def test_requires_repository() -> None: - """ - must require repository identifier to be set to start - """ - assert KeyringTrigger.REQUIRES_REPOSITORY - - def test_configuration_sections(configuration: Configuration) -> None: """ must correctly parse target list diff --git a/tests/ahriman/core/support/test_mirrorlist_trigger.py b/tests/ahriman/core/support/test_mirrorlist_trigger.py index be65fe59..7f34faaa 100644 --- a/tests/ahriman/core/support/test_mirrorlist_trigger.py +++ b/tests/ahriman/core/support/test_mirrorlist_trigger.py @@ -4,13 +4,6 @@ from ahriman.core.configuration import Configuration from ahriman.core.support import MirrorlistTrigger -def test_requires_repository() -> None: - """ - must require repository identifier to be set to start - """ - assert MirrorlistTrigger.REQUIRES_REPOSITORY - - def test_configuration_sections(configuration: Configuration) -> None: """ must correctly parse target list diff --git a/tests/ahriman/core/test_utils.py b/tests/ahriman/core/test_utils.py index c822db84..10a84aea 100644 --- a/tests/ahriman/core/test_utils.py +++ b/tests/ahriman/core/test_utils.py @@ -16,6 +16,18 @@ from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_paths import RepositoryPaths +def test_atomic_move(mocker: MockerFixture) -> None: + """ + must move file with locking + """ + filelock_mock = mocker.patch("ahriman.core.utils.filelock") + move_mock = mocker.patch("shutil.move") + + atomic_move(Path("source"), Path("destination")) + filelock_mock.assert_called_once_with(Path("destination")) + move_mock.assert_called_once_with(Path("source"), Path("destination")) + + def test_check_output(mocker: MockerFixture) -> None: """ must run command and log result @@ -235,6 +247,30 @@ def test_extract_user() -> None: assert extract_user() == "doas" +def test_filelock(tmp_path: Path) -> None: + """ + must acquire lock and remove lock file after + """ + local = tmp_path / "local" + lock = local.with_name(f".{local.name}.lock") + + with filelock(local): + assert lock.exists() + assert not lock.exists() + + +def test_filelock_cleanup_on_missing(tmp_path: Path) -> None: + """ + must not fail if lock file is already removed + """ + local = tmp_path / "local" + lock = local.with_name(f".{local.name}.lock") + + with filelock(local): + lock.unlink(missing_ok=True) + assert not lock.exists() + + def test_filter_json(package_ahriman: Package) -> None: """ must filter fields by known list @@ -470,6 +506,23 @@ def test_srcinfo_property_list() -> None: assert srcinfo_property_list("key", {"key_x86_64": ["overrides"]}, {}, architecture="x86_64") == ["overrides"] +def test_symlink_relative(mocker: MockerFixture) -> None: + """ + must create symlinks with relative paths + """ + symlink_mock = mocker.patch("pathlib.Path.symlink_to") + + symlink_relative(Path("a"), Path("b")) + symlink_relative(Path("root/a"), Path("root/c")) + symlink_relative(Path("root/sub/a"), Path("root/c")) + + symlink_mock.assert_has_calls([ + MockCall(Path("b")), + MockCall(Path("c")), + MockCall(Path("../c")), + ]) + + def test_trim_package() -> None: """ must trim package version diff --git a/tests/ahriman/core/triggers/test_trigger.py b/tests/ahriman/core/triggers/test_trigger.py index f4804d90..b6658e86 100644 --- a/tests/ahriman/core/triggers/test_trigger.py +++ b/tests/ahriman/core/triggers/test_trigger.py @@ -58,7 +58,7 @@ def test_configuration_schema_no_schema(configuration: Configuration) -> None: assert ReportTrigger.configuration_schema(configuration) == {} -def test_configuration_schema_empty(configuration: Configuration) -> None: +def test_configuration_schema_empty() -> None: """ must return default schema if no configuration set """ diff --git a/tests/ahriman/core/upload/test_upload_trigger.py b/tests/ahriman/core/upload/test_upload_trigger.py index 444b31dc..f49bc02c 100644 --- a/tests/ahriman/core/upload/test_upload_trigger.py +++ b/tests/ahriman/core/upload/test_upload_trigger.py @@ -5,13 +5,6 @@ from ahriman.core.upload import UploadTrigger from ahriman.models.result import Result -def test_requires_repository() -> None: - """ - must require repository identifier to be set to start - """ - assert UploadTrigger.REQUIRES_REPOSITORY - - def test_configuration_sections(configuration: Configuration) -> None: """ must correctly parse target list diff --git a/tests/ahriman/models/conftest.py b/tests/ahriman/models/conftest.py index ff68a1cc..165e43c2 100644 --- a/tests/ahriman/models/conftest.py +++ b/tests/ahriman/models/conftest.py @@ -4,16 +4,12 @@ from pathlib import Path from unittest.mock import MagicMock, PropertyMock from ahriman import __version__ -from ahriman.core.alpm.remote import AUR from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.counters import Counters from ahriman.models.filesystem_package import FilesystemPackage from ahriman.models.internal_status import InternalStatus -from ahriman.models.package import Package from ahriman.models.package_description import PackageDescription -from ahriman.models.package_source import PackageSource from ahriman.models.pkgbuild import Pkgbuild -from ahriman.models.remote_source import RemoteSource from ahriman.models.repository_stats import RepositoryStats diff --git a/tests/ahriman/models/test_package.py b/tests/ahriman/models/test_package.py index ae35f3d0..1843dbf4 100644 --- a/tests/ahriman/models/test_package.py +++ b/tests/ahriman/models/test_package.py @@ -353,6 +353,15 @@ def test_build_status_pretty_print(package_ahriman: Package) -> None: assert isinstance(package_ahriman.pretty_print(), str) +def test_vercmp(package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must call vercmp + """ + vercmp_mock = mocker.patch("ahriman.models.package.vercmp") + package_ahriman.vercmp("version") + vercmp_mock.assert_called_once_with(package_ahriman.version, "version") + + def test_with_packages(package_ahriman: Package, package_python_schedule: Package, pacman: Pacman, mocker: MockerFixture) -> None: """ diff --git a/tests/ahriman/models/test_repository_paths.py b/tests/ahriman/models/test_repository_paths.py index 3efed6b3..303819f5 100644 --- a/tests/ahriman/models/test_repository_paths.py +++ b/tests/ahriman/models/test_repository_paths.py @@ -185,6 +185,14 @@ def test_known_repositories_empty(repository_paths: RepositoryPaths, mocker: Moc iterdir_mock.assert_not_called() +def test_archive_for(repository_paths: RepositoryPaths, package_ahriman: Package) -> None: + """ + must correctly define archive path + """ + path = repository_paths.archive_for(package_ahriman.base) + assert path == repository_paths.archive / "packages" / "a" / package_ahriman.base + + def test_cache_for(repository_paths: RepositoryPaths, package_ahriman: Package) -> None: """ must return correct path for cache directory @@ -194,6 +202,29 @@ def test_cache_for(repository_paths: RepositoryPaths, package_ahriman: Package) assert path.parent == repository_paths.cache +def test_ensure_exists(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: + """ + must create directory if it doesn't exist + """ + owner_guard_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.preserve_owner") + mkdir_mock = mocker.patch("pathlib.Path.mkdir") + + repository_paths.ensure_exists(repository_paths.archive) + owner_guard_mock.assert_called_once_with() + mkdir_mock.assert_called_once_with(mode=0o755, parents=True) + + +def test_ensure_exists_skip(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: + """ + must do not create directory if it already exists + """ + mocker.patch("pathlib.Path.is_dir", return_value=True) + mkdir_mock = mocker.patch("pathlib.Path.mkdir") + + repository_paths.ensure_exists(repository_paths.archive) + mkdir_mock.assert_not_called() + + def test_preserve_owner(tmp_path: Path, repository_id: RepositoryId, mocker: MockerFixture) -> None: """ must preserve file owner during operations @@ -248,8 +279,8 @@ def test_tree_clear(repository_paths: RepositoryPaths, package_ahriman: Package, must remove any package related files """ paths = { - getattr(repository_paths, prop)(package_ahriman.base) - for prop in dir(repository_paths) if prop.endswith("_for") + repository_paths.cache_for(package_ahriman.base), + repository_paths.archive_for(package_ahriman.base), } rmtree_mock = mocker.patch("shutil.rmtree") @@ -269,6 +300,7 @@ def test_tree_create(repository_paths: RepositoryPaths, mocker: MockerFixture) - for prop in dir(repository_paths) if not prop.startswith("_") and prop not in ( + "archive_for", "build_root", "logger_name", "logger", @@ -282,8 +314,12 @@ def test_tree_create(repository_paths: RepositoryPaths, mocker: MockerFixture) - owner_guard_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.preserve_owner") repository_paths.tree_create() - mkdir_mock.assert_has_calls([MockCall(mode=0o755, parents=True, exist_ok=True) for _ in paths], any_order=True) - owner_guard_mock.assert_called_once_with() + mkdir_mock.assert_has_calls([MockCall(mode=0o755, parents=True) for _ in paths], any_order=True) + owner_guard_mock.assert_has_calls([ + MockCall(), + MockCall().__enter__(), + MockCall().__exit__(None, None, None) + ] * len(paths)) def test_tree_create_skip(mocker: MockerFixture) -> None: diff --git a/tests/ahriman/web/views/v1/service/test_view_v1_service_logs.py b/tests/ahriman/web/views/v1/service/test_view_v1_service_logs.py index 131c4032..60648365 100644 --- a/tests/ahriman/web/views/v1/service/test_view_v1_service_logs.py +++ b/tests/ahriman/web/views/v1/service/test_view_v1_service_logs.py @@ -66,7 +66,7 @@ async def test_delete_partially(client: TestClient, package_ahriman: Package) -> assert json -async def test_delete_exception(client: TestClient, package_ahriman: Package) -> None: +async def test_delete_exception(client: TestClient) -> None: """ must raise exception on invalid payload """ diff --git a/tests/ahriman/web/views/v1/service/test_view_v1_service_upload.py b/tests/ahriman/web/views/v1/service/test_view_v1_service_upload.py index 7ab7f41a..c8bef0f6 100644 --- a/tests/ahriman/web/views/v1/service/test_view_v1_service_upload.py +++ b/tests/ahriman/web/views/v1/service/test_view_v1_service_upload.py @@ -109,7 +109,7 @@ async def test_post(client: TestClient, repository_paths: RepositoryPaths, mocke local = Path("local") save_mock = pytest.helpers.patch_view(client.app, "save_file", AsyncMock(return_value=("filename", local / ".filename"))) - rename_mock = mocker.patch("pathlib.Path.rename") + rename_mock = mocker.patch("ahriman.web.views.v1.service.upload.atomic_move") # no content validation here because it has invalid schema data = FormData() @@ -118,7 +118,7 @@ async def test_post(client: TestClient, repository_paths: RepositoryPaths, mocke response = await client.post("/api/v1/service/upload", data=data) assert response.ok save_mock.assert_called_once_with(pytest.helpers.anyvar(int), repository_paths.packages, max_body_size=None) - rename_mock.assert_called_once_with(local / "filename") + rename_mock.assert_called_once_with(local / ".filename", local / "filename") async def test_post_with_sig(client: TestClient, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: @@ -131,7 +131,7 @@ async def test_post_with_sig(client: TestClient, repository_paths: RepositoryPat ("filename", local / ".filename"), ("filename.sig", local / ".filename.sig"), ])) - rename_mock = mocker.patch("pathlib.Path.rename") + rename_mock = mocker.patch("ahriman.web.views.v1.service.upload.atomic_move") # no content validation here because it has invalid schema data = FormData() @@ -145,8 +145,8 @@ async def test_post_with_sig(client: TestClient, repository_paths: RepositoryPat MockCall(pytest.helpers.anyvar(int), repository_paths.packages, max_body_size=None), ]) rename_mock.assert_has_calls([ - MockCall(local / "filename"), - MockCall(local / "filename.sig"), + MockCall(local / ".filename", local / "filename"), + MockCall(local / ".filename.sig", local / "filename.sig"), ]) diff --git a/tests/testresources/core/ahriman.ini b/tests/testresources/core/ahriman.ini index 522e85dc..36715f8f 100644 --- a/tests/testresources/core/ahriman.ini +++ b/tests/testresources/core/ahriman.ini @@ -10,6 +10,9 @@ root = / sync_files_database = no use_ahriman_cache = no +[archive] +keep_built_packages = 3 + [auth] client_id = client_id client_secret = client_secret