diff --git a/package/share/ahriman/settings/ahriman.ini.d/00-triggers.ini b/package/share/ahriman/settings/ahriman.ini.d/00-triggers.ini index ae29f15c..b097cc3a 100644 --- a/package/share/ahriman/settings/ahriman.ini.d/00-triggers.ini +++ b/package/share/ahriman/settings/ahriman.ini.d/00-triggers.ini @@ -1,5 +1,6 @@ [build] ; List of well-known triggers. Used only for configuration purposes. +triggers_known[] = ahriman.core.archive.ArchiveTrigger triggers_known[] = ahriman.core.distributed.WorkerLoaderTrigger triggers_known[] = ahriman.core.distributed.WorkerTrigger triggers_known[] = ahriman.core.support.KeyringTrigger diff --git a/src/ahriman/application/handlers/status.py b/src/ahriman/application/handlers/status.py index 235cca54..5273672f 100644 --- a/src/ahriman/application/handlers/status.py +++ b/src/ahriman/application/handlers/status.py @@ -66,7 +66,7 @@ class Status(Handler): Status.check_status(args.exit_code, packages) comparator: Callable[[tuple[Package, BuildStatus]], Comparable] = lambda item: item[0].base - filter_fn: Callable[[tuple[Package, BuildStatus]], bool] =\ + filter_fn: Callable[[tuple[Package, BuildStatus]], bool] = \ lambda item: args.status is None or item[1].status == args.status for package, package_status in sorted(filter(filter_fn, packages), key=comparator): PackagePrinter(package, package_status)(verbose=args.info) diff --git a/src/ahriman/core/alpm/repo.py b/src/ahriman/core/alpm/repo.py index 3d8c1839..380943e6 100644 --- a/src/ahriman/core/alpm/repo.py +++ b/src/ahriman/core/alpm/repo.py @@ -88,22 +88,24 @@ class Repo(LazyLogging): check_output("repo-add", *self.sign_args, str(self.repo_path), cwd=self.root, logger=self.logger, user=self.uid) - def remove(self, package: str, filename: Path) -> None: + def remove(self, package_name: str | None, filename: Path) -> None: """ remove package from repository Args: - package(str): package name to remove + package_name(str | None): package name to remove. If none set, it will be guessed from filename filename(Path): package filename to remove """ + package_name = package_name or filename.name.rsplit("-", maxsplit=3)[0] + # remove package and signature (if any) from filesystem for full_path in self.root.glob(f"**/{filename.name}*"): full_path.unlink() # remove package from registry check_output( - "repo-remove", *self.sign_args, str(self.repo_path), package, - exception=BuildError.from_process(package), + "repo-remove", *self.sign_args, str(self.repo_path), package_name, + exception=BuildError.from_process(package_name), cwd=self.root, logger=self.logger, user=self.uid, diff --git a/src/ahriman/core/archive/__init__.py b/src/ahriman/core/archive/__init__.py index 7413eea9..3862fe71 100644 --- a/src/ahriman/core/archive/__init__.py +++ b/src/ahriman/core/archive/__init__.py @@ -17,3 +17,4 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +from ahriman.core.archive.archive_trigger import ArchiveTrigger diff --git a/src/ahriman/core/archive/archive_tree.py b/src/ahriman/core/archive/archive_tree.py new file mode 100644 index 00000000..dfa7500e --- /dev/null +++ b/src/ahriman/core/archive/archive_tree.py @@ -0,0 +1,130 @@ +# +# Copyright (c) 2021-2025 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import datetime + +from pathlib import Path + +from ahriman.core.alpm.repo import Repo +from ahriman.core.log import LazyLogging +from ahriman.core.utils import utcnow, walk +from ahriman.models.package import Package +from ahriman.models.repository_paths import RepositoryPaths + + +class ArchiveTree(LazyLogging): + """ + wrapper around archive tree + + Attributes: + paths(RepositoryPaths): repository paths instance + repository_id(RepositoryId): repository unique identifier + sign_args(list[str]): additional args which have to be used to sign repository archive + """ + + def __init__(self, repository_path: RepositoryPaths, sign_args: list[str]) -> None: + """ + Args: + repository_path(RepositoryPaths): repository paths instance + sign_args(list[str]): additional args which have to be used to sign repository archive + """ + self.paths = repository_path + self.repository_id = repository_path.repository_id + self.sign_args = sign_args + + def repository_for(self, date: datetime.date | None = None) -> Path: + """ + get full path to repository at the specified date + + Args: + date(datetime.date | None, optional): date to generate path. If none supplied then today will be used + (Default value = None) + + Returns: + Path: path to the repository root + """ + date = date or utcnow().date() + return ( + self.paths.archive + / "repos" + / date.strftime("%Y") + / date.strftime("%m") + / date.strftime("%d") + / self.repository_id.name + / self.repository_id.architecture + ) + + def symlinks_create(self, packages: list[Package]) -> None: + """ + create symlinks for the specified packages in today's repository + + Args: + packages(list[Package]): list of packages to be updated + """ + root = self.repository_for() + repo = Repo(self.repository_id.name, self.paths, self.sign_args, root) + + for package in packages: + archive = self.paths.archive_for(package.base) + + for package_name, single in package.packages.items(): + if single.filename is None: + self.logger.warning("received empty package filename for %s", package_name) + continue + + has_file = False + for file in archive.glob(f"{single.filename}*"): + symlink = root / file.name + if symlink.exists(): + continue # symlink is already created, skip processing + has_file = True + symlink.symlink_to(file.relative_to(symlink.parent, walk_up=True)) + + if has_file: + repo.add(root / single.filename) + + def symlinks_fix(self) -> None: + """ + remove broken symlinks across repositories for all dates + """ + for path in walk(self.paths.archive / "repos"): + root = path.parent + *_, name, architecture = root.parts + if self.repository_id.name != name or self.repository_id.architecture != architecture: + continue # we only process same name repositories + + if not path.is_symlink(): + continue # find symlinks only + if path.exists(): + continue # filter out not broken symlinks + + Repo(self.repository_id.name, self.paths, self.sign_args, root).remove(None, path) + + def tree_create(self) -> None: + """ + create repository tree for current repository + """ + root = self.repository_for() + if root.exists(): + return + + with self.paths.preserve_owner(self.paths.archive): + root.mkdir(0o755, parents=True) + # init empty repository here + Repo(self.repository_id.name, self.paths, self.sign_args, root).init() diff --git a/src/ahriman/core/archive/archive_trigger.py b/src/ahriman/core/archive/archive_trigger.py index 14e188ac..935974f7 100644 --- a/src/ahriman/core/archive/archive_trigger.py +++ b/src/ahriman/core/archive/archive_trigger.py @@ -17,21 +17,23 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from pathlib import Path - +from ahriman.core import context +from ahriman.core.archive.archive_tree import ArchiveTree from ahriman.core.configuration import Configuration +from ahriman.core.sign.gpg import GPG from ahriman.core.triggers import Trigger from ahriman.models.package import Package from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result -class ArchiveRotationTrigger(Trigger): +class ArchiveTrigger(Trigger): """ archive repository extension Attributes: paths(RepositoryPaths): repository paths instance + tree(ArchiveTree): archive tree wrapper """ def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: @@ -44,9 +46,8 @@ class ArchiveRotationTrigger(Trigger): self.paths = configuration.repository_paths - @property - def repos_path(self) -> Path: - return self.paths.archive / "repos" + ctx = context.get() + self.tree = ArchiveTree(self.paths, ctx.get(GPG).repository_sign_args) def on_result(self, result: Result, packages: list[Package]) -> None: """ @@ -56,10 +57,16 @@ class ArchiveRotationTrigger(Trigger): result(Result): build result packages(list[Package]): list of all available packages """ + self.tree.symlinks_create(packages) def on_start(self) -> None: """ trigger action which will be called at the start of the application """ - with self.paths.preserve_owner(self.repos_path): - self.repos_path.mkdir(mode=0o755, exist_ok=True) + self.tree.tree_create() + + def on_stop(self) -> None: + """ + trigger action which will be called before the stop of the application + """ + self.tree.symlinks_fix() diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py index 390dace1..89a806cb 100644 --- a/src/ahriman/core/repository/executor.py +++ b/src/ahriman/core/repository/executor.py @@ -80,7 +80,7 @@ class Executor(PackageInfo, Cleaner): package_base(str): package base name """ if description.filename is None: - self.logger.warning("received empty package name for base %s", package_base) + self.logger.warning("received empty package filename for base %s", package_base) return # suppress type checking, it never can be none actually if (safe := safe_filename(description.filename)) != description.filename: @@ -161,7 +161,7 @@ class Executor(PackageInfo, Cleaner): packager_key(str | None): packager key identifier """ if filename is None: - self.logger.warning("received empty package name for base %s", package_base) + self.logger.warning("received empty package filename for base %s", package_base) return # suppress type checking, it never can be none actually # in theory, it might be NOT packages directory, but we suppose it is diff --git a/subpackages.py b/subpackages.py index 1e7659fb..ec871c39 100644 --- a/subpackages.py +++ b/subpackages.py @@ -37,6 +37,7 @@ SUBPACKAGES = { "ahriman-triggers": [ prefix / "share" / "ahriman" / "settings" / "ahriman.ini.d" / "00-triggers.ini", site_packages / "ahriman" / "application" / "handlers" / "triggers_support.py", + site_packages / "ahriman" / "core" / "archive", site_packages / "ahriman" / "core" / "distributed", site_packages / "ahriman" / "core" / "support", ], diff --git a/tests/ahriman/core/alpm/test_repo.py b/tests/ahriman/core/alpm/test_repo.py index 22f27997..6e406e47 100644 --- a/tests/ahriman/core/alpm/test_repo.py +++ b/tests/ahriman/core/alpm/test_repo.py @@ -4,6 +4,7 @@ from pathlib import Path from pytest_mock import MockerFixture from ahriman.core.alpm.repo import Repo +from ahriman.models.package import Package from ahriman.models.repository_paths import RepositoryPaths @@ -56,21 +57,37 @@ def test_repo_init(repo: Repo, mocker: MockerFixture) -> None: assert check_output_mock.call_args[0][0] == "repo-add" -def test_repo_remove(repo: Repo, mocker: MockerFixture) -> None: +def test_repo_remove(repo: Repo, package_ahriman: Package,mocker: MockerFixture) -> None: """ - must call repo-remove on package addition + must call repo-remove on package removal """ + filepath = package_ahriman.packages[package_ahriman.base].filepath mocker.patch("pathlib.Path.glob", return_value=[]) check_output_mock = mocker.patch("ahriman.core.alpm.repo.check_output") - repo.remove("package", Path("package.pkg.tar.xz")) + repo.remove(package_ahriman.base, filepath) check_output_mock.assert_called_once() # it will be checked later assert check_output_mock.call_args[0][0] == "repo-remove" + assert package_ahriman.base in check_output_mock.call_args[0] + + +def test_repo_remove_guess_package(repo: Repo, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must call repo-remove on package removal if no package name set + """ + filepath = package_ahriman.packages[package_ahriman.base].filepath + mocker.patch("pathlib.Path.glob", return_value=[]) + check_output_mock = mocker.patch("ahriman.core.alpm.repo.check_output") + + repo.remove(None, filepath) + check_output_mock.assert_called_once() # it will be checked later + assert check_output_mock.call_args[0][0] == "repo-remove" + assert package_ahriman.base in check_output_mock.call_args[0] def test_repo_remove_fail_no_file(repo: Repo, mocker: MockerFixture) -> None: """ - must fail on missing file + must fail removal on missing file """ mocker.patch("pathlib.Path.glob", return_value=[Path("package.pkg.tar.xz")]) mocker.patch("pathlib.Path.unlink", side_effect=FileNotFoundError) diff --git a/tests/ahriman/core/archive/conftest.py b/tests/ahriman/core/archive/conftest.py new file mode 100644 index 00000000..015e0a93 --- /dev/null +++ b/tests/ahriman/core/archive/conftest.py @@ -0,0 +1,40 @@ +import pytest + +from pytest_mock import MockerFixture + +from ahriman.core.archive import ArchiveTrigger +from ahriman.core.archive.archive_tree import ArchiveTree +from ahriman.core.configuration import Configuration +from ahriman.core.sign.gpg import GPG + + +@pytest.fixture +def archive_tree(configuration: Configuration) -> ArchiveTree: + """ + archive tree fixture + + Args: + configuration(Configuration): configuration fixture + + Returns: + ArchiveTree: archive tree test instance + """ + return ArchiveTree(configuration.repository_paths, []) + + +@pytest.fixture +def archive_trigger(configuration: Configuration, gpg: GPG, mocker: MockerFixture) -> ArchiveTrigger: + """ + archive trigger fixture + + Args: + configuration(Configuration): configuration fixture + gpg(GPG): GPG fixture + mocker(MockerFixture): mocker object + + Returns: + ArchiveTrigger: archive trigger test instance + """ + mocker.patch("ahriman.core._Context.get", return_value=GPG) + _, repository_id = configuration.check_loaded() + return ArchiveTrigger(repository_id, configuration) \ No newline at end of file diff --git a/tests/ahriman/core/archive/test_archive_tree.py b/tests/ahriman/core/archive/test_archive_tree.py new file mode 100644 index 00000000..776c141b --- /dev/null +++ b/tests/ahriman/core/archive/test_archive_tree.py @@ -0,0 +1,88 @@ +from dataclasses import replace +from pathlib import Path +from pytest_mock import MockerFixture + +from ahriman.core.archive.archive_tree import ArchiveTree +from ahriman.core.utils import utcnow +from ahriman.models.package import Package + + +def test_repository_for(archive_tree: ArchiveTree) -> None: + """ + must correctly generate path to repository + """ + path = archive_tree.repository_for() + assert path.is_relative_to(archive_tree.paths.archive / "repos") + assert (archive_tree.repository_id.name, archive_tree.repository_id.architecture) == path.parts[-2:] + assert set(map("{:02d}".format, utcnow().timetuple()[:3])).issubset(path.parts) + + +def test_symlinks_create(archive_tree: ArchiveTree, package_ahriman: Package, package_python_schedule: Package, + mocker: MockerFixture) -> None: + """ + must create symlinks + """ + _original_exists = Path.exists + def exists_mock(path: Path) -> bool: + if path.name in (package.filename for package in package_python_schedule.packages.values()): + return True + return _original_exists(path) + + symlinks_mock = mocker.patch("pathlib.Path.symlink_to") + add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add") + mocker.patch("pathlib.Path.glob", autospec=True, side_effect=lambda path, name: [path / name[:-1]]) + mocker.patch("pathlib.Path.exists", autospec=True, side_effect=exists_mock) + + archive_tree.symlinks_create([package_ahriman, package_python_schedule]) + symlinks_mock.assert_called_once_with( + Path("..") / + ".." / + ".." / + ".." / + ".." / + ".." / + archive_tree.paths.archive_for(package_ahriman.base) + .relative_to(archive_tree.paths.root) + .relative_to("archive") / + package_ahriman.packages[package_ahriman.base].filename + ) + add_mock.assert_called_once_with( + archive_tree.repository_for() / package_ahriman.packages[package_ahriman.base].filename + ) + + +def test_symlinks_create_empty_filename(archive_tree: ArchiveTree, package_ahriman: Package, + mocker: MockerFixture) -> None: + """ + must skip symlinks creation if filename is not set + """ + package_ahriman.packages[package_ahriman.base].filename = None + symlinks_mock = mocker.patch("pathlib.Path.symlink_to") + + archive_tree.symlinks_create([package_ahriman]) + symlinks_mock.assert_not_called() + + +def test_tree_create(archive_tree: ArchiveTree, mocker: MockerFixture) -> None: + """ + must create repository root if not exists + """ + owner_guard_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.preserve_owner") + mkdir_mock = mocker.patch("pathlib.Path.mkdir") + init_mock = mocker.patch("ahriman.core.alpm.repo.Repo.init") + + archive_tree.tree_create() + owner_guard_mock.assert_called_once_with(archive_tree.paths.archive) + mkdir_mock.assert_called_once_with(0o755, parents=True) + init_mock.assert_called_once_with() + + +def test_tree_create_exists(archive_tree: ArchiveTree, mocker: MockerFixture) -> None: + """ + must skip directory creation if already exists + """ + mocker.patch("pathlib.Path.exists", return_value=True) + mkdir_mock = mocker.patch("pathlib.Path.mkdir") + + archive_tree.tree_create() + mkdir_mock.assert_not_called() diff --git a/tests/ahriman/core/archive/test_archive_trigger.py b/tests/ahriman/core/archive/test_archive_trigger.py new file mode 100644 index 00000000..5f257ddc --- /dev/null +++ b/tests/ahriman/core/archive/test_archive_trigger.py @@ -0,0 +1,32 @@ +from pytest_mock import MockerFixture + +from ahriman.core.archive import ArchiveTrigger +from ahriman.models.package import Package +from ahriman.models.result import Result + + +def test_on_result(archive_trigger: ArchiveTrigger, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must create symlinks for actual repository + """ + symlinks_mock = mocker.patch("ahriman.core.archive.archive_tree.ArchiveTree.symlinks_create") + archive_trigger.on_result(Result(), [package_ahriman]) + symlinks_mock.assert_called_once_with([package_ahriman]) + + +def test_on_start(archive_trigger: ArchiveTrigger, mocker: MockerFixture) -> None: + """ + must create repository tree on load + """ + tree_mock = mocker.patch("ahriman.core.archive.archive_tree.ArchiveTree.tree_create") + archive_trigger.on_start() + tree_mock.assert_called_once_with() + + +def test_on_stop(archive_trigger: ArchiveTrigger, mocker: MockerFixture) -> None: + """ + must create repository tree on load + """ + symlinks_mock = mocker.patch("ahriman.core.archive.archive_tree.ArchiveTree.symlinks_fix") + archive_trigger.on_stop() + symlinks_mock.assert_called_once_with() diff --git a/tests/ahriman/core/repository/test_executor.py b/tests/ahriman/core/repository/test_executor.py index bd59cb36..e7a7a9f8 100644 --- a/tests/ahriman/core/repository/test_executor.py +++ b/tests/ahriman/core/repository/test_executor.py @@ -186,9 +186,7 @@ def test_package_update(executor: Executor, package_ahriman: Package, user: User Path("..") / ".." / ".." / - executor.paths.archive_for( - package_ahriman.base).relative_to( - executor.paths.root) / + executor.paths.archive_for(package_ahriman.base).relative_to(executor.paths.root) / filepath) # must add package repo_add_mock.assert_called_once_with(executor.paths.repository / filepath)