add archive trigger

package/share/ahriman/settings/ahriman.ini.d/00-triggers.ini

@@ -1,5 +1,6 @@
 [build]
 ; List of well-known triggers. Used only for configuration purposes.
+triggers_known[] = ahriman.core.archive.ArchiveTrigger
 triggers_known[] = ahriman.core.distributed.WorkerLoaderTrigger
 triggers_known[] = ahriman.core.distributed.WorkerTrigger
 triggers_known[] = ahriman.core.support.KeyringTrigger

src/ahriman/application/handlers/status.py

@@ -66,7 +66,7 @@ class Status(Handler):
         Status.check_status(args.exit_code, packages)
 
         comparator: Callable[[tuple[Package, BuildStatus]], Comparable] = lambda item: item[0].base
-        filter_fn: Callable[[tuple[Package, BuildStatus]], bool] =\
+        filter_fn: Callable[[tuple[Package, BuildStatus]], bool] = \
             lambda item: args.status is None or item[1].status == args.status
         for package, package_status in sorted(filter(filter_fn, packages), key=comparator):
             PackagePrinter(package, package_status)(verbose=args.info)

src/ahriman/core/alpm/repo.py

@@ -88,22 +88,24 @@ class Repo(LazyLogging):
         check_output("repo-add", *self.sign_args, str(self.repo_path),
                      cwd=self.root, logger=self.logger, user=self.uid)
 
-    def remove(self, package: str, filename: Path) -> None:
+    def remove(self, package_name: str | None, filename: Path) -> None:
         """
         remove package from repository
 
         Args:
-            package(str): package name to remove
+            package_name(str | None): package name to remove. If none set, it will be guessed from filename
             filename(Path): package filename to remove
         """
+        package_name = package_name or filename.name.rsplit("-", maxsplit=3)[0]
+
         # remove package and signature (if any) from filesystem
         for full_path in self.root.glob(f"**/{filename.name}*"):
             full_path.unlink()
 
         # remove package from registry
         check_output(
-            "repo-remove", *self.sign_args, str(self.repo_path), package,
-            exception=BuildError.from_process(package),
+            "repo-remove", *self.sign_args, str(self.repo_path), package_name,
+            exception=BuildError.from_process(package_name),
             cwd=self.root,
             logger=self.logger,
             user=self.uid,

src/ahriman/core/archive/__init__.py

@@ -17,3 +17,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+from ahriman.core.archive.archive_trigger import ArchiveTrigger

src/ahriman/core/archive/archive_tree.py

@@ -0,0 +1,130 @@
+#
+# Copyright (c) 2021-2025 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import datetime
+
+from pathlib import Path
+
+from ahriman.core.alpm.repo import Repo
+from ahriman.core.log import LazyLogging
+from ahriman.core.utils import utcnow, walk
+from ahriman.models.package import Package
+from ahriman.models.repository_paths import RepositoryPaths
+
+
+class ArchiveTree(LazyLogging):
+    """
+    wrapper around archive tree
+
+    Attributes:
+        paths(RepositoryPaths): repository paths instance
+        repository_id(RepositoryId): repository unique identifier
+        sign_args(list[str]): additional args which have to be used to sign repository archive
+    """
+
+    def __init__(self, repository_path: RepositoryPaths, sign_args: list[str]) -> None:
+        """
+        Args:
+            repository_path(RepositoryPaths): repository paths instance
+            sign_args(list[str]): additional args which have to be used to sign repository archive
+        """
+        self.paths = repository_path
+        self.repository_id = repository_path.repository_id
+        self.sign_args = sign_args
+
+    def repository_for(self, date: datetime.date | None = None) -> Path:
+        """
+        get full path to repository at the specified date
+
+        Args:
+            date(datetime.date | None, optional): date to generate path. If none supplied then today will be used
+                (Default value = None)
+
+        Returns:
+            Path: path to the repository root
+        """
+        date = date or utcnow().date()
+        return (
+            self.paths.archive
+            / "repos"
+            / date.strftime("%Y")
+            / date.strftime("%m")
+            / date.strftime("%d")
+            / self.repository_id.name
+            / self.repository_id.architecture
+        )
+
+    def symlinks_create(self, packages: list[Package]) -> None:
+        """
+        create symlinks for the specified packages in today's repository
+
+        Args:
+            packages(list[Package]): list of packages to be updated
+        """
+        root = self.repository_for()
+        repo = Repo(self.repository_id.name, self.paths, self.sign_args, root)
+
+        for package in packages:
+            archive = self.paths.archive_for(package.base)
+
+            for package_name, single in package.packages.items():
+                if single.filename is None:
+                    self.logger.warning("received empty package filename for %s", package_name)
+                    continue
+
+                has_file = False
+                for file in archive.glob(f"{single.filename}*"):
+                    symlink = root / file.name
+                    if symlink.exists():
+                        continue  # symlink is already created, skip processing
+                    has_file = True
+                    symlink.symlink_to(file.relative_to(symlink.parent, walk_up=True))
+
+                if has_file:
+                    repo.add(root / single.filename)
+
+    def symlinks_fix(self) -> None:
+        """
+        remove broken symlinks across repositories for all dates
+        """
+        for path in walk(self.paths.archive / "repos"):
+            root = path.parent
+            *_, name, architecture = root.parts
+            if self.repository_id.name != name or self.repository_id.architecture != architecture:
+                continue  # we only process same name repositories
+
+            if not path.is_symlink():
+                continue  # find symlinks only
+            if path.exists():
+                continue  # filter out not broken symlinks
+
+            Repo(self.repository_id.name, self.paths, self.sign_args, root).remove(None, path)
+
+    def tree_create(self) -> None:
+        """
+        create repository tree for current repository
+        """
+        root = self.repository_for()
+        if root.exists():
+            return
+
+        with self.paths.preserve_owner(self.paths.archive):
+            root.mkdir(0o755, parents=True)
+            # init empty repository here
+            Repo(self.repository_id.name, self.paths, self.sign_args, root).init()

src/ahriman/core/archive/archive_trigger.py

@@ -17,21 +17,23 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from pathlib import Path
-
+from ahriman.core import context
+from ahriman.core.archive.archive_tree import ArchiveTree
 from ahriman.core.configuration import Configuration
+from ahriman.core.sign.gpg import GPG
 from ahriman.core.triggers import Trigger
 from ahriman.models.package import Package
 from ahriman.models.repository_id import RepositoryId
 from ahriman.models.result import Result
 
 
-class ArchiveRotationTrigger(Trigger):
+class ArchiveTrigger(Trigger):
     """
     archive repository extension
 
     Attributes:
         paths(RepositoryPaths): repository paths instance
+        tree(ArchiveTree): archive tree wrapper
     """
 
     def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
@@ -44,9 +46,8 @@ class ArchiveRotationTrigger(Trigger):
 
         self.paths = configuration.repository_paths
 
-    @property
-    def repos_path(self) -> Path:
-        return self.paths.archive / "repos"
+        ctx = context.get()
+        self.tree = ArchiveTree(self.paths, ctx.get(GPG).repository_sign_args)
 
     def on_result(self, result: Result, packages: list[Package]) -> None:
         """
@@ -56,10 +57,16 @@ class ArchiveRotationTrigger(Trigger):
             result(Result): build result
             packages(list[Package]): list of all available packages
         """
+        self.tree.symlinks_create(packages)
 
     def on_start(self) -> None:
         """
         trigger action which will be called at the start of the application
         """
-        with self.paths.preserve_owner(self.repos_path):
-            self.repos_path.mkdir(mode=0o755, exist_ok=True)
+        self.tree.tree_create()
+
+    def on_stop(self) -> None:
+        """
+        trigger action which will be called before the stop of the application
+        """
+        self.tree.symlinks_fix()

src/ahriman/core/repository/executor.py

@@ -80,7 +80,7 @@ class Executor(PackageInfo, Cleaner):
             package_base(str): package base name
         """
         if description.filename is None:
-            self.logger.warning("received empty package name for base %s", package_base)
+            self.logger.warning("received empty package filename for base %s", package_base)
             return  # suppress type checking, it never can be none actually
 
         if (safe := safe_filename(description.filename)) != description.filename:
@@ -161,7 +161,7 @@ class Executor(PackageInfo, Cleaner):
             packager_key(str | None): packager key identifier
         """
         if filename is None:
-            self.logger.warning("received empty package name for base %s", package_base)
+            self.logger.warning("received empty package filename for base %s", package_base)
             return  # suppress type checking, it never can be none actually
 
         # in theory, it might be NOT packages directory, but we suppose it is

subpackages.py

@@ -37,6 +37,7 @@ SUBPACKAGES = {
     "ahriman-triggers": [
         prefix / "share" / "ahriman" / "settings" / "ahriman.ini.d" / "00-triggers.ini",
         site_packages / "ahriman" / "application" / "handlers" / "triggers_support.py",
+        site_packages / "ahriman" / "core" / "archive",
         site_packages / "ahriman" / "core" / "distributed",
         site_packages / "ahriman" / "core" / "support",
     ],

tests/ahriman/core/alpm/test_repo.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from pytest_mock import MockerFixture
 
 from ahriman.core.alpm.repo import Repo
+from ahriman.models.package import Package
 from ahriman.models.repository_paths import RepositoryPaths
 
 
@@ -56,21 +57,37 @@ def test_repo_init(repo: Repo, mocker: MockerFixture) -> None:
     assert check_output_mock.call_args[0][0] == "repo-add"
 
 
-def test_repo_remove(repo: Repo, mocker: MockerFixture) -> None:
+def test_repo_remove(repo: Repo, package_ahriman: Package, mocker: MockerFixture) -> None:
     """
-    must call repo-remove on package addition
+    must call repo-remove on package removal
     """
+    filepath = package_ahriman.packages[package_ahriman.base].filepath
     mocker.patch("pathlib.Path.glob", return_value=[])
     check_output_mock = mocker.patch("ahriman.core.alpm.repo.check_output")
 
-    repo.remove("package", Path("package.pkg.tar.xz"))
+    repo.remove(package_ahriman.base, filepath)
     check_output_mock.assert_called_once()  # it will be checked later
     assert check_output_mock.call_args[0][0] == "repo-remove"
+    assert package_ahriman.base in check_output_mock.call_args[0]
+
+
+def test_repo_remove_guess_package(repo: Repo, package_ahriman: Package, mocker: MockerFixture) -> None:
+    """
+    must call repo-remove on package removal if no package name set
+    """
+    filepath = package_ahriman.packages[package_ahriman.base].filepath
+    mocker.patch("pathlib.Path.glob", return_value=[])
+    check_output_mock = mocker.patch("ahriman.core.alpm.repo.check_output")
+
+    repo.remove(None, filepath)
+    check_output_mock.assert_called_once()  # it will be checked later
+    assert check_output_mock.call_args[0][0] == "repo-remove"
+    assert package_ahriman.base in check_output_mock.call_args[0]
 
 
 def test_repo_remove_fail_no_file(repo: Repo, mocker: MockerFixture) -> None:
     """
-    must fail on missing file
+    must fail removal on missing file
     """
     mocker.patch("pathlib.Path.glob", return_value=[Path("package.pkg.tar.xz")])
     mocker.patch("pathlib.Path.unlink", side_effect=FileNotFoundError)

tests/ahriman/core/archive/conftest.py

@@ -0,0 +1,40 @@
+import pytest
+
+from pytest_mock import MockerFixture
+
+from ahriman.core.archive import ArchiveTrigger
+from ahriman.core.archive.archive_tree import ArchiveTree
+from ahriman.core.configuration import Configuration
+from ahriman.core.sign.gpg import GPG
+
+
+@pytest.fixture
+def archive_tree(configuration: Configuration) -> ArchiveTree:
+    """
+    archive tree fixture
+
+    Args:
+        configuration(Configuration): configuration fixture
+
+    Returns:
+        ArchiveTree: archive tree test instance
+    """
+    return ArchiveTree(configuration.repository_paths, [])
+
+
+@pytest.fixture
+def archive_trigger(configuration: Configuration, gpg: GPG, mocker: MockerFixture) -> ArchiveTrigger:
+    """
+    archive trigger fixture
+
+    Args:
+        configuration(Configuration): configuration fixture
+        gpg(GPG): GPG fixture
+        mocker(MockerFixture): mocker object
+
+    Returns:
+        ArchiveTrigger: archive trigger test instance
+    """
+    mocker.patch("ahriman.core._Context.get", return_value=GPG)
+    _, repository_id = configuration.check_loaded()
+    return ArchiveTrigger(repository_id, configuration)

tests/ahriman/core/archive/test_archive_tree.py

@@ -0,0 +1,135 @@
+from pathlib import Path
+from pytest_mock import MockerFixture
+
+from ahriman.core.archive.archive_tree import ArchiveTree
+from ahriman.core.utils import utcnow
+from ahriman.models.package import Package
+
+
+def test_repository_for(archive_tree: ArchiveTree) -> None:
+    """
+    must correctly generate path to repository
+    """
+    path = archive_tree.repository_for()
+    assert path.is_relative_to(archive_tree.paths.archive / "repos")
+    assert (archive_tree.repository_id.name, archive_tree.repository_id.architecture) == path.parts[-2:]
+    assert set(map("{:02d}".format, utcnow().timetuple()[:3])).issubset(path.parts)
+
+
+def test_symlinks_create(archive_tree: ArchiveTree, package_ahriman: Package, package_python_schedule: Package,
+                         mocker: MockerFixture) -> None:
+    """
+    must create symlinks
+    """
+    _original_exists = Path.exists
+
+    def exists_mock(path: Path) -> bool:
+        if path.name in (package.filename for package in package_python_schedule.packages.values()):
+            return True
+        return _original_exists(path)
+
+    symlinks_mock = mocker.patch("pathlib.Path.symlink_to")
+    add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add")
+    mocker.patch("pathlib.Path.glob", autospec=True, side_effect=lambda path, name: [path / name[:-1]])
+    mocker.patch("pathlib.Path.exists", autospec=True, side_effect=exists_mock)
+
+    archive_tree.symlinks_create([package_ahriman, package_python_schedule])
+    symlinks_mock.assert_called_once_with(
+        Path("..") /
+        ".." /
+        ".." /
+        ".." /
+        ".." /
+        ".." /
+        archive_tree.paths.archive_for(package_ahriman.base)
+        .relative_to(archive_tree.paths.root)
+        .relative_to("archive") /
+        package_ahriman.packages[package_ahriman.base].filename
+    )
+    add_mock.assert_called_once_with(
+        archive_tree.repository_for() / package_ahriman.packages[package_ahriman.base].filename
+    )
+
+
+def test_symlinks_create_empty_filename(archive_tree: ArchiveTree, package_ahriman: Package,
+                                        mocker: MockerFixture) -> None:
+    """
+    must skip symlinks creation if filename is not set
+    """
+    package_ahriman.packages[package_ahriman.base].filename = None
+    symlinks_mock = mocker.patch("pathlib.Path.symlink_to")
+
+    archive_tree.symlinks_create([package_ahriman])
+    symlinks_mock.assert_not_called()
+
+
+def test_symlinks_fix(archive_tree: ArchiveTree, mocker: MockerFixture) -> None:
+    """
+    must fix broken symlinks
+    """
+    _original_exists = Path.exists
+
+    def exists_mock(path: Path) -> bool:
+        if path.name == "symlink":
+            return True
+        return _original_exists(path)
+
+    mocker.patch("pathlib.Path.is_symlink", side_effect=[True, True, False])
+    mocker.patch("pathlib.Path.exists", autospec=True, side_effect=exists_mock)
+    walk_mock = mocker.patch("ahriman.core.archive.archive_tree.walk", return_value=[
+        archive_tree.repository_for() / filename
+        for filename in ("symlink", "broken_symlink", "file")
+    ])
+    remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove")
+
+    archive_tree.symlinks_fix()
+    walk_mock.assert_called_once_with(archive_tree.paths.archive / "repos")
+    remove_mock.assert_called_once_with(None, archive_tree.repository_for() / "broken_symlink")
+
+
+def test_symlinks_fix_foreign_repository(archive_tree: ArchiveTree, mocker: MockerFixture) -> None:
+    """
+    must skip symlinks check if repository name or architecture doesn't match
+    """
+    _original_exists = Path.exists
+
+    def exists_mock(path: Path) -> bool:
+        if path.name == "symlink":
+            return True
+        return _original_exists(path)
+
+    mocker.patch("pathlib.Path.is_symlink", side_effect=[True, True, False])
+    mocker.patch("pathlib.Path.exists", autospec=True, side_effect=exists_mock)
+    mocker.patch("ahriman.core.archive.archive_tree.walk", return_value=[
+        archive_tree.repository_for().with_name("i686") / filename
+        for filename in ("symlink", "broken_symlink", "file")
+    ])
+    remove_mock = mocker.patch("ahriman.core.alpm.repo.Repo.remove")
+
+    archive_tree.symlinks_fix()
+    remove_mock.assert_not_called()
+
+
+def test_tree_create(archive_tree: ArchiveTree, mocker: MockerFixture) -> None:
+    """
+    must create repository root if not exists
+    """
+    owner_guard_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.preserve_owner")
+    mkdir_mock = mocker.patch("pathlib.Path.mkdir")
+    init_mock = mocker.patch("ahriman.core.alpm.repo.Repo.init")
+
+    archive_tree.tree_create()
+    owner_guard_mock.assert_called_once_with(archive_tree.paths.archive)
+    mkdir_mock.assert_called_once_with(0o755, parents=True)
+    init_mock.assert_called_once_with()
+
+
+def test_tree_create_exists(archive_tree: ArchiveTree, mocker: MockerFixture) -> None:
+    """
+    must skip directory creation if already exists
+    """
+    mocker.patch("pathlib.Path.exists", return_value=True)
+    mkdir_mock = mocker.patch("pathlib.Path.mkdir")
+
+    archive_tree.tree_create()
+    mkdir_mock.assert_not_called()

tests/ahriman/core/archive/test_archive_trigger.py

@@ -0,0 +1,32 @@
+from pytest_mock import MockerFixture
+
+from ahriman.core.archive import ArchiveTrigger
+from ahriman.models.package import Package
+from ahriman.models.result import Result
+
+
+def test_on_result(archive_trigger: ArchiveTrigger, package_ahriman: Package, mocker: MockerFixture) -> None:
+    """
+    must create symlinks for actual repository
+    """
+    symlinks_mock = mocker.patch("ahriman.core.archive.archive_tree.ArchiveTree.symlinks_create")
+    archive_trigger.on_result(Result(), [package_ahriman])
+    symlinks_mock.assert_called_once_with([package_ahriman])
+
+
+def test_on_start(archive_trigger: ArchiveTrigger, mocker: MockerFixture) -> None:
+    """
+    must create repository tree on load
+    """
+    tree_mock = mocker.patch("ahriman.core.archive.archive_tree.ArchiveTree.tree_create")
+    archive_trigger.on_start()
+    tree_mock.assert_called_once_with()
+
+
+def test_on_stop(archive_trigger: ArchiveTrigger, mocker: MockerFixture) -> None:
+    """
+    must create repository tree on load
+    """
+    symlinks_mock = mocker.patch("ahriman.core.archive.archive_tree.ArchiveTree.symlinks_fix")
+    archive_trigger.on_stop()
+    symlinks_mock.assert_called_once_with()

tests/ahriman/core/repository/test_executor.py

@@ -186,9 +186,7 @@ def test_package_update(executor: Executor, package_ahriman: Package, user: User
         Path("..") /
         ".." /
         ".." /
-        executor.paths.archive_for(
-            package_ahriman.base).relative_to(
-            executor.paths.root) /
+        executor.paths.archive_for(package_ahriman.base).relative_to(executor.paths.root) /
         filepath)
     # must add package
     repo_add_mock.assert_called_once_with(executor.paths.repository / filepath)