tree demo

docs/configuration.rst

@@ -65,6 +65,8 @@ will try to read value from ``SECRET`` environment variable. In case if the requ
 
 will eventually lead ``key`` option in section ``section1`` to be set to the value of ``HOME`` environment variable (if available).
 
+Moreover, configuration can be read from environment variables directly by following the same naming convention, e.g. in the example above, one can have environment variable named ``section1:key`` (e.g. ``section1:key=$HOME``) and it will be substituted to the configuration with the highest priority.
+
 There is also additional subcommand which will allow to validate configuration and print found errors. In order to do so, run ``service-config-validate`` subcommand, e.g.:
 
 .. code-block:: shell

src/ahriman/core/alpm/repo.py

@@ -31,20 +31,21 @@ class Repo(LazyLogging):
 
     Attributes:
         name(str): repository name
-        paths(RepositoryPaths): repository paths instance
+        root(Path): repository root
         sign_args(list[str]): additional args which have to be used to sign repository archive
         uid(int): uid of the repository owner user
     """
 
-    def __init__(self, name: str, paths: RepositoryPaths, sign_args: list[str]) -> None:
+    def __init__(self, name: str, paths: RepositoryPaths, sign_args: list[str], root: Path | None = None) -> None:
         """
         Args:
             name(str): repository name
             paths(RepositoryPaths): repository paths instance
             sign_args(list[str]): additional args which have to be used to sign repository archive
+            root(Path | None, optional): repository root. If none set, the default will be used (Default value = None)
         """
         self.name = name
-        self.paths = paths
+        self.root = root or paths.repository
         self.uid, _ = paths.root_owner
         self.sign_args = sign_args
 
@@ -56,28 +57,36 @@ class Repo(LazyLogging):
         Returns:
             Path: path to repository database
         """
-        return self.paths.repository / f"{self.name}.db.tar.gz"
+        return self.root / f"{self.name}.db.tar.gz"
 
-    def add(self, path: Path) -> None:
+    def add(self, path: Path, remove: bool = True) -> None:
         """
         add new package to repository
 
         Args:
             path(Path): path to archive to add
+            remove(bool, optional): whether to remove old packages or not (Default value = True)
         """
+        command = ["repo-add", *self.sign_args]
+        if remove:
+            command.extend(["--remove"])
+        command.extend([str(self.repo_path), str(path)])
+
+        # add to repository
         check_output(
-            "repo-add", *self.sign_args, "-R", str(self.repo_path), str(path),
+            *command,
             exception=BuildError.from_process(path.name),
-            cwd=self.paths.repository,
+            cwd=self.root,
             logger=self.logger,
-            user=self.uid)
+            user=self.uid,
+        )
 
     def init(self) -> None:
         """
         create empty repository database. It just calls add with empty arguments
         """
         check_output("repo-add", *self.sign_args, str(self.repo_path),
-                     cwd=self.paths.repository, logger=self.logger, user=self.uid)
+                     cwd=self.root, logger=self.logger, user=self.uid)
 
     def remove(self, package: str, filename: Path) -> None:
         """
@@ -88,13 +97,14 @@ class Repo(LazyLogging):
             filename(Path): package filename to remove
         """
         # remove package and signature (if any) from filesystem
-        for full_path in self.paths.repository.glob(f"{filename}*"):
+        for full_path in self.root.glob(f"**/{filename}*"):
             full_path.unlink()
 
         # remove package from registry
         check_output(
             "repo-remove", *self.sign_args, str(self.repo_path), package,
             exception=BuildError.from_process(package),
-            cwd=self.paths.repository,
+            cwd=self.root,
             logger=self.logger,
-            user=self.uid)
+            user=self.uid,
+        )

src/ahriman/core/configuration/configuration.py

@@ -19,6 +19,7 @@
 #
 # pylint: disable=too-many-public-methods
 import configparser
+import os
 import shlex
 import sys
 
@@ -164,6 +165,7 @@ class Configuration(configparser.RawConfigParser):
         """
         configuration = cls()
         configuration.load(path)
+        configuration.load_environment()
         configuration.merge_sections(repository_id)
         return configuration
 
@@ -288,6 +290,16 @@ class Configuration(configparser.RawConfigParser):
         self.read(self.path)
         self.load_includes()  # load includes
 
+    def load_environment(self) -> None:
+        """
+        load environment variables into configuration
+        """
+        for name, value in os.environ.items():
+            if ":" not in name:
+                continue
+            section, key = name.rsplit(":", maxsplit=1)
+            self.set_option(section, key, value)
+
     def load_includes(self, path: Path | None = None) -> None:
         """
         load configuration includes from specified path
@@ -356,11 +368,16 @@ class Configuration(configparser.RawConfigParser):
         """
         reload configuration if possible or raise exception otherwise
         """
+        # get current properties and validate input
         path, repository_id = self.check_loaded()
-        for section in self.sections():  # clear current content
+
+        # clear current content
+        for section in self.sections():
             self.remove_section(section)
-        self.load(path)
-        self.merge_sections(repository_id)
+
+        # create another instance and copy values from there
+        instance = self.from_path(path, repository_id)
+        self.copy_from(instance)
 
     def set_option(self, section: str, option: str, value: str) -> None:
         """

src/ahriman/core/repository/executor.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-import shutil
+import shutil  # shutil.move is used here to ensure cross fs file movement
 
 from collections.abc import Iterable
 from pathlib import Path
@@ -41,6 +41,101 @@ class Executor(PackageInfo, Cleaner):
     trait for common repository update processes
     """
 
+    def _archive_remove(self, description: PackageDescription, package_base: str) -> None:
+        """
+        rename package archive removing special symbols
+
+        Args:
+            description(PackageDescription): package description
+            package_base(str): package base name
+        """
+        if description.filename is None:
+            self.logger.warning("received empty package name for base %s", package_base)
+            return  # suppress type checking, it never can be none actually
+
+        if (safe := safe_filename(description.filename)) != description.filename:
+            (self.paths.packages / description.filename).rename(self.paths.packages / safe)
+            description.filename = safe
+
+    def _package_build(self, package: Package, path: Path, packager: str | None,
+                       local_version: str | None) -> str | None:
+        """
+        build single package
+
+        Args:
+            package(Package): package to build
+            path(Path): path to directory with package files
+            packager(str | None): packager identifier used for this package
+            local_version(str | None): local version of the package
+
+        Returns:
+            str | None: current commit sha if available
+        """
+        self.reporter.set_building(package.base)
+
+        task = Task(package, self.configuration, self.architecture, self.paths)
+        patches = self.reporter.package_patches_get(package.base, None)
+        commit_sha = task.init(path, patches, local_version)
+        built = task.build(path, PACKAGER=packager)
+
+        package.with_packages(built, self.pacman)
+        for src in built:
+            dst = self.paths.packages / src.name
+            shutil.move(src, dst)
+
+        return commit_sha
+
+    def _package_remove(self, package_name: str, path: Path) -> None:
+        """
+        remove single package from repository
+
+        Args:
+            package_name(str): package name
+            path(Path): path to package archive
+        """
+        try:
+            self.repo.remove(package_name, path)
+        except Exception:
+            self.logger.exception("could not remove %s", package_name)
+
+    def _package_remove_base(self, package_base: str) -> None:
+        """
+        remove package base from repository
+
+        Args:
+            package_base(str): package base name:
+        """
+        try:
+            with self.in_event(package_base, EventType.PackageRemoved):
+                self.reporter.package_remove(package_base)
+        except Exception:
+            self.logger.exception("could not remove base %s", package_base)
+
+    def _package_update(self, filename: str | None, package_base: str, packager_key: str | None) -> None:
+        """
+        update built package in repository database
+
+        Args:
+            filename(str | None): archive filename
+            package_base(str): package base name
+            packager_key(str | None): packager key identifier
+        """
+        if filename is None:
+            self.logger.warning("received empty package name for base %s", package_base)
+            return  # suppress type checking, it never can be none actually
+
+        # in theory, it might be NOT packages directory, but we suppose it is
+        full_path = self.paths.packages / filename
+        files = self.sign.process_sign_package(full_path, packager_key)
+
+        for src in files:
+            archive = self.paths.archive_for(package_base) / src.name
+            shutil.move(src, archive)  # move package to archive directory
+            if not (symlink := self.paths.repository / archive.name).exists():
+                symlink.symlink_to(archive.relative_to(symlink.parent, walk_up=True))  # create link to archive
+
+        self.repo.add(self.paths.repository / filename)
+
     def process_build(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
                       bump_pkgrel: bool = False) -> Result:
         """
@@ -55,21 +150,6 @@ class Executor(PackageInfo, Cleaner):
         Returns:
             Result: build result
         """
-        def build_single(package: Package, local_path: Path, packager_id: str | None) -> str | None:
-            self.reporter.set_building(package.base)
-            task = Task(package, self.configuration, self.architecture, self.paths)
-            local_version = local_versions.get(package.base) if bump_pkgrel else None
-            patches = self.reporter.package_patches_get(package.base, None)
-            commit_sha = task.init(local_path, patches, local_version)
-            built = task.build(local_path, PACKAGER=packager_id)
-
-            package.with_packages(built, self.pacman)
-            for src in built:
-                dst = self.paths.packages / src.name
-                shutil.move(src, dst)
-
-            return commit_sha
-
         packagers = packagers or Packagers()
         local_versions = {package.base: package.version for package in self.packages()}
 
@@ -80,16 +160,21 @@ class Executor(PackageInfo, Cleaner):
                 try:
                     with self.in_event(single.base, EventType.PackageUpdated, failure=EventType.PackageUpdateFailed):
                         packager = self.packager(packagers, single.base)
-                        last_commit_sha = build_single(single, Path(dir_name), packager.packager_id)
+                        local_version = local_versions.get(single.base) if bump_pkgrel else None
+                        commit_sha = self._package_build(single, Path(dir_name), packager.packager_id, local_version)
+
                         # update commit hash for changes keeping current diff if there is any
                         changes = self.reporter.package_changes_get(single.base)
-                        self.reporter.package_changes_update(single.base, Changes(last_commit_sha, changes.changes))
+                        self.reporter.package_changes_update(single.base, Changes(commit_sha, changes.changes))
+
                         # update dependencies list
                         package_archive = PackageArchive(self.paths.build_root, single, self.pacman, self.scan_paths)
                         dependencies = package_archive.depends_on()
                         self.reporter.package_dependencies_update(single.base, dependencies)
+
                         # update result set
                         result.add_updated(single)
+
                 except Exception:
                     self.reporter.set_failed(single.base)
                     result.add_failed(single)
@@ -107,19 +192,6 @@ class Executor(PackageInfo, Cleaner):
         Returns:
             Result: remove result
         """
-        def remove_base(package_base: str) -> None:
-            try:
-                with self.in_event(package_base, EventType.PackageRemoved):
-                    self.reporter.package_remove(package_base)
-            except Exception:
-                self.logger.exception("could not remove base %s", package_base)
-
-        def remove_package(package: str, archive_path: Path) -> None:
-            try:
-                self.repo.remove(package, archive_path)  # remove the package itself
-            except Exception:
-                self.logger.exception("could not remove %s", package)
-
         packages_to_remove: dict[str, Path] = {}
         bases_to_remove: list[str] = []
 
@@ -136,6 +208,7 @@ class Executor(PackageInfo, Cleaner):
                 })
                 bases_to_remove.append(local.base)
                 result.add_removed(local)
+
             elif requested.intersection(local.packages.keys()):
                 packages_to_remove.update({
                     package: properties.filepath
@@ -152,11 +225,11 @@ class Executor(PackageInfo, Cleaner):
 
         # remove packages from repository files
         for package, filename in packages_to_remove.items():
-            remove_package(package, filename)
+            self._package_remove(package, filename)
 
         # remove bases from registered
         for package in bases_to_remove:
-            remove_base(package)
+            self._package_remove_base(package)
 
         return result
 
@@ -172,27 +245,6 @@ class Executor(PackageInfo, Cleaner):
         Returns:
             Result: path to repository database
         """
-        def rename(archive: PackageDescription, package_base: str) -> None:
-            if archive.filename is None:
-                self.logger.warning("received empty package name for base %s", package_base)
-                return  # suppress type checking, it never can be none actually
-            if (safe := safe_filename(archive.filename)) != archive.filename:
-                shutil.move(self.paths.packages / archive.filename, self.paths.packages / safe)
-                archive.filename = safe
-
-        def update_single(name: str | None, package_base: str, packager_key: str | None) -> None:
-            if name is None:
-                self.logger.warning("received empty package name for base %s", package_base)
-                return  # suppress type checking, it never can be none actually
-            # in theory, it might be NOT packages directory, but we suppose it is
-            full_path = self.paths.packages / name
-            files = self.sign.process_sign_package(full_path, packager_key)
-            for src in files:
-                dst = self.paths.repository / safe_filename(src.name)
-                shutil.move(src, dst)
-            package_path = self.paths.repository / safe_filename(name)
-            self.repo.add(package_path)
-
         current_packages = {package.base: package for package in self.packages()}
         local_versions = {package_base: package.version for package_base, package in current_packages.items()}
 
@@ -207,8 +259,8 @@ class Executor(PackageInfo, Cleaner):
                     packager = self.packager(packagers, local.base)
 
                     for description in local.packages.values():
-                        rename(description, local.base)
-                        update_single(description.filename, local.base, packager.key)
+                        self._archive_remove(description, local.base)
+                        self._package_update(description.filename, local.base, packager.key)
                     self.reporter.set_success(local)
                     result.add_updated(local)
 
@@ -216,12 +268,13 @@ class Executor(PackageInfo, Cleaner):
                     if local.base in current_packages:
                         current_package_archives = set(current_packages[local.base].packages.keys())
                     removed_packages.extend(current_package_archives.difference(local.packages))
+
                 except Exception:
                     self.reporter.set_failed(local.base)
                     result.add_failed(local)
                     self.logger.exception("could not process %s", local.base)
-        self.clear_packages()
 
+        self.clear_packages()
         self.process_remove(removed_packages)
 
         return result

src/ahriman/models/repository_paths.py

@@ -85,6 +85,16 @@ class RepositoryPaths(LazyLogging):
                 return Path(self.repository_id.architecture)  # legacy tree suffix
         return Path(self.repository_id.name) / self.repository_id.architecture
 
+    @property
+    def archive(self) -> Path:
+        """
+        archive directory root
+
+        Returns:
+            Path: archive directory root
+        """
+        return self.root / "archive" / self._suffix
+
     @property
     def build_root(self) -> Path:
         """
@@ -249,6 +259,23 @@ class RepositoryPaths(LazyLogging):
             set_owner(path)
             path = path.parent
 
+    def archive_for(self, package_base: str) -> Path:
+        """
+        get path to archive specified search criteria
+
+        Args:
+            package_base(str): package base name
+
+        Returns:
+            Path: path to archive directory for package base
+        """
+        directory = self.archive / "packages" / package_base[0] / package_base
+        if not directory.is_dir():  # create if not exists
+            with self.preserve_owner(self.archive):
+                directory.mkdir(mode=0o755, parents=True)
+
+        return directory
+
     def cache_for(self, package_base: str) -> Path:
         """
         get path to cached PKGBUILD and package sources for the package base
@@ -320,6 +347,7 @@ class RepositoryPaths(LazyLogging):
 
         with self.preserve_owner():
             for directory in (
+                    self.archive,
                     self.cache,
                     self.chroot,
                     self.packages,

tests/ahriman/core/configuration/test_configuration.py

@@ -1,8 +1,8 @@
 import configparser
-from io import StringIO
-
 import pytest
+import os
 
+from io import StringIO
 from pathlib import Path
 from pytest_mock import MockerFixture
 from unittest.mock import call as MockCall
@@ -42,12 +42,16 @@ def test_from_path(repository_id: RepositoryId, mocker: MockerFixture) -> None:
     mocker.patch("ahriman.core.configuration.Configuration.get", return_value="ahriman.ini.d")
     read_mock = mocker.patch("ahriman.core.configuration.Configuration.read")
     load_includes_mock = mocker.patch("ahriman.core.configuration.Configuration.load_includes")
+    merge_mock = mocker.patch("ahriman.core.configuration.Configuration.merge_sections")
+    environment_mock = mocker.patch("ahriman.core.configuration.Configuration.load_environment")
     path = Path("path")
 
     configuration = Configuration.from_path(path, repository_id)
     assert configuration.path == path
     read_mock.assert_called_once_with(path)
     load_includes_mock.assert_called_once_with()
+    merge_mock.assert_called_once_with(repository_id)
+    environment_mock.assert_called_once_with()
 
 
 def test_from_path_file_missing(repository_id: RepositoryId, mocker: MockerFixture) -> None:
@@ -324,6 +328,18 @@ def test_gettype_from_section_no_section(configuration: Configuration) -> None:
         configuration.gettype("rsync:x86_64", configuration.repository_id)
 
 
+def test_load_environment(configuration: Configuration) -> None:
+    """
+    must load environment variables
+    """
+    os.environ["section:key"] = "value1"
+    os.environ["section:identifier:key"] = "value2"
+
+    configuration.load_environment()
+    assert configuration.get("section", "key") == "value1"
+    assert configuration.get("section:identifier", "key") == "value2"
+
+
 def test_load_includes(mocker: MockerFixture) -> None:
     """
     must load includes
@@ -444,10 +460,12 @@ def test_reload(configuration: Configuration, mocker: MockerFixture) -> None:
     """
     load_mock = mocker.patch("ahriman.core.configuration.Configuration.load")
     merge_mock = mocker.patch("ahriman.core.configuration.Configuration.merge_sections")
+    environment_mock = mocker.patch("ahriman.core.configuration.Configuration.load_environment")
 
     configuration.reload()
     load_mock.assert_called_once_with(configuration.path)
     merge_mock.assert_called_once_with(configuration.repository_id)
+    environment_mock.assert_called_once_with()
 
 
 def test_reload_clear(configuration: Configuration, mocker: MockerFixture) -> None: