diff --git a/src/ahriman/core/alpm/repo.py b/src/ahriman/core/alpm/repo.py
index e7810b33..fa657458 100644
--- a/src/ahriman/core/alpm/repo.py
+++ b/src/ahriman/core/alpm/repo.py
@@ -31,20 +31,21 @@ class Repo(LazyLogging):
Attributes:
name(str): repository name
- paths(RepositoryPaths): repository paths instance
+ root(Path): repository root
sign_args(list[str]): additional args which have to be used to sign repository archive
uid(int): uid of the repository owner user
"""
- def __init__(self, name: str, paths: RepositoryPaths, sign_args: list[str]) -> None:
+ def __init__(self, name: str, paths: RepositoryPaths, sign_args: list[str], root: Path | None = None) -> None:
"""
Args:
name(str): repository name
paths(RepositoryPaths): repository paths instance
sign_args(list[str]): additional args which have to be used to sign repository archive
+ root(Path | None, optional): repository root. If none set, the default will be used (Default value = None)
"""
self.name = name
- self.paths = paths
+ self.root = root or paths.repository
self.uid, _ = paths.root_owner
self.sign_args = sign_args
@@ -56,28 +57,36 @@ class Repo(LazyLogging):
Returns:
Path: path to repository database
"""
- return self.paths.repository / f"{self.name}.db.tar.gz"
+ return self.root / f"{self.name}.db.tar.gz"
- def add(self, path: Path) -> None:
+ def add(self, path: Path, remove: bool = True) -> None:
"""
add new package to repository
Args:
path(Path): path to archive to add
+ remove(bool, optional): whether to remove old packages or not (Default value = True)
"""
+ command = ["repo-add", *self.sign_args]
+ if remove:
+ command.extend(["--remove"])
+ command.extend([str(self.repo_path), str(path)])
+
+ # add to repository
check_output(
- "repo-add", *self.sign_args, "-R", str(self.repo_path), str(path),
+ *command,
exception=BuildError.from_process(path.name),
- cwd=self.paths.repository,
+ cwd=self.root,
logger=self.logger,
- user=self.uid)
+ user=self.uid,
+ )
def init(self) -> None:
"""
create empty repository database. It just calls add with empty arguments
"""
check_output("repo-add", *self.sign_args, str(self.repo_path),
- cwd=self.paths.repository, logger=self.logger, user=self.uid)
+ cwd=self.root, logger=self.logger, user=self.uid)
def remove(self, package: str, filename: Path) -> None:
"""
@@ -88,13 +97,14 @@ class Repo(LazyLogging):
filename(Path): package filename to remove
"""
# remove package and signature (if any) from filesystem
- for full_path in self.paths.repository.glob(f"{filename}*"):
+ for full_path in self.root.glob(f"**/{filename}*"):
full_path.unlink()
# remove package from registry
check_output(
"repo-remove", *self.sign_args, str(self.repo_path), package,
exception=BuildError.from_process(package),
- cwd=self.paths.repository,
+ cwd=self.root,
logger=self.logger,
- user=self.uid)
+ user=self.uid,
+ )
diff --git a/src/ahriman/core/database/migrations/m016_archive.py b/src/ahriman/core/database/migrations/m016_archive.py
new file mode 100644
index 00000000..47711ef0
--- /dev/null
+++ b/src/ahriman/core/database/migrations/m016_archive.py
@@ -0,0 +1,85 @@
+#
+# Copyright (c) 2021-2025 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+import argparse
+import shutil
+
+from dataclasses import replace
+from sqlite3 import Connection
+
+from ahriman.application.handlers.handler import Handler
+from ahriman.core.alpm.pacman import Pacman
+from ahriman.core.configuration import Configuration
+from ahriman.models.package import Package
+from ahriman.models.pacman_synchronization import PacmanSynchronization
+from ahriman.models.repository_paths import RepositoryPaths
+
+
+__all__ = ["migrate_data"]
+
+
+def migrate_data(connection: Connection, configuration: Configuration) -> None:
+ """
+ perform data migration
+
+ Args:
+ connection(Connection): database connection
+ configuration(Configuration): configuration instance
+ """
+ del connection
+
+ config_path, _ = configuration.check_loaded()
+ args = argparse.Namespace(configuration=config_path, repository_id=None, repository=None, architecture=None)
+
+ for repository_id in Handler.repositories_extract(args):
+ paths = replace(configuration.repository_paths, repository_id=repository_id)
+ pacman = Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled)
+
+ # create archive directory if required
+ if not paths.archive.is_dir():
+ with paths.preserve_owner(paths.root / "archive"):
+ paths.archive.mkdir(mode=0o755, parents=True)
+
+ move_packages(paths, pacman)
+
+
+def move_packages(repository_paths: RepositoryPaths, pacman: Pacman) -> None:
+ """
+ move packages from repository to archive and create symbolic links
+
+ Args:
+ repository_paths(RepositoryPaths): repository paths instance
+ pacman(Pacman): alpm wrapper instance
+ """
+ for source in repository_paths.repository.iterdir():
+ if not source.is_file(follow_symlinks=False):
+ continue # skip symbolic links if any
+
+ filename = source.name
+ if filename.startswith(".") or ".pkg." not in filename:
+ # we don't use package_like method here, because it also filters out signatures
+ continue
+ package = Package.from_archive(source, pacman)
+
+ # move package to the archive directory
+ target = repository_paths.archive_for(package.base) / filename
+ shutil.move(source, target)
+
+ # create symlink to the archive
+ source.symlink_to(target.relative_to(source.parent, walk_up=True))
diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py
index 41755e79..fec51446 100644
--- a/src/ahriman/core/repository/executor.py
+++ b/src/ahriman/core/repository/executor.py
@@ -17,7 +17,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#
-import shutil
+import shutil # shutil.move is used here to ensure cross fs file movement
from collections.abc import Iterable
from pathlib import Path
@@ -41,6 +41,101 @@ class Executor(PackageInfo, Cleaner):
trait for common repository update processes
"""
+ def _archive_remove(self, description: PackageDescription, package_base: str) -> None:
+ """
+ rename package archive removing special symbols
+
+ Args:
+ description(PackageDescription): package description
+ package_base(str): package base name
+ """
+ if description.filename is None:
+ self.logger.warning("received empty package name for base %s", package_base)
+ return # suppress type checking, it never can be none actually
+
+ if (safe := safe_filename(description.filename)) != description.filename:
+ (self.paths.packages / description.filename).rename(self.paths.packages / safe)
+ description.filename = safe
+
+ def _package_build(self, package: Package, path: Path, packager: str | None,
+ local_version: str | None) -> str | None:
+ """
+ build single package
+
+ Args:
+ package(Package): package to build
+ path(Path): path to directory with package files
+ packager(str | None): packager identifier used for this package
+ local_version(str | None): local version of the package
+
+ Returns:
+ str | None: current commit sha if available
+ """
+ self.reporter.set_building(package.base)
+
+ task = Task(package, self.configuration, self.architecture, self.paths)
+ patches = self.reporter.package_patches_get(package.base, None)
+ commit_sha = task.init(path, patches, local_version)
+ built = task.build(path, PACKAGER=packager)
+
+ package.with_packages(built, self.pacman)
+ for src in built:
+ dst = self.paths.packages / src.name
+ shutil.move(src, dst)
+
+ return commit_sha
+
+ def _package_remove(self, package_name: str, path: Path) -> None:
+ """
+ remove single package from repository
+
+ Args:
+ package_name(str): package name
+ path(Path): path to package archive
+ """
+ try:
+ self.repo.remove(package_name, path)
+ except Exception:
+ self.logger.exception("could not remove %s", package_name)
+
+ def _package_remove_base(self, package_base: str) -> None:
+ """
+ remove package base from repository
+
+ Args:
+ package_base(str): package base name:
+ """
+ try:
+ with self.in_event(package_base, EventType.PackageRemoved):
+ self.reporter.package_remove(package_base)
+ except Exception:
+ self.logger.exception("could not remove base %s", package_base)
+
+ def _package_update(self, filename: str | None, package_base: str, packager_key: str | None) -> None:
+ """
+ update built package in repository database
+
+ Args:
+ filename(str | None): archive filename
+ package_base(str): package base name
+ packager_key(str | None): packager key identifier
+ """
+ if filename is None:
+ self.logger.warning("received empty package name for base %s", package_base)
+ return # suppress type checking, it never can be none actually
+
+ # in theory, it might be NOT packages directory, but we suppose it is
+ full_path = self.paths.packages / filename
+ files = self.sign.process_sign_package(full_path, packager_key)
+
+ for src in files:
+ archive = self.paths.archive_for(package_base) / src.name
+ shutil.move(src, archive) # move package to archive directory
+ if not (symlink := self.paths.repository / archive.name).exists():
+ symlink.symlink_to(archive.relative_to(symlink.parent, walk_up=True)) # create link to archive
+
+ self.repo.add(self.paths.repository / filename)
+
def process_build(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
bump_pkgrel: bool = False) -> Result:
"""
@@ -55,21 +150,6 @@ class Executor(PackageInfo, Cleaner):
Returns:
Result: build result
"""
- def build_single(package: Package, local_path: Path, packager_id: str | None) -> str | None:
- self.reporter.set_building(package.base)
- task = Task(package, self.configuration, self.architecture, self.paths)
- local_version = local_versions.get(package.base) if bump_pkgrel else None
- patches = self.reporter.package_patches_get(package.base, None)
- commit_sha = task.init(local_path, patches, local_version)
- built = task.build(local_path, PACKAGER=packager_id)
-
- package.with_packages(built, self.pacman)
- for src in built:
- dst = self.paths.packages / src.name
- shutil.move(src, dst)
-
- return commit_sha
-
packagers = packagers or Packagers()
local_versions = {package.base: package.version for package in self.packages()}
@@ -80,16 +160,21 @@ class Executor(PackageInfo, Cleaner):
try:
with self.in_event(single.base, EventType.PackageUpdated, failure=EventType.PackageUpdateFailed):
packager = self.packager(packagers, single.base)
- last_commit_sha = build_single(single, Path(dir_name), packager.packager_id)
+ local_version = local_versions.get(single.base) if bump_pkgrel else None
+ commit_sha = self._package_build(single, Path(dir_name), packager.packager_id, local_version)
+
# update commit hash for changes keeping current diff if there is any
changes = self.reporter.package_changes_get(single.base)
- self.reporter.package_changes_update(single.base, Changes(last_commit_sha, changes.changes))
+ self.reporter.package_changes_update(single.base, Changes(commit_sha, changes.changes))
+
# update dependencies list
package_archive = PackageArchive(self.paths.build_root, single, self.pacman, self.scan_paths)
dependencies = package_archive.depends_on()
self.reporter.package_dependencies_update(single.base, dependencies)
+
# update result set
result.add_updated(single)
+
except Exception:
self.reporter.set_failed(single.base)
result.add_failed(single)
@@ -107,19 +192,6 @@ class Executor(PackageInfo, Cleaner):
Returns:
Result: remove result
"""
- def remove_base(package_base: str) -> None:
- try:
- with self.in_event(package_base, EventType.PackageRemoved):
- self.reporter.package_remove(package_base)
- except Exception:
- self.logger.exception("could not remove base %s", package_base)
-
- def remove_package(package: str, archive_path: Path) -> None:
- try:
- self.repo.remove(package, archive_path) # remove the package itself
- except Exception:
- self.logger.exception("could not remove %s", package)
-
packages_to_remove: dict[str, Path] = {}
bases_to_remove: list[str] = []
@@ -136,6 +208,7 @@ class Executor(PackageInfo, Cleaner):
})
bases_to_remove.append(local.base)
result.add_removed(local)
+
elif requested.intersection(local.packages.keys()):
packages_to_remove.update({
package: properties.filepath
@@ -152,11 +225,11 @@ class Executor(PackageInfo, Cleaner):
# remove packages from repository files
for package, filename in packages_to_remove.items():
- remove_package(package, filename)
+ self._package_remove(package, filename)
# remove bases from registered
for package in bases_to_remove:
- remove_base(package)
+ self._package_remove_base(package)
return result
@@ -172,27 +245,6 @@ class Executor(PackageInfo, Cleaner):
Returns:
Result: path to repository database
"""
- def rename(archive: PackageDescription, package_base: str) -> None:
- if archive.filename is None:
- self.logger.warning("received empty package name for base %s", package_base)
- return # suppress type checking, it never can be none actually
- if (safe := safe_filename(archive.filename)) != archive.filename:
- shutil.move(self.paths.packages / archive.filename, self.paths.packages / safe)
- archive.filename = safe
-
- def update_single(name: str | None, package_base: str, packager_key: str | None) -> None:
- if name is None:
- self.logger.warning("received empty package name for base %s", package_base)
- return # suppress type checking, it never can be none actually
- # in theory, it might be NOT packages directory, but we suppose it is
- full_path = self.paths.packages / name
- files = self.sign.process_sign_package(full_path, packager_key)
- for src in files:
- dst = self.paths.repository / safe_filename(src.name)
- shutil.move(src, dst)
- package_path = self.paths.repository / safe_filename(name)
- self.repo.add(package_path)
-
current_packages = {package.base: package for package in self.packages()}
local_versions = {package_base: package.version for package_base, package in current_packages.items()}
@@ -207,8 +259,8 @@ class Executor(PackageInfo, Cleaner):
packager = self.packager(packagers, local.base)
for description in local.packages.values():
- rename(description, local.base)
- update_single(description.filename, local.base, packager.key)
+ self._archive_remove(description, local.base)
+ self._package_update(description.filename, local.base, packager.key)
self.reporter.set_success(local)
result.add_updated(local)
@@ -216,12 +268,13 @@ class Executor(PackageInfo, Cleaner):
if local.base in current_packages:
current_package_archives = set(current_packages[local.base].packages.keys())
removed_packages.extend(current_package_archives.difference(local.packages))
+
except Exception:
self.reporter.set_failed(local.base)
result.add_failed(local)
self.logger.exception("could not process %s", local.base)
- self.clear_packages()
+ self.clear_packages()
self.process_remove(removed_packages)
return result
diff --git a/src/ahriman/models/repository_paths.py b/src/ahriman/models/repository_paths.py
index 7e8ae558..2084cb9d 100644
--- a/src/ahriman/models/repository_paths.py
+++ b/src/ahriman/models/repository_paths.py
@@ -85,6 +85,16 @@ class RepositoryPaths(LazyLogging):
return Path(self.repository_id.architecture) # legacy tree suffix
return Path(self.repository_id.name) / self.repository_id.architecture
+ @property
+ def archive(self) -> Path:
+ """
+ archive directory root
+
+ Returns:
+ Path: archive directory root
+ """
+ return self.root / "archive" / self._suffix
+
@property
def build_root(self) -> Path:
"""
@@ -249,6 +259,23 @@ class RepositoryPaths(LazyLogging):
set_owner(path)
path = path.parent
+ def archive_for(self, package_base: str) -> Path:
+ """
+ get path to archive specified search criteria
+
+ Args:
+ package_base(str): package base name
+
+ Returns:
+ Path: path to archive directory for package base
+ """
+ directory = self.archive / "packages" / package_base[0] / package_base
+ if not directory.is_dir(): # create if not exists
+ with self.preserve_owner(self.archive):
+ directory.mkdir(mode=0o755, parents=True)
+
+ return directory
+
def cache_for(self, package_base: str) -> Path:
"""
get path to cached PKGBUILD and package sources for the package base
@@ -320,6 +347,7 @@ class RepositoryPaths(LazyLogging):
with self.preserve_owner():
for directory in (
+ self.archive,
self.cache,
self.chroot,
self.packages,