From 099e28189f01bfc44c9006152f1c19da4a12d0da Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Wed, 16 Jul 2025 01:42:24 +0300 Subject: [PATCH] tree demo --- src/ahriman/core/alpm/repo.py | 34 ++-- .../core/database/migrations/m016_archive.py | 85 +++++++++ src/ahriman/core/repository/executor.py | 167 ++++++++++++------ src/ahriman/models/repository_paths.py | 28 +++ 4 files changed, 245 insertions(+), 69 deletions(-) create mode 100644 src/ahriman/core/database/migrations/m016_archive.py diff --git a/src/ahriman/core/alpm/repo.py b/src/ahriman/core/alpm/repo.py index e7810b33..fa657458 100644 --- a/src/ahriman/core/alpm/repo.py +++ b/src/ahriman/core/alpm/repo.py @@ -31,20 +31,21 @@ class Repo(LazyLogging): Attributes: name(str): repository name - paths(RepositoryPaths): repository paths instance + root(Path): repository root sign_args(list[str]): additional args which have to be used to sign repository archive uid(int): uid of the repository owner user """ - def __init__(self, name: str, paths: RepositoryPaths, sign_args: list[str]) -> None: + def __init__(self, name: str, paths: RepositoryPaths, sign_args: list[str], root: Path | None = None) -> None: """ Args: name(str): repository name paths(RepositoryPaths): repository paths instance sign_args(list[str]): additional args which have to be used to sign repository archive + root(Path | None, optional): repository root. If none set, the default will be used (Default value = None) """ self.name = name - self.paths = paths + self.root = root or paths.repository self.uid, _ = paths.root_owner self.sign_args = sign_args @@ -56,28 +57,36 @@ class Repo(LazyLogging): Returns: Path: path to repository database """ - return self.paths.repository / f"{self.name}.db.tar.gz" + return self.root / f"{self.name}.db.tar.gz" - def add(self, path: Path) -> None: + def add(self, path: Path, remove: bool = True) -> None: """ add new package to repository Args: path(Path): path to archive to add + remove(bool, optional): whether to remove old packages or not (Default value = True) """ + command = ["repo-add", *self.sign_args] + if remove: + command.extend(["--remove"]) + command.extend([str(self.repo_path), str(path)]) + + # add to repository check_output( - "repo-add", *self.sign_args, "-R", str(self.repo_path), str(path), + *command, exception=BuildError.from_process(path.name), - cwd=self.paths.repository, + cwd=self.root, logger=self.logger, - user=self.uid) + user=self.uid, + ) def init(self) -> None: """ create empty repository database. It just calls add with empty arguments """ check_output("repo-add", *self.sign_args, str(self.repo_path), - cwd=self.paths.repository, logger=self.logger, user=self.uid) + cwd=self.root, logger=self.logger, user=self.uid) def remove(self, package: str, filename: Path) -> None: """ @@ -88,13 +97,14 @@ class Repo(LazyLogging): filename(Path): package filename to remove """ # remove package and signature (if any) from filesystem - for full_path in self.paths.repository.glob(f"{filename}*"): + for full_path in self.root.glob(f"**/{filename}*"): full_path.unlink() # remove package from registry check_output( "repo-remove", *self.sign_args, str(self.repo_path), package, exception=BuildError.from_process(package), - cwd=self.paths.repository, + cwd=self.root, logger=self.logger, - user=self.uid) + user=self.uid, + ) diff --git a/src/ahriman/core/database/migrations/m016_archive.py b/src/ahriman/core/database/migrations/m016_archive.py new file mode 100644 index 00000000..47711ef0 --- /dev/null +++ b/src/ahriman/core/database/migrations/m016_archive.py @@ -0,0 +1,85 @@ +# +# Copyright (c) 2021-2025 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import argparse +import shutil + +from dataclasses import replace +from sqlite3 import Connection + +from ahriman.application.handlers.handler import Handler +from ahriman.core.alpm.pacman import Pacman +from ahriman.core.configuration import Configuration +from ahriman.models.package import Package +from ahriman.models.pacman_synchronization import PacmanSynchronization +from ahriman.models.repository_paths import RepositoryPaths + + +__all__ = ["migrate_data"] + + +def migrate_data(connection: Connection, configuration: Configuration) -> None: + """ + perform data migration + + Args: + connection(Connection): database connection + configuration(Configuration): configuration instance + """ + del connection + + config_path, _ = configuration.check_loaded() + args = argparse.Namespace(configuration=config_path, repository_id=None, repository=None, architecture=None) + + for repository_id in Handler.repositories_extract(args): + paths = replace(configuration.repository_paths, repository_id=repository_id) + pacman = Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled) + + # create archive directory if required + if not paths.archive.is_dir(): + with paths.preserve_owner(paths.root / "archive"): + paths.archive.mkdir(mode=0o755, parents=True) + + move_packages(paths, pacman) + + +def move_packages(repository_paths: RepositoryPaths, pacman: Pacman) -> None: + """ + move packages from repository to archive and create symbolic links + + Args: + repository_paths(RepositoryPaths): repository paths instance + pacman(Pacman): alpm wrapper instance + """ + for source in repository_paths.repository.iterdir(): + if not source.is_file(follow_symlinks=False): + continue # skip symbolic links if any + + filename = source.name + if filename.startswith(".") or ".pkg." not in filename: + # we don't use package_like method here, because it also filters out signatures + continue + package = Package.from_archive(source, pacman) + + # move package to the archive directory + target = repository_paths.archive_for(package.base) / filename + shutil.move(source, target) + + # create symlink to the archive + source.symlink_to(target.relative_to(source.parent, walk_up=True)) diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py index 41755e79..fec51446 100644 --- a/src/ahriman/core/repository/executor.py +++ b/src/ahriman/core/repository/executor.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -import shutil +import shutil # shutil.move is used here to ensure cross fs file movement from collections.abc import Iterable from pathlib import Path @@ -41,6 +41,101 @@ class Executor(PackageInfo, Cleaner): trait for common repository update processes """ + def _archive_remove(self, description: PackageDescription, package_base: str) -> None: + """ + rename package archive removing special symbols + + Args: + description(PackageDescription): package description + package_base(str): package base name + """ + if description.filename is None: + self.logger.warning("received empty package name for base %s", package_base) + return # suppress type checking, it never can be none actually + + if (safe := safe_filename(description.filename)) != description.filename: + (self.paths.packages / description.filename).rename(self.paths.packages / safe) + description.filename = safe + + def _package_build(self, package: Package, path: Path, packager: str | None, + local_version: str | None) -> str | None: + """ + build single package + + Args: + package(Package): package to build + path(Path): path to directory with package files + packager(str | None): packager identifier used for this package + local_version(str | None): local version of the package + + Returns: + str | None: current commit sha if available + """ + self.reporter.set_building(package.base) + + task = Task(package, self.configuration, self.architecture, self.paths) + patches = self.reporter.package_patches_get(package.base, None) + commit_sha = task.init(path, patches, local_version) + built = task.build(path, PACKAGER=packager) + + package.with_packages(built, self.pacman) + for src in built: + dst = self.paths.packages / src.name + shutil.move(src, dst) + + return commit_sha + + def _package_remove(self, package_name: str, path: Path) -> None: + """ + remove single package from repository + + Args: + package_name(str): package name + path(Path): path to package archive + """ + try: + self.repo.remove(package_name, path) + except Exception: + self.logger.exception("could not remove %s", package_name) + + def _package_remove_base(self, package_base: str) -> None: + """ + remove package base from repository + + Args: + package_base(str): package base name: + """ + try: + with self.in_event(package_base, EventType.PackageRemoved): + self.reporter.package_remove(package_base) + except Exception: + self.logger.exception("could not remove base %s", package_base) + + def _package_update(self, filename: str | None, package_base: str, packager_key: str | None) -> None: + """ + update built package in repository database + + Args: + filename(str | None): archive filename + package_base(str): package base name + packager_key(str | None): packager key identifier + """ + if filename is None: + self.logger.warning("received empty package name for base %s", package_base) + return # suppress type checking, it never can be none actually + + # in theory, it might be NOT packages directory, but we suppose it is + full_path = self.paths.packages / filename + files = self.sign.process_sign_package(full_path, packager_key) + + for src in files: + archive = self.paths.archive_for(package_base) / src.name + shutil.move(src, archive) # move package to archive directory + if not (symlink := self.paths.repository / archive.name).exists(): + symlink.symlink_to(archive.relative_to(symlink.parent, walk_up=True)) # create link to archive + + self.repo.add(self.paths.repository / filename) + def process_build(self, updates: Iterable[Package], packagers: Packagers | None = None, *, bump_pkgrel: bool = False) -> Result: """ @@ -55,21 +150,6 @@ class Executor(PackageInfo, Cleaner): Returns: Result: build result """ - def build_single(package: Package, local_path: Path, packager_id: str | None) -> str | None: - self.reporter.set_building(package.base) - task = Task(package, self.configuration, self.architecture, self.paths) - local_version = local_versions.get(package.base) if bump_pkgrel else None - patches = self.reporter.package_patches_get(package.base, None) - commit_sha = task.init(local_path, patches, local_version) - built = task.build(local_path, PACKAGER=packager_id) - - package.with_packages(built, self.pacman) - for src in built: - dst = self.paths.packages / src.name - shutil.move(src, dst) - - return commit_sha - packagers = packagers or Packagers() local_versions = {package.base: package.version for package in self.packages()} @@ -80,16 +160,21 @@ class Executor(PackageInfo, Cleaner): try: with self.in_event(single.base, EventType.PackageUpdated, failure=EventType.PackageUpdateFailed): packager = self.packager(packagers, single.base) - last_commit_sha = build_single(single, Path(dir_name), packager.packager_id) + local_version = local_versions.get(single.base) if bump_pkgrel else None + commit_sha = self._package_build(single, Path(dir_name), packager.packager_id, local_version) + # update commit hash for changes keeping current diff if there is any changes = self.reporter.package_changes_get(single.base) - self.reporter.package_changes_update(single.base, Changes(last_commit_sha, changes.changes)) + self.reporter.package_changes_update(single.base, Changes(commit_sha, changes.changes)) + # update dependencies list package_archive = PackageArchive(self.paths.build_root, single, self.pacman, self.scan_paths) dependencies = package_archive.depends_on() self.reporter.package_dependencies_update(single.base, dependencies) + # update result set result.add_updated(single) + except Exception: self.reporter.set_failed(single.base) result.add_failed(single) @@ -107,19 +192,6 @@ class Executor(PackageInfo, Cleaner): Returns: Result: remove result """ - def remove_base(package_base: str) -> None: - try: - with self.in_event(package_base, EventType.PackageRemoved): - self.reporter.package_remove(package_base) - except Exception: - self.logger.exception("could not remove base %s", package_base) - - def remove_package(package: str, archive_path: Path) -> None: - try: - self.repo.remove(package, archive_path) # remove the package itself - except Exception: - self.logger.exception("could not remove %s", package) - packages_to_remove: dict[str, Path] = {} bases_to_remove: list[str] = [] @@ -136,6 +208,7 @@ class Executor(PackageInfo, Cleaner): }) bases_to_remove.append(local.base) result.add_removed(local) + elif requested.intersection(local.packages.keys()): packages_to_remove.update({ package: properties.filepath @@ -152,11 +225,11 @@ class Executor(PackageInfo, Cleaner): # remove packages from repository files for package, filename in packages_to_remove.items(): - remove_package(package, filename) + self._package_remove(package, filename) # remove bases from registered for package in bases_to_remove: - remove_base(package) + self._package_remove_base(package) return result @@ -172,27 +245,6 @@ class Executor(PackageInfo, Cleaner): Returns: Result: path to repository database """ - def rename(archive: PackageDescription, package_base: str) -> None: - if archive.filename is None: - self.logger.warning("received empty package name for base %s", package_base) - return # suppress type checking, it never can be none actually - if (safe := safe_filename(archive.filename)) != archive.filename: - shutil.move(self.paths.packages / archive.filename, self.paths.packages / safe) - archive.filename = safe - - def update_single(name: str | None, package_base: str, packager_key: str | None) -> None: - if name is None: - self.logger.warning("received empty package name for base %s", package_base) - return # suppress type checking, it never can be none actually - # in theory, it might be NOT packages directory, but we suppose it is - full_path = self.paths.packages / name - files = self.sign.process_sign_package(full_path, packager_key) - for src in files: - dst = self.paths.repository / safe_filename(src.name) - shutil.move(src, dst) - package_path = self.paths.repository / safe_filename(name) - self.repo.add(package_path) - current_packages = {package.base: package for package in self.packages()} local_versions = {package_base: package.version for package_base, package in current_packages.items()} @@ -207,8 +259,8 @@ class Executor(PackageInfo, Cleaner): packager = self.packager(packagers, local.base) for description in local.packages.values(): - rename(description, local.base) - update_single(description.filename, local.base, packager.key) + self._archive_remove(description, local.base) + self._package_update(description.filename, local.base, packager.key) self.reporter.set_success(local) result.add_updated(local) @@ -216,12 +268,13 @@ class Executor(PackageInfo, Cleaner): if local.base in current_packages: current_package_archives = set(current_packages[local.base].packages.keys()) removed_packages.extend(current_package_archives.difference(local.packages)) + except Exception: self.reporter.set_failed(local.base) result.add_failed(local) self.logger.exception("could not process %s", local.base) - self.clear_packages() + self.clear_packages() self.process_remove(removed_packages) return result diff --git a/src/ahriman/models/repository_paths.py b/src/ahriman/models/repository_paths.py index 7e8ae558..2084cb9d 100644 --- a/src/ahriman/models/repository_paths.py +++ b/src/ahriman/models/repository_paths.py @@ -85,6 +85,16 @@ class RepositoryPaths(LazyLogging): return Path(self.repository_id.architecture) # legacy tree suffix return Path(self.repository_id.name) / self.repository_id.architecture + @property + def archive(self) -> Path: + """ + archive directory root + + Returns: + Path: archive directory root + """ + return self.root / "archive" / self._suffix + @property def build_root(self) -> Path: """ @@ -249,6 +259,23 @@ class RepositoryPaths(LazyLogging): set_owner(path) path = path.parent + def archive_for(self, package_base: str) -> Path: + """ + get path to archive specified search criteria + + Args: + package_base(str): package base name + + Returns: + Path: path to archive directory for package base + """ + directory = self.archive / "packages" / package_base[0] / package_base + if not directory.is_dir(): # create if not exists + with self.preserve_owner(self.archive): + directory.mkdir(mode=0o755, parents=True) + + return directory + def cache_for(self, package_base: str) -> Path: """ get path to cached PKGBUILD and package sources for the package base @@ -320,6 +347,7 @@ class RepositoryPaths(LazyLogging): with self.preserve_owner(): for directory in ( + self.archive, self.cache, self.chroot, self.packages,