diff --git a/src/ahriman/core/alpm/repo.py b/src/ahriman/core/alpm/repo.py index e7810b33..982c4f01 100644 --- a/src/ahriman/core/alpm/repo.py +++ b/src/ahriman/core/alpm/repo.py @@ -48,6 +48,16 @@ class Repo(LazyLogging): self.uid, _ = paths.root_owner self.sign_args = sign_args + @property + def repo_archive_path(self) -> Path: + """ + get full path to the repository archive + + Returns: + Path: path to repository archive + """ + return self.paths.archive / f"{self.name}.db.tar.gz" + @property def repo_path(self) -> Path: """ @@ -65,19 +75,38 @@ class Repo(LazyLogging): Args: path(Path): path to archive to add """ + # add to archive first check_output( - "repo-add", *self.sign_args, "-R", str(self.repo_path), str(path), + "repo-add", *self.sign_args, str(self.repo_archive_path), str(path), + exception=BuildError.from_process(path.name), + cwd=self.paths.archive, + logger=self.logger, + user=self.uid, + ) + + # create symlinks to the main repository + for full_path in path.parent.glob(f"{path.name}*"): # grab both packages and signatures + symlink = self.paths.repository / full_path.name + if symlink.exists(): + continue # skip already created symlinks (or files) + symlink.symlink_to(full_path.relative_to(symlink.parent, walk_up=True)) + + # add to repository + check_output( + "repo-add", *self.sign_args, "--remove", str(self.repo_path), str(path), exception=BuildError.from_process(path.name), cwd=self.paths.repository, logger=self.logger, - user=self.uid) + user=self.uid, + ) def init(self) -> None: """ create empty repository database. It just calls add with empty arguments """ - check_output("repo-add", *self.sign_args, str(self.repo_path), - cwd=self.paths.repository, logger=self.logger, user=self.uid) + for root in (self.repo_path, self.repo_archive_path): + check_output("repo-add", *self.sign_args, str(root), + cwd=root, logger=self.logger, user=self.uid) def remove(self, package: str, filename: Path) -> None: """ @@ -88,7 +117,7 @@ class Repo(LazyLogging): filename(Path): package filename to remove """ # remove package and signature (if any) from filesystem - for full_path in self.paths.repository.glob(f"{filename}*"): + for full_path in self.paths.repository.glob(f"**/{filename}*"): full_path.unlink() # remove package from registry @@ -97,4 +126,5 @@ class Repo(LazyLogging): exception=BuildError.from_process(package), cwd=self.paths.repository, logger=self.logger, - user=self.uid) + user=self.uid, + ) diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py index 41755e79..2dfbd448 100644 --- a/src/ahriman/core/repository/executor.py +++ b/src/ahriman/core/repository/executor.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -import shutil +import shutil # shutil.move is used here to ensure cross fs file movement from collections.abc import Iterable from pathlib import Path @@ -41,6 +41,102 @@ class Executor(PackageInfo, Cleaner): trait for common repository update processes """ + def _archive_remove(self, description: PackageDescription, package_base: str) -> None: + """ + rename package archive removing special symbols + + Args: + description(PackageDescription): package description + package_base(str): package base name + """ + if description.filename is None: + self.logger.warning("received empty package name for base %s", package_base) + return # suppress type checking, it never can be none actually + + if (safe := safe_filename(description.filename)) != description.filename: + (self.paths.packages / description.filename).rename(self.paths.packages / safe) + description.filename = safe + + def _package_build(self, package: Package, path: Path, packager: str | None, + local_version: str | None) -> str | None: + """ + build single package + + Args: + package(Package): package to build + path(Path): path to directory with package files + packager(str | None): packager identifier used for this package + local_version(str | None): local version of the package + + Returns: + str | None: current commit sha if available + """ + self.reporter.set_building(package.base) + + task = Task(package, self.configuration, self.architecture, self.paths) + patches = self.reporter.package_patches_get(package.base, None) + commit_sha = task.init(path, patches, local_version) + built = task.build(path, PACKAGER=packager) + + package.with_packages(built, self.pacman) + for src in built: + dst = self.paths.packages / src.name + shutil.move(src, dst) + + return commit_sha + + def _package_remove(self, package_name: str, path: Path) -> None: + """ + remove single package from repository + + Args: + package_name(str): package name + path(Path): path to package archive + """ + try: + self.repo.remove(package_name, path) + except Exception: + self.logger.exception("could not remove %s", package_name) + + def _package_remove_base(self, package_base: str) -> None: + """ + remove package base from repository + + Args: + package_base(str): package base name: + """ + try: + with self.in_event(package_base, EventType.PackageRemoved): + self.reporter.package_remove(package_base) + except Exception: + self.logger.exception("could not remove base %s", package_base) + + def _package_update(self, filename: str | None, package_base: str, packager_key: str | None) -> None: + """ + update built package in repository database + + Args: + filename(str | None): archive filename + package_base(str): package base name + packager_key(str | None): packager key identifier + """ + if filename is None: + self.logger.warning("received empty package name for base %s", package_base) + return # suppress type checking, it never can be none actually + + # in theory, it might be NOT packages directory, but we suppose it is + full_path = self.paths.packages / filename + files = self.sign.process_sign_package(full_path, packager_key) + + for src in files: + archive = self.paths.archive_for(package_base) / src.name + shutil.move(src, archive) + if not (symlink := self.paths.repository / archive.name).exists(): + symlink.symlink_to(archive.relative_to(symlink.parent, walk_up=True)) + + package_path = self.paths.archive / filename + self.repo.add(package_path) + def process_build(self, updates: Iterable[Package], packagers: Packagers | None = None, *, bump_pkgrel: bool = False) -> Result: """ @@ -55,21 +151,6 @@ class Executor(PackageInfo, Cleaner): Returns: Result: build result """ - def build_single(package: Package, local_path: Path, packager_id: str | None) -> str | None: - self.reporter.set_building(package.base) - task = Task(package, self.configuration, self.architecture, self.paths) - local_version = local_versions.get(package.base) if bump_pkgrel else None - patches = self.reporter.package_patches_get(package.base, None) - commit_sha = task.init(local_path, patches, local_version) - built = task.build(local_path, PACKAGER=packager_id) - - package.with_packages(built, self.pacman) - for src in built: - dst = self.paths.packages / src.name - shutil.move(src, dst) - - return commit_sha - packagers = packagers or Packagers() local_versions = {package.base: package.version for package in self.packages()} @@ -80,16 +161,21 @@ class Executor(PackageInfo, Cleaner): try: with self.in_event(single.base, EventType.PackageUpdated, failure=EventType.PackageUpdateFailed): packager = self.packager(packagers, single.base) - last_commit_sha = build_single(single, Path(dir_name), packager.packager_id) + local_version = local_versions.get(single.base) if bump_pkgrel else None + commit_sha = self._package_build(single, Path(dir_name), packager.packager_id, local_version) + # update commit hash for changes keeping current diff if there is any changes = self.reporter.package_changes_get(single.base) - self.reporter.package_changes_update(single.base, Changes(last_commit_sha, changes.changes)) + self.reporter.package_changes_update(single.base, Changes(commit_sha, changes.changes)) + # update dependencies list package_archive = PackageArchive(self.paths.build_root, single, self.pacman, self.scan_paths) dependencies = package_archive.depends_on() self.reporter.package_dependencies_update(single.base, dependencies) + # update result set result.add_updated(single) + except Exception: self.reporter.set_failed(single.base) result.add_failed(single) @@ -107,19 +193,6 @@ class Executor(PackageInfo, Cleaner): Returns: Result: remove result """ - def remove_base(package_base: str) -> None: - try: - with self.in_event(package_base, EventType.PackageRemoved): - self.reporter.package_remove(package_base) - except Exception: - self.logger.exception("could not remove base %s", package_base) - - def remove_package(package: str, archive_path: Path) -> None: - try: - self.repo.remove(package, archive_path) # remove the package itself - except Exception: - self.logger.exception("could not remove %s", package) - packages_to_remove: dict[str, Path] = {} bases_to_remove: list[str] = [] @@ -136,6 +209,7 @@ class Executor(PackageInfo, Cleaner): }) bases_to_remove.append(local.base) result.add_removed(local) + elif requested.intersection(local.packages.keys()): packages_to_remove.update({ package: properties.filepath @@ -152,11 +226,11 @@ class Executor(PackageInfo, Cleaner): # remove packages from repository files for package, filename in packages_to_remove.items(): - remove_package(package, filename) + self._package_remove(package, filename) # remove bases from registered for package in bases_to_remove: - remove_base(package) + self._package_remove_base(package) return result @@ -172,27 +246,6 @@ class Executor(PackageInfo, Cleaner): Returns: Result: path to repository database """ - def rename(archive: PackageDescription, package_base: str) -> None: - if archive.filename is None: - self.logger.warning("received empty package name for base %s", package_base) - return # suppress type checking, it never can be none actually - if (safe := safe_filename(archive.filename)) != archive.filename: - shutil.move(self.paths.packages / archive.filename, self.paths.packages / safe) - archive.filename = safe - - def update_single(name: str | None, package_base: str, packager_key: str | None) -> None: - if name is None: - self.logger.warning("received empty package name for base %s", package_base) - return # suppress type checking, it never can be none actually - # in theory, it might be NOT packages directory, but we suppose it is - full_path = self.paths.packages / name - files = self.sign.process_sign_package(full_path, packager_key) - for src in files: - dst = self.paths.repository / safe_filename(src.name) - shutil.move(src, dst) - package_path = self.paths.repository / safe_filename(name) - self.repo.add(package_path) - current_packages = {package.base: package for package in self.packages()} local_versions = {package_base: package.version for package_base, package in current_packages.items()} @@ -207,8 +260,8 @@ class Executor(PackageInfo, Cleaner): packager = self.packager(packagers, local.base) for description in local.packages.values(): - rename(description, local.base) - update_single(description.filename, local.base, packager.key) + self._archive_remove(description, local.base) + self._package_update(description.filename, local.base, packager.key) self.reporter.set_success(local) result.add_updated(local) @@ -216,12 +269,13 @@ class Executor(PackageInfo, Cleaner): if local.base in current_packages: current_package_archives = set(current_packages[local.base].packages.keys()) removed_packages.extend(current_package_archives.difference(local.packages)) + except Exception: self.reporter.set_failed(local.base) result.add_failed(local) self.logger.exception("could not process %s", local.base) - self.clear_packages() + self.clear_packages() self.process_remove(removed_packages) return result diff --git a/src/ahriman/models/repository_paths.py b/src/ahriman/models/repository_paths.py index 7e8ae558..033f0074 100644 --- a/src/ahriman/models/repository_paths.py +++ b/src/ahriman/models/repository_paths.py @@ -85,6 +85,16 @@ class RepositoryPaths(LazyLogging): return Path(self.repository_id.architecture) # legacy tree suffix return Path(self.repository_id.name) / self.repository_id.architecture + @property + def archive(self) -> Path: + """ + archive directory root + + Returns: + Path: archive directory root + """ + return self.root / "archive" / self._suffix + @property def build_root(self) -> Path: """ @@ -249,6 +259,23 @@ class RepositoryPaths(LazyLogging): set_owner(path) path = path.parent + def archive_for(self, package_base: str) -> Path: + """ + get path to archive specified search criteria + + Args: + package_base(str): package base name + + Returns: + Path: path to archive directory for package base + """ + directory = self.archive / package_base[0] / package_base + if not directory.is_dir(): # create if not exists + with self.preserve_owner(self.archive): + directory.mkdir(mode=0o755, parents=True) + + return directory + def cache_for(self, package_base: str) -> Path: """ get path to cached PKGBUILD and package sources for the package base @@ -320,6 +347,7 @@ class RepositoryPaths(LazyLogging): with self.preserve_owner(): for directory in ( + self.archive, self.cache, self.chroot, self.packages,