arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2026-03-14 05:53:39 +00:00
Code Issues Projects Releases Wiki Activity

feat: archive package tree implementation (#153)

Browse Source 
* store built packages in archive tree instead of repository

* write tests to support new changes

* implement atomic_move method, move files only with lock

* use generic packages tree for all repos

* lookup through archive packages before build

* add archive trigger

* add archive trigger

* regenerate docs

* gpg loader fix

* support requires repostory flag

* drop excess REQUIRES_REPOSITORY

* simplify symlionk creation

* remove generators

* fix sttyle

* add separate function for symlinks creation

* fix rebase

* add note about slicing

* smol refactoring of archive_tree class

* remove duplicate code

* fix typos

* few review fixes

* monor fixes and typos

* clean empty directories

* remove side effect from getter

* drop recursive remove

* ensure_exists now accepts only argument

* add package like guard to symlinks fix

* speedup archive_lookup processing by iterrupting cycle

* remove custom filelock

* fix naming

* remove remove flag from repo

* review fixes

* restore wrapper around filelock

* extract repository explorer to separate class

* docs update

* fix ide findings
This commit is contained in:
Evgenii Alekseev
2026-02-16 00:12:51 +02:00
committed by GitHub
parent 6a2454548d
commit 2d6d42f969
71 changed files with 1876 additions and 363 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/ahriman/application/handlers/handler.py
View File

@@ -20,7 +20,7 @@
import argparse
import logging
from collections.abc import Callable, Iterable
from collections.abc import Callable
from multiprocessing import Pool
from typing import ClassVar, TypeVar
@@ -28,9 +28,9 @@ from ahriman.application.lock import Lock
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import ExitCode, MissingArchitectureError, MultipleArchitecturesError
from ahriman.core.log.log_loader import LogLoader
from ahriman.core.repository import Explorer
from ahriman.core.types import ExplicitBool
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
# this workaround is for several things
@@ -169,11 +169,6 @@ class Handler:
Raises:
MissingArchitectureError: if no architecture set and automatic detection is not allowed or failed
"""
configuration = Configuration()
configuration.load(args.configuration)
# pylint, wtf???
root = configuration.getpath("repository", "root") # pylint: disable=assignment-from-no-return
# preparse systemd repository-id argument
# we are using unescaped values, so / is not allowed here, because it is impossible to separate if from dashes
if args.repository_id is not None:
@@ -184,27 +179,10 @@ class Handler:
if repository_parts:
args.repository = "-".join(repository_parts) # replace slash with dash
# extract repository names first
if (from_args := args.repository) is not None:
repositories: Iterable[str] = [from_args]
elif from_filesystem := RepositoryPaths.known_repositories(root):
repositories = from_filesystem
else: # try to read configuration now
repositories = [configuration.get("repository", "name")]
configuration = Configuration()
configuration.load(args.configuration)
repositories = Explorer.repositories_extract(configuration, args.repository, args.architecture)
# extract architecture names
if (architecture := args.architecture) is not None:
parsed = set(
RepositoryId(architecture, repository)
for repository in repositories
)
else: # try to read from file system
parsed = set(
RepositoryId(architecture, repository)
for repository in repositories
for architecture in RepositoryPaths.known_architectures(root, repository)
)
if not parsed:
if not repositories:
raise MissingArchitectureError(args.command)
return sorted(parsed)
return sorted(repositories)

2
src/ahriman/application/handlers/status.py
View File

@@ -66,7 +66,7 @@ class Status(Handler):
Status.check_status(args.exit_code, packages)
comparator: Callable[[tuple[Package, BuildStatus]], Comparable] = lambda item: item[0].base
filter_fn: Callable[[tuple[Package, BuildStatus]], bool] =\
filter_fn: Callable[[tuple[Package, BuildStatus]], bool] = \
lambda item: args.status is None or item[1].status == args.status
for package, package_status in sorted(filter(filter_fn, packages), key=comparator):
PackagePrinter(package, package_status)(verbose=args.info)

18
src/ahriman/application/handlers/tree_migrate.py
View File

@@ -21,6 +21,7 @@ import argparse
from ahriman.application.handlers.handler import Handler, SubParserAction
from ahriman.core.configuration import Configuration
from ahriman.core.utils import symlink_relative, walk
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
@@ -49,6 +50,7 @@ class TreeMigrate(Handler):
target_tree.tree_create()
# perform migration
TreeMigrate.tree_move(current_tree, target_tree)
TreeMigrate.symlinks_fix(target_tree)
@staticmethod
def _set_service_tree_migrate_parser(root: SubParserAction) -> argparse.ArgumentParser:
@@ -66,6 +68,22 @@ class TreeMigrate(Handler):
parser.set_defaults(lock=None, quiet=True, report=False)
return parser
@staticmethod
def symlinks_fix(paths: RepositoryPaths) -> None:
"""
fix package archive symlinks
Args:
paths(RepositoryPaths): new repository paths
"""
archives = {path.name: path for path in walk(paths.archive)}
for symlink in walk(paths.repository):
if symlink.exists(): # no need to check for symlinks as we have just walked through the tree
continue
if (source_archive := archives.get(symlink.name)) is not None:
symlink.unlink()
symlink_relative(symlink, source_archive)
@staticmethod
def tree_move(from_tree: RepositoryPaths, to_tree: RepositoryPaths) -> None:
"""

33
src/ahriman/core/alpm/repo.py
View File

@@ -31,20 +31,21 @@ class Repo(LazyLogging):
Attributes:
name(str): repository name
paths(RepositoryPaths): repository paths instance
root(Path): repository root
sign_args(list[str]): additional args which have to be used to sign repository archive
uid(int): uid of the repository owner user
"""
def __init__(self, name: str, paths: RepositoryPaths, sign_args: list[str]) -> None:
def __init__(self, name: str, paths: RepositoryPaths, sign_args: list[str], root: Path | None = None) -> None:
"""
Args:
name(str): repository name
paths(RepositoryPaths): repository paths instance
sign_args(list[str]): additional args which have to be used to sign repository archive
root(Path | None, optional): repository root. If none set, the default will be used (Default value = None)
"""
self.name = name
self.paths = paths
self.root = root or paths.repository
self.uid, _ = paths.root_owner
self.sign_args = sign_args
@@ -56,7 +57,7 @@ class Repo(LazyLogging):
Returns:
Path: path to repository database
"""
return self.paths.repository / f"{self.name}.db.tar.gz"
return self.root / f"{self.name}.db.tar.gz"
def add(self, path: Path) -> None:
"""
@@ -66,35 +67,37 @@ class Repo(LazyLogging):
path(Path): path to archive to add
"""
check_output(
"repo-add", *self.sign_args, "-R", str(self.repo_path), str(path),
"repo-add", *self.sign_args, "--remove", str(self.repo_path), str(path),
exception=BuildError.from_process(path.name),
cwd=self.paths.repository,
cwd=self.root,
logger=self.logger,
user=self.uid)
user=self.uid,
)
def init(self) -> None:
"""
create empty repository database. It just calls add with empty arguments
"""
check_output("repo-add", *self.sign_args, str(self.repo_path),
cwd=self.paths.repository, logger=self.logger, user=self.uid)
cwd=self.root, logger=self.logger, user=self.uid)
def remove(self, package: str, filename: Path) -> None:
def remove(self, package_name: str, filename: Path) -> None:
"""
remove package from repository
Args:
package(str): package name to remove
package_name(str): package name to remove
filename(Path): package filename to remove
"""
# remove package and signature (if any) from filesystem
for full_path in self.paths.repository.glob(f"{filename}*"):
for full_path in self.root.glob(f"{filename.name}*"):
full_path.unlink()
# remove package from registry
check_output(
"repo-remove", *self.sign_args, str(self.repo_path), package,
exception=BuildError.from_process(package),
cwd=self.paths.repository,
"repo-remove", *self.sign_args, str(self.repo_path), package_name,
exception=BuildError.from_process(package_name),
cwd=self.root,
logger=self.logger,
user=self.uid)
user=self.uid,
)

20
src/ahriman/core/archive/__init__.py Normal file
View File

@@ -0,0 +1,20 @@
#
# Copyright (c) 2021-2025 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.archive.archive_trigger import ArchiveTrigger

185
src/ahriman/core/archive/archive_tree.py Normal file
View File

@@ -0,0 +1,185 @@
#
# Copyright (c) 2021-2025 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import datetime
from collections.abc import Iterator
from pathlib import Path
from ahriman.core.alpm.repo import Repo
from ahriman.core.log import LazyLogging
from ahriman.core.utils import package_like, symlink_relative, utcnow, walk
from ahriman.models.package import Package
from ahriman.models.package_description import PackageDescription
from ahriman.models.repository_paths import RepositoryPaths
class ArchiveTree(LazyLogging):
"""
wrapper around archive tree
Attributes:
paths(RepositoryPaths): repository paths instance
repository_id(RepositoryId): repository unique identifier
sign_args(list[str]): additional args which have to be used to sign repository archive
"""
def __init__(self, repository_path: RepositoryPaths, sign_args: list[str]) -> None:
"""
Args:
repository_path(RepositoryPaths): repository paths instance
sign_args(list[str]): additional args which have to be used to sign repository archive
"""
self.paths = repository_path
self.repository_id = repository_path.repository_id
self.sign_args = sign_args
@staticmethod
def _package_symlinks_create(package_description: PackageDescription, root: Path, archive: Path) -> bool:
"""
process symlinks creation for single package
Args:
package_description(PackageDescription): archive descriptor
root(Path): path to the archive repository root
archive(Path): path to directory with archives
Returns:
bool: ``True`` if symlinks were created and ``False`` otherwise
"""
symlinks_created = False
# here we glob for archive itself and signature if any
for file in archive.glob(f"{package_description.filename}*"):
try:
symlink_relative(root / file.name, file)
symlinks_created = True
except FileExistsError:
continue # symlink is already created, skip processing
return symlinks_created
def _repo(self, root: Path) -> Repo:
"""
constructs :class:`ahriman.core.alpm.repo.Repo` object for given path
Args:
root(Path): root of the repository
Returns:
Repo: constructed object with correct properties
"""
return Repo(self.repository_id.name, self.paths, self.sign_args, root)
def directories_fix(self, paths: set[Path]) -> None:
"""
remove empty repository directories recursively
Args:
paths(set[Path]): repositories to check
"""
root = self.paths.archive / "repos"
for repository in paths:
parents = [repository] + list(repository.parents[:-1])
for parent in parents:
path = root / parent
if list(path.iterdir()):
continue # directory is not empty
path.rmdir()
def repository_for(self, date: datetime.date | None = None) -> Path:
"""
get full path to repository at the specified date
Args:
date(datetime.date | None, optional): date to generate path. If none supplied then today will be used
(Default value = None)
Returns:
Path: path to the repository root
"""
date = date or utcnow().date()
return (
self.paths.archive
/ "repos"
/ date.strftime("%Y")
/ date.strftime("%m")
/ date.strftime("%d")
/ self.repository_id.name
/ self.repository_id.architecture
)
def symlinks_create(self, packages: list[Package]) -> None:
"""
create symlinks for the specified packages in today's repository
Args:
packages(list[Package]): list of packages to be updated
"""
root = self.repository_for()
repo = self._repo(root)
for package in packages:
archive = self.paths.archive_for(package.base)
for package_name, single in package.packages.items():
if single.filename is None:
self.logger.warning("received empty package filename for %s", package_name)
continue
if self._package_symlinks_create(single, root, archive):
repo.add(root / single.filename)
def symlinks_fix(self) -> Iterator[Path]:
"""
remove broken symlinks across repositories for all dates
Yields:
Path: path of the sub-repository with removed symlinks
"""
for path in walk(self.paths.archive / "repos"):
root = path.parent
*_, name, architecture = root.parts
if self.repository_id.name != name or self.repository_id.architecture != architecture:
continue # we only process same name repositories
if not package_like(path):
continue
if not path.is_symlink():
continue # find symlinks only
if path.exists():
continue # filter out not broken symlinks
# here we don't have access to original archive, so we have to guess name based on archive name
# normally it should be fine to do so
package_name = path.name.rsplit("-", maxsplit=3)[0]
self._repo(root).remove(package_name, path)
yield path.parent.relative_to(self.paths.archive / "repos")
def tree_create(self) -> None:
"""
create repository tree for current repository
"""
root = self.repository_for()
if root.exists():
return
with self.paths.preserve_owner():
root.mkdir(0o755, parents=True)
# init empty repository here
self._repo(root).init()

70
src/ahriman/core/archive/archive_trigger.py Normal file
View File

@@ -0,0 +1,70 @@
#
# Copyright (c) 2021-2025 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.archive.archive_tree import ArchiveTree
from ahriman.core.configuration import Configuration
from ahriman.core.sign.gpg import GPG
from ahriman.core.triggers import Trigger
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
class ArchiveTrigger(Trigger):
"""
archive repository extension
Attributes:
paths(RepositoryPaths): repository paths instance
tree(ArchiveTree): archive tree wrapper
"""
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
Args:
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, repository_id, configuration)
self.paths = configuration.repository_paths
self.tree = ArchiveTree(self.paths, GPG(configuration).repository_sign_args)
def on_result(self, result: Result, packages: list[Package]) -> None:
"""
run trigger
Args:
result(Result): build result
packages(list[Package]): list of all available packages
"""
self.tree.symlinks_create(packages)
def on_start(self) -> None:
"""
trigger action which will be called at the start of the application
"""
self.tree.tree_create()
def on_stop(self) -> None:
"""
trigger action which will be called before the stop of the application
"""
repositories = set(self.tree.symlinks_fix())
self.tree.directories_fix(repositories)

5
src/ahriman/core/build_tools/package_version.py
View File

@@ -17,8 +17,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from pyalpm import vercmp # type: ignore[import-not-found]
from ahriman.core.build_tools.task import Task
from ahriman.core.configuration import Configuration
from ahriman.core.log import LazyLogging
@@ -113,5 +111,4 @@ class PackageVersion(LazyLogging):
else:
remote_version = remote.version
result: int = vercmp(self.package.version, remote_version)
return result < 0
return self.package.vercmp(remote_version) < 0

81
src/ahriman/core/database/migrations/m016_archive.py Normal file
View File

@@ -0,0 +1,81 @@
#
# Copyright (c) 2021-2025 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import replace
from sqlite3 import Connection
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.configuration import Configuration
from ahriman.core.repository import Explorer
from ahriman.core.sign.gpg import GPG
from ahriman.core.utils import atomic_move, package_like, symlink_relative
from ahriman.models.package import Package
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.repository_paths import RepositoryPaths
__all__ = ["migrate_data"]
def migrate_data(connection: Connection, configuration: Configuration) -> None:
"""
perform data migration
Args:
connection(Connection): database connection
configuration(Configuration): configuration instance
"""
del connection
for repository_id in Explorer.repositories_extract(configuration):
paths = replace(configuration.repository_paths, repository_id=repository_id)
pacman = Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled)
# create archive directory if required
if not paths.archive.is_dir():
with paths.preserve_owner():
paths.archive.mkdir(mode=0o755, parents=True)
move_packages(paths, pacman)
def move_packages(repository_paths: RepositoryPaths, pacman: Pacman) -> None:
"""
move packages from repository to archive and create symbolic links
Args:
repository_paths(RepositoryPaths): repository paths instance
pacman(Pacman): alpm wrapper instance
"""
for archive in filter(package_like, repository_paths.repository.iterdir()):
if not archive.is_file(follow_symlinks=False):
continue # skip symbolic links if any
package = Package.from_archive(archive, pacman)
artifacts = [archive]
# check if there are signatures for this package and append it here too
if (signature := GPG.signature(archive)).exists():
artifacts.append(signature)
for source in artifacts:
target = repository_paths.ensure_exists(repository_paths.archive_for(package.base)) / source.name
# move package to the archive directory
atomic_move(source, target)
# create symlink to the archive
symlink_relative(source, target)

12
src/ahriman/core/database/sqlite.py
View File

@@ -25,8 +25,16 @@ from typing import Self
from ahriman.core.configuration import Configuration
from ahriman.core.database.migrations import Migrations
from ahriman.core.database.operations import AuthOperations, BuildOperations, ChangesOperations, \
DependenciesOperations, EventOperations, LogsOperations, PackageOperations, PatchOperations
from ahriman.core.database.operations import (
AuthOperations,
BuildOperations,
ChangesOperations,
DependenciesOperations,
EventOperations,
LogsOperations,
PackageOperations,
PatchOperations,
)
from ahriman.models.repository_id import RepositoryId

1
src/ahriman/core/housekeeping/__init__.py
View File

@@ -17,4 +17,5 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.housekeeping.archive_rotation_trigger import ArchiveRotationTrigger
from ahriman.core.housekeeping.logs_rotation_trigger import LogsRotationTrigger

116
src/ahriman/core/housekeeping/archive_rotation_trigger.py Normal file
View File

@@ -0,0 +1,116 @@
#
# Copyright (c) 2021-2025 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from collections.abc import Callable
from functools import cmp_to_key
from ahriman.core import context
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.configuration import Configuration
from ahriman.core.triggers import Trigger
from ahriman.core.utils import package_like
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
class ArchiveRotationTrigger(Trigger):
"""
remove packages from archive
Attributes:
keep_built_packages(int): number of last packages to keep
paths(RepositoryPaths): repository paths instance
"""
CONFIGURATION_SCHEMA = {
"archive": {
"type": "dict",
"schema": {
"keep_built_packages": {
"type": "integer",
"required": True,
"coerce": "integer",
"min": 0,
},
},
},
}
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
Args:
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, repository_id, configuration)
section = next(iter(self.configuration_sections(configuration)))
self.keep_built_packages = max(configuration.getint(section, "keep_built_packages"), 0)
self.paths = configuration.repository_paths
@classmethod
def configuration_sections(cls, configuration: Configuration) -> list[str]:
"""
extract configuration sections from configuration
Args:
configuration(Configuration): configuration instance
Returns:
list[str]: read configuration sections belong to this trigger
"""
return list(cls.CONFIGURATION_SCHEMA.keys())
def archives_remove(self, package: Package, pacman: Pacman) -> None:
"""
remove older versions of the specified package
Args:
package(Package): package which has been updated to check for older versions
pacman(Pacman): alpm wrapper instance
"""
packages: dict[tuple[str, str], Package] = {}
# we can't use here load_archives, because it ignores versions
for full_path in filter(package_like, self.paths.archive_for(package.base).iterdir()):
local = Package.from_archive(full_path, pacman)
packages.setdefault((local.base, local.version), local).packages.update(local.packages)
comparator: Callable[[Package, Package], int] = lambda left, right: left.vercmp(right.version)
to_remove = sorted(packages.values(), key=cmp_to_key(comparator))
# 0 will implicitly be translated into [:0], meaning we keep all packages
for single in to_remove[:-self.keep_built_packages]:
self.logger.info("removing version %s of package %s", single.version, single.base)
for archive in single.packages.values():
for path in self.paths.archive_for(single.base).glob(f"{archive.filename}*"):
path.unlink()
def on_result(self, result: Result, packages: list[Package]) -> None:
"""
run trigger
Args:
result(Result): build result
packages(list[Package]): list of all available packages
"""
ctx = context.get()
pacman = ctx.get(Pacman)
for package in result.success:
self.archives_remove(package, pacman)

1
src/ahriman/core/housekeeping/logs_rotation_trigger.py
View File

@@ -47,7 +47,6 @@ class LogsRotationTrigger(Trigger):
},
},
}
REQUIRES_REPOSITORY = True
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""

1
src/ahriman/core/report/report_trigger.py
View File

@@ -336,7 +336,6 @@ class ReportTrigger(Trigger):
},
},
}
REQUIRES_REPOSITORY = True
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""

1
src/ahriman/core/repository/__init__.py
View File

@@ -17,4 +17,5 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.repository.explorer import Explorer
from ahriman.core.repository.repository import Repository

206
src/ahriman/core/repository/executor.py
View File

@@ -27,7 +27,7 @@ from ahriman.core.build_tools.package_archive import PackageArchive
from ahriman.core.build_tools.task import Task
from ahriman.core.repository.cleaner import Cleaner
from ahriman.core.repository.package_info import PackageInfo
from ahriman.core.utils import safe_filename
from ahriman.core.utils import atomic_move, filelock, list_flatmap, package_like, safe_filename, symlink_relative
from ahriman.models.changes import Changes
from ahriman.models.event import EventType
from ahriman.models.package import Package
@@ -41,6 +41,140 @@ class Executor(PackageInfo, Cleaner):
trait for common repository update processes
"""
def _archive_lookup(self, package: Package) -> list[Path]:
"""
check if there is a rebuilt package already
Args:
package(Package): package to check
Returns:
list[Path]: list of built packages and signatures if available, empty list otherwise
"""
archive = self.paths.archive_for(package.base)
if not archive.is_dir():
return []
for path in filter(package_like, archive.iterdir()):
# check if package version is the same
built = Package.from_archive(path, self.pacman)
if built.version != package.version:
continue
packages = built.packages.values()
# all packages must be either any or same architecture
if not all(single.architecture in ("any", self.architecture) for single in packages):
continue
return list_flatmap(packages, lambda single: archive.glob(f"{single.filename}*"))
return []
def _archive_rename(self, description: PackageDescription, package_base: str) -> None:
"""
rename package archive removing special symbols
Args:
description(PackageDescription): package description
package_base(str): package base name
"""
if description.filename is None:
self.logger.warning("received empty package filename for base %s", package_base)
return # suppress type checking, it never can be none actually
if (safe := safe_filename(description.filename)) != description.filename:
atomic_move(self.paths.packages / description.filename, self.paths.packages / safe)
description.filename = safe
def _package_build(self, package: Package, path: Path, packager: str | None,
local_version: str | None) -> str | None:
"""
build single package
Args:
package(Package): package to build
path(Path): path to directory with package files
packager(str | None): packager identifier used for this package
local_version(str | None): local version of the package
Returns:
str | None: current commit sha if available
"""
self.reporter.set_building(package.base)
task = Task(package, self.configuration, self.architecture, self.paths)
patches = self.reporter.package_patches_get(package.base, None)
commit_sha = task.init(path, patches, local_version)
loaded_package = Package.from_build(path, self.architecture, None)
if prebuilt := list(self._archive_lookup(loaded_package)):
self.logger.info("using prebuilt packages for %s-%s", loaded_package.base, loaded_package.version)
built = []
for artifact in prebuilt:
with filelock(artifact):
shutil.copy(artifact, path)
built.append(path / artifact.name)
else:
built = task.build(path, PACKAGER=packager)
package.with_packages(built, self.pacman)
for src in built:
dst = self.paths.packages / src.name
atomic_move(src, dst)
return commit_sha
def _package_remove(self, package_name: str, path: Path) -> None:
"""
remove single package from repository
Args:
package_name(str): package name
path(Path): path to package archive
"""
try:
self.repo.remove(package_name, path)
except Exception:
self.logger.exception("could not remove %s", package_name)
def _package_remove_base(self, package_base: str) -> None:
"""
remove package base from repository
Args:
package_base(str): package base name
"""
try:
with self.in_event(package_base, EventType.PackageRemoved):
self.reporter.package_remove(package_base)
except Exception:
self.logger.exception("could not remove base %s", package_base)
def _package_update(self, filename: str | None, package_base: str, packager_key: str | None) -> None:
"""
update built package in repository database
Args:
filename(str | None): archive filename
package_base(str): package base name
packager_key(str | None): packager key identifier
"""
if filename is None:
self.logger.warning("received empty package filename for base %s", package_base)
return # suppress type checking, it never can be none actually
# in theory, it might be NOT packages directory, but we suppose it is
full_path = self.paths.packages / filename
files = self.sign.process_sign_package(full_path, packager_key)
for src in files:
dst = self.paths.ensure_exists(self.paths.archive_for(package_base)) / src.name
atomic_move(src, dst) # move package to archive directory
if not (symlink := self.paths.repository / dst.name).exists():
symlink_relative(symlink, dst) # create link to archive
self.repo.add(self.paths.repository / filename)
def process_build(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
bump_pkgrel: bool = False) -> Result:
"""
@@ -55,21 +189,6 @@ class Executor(PackageInfo, Cleaner):
Returns:
Result: build result
"""
def build_single(package: Package, local_path: Path, packager_id: str | None) -> str | None:
self.reporter.set_building(package.base)
task = Task(package, self.configuration, self.architecture, self.paths)
local_version = local_versions.get(package.base) if bump_pkgrel else None
patches = self.reporter.package_patches_get(package.base, None)
commit_sha = task.init(local_path, patches, local_version)
built = task.build(local_path, PACKAGER=packager_id)
package.with_packages(built, self.pacman)
for src in built:
dst = self.paths.packages / src.name
shutil.move(src, dst)
return commit_sha
packagers = packagers or Packagers()
local_versions = {package.base: package.version for package in self.packages()}
@@ -80,16 +199,21 @@ class Executor(PackageInfo, Cleaner):
try:
with self.in_event(single.base, EventType.PackageUpdated, failure=EventType.PackageUpdateFailed):
packager = self.packager(packagers, single.base)
last_commit_sha = build_single(single, Path(dir_name), packager.packager_id)
local_version = local_versions.get(single.base) if bump_pkgrel else None
commit_sha = self._package_build(single, Path(dir_name), packager.packager_id, local_version)
# update commit hash for changes keeping current diff if there is any
changes = self.reporter.package_changes_get(single.base)
self.reporter.package_changes_update(single.base, Changes(last_commit_sha, changes.changes))
self.reporter.package_changes_update(single.base, Changes(commit_sha, changes.changes))
# update dependencies list
package_archive = PackageArchive(self.paths.build_root, single, self.pacman, self.scan_paths)
dependencies = package_archive.depends_on()
self.reporter.package_dependencies_update(single.base, dependencies)
# update result set
result.add_updated(single)
except Exception:
self.reporter.set_failed(single.base)
result.add_failed(single)
@@ -107,19 +231,6 @@ class Executor(PackageInfo, Cleaner):
Returns:
Result: remove result
"""
def remove_base(package_base: str) -> None:
try:
with self.in_event(package_base, EventType.PackageRemoved):
self.reporter.package_remove(package_base)
except Exception:
self.logger.exception("could not remove base %s", package_base)
def remove_package(package: str, archive_path: Path) -> None:
try:
self.repo.remove(package, archive_path) # remove the package itself
except Exception:
self.logger.exception("could not remove %s", package)
packages_to_remove: dict[str, Path] = {}
bases_to_remove: list[str] = []
@@ -136,6 +247,7 @@ class Executor(PackageInfo, Cleaner):
})
bases_to_remove.append(local.base)
result.add_removed(local)
elif requested.intersection(local.packages.keys()):
packages_to_remove.update({
package: properties.filepath
@@ -152,11 +264,11 @@ class Executor(PackageInfo, Cleaner):
# remove packages from repository files
for package, filename in packages_to_remove.items():
remove_package(package, filename)
self._package_remove(package, filename)
# remove bases from registered
for package in bases_to_remove:
remove_base(package)
self._package_remove_base(package)
return result
@@ -172,27 +284,6 @@ class Executor(PackageInfo, Cleaner):
Returns:
Result: path to repository database
"""
def rename(archive: PackageDescription, package_base: str) -> None:
if archive.filename is None:
self.logger.warning("received empty package name for base %s", package_base)
return # suppress type checking, it never can be none actually
if (safe := safe_filename(archive.filename)) != archive.filename:
shutil.move(self.paths.packages / archive.filename, self.paths.packages / safe)
archive.filename = safe
def update_single(name: str | None, package_base: str, packager_key: str | None) -> None:
if name is None:
self.logger.warning("received empty package name for base %s", package_base)
return # suppress type checking, it never can be none actually
# in theory, it might be NOT packages directory, but we suppose it is
full_path = self.paths.packages / name
files = self.sign.process_sign_package(full_path, packager_key)
for src in files:
dst = self.paths.repository / safe_filename(src.name)
shutil.move(src, dst)
package_path = self.paths.repository / safe_filename(name)
self.repo.add(package_path)
current_packages = {package.base: package for package in self.packages()}
local_versions = {package_base: package.version for package_base, package in current_packages.items()}
@@ -207,8 +298,8 @@ class Executor(PackageInfo, Cleaner):
packager = self.packager(packagers, local.base)
for description in local.packages.values():
rename(description, local.base)
update_single(description.filename, local.base, packager.key)
self._archive_rename(description, local.base)
self._package_update(description.filename, local.base, packager.key)
self.reporter.set_success(local)
result.add_updated(local)
@@ -216,12 +307,13 @@ class Executor(PackageInfo, Cleaner):
if local.base in current_packages:
current_package_archives = set(current_packages[local.base].packages.keys())
removed_packages.extend(current_package_archives.difference(local.packages))
except Exception:
self.reporter.set_failed(local.base)
result.add_failed(local)
self.logger.exception("could not process %s", local.base)
self.clear_packages()
self.clear_packages()
self.process_remove(removed_packages)
return result

70
src/ahriman/core/repository/explorer.py Normal file
View File

@@ -0,0 +1,70 @@
#
# Copyright (c) 2021-2026 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from collections.abc import Iterable
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
class Explorer:
"""
helper to read filesystem and find created repositories
"""
@staticmethod
def repositories_extract(configuration: Configuration, repository: str | None = None,
architecture: str | None = None) -> list[RepositoryId]:
"""
get known architectures
Args:
configuration(Configuration): configuration instance
repository(str | None, optional): predefined repository name if available (Default value = None)
architecture(str | None, optional): predefined repository architecture if available (Default value = None)
Returns:
list[RepositoryId]: list of repository names and architectures for which tree is created
"""
# pylint, wtf???
root = configuration.getpath("repository", "root") # pylint: disable=assignment-from-no-return
# extract repository names first
if repository is not None:
repositories: Iterable[str] = [repository]
elif from_filesystem := RepositoryPaths.known_repositories(root):
repositories = from_filesystem
else: # try to read configuration now
repositories = [configuration.get("repository", "name")]
# extract architecture names
if architecture is not None:
parsed = set(
RepositoryId(architecture, repository)
for repository in repositories
)
else: # try to read from file system
parsed = set(
RepositoryId(architecture, repository)
for repository in repositories
for architecture in RepositoryPaths.known_architectures(root, repository)
)
return sorted(parsed)

1
src/ahriman/core/support/keyring_trigger.py
View File

@@ -103,7 +103,6 @@ class KeyringTrigger(Trigger):
},
},
}
REQUIRES_REPOSITORY = True
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""

1
src/ahriman/core/support/mirrorlist_trigger.py
View File

@@ -90,7 +90,6 @@ class MirrorlistTrigger(Trigger):
},
},
}
REQUIRES_REPOSITORY = True
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""

1
src/ahriman/core/upload/upload_trigger.py
View File

@@ -160,7 +160,6 @@ class UploadTrigger(Trigger):
},
},
}
REQUIRES_REPOSITORY = True
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""

57
src/ahriman/core/utils.py
View File

@@ -18,6 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# pylint: disable=too-many-lines
import contextlib
import datetime
import io
import itertools
@@ -25,11 +26,13 @@ import logging
import os
import re
import selectors
import shutil
import subprocess
from collections.abc import Callable, Iterable, Iterator, Mapping
from dataclasses import asdict
from enum import Enum
from filelock import FileLock
from pathlib import Path
from pwd import getpwuid
from typing import Any, IO, TypeVar
@@ -39,11 +42,13 @@ from ahriman.core.types import Comparable
__all__ = [
"atomic_move",
"check_output",
"check_user",
"dataclass_view",
"enum_values",
"extract_user",
"filelock",
"filter_json",
"full_version",
"list_flatmap",
@@ -58,6 +63,7 @@ __all__ = [
"safe_filename",
"srcinfo_property",
"srcinfo_property_list",
"symlink_relative",
"trim_package",
"utcnow",
"walk",
@@ -68,6 +74,25 @@ R = TypeVar("R", bound=Comparable)
T = TypeVar("T")
def atomic_move(src: Path, dst: Path) -> None:
"""
move file from ``source`` location to ``destination``. This method uses lock and :func:`shutil.move` to ensure that
file will be copied (if not rename) atomically. This method blocks execution until lock is available
Args:
src(Path): path to the source file
dst(Path): path to the destination
Examples:
This method is a drop-in replacement for :func:`shutil.move` (except it doesn't allow to override copy method)
which first locking destination file. To use it simply call method with arguments::
>>> atomic_move(src, dst)
"""
with filelock(dst):
shutil.move(src, dst)
# pylint: disable=too-many-locals
def check_output(*args: str, exception: Exception | Callable[[int, list[str], str, str], Exception] | None = None,
cwd: Path | None = None, input_data: str | None = None,
@@ -239,6 +264,25 @@ def extract_user() -> str | None:
return os.getenv("SUDO_USER") or os.getenv("DOAS_USER") or os.getenv("USER")
@contextlib.contextmanager
def filelock(path: Path) -> Iterator[FileLock]:
"""
wrapper around :class:`filelock.FileLock`, which also removes locks afterward
Args:
path(Path): path to lock on. The lock file will be created as ``.{path.name}.lock``
Yields:
FileLock: acquired file lock instance
"""
lock_path = path.with_name(f".{path.name}.lock")
try:
with FileLock(lock_path) as lock:
yield lock
finally:
lock_path.unlink(missing_ok=True)
def filter_json(source: dict[str, Any], known_fields: Iterable[str]) -> dict[str, Any]:
"""
filter json object by fields used for json-to-object conversion
@@ -279,7 +323,7 @@ def full_version(epoch: str | int | None, pkgver: str, pkgrel: str) -> str:
return f"{prefix}{pkgver}-{pkgrel}"
def list_flatmap(source: Iterable[T], extractor: Callable[[T], list[R]]) -> list[R]:
def list_flatmap(source: Iterable[T], extractor: Callable[[T], Iterable[R]]) -> list[R]:
"""
extract elements from list of lists, flatten them and apply ``extractor``
@@ -510,6 +554,17 @@ def srcinfo_property_list(key: str, srcinfo: Mapping[str, Any], package_srcinfo:
return values
def symlink_relative(symlink: Path, source: Path) -> None:
"""
create symlink with relative path to the target directory
Args:
symlink(Path): path to symlink to create
source(Path): source file to be symlinked
"""
symlink.symlink_to(source.relative_to(symlink.parent, walk_up=True))
def trim_package(package_name: str) -> str:
"""
remove version bound and description from package name. Pacman allows to specify version bound (=, <=, >= etc.) for

15
src/ahriman/models/package.py
View File

@@ -357,7 +357,7 @@ class Package(LazyLogging):
if local_version is None:
return None # local version not found, keep upstream pkgrel
if vercmp(self.version, local_version) > 0:
if self.vercmp(local_version) > 0:
return None # upstream version is newer than local one, keep upstream pkgrel
*_, local_pkgrel = parse_version(local_version)
@@ -378,6 +378,19 @@ class Package(LazyLogging):
details = "" if self.is_single_package else f""" ({" ".join(sorted(self.packages.keys()))})"""
return f"{self.base}{details}"
def vercmp(self, version: str) -> int:
"""
typed wrapper around :func:`pyalpm.vercmp()`
Args:
version(str): version to compare
Returns:
int: negative if current version is less than provided, positive if greater than and zero if equals
"""
result: int = vercmp(self.version, version)
return result
def view(self) -> dict[str, Any]:
"""
generate json package view

62
src/ahriman/models/repository_paths.py
View File

@@ -85,6 +85,16 @@ class RepositoryPaths(LazyLogging):
return Path(self.repository_id.architecture) # legacy tree suffix
return Path(self.repository_id.name) / self.repository_id.architecture
@property
def archive(self) -> Path:
"""
archive directory root
Returns:
Path: archive directory root
"""
return self.root / "archive"
@property
def build_root(self) -> Path:
"""
@@ -208,6 +218,18 @@ class RepositoryPaths(LazyLogging):
return set(walk(instance))
def archive_for(self, package_base: str) -> Path:
"""
get path to archive specified search criteria
Args:
package_base(str): package base name
Returns:
Path: path to archive directory for package base
"""
return self.archive / "packages" / package_base[0] / package_base
def cache_for(self, package_base: str) -> Path:
"""
get path to cached PKGBUILD and package sources for the package base
@@ -220,6 +242,27 @@ class RepositoryPaths(LazyLogging):
"""
return self.cache / package_base
def ensure_exists(self, directory: Path) -> Path:
"""
get path based on ``directory`` callable provided and ensure it exists
Args:
directory(Path): path to directory to check
Returns:
Path: original path based on extractor provided. Directory will always exist
Examples:
This method calls directory accessor and then checks if there is a directory and - otherwise - creates it::
>>> paths.ensure_exists(paths.archive_for(package_base))
"""
if not directory.is_dir():
with self.preserve_owner():
directory.mkdir(mode=0o755, parents=True)
return directory
@contextlib.contextmanager
def preserve_owner(self) -> Iterator[None]:
"""
@@ -265,6 +308,7 @@ class RepositoryPaths(LazyLogging):
"""
for directory in (
self.cache_for(package_base),
self.archive_for(package_base),
):
shutil.rmtree(directory, ignore_errors=True)
@@ -275,12 +319,12 @@ class RepositoryPaths(LazyLogging):
if self.repository_id.is_empty:
return # do not even try to create tree in case if no repository id set
with self.preserve_owner():
for directory in (
self.cache,
self.chroot,
self.packages,
self.pacman,
self.repository,
):
directory.mkdir(mode=0o755, parents=True, exist_ok=True)
for directory in (
self.archive,
self.cache,
self.chroot,
self.packages,
self.pacman,
self.repository,
):
self.ensure_exists(directory)

14
src/ahriman/web/middlewares/exception_handler.py
View File

@@ -21,8 +21,18 @@ import aiohttp_jinja2
import logging
from aiohttp.typedefs import Middleware
from aiohttp.web import HTTPClientError, HTTPException, HTTPMethodNotAllowed, HTTPNoContent, HTTPServerError, \
HTTPUnauthorized, Request, StreamResponse, json_response, middleware
from aiohttp.web import (
HTTPClientError,
HTTPException,
HTTPMethodNotAllowed,
HTTPNoContent,
HTTPServerError,
HTTPUnauthorized,
Request,
StreamResponse,
json_response,
middleware,
)
from ahriman.web.middlewares import HandlerType

8
src/ahriman/web/views/v1/packages/package.py
View File

@@ -25,8 +25,12 @@ from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package
from ahriman.models.user_access import UserAccess
from ahriman.web.apispec.decorators import apidocs
from ahriman.web.schemas import PackageNameSchema, PackageStatusSchema, PackageStatusSimplifiedSchema, \
RepositoryIdSchema
from ahriman.web.schemas import (
PackageNameSchema,
PackageStatusSchema,
PackageStatusSimplifiedSchema,
RepositoryIdSchema,
)
from ahriman.web.views.base import BaseView
from ahriman.web.views.status_view_guard import StatusViewGuard

5
src/ahriman/web/views/v1/service/upload.py
View File

@@ -26,6 +26,7 @@ from tempfile import NamedTemporaryFile
from typing import ClassVar
from ahriman.core.configuration import Configuration
from ahriman.core.utils import atomic_move
from ahriman.models.repository_paths import RepositoryPaths
from ahriman.models.user_access import UserAccess
from ahriman.web.apispec.decorators import apidocs
@@ -152,10 +153,8 @@ class UploadView(BaseView):
files.append(await self.save_file(part, target, max_body_size=max_body_size))
# and now we can rename files, which is relatively fast operation
# it is probably good way to call lock here, however
for filename, current_location in files:
target_location = current_location.parent / filename
current_location.rename(target_location)
atomic_move(current_location, target_location)
raise HTTPCreated