From e8896423d347b4b1a1fce9e9797b1d31fee59df9 Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Mon, 12 Feb 2024 16:09:04 +0200 Subject: [PATCH] load local database too in pacman wrapper --- .../application/application/application.py | 3 ++ src/ahriman/core/alpm/pacman.py | 45 +++++++++++-------- src/ahriman/core/alpm/pacman_database.py | 29 ++++++++++-- tests/ahriman/core/alpm/test_pacman.py | 29 +++++++----- .../ahriman/core/alpm/test_pacman_database.py | 29 ++++++++++++ 5 files changed, 103 insertions(+), 32 deletions(-) diff --git a/src/ahriman/application/application/application.py b/src/ahriman/application/application/application.py index 90e2b535..4ec45a61 100644 --- a/src/ahriman/application/application/application.py +++ b/src/ahriman/application/application/application.py @@ -62,10 +62,13 @@ class Application(ApplicationPackages, ApplicationRepository): """ known_packages: set[str] = set() # local set + # this action is not really needed in case if ``alpm.use_ahriman_cache`` set to yes, because pacman + # will eventually contain all the local packages for base in self.repository.packages(): for package, properties in base.packages.items(): known_packages.add(package) known_packages.update(properties.provides) + # known pacman databases known_packages.update(self.repository.pacman.packages()) return known_packages diff --git a/src/ahriman/core/alpm/pacman.py b/src/ahriman/core/alpm/pacman.py index b35b611e..66b52269 100644 --- a/src/ahriman/core/alpm/pacman.py +++ b/src/ahriman/core/alpm/pacman.py @@ -23,7 +23,7 @@ import tarfile from collections.abc import Generator, Iterable from functools import cached_property from pathlib import Path -from pyalpm import DB, Handle, Package, SIG_PACKAGE # type: ignore[import-not-found] +from pyalpm import DB, Handle, Package, SIG_DATABASE_OPTIONAL, SIG_PACKAGE_OPTIONAL # type: ignore[import-not-found] from string import Template from ahriman.core.alpm.pacman_database import PacmanDatabase @@ -32,7 +32,6 @@ from ahriman.core.log import LazyLogging from ahriman.core.util import trim_package from ahriman.models.pacman_synchronization import PacmanSynchronization from ahriman.models.repository_id import RepositoryId -from ahriman.models.repository_paths import RepositoryPaths class Pacman(LazyLogging): @@ -43,6 +42,7 @@ class Pacman(LazyLogging): configuration(Configuration): configuration instance refresh_database(PacmanSynchronization): synchronize local cache to remote repository_id(RepositoryId): repository unique identifier + repository_path(RepositoryPaths): repository paths instance """ def __init__(self, repository_id: RepositoryId, configuration: Configuration, *, @@ -57,6 +57,7 @@ class Pacman(LazyLogging): """ self.configuration = configuration self.repository_id = repository_id + self.repository_paths = configuration.repository_paths self.refresh_database = refresh_database @@ -82,23 +83,25 @@ class Pacman(LazyLogging): """ pacman_root = self.configuration.getpath("alpm", "database") use_ahriman_cache = self.configuration.getboolean("alpm", "use_ahriman_cache") - paths = self.configuration.repository_paths - database_path = paths.pacman if use_ahriman_cache else pacman_root + database_path = self.repository_paths.pacman if use_ahriman_cache else pacman_root root = self.configuration.getpath("alpm", "root") handle = Handle(str(root), str(database_path)) for repository in self.configuration.getlist("alpm", "repositories"): database = self.database_init(handle, repository, self.repository_id.architecture) - self.database_copy(handle, database, pacman_root, paths, use_ahriman_cache=use_ahriman_cache) + self.database_copy(handle, database, pacman_root, use_ahriman_cache=use_ahriman_cache) + + # install repository database too + local_database = self.database_init(handle, self.repository_id.name, self.repository_id.architecture) + self.database_copy(handle, local_database, pacman_root, use_ahriman_cache=use_ahriman_cache) if use_ahriman_cache and refresh_database: self.database_sync(handle, force=refresh_database == PacmanSynchronization.Force) return handle - def database_copy(self, handle: Handle, database: DB, pacman_root: Path, paths: RepositoryPaths, *, - use_ahriman_cache: bool) -> None: + def database_copy(self, handle: Handle, database: DB, pacman_root: Path, *, use_ahriman_cache: bool) -> None: """ copy database from the operating system root to the ahriman local home @@ -106,7 +109,6 @@ class Pacman(LazyLogging): handle(Handle): pacman handle which will be used for database copying database(DB): pacman database instance to be copied pacman_root(Path): operating system pacman root - paths(RepositoryPaths): repository paths instance use_ahriman_cache(bool): use local ahriman cache instead of system one """ def repository_database(root: Path) -> Path: @@ -128,7 +130,7 @@ class Pacman(LazyLogging): return # database for some reason deos not exist self.logger.info("copy pacman database from operating system root to ahriman's home") shutil.copy(src, dst) - paths.chown(dst) + self.repository_paths.chown(dst) def database_init(self, handle: Handle, repository: str, architecture: str) -> DB: """ @@ -143,15 +145,21 @@ class Pacman(LazyLogging): DB: loaded pacman database instance """ self.logger.info("loading pacman database %s", repository) - database: DB = handle.register_syncdb(repository, SIG_PACKAGE) + database: DB = handle.register_syncdb(repository, SIG_DATABASE_OPTIONAL | SIG_PACKAGE_OPTIONAL) - mirror = self.configuration.get("alpm", "mirror") - # replace variables in mirror address - variables = { - "arch": architecture, - "repo": repository, - } - database.servers = [Template(mirror).safe_substitute(variables)] + if repository != self.repository_id.name: + mirror = self.configuration.get("alpm", "mirror") + # replace variables in mirror address + variables = { + "arch": architecture, + "repo": repository, + } + server = Template(mirror).safe_substitute(variables) + else: + # special case, same database, use local storage instead + server = f"file://{self.repository_paths.repository}" + + database.servers = [server] return database @@ -180,7 +188,6 @@ class Pacman(LazyLogging): dict[str, set[Path]]: map of package name to its list of files """ packages = packages or [] - repository_paths = self.configuration.repository_paths def extract(tar: tarfile.TarFile) -> Generator[tuple[str, set[Path]], None, None]: for descriptor in filter(lambda info: info.path.endswith("/files"), tar.getmembers()): @@ -196,7 +203,7 @@ class Pacman(LazyLogging): result: dict[str, set[Path]] = {} for database in self.handle.get_syncdbs(): - database_file = repository_paths.pacman / "sync" / f"{database.name}.files.tar.gz" + database_file = self.repository_paths.pacman / "sync" / f"{database.name}.files.tar.gz" if not database_file.is_file(): continue # no database file found with tarfile.open(database_file, "r:gz") as archive: diff --git a/src/ahriman/core/alpm/pacman_database.py b/src/ahriman/core/alpm/pacman_database.py index d99bbdd2..3c99f07c 100644 --- a/src/ahriman/core/alpm/pacman_database.py +++ b/src/ahriman/core/alpm/pacman_database.py @@ -18,10 +18,12 @@ # along with this program. If not, see . # import os +import shutil from email.utils import parsedate_to_datetime from pathlib import Path from pyalpm import DB # type: ignore[import-not-found] +from urllib.parse import urlparse from ahriman.core.configuration import Configuration from ahriman.core.exceptions import PacmanError @@ -57,6 +59,16 @@ class PacmanDatabase(SyncHttpClient): self.sync_files_database = configuration.getboolean("alpm", "sync_files_database") + def copy(self, remote_path: Path, local_path: Path) -> None: + """ + copy local database file + + Args: + remote_path(Path): path to source (remote) file + local_path(Path): path to locally stored file + """ + shutil.copy(remote_path, local_path) + def download(self, url: str, local_path: Path) -> None: """ download remote file and store it to local path with the correct last modified headers @@ -131,11 +143,22 @@ class PacmanDatabase(SyncHttpClient): filename = f"{self.database.name}.files.tar.gz" url = f"{server}/{filename}" + remote_uri = urlparse(url) local_path = Path(self.repository_paths.pacman / "sync" / filename) - if not force and not self.is_outdated(url, local_path): - return - self.download(url, local_path) + match remote_uri.scheme: + case "http" | "https": + if not force and not self.is_outdated(url, local_path): + return + + self.download(url, local_path) + + case "file": + # just copy file as it is relatively cheap operation, no need to check timestamps + self.copy(Path(remote_uri.path), local_path) + + case other: + raise PacmanError(f"Unknown or unsupported URL scheme {other}") def sync_packages(self, *, force: bool) -> None: """ diff --git a/tests/ahriman/core/alpm/test_pacman.py b/tests/ahriman/core/alpm/test_pacman.py index adedce65..d0dce96a 100644 --- a/tests/ahriman/core/alpm/test_pacman.py +++ b/tests/ahriman/core/alpm/test_pacman.py @@ -49,7 +49,7 @@ def test_init_with_local_cache_forced(configuration: Configuration, mocker: Mock sync_mock.assert_called_once_with(pytest.helpers.anyvar(int), force=True) -def test_database_copy(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: +def test_database_copy(pacman: Pacman, mocker: MockerFixture) -> None: """ must copy database from root """ @@ -63,13 +63,13 @@ def test_database_copy(pacman: Pacman, repository_paths: RepositoryPaths, mocker copy_mock = mocker.patch("shutil.copy") chown_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.chown") - pacman.database_copy(pacman.handle, database, path, repository_paths, use_ahriman_cache=True) + pacman.database_copy(pacman.handle, database, path, use_ahriman_cache=True) mkdir_mock.assert_called_once_with(mode=0o755, exist_ok=True) copy_mock.assert_called_once_with(path / "sync" / "core.db", dst_path) chown_mock.assert_called_once_with(dst_path) -def test_database_copy_skip(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: +def test_database_copy_skip(pacman: Pacman, mocker: MockerFixture) -> None: """ must do not copy database from root if local cache is disabled """ @@ -80,11 +80,11 @@ def test_database_copy_skip(pacman: Pacman, repository_paths: RepositoryPaths, m mocker.patch("pathlib.Path.is_file", autospec=True, side_effect=lambda p: p.is_relative_to(path)) copy_mock = mocker.patch("shutil.copy") - pacman.database_copy(pacman.handle, database, path, repository_paths, use_ahriman_cache=False) + pacman.database_copy(pacman.handle, database, path, use_ahriman_cache=False) copy_mock.assert_not_called() -def test_database_copy_no_directory(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: +def test_database_copy_no_directory(pacman: Pacman, mocker: MockerFixture) -> None: """ must do not copy database if local cache already exists """ @@ -95,11 +95,11 @@ def test_database_copy_no_directory(pacman: Pacman, repository_paths: Repository mocker.patch("pathlib.Path.is_file", autospec=True, side_effect=lambda p: p.is_relative_to(path)) copy_mock = mocker.patch("shutil.copy") - pacman.database_copy(pacman.handle, database, path, repository_paths, use_ahriman_cache=True) + pacman.database_copy(pacman.handle, database, path, use_ahriman_cache=True) copy_mock.assert_not_called() -def test_database_copy_no_root_file(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: +def test_database_copy_no_root_file(pacman: Pacman, mocker: MockerFixture) -> None: """ must do not copy database if no repository file exists in filesystem """ @@ -110,11 +110,11 @@ def test_database_copy_no_root_file(pacman: Pacman, repository_paths: Repository mocker.patch("pathlib.Path.is_file", return_value=False) copy_mock = mocker.patch("shutil.copy") - pacman.database_copy(pacman.handle, database, path, repository_paths, use_ahriman_cache=True) + pacman.database_copy(pacman.handle, database, path, use_ahriman_cache=True) copy_mock.assert_not_called() -def test_database_copy_database_exist(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: +def test_database_copy_database_exist(pacman: Pacman, mocker: MockerFixture) -> None: """ must do not copy database if local cache already exists """ @@ -124,7 +124,7 @@ def test_database_copy_database_exist(pacman: Pacman, repository_paths: Reposito mocker.patch("pathlib.Path.is_file", return_value=True) copy_mock = mocker.patch("shutil.copy") - pacman.database_copy(pacman.handle, database, Path("root"), repository_paths, use_ahriman_cache=True) + pacman.database_copy(pacman.handle, database, Path("root"), use_ahriman_cache=True) copy_mock.assert_not_called() @@ -136,6 +136,15 @@ def test_database_init(pacman: Pacman, configuration: Configuration) -> None: assert database.servers == ["https://geo.mirror.pkgbuild.com/testing/os/x86_64"] +def test_database_init_local(pacman: Pacman, configuration: Configuration) -> None: + """ + must set file protocol for local databases + """ + _, repository_id = configuration.check_loaded() + database = pacman.database_init(MagicMock(), repository_id.name, repository_id.architecture) + assert database.servers == [f"file://{configuration.repository_paths.repository}"] + + def test_database_sync(pacman: Pacman, mocker: MockerFixture) -> None: """ must sync databases diff --git a/tests/ahriman/core/alpm/test_pacman_database.py b/tests/ahriman/core/alpm/test_pacman_database.py index be24ac96..53a03c67 100644 --- a/tests/ahriman/core/alpm/test_pacman_database.py +++ b/tests/ahriman/core/alpm/test_pacman_database.py @@ -8,6 +8,15 @@ from ahriman.core.alpm.pacman_database import PacmanDatabase from ahriman.core.exceptions import PacmanError +def test_copy(pacman_database: PacmanDatabase, mocker: MockerFixture) -> None: + """ + must copy loca database file + """ + copy_mock = mocker.patch("shutil.copy") + pacman_database.copy(Path("remote"), Path("local")) + copy_mock.assert_called_once_with(Path("remote"), Path("local")) + + def test_download(pacman_database: PacmanDatabase, mocker: MockerFixture) -> None: """ must download database by remote url @@ -163,6 +172,26 @@ def test_sync_files_force(pacman_database: PacmanDatabase, mocker: MockerFixture "https://geo.mirror.pkgbuild.com/core/os/x86_64/core.files.tar.gz", pytest.helpers.anyvar(int)) +def test_sync_files_local(pacman_database: PacmanDatabase, mocker: MockerFixture) -> None: + """ + must copy local files instead of downloading them + """ + pacman_database.database.servers = ["file:///var"] + copy_mock = mocker.patch("ahriman.core.alpm.pacman_database.PacmanDatabase.copy") + + pacman_database.sync_files(force=False) + copy_mock.assert_called_once_with(Path("/var/core.files.tar.gz"), pytest.helpers.anyvar(int)) + + +def test_sync_files_unknown_source(pacman_database: PacmanDatabase) -> None: + """ + must raise an exception in case if server scheme is unsupported + """ + pacman_database.database.servers = ["some random string"] + with pytest.raises(PacmanError): + pacman_database.sync_files(force=False) + + def test_sync_packages(pacman_database: PacmanDatabase) -> None: """ must sync packages by using pyalpm method