arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-04-24 07:17:17 +00:00
Code Issues Projects Releases Wiki Activity

load local database too in pacman wrapper

Browse Source
This commit is contained in:
Evgenii Alekseev 2024-02-12 16:09:04 +02:00
parent 48b7bafbe4
commit e8896423d3
5 changed files with 103 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/ahriman/application/application/application.py
View File

@ -62,10 +62,13 @@ class Application(ApplicationPackages, ApplicationRepository):
"""
known_packages: set[str] = set()
# local set
# this action is not really needed in case if ``alpm.use_ahriman_cache`` set to yes, because pacman
# will eventually contain all the local packages
for base in self.repository.packages():
for package, properties in base.packages.items():
known_packages.add(package)
known_packages.update(properties.provides)
# known pacman databases
known_packages.update(self.repository.pacman.packages())
return known_packages

45
src/ahriman/core/alpm/pacman.py
View File

@ -23,7 +23,7 @@ import tarfile
from collections.abc import Generator, Iterable
from functools import cached_property
from pathlib import Path
from pyalpm import DB, Handle, Package, SIG_PACKAGE # type: ignore[import-not-found]
from pyalpm import DB, Handle, Package, SIG_DATABASE_OPTIONAL, SIG_PACKAGE_OPTIONAL # type: ignore[import-not-found]
from string import Template
from ahriman.core.alpm.pacman_database import PacmanDatabase
@ -32,7 +32,6 @@ from ahriman.core.log import LazyLogging
from ahriman.core.util import trim_package
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
class Pacman(LazyLogging):
@ -43,6 +42,7 @@ class Pacman(LazyLogging):
configuration(Configuration): configuration instance
refresh_database(PacmanSynchronization): synchronize local cache to remote
repository_id(RepositoryId): repository unique identifier
repository_path(RepositoryPaths): repository paths instance
"""
def __init__(self, repository_id: RepositoryId, configuration: Configuration, *,
@ -57,6 +57,7 @@ class Pacman(LazyLogging):
"""
self.configuration = configuration
self.repository_id = repository_id
self.repository_paths = configuration.repository_paths
self.refresh_database = refresh_database
@ -82,23 +83,25 @@ class Pacman(LazyLogging):
"""
pacman_root = self.configuration.getpath("alpm", "database")
use_ahriman_cache = self.configuration.getboolean("alpm", "use_ahriman_cache")
paths = self.configuration.repository_paths
database_path = paths.pacman if use_ahriman_cache else pacman_root
database_path = self.repository_paths.pacman if use_ahriman_cache else pacman_root
root = self.configuration.getpath("alpm", "root")
handle = Handle(str(root), str(database_path))
for repository in self.configuration.getlist("alpm", "repositories"):
database = self.database_init(handle, repository, self.repository_id.architecture)
self.database_copy(handle, database, pacman_root, paths, use_ahriman_cache=use_ahriman_cache)
self.database_copy(handle, database, pacman_root, use_ahriman_cache=use_ahriman_cache)
# install repository database too
local_database = self.database_init(handle, self.repository_id.name, self.repository_id.architecture)
self.database_copy(handle, local_database, pacman_root, use_ahriman_cache=use_ahriman_cache)
if use_ahriman_cache and refresh_database:
self.database_sync(handle, force=refresh_database == PacmanSynchronization.Force)
return handle
def database_copy(self, handle: Handle, database: DB, pacman_root: Path, paths: RepositoryPaths, *,
use_ahriman_cache: bool) -> None:
def database_copy(self, handle: Handle, database: DB, pacman_root: Path, *, use_ahriman_cache: bool) -> None:
"""
copy database from the operating system root to the ahriman local home
@ -106,7 +109,6 @@ class Pacman(LazyLogging):
handle(Handle): pacman handle which will be used for database copying
database(DB): pacman database instance to be copied
pacman_root(Path): operating system pacman root
paths(RepositoryPaths): repository paths instance
use_ahriman_cache(bool): use local ahriman cache instead of system one
"""
def repository_database(root: Path) -> Path:
@ -128,7 +130,7 @@ class Pacman(LazyLogging):
return # database for some reason deos not exist
self.logger.info("copy pacman database from operating system root to ahriman's home")
shutil.copy(src, dst)
paths.chown(dst)
self.repository_paths.chown(dst)
def database_init(self, handle: Handle, repository: str, architecture: str) -> DB:
"""
@ -143,15 +145,21 @@ class Pacman(LazyLogging):
DB: loaded pacman database instance
"""
self.logger.info("loading pacman database %s", repository)
database: DB = handle.register_syncdb(repository, SIG_PACKAGE)
database: DB = handle.register_syncdb(repository, SIG_DATABASE_OPTIONAL | SIG_PACKAGE_OPTIONAL)
mirror = self.configuration.get("alpm", "mirror")
# replace variables in mirror address
variables = {
"arch": architecture,
"repo": repository,
}
database.servers = [Template(mirror).safe_substitute(variables)]
if repository != self.repository_id.name:
mirror = self.configuration.get("alpm", "mirror")
# replace variables in mirror address
variables = {
"arch": architecture,
"repo": repository,
}
server = Template(mirror).safe_substitute(variables)
else:
# special case, same database, use local storage instead
server = f"file://{self.repository_paths.repository}"
database.servers = [server]
return database
@ -180,7 +188,6 @@ class Pacman(LazyLogging):
dict[str, set[Path]]: map of package name to its list of files
"""
packages = packages or []
repository_paths = self.configuration.repository_paths
def extract(tar: tarfile.TarFile) -> Generator[tuple[str, set[Path]], None, None]:
for descriptor in filter(lambda info: info.path.endswith("/files"), tar.getmembers()):
@ -196,7 +203,7 @@ class Pacman(LazyLogging):
result: dict[str, set[Path]] = {}
for database in self.handle.get_syncdbs():
database_file = repository_paths.pacman / "sync" / f"{database.name}.files.tar.gz"
database_file = self.repository_paths.pacman / "sync" / f"{database.name}.files.tar.gz"
if not database_file.is_file():
continue # no database file found
with tarfile.open(database_file, "r:gz") as archive:

29
src/ahriman/core/alpm/pacman_database.py
View File

@ -18,10 +18,12 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
import shutil
from email.utils import parsedate_to_datetime
from pathlib import Path
from pyalpm import DB # type: ignore[import-not-found]
from urllib.parse import urlparse
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import PacmanError
@ -57,6 +59,16 @@ class PacmanDatabase(SyncHttpClient):
self.sync_files_database = configuration.getboolean("alpm", "sync_files_database")
def copy(self, remote_path: Path, local_path: Path) -> None:
"""
copy local database file
Args:
remote_path(Path): path to source (remote) file
local_path(Path): path to locally stored file
"""
shutil.copy(remote_path, local_path)
def download(self, url: str, local_path: Path) -> None:
"""
download remote file and store it to local path with the correct last modified headers
@ -131,11 +143,22 @@ class PacmanDatabase(SyncHttpClient):
filename = f"{self.database.name}.files.tar.gz"
url = f"{server}/{filename}"
remote_uri = urlparse(url)
local_path = Path(self.repository_paths.pacman / "sync" / filename)
if not force and not self.is_outdated(url, local_path):
return
self.download(url, local_path)
match remote_uri.scheme:
case "http" | "https":
if not force and not self.is_outdated(url, local_path):
return
self.download(url, local_path)
case "file":
# just copy file as it is relatively cheap operation, no need to check timestamps
self.copy(Path(remote_uri.path), local_path)
case other:
raise PacmanError(f"Unknown or unsupported URL scheme {other}")
def sync_packages(self, *, force: bool) -> None:
"""

29
tests/ahriman/core/alpm/test_pacman.py
View File

@ -49,7 +49,7 @@ def test_init_with_local_cache_forced(configuration: Configuration, mocker: Mock
sync_mock.assert_called_once_with(pytest.helpers.anyvar(int), force=True)
def test_database_copy(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
def test_database_copy(pacman: Pacman, mocker: MockerFixture) -> None:
"""
must copy database from root
"""
@ -63,13 +63,13 @@ def test_database_copy(pacman: Pacman, repository_paths: RepositoryPaths, mocker
copy_mock = mocker.patch("shutil.copy")
chown_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.chown")
pacman.database_copy(pacman.handle, database, path, repository_paths, use_ahriman_cache=True)
pacman.database_copy(pacman.handle, database, path, use_ahriman_cache=True)
mkdir_mock.assert_called_once_with(mode=0o755, exist_ok=True)
copy_mock.assert_called_once_with(path / "sync" / "core.db", dst_path)
chown_mock.assert_called_once_with(dst_path)
def test_database_copy_skip(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
def test_database_copy_skip(pacman: Pacman, mocker: MockerFixture) -> None:
"""
must do not copy database from root if local cache is disabled
"""
@ -80,11 +80,11 @@ def test_database_copy_skip(pacman: Pacman, repository_paths: RepositoryPaths, m
mocker.patch("pathlib.Path.is_file", autospec=True, side_effect=lambda p: p.is_relative_to(path))
copy_mock = mocker.patch("shutil.copy")
pacman.database_copy(pacman.handle, database, path, repository_paths, use_ahriman_cache=False)
pacman.database_copy(pacman.handle, database, path, use_ahriman_cache=False)
copy_mock.assert_not_called()
def test_database_copy_no_directory(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
def test_database_copy_no_directory(pacman: Pacman, mocker: MockerFixture) -> None:
"""
must do not copy database if local cache already exists
"""
@ -95,11 +95,11 @@ def test_database_copy_no_directory(pacman: Pacman, repository_paths: Repository
mocker.patch("pathlib.Path.is_file", autospec=True, side_effect=lambda p: p.is_relative_to(path))
copy_mock = mocker.patch("shutil.copy")
pacman.database_copy(pacman.handle, database, path, repository_paths, use_ahriman_cache=True)
pacman.database_copy(pacman.handle, database, path, use_ahriman_cache=True)
copy_mock.assert_not_called()
def test_database_copy_no_root_file(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
def test_database_copy_no_root_file(pacman: Pacman, mocker: MockerFixture) -> None:
"""
must do not copy database if no repository file exists in filesystem
"""
@ -110,11 +110,11 @@ def test_database_copy_no_root_file(pacman: Pacman, repository_paths: Repository
mocker.patch("pathlib.Path.is_file", return_value=False)
copy_mock = mocker.patch("shutil.copy")
pacman.database_copy(pacman.handle, database, path, repository_paths, use_ahriman_cache=True)
pacman.database_copy(pacman.handle, database, path, use_ahriman_cache=True)
copy_mock.assert_not_called()
def test_database_copy_database_exist(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
def test_database_copy_database_exist(pacman: Pacman, mocker: MockerFixture) -> None:
"""
must do not copy database if local cache already exists
"""
@ -124,7 +124,7 @@ def test_database_copy_database_exist(pacman: Pacman, repository_paths: Reposito
mocker.patch("pathlib.Path.is_file", return_value=True)
copy_mock = mocker.patch("shutil.copy")
pacman.database_copy(pacman.handle, database, Path("root"), repository_paths, use_ahriman_cache=True)
pacman.database_copy(pacman.handle, database, Path("root"), use_ahriman_cache=True)
copy_mock.assert_not_called()
@ -136,6 +136,15 @@ def test_database_init(pacman: Pacman, configuration: Configuration) -> None:
assert database.servers == ["https://geo.mirror.pkgbuild.com/testing/os/x86_64"]
def test_database_init_local(pacman: Pacman, configuration: Configuration) -> None:
"""
must set file protocol for local databases
"""
_, repository_id = configuration.check_loaded()
database = pacman.database_init(MagicMock(), repository_id.name, repository_id.architecture)
assert database.servers == [f"file://{configuration.repository_paths.repository}"]
def test_database_sync(pacman: Pacman, mocker: MockerFixture) -> None:
"""
must sync databases

29
tests/ahriman/core/alpm/test_pacman_database.py
View File

@ -8,6 +8,15 @@ from ahriman.core.alpm.pacman_database import PacmanDatabase
from ahriman.core.exceptions import PacmanError
def test_copy(pacman_database: PacmanDatabase, mocker: MockerFixture) -> None:
"""
must copy loca database file
"""
copy_mock = mocker.patch("shutil.copy")
pacman_database.copy(Path("remote"), Path("local"))
copy_mock.assert_called_once_with(Path("remote"), Path("local"))
def test_download(pacman_database: PacmanDatabase, mocker: MockerFixture) -> None:
"""
must download database by remote url
@ -163,6 +172,26 @@ def test_sync_files_force(pacman_database: PacmanDatabase, mocker: MockerFixture
"https://geo.mirror.pkgbuild.com/core/os/x86_64/core.files.tar.gz", pytest.helpers.anyvar(int))
def test_sync_files_local(pacman_database: PacmanDatabase, mocker: MockerFixture) -> None:
"""
must copy local files instead of downloading them
"""
pacman_database.database.servers = ["file:///var"]
copy_mock = mocker.patch("ahriman.core.alpm.pacman_database.PacmanDatabase.copy")
pacman_database.sync_files(force=False)
copy_mock.assert_called_once_with(Path("/var/core.files.tar.gz"), pytest.helpers.anyvar(int))
def test_sync_files_unknown_source(pacman_database: PacmanDatabase) -> None:
"""
must raise an exception in case if server scheme is unsupported
"""
pacman_database.database.servers = ["some random string"]
with pytest.raises(PacmanError):
pacman_database.sync_files(force=False)
def test_sync_packages(pacman_database: PacmanDatabase) -> None:
"""
must sync packages by using pyalpm method