add ability to use ahriman pacman database instead of system one (#71)

By default this feature is enabled. On the first run it will copy (if
exists) databases from filesystem to local cache (one per each
architecture). Later it will use this cache for all alpm operations. In
order to update this cache, some commands (mainly package building)
provide `-y`/`--refresh` option which has same semantics as pacman -Sy
does.

Note however that due to extending `Pacman` class some methods were
renamed in order to be more descriptive:
* `Pacman.all_packages` -> `Pacman.packages`
* `Pacman.get` -> `Pacman.package_get`

This commit also adds multilib repository to the default docker image
which was missed.

tests/ahriman/application/handlers/test_handler_add.py

@@ -23,6 +23,7 @@ def _default_args(args: argparse.Namespace) -> argparse.Namespace:
     args.package = []
     args.exit_code = False
     args.now = False
+    args.refresh = 0
     args.source = PackageSource.Auto
     args.without_dependencies = False
     return args

tests/ahriman/application/handlers/test_handler_update.py

@@ -28,6 +28,7 @@ def _default_args(args: argparse.Namespace) -> argparse.Namespace:
     args.no_local = False
     args.no_manual = False
     args.no_vcs = False
+    args.refresh = 0
     return args
 
 

tests/ahriman/application/test_ahriman.py

@@ -75,6 +75,18 @@ def test_subparsers_daemon(parser: argparse.ArgumentParser) -> None:
     assert args.package == []
 
 
+def test_subparsers_daemon_option_refresh(parser: argparse.ArgumentParser) -> None:
+    """
+    daemon command must count refresh options
+    """
+    args = parser.parse_args(["daemon"])
+    assert args.refresh == 0
+    args = parser.parse_args(["daemon", "-y"])
+    assert args.refresh == 1
+    args = parser.parse_args(["daemon", "-yy"])
+    assert args.refresh == 2
+
+
 def test_subparsers_daemon_option_interval(parser: argparse.ArgumentParser) -> None:
     """
     daemon command must convert interval option to int instance
@@ -155,6 +167,18 @@ def test_subparsers_package_add_architecture(parser: argparse.ArgumentParser) ->
     assert args.architecture == ["x86_64"]
 
 
+def test_subparsers_package_add_option_refresh(parser: argparse.ArgumentParser) -> None:
+    """
+    package-add command must count refresh options
+    """
+    args = parser.parse_args(["package-add", "ahriman"])
+    assert args.refresh == 0
+    args = parser.parse_args(["package-add", "ahriman", "-y"])
+    assert args.refresh == 1
+    args = parser.parse_args(["package-add", "ahriman", "-yy"])
+    assert args.refresh == 2
+
+
 def test_subparsers_package_remove_architecture(parser: argparse.ArgumentParser) -> None:
     """
     package-remove command must correctly parse architecture list
@@ -345,6 +369,18 @@ def test_subparsers_repo_check_architecture(parser: argparse.ArgumentParser) ->
     assert args.architecture == ["x86_64"]
 
 
+def test_subparsers_repo_check_option_refresh(parser: argparse.ArgumentParser) -> None:
+    """
+    repo-check command must count refresh options
+    """
+    args = parser.parse_args(["repo-check"])
+    assert args.refresh == 0
+    args = parser.parse_args(["repo-check", "-y"])
+    assert args.refresh == 1
+    args = parser.parse_args(["repo-check", "-yy"])
+    assert args.refresh == 2
+
+
 def test_subparsers_repo_clean(parser: argparse.ArgumentParser) -> None:
     """
     repo-clean command must imply quiet and unsafe
@@ -540,6 +576,18 @@ def test_subparsers_repo_update_architecture(parser: argparse.ArgumentParser) ->
     assert args.architecture == ["x86_64"]
 
 
+def test_subparsers_repo_update_option_refresh(parser: argparse.ArgumentParser) -> None:
+    """
+    repo-update command must count refresh options
+    """
+    args = parser.parse_args(["repo-update"])
+    assert args.refresh == 0
+    args = parser.parse_args(["repo-update", "-y"])
+    assert args.refresh == 1
+    args = parser.parse_args(["repo-update", "-yy"])
+    assert args.refresh == 2
+
+
 def test_subparsers_shell(parser: argparse.ArgumentParser) -> None:
     """
     shell command must imply lock and no-report

tests/ahriman/conftest.py

@@ -360,7 +360,7 @@ def pacman(configuration: Configuration) -> Pacman:
     Returns:
         Pacman: pacman wrapper test instance
     """
-    return Pacman(configuration)
+    return Pacman("x86_64", configuration, refresh_database=0)
 
 
 @pytest.fixture

tests/ahriman/core/alpm/remote/test_official_syncdb.py

@@ -11,7 +11,7 @@ def test_package_info(official_syncdb: OfficialSyncdb, aur_package_akonadi: AURP
     must return package info from the database
     """
     mocker.patch("ahriman.models.aur_package.AURPackage.from_pacman", return_value=aur_package_akonadi)
-    get_mock = mocker.patch("ahriman.core.alpm.pacman.Pacman.get", return_value=[aur_package_akonadi])
+    get_mock = mocker.patch("ahriman.core.alpm.pacman.Pacman.package_get", return_value=[aur_package_akonadi])
 
     package = official_syncdb.package_info(aur_package_akonadi.name, pacman=pacman)
     get_mock.assert_called_once_with(aur_package_akonadi.name)

tests/ahriman/core/alpm/test_pacman.py

@@ -1,31 +1,205 @@
+from pathlib import Path
+from pyalpm import error as PyalpmError
+from pytest_mock import MockerFixture
+from tempfile import TemporaryDirectory
+from unittest.mock import MagicMock
+
 from ahriman.core.alpm.pacman import Pacman
+from ahriman.core.configuration import Configuration
+from ahriman.models.repository_paths import RepositoryPaths
 
 
-def test_all_packages(pacman: Pacman) -> None:
+def test_init_with_local_cache(configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must sync repositories at the start if set
+    """
+    mocker.patch("ahriman.core.alpm.pacman.Pacman.database_copy")
+    sync_mock = mocker.patch("ahriman.core.alpm.pacman.Pacman.database_sync")
+    configuration.set_option("alpm", "use_ahriman_cache", "yes")
+
+    # pyalpm.Handle is trying to reach the directory we've asked, thus we need to patch it a bit
+    with TemporaryDirectory(ignore_cleanup_errors=True) as pacman_root:
+        mocker.patch.object(RepositoryPaths, "pacman", Path(pacman_root))
+        # during the creation pyalpm.Handle will create also version file which we would like to remove later
+        Pacman("x86_64", configuration, refresh_database=1)
+        sync_mock.assert_called_once_with(False)
+
+
+def test_init_with_local_cache_forced(configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must sync repositories at the start if set with force flag
+    """
+    mocker.patch("ahriman.core.alpm.pacman.Pacman.database_copy")
+    sync_mock = mocker.patch("ahriman.core.alpm.pacman.Pacman.database_sync")
+    configuration.set_option("alpm", "use_ahriman_cache", "yes")
+
+    # pyalpm.Handle is trying to reach the directory we've asked, thus we need to patch it a bit
+    with TemporaryDirectory(ignore_cleanup_errors=True) as pacman_root:
+        mocker.patch.object(RepositoryPaths, "pacman", Path(pacman_root))
+        # during the creation pyalpm.Handle will create also version file which we would like to remove later
+        Pacman("x86_64", configuration, refresh_database=2)
+        sync_mock.assert_called_once_with(True)
+
+
+def test_database_copy(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
+    """
+    must copy database from root
+    """
+    database = next(db for db in pacman.handle.get_syncdbs() if db.name == "core")
+    path = Path("randomname")
+    dst_path = Path("/var/lib/pacman/sync/core.db")
+    mocker.patch("pathlib.Path.is_dir", return_value=True)
+    # root database exists, local database does not
+    mocker.patch("pathlib.Path.is_file", autospec=True, side_effect=lambda p: True if p.is_relative_to(path) else False)
+    copy_mock = mocker.patch("shutil.copy")
+    chown_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.chown")
+
+    pacman.database_copy(database, path, repository_paths, use_ahriman_cache=True)
+    copy_mock.assert_called_once_with(path / "sync" / "core.db", dst_path)
+    chown_mock.assert_called_once_with(dst_path)
+
+
+def test_database_copy_skip(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
+    """
+    must do not copy database from root if local cache is disabled
+    """
+    database = next(db for db in pacman.handle.get_syncdbs() if db.name == "core")
+    path = Path("randomname")
+    mocker.patch("pathlib.Path.is_dir", return_value=True)
+    # root database exists, local database does not
+    mocker.patch("pathlib.Path.is_file", autospec=True, side_effect=lambda p: True if p.is_relative_to(path) else False)
+    copy_mock = mocker.patch("shutil.copy")
+
+    pacman.database_copy(database, path, repository_paths, use_ahriman_cache=False)
+    copy_mock.assert_not_called()
+
+
+def test_database_copy_no_directory(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
+    """
+    must do not copy database if local cache already exists
+    """
+    database = next(db for db in pacman.handle.get_syncdbs() if db.name == "core")
+    path = Path("randomname")
+    mocker.patch("pathlib.Path.is_dir", return_value=False)
+    # root database exists, local database does not
+    mocker.patch("pathlib.Path.is_file", autospec=True, side_effect=lambda p: True if p.is_relative_to(path) else False)
+    copy_mock = mocker.patch("shutil.copy")
+
+    pacman.database_copy(database, path, repository_paths, use_ahriman_cache=True)
+    copy_mock.assert_not_called()
+
+
+def test_database_copy_no_root_file(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
+    """
+    must do not copy database if no repository file exists in filesystem
+    """
+    database = next(db for db in pacman.handle.get_syncdbs() if db.name == "core")
+    path = Path("randomname")
+    mocker.patch("pathlib.Path.is_dir", return_value=True)
+    # root database does not exist, local database does not either
+    mocker.patch("pathlib.Path.is_file", return_value=False)
+    copy_mock = mocker.patch("shutil.copy")
+
+    pacman.database_copy(database, path, repository_paths, use_ahriman_cache=True)
+    copy_mock.assert_not_called()
+
+
+def test_database_copy_database_exist(pacman: Pacman, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
+    """
+    must do not copy database if local cache already exists
+    """
+    database = next(db for db in pacman.handle.get_syncdbs() if db.name == "core")
+    mocker.patch("pathlib.Path.is_dir", return_value=True)
+    # root database exists, local database does either
+    mocker.patch("pathlib.Path.is_file", return_value=True)
+    copy_mock = mocker.patch("shutil.copy")
+
+    pacman.database_copy(database, Path("root"), repository_paths, use_ahriman_cache=True)
+    copy_mock.assert_not_called()
+
+
+def test_database_init(pacman: Pacman, configuration: Configuration) -> None:
+    """
+    must init database with settings
+    """
+    mirror = configuration.get("alpm", "mirror")
+    database = pacman.database_init("test", mirror, "x86_64")
+    assert len(database.servers) == 1
+
+
+def test_database_sync(pacman: Pacman) -> None:
+    """
+    must sync databases
+    """
+    handle_mock = MagicMock()
+    core_mock = MagicMock()
+    extra_mock = MagicMock()
+    transaction_mock = MagicMock()
+    handle_mock.get_syncdbs.return_value = [core_mock, extra_mock]
+    handle_mock.init_transaction.return_value = transaction_mock
+    pacman.handle = handle_mock
+
+    pacman.database_sync(False)
+    handle_mock.init_transaction.assert_called_once_with()
+    core_mock.update.assert_called_once_with(False)
+    extra_mock.update.assert_called_once_with(False)
+    transaction_mock.release.assert_called_once_with()
+
+
+def test_database_sync_failed(pacman: Pacman) -> None:
+    """
+    must sync databases even if there was exception
+    """
+    handle_mock = MagicMock()
+    core_mock = MagicMock()
+    core_mock.update.side_effect = PyalpmError()
+    extra_mock = MagicMock()
+    handle_mock.get_syncdbs.return_value = [core_mock, extra_mock]
+    pacman.handle = handle_mock
+
+    pacman.database_sync(False)
+    extra_mock.update.assert_called_once_with(False)
+
+
+def test_database_sync_forced(pacman: Pacman) -> None:
+    """
+    must sync databases with force flag
+    """
+    handle_mock = MagicMock()
+    core_mock = MagicMock()
+    handle_mock.get_syncdbs.return_value = [core_mock]
+    pacman.handle = handle_mock
+
+    pacman.database_sync(True)
+    handle_mock.init_transaction.assert_called_once_with()
+    core_mock.update.assert_called_once_with(True)
+
+
+def test_package_get(pacman: Pacman) -> None:
+    """
+    must retrieve package
+    """
+    assert list(pacman.package_get("pacman"))
+
+
+def test_package_get_empty(pacman: Pacman) -> None:
+    """
+    must return empty packages list without exception
+    """
+    assert not list(pacman.package_get("some-random-name"))
+
+
+def test_packages(pacman: Pacman) -> None:
     """
     package list must not be empty
     """
-    packages = pacman.all_packages()
+    packages = pacman.packages()
     assert packages
     assert "pacman" in packages
 
 
-def test_all_packages_with_provides(pacman: Pacman) -> None:
+def test_packages_with_provides(pacman: Pacman) -> None:
     """
     package list must contain provides packages
     """
-    assert "sh" in pacman.all_packages()
-
-
-def test_get(pacman: Pacman) -> None:
-    """
-    must retrieve package
-    """
-    assert list(pacman.get("pacman"))
-
-
-def test_get_empty(pacman: Pacman) -> None:
-    """
-    must return empty packages list without exception
-    """
-    assert not list(pacman.get("some-random-name"))
+    assert "sh" in pacman.packages()

tests/ahriman/models/test_repository_paths.py

@@ -154,9 +154,5 @@ def test_tree_create(repository_paths: RepositoryPaths, mocker: MockerFixture) -
     chown_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.chown")
 
     repository_paths.tree_create()
-    mkdir_mock.assert_has_calls(
-        [
-            mock.call(mode=0o755, parents=True, exist_ok=True)
-            for _ in paths
-        ], any_order=True)
-    chown_mock.assert_has_calls([mock.call(getattr(repository_paths, path)) for path in paths], any_order=True)
+    mkdir_mock.assert_has_calls([mock.call(mode=0o755, parents=True, exist_ok=True) for _ in paths], any_order=True)
+    chown_mock.assert_has_calls([mock.call(pytest.helpers.anyvar(int)) for _ in paths], any_order=True)

tests/testresources/core/ahriman.ini

@@ -5,8 +5,10 @@ database = ../../../ahriman-test.db
 
 [alpm]
 database = /var/lib/pacman
+mirror = https://geo.mirror.pkgbuild.com/$repo/os/$arch
 repositories = core extra community multilib
 root = /
+use_ahriman_cache = no
 
 [auth]
 client_id = client_id