feat: optimize archive reading

Instead of trying to load every database and look for files, this commit
introduces the optimization in which, the service loads packages first,
groups them by database and load files later.

In some cases it significantly descreases times for loading files

tests/ahriman/conftest.py

@@ -4,7 +4,7 @@ import pytest
 from pathlib import Path
 from pytest_mock import MockerFixture
 from typing import Any, TypeVar
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, PropertyMock
 
 from ahriman.core.alpm.pacman import Pacman
 from ahriman.core.alpm.remote import AUR
@@ -476,6 +476,41 @@ def passwd() -> MagicMock:
     return passwd
 
 
+@pytest.fixture
+def pyalpm_package_ahriman(aur_package_ahriman: AURPackage) -> MagicMock:
+    """
+    mock object for pyalpm package
+
+    Args:
+        aur_package_ahriman(AURPackage): package fixture
+
+    Returns:
+        MagicMock: pyalpm package mock
+    """
+    mock = MagicMock()
+    db = type(mock).db = MagicMock()
+
+    type(mock).base = PropertyMock(return_value=aur_package_ahriman.package_base)
+    type(mock).builddate = PropertyMock(
+        return_value=aur_package_ahriman.last_modified.replace(tzinfo=datetime.timezone.utc).timestamp())
+    type(mock).conflicts = PropertyMock(return_value=aur_package_ahriman.conflicts)
+    type(db).name = PropertyMock(return_value="aur")
+    type(mock).depends = PropertyMock(return_value=aur_package_ahriman.depends)
+    type(mock).desc = PropertyMock(return_value=aur_package_ahriman.description)
+    type(mock).licenses = PropertyMock(return_value=aur_package_ahriman.license)
+    type(mock).makedepends = PropertyMock(return_value=aur_package_ahriman.make_depends)
+    type(mock).name = PropertyMock(return_value=aur_package_ahriman.name)
+    type(mock).optdepends = PropertyMock(return_value=aur_package_ahriman.opt_depends)
+    type(mock).checkdepends = PropertyMock(return_value=aur_package_ahriman.check_depends)
+    type(mock).packager = PropertyMock(return_value="packager")
+    type(mock).provides = PropertyMock(return_value=aur_package_ahriman.provides)
+    type(mock).version = PropertyMock(return_value=aur_package_ahriman.version)
+    type(mock).url = PropertyMock(return_value=aur_package_ahriman.url)
+    type(mock).groups = PropertyMock(return_value=aur_package_ahriman.groups)
+
+    return mock
+
+
 @pytest.fixture
 def remote_source() -> RemoteSource:
     """

tests/ahriman/core/alpm/test_pacman.py

@@ -1,3 +1,4 @@
+import pyalpm
 import pytest
 import tarfile
 
@@ -175,31 +176,12 @@ def test_database_sync_forced(pacman: Pacman, mocker: MockerFixture) -> None:
     sync_mock.assert_called_once_with(force=True)
 
 
-def test_files(pacman: Pacman, package_ahriman: Package, mocker: MockerFixture, resource_path_root: Path) -> None:
-    """
-    must load files from databases
-    """
-    handle_mock = MagicMock()
-    handle_mock.get_syncdbs.return_value = [MagicMock()]
-    pacman.handle = handle_mock
-    tarball = resource_path_root / "core" / "arcanisrepo.files.tar.gz"
-
-    with tarfile.open(tarball, "r:gz") as fd:
-        mocker.patch("pathlib.Path.is_file", return_value=True)
-        open_mock = mocker.patch("ahriman.core.alpm.pacman.tarfile.open", return_value=fd)
-
-        files = pacman.files()
-        assert len(files) == 2
-        assert package_ahriman.base in files
-        assert "usr/bin/ahriman" in files[package_ahriman.base]
-        open_mock.assert_called_once_with(pytest.helpers.anyvar(int), "r:gz")
-
-
-def test_files_package(pacman: Pacman, package_ahriman: Package, mocker: MockerFixture,
-                       resource_path_root: Path) -> None:
+def test_files_package(pacman: Pacman, package_ahriman: Package, pyalpm_package_ahriman: pyalpm.Package,
+                       mocker: MockerFixture, resource_path_root: Path) -> None:
     """
     must load files only for the specified package
     """
+    mocker.patch("ahriman.core.alpm.pacman.Pacman.package", return_value=[pyalpm_package_ahriman])
     handle_mock = MagicMock()
     handle_mock.get_syncdbs.return_value = [MagicMock()]
     pacman.handle = handle_mock
@@ -210,34 +192,35 @@ def test_files_package(pacman: Pacman, package_ahriman: Package, mocker: MockerF
         mocker.patch("pathlib.Path.is_file", return_value=True)
         mocker.patch("ahriman.core.alpm.pacman.tarfile.open", return_value=fd)
 
-        files = pacman.files(package_ahriman.base)
+        files = pacman.files([package_ahriman.base])
         assert len(files) == 1
         assert package_ahriman.base in files
 
 
-def test_files_skip(pacman: Pacman, mocker: MockerFixture) -> None:
+def test_files_skip(pacman: Pacman, pyalpm_package_ahriman: pyalpm.Package, mocker: MockerFixture) -> None:
     """
     must return empty list if no database found
     """
+    mocker.patch("ahriman.core.alpm.pacman.Pacman.package", return_value=[pyalpm_package_ahriman])
     handle_mock = MagicMock()
     handle_mock.get_syncdbs.return_value = [MagicMock()]
     pacman.handle = handle_mock
 
     mocker.patch("pathlib.Path.is_file", return_value=False)
 
-    assert not pacman.files()
+    assert not pacman.files([pyalpm_package_ahriman.name])
 
 
-def test_files_no_content(pacman: Pacman, mocker: MockerFixture) -> None:
+def test_files_no_content(pacman: Pacman, pyalpm_package_ahriman: pyalpm.Package, mocker: MockerFixture) -> None:
     """
     must skip package if no content can be loaded
     """
+    mocker.patch("ahriman.core.alpm.pacman.Pacman.package", return_value=[pyalpm_package_ahriman])
     handle_mock = MagicMock()
     handle_mock.get_syncdbs.return_value = [MagicMock()]
     pacman.handle = handle_mock
 
     tar_mock = MagicMock()
-    tar_mock.getmembers.return_value = [MagicMock()]
     tar_mock.extractfile.return_value = None
 
     open_mock = MagicMock()
@@ -246,7 +229,28 @@ def test_files_no_content(pacman: Pacman, mocker: MockerFixture) -> None:
     mocker.patch("pathlib.Path.is_file", return_value=True)
     mocker.patch("ahriman.core.alpm.pacman.tarfile.open", return_value=open_mock)
 
-    assert not pacman.files()
+    assert not pacman.files([pyalpm_package_ahriman.name])
+
+
+def test_files_no_entry(pacman: Pacman, pyalpm_package_ahriman: pyalpm.Package, mocker: MockerFixture) -> None:
+    """
+    must skip package if it wasn't found in the archive
+    """
+    mocker.patch("ahriman.core.alpm.pacman.Pacman.package", return_value=[pyalpm_package_ahriman])
+    handle_mock = MagicMock()
+    handle_mock.get_syncdbs.return_value = [MagicMock()]
+    pacman.handle = handle_mock
+
+    tar_mock = MagicMock()
+    tar_mock.extractfile.side_effect = KeyError()
+
+    open_mock = MagicMock()
+    open_mock.__enter__.return_value = tar_mock
+
+    mocker.patch("pathlib.Path.is_file", return_value=True)
+    mocker.patch("ahriman.core.alpm.pacman.tarfile.open", return_value=open_mock)
+
+    assert not pacman.files([pyalpm_package_ahriman.name])
 
 
 def test_package(pacman: Pacman) -> None:

tests/ahriman/models/conftest.py

@@ -1,4 +1,3 @@
-import datetime
 import pytest
 
 from typing import Any
@@ -8,7 +7,6 @@ from pytest_mock import MockerFixture
 from ahriman import __version__
 from ahriman.core.alpm.pacman import Pacman
 from ahriman.core.alpm.remote import AUR
-from ahriman.models.aur_package import AURPackage
 from ahriman.models.build_status import BuildStatus, BuildStatusEnum
 from ahriman.models.counters import Counters
 from ahriman.models.filesystem_package import FilesystemPackage
@@ -134,41 +132,6 @@ def pyalpm_handle(pyalpm_package_ahriman: MagicMock) -> MagicMock:
     return mock
 
 
-@pytest.fixture
-def pyalpm_package_ahriman(aur_package_ahriman: AURPackage) -> MagicMock:
-    """
-    mock object for pyalpm package
-
-    Args:
-        aur_package_ahriman(AURPackage): package fixture
-
-    Returns:
-        MagicMock: pyalpm package mock
-    """
-    mock = MagicMock()
-    db = type(mock).db = MagicMock()
-
-    type(mock).base = PropertyMock(return_value=aur_package_ahriman.package_base)
-    type(mock).builddate = PropertyMock(
-        return_value=aur_package_ahriman.last_modified.replace(tzinfo=datetime.timezone.utc).timestamp())
-    type(mock).conflicts = PropertyMock(return_value=aur_package_ahriman.conflicts)
-    type(db).name = PropertyMock(return_value="aur")
-    type(mock).depends = PropertyMock(return_value=aur_package_ahriman.depends)
-    type(mock).desc = PropertyMock(return_value=aur_package_ahriman.description)
-    type(mock).licenses = PropertyMock(return_value=aur_package_ahriman.license)
-    type(mock).makedepends = PropertyMock(return_value=aur_package_ahriman.make_depends)
-    type(mock).name = PropertyMock(return_value=aur_package_ahriman.name)
-    type(mock).optdepends = PropertyMock(return_value=aur_package_ahriman.opt_depends)
-    type(mock).checkdepends = PropertyMock(return_value=aur_package_ahriman.check_depends)
-    type(mock).packager = PropertyMock(return_value="packager")
-    type(mock).provides = PropertyMock(return_value=aur_package_ahriman.provides)
-    type(mock).version = PropertyMock(return_value=aur_package_ahriman.version)
-    type(mock).url = PropertyMock(return_value=aur_package_ahriman.url)
-    type(mock).groups = PropertyMock(return_value=aur_package_ahriman.groups)
-
-    return mock
-
-
 @pytest.fixture
 def pyalpm_package_description_ahriman(package_description_ahriman: PackageDescription) -> MagicMock:
     """