remove excess dependencies leaves

Since requests-2.32.0, the http+unix url scheme is brokek, check
https://github.com/msabramo/requests-unixsocket/issues/73 for more
details

Dockerfile

@@ -34,9 +34,9 @@ COPY "docker/install-aur-package.sh" "/usr/local/bin/install-aur-package"
 ## darcs is not installed by reasons, because it requires a lot haskell packages which dramatically increase image size
 RUN pacman -Sy --noconfirm --asdeps devtools git pyalpm python-cerberus python-inflection python-passlib python-pyelftools python-requests python-srcinfo && \
     pacman -Sy --noconfirm --asdeps base-devel python-build python-flit python-installer python-wheel && \
-    pacman -Sy --noconfirm --asdeps breezy git mercurial python-aiohttp python-boto3 python-cryptography python-jinja python-requests-unixsocket python-systemd rsync subversion && \
+    pacman -Sy --noconfirm --asdeps breezy git mercurial python-aiohttp python-boto3 python-cryptography python-jinja python-systemd rsync subversion && \
     runuser -u build -- install-aur-package python-aioauth-client python-webargs python-aiohttp-apispec-git python-aiohttp-cors \
-                                            python-aiohttp-jinja2 python-aiohttp-session python-aiohttp-security
+                                            python-aiohttp-jinja2 python-aiohttp-session python-aiohttp-security python-requests-unixsocket2
 
 ## FIXME since 1.0.4 devtools requires dbus to be run, which doesn't work now in container
 COPY "docker/systemd-nspawn.sh" "/usr/local/bin/systemd-nspawn"

docs/architecture.rst

@@ -366,7 +366,7 @@ Web application requires the following python packages to be installed:
 * Additional web features also require ``aiohttp-apispec`` (autogenerated documentation), ``aiohttp_cors`` (CORS support, required by documentation).
 * In addition, authorization feature requires ``aiohttp_security``, ``aiohttp_session`` and ``cryptography``.
 * In addition to base authorization dependencies, OAuth2 also requires ``aioauth-client`` library.
-* In addition if you would like to disable authorization for local access (recommended way in order to run the application itself with reporting support), the ``requests-unixsocket`` library is required.
+* In addition if you would like to disable authorization for local access (recommended way in order to run the application itself with reporting support), the ``requests-unixsocket2`` library is required.
 
 Middlewares
 ^^^^^^^^^^^

docs/faq.rst

@@ -1313,7 +1313,7 @@ How to enable basic authorization
    The ``salt`` parameter is optional, but recommended, and can be set to any (random) string.
 
 #.
-   In order to provide access for reporting from application instances you can (the recommended way) use unix sockets by the following configuration (note, that it requires ``python-requests-unixsocket`` package to be installed):
+   In order to provide access for reporting from application instances you can (the recommended way) use unix sockets by the following configuration (note, that it requires ``python-requests-unixsocket2`` package to be installed):
 
    .. code-block:: ini
 

package/archlinux/PKGBUILD

@@ -21,7 +21,7 @@ optdepends=('breezy: -bzr packages support'
             'python-aiohttp-session: web server with authorization'
             'python-boto3: sync to s3'
             'python-cryptography: web server with authorization'
-            'python-requests-unixsocket: client report to web server by unix socket'
+            'python-requests-unixsocket2: client report to web server by unix socket'
             'python-jinja: html report generation'
             'python-systemd: journal support'
             'rsync: sync by using rsync'

pyproject.toml

@@ -81,7 +81,7 @@ web = [
     "aiohttp_session",
     "aiohttp_security",
     "cryptography",
-    "requests-unixsocket",  # required by unix socket support
+    "requests-unixsocket2",  # required by unix socket support
     "setuptools",  # required by aiohttp-apispec
 ]
 

src/ahriman/core/repository/executor.py

@@ -80,7 +80,7 @@ class Executor(PackageInfo, Cleaner):
                     # clear changes and update commit hash
                     self.reporter.package_changes_update(single.base, Changes(last_commit_sha))
                     # update dependencies list
-                    dependencies = PackageArchive(self.paths.build_directory, single).depends_on()
+                    dependencies = PackageArchive(self.paths.build_directory, single, self.pacman).depends_on()
                     self.reporter.package_dependencies_update(single.base, dependencies)
                     # update result set
                     result.add_updated(single)

src/ahriman/models/aur_package.py

@@ -57,6 +57,7 @@ class AURPackage:
         provides(list[str]): list of packages which this package provides
         license(list[str]): list of package licenses
         keywords(list[str]): list of package keywords
+        groups(list[str]): list of package groups
 
     Examples:
         Mainly this class must be used from class methods instead of default :func:`__init__()`::
@@ -100,6 +101,7 @@ class AURPackage:
     provides: list[str] = field(default_factory=list)
     license: list[str] = field(default_factory=list)
     keywords: list[str] = field(default_factory=list)
+    groups: list[str] = field(default_factory=list)
 
     @classmethod
     def from_json(cls, dump: dict[str, Any]) -> Self:
@@ -153,6 +155,7 @@ class AURPackage:
             provides=package.provides,
             license=package.licenses,
             keywords=[],
+            groups=package.groups,
         )
 
     @classmethod
@@ -191,6 +194,7 @@ class AURPackage:
             provides=dump["provides"],
             license=dump["licenses"],
             keywords=[],
+            groups=dump["groups"],
         )
 
     @staticmethod

src/ahriman/models/dependencies.py

@@ -34,6 +34,13 @@ class Dependencies:
 
     paths: dict[str, list[str]] = field(default_factory=dict)
 
+    def __post_init__(self) -> None:
+        """
+        remove empty paths
+        """
+        paths = {path: packages for path, packages in self.paths.items() if packages}
+        object.__setattr__(self, "paths", paths)
+
     @classmethod
     def from_json(cls, dump: dict[str, Any]) -> Self:
         """

src/ahriman/models/filesystem_package.py

@@ -0,0 +1,55 @@
+#
+# Copyright (c) 2021-2024 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass(frozen=True, kw_only=True)
+class FilesystemPackage:
+    """
+    class representing a simplified model for the package installed to filesystem
+
+    Attributes:
+        package_name(str): package name
+        dependencies(list[str]): list of package dependencies
+        directories(list[Path]): list of directories this package contains
+        files(list[Path]): list of files this package contains
+        groups(list[str]): list of groups of the package
+    """
+
+    package_name: str
+    groups: set[str]
+    dependencies: set[str]
+    directories: list[Path] = field(default_factory=list)
+    files: list[Path] = field(default_factory=list)
+
+    @property
+    def is_valid(self) -> bool:
+        """
+        quick check if this package must be used for the dependencies calculation. It checks that
+        1) package is not in the base group
+
+        Returns:
+            bool: True in case if this package must be used for the dependencies calculation or False otherwise
+        """
+        return "base" not in self.groups
+
+    def __repr__(self):
+        return f'FilesystemPackage(package_name="{self.package_name}", dependencies={self.dependencies})'

src/ahriman/models/package_archive.py

@@ -23,8 +23,12 @@ from elftools.elf.elffile import ELFFile
 from pathlib import Path
 from typing import IO
 
+from ahriman.core.alpm.pacman import Pacman
+from ahriman.core.alpm.remote import OfficialSyncdb
+from ahriman.core.exceptions import UnknownPackageError
 from ahriman.core.util import walk
 from ahriman.models.dependencies import Dependencies
+from ahriman.models.filesystem_package import FilesystemPackage
 from ahriman.models.package import Package
 
 
@@ -40,6 +44,7 @@ class PackageArchive:
 
     root: Path
     package: Package
+    pacman: Pacman
 
     @staticmethod
     def dynamic_needed(binary_path: Path) -> list[str]:
@@ -90,6 +95,27 @@ class PackageArchive:
 
         return magic_bytes == expected
 
+    def _load_pacman_package(self, path: Path) -> FilesystemPackage:
+        """
+        load pacman package model from path
+
+        Args:
+            path(Path): path to package files database
+
+        Returns:
+            FilesystemPackage: generated pacman package model with empty paths
+        """
+        package_name, *_ = path.parent.name.rsplit("-", 2)
+        try:
+            pacman_package = OfficialSyncdb.info(package_name, pacman=self.pacman)
+            return FilesystemPackage(
+                package_name=package_name,
+                groups=set(pacman_package.groups),
+                dependencies=set(pacman_package.depends),
+            )
+        except UnknownPackageError:
+            return FilesystemPackage(package_name=package_name, groups=set(), dependencies=set())
+
     def depends_on(self) -> Dependencies:
         """
         extract packages and paths which are required for this package
@@ -98,17 +124,35 @@ class PackageArchive:
             Dependencies: map of the package name to set of paths used by this package
         """
         dependencies, roots = self.depends_on_paths()
+        installed_packages = self.installed_packages()
 
-        result: dict[str, list[str]] = {}
+        dependencies_per_path: dict[Path, list[FilesystemPackage]] = {}
-        for package, (directories, files) in self.installed_packages().items():
+        for package_base, package in installed_packages.items():
-            if package in self.package.packages:
+            if package_base in self.package.packages:
                 continue  # skip package itself
 
-            required_by = [directory for directory in directories if directory in roots]
+            required_by = [directory for directory in package.directories if directory in roots]
-            required_by.extend(library for library in files if library.name in dependencies)
+            required_by.extend(library for library in package.files if library.name in dependencies)
 
             for path in required_by:
-                result.setdefault(str(path), []).append(package)
+                dependencies_per_path.setdefault(path, []).append(package)
+
+        # reduce trees
+        result = {}
+        for path, packages in dependencies_per_path.items():
+            package_names = [package.package_name for package in packages]
+            result[str(path)] = [
+                package.package_name
+                for package in packages
+                # if there is any package which is dependency of this package, we can skip it here
+                # also skip packages which didn't pass validation
+                if not package.dependencies.intersection(package_names) and package.is_valid
+            ]
+
+            if str(path) == 'usr/lib/python3.12/site-packages':
+                print(package_names)
+                print(packages)
+                print(result[str(path)])
 
         return Dependencies(result)
 
@@ -130,7 +174,7 @@ class PackageArchive:
 
         return dependencies, roots
 
-    def installed_packages(self) -> dict[str, tuple[list[Path], list[Path]]]:
+    def installed_packages(self) -> dict[str, FilesystemPackage]:
         """
         extract list of the installed packages and their content
 
@@ -142,9 +186,8 @@ class PackageArchive:
 
         pacman_local_files = self.root / "var" / "lib" / "pacman" / "local"
         for path in filter(lambda fn: fn.name == "files", walk(pacman_local_files)):
-            package, *_ = path.parent.name.rsplit("-", 2)
+            package = self._load_pacman_package(path)
 
-            directories, files = [], []
             is_files = False
             for line in path.read_text(encoding="utf8").splitlines():
                 if not line:  # skip empty lines
@@ -156,10 +199,10 @@ class PackageArchive:
 
                 entry = Path(line)
                 if line.endswith("/"):  # simple check if it is directory
-                    directories.append(entry)
+                    package.directories.append(entry)
                 else:
-                    files.append(entry)
+                    package.files.append(entry)
 
-            result[package] = directories, files
+            result[package.package_name] = package
 
         return result

tests/ahriman/core/log/test_lazy_logging.py

@@ -30,7 +30,7 @@ def test_package_logger_set_reset(database: SQLite) -> None:
     database._package_logger_reset()
     record = logging.makeLogRecord({})
     with pytest.raises(AttributeError):
-        record.package_id
+        assert record.package_id
 
 
 def test_in_package_context(database: SQLite, package_ahriman: Package, mocker: MockerFixture) -> None: