From 0626078319260fd1b56b72cfa489c484cf7385eb Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Tue, 28 May 2024 18:04:46 +0300 Subject: [PATCH] remove excess dependencies leaves --- src/ahriman/core/repository/executor.py | 2 +- src/ahriman/models/aur_package.py | 4 + src/ahriman/models/dependencies.py | 7 ++ src/ahriman/models/filesystem_package.py | 50 ++++++++++++ src/ahriman/models/package_archive.py | 87 +++++++++++++++++---- tests/ahriman/core/log/test_lazy_logging.py | 2 +- 6 files changed, 135 insertions(+), 17 deletions(-) create mode 100644 src/ahriman/models/filesystem_package.py diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py index 1037d107..a8af5a29 100644 --- a/src/ahriman/core/repository/executor.py +++ b/src/ahriman/core/repository/executor.py @@ -80,7 +80,7 @@ class Executor(PackageInfo, Cleaner): # clear changes and update commit hash self.reporter.package_changes_update(single.base, Changes(last_commit_sha)) # update dependencies list - dependencies = PackageArchive(self.paths.build_directory, single).depends_on() + dependencies = PackageArchive(self.paths.build_directory, single, self.pacman).depends_on() self.reporter.package_dependencies_update(single.base, dependencies) # update result set result.add_updated(single) diff --git a/src/ahriman/models/aur_package.py b/src/ahriman/models/aur_package.py index cd73fab5..6f143955 100644 --- a/src/ahriman/models/aur_package.py +++ b/src/ahriman/models/aur_package.py @@ -57,6 +57,7 @@ class AURPackage: provides(list[str]): list of packages which this package provides license(list[str]): list of package licenses keywords(list[str]): list of package keywords + groups(list[str]): list of package groups Examples: Mainly this class must be used from class methods instead of default :func:`__init__()`:: @@ -100,6 +101,7 @@ class AURPackage: provides: list[str] = field(default_factory=list) license: list[str] = field(default_factory=list) keywords: list[str] = field(default_factory=list) + groups: list[str] = field(default_factory=list) @classmethod def from_json(cls, dump: dict[str, Any]) -> Self: @@ -153,6 +155,7 @@ class AURPackage: provides=package.provides, license=package.licenses, keywords=[], + groups=package.groups, ) @classmethod @@ -191,6 +194,7 @@ class AURPackage: provides=dump["provides"], license=dump["licenses"], keywords=[], + groups=dump["groups"], ) @staticmethod diff --git a/src/ahriman/models/dependencies.py b/src/ahriman/models/dependencies.py index b823cc1c..65b62875 100644 --- a/src/ahriman/models/dependencies.py +++ b/src/ahriman/models/dependencies.py @@ -34,6 +34,13 @@ class Dependencies: paths: dict[str, list[str]] = field(default_factory=dict) + def __post_init__(self) -> None: + """ + remove empty paths + """ + paths = {path: packages for path, packages in self.paths.items() if packages} + object.__setattr__(self, "paths", paths) + @classmethod def from_json(cls, dump: dict[str, Any]) -> Self: """ diff --git a/src/ahriman/models/filesystem_package.py b/src/ahriman/models/filesystem_package.py new file mode 100644 index 00000000..ec4285b0 --- /dev/null +++ b/src/ahriman/models/filesystem_package.py @@ -0,0 +1,50 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass(frozen=True, kw_only=True) +class FilesystemPackage: + """ + class representing a simplified model for the package installed to filesystem + + Attributes: + package_name(str): package name + depends(list[str]): list of package dependencies + directories(list[Path]): list of directories this package contains + files(list[Path]): list of files this package contains + groups(list[str]): list of groups of the package + """ + + package_name: str + groups: set[str] + depends: set[str] + directories: list[Path] = field(default_factory=list) + files: list[Path] = field(default_factory=list) + + def __repr__(self) -> str: + """ + generate string representation of object + + Returns: + str: unique string representation + """ + return f'FilesystemPackage(package_name={self.package_name}, depends={self.depends})' diff --git a/src/ahriman/models/package_archive.py b/src/ahriman/models/package_archive.py index 974f95bb..fffb8c1b 100644 --- a/src/ahriman/models/package_archive.py +++ b/src/ahriman/models/package_archive.py @@ -23,8 +23,12 @@ from elftools.elf.elffile import ELFFile from pathlib import Path from typing import IO +from ahriman.core.alpm.pacman import Pacman +from ahriman.core.alpm.remote import OfficialSyncdb +from ahriman.core.exceptions import UnknownPackageError from ahriman.core.util import walk from ahriman.models.dependencies import Dependencies +from ahriman.models.filesystem_package import FilesystemPackage from ahriman.models.package import Package @@ -40,6 +44,7 @@ class PackageArchive: root: Path package: Package + pacman: Pacman @staticmethod def dynamic_needed(binary_path: Path) -> list[str]: @@ -90,6 +95,27 @@ class PackageArchive: return magic_bytes == expected + def _load_pacman_package(self, path: Path) -> FilesystemPackage: + """ + load pacman package model from path + + Args: + path(Path): path to package files database + + Returns: + FilesystemPackage: generated pacman package model with empty paths + """ + package_name, *_ = path.parent.name.rsplit("-", 2) + try: + pacman_package = OfficialSyncdb.info(package_name, pacman=self.pacman) + return FilesystemPackage( + package_name=package_name, + groups=set(pacman_package.groups), + depends=set(pacman_package.depends), + ) + except UnknownPackageError: + return FilesystemPackage(package_name=package_name, groups=set(), depends=set()) + def depends_on(self) -> Dependencies: """ extract packages and paths which are required for this package @@ -98,17 +124,49 @@ class PackageArchive: Dependencies: map of the package name to set of paths used by this package """ dependencies, roots = self.depends_on_paths() + installed_packages = self.installed_packages() - result: dict[str, list[str]] = {} - for package, (directories, files) in self.installed_packages().items(): - if package in self.package.packages: + # build initial map of file path -> packages containing this path + # in fact, keys will contain all libraries the package linked to and all directories it contains + dependencies_per_path: dict[Path, list[FilesystemPackage]] = {} + for package_base, package in installed_packages.items(): + if package_base in self.package.packages: continue # skip package itself - required_by = [directory for directory in directories if directory in roots] - required_by.extend(library for library in files if library.name in dependencies) + required_by = [directory for directory in package.directories if directory in roots] + required_by.extend(library for library in package.files if library.name in dependencies) for path in required_by: - result.setdefault(str(path), []).append(package) + dependencies_per_path.setdefault(path, []).append(package) + + # reduce trees + result = {} + base_packages = OfficialSyncdb.info("base", pacman=self.pacman).depends + # sort items from children directories to root + for path, packages in reversed(sorted(dependencies_per_path.items())): + package_names = [package.package_name for package in packages] + reduced_packages_list = [ + package.package_name + for package in packages + # if there is any package which is dependency of this package, we can skip it here + if not package.depends.intersection(package_names) + ] + + # skip if this path belongs to the one of the base packages + if any(package in reduced_packages_list for package in base_packages): + continue + + # check if there is already parent of current path in the result and has the same packages + for children_path, children_packages in result.items(): + if not Path(children_path).is_relative_to(path): + continue + reduced_packages_list = [ + package_name + for package_name in reduced_packages_list + if package_name not in children_packages + ] + + result[str(path)] = reduced_packages_list return Dependencies(result) @@ -130,7 +188,7 @@ class PackageArchive: return dependencies, roots - def installed_packages(self) -> dict[str, tuple[list[Path], list[Path]]]: + def installed_packages(self) -> dict[str, FilesystemPackage]: """ extract list of the installed packages and their content @@ -142,24 +200,23 @@ class PackageArchive: pacman_local_files = self.root / "var" / "lib" / "pacman" / "local" for path in filter(lambda fn: fn.name == "files", walk(pacman_local_files)): - package, *_ = path.parent.name.rsplit("-", 2) + package = self._load_pacman_package(path) - directories, files = [], [] - is_files = False + is_files_section = False for line in path.read_text(encoding="utf8").splitlines(): if not line: # skip empty lines continue if line.startswith("%") and line.endswith("%"): # directive started - is_files = line == "%FILES%" - if not is_files: # not a files directive + is_files_section = line == "%FILES%" + if not is_files_section: # not a files directive continue entry = Path(line) if line.endswith("/"): # simple check if it is directory - directories.append(entry) + package.directories.append(entry) else: - files.append(entry) + package.files.append(entry) - result[package] = directories, files + result[package.package_name] = package return result diff --git a/tests/ahriman/core/log/test_lazy_logging.py b/tests/ahriman/core/log/test_lazy_logging.py index 5a05878d..d4383428 100644 --- a/tests/ahriman/core/log/test_lazy_logging.py +++ b/tests/ahriman/core/log/test_lazy_logging.py @@ -30,7 +30,7 @@ def test_package_logger_set_reset(database: SQLite) -> None: database._package_logger_reset() record = logging.makeLogRecord({}) with pytest.raises(AttributeError): - record.package_id + assert record.package_id def test_in_package_context(database: SQLite, package_ahriman: Package, mocker: MockerFixture) -> None: