From 1625fddccd7873be179331c0e40e97d5fbfadfb6 Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Wed, 31 May 2023 16:10:11 +0300 Subject: [PATCH] support check dependencies --- CONTRIBUTING.md | 2 +- docs/ahriman.core.database.migrations.rst | 8 ++ src/ahriman/application/handlers/setup.py | 2 +- .../database/migrations/m007_check_depends.py | 80 +++++++++++++++++++ .../database/operations/package_operations.py | 6 +- src/ahriman/core/sign/gpg.py | 2 +- src/ahriman/models/aur_package.py | 4 + src/ahriman/models/package.py | 45 +++++++++-- src/ahriman/models/package_description.py | 5 ++ .../web/schemas/package_properties_schema.py | 4 + .../application/test_application.py | 2 +- tests/ahriman/conftest.py | 6 ++ .../migrations/test_m007_check_depends.py | 52 ++++++++++++ tests/ahriman/core/upload/test_http_upload.py | 2 +- tests/ahriman/core/upload/test_s3.py | 2 +- tests/ahriman/models/conftest.py | 2 + tests/ahriman/models/test_package.py | 23 +++++- .../testresources/models/package_ahriman_aur | 3 + .../models/package_ahriman_srcinfo | 1 + 19 files changed, 234 insertions(+), 17 deletions(-) create mode 100644 src/ahriman/core/database/migrations/m007_check_depends.py create mode 100644 tests/ahriman/core/database/migrations/test_m007_check_depends.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ca945987..ff76eaf1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,7 +34,7 @@ Again, the most checks can be performed by `make check` command, though some add do foo. With very very very long docstring - Note: + Notes: Very important note about this function Args: diff --git a/docs/ahriman.core.database.migrations.rst b/docs/ahriman.core.database.migrations.rst index c35d10f7..32d278a8 100644 --- a/docs/ahriman.core.database.migrations.rst +++ b/docs/ahriman.core.database.migrations.rst @@ -60,6 +60,14 @@ ahriman.core.database.migrations.m006\_packages\_architecture\_required module :no-undoc-members: :show-inheritance: +ahriman.core.database.migrations.m007\_check\_depends module +------------------------------------------------------------ + +.. automodule:: ahriman.core.database.migrations.m007_check_depends + :members: + :no-undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/src/ahriman/application/handlers/setup.py b/src/ahriman/application/handlers/setup.py index 8db02618..15de7a50 100644 --- a/src/ahriman/application/handlers/setup.py +++ b/src/ahriman/application/handlers/setup.py @@ -136,7 +136,7 @@ class Setup(Handler): """ create configuration for devtools based on ``source`` configuration - Note: + Notes: devtools does not allow to specify the pacman configuration, thus we still have to use configuration in /usr Args: diff --git a/src/ahriman/core/database/migrations/m007_check_depends.py b/src/ahriman/core/database/migrations/m007_check_depends.py new file mode 100644 index 00000000..2bf57903 --- /dev/null +++ b/src/ahriman/core/database/migrations/m007_check_depends.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from sqlite3 import Connection + +from ahriman.core.alpm.pacman import Pacman +from ahriman.core.configuration import Configuration +from ahriman.core.util import package_like +from ahriman.models.package import Package +from ahriman.models.pacman_synchronization import PacmanSynchronization + + +__all__ = ["migrate_data", "steps"] + + +steps = [ + """ + alter table packages add column check_depends json + """, +] + + +def migrate_data(connection: Connection, configuration: Configuration) -> None: + """ + perform data migration + + Args: + connection(Connection): database connection + configuration(Configuration): configuration instance + """ + migrate_package_check_depends(connection, configuration) + + +def migrate_package_check_depends(connection: Connection, configuration: Configuration) -> None: + """ + migrate package check depends fields + + Args: + connection(Connection): database connection + configuration(Configuration): configuration instance + """ + if not configuration.repository_paths.repository.is_dir(): + return + + _, architecture = configuration.check_loaded() + pacman = Pacman(architecture, configuration, refresh_database=PacmanSynchronization.Disabled) + + package_list = [] + for full_path in filter(package_like, configuration.repository_paths.repository.iterdir()): + base = Package.from_archive(full_path, pacman, remote=None) + for package, description in base.packages.items(): + package_list.append({ + "check_depends": description.check_depends, + "package": package, + }) + + connection.executemany( + """ + update packages set + check_depends = :check_depends + where package = :package + """, + package_list + ) diff --git a/src/ahriman/core/database/operations/package_operations.py b/src/ahriman/core/database/operations/package_operations.py index c6b10d18..ee417c7d 100644 --- a/src/ahriman/core/database/operations/package_operations.py +++ b/src/ahriman/core/database/operations/package_operations.py @@ -113,17 +113,17 @@ class PackageOperations(Operations): (package, package_base, architecture, archive_size, build_date, depends, description, filename, "groups", installed_size, licenses, provides, - url, make_depends, opt_depends) + url, make_depends, opt_depends, check_depends) values (:package, :package_base, :architecture, :archive_size, :build_date, :depends, :description, :filename, :groups, :installed_size, :licenses, :provides, - :url, :make_depends, :opt_depends) + :url, :make_depends, :opt_depends, :check_depends) on conflict (package, architecture) do update set package_base = :package_base, archive_size = :archive_size, build_date = :build_date, depends = :depends, description = :description, filename = :filename, "groups" = :groups, installed_size = :installed_size, licenses = :licenses, provides = :provides, - url = :url, make_depends = :make_depends, opt_depends = :opt_depends + url = :url, make_depends = :make_depends, opt_depends = :opt_depends, check_depends = :check_depends """, package_list) diff --git a/src/ahriman/core/sign/gpg.py b/src/ahriman/core/sign/gpg.py index be167c2c..b5691a8d 100644 --- a/src/ahriman/core/sign/gpg.py +++ b/src/ahriman/core/sign/gpg.py @@ -220,7 +220,7 @@ class GPG(LazyLogging): """ sign repository if required by configuration - Note: + Notes: More likely you just want to pass ``repository_sign_args`` to repo wrapper Args: diff --git a/src/ahriman/models/aur_package.py b/src/ahriman/models/aur_package.py index 851ae17d..fb7cc82d 100644 --- a/src/ahriman/models/aur_package.py +++ b/src/ahriman/models/aur_package.py @@ -52,6 +52,7 @@ class AURPackage: depends(list[str]): list of package dependencies make_depends(l[str]): list of package make dependencies opt_depends(list[str]): list of package optional dependencies + check_depends(list[str]): list of package test dependencies conflicts(list[str]): conflicts list for the package provides(list[str]): list of packages which this package provides license(list[str]): list of package licenses @@ -94,6 +95,7 @@ class AURPackage: depends: list[str] = field(default_factory=list) make_depends: list[str] = field(default_factory=list) opt_depends: list[str] = field(default_factory=list) + check_depends: list[str] = field(default_factory=list) conflicts: list[str] = field(default_factory=list) provides: list[str] = field(default_factory=list) license: list[str] = field(default_factory=list) @@ -146,6 +148,7 @@ class AURPackage: depends=package.depends, make_depends=package.makedepends, opt_depends=package.optdepends, + check_depends=package.checkdepends, conflicts=package.conflicts, provides=package.provides, license=package.licenses, @@ -185,6 +188,7 @@ class AURPackage: depends=dump["depends"], make_depends=dump["makedepends"], opt_depends=dump["optdepends"], + check_depends=dump["checkdepends"], conflicts=dump["conflicts"], provides=dump["provides"], license=dump["licenses"], diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 0dc15e15..9f63e2a5 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -22,7 +22,7 @@ from __future__ import annotations import copy -from collections.abc import Generator, Iterable +from collections.abc import Callable, Generator, Iterable from dataclasses import asdict, dataclass from pathlib import Path from pyalpm import vercmp # type: ignore[import] @@ -88,7 +88,7 @@ class Package(LazyLogging): Returns: list[str]: sum of dependencies per each package """ - return sorted(set(sum((package.depends for package in self.packages.values()), start=[]))) + return self._package_list_property(lambda package: package.depends) @property def depends_build(self) -> set[str]: @@ -98,7 +98,17 @@ class Package(LazyLogging): Returns: set[str]: full dependencies list used by devtools """ - return (set(self.depends) | set(self.depends_make)).difference(self.packages_full) + return (set(self.depends) | set(self.depends_make) | set(self.depends_check)).difference(self.packages_full) + + @property + def depends_check(self) -> list[str]: + """ + get package test dependencies + + Returns: + list[str]: sum of test dependencies per each package + """ + return self._package_list_property(lambda package: package.check_depends) @property def depends_make(self) -> list[str]: @@ -108,7 +118,7 @@ class Package(LazyLogging): Returns: list[str]: sum of make dependencies per each package """ - return sorted(set(sum((package.make_depends for package in self.packages.values()), start=[]))) + return self._package_list_property(lambda package: package.make_depends) @property def depends_opt(self) -> list[str]: @@ -118,7 +128,7 @@ class Package(LazyLogging): Returns: list[str]: sum of optional dependencies per each package """ - return sorted(set(sum((package.opt_depends for package in self.packages.values()), start=[]))) + return self._package_list_property(lambda package: package.opt_depends) @property def groups(self) -> list[str]: @@ -128,7 +138,7 @@ class Package(LazyLogging): Returns: list[str]: sum of groups per each package """ - return sorted(set(sum((package.groups for package in self.packages.values()), start=[]))) + return self._package_list_property(lambda package: package.groups) @property def is_single_package(self) -> bool: @@ -163,7 +173,7 @@ class Package(LazyLogging): Returns: list[str]: sum of licenses per each package """ - return sorted(set(sum((package.licenses for package in self.packages.values()), start=[]))) + return self._package_list_property(lambda package: package.licenses) @property def packages_full(self) -> list[str]: @@ -241,6 +251,7 @@ class Package(LazyLogging): depends=srcinfo_property_list("depends", srcinfo, properties, architecture=architecture), make_depends=srcinfo_property_list("makedepends", srcinfo, properties, architecture=architecture), opt_depends=srcinfo_property_list("optdepends", srcinfo, properties, architecture=architecture), + check_depends=srcinfo_property_list("checkdepends", srcinfo, properties, architecture=architecture), ) for package, properties in srcinfo["packages"].items() } @@ -351,6 +362,26 @@ class Package(LazyLogging): raise PackageInfoError(errors) return set(srcinfo.get("arch", [])) + def _package_list_property(self, extractor: Callable[[PackageDescription], list[str]]) -> list[str]: + """ + extract list property from single packages and combine them into one list + + Notes: + Basically this method is generic for type of ``list[T]``, but there is no trait ``Comparable`` in default + packages, thus we limit this method only to new types + + Args: + extractor(Callable[[PackageDescription], list[str]): package property extractor + + Returns: + list[str]: combined list of unique entries in properties list + """ + def generator() -> Generator[str, None, None]: + for package in self.packages.values(): + yield from extractor(package) + + return sorted(set(generator())) + def actual_version(self, paths: RepositoryPaths) -> str: """ additional method to handle VCS package versions diff --git a/src/ahriman/models/package_description.py b/src/ahriman/models/package_description.py index 6f627922..18dd96cb 100644 --- a/src/ahriman/models/package_description.py +++ b/src/ahriman/models/package_description.py @@ -35,6 +35,7 @@ class PackageDescription: architecture(str | None): package architecture archive_size(int | None): package archive size build_date(int | None): package build date + check_depends(list[str]): package dependencies list used for check functions depends(list[str]): package dependencies list opt_depends(list[str]): optional package dependencies list make_depends(list[str]): package dependencies list used for building @@ -70,6 +71,7 @@ class PackageDescription: depends: list[str] = field(default_factory=list) make_depends: list[str] = field(default_factory=list) opt_depends: list[str] = field(default_factory=list) + check_depends: list[str] = field(default_factory=list) description: str | None = None filename: str | None = None groups: list[str] = field(default_factory=list) @@ -85,6 +87,7 @@ class PackageDescription: self.depends = [trim_package(package) for package in self.depends] self.opt_depends = [trim_package(package) for package in self.opt_depends] self.make_depends = [trim_package(package) for package in self.make_depends] + self.check_depends = [trim_package(package) for package in self.check_depends] @property def filepath(self) -> Path | None: @@ -111,6 +114,7 @@ class PackageDescription: depends=package.depends, make_depends=package.make_depends, opt_depends=package.opt_depends, + check_depends=package.check_depends, description=package.description, licenses=package.license, provides=package.provides, @@ -151,6 +155,7 @@ class PackageDescription: depends=package.depends, make_depends=package.makedepends, opt_depends=package.optdepends, + check_depends=package.checkdepends, description=package.desc, filename=path.name, groups=package.groups, diff --git a/src/ahriman/web/schemas/package_properties_schema.py b/src/ahriman/web/schemas/package_properties_schema.py index 8d6c0520..4d134361 100644 --- a/src/ahriman/web/schemas/package_properties_schema.py +++ b/src/ahriman/web/schemas/package_properties_schema.py @@ -49,6 +49,10 @@ class PackagePropertiesSchema(Schema): "description": "Package optional dependencies list", "example": ["python-aiohttp"], }) + check_depends = fields.List(fields.String(), metadata={ + "description": "Package test dependencies list", + "example": ["python-pytest"], + }) description = fields.String(metadata={ "description": "Package description", "example": "ArcH linux ReposItory MANager", diff --git a/tests/ahriman/application/application/test_application.py b/tests/ahriman/application/application/test_application.py index 2a61eafc..04ba48a8 100644 --- a/tests/ahriman/application/application/test_application.py +++ b/tests/ahriman/application/application/test_application.py @@ -74,7 +74,7 @@ def test_with_dependencies(application: Application, package_ahriman: Package, p package_mock = mocker.patch("ahriman.models.package.Package.from_aur", side_effect=lambda p, _: packages[p]) packages_mock = mocker.patch("ahriman.application.application.Application._known_packages", - return_value=["devtools", "python-build"]) + return_value=["devtools", "python-build", "python-pytest"]) result = application.with_dependencies([package_ahriman], process_dependencies=True) assert {package.base: package for package in result} == packages diff --git a/tests/ahriman/conftest.py b/tests/ahriman/conftest.py index f4467442..42fde054 100644 --- a/tests/ahriman/conftest.py +++ b/tests/ahriman/conftest.py @@ -146,6 +146,9 @@ def aur_package_ahriman() -> AURPackage: "rsync", "subversion", ], + check_depends=[ + "python-pytest", + ], conflicts=[], provides=[], license=["GPL3"], @@ -335,6 +338,9 @@ def package_description_ahriman() -> PackageDescription: "rsync", "subversion", ], + check_depends=[ + "python-pytest", + ], description="ArcH linux ReposItory MANager", filename="ahriman-2.6.0-1-any.pkg.tar.zst", groups=[], diff --git a/tests/ahriman/core/database/migrations/test_m007_check_depends.py b/tests/ahriman/core/database/migrations/test_m007_check_depends.py new file mode 100644 index 00000000..aa7525fd --- /dev/null +++ b/tests/ahriman/core/database/migrations/test_m007_check_depends.py @@ -0,0 +1,52 @@ +import pytest + +from pytest_mock import MockerFixture +from sqlite3 import Connection + +from ahriman.core.configuration import Configuration +from ahriman.core.database.migrations.m007_check_depends import migrate_data, migrate_package_check_depends, steps +from ahriman.models.package import Package + + +def test_migration_check_depends() -> None: + """ + migration must not be empty + """ + assert steps + + +def test_migrate_data(connection: Connection, configuration: Configuration, mocker: MockerFixture) -> None: + """ + must perform data migration + """ + depends_mock = mocker.patch("ahriman.core.database.migrations.m007_check_depends.migrate_package_check_depends") + migrate_data(connection, configuration) + depends_mock.assert_called_once_with(connection, configuration) + + +def test_migrate_package_depends(connection: Connection, configuration: Configuration, package_ahriman: Package, + mocker: MockerFixture) -> None: + """ + must update make and opt depends list + """ + mocker.patch("pathlib.Path.is_dir", return_value=True) + mocker.patch("pathlib.Path.iterdir", return_value=[package_ahriman.packages[package_ahriman.base].filepath]) + package_mock = mocker.patch("ahriman.models.package.Package.from_archive", return_value=package_ahriman) + + migrate_package_check_depends(connection, configuration) + package_mock.assert_called_once_with( + package_ahriman.packages[package_ahriman.base].filepath, pytest.helpers.anyvar(int), remote=None) + connection.executemany.assert_called_once_with(pytest.helpers.anyvar(str, strict=True), [{ + "check_depends": package_ahriman.packages[package_ahriman.base].check_depends, + "package": package_ahriman.base, + }]) + + +def test_migrate_package_depends_skip(connection: Connection, configuration: Configuration, + mocker: MockerFixture) -> None: + """ + must skip update make and opt depends list if no repository directory found + """ + mocker.patch("pathlib.Path.is_dir", return_value=False) + migrate_package_check_depends(connection, configuration) + connection.executemany.assert_not_called() diff --git a/tests/ahriman/core/upload/test_http_upload.py b/tests/ahriman/core/upload/test_http_upload.py index e6e977f5..544c5562 100644 --- a/tests/ahriman/core/upload/test_http_upload.py +++ b/tests/ahriman/core/upload/test_http_upload.py @@ -22,7 +22,7 @@ def test_calculate_hash_small(resource_path_root: Path) -> None: must calculate checksum for path which is single chunk """ path = resource_path_root / "models" / "package_ahriman_srcinfo" - assert HttpUpload.calculate_hash(path) == "79b0f84e0232ed34fd191a85c383ecc5" + assert HttpUpload.calculate_hash(path) == "2635e2898452d594025517cfe529b1f2" def test_get_body_get_hashes() -> None: diff --git a/tests/ahriman/core/upload/test_s3.py b/tests/ahriman/core/upload/test_s3.py index 7442fd71..979a1fce 100644 --- a/tests/ahriman/core/upload/test_s3.py +++ b/tests/ahriman/core/upload/test_s3.py @@ -30,7 +30,7 @@ def test_calculate_etag_small(resource_path_root: Path) -> None: must calculate checksum for path which is single chunk """ path = resource_path_root / "models" / "package_ahriman_srcinfo" - assert S3.calculate_etag(path, _chunk_size) == "79b0f84e0232ed34fd191a85c383ecc5" + assert S3.calculate_etag(path, _chunk_size) == "2635e2898452d594025517cfe529b1f2" def test_files_remove(s3_remote_objects: list[Any]) -> None: diff --git a/tests/ahriman/models/conftest.py b/tests/ahriman/models/conftest.py index ef5e9c78..15c8bf8b 100644 --- a/tests/ahriman/models/conftest.py +++ b/tests/ahriman/models/conftest.py @@ -115,6 +115,7 @@ def pyalpm_package_ahriman(aur_package_ahriman: AURPackage) -> MagicMock: type(mock).makedepends = PropertyMock(return_value=aur_package_ahriman.make_depends) type(mock).name = PropertyMock(return_value=aur_package_ahriman.name) type(mock).optdepends = PropertyMock(return_value=aur_package_ahriman.opt_depends) + type(mock).checkdepends = PropertyMock(return_value=aur_package_ahriman.check_depends) type(mock).provides = PropertyMock(return_value=aur_package_ahriman.provides) type(mock).version = PropertyMock(return_value=aur_package_ahriman.version) type(mock).url = PropertyMock(return_value=aur_package_ahriman.url) @@ -139,6 +140,7 @@ def pyalpm_package_description_ahriman(package_description_ahriman: PackageDescr type(mock).depends = PropertyMock(return_value=package_description_ahriman.depends) type(mock).makedepends = PropertyMock(return_value=package_description_ahriman.make_depends) type(mock).optdepends = PropertyMock(return_value=package_description_ahriman.opt_depends) + type(mock).checkdepends = PropertyMock(return_value=package_description_ahriman.check_depends) type(mock).desc = PropertyMock(return_value=package_description_ahriman.description) type(mock).groups = PropertyMock(return_value=package_description_ahriman.groups) type(mock).isize = PropertyMock(return_value=package_description_ahriman.installed_size) diff --git a/tests/ahriman/models/test_package.py b/tests/ahriman/models/test_package.py index 932b5125..633cbe2e 100644 --- a/tests/ahriman/models/test_package.py +++ b/tests/ahriman/models/test_package.py @@ -36,6 +36,10 @@ def test_depends_build(package_ahriman: Package, package_python_schedule: Packag set(package_ahriman.depends_build).intersection(package.make_depends) for package in package_ahriman.packages.values() ) + assert all( + set(package_ahriman.depends_build).intersection(package.check_depends) + for package in package_ahriman.packages.values() + ) assert all( set(package_python_schedule.depends_build).intersection(package.depends) @@ -53,7 +57,21 @@ def test_depends_build_with_version_and_overlap(mocker: MockerFixture, resource_ mocker.patch("ahriman.models.package.Package._check_output", return_value=srcinfo) package_gcc10 = Package.from_build(Path("local"), "x86_64") - assert package_gcc10.depends_build == {"glibc", "doxygen", "binutils", "git", "libmpc", "python", "zstd"} + assert package_gcc10.depends_build == { + "glibc", "zstd", # depends + "doxygen", "binutils", "git", "libmpc", "python", # make depends + "dejagnu", "inetutils", # check depends + } + + +def test_depends_check(package_ahriman: Package) -> None: + """ + must return list of test dependencies + """ + assert all( + set(package_ahriman.depends_check).intersection(package.check_depends) + for package in package_ahriman.packages.values() + ) def test_depends_make(package_ahriman: Package) -> None: @@ -183,16 +201,19 @@ def test_from_build_multiple_packages(mocker: MockerFixture, resource_path_root: depends=["gcc10-libs=10.3.0-2", "binutils>=2.28", "libmpc", "zstd"], make_depends=["binutils", "doxygen", "git", "libmpc", "python"], opt_depends=[], + check_depends=["dejagnu", "inetutils"], ), "gcc10-libs": PackageDescription( depends=["glibc>=2.27"], make_depends=["binutils", "doxygen", "git", "libmpc", "python"], opt_depends=[], + check_depends=["dejagnu", "inetutils"], ), "gcc10-fortran": PackageDescription( depends=["gcc10=10.3.0-2"], make_depends=["binutils", "doxygen", "git", "libmpc", "python"], opt_depends=[], + check_depends=["dejagnu", "inetutils"], ), } diff --git a/tests/testresources/models/package_ahriman_aur b/tests/testresources/models/package_ahriman_aur index 9b5d5818..f294f1dc 100644 --- a/tests/testresources/models/package_ahriman_aur +++ b/tests/testresources/models/package_ahriman_aur @@ -2,6 +2,9 @@ "resultcount": 1, "results": [ { + "CheckDepends": [ + "python-pytest" + ], "Depends": [ "devtools", "git", diff --git a/tests/testresources/models/package_ahriman_srcinfo b/tests/testresources/models/package_ahriman_srcinfo index a0f854b0..214a5491 100644 --- a/tests/testresources/models/package_ahriman_srcinfo +++ b/tests/testresources/models/package_ahriman_srcinfo @@ -5,6 +5,7 @@ pkgbase = ahriman url = https://github.com/arcan1s/ahriman arch = any license = GPL3 + checkdepends = python-pytest makedepends = python-build makedepends = python-installer makedepends = python-wheel