diff --git a/src/ahriman/core/build_tools/sources.py b/src/ahriman/core/build_tools/sources.py index 93d90c06..3971ab22 100644 --- a/src/ahriman/core/build_tools/sources.py +++ b/src/ahriman/core/build_tools/sources.py @@ -125,6 +125,12 @@ class Sources(LazyLogging): Sources._check_output("git", "init", "--initial-branch", instance.DEFAULT_BRANCH, cwd=sources_dir, logger=instance.logger) + # extract local files... + files = ["PKGBUILD", ".SRCINFO"] + [str(path) for path in Package.local_files(sources_dir)] + instance.add(sources_dir, *files) + # ...and commit them + instance.commit(sources_dir, author="ahriman ") + @staticmethod def load(sources_dir: Path, package: Package, patches: list[PkgbuildPatch], paths: RepositoryPaths) -> None: """ diff --git a/src/ahriman/core/util.py b/src/ahriman/core/util.py index e58ae0c2..da1de322 100644 --- a/src/ahriman/core/util.py +++ b/src/ahriman/core/util.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +# pylint: disable=too-many-lines import datetime import io import itertools @@ -48,6 +49,8 @@ __all__ = [ "pretty_datetime", "pretty_size", "safe_filename", + "srcinfo_property", + "srcinfo_property_list", "trim_package", "utcnow", "walk", @@ -326,6 +329,47 @@ def safe_filename(source: str) -> str: return re.sub(r"[^A-Za-z\d\-._~:\[\]@]", "-", source) +def srcinfo_property(key: str, srcinfo: dict[str, Any], package_srcinfo: dict[str, Any], *, + default: Any = None) -> Any: + """ + extract property from SRCINFO. This method extracts property from package if this property is presented in + ``package``. Otherwise, it looks for the same property in root srcinfo. If none found, the default value will be + returned + + Args: + key(str): key to extract from srcinfo + srcinfo(dict[str, Any]): root structure of SRCINFO + package_srcinfo(dict[str, Any]): package specific SRCINFO + default(Any, optional): the default value for the specified key (Default value = None) + + Returns: + Any: extracted value from SRCINFO + """ + return package_srcinfo.get(key) or srcinfo.get(key) or default + + +def srcinfo_property_list(key: str, srcinfo: dict[str, Any], package_srcinfo: dict[str, Any], *, + architecture: str | None = None) -> list[Any]: + """ + extract list property from SRCINFO. Unlike ``srcinfo_property`` it supposes that default return value is always + empty list. If ``architecture`` is supplied, then it will try to lookup for architecture specific values and will + append it at the end of result + + Args: + key(str): key to extract from srcinfo + srcinfo(dict[str, Any]): root structure of SRCINFO + package_srcinfo(dict[str, Any]): package specific SRCINFO + architecture(str | None, optional): package architecture if set (Default value = None) + + Returns: + list[Any]: list of extracted properties from SRCINFO + """ + values: list[Any] = srcinfo_property(key, srcinfo, package_srcinfo, default=[]) + if architecture is not None: + values.extend(srcinfo_property(f"{key}_{architecture}", srcinfo, package_srcinfo, default=[])) + return values + + def trim_package(package_name: str) -> str: """ remove version bound and description from package name. Pacman allows to specify version bound (=, <=, >= etc) for diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 64315992..0dc15e15 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -22,18 +22,19 @@ from __future__ import annotations import copy -from collections.abc import Iterable +from collections.abc import Generator, Iterable from dataclasses import asdict, dataclass from pathlib import Path from pyalpm import vercmp # type: ignore[import] from srcinfo.parse import parse_srcinfo # type: ignore[import] from typing import Any, Self +from urllib.parse import urlparse from ahriman.core.alpm.pacman import Pacman from ahriman.core.alpm.remote import AUR, Official, OfficialSyncdb from ahriman.core.exceptions import PackageInfoError from ahriman.core.log import LazyLogging -from ahriman.core.util import check_output, full_version, utcnow +from ahriman.core.util import check_output, full_version, srcinfo_property_list, utcnow from ahriman.models.package_description import PackageDescription from ahriman.models.package_source import PackageSource from ahriman.models.remote_source import RemoteSource @@ -235,23 +236,25 @@ class Package(LazyLogging): if errors: raise PackageInfoError(errors) - def get_property(key: str, properties: dict[str, Any], default: Any) -> Any: - return properties.get(key) or srcinfo.get(key) or default - - def get_list(key: str, properties: dict[str, Any]) -> Any: - return get_property(key, properties, []) + get_property(f"{key}_{architecture}", properties, []) - packages = { package: PackageDescription( - depends=get_list("depends", properties), - make_depends=get_list("makedepends", properties), - opt_depends=get_list("optdepends", properties), + depends=srcinfo_property_list("depends", srcinfo, properties, architecture=architecture), + make_depends=srcinfo_property_list("makedepends", srcinfo, properties, architecture=architecture), + opt_depends=srcinfo_property_list("optdepends", srcinfo, properties, architecture=architecture), ) for package, properties in srcinfo["packages"].items() } version = full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"]) - return cls(base=srcinfo["pkgbase"], version=version, remote=None, packages=packages) + remote = RemoteSource( + git_url=path.absolute().as_uri(), + web_url="", + path=".", + branch="master", + source=PackageSource.Local, + ) + + return cls(base=srcinfo["pkgbase"], version=version, remote=remote, packages=packages) @classmethod def from_json(cls, dump: dict[str, Any]) -> Self: @@ -293,6 +296,41 @@ class Package(LazyLogging): remote=remote, packages={package.name: PackageDescription.from_aur(package)}) + @staticmethod + def local_files(path: Path) -> Generator[Path, None, None]: + """ + extract list of local files + + Args: + path(Path): path to package sources directory + + Returns: + Generator[Path, None, None]: list of paths of files which belong to the package and distributed together + with this tarball. All paths are relative to the ``path`` + """ + srcinfo_source = Package._check_output("makepkg", "--printsrcinfo", cwd=path) + srcinfo, errors = parse_srcinfo(srcinfo_source) + if errors: + raise PackageInfoError(errors) + + # we could use arch property, but for consistency it is better to call special method + architectures = Package.supported_architectures(path) + + for architecture in architectures: + for source in srcinfo_property_list("source", srcinfo, {}, architecture=architecture): + if "::" in source: + _, source = source.split("::", 1) # in case if filename is specified, remove it + + if urlparse(source).scheme: + # basically file schema should use absolute path which is impossible if we are distributing + # files together with PKGBUILD. In this case we are going to skip it also + continue + + yield Path(source) + + if (install := srcinfo.get("install", None)) is not None: + yield Path(install) + @staticmethod def supported_architectures(path: Path) -> set[str]: """ diff --git a/tests/ahriman/core/build_tools/test_sources.py b/tests/ahriman/core/build_tools/test_sources.py index 3d950e69..67adfbe6 100644 --- a/tests/ahriman/core/build_tools/test_sources.py +++ b/tests/ahriman/core/build_tools/test_sources.py @@ -135,12 +135,17 @@ def test_init(mocker: MockerFixture) -> None: """ must create empty repository at the specified path """ + mocker.patch("ahriman.models.package.Package.local_files", return_value=[Path("local")]) + add_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.add") check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output") + commit_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.commit") local = Path("local") Sources.init(local) check_output_mock.assert_called_once_with("git", "init", "--initial-branch", Sources.DEFAULT_BRANCH, cwd=local, logger=pytest.helpers.anyvar(int)) + add_mock.assert_called_once_with(local, "PKGBUILD", ".SRCINFO", "local") + commit_mock.assert_called_once_with(local, author="ahriman ") def test_load(package_ahriman: Package, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/core/test_util.py b/tests/ahriman/core/test_util.py index 3aa6bd43..a53a3d0c 100644 --- a/tests/ahriman/core/test_util.py +++ b/tests/ahriman/core/test_util.py @@ -12,7 +12,8 @@ from unittest.mock import MagicMock from ahriman.core.exceptions import BuildError, OptionError, UnsafeRunError from ahriman.core.util import check_output, check_user, enum_values, exception_response_text, filter_json, \ - full_version, package_like, partition, pretty_datetime, pretty_size, safe_filename, trim_package, utcnow, walk + full_version, package_like, partition, pretty_datetime, pretty_size, safe_filename, srcinfo_property, \ + srcinfo_property_list, trim_package, utcnow, walk from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.repository_paths import RepositoryPaths @@ -331,6 +332,31 @@ def test_safe_filename() -> None: assert safe_filename("tolua++-1.0.93-4-x86_64.pkg.tar.zst") == "tolua---1.0.93-4-x86_64.pkg.tar.zst" +def test_srcinfo_property() -> None: + """ + must correctly extract properties + """ + assert srcinfo_property("key", {"key": "root"}, {"key": "overrides"}, default="default") == "overrides" + assert srcinfo_property("key", {"key": "root"}, {}, default="default") == "root" + assert srcinfo_property("key", {}, {"key": "overrides"}, default="default") == "overrides" + assert srcinfo_property("key", {}, {}, default="default") == "default" + assert srcinfo_property("key", {}, {}) is None + + +def test_srcinfo_property_list() -> None: + """ + must correctly extract property list + """ + assert srcinfo_property_list("key", {"key": ["root"]}, {"key": ["overrides"]}) == ["overrides"] + assert srcinfo_property_list("key", {"key": ["root"]}, {"key_x86_64": ["overrides"]}, architecture="x86_64") == [ + "root", "overrides" + ] + assert srcinfo_property_list("key", {"key": ["root"], "key_x86_64": ["overrides"]}, {}, architecture="x86_64") == [ + "root", "overrides" + ] + assert srcinfo_property_list("key", {"key_x86_64": ["overrides"]}, {}, architecture="x86_64") == ["overrides"] + + def test_trim_package() -> None: """ must trim package version diff --git a/tests/ahriman/models/test_package.py b/tests/ahriman/models/test_package.py index 123e061d..932b5125 100644 --- a/tests/ahriman/models/test_package.py +++ b/tests/ahriman/models/test_package.py @@ -2,6 +2,7 @@ import pytest from pathlib import Path from pytest_mock import MockerFixture +from srcinfo.parse import parse_srcinfo from unittest.mock import MagicMock from ahriman.core.alpm.pacman import Pacman @@ -165,7 +166,7 @@ def test_from_build(package_ahriman: Package, mocker: MockerFixture, resource_pa package = Package.from_build(Path("path"), "x86_64") assert package_ahriman.packages.keys() == package.packages.keys() package_ahriman.packages = package.packages # we are not going to test PackageDescription here - package_ahriman.remote = None + package_ahriman.remote = package.remote assert package_ahriman == package @@ -269,6 +270,70 @@ def test_from_official(package_ahriman: Package, aur_package_ahriman: AURPackage assert package_ahriman.packages.keys() == package.packages.keys() +def test_local_files(mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must extract local file sources + """ + srcinfo = (resource_path_root / "models" / "package_yay_srcinfo").read_text() + parsed_srcinfo, _ = parse_srcinfo(srcinfo) + parsed_srcinfo["source"] = ["local-file.tar.gz"] + mocker.patch("ahriman.models.package.parse_srcinfo", return_value=(parsed_srcinfo, [])) + mocker.patch("ahriman.models.package.Package._check_output", return_value=srcinfo) + mocker.patch("ahriman.models.package.Package.supported_architectures", return_value=["any"]) + + assert list(Package.local_files(Path("path"))) == [Path("local-file.tar.gz")] + + +def test_local_files_empty(mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must extract empty local files list when there is no local files + """ + srcinfo = (resource_path_root / "models" / "package_yay_srcinfo").read_text() + mocker.patch("ahriman.models.package.Package._check_output", return_value=srcinfo) + mocker.patch("ahriman.models.package.Package.supported_architectures", return_value=["any"]) + + assert list(Package.local_files(Path("path"))) == [] + + +def test_local_files_error(mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must raise exception on package parsing for local sources + """ + mocker.patch("ahriman.models.package.Package._check_output", return_value="") + mocker.patch("ahriman.models.package.parse_srcinfo", return_value=({"packages": {}}, ["an error"])) + + with pytest.raises(PackageInfoError): + list(Package.local_files(Path("path"))) + + +def test_local_files_schema(mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must skip local file source when file schema is used + """ + srcinfo = (resource_path_root / "models" / "package_yay_srcinfo").read_text() + parsed_srcinfo, _ = parse_srcinfo(srcinfo) + parsed_srcinfo["source"] = ["file:///local-file.tar.gz"] + mocker.patch("ahriman.models.package.parse_srcinfo", return_value=(parsed_srcinfo, [])) + mocker.patch("ahriman.models.package.Package._check_output", return_value="") + mocker.patch("ahriman.models.package.Package.supported_architectures", return_value=["any"]) + + assert list(Package.local_files(Path("path"))) == [] + + +def test_local_files_with_install(mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must extract local file sources with install file + """ + srcinfo = (resource_path_root / "models" / "package_yay_srcinfo").read_text() + parsed_srcinfo, _ = parse_srcinfo(srcinfo) + parsed_srcinfo["install"] = "install" + mocker.patch("ahriman.models.package.parse_srcinfo", return_value=(parsed_srcinfo, [])) + mocker.patch("ahriman.models.package.Package._check_output", return_value="") + mocker.patch("ahriman.models.package.Package.supported_architectures", return_value=["any"]) + + assert list(Package.local_files(Path("path"))) == [Path("install")] + + def test_supported_architectures(mocker: MockerFixture, resource_path_root: Path) -> None: """ must generate list of available architectures