From 237fec3f851591f9c51c055385f2011e4906e3cd Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Tue, 8 Aug 2023 02:30:44 +0300 Subject: [PATCH] fix issues with remote pull triggers (see #103) * The issue appears when repository contains PKGBUILD in root. In this case it will copy tree with loosing package information, because the repository will be cloned to temporary path with random generated name * The issue appears when branch which is different from master is used for any reposittory with git files (e.g. single-pkgbuild repo or repo with submodules) --- src/ahriman/core/gitremote/remote_pull.py | 33 +++++++++-- .../core/gitremote/remote_pull_trigger.py | 2 +- src/ahriman/core/gitremote/remote_push.py | 1 + .../core/gitremote/test_remote_pull.py | 56 ++++++++++++------- 4 files changed, 66 insertions(+), 26 deletions(-) diff --git a/src/ahriman/core/gitremote/remote_pull.py b/src/ahriman/core/gitremote/remote_pull.py index 6674a21a..2c0e1c07 100644 --- a/src/ahriman/core/gitremote/remote_pull.py +++ b/src/ahriman/core/gitremote/remote_pull.py @@ -27,6 +27,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.exceptions import GitRemoteError from ahriman.core.log import LazyLogging from ahriman.core.util import walk +from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.remote_source import RemoteSource @@ -36,16 +37,18 @@ class RemotePull(LazyLogging): fetch PKGBUILDs from remote repository and use them for following actions Attributes: + architecture(str): repository architecture remote_source(RemoteSource): repository remote source (remote pull url and branch) repository_paths(RepositoryPaths): repository paths instance """ - def __init__(self, configuration: Configuration, section: str) -> None: + def __init__(self, configuration: Configuration, architecture: str, section: str) -> None: """ default constructor Args: configuration(Configuration): configuration instance + architecture(str): repository architecture section(str): settings section name """ self.remote_source = RemoteSource( @@ -55,8 +58,30 @@ class RemotePull(LazyLogging): branch=configuration.get(section, "pull_branch", fallback="master"), source=PackageSource.Local, ) + self.architecture = architecture self.repository_paths = configuration.repository_paths + def package_copy(self, pkgbuild_path: Path) -> None: + """ + copy single PKGBUILD content to the repository tree + + Args: + pkgbuild_path(Path): path to PKGBUILD to copy + """ + cloned_pkgbuild_dir = pkgbuild_path.parent + + # load package from the PKGBUILD, because it might be possible that name doesn't match + # e.g. if we have just cloned repo with just one PKGBUILD + package = Package.from_build(cloned_pkgbuild_dir, self.architecture, None) + package_base = package.base + local_pkgbuild_dir = self.repository_paths.cache_for(package_base) + + # copy source ignoring the git files + shutil.copytree(cloned_pkgbuild_dir, local_pkgbuild_dir, + ignore=shutil.ignore_patterns(".git*"), dirs_exist_ok=True) + # initialized git repository is required for local sources + Sources.init(local_pkgbuild_dir) + def repo_clone(self) -> None: """ clone repository from remote source @@ -74,11 +99,7 @@ class RemotePull(LazyLogging): clone_dir(Path): path to temporary cloned directory """ for pkgbuild_path in filter(lambda path: path.name == "PKGBUILD", walk(clone_dir)): - cloned_pkgbuild_dir = pkgbuild_path.parent - package_base = cloned_pkgbuild_dir.name - local_pkgbuild_dir = self.repository_paths.cache_for(package_base) - shutil.copytree(cloned_pkgbuild_dir, local_pkgbuild_dir, dirs_exist_ok=True) - Sources.init(local_pkgbuild_dir) # initialized git repository is required for local sources + self.package_copy(pkgbuild_path) def run(self) -> None: """ diff --git a/src/ahriman/core/gitremote/remote_pull_trigger.py b/src/ahriman/core/gitremote/remote_pull_trigger.py index e9c400a5..b850cc77 100644 --- a/src/ahriman/core/gitremote/remote_pull_trigger.py +++ b/src/ahriman/core/gitremote/remote_pull_trigger.py @@ -87,5 +87,5 @@ class RemotePullTrigger(Trigger): for target in self.targets: section, _ = self.configuration.gettype( target, self.architecture, fallback=self.CONFIGURATION_SCHEMA_FALLBACK) - runner = RemotePull(self.configuration, section) + runner = RemotePull(self.configuration, self.architecture, section) runner.run() diff --git a/src/ahriman/core/gitremote/remote_push.py b/src/ahriman/core/gitremote/remote_push.py index 25d43a57..2e175f47 100644 --- a/src/ahriman/core/gitremote/remote_push.py +++ b/src/ahriman/core/gitremote/remote_push.py @@ -79,6 +79,7 @@ class RemotePush(LazyLogging): package_target_dir = target_dir / package.base shutil.rmtree(package_target_dir, ignore_errors=True) # ...secondly, we clone whole tree... + # fetch is used intentionally here in order to avoid copying downloaded blobs Sources.fetch(package_target_dir, package.remote) # ...and last, but not least, we remove the dot-git directory... for git_file in package_target_dir.glob(".git*"): diff --git a/tests/ahriman/core/gitremote/test_remote_pull.py b/tests/ahriman/core/gitremote/test_remote_pull.py index 78bef6ee..8382c70f 100644 --- a/tests/ahriman/core/gitremote/test_remote_pull.py +++ b/tests/ahriman/core/gitremote/test_remote_pull.py @@ -7,6 +7,7 @@ from unittest.mock import call as MockCall from ahriman.core.configuration import Configuration from ahriman.core.exceptions import GitRemoteError from ahriman.core.gitremote.remote_pull import RemotePull +from ahriman.models.package import Package def test_repo_clone(configuration: Configuration, mocker: MockerFixture) -> None: @@ -15,36 +16,53 @@ def test_repo_clone(configuration: Configuration, mocker: MockerFixture) -> None """ fetch_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.fetch") copy_mock = mocker.patch("ahriman.core.gitremote.remote_pull.RemotePull.repo_copy") - runner = RemotePull(configuration, "gitremote") + runner = RemotePull(configuration, "x86_64", "gitremote") runner.repo_clone() fetch_mock.assert_called_once_with(pytest.helpers.anyvar(int), runner.remote_source) copy_mock.assert_called_once_with(pytest.helpers.anyvar(int)) +def test_package_copy(configuration: Configuration, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must copy single package + """ + package_mock = mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman) + patterns = object() + ignore_patterns_mock = mocker.patch("shutil.ignore_patterns", return_value=patterns) + copytree_mock = mocker.patch("shutil.copytree") + init_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.init") + runner = RemotePull(configuration, "x86_64", "gitremote") + local = Path("local") + + runner.package_copy(local / "PKGBUILD") + package_mock.assert_called_once_with(local, "x86_64", None) + ignore_patterns_mock.assert_called_once_with(".git*") + copytree_mock.assert_called_once_with( + local, configuration.repository_paths.cache_for(package_ahriman.base), + ignore=patterns, dirs_exist_ok=True) + init_mock.assert_called_once_with(configuration.repository_paths.cache_for(package_ahriman.base)) + + def test_repo_copy(configuration: Configuration, mocker: MockerFixture) -> None: """ must copy repository tree from temporary directory to the local cache """ + local = Path("local") mocker.patch("ahriman.core.gitremote.remote_pull.walk", return_value=[ - Path("local") / "package1" / "PKGBUILD", - Path("local") / "package1" / ".SRCINFO", - Path("local") / "package2" / ".SRCINFO", - Path("local") / "package3" / "PKGBUILD", - Path("local") / "package3" / ".SRCINFO", + local / "package1" / "PKGBUILD", + local / "package1" / ".SRCINFO", + local / "package2" / ".SRCINFO", + local / "package3" / "PKGBUILD", + local / "package3" / ".SRCINFO", ]) - copytree_mock = mocker.patch("shutil.copytree") - init_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.init") - runner = RemotePull(configuration, "gitremote") + copy_mock = mocker.patch("ahriman.core.gitremote.remote_pull.RemotePull.package_copy") + runner = RemotePull(configuration, "x86_64", "gitremote") - runner.repo_copy(Path("local")) - copytree_mock.assert_has_calls([ - MockCall(Path("local") / "package1", configuration.repository_paths.cache_for("package1"), dirs_exist_ok=True), - MockCall(Path("local") / "package3", configuration.repository_paths.cache_for("package3"), dirs_exist_ok=True), - ]) - init_mock.assert_has_calls([ - MockCall(configuration.repository_paths.cache_for("package1")), - MockCall(configuration.repository_paths.cache_for("package3")), + runner.repo_copy(local) + copy_mock.assert_has_calls([ + MockCall(local / "package1" / "PKGBUILD"), + MockCall(local / "package3" / "PKGBUILD"), ]) @@ -53,7 +71,7 @@ def test_run(configuration: Configuration, mocker: MockerFixture) -> None: must clone repo on run """ clone_mock = mocker.patch("ahriman.core.gitremote.remote_pull.RemotePull.repo_clone") - runner = RemotePull(configuration, "gitremote") + runner = RemotePull(configuration, "x86_64", "gitremote") runner.run() clone_mock.assert_called_once_with() @@ -64,7 +82,7 @@ def test_run_failed(configuration: Configuration, mocker: MockerFixture) -> None must reraise exception on error occurred """ mocker.patch("ahriman.core.gitremote.remote_pull.RemotePull.repo_clone", side_effect=Exception()) - runner = RemotePull(configuration, "gitremote") + runner = RemotePull(configuration, "x86_64", "gitremote") with pytest.raises(GitRemoteError): runner.run()