From 086885fb0e67e37f1dd56837bf3f0308b5306d47 Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Tue, 12 Oct 2021 02:42:06 +0300 Subject: [PATCH] add ability to add manually stored packages --- src/ahriman/application/application.py | 33 ++++--- src/ahriman/core/build_tools/sources.py | 89 ++++++++++++------- src/ahriman/core/repository/executor.py | 16 +++- src/ahriman/core/repository/properties.py | 2 +- src/ahriman/models/package_source.py | 9 +- src/ahriman/models/repository_paths.py | 43 ++++++--- .../application/handlers/test_handler_init.py | 4 +- .../core/repository/test_properties.py | 8 +- tests/ahriman/models/test_repository_paths.py | 42 ++++----- 9 files changed, 155 insertions(+), 91 deletions(-) diff --git a/src/ahriman/application/application.py b/src/ahriman/application/application.py index 241f473a..cbc11c58 100644 --- a/src/ahriman/application/application.py +++ b/src/ahriman/application/application.py @@ -105,21 +105,29 @@ class Application: :param without_dependencies: if set, dependency check will be disabled """ known_packages = self._known_packages() + aur_url = self.configuration.get("alpm", "aur_url") + + def add_archive(src: Path) -> None: + dst = self.repository.paths.packages / src.name + shutil.move(src, dst) def add_directory(path: Path) -> None: for full_path in filter(package_like, path.iterdir()): add_archive(full_path) - def add_manual(src: str) -> Path: - package = Package.load(src, self.repository.pacman, self.configuration.get("alpm", "aur_url")) + def add_local(path: Path) -> Path: + package = Package.load(path, self.repository.pacman, aur_url) + Sources.init(path) + shutil.copytree(path, self.repository.paths.manual_for(package.base)) # copy package for the build + shutil.copytree(path, self.repository.paths.cache_for(package.base)) # copy package to store in caches + return self.repository.paths.manual_for(package.base) + + def add_remote(src: str) -> Path: + package = Package.load(src, self.repository.pacman, aur_url) Sources.load(self.repository.paths.manual_for(package.base), package.git_url, self.repository.paths.patches_for(package.base)) return self.repository.paths.manual_for(package.base) - def add_archive(src: Path) -> None: - dst = self.repository.paths.packages / src.name - shutil.move(src, dst) - def process_dependencies(path: Path) -> None: if without_dependencies: return @@ -128,12 +136,15 @@ class Application: def process_single(src: str) -> None: resolved_source = source.resolve(src) - if resolved_source == PackageSource.Directory: - add_directory(Path(src)) - elif resolved_source == PackageSource.Archive: + if resolved_source == PackageSource.Archive: add_archive(Path(src)) - else: - path = add_manual(src) + elif resolved_source == PackageSource.AUR: + path = add_remote(src) + process_dependencies(path) + elif resolved_source == PackageSource.Directory: + add_directory(Path(src)) + elif resolved_source == PackageSource.Local: + path = add_local(Path(src)) process_dependencies(path) for name in names: diff --git a/src/ahriman/core/build_tools/sources.py b/src/ahriman/core/build_tools/sources.py index 604b3c23..780c167f 100644 --- a/src/ahriman/core/build_tools/sources.py +++ b/src/ahriman/core/build_tools/sources.py @@ -33,72 +33,97 @@ class Sources: logger = logging.getLogger("build_details") + _branch = "master" # in case if BLM would like to change it _check_output = check_output @staticmethod - def add(local_path: Path, *pattern: str) -> None: + def add(sources_dir: Path, *pattern: str) -> None: """ track found files via git - :param local_path: local path to git repository + :param sources_dir: local path to git repository :param pattern: glob patterns """ # glob directory to find files which match the specified patterns found_files: List[Path] = [] for glob in pattern: - found_files.extend(local_path.glob(glob)) + found_files.extend(sources_dir.glob(glob)) Sources.logger.info("found matching files %s", found_files) # add them to index - Sources._check_output("git", "add", "--intent-to-add", *[str(fn.relative_to(local_path)) for fn in found_files], - exception=None, cwd=local_path, logger=Sources.logger) + Sources._check_output("git", "add", "--intent-to-add", + *[str(fn.relative_to(sources_dir)) for fn in found_files], + exception=None, cwd=sources_dir, logger=Sources.logger) @staticmethod - def diff(local_path: Path, patch_path: Path) -> None: + def branches(sources_dir: Path) -> List[str]: + """ + list current branches. Currently this method is used to define if there is initialized git repository + :param sources_dir: local path to git repository + :return: sorted list of available branches + """ + branches = Sources._check_output("git", "branch", exception=None, cwd=sources_dir, logger=Sources.logger) + return sorted(branches.splitlines()) + + @staticmethod + def diff(sources_dir: Path, patch_path: Path) -> None: """ generate diff from the current version and write it to the output file - :param local_path: local path to git repository + :param sources_dir: local path to git repository :param patch_path: path to result patch """ - patch = Sources._check_output("git", "diff", exception=None, cwd=local_path, logger=Sources.logger) + patch = Sources._check_output("git", "diff", exception=None, cwd=sources_dir, logger=Sources.logger) patch_path.write_text(patch) @staticmethod - def fetch(local_path: Path, remote: str, branch: str = "master") -> None: + def fetch(sources_dir: Path, remote: str) -> None: """ either clone repository or update it to origin/`branch` - :param local_path: local path to fetch + :param sources_dir: local path to fetch :param remote: remote target (from where to fetch) - :param branch: branch name to checkout, master by default """ # local directory exists and there is .git directory - if (local_path / ".git").is_dir(): - Sources.logger.info("update HEAD to remote to %s", local_path) - Sources._check_output("git", "fetch", "origin", branch, - exception=None, cwd=local_path, logger=Sources.logger) + is_initialized_git = (sources_dir / ".git").is_dir() + if is_initialized_git and not Sources.branches(sources_dir): + # there is git repository, but no remote configured so far + return + + if is_initialized_git: + Sources.logger.info("update HEAD to remote to %s", sources_dir) + Sources._check_output("git", "fetch", "origin", Sources._branch, + exception=None, cwd=sources_dir, logger=Sources.logger) else: - Sources.logger.info("clone remote %s to %s", remote, local_path) - Sources._check_output("git", "clone", remote, str(local_path), exception=None, logger=Sources.logger) + Sources.logger.info("clone remote %s to %s", remote, sources_dir) + Sources._check_output("git", "clone", remote, str(sources_dir), exception=None, logger=Sources.logger) # and now force reset to our branch - Sources._check_output("git", "checkout", "--force", branch, - exception=None, cwd=local_path, logger=Sources.logger) - Sources._check_output("git", "reset", "--hard", f"origin/{branch}", - exception=None, cwd=local_path, logger=Sources.logger) + Sources._check_output("git", "checkout", "--force", Sources._branch, + exception=None, cwd=sources_dir, logger=Sources.logger) + Sources._check_output("git", "reset", "--hard", f"origin/{Sources._branch}", + exception=None, cwd=sources_dir, logger=Sources.logger) @staticmethod - def load(local_path: Path, remote: str, patch_dir: Path) -> None: + def init(sources_dir: Path) -> None: + """ + create empty git repository at the specified path + :param sources_dir: local path to sources + """ + Sources._check_output("git", "init", "--initial-branch", Sources._branch, + exception=None, cwd=sources_dir, logger=Sources.logger) + + @staticmethod + def load(sources_dir: Path, remote: str, patch_dir: Path) -> None: """ fetch sources from remote and apply patches - :param local_path: local path to fetch + :param sources_dir: local path to fetch :param remote: remote target (from where to fetch) :param patch_dir: path to directory with package patches """ - Sources.fetch(local_path, remote) - Sources.patch_apply(local_path, patch_dir) + Sources.fetch(sources_dir, remote) + Sources.patch_apply(sources_dir, patch_dir) @staticmethod - def patch_apply(local_path: Path, patch_dir: Path) -> None: + def patch_apply(sources_dir: Path, patch_dir: Path) -> None: """ apply patches if any - :param local_path: local path to directory with git sources + :param sources_dir: local path to directory with git sources :param patch_dir: path to directory with package patches """ # check if even there are patches @@ -110,15 +135,15 @@ class Sources: for patch in patches: Sources.logger.info("apply patch %s", patch.name) Sources._check_output("git", "apply", "--ignore-space-change", "--ignore-whitespace", str(patch), - exception=None, cwd=local_path, logger=Sources.logger) + exception=None, cwd=sources_dir, logger=Sources.logger) @staticmethod - def patch_create(local_path: Path, patch_path: Path, *pattern: str) -> None: + def patch_create(sources_dir: Path, patch_path: Path, *pattern: str) -> None: """ create patch set for the specified local path - :param local_path: local path to git repository + :param sources_dir: local path to git repository :param patch_path: path to result patch :param pattern: glob patterns """ - Sources.add(local_path, *pattern) - Sources.diff(local_path, patch_path) + Sources.add(sources_dir, *pattern) + Sources.diff(sources_dir, patch_path) diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py index 7115cffe..c698743d 100644 --- a/src/ahriman/core/repository/executor.py +++ b/src/ahriman/core/repository/executor.py @@ -72,9 +72,16 @@ class Executor(Cleaner): :param packages: list of package names or bases to remove :return: path to repository database """ - def remove_single(package: str, fn: Path) -> None: + def remove_base(package_base: str) -> None: try: - self.repo.remove(package, fn) + self.paths.tree_clear(package_base) # remove all internal files + self.reporter.remove(package_base) # we only update status page in case of base removal + except Exception: + self.logger.exception("could not remove base %s", package_base) + + def remove_package(package: str, fn: Path) -> None: + try: + self.repo.remove(package, fn) # remove the package itself except Exception: self.logger.exception("could not remove %s", package) @@ -86,7 +93,7 @@ class Executor(Cleaner): for package, properties in local.packages.items() if properties.filename is not None } - self.reporter.remove(local.base) # we only update status page in case of base removal + remove_base(local.base) elif requested.intersection(local.packages.keys()): to_remove = { package: Path(properties.filename) @@ -95,8 +102,9 @@ class Executor(Cleaner): } else: to_remove = {} + for package, filename in to_remove.items(): - remove_single(package, filename) + remove_package(package, filename) return self.repo.repo_path diff --git a/src/ahriman/core/repository/properties.py b/src/ahriman/core/repository/properties.py index 88d860e4..bd0ff5ee 100644 --- a/src/ahriman/core/repository/properties.py +++ b/src/ahriman/core/repository/properties.py @@ -58,7 +58,7 @@ class Properties: self.name = configuration.get("repository", "name") self.paths = RepositoryPaths(configuration.getpath("repository", "root"), architecture) - self.paths.create_tree() + self.paths.tree_create() self.ignore_list = configuration.getlist("build", "ignore_packages", fallback=[]) self.pacman = Pacman(configuration) diff --git a/src/ahriman/models/package_source.py b/src/ahriman/models/package_source.py index b40c9dbb..8bd23c1f 100644 --- a/src/ahriman/models/package_source.py +++ b/src/ahriman/models/package_source.py @@ -30,14 +30,16 @@ class PackageSource(Enum): package source for addition enumeration :cvar Auto: automatically determine type of the source :cvar Archive: source is a package archive - :cvar Directory: source is a directory which contains packages :cvar AUR: source is an AUR package for which it should search + :cvar Directory: source is a directory which contains packages + :cvar Local: source is locally stored PKGBUILD """ Auto = "auto" Archive = "archive" - Directory = "directory" AUR = "aur" + Directory = "directory" + Local = "local" def resolve(self, source: str) -> PackageSource: """ @@ -47,7 +49,10 @@ class PackageSource(Enum): """ if self != PackageSource.Auto: return self + maybe_path = Path(source) + if (maybe_path / "PKGBUILD").is_file(): + return PackageSource.Local if maybe_path.is_dir(): return PackageSource.Directory if maybe_path.is_file() and package_like(maybe_path): diff --git a/src/ahriman/models/repository_paths.py b/src/ahriman/models/repository_paths.py index 1025c111..7dac934a 100644 --- a/src/ahriman/models/repository_paths.py +++ b/src/ahriman/models/repository_paths.py @@ -19,6 +19,8 @@ # from __future__ import annotations +import shutil + from dataclasses import dataclass from pathlib import Path from typing import Set, Type @@ -101,24 +103,12 @@ class RepositoryPaths: def cache_for(self, package_base: str) -> Path: """ - get cache path for specific package base + get path to cached PKGBUILD and package sources for the package base :param package_base: package base name :return: full path to directory for specified package base cache """ return self.cache / package_base - def create_tree(self) -> None: - """ - create ahriman working tree - """ - self.cache.mkdir(mode=0o755, parents=True, exist_ok=True) - self.chroot.mkdir(mode=0o755, parents=True, exist_ok=True) - self.manual.mkdir(mode=0o755, parents=True, exist_ok=True) - self.packages.mkdir(mode=0o755, parents=True, exist_ok=True) - self.patches.mkdir(mode=0o755, parents=True, exist_ok=True) - self.repository.mkdir(mode=0o755, parents=True, exist_ok=True) - self.sources.mkdir(mode=0o755, parents=True, exist_ok=True) - def manual_for(self, package_base: str) -> Path: """ get manual path for specific package base @@ -137,8 +127,33 @@ class RepositoryPaths: def sources_for(self, package_base: str) -> Path: """ - get sources path for specific package base + get path to directory from where build will start for the package base :param package_base: package base name :return: full path to directory for specified package base sources """ return self.sources / package_base + + def tree_clear(self, package_base: str) -> None: + """ + clear package specific files + :param package_base: package base name + """ + for directory in ( + self.cache_for(package_base), + self.manual_for(package_base), + self.patches_for(package_base)): + shutil.rmtree(directory, ignore_errors=True) + + def tree_create(self) -> None: + """ + create ahriman working tree + """ + for directory in ( + self.cache, + self.chroot, + self.manual, + self.packages, + self.patches, + self.repository, + self.sources): + directory.mkdir(mode=0o755, parents=True, exist_ok=True) diff --git a/tests/ahriman/application/handlers/test_handler_init.py b/tests/ahriman/application/handlers/test_handler_init.py index 3613b2bb..6d66c4de 100644 --- a/tests/ahriman/application/handlers/test_handler_init.py +++ b/tests/ahriman/application/handlers/test_handler_init.py @@ -10,11 +10,11 @@ def test_run(args: argparse.Namespace, configuration: Configuration, mocker: Moc """ must run command """ - create_tree_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree") + tree_create_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create") init_mock = mocker.patch("ahriman.core.alpm.repo.Repo.init") Init.run(args, "x86_64", configuration, True) - create_tree_mock.assert_called_once() + tree_create_mock.assert_called_once() init_mock.assert_called_once() diff --git a/tests/ahriman/core/repository/test_properties.py b/tests/ahriman/core/repository/test_properties.py index cda9ca4c..262138cf 100644 --- a/tests/ahriman/core/repository/test_properties.py +++ b/tests/ahriman/core/repository/test_properties.py @@ -9,17 +9,17 @@ def test_create_tree_on_load(configuration: Configuration, mocker: MockerFixture """ must create tree on load """ - create_tree_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree") + tree_create_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create") Properties("x86_64", configuration, True) - create_tree_mock.assert_called_once() + tree_create_mock.assert_called_once() def test_create_dummy_report_client(configuration: Configuration, mocker: MockerFixture) -> None: """ must create dummy report client if report is disabled """ - mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree") + mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create") load_mock = mocker.patch("ahriman.core.status.client.Client.load") properties = Properties("x86_64", configuration, True) @@ -31,7 +31,7 @@ def test_create_full_report_client(configuration: Configuration, mocker: MockerF """ must create load report client if report is enabled """ - mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree") + mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create") load_mock = mocker.patch("ahriman.core.status.client.Client.load") Properties("x86_64", configuration, False) diff --git a/tests/ahriman/models/test_repository_paths.py b/tests/ahriman/models/test_repository_paths.py index c91058fc..bbda6fec 100644 --- a/tests/ahriman/models/test_repository_paths.py +++ b/tests/ahriman/models/test_repository_paths.py @@ -23,27 +23,6 @@ def test_cache_for(repository_paths: RepositoryPaths, package_ahriman: Package) assert path.parent == repository_paths.cache -def test_create_tree(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: - """ - must create whole tree - """ - paths = { - prop - for prop in dir(repository_paths) - if not prop.startswith("_") - and not prop.endswith("_for") - and prop not in ("architecture", "create_tree", "known_architectures", "root") - } - mkdir_mock = mocker.patch("pathlib.Path.mkdir") - - repository_paths.create_tree() - mkdir_mock.assert_has_calls( - [ - mock.call(mode=0o755, parents=True, exist_ok=True) - for _ in paths - ]) - - def test_manual_for(repository_paths: RepositoryPaths, package_ahriman: Package) -> None: """ must return correct path for manual directory @@ -69,3 +48,24 @@ def test_sources_for(repository_paths: RepositoryPaths, package_ahriman: Package path = repository_paths.sources_for(package_ahriman.base) assert path.name == package_ahriman.base assert path.parent == repository_paths.sources + + +def test_tree_create(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: + """ + must create whole tree + """ + paths = { + prop + for prop in dir(repository_paths) + if not prop.startswith("_") + and not prop.endswith("_for") + and prop not in ("architecture", "known_architectures", "root", "tree_clear", "tree_create") + } + mkdir_mock = mocker.patch("pathlib.Path.mkdir") + + repository_paths.tree_create() + mkdir_mock.assert_has_calls( + [ + mock.call(mode=0o755, parents=True, exist_ok=True) + for _ in paths + ])