add ability to add manually stored packages

src/ahriman/application/application.py

@@ -105,21 +105,29 @@ class Application:
         :param without_dependencies: if set, dependency check will be disabled
         """
         known_packages = self._known_packages()
+        aur_url = self.configuration.get("alpm", "aur_url")
+
+        def add_archive(src: Path) -> None:
+            dst = self.repository.paths.packages / src.name
+            shutil.move(src, dst)
 
         def add_directory(path: Path) -> None:
             for full_path in filter(package_like, path.iterdir()):
                 add_archive(full_path)
 
-        def add_manual(src: str) -> Path:
+        def add_local(path: Path) -> Path:
-            package = Package.load(src, self.repository.pacman, self.configuration.get("alpm", "aur_url"))
+            package = Package.load(path, self.repository.pacman, aur_url)
+            Sources.init(path)
+            shutil.copytree(path, self.repository.paths.manual_for(package.base))  # copy package for the build
+            shutil.copytree(path, self.repository.paths.cache_for(package.base))  # copy package to store in caches
+            return self.repository.paths.manual_for(package.base)
+
+        def add_remote(src: str) -> Path:
+            package = Package.load(src, self.repository.pacman, aur_url)
             Sources.load(self.repository.paths.manual_for(package.base), package.git_url,
                          self.repository.paths.patches_for(package.base))
             return self.repository.paths.manual_for(package.base)
 
-        def add_archive(src: Path) -> None:
-            dst = self.repository.paths.packages / src.name
-            shutil.move(src, dst)
-
         def process_dependencies(path: Path) -> None:
             if without_dependencies:
                 return
@@ -128,12 +136,15 @@ class Application:
 
         def process_single(src: str) -> None:
             resolved_source = source.resolve(src)
-            if resolved_source == PackageSource.Directory:
+            if resolved_source == PackageSource.Archive:
-                add_directory(Path(src))
-            elif resolved_source == PackageSource.Archive:
                 add_archive(Path(src))
-            else:
+            elif resolved_source == PackageSource.AUR:
-                path = add_manual(src)
+                path = add_remote(src)
+                process_dependencies(path)
+            elif resolved_source == PackageSource.Directory:
+                add_directory(Path(src))
+            elif resolved_source == PackageSource.Local:
+                path = add_local(Path(src))
                 process_dependencies(path)
 
         for name in names:

src/ahriman/core/build_tools/sources.py

@@ -33,72 +33,97 @@ class Sources:
 
     logger = logging.getLogger("build_details")
 
+    _branch = "master"  # in case if BLM would like to change it
     _check_output = check_output
 
     @staticmethod
-    def add(local_path: Path, *pattern: str) -> None:
+    def add(sources_dir: Path, *pattern: str) -> None:
         """
         track found files via git
-        :param local_path: local path to git repository
+        :param sources_dir: local path to git repository
         :param pattern: glob patterns
         """
         # glob directory to find files which match the specified patterns
         found_files: List[Path] = []
         for glob in pattern:
-            found_files.extend(local_path.glob(glob))
+            found_files.extend(sources_dir.glob(glob))
         Sources.logger.info("found matching files %s", found_files)
         # add them to index
-        Sources._check_output("git", "add", "--intent-to-add", *[str(fn.relative_to(local_path)) for fn in found_files],
+        Sources._check_output("git", "add", "--intent-to-add",
-                              exception=None, cwd=local_path, logger=Sources.logger)
+                              *[str(fn.relative_to(sources_dir)) for fn in found_files],
+                              exception=None, cwd=sources_dir, logger=Sources.logger)
 
     @staticmethod
-    def diff(local_path: Path, patch_path: Path) -> None:
+    def branches(sources_dir: Path) -> List[str]:
+        """
+        list current branches. Currently this method is used to define if there is initialized git repository
+        :param sources_dir: local path to git repository
+        :return: sorted list of available branches
+        """
+        branches = Sources._check_output("git", "branch", exception=None, cwd=sources_dir, logger=Sources.logger)
+        return sorted(branches.splitlines())
+
+    @staticmethod
+    def diff(sources_dir: Path, patch_path: Path) -> None:
         """
         generate diff from the current version and write it to the output file
-        :param local_path: local path to git repository
+        :param sources_dir: local path to git repository
         :param patch_path: path to result patch
         """
-        patch = Sources._check_output("git", "diff", exception=None, cwd=local_path, logger=Sources.logger)
+        patch = Sources._check_output("git", "diff", exception=None, cwd=sources_dir, logger=Sources.logger)
         patch_path.write_text(patch)
 
     @staticmethod
-    def fetch(local_path: Path, remote: str, branch: str = "master") -> None:
+    def fetch(sources_dir: Path, remote: str) -> None:
         """
         either clone repository or update it to origin/`branch`
-        :param local_path: local path to fetch
+        :param sources_dir: local path to fetch
         :param remote: remote target (from where to fetch)
-        :param branch: branch name to checkout, master by default
         """
         # local directory exists and there is .git directory
-        if (local_path / ".git").is_dir():
+        is_initialized_git = (sources_dir / ".git").is_dir()
-            Sources.logger.info("update HEAD to remote to %s", local_path)
+        if is_initialized_git and not Sources.branches(sources_dir):
-            Sources._check_output("git", "fetch", "origin", branch,
+            # there is git repository, but no remote configured so far
-                                  exception=None, cwd=local_path, logger=Sources.logger)
+            return
+
+        if is_initialized_git:
+            Sources.logger.info("update HEAD to remote to %s", sources_dir)
+            Sources._check_output("git", "fetch", "origin", Sources._branch,
+                                  exception=None, cwd=sources_dir, logger=Sources.logger)
         else:
-            Sources.logger.info("clone remote %s to %s", remote, local_path)
+            Sources.logger.info("clone remote %s to %s", remote, sources_dir)
-            Sources._check_output("git", "clone", remote, str(local_path), exception=None, logger=Sources.logger)
+            Sources._check_output("git", "clone", remote, str(sources_dir), exception=None, logger=Sources.logger)
         # and now force reset to our branch
-        Sources._check_output("git", "checkout", "--force", branch,
+        Sources._check_output("git", "checkout", "--force", Sources._branch,
-                              exception=None, cwd=local_path, logger=Sources.logger)
+                              exception=None, cwd=sources_dir, logger=Sources.logger)
-        Sources._check_output("git", "reset", "--hard", f"origin/{branch}",
+        Sources._check_output("git", "reset", "--hard", f"origin/{Sources._branch}",
-                              exception=None, cwd=local_path, logger=Sources.logger)
+                              exception=None, cwd=sources_dir, logger=Sources.logger)
 
     @staticmethod
-    def load(local_path: Path, remote: str, patch_dir: Path) -> None:
+    def init(sources_dir: Path) -> None:
+        """
+        create empty git repository at the specified path
+        :param sources_dir: local path to sources
+        """
+        Sources._check_output("git", "init", "--initial-branch", Sources._branch,
+                              exception=None, cwd=sources_dir, logger=Sources.logger)
+
+    @staticmethod
+    def load(sources_dir: Path, remote: str, patch_dir: Path) -> None:
         """
         fetch sources from remote and apply patches
-        :param local_path: local path to fetch
+        :param sources_dir: local path to fetch
         :param remote: remote target (from where to fetch)
         :param patch_dir: path to directory with package patches
         """
-        Sources.fetch(local_path, remote)
+        Sources.fetch(sources_dir, remote)
-        Sources.patch_apply(local_path, patch_dir)
+        Sources.patch_apply(sources_dir, patch_dir)
 
     @staticmethod
-    def patch_apply(local_path: Path, patch_dir: Path) -> None:
+    def patch_apply(sources_dir: Path, patch_dir: Path) -> None:
         """
         apply patches if any
-        :param local_path: local path to directory with git sources
+        :param sources_dir: local path to directory with git sources
         :param patch_dir: path to directory with package patches
         """
         # check if even there are patches
@@ -110,15 +135,15 @@ class Sources:
         for patch in patches:
             Sources.logger.info("apply patch %s", patch.name)
             Sources._check_output("git", "apply", "--ignore-space-change", "--ignore-whitespace", str(patch),
-                                  exception=None, cwd=local_path, logger=Sources.logger)
+                                  exception=None, cwd=sources_dir, logger=Sources.logger)
 
     @staticmethod
-    def patch_create(local_path: Path, patch_path: Path, *pattern: str) -> None:
+    def patch_create(sources_dir: Path, patch_path: Path, *pattern: str) -> None:
         """
         create patch set for the specified local path
-        :param local_path: local path to git repository
+        :param sources_dir: local path to git repository
         :param patch_path: path to result patch
         :param pattern: glob patterns
         """
-        Sources.add(local_path, *pattern)
+        Sources.add(sources_dir, *pattern)
-        Sources.diff(local_path, patch_path)
+        Sources.diff(sources_dir, patch_path)

src/ahriman/core/repository/executor.py

@@ -72,9 +72,16 @@ class Executor(Cleaner):
         :param packages: list of package names or bases to remove
         :return: path to repository database
         """
-        def remove_single(package: str, fn: Path) -> None:
+        def remove_base(package_base: str) -> None:
             try:
-                self.repo.remove(package, fn)
+                self.paths.tree_clear(package_base)  # remove all internal files
+                self.reporter.remove(package_base)  # we only update status page in case of base removal
+            except Exception:
+                self.logger.exception("could not remove base %s", package_base)
+
+        def remove_package(package: str, fn: Path) -> None:
+            try:
+                self.repo.remove(package, fn)  # remove the package itself
             except Exception:
                 self.logger.exception("could not remove %s", package)
 
@@ -86,7 +93,7 @@ class Executor(Cleaner):
                     for package, properties in local.packages.items()
                     if properties.filename is not None
                 }
-                self.reporter.remove(local.base)  # we only update status page in case of base removal
+                remove_base(local.base)
             elif requested.intersection(local.packages.keys()):
                 to_remove = {
                     package: Path(properties.filename)
@@ -95,8 +102,9 @@ class Executor(Cleaner):
                 }
             else:
                 to_remove = {}
+
             for package, filename in to_remove.items():
-                remove_single(package, filename)
+                remove_package(package, filename)
 
         return self.repo.repo_path
 

src/ahriman/core/repository/properties.py

@@ -58,7 +58,7 @@ class Properties:
         self.name = configuration.get("repository", "name")
 
         self.paths = RepositoryPaths(configuration.getpath("repository", "root"), architecture)
-        self.paths.create_tree()
+        self.paths.tree_create()
 
         self.ignore_list = configuration.getlist("build", "ignore_packages", fallback=[])
         self.pacman = Pacman(configuration)

src/ahriman/models/package_source.py

@@ -30,14 +30,16 @@ class PackageSource(Enum):
     package source for addition enumeration
     :cvar Auto: automatically determine type of the source
     :cvar Archive: source is a package archive
-    :cvar Directory: source is a directory which contains packages
     :cvar AUR: source is an AUR package for which it should search
+    :cvar Directory: source is a directory which contains packages
+    :cvar Local: source is locally stored PKGBUILD
     """
 
     Auto = "auto"
     Archive = "archive"
-    Directory = "directory"
     AUR = "aur"
+    Directory = "directory"
+    Local = "local"
 
     def resolve(self, source: str) -> PackageSource:
         """
@@ -47,7 +49,10 @@ class PackageSource(Enum):
         """
         if self != PackageSource.Auto:
             return self
+
         maybe_path = Path(source)
+        if (maybe_path / "PKGBUILD").is_file():
+            return PackageSource.Local
         if maybe_path.is_dir():
             return PackageSource.Directory
         if maybe_path.is_file() and package_like(maybe_path):

src/ahriman/models/repository_paths.py

@@ -19,6 +19,8 @@
 #
 from __future__ import annotations
 
+import shutil
+
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Set, Type
@@ -101,24 +103,12 @@ class RepositoryPaths:
 
     def cache_for(self, package_base: str) -> Path:
         """
-        get cache path for specific package base
+        get path to cached PKGBUILD and package sources for the package base
         :param package_base: package base name
         :return: full path to directory for specified package base cache
         """
         return self.cache / package_base
 
-    def create_tree(self) -> None:
-        """
-        create ahriman working tree
-        """
-        self.cache.mkdir(mode=0o755, parents=True, exist_ok=True)
-        self.chroot.mkdir(mode=0o755, parents=True, exist_ok=True)
-        self.manual.mkdir(mode=0o755, parents=True, exist_ok=True)
-        self.packages.mkdir(mode=0o755, parents=True, exist_ok=True)
-        self.patches.mkdir(mode=0o755, parents=True, exist_ok=True)
-        self.repository.mkdir(mode=0o755, parents=True, exist_ok=True)
-        self.sources.mkdir(mode=0o755, parents=True, exist_ok=True)
-
     def manual_for(self, package_base: str) -> Path:
         """
         get manual path for specific package base
@@ -137,8 +127,33 @@ class RepositoryPaths:
 
     def sources_for(self, package_base: str) -> Path:
         """
-        get sources path for specific package base
+        get path to directory from where build will start for the package base
         :param package_base: package base name
         :return: full path to directory for specified package base sources
         """
         return self.sources / package_base
+
+    def tree_clear(self, package_base: str) -> None:
+        """
+        clear package specific files
+        :param package_base: package base name
+        """
+        for directory in (
+                self.cache_for(package_base),
+                self.manual_for(package_base),
+                self.patches_for(package_base)):
+            shutil.rmtree(directory, ignore_errors=True)
+
+    def tree_create(self) -> None:
+        """
+        create ahriman working tree
+        """
+        for directory in (
+                self.cache,
+                self.chroot,
+                self.manual,
+                self.packages,
+                self.patches,
+                self.repository,
+                self.sources):
+            directory.mkdir(mode=0o755, parents=True, exist_ok=True)

tests/ahriman/application/handlers/test_handler_init.py

@@ -10,11 +10,11 @@ def test_run(args: argparse.Namespace, configuration: Configuration, mocker: Moc
     """
     must run command
     """
-    create_tree_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree")
+    tree_create_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
     init_mock = mocker.patch("ahriman.core.alpm.repo.Repo.init")
 
     Init.run(args, "x86_64", configuration, True)
-    create_tree_mock.assert_called_once()
+    tree_create_mock.assert_called_once()
     init_mock.assert_called_once()
 
 

tests/ahriman/core/repository/test_properties.py

@@ -9,17 +9,17 @@ def test_create_tree_on_load(configuration: Configuration, mocker: MockerFixture
     """
     must create tree on load
     """
-    create_tree_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree")
+    tree_create_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
     Properties("x86_64", configuration, True)
 
-    create_tree_mock.assert_called_once()
+    tree_create_mock.assert_called_once()
 
 
 def test_create_dummy_report_client(configuration: Configuration, mocker: MockerFixture) -> None:
     """
     must create dummy report client if report is disabled
     """
-    mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree")
+    mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
     load_mock = mocker.patch("ahriman.core.status.client.Client.load")
     properties = Properties("x86_64", configuration, True)
 
@@ -31,7 +31,7 @@ def test_create_full_report_client(configuration: Configuration, mocker: MockerF
     """
     must create load report client if report is enabled
     """
-    mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree")
+    mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
     load_mock = mocker.patch("ahriman.core.status.client.Client.load")
     Properties("x86_64", configuration, False)
 

tests/ahriman/models/test_repository_paths.py

@@ -23,27 +23,6 @@ def test_cache_for(repository_paths: RepositoryPaths, package_ahriman: Package)
     assert path.parent == repository_paths.cache
 
 
-def test_create_tree(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
-    """
-    must create whole tree
-    """
-    paths = {
-        prop
-        for prop in dir(repository_paths)
-        if not prop.startswith("_")
-        and not prop.endswith("_for")
-        and prop not in ("architecture", "create_tree", "known_architectures", "root")
-    }
-    mkdir_mock = mocker.patch("pathlib.Path.mkdir")
-
-    repository_paths.create_tree()
-    mkdir_mock.assert_has_calls(
-        [
-            mock.call(mode=0o755, parents=True, exist_ok=True)
-            for _ in paths
-        ])
-
-
 def test_manual_for(repository_paths: RepositoryPaths, package_ahriman: Package) -> None:
     """
     must return correct path for manual directory
@@ -69,3 +48,24 @@ def test_sources_for(repository_paths: RepositoryPaths, package_ahriman: Package
     path = repository_paths.sources_for(package_ahriman.base)
     assert path.name == package_ahriman.base
     assert path.parent == repository_paths.sources
+
+
+def test_tree_create(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
+    """
+    must create whole tree
+    """
+    paths = {
+        prop
+        for prop in dir(repository_paths)
+        if not prop.startswith("_")
+        and not prop.endswith("_for")
+        and prop not in ("architecture", "known_architectures", "root", "tree_clear", "tree_create")
+    }
+    mkdir_mock = mocker.patch("pathlib.Path.mkdir")
+
+    repository_paths.tree_create()
+    mkdir_mock.assert_has_calls(
+        [
+            mock.call(mode=0o755, parents=True, exist_ok=True)
+            for _ in paths
+        ])