add ability to add manually stored packages (#40)

* add ability to add manually stored packages

* update tests

* handle manual packages in remove-unknown method

* live fixes

also rename branches to has_remotes method and change return type

src/ahriman/application/ahriman.py

@@ -137,9 +137,16 @@ def _set_package_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
     """
     parser = root.add_parser("package-add", aliases=["add"], help="add package", description="add package",
                              epilog="This subcommand should be used for new package addition. It also supports flag "
-                                    "--now in case if you would like to build the package immediately.",
+                                    "--now in case if you would like to build the package immediately. "
+                                    "You can add new package from one of supported sources: "
+                                    "1) if it is already built package you can specify the path to the archive; "
+                                    "2) you can also add built packages from the directory (e.g. during the migration "
+                                    "from another repository source); "
+                                    "3) it is also possible to add package from local PKGBUILD, but in this case it "
+                                    "will be ignored during the next automatic updates; "
+                                    "4) and finally you can add package from AUR.",
                              formatter_class=_formatter)
-    parser.add_argument("package", help="package base/name or archive path", nargs="+")
+    parser.add_argument("package", help="package base/name or path to local files", nargs="+")
     parser.add_argument("-n", "--now", help="run update function after", action="store_true")
     parser.add_argument("-s", "--source", help="package source",
                         type=PackageSource, choices=PackageSource, default=PackageSource.Auto)

src/ahriman/application/application.py

@@ -105,21 +105,30 @@ class Application:
         :param without_dependencies: if set, dependency check will be disabled
         """
         known_packages = self._known_packages()
+        aur_url = self.configuration.get("alpm", "aur_url")
+
+        def add_archive(src: Path) -> None:
+            dst = self.repository.paths.packages / src.name
+            shutil.move(src, dst)
 
         def add_directory(path: Path) -> None:
             for full_path in filter(package_like, path.iterdir()):
                 add_archive(full_path)
 
-        def add_manual(src: str) -> Path:
-            package = Package.load(src, self.repository.pacman, self.configuration.get("alpm", "aur_url"))
+        def add_local(path: Path) -> Path:
+            package = Package.load(path, self.repository.pacman, aur_url)
+            cache_dir = self.repository.paths.cache_for(package.base)
+            shutil.copytree(path, cache_dir)  # copy package to store in caches
+            Sources.init(cache_dir)  # we need to run init command in directory where we do have permissions
+            shutil.copytree(cache_dir, self.repository.paths.manual_for(package.base))  # copy package for the build
+            return self.repository.paths.manual_for(package.base)
+
+        def add_remote(src: str) -> Path:
+            package = Package.load(src, self.repository.pacman, aur_url)
             Sources.load(self.repository.paths.manual_for(package.base), package.git_url,
                          self.repository.paths.patches_for(package.base))
             return self.repository.paths.manual_for(package.base)
 
-        def add_archive(src: Path) -> None:
-            dst = self.repository.paths.packages / src.name
-            shutil.move(src, dst)
-
         def process_dependencies(path: Path) -> None:
             if without_dependencies:
                 return
@@ -128,12 +137,15 @@ class Application:
 
         def process_single(src: str) -> None:
             resolved_source = source.resolve(src)
-            if resolved_source == PackageSource.Directory:
-                add_directory(Path(src))
-            elif resolved_source == PackageSource.Archive:
+            if resolved_source == PackageSource.Archive:
                 add_archive(Path(src))
-            else:
-                path = add_manual(src)
+            elif resolved_source == PackageSource.AUR:
+                path = add_remote(src)
+                process_dependencies(path)
+            elif resolved_source == PackageSource.Directory:
+                add_directory(Path(src))
+            elif resolved_source == PackageSource.Local:
+                path = add_local(Path(src))
                 process_dependencies(path)
 
         for name in names:
@@ -213,13 +225,22 @@ class Application:
         get packages which were not found in AUR
         :return: unknown package list
         """
-        packages = []
-        for base in self.repository.packages():
+        def has_aur(package_base: str, aur_url: str) -> bool:
             try:
-                _ = Package.from_aur(base.base, base.aur_url)
+                _ = Package.from_aur(package_base, aur_url)
             except Exception:
-                packages.append(base)
-        return packages
+                return False
+            return True
+
+        def has_local(package_base: str) -> bool:
+            cache_dir = self.repository.paths.cache_for(package_base)
+            return cache_dir.is_dir() and not Sources.has_remotes(cache_dir)
+
+        return [
+            package
+            for package in self.repository.packages()
+            if not has_aur(package.base, package.aur_url) and not has_local(package.base)
+        ]
 
     def update(self, updates: Iterable[Package]) -> None:
         """

src/ahriman/core/build_tools/sources.py

@@ -33,72 +33,98 @@ class Sources:
 
     logger = logging.getLogger("build_details")
 
+    _branch = "master"  # in case if BLM would like to change it
     _check_output = check_output
 
     @staticmethod
-    def add(local_path: Path, *pattern: str) -> None:
+    def add(sources_dir: Path, *pattern: str) -> None:
         """
         track found files via git
-        :param local_path: local path to git repository
+        :param sources_dir: local path to git repository
         :param pattern: glob patterns
         """
         # glob directory to find files which match the specified patterns
         found_files: List[Path] = []
         for glob in pattern:
-            found_files.extend(local_path.glob(glob))
+            found_files.extend(sources_dir.glob(glob))
         Sources.logger.info("found matching files %s", found_files)
         # add them to index
-        Sources._check_output("git", "add", "--intent-to-add", *[str(fn.relative_to(local_path)) for fn in found_files],
-                              exception=None, cwd=local_path, logger=Sources.logger)
+        Sources._check_output("git", "add", "--intent-to-add",
+                              *[str(fn.relative_to(sources_dir)) for fn in found_files],
+                              exception=None, cwd=sources_dir, logger=Sources.logger)
 
     @staticmethod
-    def diff(local_path: Path, patch_path: Path) -> None:
+    def diff(sources_dir: Path, patch_path: Path) -> None:
         """
         generate diff from the current version and write it to the output file
-        :param local_path: local path to git repository
+        :param sources_dir: local path to git repository
         :param patch_path: path to result patch
         """
-        patch = Sources._check_output("git", "diff", exception=None, cwd=local_path, logger=Sources.logger)
+        patch = Sources._check_output("git", "diff", exception=None, cwd=sources_dir, logger=Sources.logger)
         patch_path.write_text(patch)
 
     @staticmethod
-    def fetch(local_path: Path, remote: str, branch: str = "master") -> None:
+    def fetch(sources_dir: Path, remote: str) -> None:
         """
         either clone repository or update it to origin/`branch`
-        :param local_path: local path to fetch
+        :param sources_dir: local path to fetch
         :param remote: remote target (from where to fetch)
-        :param branch: branch name to checkout, master by default
         """
         # local directory exists and there is .git directory
-        if (local_path / ".git").is_dir():
-            Sources.logger.info("update HEAD to remote to %s", local_path)
-            Sources._check_output("git", "fetch", "origin", branch,
-                                  exception=None, cwd=local_path, logger=Sources.logger)
+        is_initialized_git = (sources_dir / ".git").is_dir()
+        if is_initialized_git and not Sources.has_remotes(sources_dir):
+            # there is git repository, but no remote configured so far
+            Sources.logger.info("skip update at %s because there are no branches configured", sources_dir)
+            return
+
+        if is_initialized_git:
+            Sources.logger.info("update HEAD to remote at %s", sources_dir)
+            Sources._check_output("git", "fetch", "origin", Sources._branch,
+                                  exception=None, cwd=sources_dir, logger=Sources.logger)
         else:
-            Sources.logger.info("clone remote %s to %s", remote, local_path)
-            Sources._check_output("git", "clone", remote, str(local_path), exception=None, logger=Sources.logger)
+            Sources.logger.info("clone remote %s to %s", remote, sources_dir)
+            Sources._check_output("git", "clone", remote, str(sources_dir), exception=None, logger=Sources.logger)
         # and now force reset to our branch
-        Sources._check_output("git", "checkout", "--force", branch,
-                              exception=None, cwd=local_path, logger=Sources.logger)
-        Sources._check_output("git", "reset", "--hard", f"origin/{branch}",
-                              exception=None, cwd=local_path, logger=Sources.logger)
+        Sources._check_output("git", "checkout", "--force", Sources._branch,
+                              exception=None, cwd=sources_dir, logger=Sources.logger)
+        Sources._check_output("git", "reset", "--hard", f"origin/{Sources._branch}",
+                              exception=None, cwd=sources_dir, logger=Sources.logger)
 
     @staticmethod
-    def load(local_path: Path, remote: str, patch_dir: Path) -> None:
+    def has_remotes(sources_dir: Path) -> bool:
+        """
+        check if there are remotes for the repository
+        :param sources_dir: local path to git repository
+        :return: True in case if there is any remote and false otherwise
+        """
+        remotes = Sources._check_output("git", "remote", exception=None, cwd=sources_dir, logger=Sources.logger)
+        return bool(remotes)
+
+    @staticmethod
+    def init(sources_dir: Path) -> None:
+        """
+        create empty git repository at the specified path
+        :param sources_dir: local path to sources
+        """
+        Sources._check_output("git", "init", "--initial-branch", Sources._branch,
+                              exception=None, cwd=sources_dir, logger=Sources.logger)
+
+    @staticmethod
+    def load(sources_dir: Path, remote: str, patch_dir: Path) -> None:
         """
         fetch sources from remote and apply patches
-        :param local_path: local path to fetch
+        :param sources_dir: local path to fetch
         :param remote: remote target (from where to fetch)
         :param patch_dir: path to directory with package patches
         """
-        Sources.fetch(local_path, remote)
-        Sources.patch_apply(local_path, patch_dir)
+        Sources.fetch(sources_dir, remote)
+        Sources.patch_apply(sources_dir, patch_dir)
 
     @staticmethod
-    def patch_apply(local_path: Path, patch_dir: Path) -> None:
+    def patch_apply(sources_dir: Path, patch_dir: Path) -> None:
         """
         apply patches if any
-        :param local_path: local path to directory with git sources
+        :param sources_dir: local path to directory with git sources
         :param patch_dir: path to directory with package patches
         """
         # check if even there are patches
@@ -110,15 +136,15 @@ class Sources:
         for patch in patches:
             Sources.logger.info("apply patch %s", patch.name)
             Sources._check_output("git", "apply", "--ignore-space-change", "--ignore-whitespace", str(patch),
-                                  exception=None, cwd=local_path, logger=Sources.logger)
+                                  exception=None, cwd=sources_dir, logger=Sources.logger)
 
     @staticmethod
-    def patch_create(local_path: Path, patch_path: Path, *pattern: str) -> None:
+    def patch_create(sources_dir: Path, patch_path: Path, *pattern: str) -> None:
         """
         create patch set for the specified local path
-        :param local_path: local path to git repository
+        :param sources_dir: local path to git repository
         :param patch_path: path to result patch
         :param pattern: glob patterns
         """
-        Sources.add(local_path, *pattern)
-        Sources.diff(local_path, patch_path)
+        Sources.add(sources_dir, *pattern)
+        Sources.diff(sources_dir, patch_path)

src/ahriman/core/repository/executor.py

@@ -72,9 +72,16 @@ class Executor(Cleaner):
         :param packages: list of package names or bases to remove
         :return: path to repository database
         """
-        def remove_single(package: str, fn: Path) -> None:
+        def remove_base(package_base: str) -> None:
             try:
-                self.repo.remove(package, fn)
+                self.paths.tree_clear(package_base)  # remove all internal files
+                self.reporter.remove(package_base)  # we only update status page in case of base removal
+            except Exception:
+                self.logger.exception("could not remove base %s", package_base)
+
+        def remove_package(package: str, fn: Path) -> None:
+            try:
+                self.repo.remove(package, fn)  # remove the package itself
             except Exception:
                 self.logger.exception("could not remove %s", package)
 
@@ -86,7 +93,7 @@ class Executor(Cleaner):
                     for package, properties in local.packages.items()
                     if properties.filename is not None
                 }
-                self.reporter.remove(local.base)  # we only update status page in case of base removal
+                remove_base(local.base)
             elif requested.intersection(local.packages.keys()):
                 to_remove = {
                     package: Path(properties.filename)
@@ -95,8 +102,9 @@ class Executor(Cleaner):
                 }
             else:
                 to_remove = {}
+
             for package, filename in to_remove.items():
-                remove_single(package, filename)
+                remove_package(package, filename)
 
         return self.repo.repo_path
 

src/ahriman/core/repository/properties.py

@@ -58,7 +58,7 @@ class Properties:
         self.name = configuration.get("repository", "name")
 
         self.paths = RepositoryPaths(configuration.getpath("repository", "root"), architecture)
-        self.paths.create_tree()
+        self.paths.tree_create()
 
         self.ignore_list = configuration.getlist("build", "ignore_packages", fallback=[])
         self.pacman = Pacman(configuration)

src/ahriman/models/package_source.py

@@ -30,14 +30,16 @@ class PackageSource(Enum):
     package source for addition enumeration
     :cvar Auto: automatically determine type of the source
     :cvar Archive: source is a package archive
-    :cvar Directory: source is a directory which contains packages
     :cvar AUR: source is an AUR package for which it should search
+    :cvar Directory: source is a directory which contains packages
+    :cvar Local: source is locally stored PKGBUILD
     """
 
     Auto = "auto"
     Archive = "archive"
-    Directory = "directory"
     AUR = "aur"
+    Directory = "directory"
+    Local = "local"
 
     def resolve(self, source: str) -> PackageSource:
         """
@@ -47,7 +49,10 @@ class PackageSource(Enum):
         """
         if self != PackageSource.Auto:
             return self
+
         maybe_path = Path(source)
+        if (maybe_path / "PKGBUILD").is_file():
+            return PackageSource.Local
         if maybe_path.is_dir():
             return PackageSource.Directory
         if maybe_path.is_file() and package_like(maybe_path):

src/ahriman/models/repository_paths.py

@@ -19,6 +19,8 @@
 #
 from __future__ import annotations
 
+import shutil
+
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Set, Type
@@ -101,24 +103,12 @@ class RepositoryPaths:
 
     def cache_for(self, package_base: str) -> Path:
         """
-        get cache path for specific package base
+        get path to cached PKGBUILD and package sources for the package base
         :param package_base: package base name
         :return: full path to directory for specified package base cache
         """
         return self.cache / package_base
 
-    def create_tree(self) -> None:
-        """
-        create ahriman working tree
-        """
-        self.cache.mkdir(mode=0o755, parents=True, exist_ok=True)
-        self.chroot.mkdir(mode=0o755, parents=True, exist_ok=True)
-        self.manual.mkdir(mode=0o755, parents=True, exist_ok=True)
-        self.packages.mkdir(mode=0o755, parents=True, exist_ok=True)
-        self.patches.mkdir(mode=0o755, parents=True, exist_ok=True)
-        self.repository.mkdir(mode=0o755, parents=True, exist_ok=True)
-        self.sources.mkdir(mode=0o755, parents=True, exist_ok=True)
-
     def manual_for(self, package_base: str) -> Path:
         """
         get manual path for specific package base
@@ -137,8 +127,34 @@ class RepositoryPaths:
 
     def sources_for(self, package_base: str) -> Path:
         """
-        get sources path for specific package base
+        get path to directory from where build will start for the package base
         :param package_base: package base name
         :return: full path to directory for specified package base sources
         """
         return self.sources / package_base
+
+    def tree_clear(self, package_base: str) -> None:
+        """
+        clear package specific files
+        :param package_base: package base name
+        """
+        for directory in (
+                self.cache_for(package_base),
+                self.manual_for(package_base),
+                self.patches_for(package_base),
+                self.sources_for(package_base)):
+            shutil.rmtree(directory, ignore_errors=True)
+
+    def tree_create(self) -> None:
+        """
+        create ahriman working tree
+        """
+        for directory in (
+                self.cache,
+                self.chroot,
+                self.manual,
+                self.packages,
+                self.patches,
+                self.repository,
+                self.sources):
+            directory.mkdir(mode=0o755, parents=True, exist_ok=True)