smart fetch & vcs cache

src/ahriman/core/build_tools/task.py

@@ -58,6 +58,13 @@ class Task:
         self.makepkg_flags = config.getlist(section, 'makepkg_flags')
         self.makechrootpkg_flags = config.getlist(section, 'makechrootpkg_flags')
 
+    @property
+    def cache_path(self) -> str:
+        '''
+        :return: path to cached packages
+        '''
+        return os.path.join(self.paths.cache, self.package.base)
+
     @property
     def git_path(self) -> str:
         '''
@@ -66,14 +73,19 @@ class Task:
         return os.path.join(self.paths.sources, self.package.base)
 
     @staticmethod
-    def fetch(local: str, remote: str) -> None:
+    def fetch(local: str, remote: str, branch: str = 'master') -> None:
         '''
-        fetch package from git
+        either clone repository or update it to origin/`branch`
         :param local: local path to fetch
         :param remote: remote target (from where to fetch)
+        :param branch: branch name to checkout, master by default
         '''
-        shutil.rmtree(local, ignore_errors=True)  # remove in case if file exists
-        check_output('git', 'clone', remote, local, exception=None)
+        if os.path.isdir(local):
+            check_output('git', 'fetch', 'origin', branch, cwd=local, exception=None)
+        else:
+            check_output('git', 'clone', remote, local, exception=None)
+        # and now force reset to our branch
+        check_output('git', 'reset', '--hard', f'origin/{branch}', cwd=local, exception=None)
 
     def build(self) -> List[str]:
         '''
@@ -97,10 +109,13 @@ class Task:
                             exception=BuildFailed(self.package.base),
                             cwd=self.git_path).splitlines()
 
-    def clone(self, path: Optional[str] = None) -> None:
+    def init(self, path: Optional[str] = None) -> None:
         '''
         fetch package from git
         :param path: optional local path to fetch. If not set default path will be used
         '''
         git_path = path or self.git_path
+        if os.path.isdir(self.cache_path):
+            # no need to clone whole repository, just copy from cache first
+            shutil.copytree(self.cache_path, git_path)
         return Task.fetch(git_path, self.package.git_url)

src/ahriman/core/repository.py

@@ -130,7 +130,7 @@ class Repository:
         def build_single(package: Package) -> None:
             self.reporter.set_building(package.base)
             task = Task(package, self.architecture, self.config, self.paths)
-            task.clone()
+            task.init()
             built = task.build()
             for src in built:
                 dst = os.path.join(self.paths.packages, os.path.basename(src))
@@ -238,7 +238,7 @@ class Repository:
 
             try:
                 remote = Package.load(local.base, self.pacman, self.aur_url)
-                if local.is_outdated(remote):
+                if local.is_outdated(remote, self.paths):
                     result.append(remote)
                     self.reporter.set_pending(local.base)
             except Exception:

src/ahriman/models/package.py

@@ -21,8 +21,6 @@ from __future__ import annotations
 
 import aur  # type: ignore
 import os
-import shutil
-import tempfile
 
 from dataclasses import dataclass
 from pyalpm import vercmp  # type: ignore
@@ -33,6 +31,7 @@ from ahriman.core.alpm.pacman import Pacman
 from ahriman.core.exceptions import InvalidPackageInfo
 from ahriman.core.util import check_output
 from ahriman.models.package_desciption import PackageDescription
+from ahriman.models.repository_paths import RepositoryPaths
 
 
 @dataclass
@@ -76,30 +75,28 @@ class Package:
         '''
         return f'{self.aur_url}/packages/{self.base}'
 
-    def actual_version(self) -> str:
+    def actual_version(self, paths: RepositoryPaths) -> str:
         '''
         additional method to handle VCS package versions
+        :param paths: repository paths instance
         :return: package version if package is not VCS and current version according to VCS otherwise
         '''
         if not self.is_vcs:
             return self.version
 
         from ahriman.core.build_tools.task import Task
-        clone_dir = tempfile.mkdtemp()
-        try:
-            Task.fetch(clone_dir, self.git_url)
-            # update pkgver first
-            check_output('makepkg', '--nodeps', '--nobuild',
-                         exception=None, cwd=clone_dir)
-            # generate new .SRCINFO and put it to parser
-            src_info_source = check_output('makepkg', '--printsrcinfo',
-                                           exception=None, cwd=clone_dir)
-            src_info, errors = parse_srcinfo(src_info_source)
-            if errors:
-                raise InvalidPackageInfo(errors)
-            return self.full_version(src_info.get('epoch'), src_info['pkgver'], src_info['pkgrel'])
-        finally:
-            shutil.rmtree(clone_dir, ignore_errors=True)
+        clone_dir = os.path.join(paths.cache, self.base)
+
+        Task.fetch(clone_dir, self.git_url)
+        # update pkgver first
+        check_output('makepkg', '--nodeps', '--nobuild', exception=None, cwd=clone_dir)
+        # generate new .SRCINFO and put it to parser
+        src_info_source = check_output('makepkg', '--printsrcinfo', exception=None, cwd=clone_dir)
+        src_info, errors = parse_srcinfo(src_info_source)
+        if errors:
+            raise InvalidPackageInfo(errors)
+
+        return self.full_version(src_info.get('epoch'), src_info['pkgver'], src_info['pkgrel'])
 
     @classmethod
     def from_archive(cls: Type[Package], path: str, pacman: Pacman, aur_url: str) -> Package:
@@ -196,12 +193,13 @@ class Package:
         except Exception as e:
             raise InvalidPackageInfo(str(e))
 
-    def is_outdated(self, remote: Package) -> bool:
+    def is_outdated(self, remote: Package, paths: RepositoryPaths) -> bool:
         '''
         check if package is out-of-dated
         :param remote: package properties from remote source
+        :param paths: repository paths instance. Required for VCS packages cache
         :return: True if the package is out-of-dated and False otherwise
         '''
-        remote_version = remote.actual_version()  # either normal version or updated VCS
+        remote_version = remote.actual_version(paths)  # either normal version or updated VCS
         result: int = vercmp(self.version, remote_version)
         return result < 0

src/ahriman/models/repository_paths.py

@@ -33,6 +33,13 @@ class RepositoryPaths:
     root: str
     architecture: str
 
+    @property
+    def cache(self) -> str:
+        '''
+        :return: directory for packages cache (mainly used for VCS packages)
+        '''
+        return os.path.join(self.root, 'cache')
+
     @property
     def chroot(self) -> str:
         '''
@@ -73,6 +80,7 @@ class RepositoryPaths:
         '''
         create ahriman working tree
         '''
+        os.makedirs(self.cache, mode=0o755, exist_ok=True)
         os.makedirs(self.chroot, mode=0o755, exist_ok=True)
         os.makedirs(self.manual, mode=0o755, exist_ok=True)
         os.makedirs(self.packages, mode=0o755, exist_ok=True)