Release 0.13.0

package/archlinux/PKGBUILD

@@ -1,7 +1,7 @@
 # Maintainer: Evgeniy Alekseev
 
 pkgname='ahriman'
-pkgver=0.12.2
+pkgver=0.13.0
 pkgrel=1
 pkgdesc="ArcHlinux ReposItory MANager"
 arch=('any')
@@ -23,7 +23,7 @@ optdepends=('aws-cli: sync to s3'
 source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz"
         'ahriman.sysusers'
         'ahriman.tmpfiles')
-sha512sums=('8a140f11819a103b50bc8e8477f0d49a2c7468da4fbcaebc9b0519b29d964b9ca04224d171e3a6ecf00e371d09427c45d363e781d2df82cf0c6cbe9d9c829cfe'
+sha512sums=('b835d745fb77e400ca31ba4d93547b7db8e9dfe5d6c04b60e3953efeeaa7f561a1c60b2ade2684d3c7ba9a87e470c65610f33340315f192661c1676746b91298'
             '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
             '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')
 backup=('etc/ahriman.ini'

package/share/ahriman/repo-index.jinja2

@@ -33,14 +33,18 @@
                     <tr class="header">
                         <th>package</th>
                         <th>version</th>
+                        <th>archive size</th>
                         <th>installed size</th>
+                        <th>build date</th>
                     </tr>
 
                     {% for package in packages %}
                         <tr class="package">
                             <td class="include-search"><a href="{{ link_path|e }}/{{ package.filename|e }}" title="{{ package.name|e }}">{{ package.name|e }}</a></td>
                             <td>{{ package.version|e }}</td>
+                            <td>{{ package.archive_size|e }}</td>
                             <td>{{ package.installed_size|e }}</td>
+                            <td>{{ package.build_date|e }}</td>
                         </tr>
                     {% endfor %}
                 </table>

src/ahriman/application/ahriman.py

@@ -49,6 +49,16 @@ def add(args: argparse.Namespace, architecture: str, config: Configuration) -> N
     Application(architecture, config).add(args.package, args.without_dependencies)
 
 
+def clean(args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    '''
+    clean caches callback
+    :param args: command line args
+    :param architecture: repository architecture
+    :param config: configuration instance
+    '''
+    Application(architecture, config).clean()
+
+
 def rebuild(args: argparse.Namespace, architecture: str, config: Configuration) -> None:
     '''
     world rebuild callback
@@ -141,6 +151,9 @@ if __name__ == '__main__':
     check_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true')
     check_parser.set_defaults(fn=update, no_aur=False, no_manual=True, dry_run=True)
 
+    clean_parser = subparsers.add_parser('clean', description='clear all local caches')
+    clean_parser.set_defaults(fn=clean)
+
     rebuild_parser = subparsers.add_parser('rebuild', description='rebuild whole repository')
     rebuild_parser.set_defaults(fn=rebuild)
 

src/ahriman/application/application.py

@@ -17,13 +17,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from __future__ import annotations
-
 import logging
 import os
 import shutil
 
-from typing import Callable, Iterable, List, Optional, Set, Type
+from typing import Callable, Iterable, List, Optional, Set
 
 from ahriman.core.build_tools.task import Task
 from ahriman.core.configuration import Configuration
@@ -128,6 +126,16 @@ class Application:
         for name in names:
             process_single(name)
 
+    def clean(self) -> None:
+        '''
+        run all clean methods
+        '''
+        self.repository._clear_build()
+        self.repository._clear_cache()
+        self.repository._clear_chroot()
+        self.repository._clear_manual()
+        self.repository._clear_packages()
+
     def remove(self, names: Iterable[str]) -> None:
         '''
         remove packages from repository

src/ahriman/core/build_tools/task.py

@@ -58,6 +58,13 @@ class Task:
         self.makepkg_flags = config.getlist(section, 'makepkg_flags')
         self.makechrootpkg_flags = config.getlist(section, 'makechrootpkg_flags')
 
+    @property
+    def cache_path(self) -> str:
+        '''
+        :return: path to cached packages
+        '''
+        return os.path.join(self.paths.cache, self.package.base)
+
     @property
     def git_path(self) -> str:
         '''
@@ -66,14 +73,19 @@ class Task:
         return os.path.join(self.paths.sources, self.package.base)
 
     @staticmethod
-    def fetch(local: str, remote: str) -> None:
+    def fetch(local: str, remote: str, branch: str = 'master') -> None:
         '''
-        fetch package from git
+        either clone repository or update it to origin/`branch`
         :param local: local path to fetch
         :param remote: remote target (from where to fetch)
+        :param branch: branch name to checkout, master by default
         '''
-        shutil.rmtree(local, ignore_errors=True)  # remove in case if file exists
+        if os.path.isdir(local):
+            check_output('git', 'fetch', 'origin', branch, cwd=local, exception=None)
+        else:
             check_output('git', 'clone', remote, local, exception=None)
+        # and now force reset to our branch
+        check_output('git', 'reset', '--hard', f'origin/{branch}', cwd=local, exception=None)
 
     def build(self) -> List[str]:
         '''
@@ -97,10 +109,13 @@ class Task:
                             exception=BuildFailed(self.package.base),
                             cwd=self.git_path).splitlines()
 
-    def clone(self, path: Optional[str] = None) -> None:
+    def init(self, path: Optional[str] = None) -> None:
         '''
         fetch package from git
         :param path: optional local path to fetch. If not set default path will be used
         '''
         git_path = path or self.git_path
+        if os.path.isdir(self.cache_path):
+            # no need to clone whole repository, just copy from cache first
+            shutil.copytree(self.cache_path, git_path)
         return Task.fetch(git_path, self.package.git_url)

src/ahriman/core/report/html.py

@@ -24,8 +24,8 @@ from typing import Callable, Dict, Iterable
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.report.report import Report
+from ahriman.core.util import pretty_size, pretty_datetime
 from ahriman.models.package import Package
-from ahriman.models.package_desciption import PackageDescription
 from ahriman.models.sign_settings import SignSettings
 
 
@@ -39,7 +39,7 @@ class HTML(Report):
         link_path - prefix fo packages to download, string, required
         has_package_signed - True in case if package sign enabled, False otherwise, required
         has_repo_signed - True in case if repository database sign enabled, False otherwise, required
-        packages - sorted list of packages properties: filename, installed_size, name, version. Required
+        packages - sorted list of packages properties: archive_size, build_date, filename, installed_size, name, version. Required
         pgp_key - default PGP key ID, string, optional
         repository - repository name, string, required
 
@@ -85,8 +85,10 @@ class HTML(Report):
 
         content = [
             {
+                'archive_size': pretty_size(properties.archive_size),
+                'build_date': pretty_datetime(properties.build_date),
                 'filename': properties.filename,
-                'installed_size': PackageDescription.size_to_str(properties.installed_size),
+                'installed_size': pretty_size(properties.installed_size),
                 'name': package,
                 'version': base.version
             } for base in packages for package, properties in base.packages.items()

src/ahriman/core/repository.py

@@ -79,6 +79,20 @@ class Repository:
         for package in os.listdir(self.paths.sources):
             shutil.rmtree(os.path.join(self.paths.sources, package))
 
+    def _clear_cache(self) -> None:
+        '''
+        clear cache directory
+        '''
+        for package in os.listdir(self.paths.cache):
+            shutil.rmtree(os.path.join(self.paths.cache, package))
+
+    def _clear_chroot(self) -> None:
+        '''
+        clear cache directory. Warning: this method is architecture independent and will clear every chroot
+        '''
+        for chroot in os.listdir(self.paths.chroot):
+            shutil.rmtree(os.path.join(self.paths.chroot, chroot))
+
     def _clear_manual(self) -> None:
         '''
         clear directory with manual package updates
@@ -130,7 +144,7 @@ class Repository:
         def build_single(package: Package) -> None:
             self.reporter.set_building(package.base)
             task = Task(package, self.architecture, self.config, self.paths)
-            task.clone()
+            task.init()
             built = task.build()
             for src in built:
                 dst = os.path.join(self.paths.packages, os.path.basename(src))
@@ -238,7 +252,7 @@ class Repository:
 
             try:
                 remote = Package.load(local.base, self.pacman, self.aur_url)
-                if local.is_outdated(remote):
+                if local.is_outdated(remote, self.paths):
                     result.append(remote)
                     self.reporter.set_pending(local.base)
             except Exception:

src/ahriman/core/util.py

@@ -17,11 +17,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import datetime
 import subprocess
 
 from logging import Logger
 from typing import Optional
 
+from ahriman.core.exceptions import InvalidOption
+
 
 def check_output(*args: str, exception: Optional[Exception],
                  cwd: Optional[str] = None, stderr: int = subprocess.STDOUT,
@@ -55,3 +58,37 @@ def package_like(filename: str) -> bool:
     :return: True in case if name contains `.pkg.` and not signature, False otherwise
     '''
     return '.pkg.' in filename and not filename.endswith('.sig')
+
+
+def pretty_datetime(timestamp: Optional[datetime.datetime]) -> str:
+    '''
+    convert datetime object to string
+    :param timestamp: datetime to convert
+    :return: pretty printable datetime as string
+    '''
+    return '' if timestamp is None else timestamp.strftime('%Y-%m-%d %H:%M:%S')
+
+
+def pretty_size(size: Optional[float], level: int = 0) -> str:
+    '''
+    convert size to string
+    :param size: size to convert
+    :param level: represents current units, 0 is B, 1 is KiB etc
+    :return: pretty printable size as string
+    '''
+    def str_level() -> str:
+        if level == 0:
+            return 'B'
+        elif level == 1:
+            return 'KiB'
+        elif level == 2:
+            return 'MiB'
+        elif level == 3:
+            return 'GiB'
+        raise InvalidOption(level)  # I hope it will not be more than 1024 GiB
+
+    if size is None:
+        return ''
+    elif size < 1024:
+        return f'{round(size, 2)} {str_level()}'
+    return pretty_size(size / 1024, level + 1)

src/ahriman/models/build_status.py

@@ -70,11 +70,4 @@ class BuildStatus:
         :param timestamp: build status timestamp. Current timestamp will be used if not set
         '''
         self.status = BuildStatusEnum(status) if status else BuildStatusEnum.Unknown
-        self._timestamp = timestamp or datetime.datetime.utcnow()
+        self.timestamp = timestamp or datetime.datetime.utcnow()
-
-    @property
-    def timestamp(self) -> str:
-        '''
-        :return: string representation of build status timestamp
-        '''
-        return self._timestamp.strftime('%Y-%m-%d %H:%M:%S')

src/ahriman/models/package.py

@@ -20,9 +20,8 @@
 from __future__ import annotations
 
 import aur  # type: ignore
+import datetime
 import os
-import shutil
-import tempfile
 
 from dataclasses import dataclass
 from pyalpm import vercmp  # type: ignore
@@ -33,6 +32,7 @@ from ahriman.core.alpm.pacman import Pacman
 from ahriman.core.exceptions import InvalidPackageInfo
 from ahriman.core.util import check_output
 from ahriman.models.package_desciption import PackageDescription
+from ahriman.models.repository_paths import RepositoryPaths
 
 
 @dataclass
@@ -76,30 +76,28 @@ class Package:
         '''
         return f'{self.aur_url}/packages/{self.base}'
 
-    def actual_version(self) -> str:
+    def actual_version(self, paths: RepositoryPaths) -> str:
         '''
         additional method to handle VCS package versions
+        :param paths: repository paths instance
         :return: package version if package is not VCS and current version according to VCS otherwise
         '''
         if not self.is_vcs:
             return self.version
 
         from ahriman.core.build_tools.task import Task
-        clone_dir = tempfile.mkdtemp()
+        clone_dir = os.path.join(paths.cache, self.base)
-        try:
+
         Task.fetch(clone_dir, self.git_url)
         # update pkgver first
-            check_output('makepkg', '--nodeps', '--nobuild',
+        check_output('makepkg', '--nodeps', '--nobuild', exception=None, cwd=clone_dir)
-                         exception=None, cwd=clone_dir)
         # generate new .SRCINFO and put it to parser
-            src_info_source = check_output('makepkg', '--printsrcinfo',
+        src_info_source = check_output('makepkg', '--printsrcinfo', exception=None, cwd=clone_dir)
-                                           exception=None, cwd=clone_dir)
         src_info, errors = parse_srcinfo(src_info_source)
         if errors:
             raise InvalidPackageInfo(errors)
+
         return self.full_version(src_info.get('epoch'), src_info['pkgver'], src_info['pkgrel'])
-        finally:
-            shutil.rmtree(clone_dir, ignore_errors=True)
 
     @classmethod
     def from_archive(cls: Type[Package], path: str, pacman: Pacman, aur_url: str) -> Package:
@@ -111,7 +109,8 @@ class Package:
         :return: package properties
         '''
         package = pacman.handle.load_pkg(path)
-        properties = PackageDescription(os.path.basename(path), package.isize)
+        build_date = datetime.datetime.fromtimestamp(package.builddate)
+        properties = PackageDescription(package.size, build_date, os.path.basename(path), package.isize)
         return cls(package.base, package.version, aur_url, {package.name: properties})
 
     @classmethod
@@ -196,12 +195,13 @@ class Package:
         except Exception as e:
             raise InvalidPackageInfo(str(e))
 
-    def is_outdated(self, remote: Package) -> bool:
+    def is_outdated(self, remote: Package, paths: RepositoryPaths) -> bool:
         '''
         check if package is out-of-dated
         :param remote: package properties from remote source
+        :param paths: repository paths instance. Required for VCS packages cache
         :return: True if the package is out-of-dated and False otherwise
         '''
-        remote_version = remote.actual_version()  # either normal version or updated VCS
+        remote_version = remote.actual_version(paths)  # either normal version or updated VCS
         result: int = vercmp(self.version, remote_version)
         return result < 0

src/ahriman/models/package_desciption.py

@@ -17,41 +17,23 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import datetime
+
 from dataclasses import dataclass
 from typing import Optional
 
-from ahriman.core.exceptions import InvalidOption
-
 
 @dataclass
 class PackageDescription:
     '''
     package specific properties
+    :ivar archive_size: package archive size
+    :ivar build_date: package build date
+    :ivar filename: package archive name
+    :ivar installed_size: package installed size
     '''
+
+    archive_size: Optional[int] = None
+    build_date: Optional[datetime.datetime] = None
     filename: Optional[str] = None
     installed_size: Optional[int] = None
-
-    @staticmethod
-    def size_to_str(size: Optional[float], level: int = 0) -> str:
-        '''
-        convert size to string
-        :param size: size to convert
-        :param level: represents current units, 0 is B, 1 is KiB etc
-        :return: pretty printable size as string
-        '''
-        def str_level() -> str:
-            if level == 0:
-                return 'B'
-            elif level == 1:
-                return 'KiB'
-            elif level == 2:
-                return 'MiB'
-            elif level == 3:
-                return 'GiB'
-            raise InvalidOption(level)
-
-        if size is None:
-            return ''
-        elif size < 1024:
-            return f'{round(size, 2)} {str_level()}'
-        return PackageDescription.size_to_str(size / 1024, level + 1)

src/ahriman/models/report_settings.py

@@ -27,7 +27,7 @@ from ahriman.core.exceptions import InvalidOption
 class ReportSettings(Enum):
     '''
     report targets enumeration
-    :ivar HTML: html report generation
+    :cvar HTML: html report generation
     '''
 
     HTML = auto()

src/ahriman/models/repository_paths.py

@@ -33,6 +33,13 @@ class RepositoryPaths:
     root: str
     architecture: str
 
+    @property
+    def cache(self) -> str:
+        '''
+        :return: directory for packages cache (mainly used for VCS packages)
+        '''
+        return os.path.join(self.root, 'cache')
+
     @property
     def chroot(self) -> str:
         '''
@@ -73,6 +80,7 @@ class RepositoryPaths:
         '''
         create ahriman working tree
         '''
+        os.makedirs(self.cache, mode=0o755, exist_ok=True)
         os.makedirs(self.chroot, mode=0o755, exist_ok=True)
         os.makedirs(self.manual, mode=0o755, exist_ok=True)
         os.makedirs(self.packages, mode=0o755, exist_ok=True)

src/ahriman/models/sign_settings.py

@@ -27,8 +27,8 @@ from ahriman.core.exceptions import InvalidOption
 class SignSettings(Enum):
     '''
     sign targets enumeration
-    :ivar SignPackages: sign each package
+    :cvar SignPackages: sign each package
-    :ivar SignRepository: sign repository database file
+    :cvar SignRepository: sign repository database file
     '''
 
     SignPackages = auto()

src/ahriman/models/upload_settings.py

@@ -27,8 +27,8 @@ from ahriman.core.exceptions import InvalidOption
 class UploadSettings(Enum):
     '''
     remote synchronization targets enumeration
-    :ivar Rsync: sync via rsync
+    :cvar Rsync: sync via rsync
-    :ivar S3: sync to Amazon S3
+    :cvar S3: sync to Amazon S3
     '''
 
     Rsync = auto()

src/ahriman/version.py

@@ -17,4 +17,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-__version__ = '0.12.2'
+__version__ = '0.13.0'

src/ahriman/web/views/index.py

@@ -23,6 +23,7 @@ from typing import Any, Dict
 
 import ahriman.version as version
 
+from ahriman.core.util import pretty_datetime
 from ahriman.web.views.base import BaseView
 
 
@@ -52,7 +53,7 @@ class IndexView(BaseView):
                 'base': package.base,
                 'packages': [p for p in sorted(package.packages)],
                 'status': status.status.value,
-                'timestamp': status.timestamp,
+                'timestamp': pretty_datetime(status.timestamp),
                 'version': package.version,
                 'web_url': package.web_url
             } for package, status in sorted(self.service.packages, key=lambda item: item[0].base)
@@ -60,7 +61,7 @@ class IndexView(BaseView):
         service = {
             'status': self.service.status.status.value,
             'status_color': self.service.status.status.badges_color(),
-            'timestamp': self.service.status.timestamp
+            'timestamp': pretty_datetime(self.service.status.timestamp)
         }
 
         return {