Release 0.14.0

directory

README.md

@@ -15,13 +15,54 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github
 ## Installation and run
 
 * Install package as usual.
-* Change settings if required, see `CONFIGURING.md` for more details.
-* Create `/var/lib/ahriman/.makepkg.conf` with `makepkg.conf` overrides if required (at least you might want to set `PACKAGER`).
-* Configure build tools (it might be required if your package will use any custom repositories):
-    * create build command, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/custom-x86_64-build` (you can choose any name for command);
-    * create configuration file, e.g. `cp /usr/share/devtools/pacman-{extra,custom}.conf`;
-    * change configuration file, add your own repository, add multilib repository etc;
-    * set `build.build_command` setting to point to your command;
-    * configure `/etc/sudoers.d/ahriman` to allow running command without password.
-* Start and enable `ahriman.timer` via `systemctl`.
-* Add packages by using `ahriman add {package}` command.
+* Change settings if required, see [CONFIGURING](CONFIGURING.md) for more details.
+* Create `/var/lib/ahriman/.makepkg.conf` with `makepkg.conf` overrides if required (at least you might want to set `PACKAGER`):
+
+    ```shell
+    echo 'PACKAGES="John Doe <john@doe.com>"' | sudo -u ahriman tee -a /var/lib/ahriman/.makepkg.conf
+    ```
+
+* Configure build tools (it is required for correct dependency management system):
+
+    * create build command, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build` (you can choose any name for command, basically it should be `{name}-{arch}-build`);
+    * create configuration file, e.g. `cp /usr/share/devtools/pacman-{extra,ahriman}.conf` (same as previous `pacman-{name}.conf`);
+    * change configuration file, add your own repository, add multilib repository etc. Hint: you can use `Include` option as well;
+    * set `build_command` option to point to your command;
+    * configure `/etc/sudoers.d/ahriman` to allow running command without a password.
+
+    ```shell
+    ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build
+    cp /usr/share/devtools/pacman-{extra,ahriman}.conf
+
+    echo '[multilib]' | tee -a /usr/share/devtools/pacman-ahriman.conf
+    echo 'Include = /etc/pacman.d/mirrorlist' | tee -a /usr/share/devtools/pacman-ahriman.conf
+
+    echo '[aur-clone]' | tee -a /usr/share/devtools/pacman-ahriman.conf
+    echo 'SigLevel = Optional TrustAll' | tee -a /usr/share/devtools/pacman-ahriman.conf
+    echo 'Server = file:///var/lib/ahriman/repository/$arch' | tee -a /usr/share/devtools/pacman-ahriman.conf
+
+    echo '[build]' | tee -a /etc/ahriman.ini.d/build.ini
+    echo 'build_command = ahriman-x86_64-build' | tee -a /etc/ahriman.ini.d/build.ini
+
+    echo 'Cmnd_Alias CARCHBUILD_CMD = /usr/local/bin/ahriman-x86_64-build *' | tee -a /etc/sudoers.d/ahriman
+    echo 'ahriman ALL=(ALL) NOPASSWD: CARCHBUILD_CMD' | tee -a /etc/sudoers.d/ahriman
+    chmod 400 /etc/sudoers.d/ahriman
+    ```
+
+* Start and enable `ahriman@.timer` via `systemctl`:
+
+    ```shell
+    systemctl enable --now ahriman@x86_64.timer
+    ```
+
+* Start and enable status page:
+
+    ```shell
+    systemctl enable --now ahriman-web@x86_64
+    ```
+
+* Add packages by using `ahriman add {package}` command:
+
+    ```shell
+    sudo -u ahriman ahriman -a x86_64 add yay
+    ```

package/archlinux/PKGBUILD

@@ -1,7 +1,7 @@
 # Maintainer: Evgeniy Alekseev
 
 pkgname='ahriman'
-pkgver=0.12.1
+pkgver=0.14.0
 pkgrel=1
 pkgdesc="ArcHlinux ReposItory MANager"
 arch=('any')
@@ -23,7 +23,7 @@ optdepends=('aws-cli: sync to s3'
 source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz"
         'ahriman.sysusers'
         'ahriman.tmpfiles')
-sha512sums=('21a1a216af25aeb580f5f2f574e9eed6eb79b74a1038c40a2e1594f18ca0afcd8a64358d11494b87a9a9ea25d5f6be27c39591b124a6ae2d7d4e78fbe44fdd1d'
+sha512sums=('5d35e9173fdf3809e5b73fcac2976ccf2c528ad23e7faed17389647b728a609b037288b1d438413690fffad7c24d9abbdae0419ee575053fd1d0d88dae6d01a8'
             '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
             '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')
 backup=('etc/ahriman.ini'

package/etc/ahriman.ini.d/logging.ini

@@ -2,17 +2,11 @@
 keys = root,builder,build_details,http
 
 [handlers]
-keys = console_handler,build_file_handler,file_handler,http_handler
+keys = build_file_handler,file_handler,http_handler
 
 [formatters]
 keys = generic_format
 
-[handler_console_handler]
-class = StreamHandler
-level = DEBUG
-formatter = generic_format
-args = (sys.stdout,)
-
 [handler_file_handler]
 class = logging.handlers.RotatingFileHandler
 level = DEBUG

package/share/ahriman/build-status.jinja2

@@ -11,7 +11,11 @@
 
     <body>
         <div class="root">
-            <h1>ahriman {{ version|e }} ({{ architecture|e }})<sup class="service-{{ service.status|e }}" title="{{ service.timestamp }}">{{ service.status|e }}</sup></h1>
+            <h1>ahriman
+                <img src="https://img.shields.io/badge/version-{{ version|e }}-informational" alt="{{ version|e }}">
+                <img src="https://img.shields.io/badge/architecture-{{ architecture|e }}-informational" alt="{{ architecture|e }}">
+                <img src="https://img.shields.io/badge/service%20status-{{ service.status|e }}-{{ service.status_color|e }}" alt="{{ service.status|e }}" title="{{ service.timestamp|e }}">
+            </h1>
 
             {% include "search-line.jinja2" %}
 
@@ -31,11 +35,19 @@
                             <td class="include-search">{{ package.packages|join("<br>"|safe) }}</td>
                             <td>{{ package.version|e }}</td>
                             <td>{{ package.timestamp|e }}</td>
-                            <td class="package-{{ package.status|e }}">{{ package.status|e }}</td>
+                            <td class="status package-{{ package.status|e }}">{{ package.status|e }}</td>
                         </tr>
                     {% endfor %}
                 </table>
             </section>
+
+            <footer>
+                <ul class="navigation">
+                    <li><a href="https://github.com/arcan1s/ahriman" title="sources">ahriman</a></li>
+                    <li><a href="https://github.com/arcan1s/ahriman/releases" title="releases list">releases</a></li>
+                    <li><a href="https://github.com/arcan1s/ahriman/issues" title="issues tracker">report a bug</a></li>
+                </ul>
+            </footer>
         </div>
     </body>
 

package/share/ahriman/repo-index.jinja2

@@ -33,22 +33,30 @@
                     <tr class="header">
                         <th>package</th>
                         <th>version</th>
+                        <th>archive size</th>
                         <th>installed size</th>
+                        <th>build date</th>
                     </tr>
 
                     {% for package in packages %}
                         <tr class="package">
                             <td class="include-search"><a href="{{ link_path|e }}/{{ package.filename|e }}" title="{{ package.name|e }}">{{ package.name|e }}</a></td>
                             <td>{{ package.version|e }}</td>
+                            <td>{{ package.archive_size|e }}</td>
                             <td>{{ package.installed_size|e }}</td>
+                            <td>{{ package.build_date|e }}</td>
                         </tr>
                     {% endfor %}
                 </table>
             </section>
 
-            {% if homepage is not none %}
-                <footer><a href="{{ homepage|e }}" title="homepage">Homepage</a></footer>
-            {% endif %}
+            <footer>
+                <ul class="navigation">
+                    {% if homepage is not none %}
+                        <li><a href="{{ homepage|e }}" title="homepage">Homepage</a></li>
+                    {% endif %}
+                </ul>
+            </footer>
         </div>
     </body>
 </html>

package/share/ahriman/style.jinja2

@@ -5,6 +5,11 @@
         --color-pending: 255, 255, 146;
         --color-success: 94, 255, 94;
         --color-unknown: 225, 225, 225;
+
+        --color-header: 200, 200, 255;
+        --color-hover: 255, 255, 225;
+        --color-line-blue: 235, 235, 255;
+        --color-line-white: 255, 255, 255;
     }
 
     @keyframes blink-building {
@@ -26,7 +31,7 @@
         padding: 15px 15% 0;
     }
 
-    section.element {
+    section.element, footer {
         width: 100%;
         padding: 10px 0;
     }
@@ -40,19 +45,23 @@
     }
 
     tr.package:nth-child(odd) {
-        background-color: rgba(255, 255, 255, 1);
+        background-color: rgba(var(--color-line-white), 1.0);
     }
 
     tr.package:nth-child(even) {
-        background-color: rgba(235, 235, 255, 1);
+        background-color: rgba(var(--color-line-blue), 1.0);
     }
 
     tr.package:hover {
-        background-color: rgba(255, 255, 225, 1);
+        background-color: rgba(var(--color-hover), 1.0);
     }
 
     tr.header{
-        background-color: rgba(200, 200, 255, 1);
+        background-color: rgba(var(--color-header), 1.0);
+    }
+
+    td.status {
+        text-align: center;
     }
 
     td.package-unknown {
@@ -76,12 +85,10 @@
         background-color: rgba(var(--color-success), 1.0);
     }
 
-    sup.service-unknown {
-        font-weight: lighter;
+    li.service-unknown {
         background-color: rgba(var(--color-unknown), 1.0);
     }
-    sup.service-building {
-        font-weight: lighter;
+    li.service-building {
         background-color: rgba(var(--color-building), 1.0);
         animation-name: blink-building;
         animation-duration: 1s;
@@ -89,12 +96,41 @@
         animation-iteration-count: infinite;
         animation-direction: alternate;
     }
-    sup.service-failed {
-        font-weight: lighter;
+    li.service-failed {
         background-color: rgba(var(--color-failed), 1.0);
     }
-    sup.service-success {
-        font-weight: lighter;
+    li.service-success {
         background-color: rgba(var(--color-success), 1.0);
     }
+
+    ul.navigation {
+        list-style-type: none;
+        margin: 0;
+        padding: 0;
+        overflow: hidden;
+        background-color: rgba(var(--color-header), 1.0);
+    }
+
+    ul.navigation li {
+        float: left;
+    }
+
+    ul.navigation li.status {
+        display: block;
+        text-align: center;
+        text-decoration: none;
+        padding: 14px 16px;
+    }
+
+    ul.navigation li a {
+        display: block;
+        color: black;
+        text-align: center;
+        text-decoration: none;
+        padding: 14px 16px;
+    }
+
+    ul.navigation li a:hover {
+        background-color: rgba(var(--color-hover), 1.0);
+    }
 </style>

src/ahriman/application/ahriman.py

@@ -18,6 +18,10 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import argparse
+import logging
+import sys
+
+from multiprocessing import Pool
 
 import ahriman.version as version
 
@@ -26,88 +30,143 @@ from ahriman.application.lock import Lock
 from ahriman.core.configuration import Configuration
 
 
-def add(args: argparse.Namespace, config: Configuration) -> None:
+def _call(args: argparse.Namespace, architecture: str, config: Configuration) -> bool:
+    '''
+    additional function to wrap all calls for multiprocessing library
+    :param args: command line args
+    :param architecture: repository architecture
+    :param config: configuration instance
+    :return: True on success, False otherwise
+    '''
+    try:
+        with Lock(args.lock, architecture, args.force, args.unsafe, config):
+            args.fn(args, architecture, config)
+        return True
+    except Exception:
+        logging.getLogger('root').exception('process exception', exc_info=True)
+        return False
+
+
+def add(args: argparse.Namespace, architecture: str, config: Configuration) -> None:
     '''
     add packages callback
     :param args: command line args
+    :param architecture: repository architecture
     :param config: configuration instance
     '''
-    Application.from_args(args, config).add(args.package, args.without_dependencies)
+    Application(architecture, config).add(args.package, args.without_dependencies)
 
 
-def rebuild(args: argparse.Namespace, config: Configuration) -> None:
+def clean(args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    '''
+    clean caches callback
+    :param args: command line args
+    :param architecture: repository architecture
+    :param config: configuration instance
+    '''
+    Application(architecture, config).clean(args.no_build, args.no_cache, args.no_chroot,
+                                            args.no_manual, args.no_packages)
+
+
+def dump_config(args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    '''
+    configuration dump callback
+    :param args: command line args
+    :param architecture: repository architecture
+    :param config: configuration instance
+    '''
+    result = config.dump(architecture)
+    for section, values in sorted(result.items()):
+        print(f'[{section}]')
+        for key, value in sorted(values.items()):
+            print(f'{key} = {value}')
+        print()
+
+
+def rebuild(args: argparse.Namespace, architecture: str, config: Configuration) -> None:
     '''
     world rebuild callback
     :param args: command line args
+    :param architecture: repository architecture
     :param config: configuration instance
     '''
-    app = Application.from_args(args, config)
+    app = Application(architecture, config)
     packages = app.repository.packages()
     app.update(packages)
 
 
-def remove(args: argparse.Namespace, config: Configuration) -> None:
+def remove(args: argparse.Namespace, architecture: str, config: Configuration) -> None:
     '''
     remove packages callback
     :param args: command line args
+    :param architecture: repository architecture
     :param config: configuration instance
     '''
-    Application.from_args(args, config).remove(args.package)
+    Application(architecture, config).remove(args.package)
 
 
-def report(args: argparse.Namespace, config: Configuration) -> None:
+def report(args: argparse.Namespace, architecture: str, config: Configuration) -> None:
     '''
     generate report callback
     :param args: command line args
+    :param architecture: repository architecture
     :param config: configuration instance
     '''
-    Application.from_args(args, config).report(args.target)
+    Application(architecture, config).report(args.target)
 
 
-def sync(args: argparse.Namespace, config: Configuration) -> None:
+def sync(args: argparse.Namespace, architecture: str, config: Configuration) -> None:
     '''
     sync to remote server callback
     :param args: command line args
+    :param architecture: repository architecture
     :param config: configuration instance
     '''
-    Application.from_args(args, config).sync(args.target)
+    Application(architecture, config).sync(args.target)
 
 
-def update(args: argparse.Namespace, config: Configuration) -> None:
+def update(args: argparse.Namespace, architecture: str, config: Configuration) -> None:
     '''
     update packages callback
     :param args: command line args
+    :param architecture: repository architecture
     :param config: configuration instance
     '''
     # typing workaround
     def log_fn(line: str) -> None:
-        return print(line) if args.dry_run else app.logger.info(line)
+        return print(line) if args.dry_run else application.logger.info(line)
 
-    app = Application.from_args(args, config)
-    packages = app.get_updates(args.package, args.no_aur, args.no_manual, args.no_vcs, log_fn)
+    application = Application(architecture, config)
+    packages = application.get_updates(args.package, args.no_aur, args.no_manual, args.no_vcs, log_fn)
     if args.dry_run:
         return
 
-    app.update(packages)
+    application.update(packages)
 
 
-def web(args: argparse.Namespace, config: Configuration) -> None:
+def web(args: argparse.Namespace, architecture: str, config: Configuration) -> None:
     '''
     web server callback
     :param args: command line args
+    :param architecture: repository architecture
     :param config: configuration instance
     '''
     from ahriman.web.web import run_server, setup_service
-    app = setup_service(args.architecture, config)
-    run_server(app, args.architecture)
+    application = setup_service(architecture, config)
+    run_server(application, architecture)
 
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(prog='ahriman', description='ArcHlinux ReposItory MANager')
-    parser.add_argument('-a', '--architecture', help='target architecture', required=True)
+    parser.add_argument(
+        '-a',
+        '--architecture',
+        help='target architectures (can be used multiple times)',
+        action='append')
     parser.add_argument('-c', '--config', help='configuration path', default='/etc/ahriman.ini')
     parser.add_argument('--force', help='force run, remove file lock', action='store_true')
     parser.add_argument('--lock', help='lock file', default='/tmp/ahriman.lock')
+    parser.add_argument('--unsafe', help='allow to run ahriman as non-ahriman user', action='store_true')
     parser.add_argument('-v', '--version', action='version', version=version.__version__)
     subparsers = parser.add_subparsers(title='command')
 
@@ -118,7 +177,22 @@ if __name__ == '__main__':
 
     check_parser = subparsers.add_parser('check', description='check for updates')
     check_parser.add_argument('package', help='filter check by packages', nargs='*')
-    check_parser.set_defaults(fn=update, no_aur=False, no_manual=True, no_vcs=False, dry_run=True)
+    check_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true')
+    check_parser.set_defaults(fn=update, no_aur=False, no_manual=True, dry_run=True)
+
+    clean_parser = subparsers.add_parser('clean', description='clear all local caches')
+    clean_parser.add_argument('--no-build', help='do not clear directory with package sources', action='store_true')
+    clean_parser.add_argument('--no-cache', help='do not clear directory with package caches', action='store_true')
+    clean_parser.add_argument('--no-chroot', help='do not clear build chroot', action='store_true')
+    clean_parser.add_argument(
+        '--no-manual',
+        help='do not clear directory with manually added packages',
+        action='store_true')
+    clean_parser.add_argument('--no-packages', help='do not clear directory with built packages', action='store_true')
+    clean_parser.set_defaults(fn=clean)
+
+    config_parser = subparsers.add_parser('config', description='dump configuration for specified architecture')
+    config_parser.set_defaults(fn=dump_config)
 
     rebuild_parser = subparsers.add_parser('rebuild', description='rebuild whole repository')
     rebuild_parser.set_defaults(fn=rebuild)
@@ -153,5 +227,7 @@ if __name__ == '__main__':
         exit(1)
 
     config = Configuration.from_path(args.config)
-    with Lock(args.lock, args.architecture, args.force, config):
-        args.fn(args, config)
+    with Pool(len(args.architecture)) as pool:
+        result = pool.starmap(_call, [(args, architecture, config) for architecture in args.architecture])
+
+    sys.exit(0 if all(result) else 1)

src/ahriman/application/application.py

@@ -17,14 +17,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from __future__ import annotations
-
-import argparse
 import logging
 import os
 import shutil
 
-from typing import Callable, Iterable, List, Optional, Set, Type
+from typing import Callable, Iterable, List, Optional, Set
 
 from ahriman.core.build_tools.task import Task
 from ahriman.core.configuration import Configuration
@@ -53,16 +50,6 @@ class Application:
         self.architecture = architecture
         self.repository = Repository(architecture, config)
 
-    @classmethod
-    def from_args(cls: Type[Application], args: argparse.Namespace, config: Configuration) -> Application:
-        '''
-        constructor which has to be used to build instance from command line args
-        :param args: command line args
-        :param config: configuration instance
-        :return: application instance
-        '''
-        return cls(args.architecture, config)
-
     def _known_packages(self) -> Set[str]:
         '''
         load packages from repository and pacman repositories
@@ -139,6 +126,26 @@ class Application:
         for name in names:
             process_single(name)
 
+    def clean(self, no_build: bool, no_cache: bool, no_chroot: bool, no_manual: bool, no_packages: bool) -> None:
+        '''
+        run all clean methods. Warning: some functions might not be available under non-root
+        :param no_build: do not clear directory with package sources
+        :param no_cache: do not clear directory with package caches
+        :param no_chroot: do not clear build chroot
+        :param no_manual: do not clear directory with manually added packages
+        :param no_packages: do not clear directory with built packages
+        '''
+        if not no_build:
+            self.repository._clear_build()
+        if not no_cache:
+            self.repository._clear_cache()
+        if not no_chroot:
+            self.repository._clear_chroot()
+        if not no_manual:
+            self.repository._clear_manual()
+        if not no_packages:
+            self.repository._clear_packages()
+
     def remove(self, names: Iterable[str]) -> None:
         '''
         remove packages from repository

src/ahriman/application/lock.py

@@ -25,7 +25,7 @@ from types import TracebackType
 from typing import Literal, Optional, Type
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.exceptions import DuplicateRun
+from ahriman.core.exceptions import DuplicateRun, UnsafeRun
 from ahriman.core.watcher.client import Client
 from ahriman.models.build_status import BuildStatusEnum
 
@@ -36,30 +36,38 @@ class Lock:
     :ivar force: remove lock file on start if any
     :ivar path: path to lock file if any
     :ivar reporter: build status reporter instance
+    :ivar root: repository root (i.e. ahriman home)
+    :ivar unsafe: skip user check
     '''
 
-    def __init__(self, path: Optional[str], architecture: str, force: bool, config: Configuration) -> None:
+    def __init__(self, path: Optional[str], architecture: str, force: bool, unsafe: bool,
+                 config: Configuration) -> None:
         '''
         default constructor
         :param path: optional path to lock file, if empty no file lock will be used
         :param architecture: repository architecture
         :param force: remove lock file on start if any
+        :param unsafe: skip user check
         :param config: configuration instance
         '''
         self.path = f'{path}_{architecture}' if path is not None else None
         self.force = force
+        self.unsafe = unsafe
 
+        self.root = config.get('repository', 'root')
         self.reporter = Client.load(architecture, config)
 
     def __enter__(self) -> Lock:
         '''
         default workflow is the following:
 
+            check user UID
             remove lock file if force flag is set
             check if there is lock file
             create lock file
             report to web if enabled
         '''
+        self.check_user()
         if self.force:
             self.remove()
         self.check()
@@ -90,6 +98,17 @@ class Lock:
         if os.path.exists(self.path):
             raise DuplicateRun()
 
+    def check_user(self) -> None:
+        '''
+        check if current user is actually owner of ahriman root
+        '''
+        if self.unsafe:
+            return
+        current_uid = os.getuid()
+        root_uid = os.stat(self.root).st_uid
+        if current_uid != root_uid:
+            raise UnsafeRun(current_uid, root_uid)
+
     def create(self) -> None:
         '''
         create lock file

src/ahriman/core/build_tools/task.py

@@ -58,6 +58,13 @@ class Task:
         self.makepkg_flags = config.getlist(section, 'makepkg_flags')
         self.makechrootpkg_flags = config.getlist(section, 'makechrootpkg_flags')
 
+    @property
+    def cache_path(self) -> str:
+        '''
+        :return: path to cached packages
+        '''
+        return os.path.join(self.paths.cache, self.package.base)
+
     @property
     def git_path(self) -> str:
         '''
@@ -66,14 +73,21 @@ class Task:
         return os.path.join(self.paths.sources, self.package.base)
 
     @staticmethod
-    def fetch(local: str, remote: str) -> None:
+    def fetch(local: str, remote: str, branch: str = 'master') -> None:
         '''
-        fetch package from git
+        either clone repository or update it to origin/`branch`
         :param local: local path to fetch
         :param remote: remote target (from where to fetch)
+        :param branch: branch name to checkout, master by default
         '''
-        shutil.rmtree(local, ignore_errors=True)  # remove in case if file exists
-        check_output('git', 'clone', remote, local, exception=None)
+        logger = logging.getLogger('build_details')
+        # local directory exists and there is .git directory
+        if os.path.isdir(os.path.join(local, '.git')):
+            check_output('git', 'fetch', 'origin', branch, exception=None, cwd=local, logger=logger)
+        else:
+            check_output('git', 'clone', remote, local, exception=None, logger=logger)
+        # and now force reset to our branch
+        check_output('git', 'reset', '--hard', f'origin/{branch}', exception=None, cwd=local, logger=logger)
 
     def build(self) -> List[str]:
         '''
@@ -95,12 +109,16 @@ class Task:
         # well it is not actually correct, but we can deal with it
         return check_output('makepkg', '--packagelist',
                             exception=BuildFailed(self.package.base),
-                            cwd=self.git_path).splitlines()
+                            cwd=self.git_path,
+                            logger=self.build_logger).splitlines()
 
-    def clone(self, path: Optional[str] = None) -> None:
+    def init(self, path: Optional[str] = None) -> None:
         '''
         fetch package from git
         :param path: optional local path to fetch. If not set default path will be used
         '''
         git_path = path or self.git_path
+        if os.path.isdir(self.cache_path):
+            # no need to clone whole repository, just copy from cache first
+            shutil.copytree(self.cache_path, git_path)
         return Task.fetch(git_path, self.package.git_url)

src/ahriman/core/configuration.py

@@ -20,18 +20,29 @@
 from __future__ import annotations
 
 import configparser
+import logging
 import os
 
 from logging.config import fileConfig
-from typing import List, Optional, Type
+from typing import Dict, List, Optional, Type
 
 
 class Configuration(configparser.RawConfigParser):
     '''
     extension for built-in configuration parser
     :ivar path: path to root configuration file
+    :cvar ARCHITECTURE_SPECIFIC_SECTIONS: known sections which can be architecture specific (required by dump)
+    :cvar DEFAULT_LOG_FORMAT: default log format (in case of fallback)
+    :cvar DEFAULT_LOG_LEVEL: default log level (in case of fallback)
+    :cvar STATIC_SECTIONS: known sections which are not architecture specific (required by dump)
     '''
 
+    DEFAULT_LOG_FORMAT = '%(asctime)s : %(levelname)s : %(funcName)s : %(message)s'
+    DEFAULT_LOG_LEVEL = logging.DEBUG
+
+    STATIC_SECTIONS = ['alpm', 'report', 'repository', 'settings', 'upload']
+    ARCHITECTURE_SPECIFIC_SECTIONS = ['build', 'html', 'rsync', 's3', 'sign', 'web']
+
     def __init__(self) -> None:
         '''
         default constructor
@@ -58,6 +69,25 @@ class Configuration(configparser.RawConfigParser):
         config.load_logging()
         return config
 
+    def dump(self, architecture: str) -> Dict[str, Dict[str, str]]:
+        '''
+        dump configuration to dictionary
+        :param architecture: repository architecture
+        :return: configuration dump for specific architecture
+        '''
+        result: Dict[str, Dict[str, str]] = {}
+        for section in Configuration.STATIC_SECTIONS:
+            if not self.has_section(section):
+                continue
+            result[section] = dict(self[section])
+        for group in Configuration.ARCHITECTURE_SPECIFIC_SECTIONS:
+            section = self.get_section_name(group, architecture)
+            if not self.has_section(section):
+                continue
+            result[section] = dict(self[section])
+
+        return result
+
     def getlist(self, section: str, key: str) -> List[str]:
         '''
         get space separated string list option
@@ -103,4 +133,9 @@ class Configuration(configparser.RawConfigParser):
         '''
         setup logging settings from configuration
         '''
-        fileConfig(self.get('settings', 'logging'))
+        try:
+            fileConfig(self.get('settings', 'logging'))
+        except PermissionError:
+            logging.basicConfig(filename=None, format=Configuration.DEFAULT_LOG_FORMAT,
+                                level=Configuration.DEFAULT_LOG_LEVEL)
+            logging.error('could not create logfile, fallback to stderr', exc_info=True)

src/ahriman/core/exceptions.py

@@ -105,3 +105,19 @@ class SyncFailed(Exception):
         default constructor
         '''
         Exception.__init__(self, 'Sync failed')
+
+
+class UnsafeRun(Exception):
+    '''
+    exception which will be raised in case if user is not owner of repository
+    '''
+
+    def __init__(self, current_uid: int, root_uid: int) -> None:
+        '''
+        default constructor
+        '''
+        Exception.__init__(
+            self,
+            f'''Current UID {current_uid} differs from root owner {root_uid}.
+Note that for the most actions it is unsafe to run application as different user.
+If you are 100% sure that it must be there try --unsafe option''')

src/ahriman/core/report/html.py

@@ -24,8 +24,8 @@ from typing import Callable, Dict, Iterable
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.report.report import Report
+from ahriman.core.util import pretty_size, pretty_datetime
 from ahriman.models.package import Package
-from ahriman.models.package_desciption import PackageDescription
 from ahriman.models.sign_settings import SignSettings
 
 
@@ -39,7 +39,7 @@ class HTML(Report):
         link_path - prefix fo packages to download, string, required
         has_package_signed - True in case if package sign enabled, False otherwise, required
         has_repo_signed - True in case if repository database sign enabled, False otherwise, required
-        packages - sorted list of packages properties: filename, installed_size, name, version. Required
+        packages - sorted list of packages properties: archive_size, build_date, filename, installed_size, name, version. Required
         pgp_key - default PGP key ID, string, optional
         repository - repository name, string, required
 
@@ -85,8 +85,10 @@ class HTML(Report):
 
         content = [
             {
+                'archive_size': pretty_size(properties.archive_size),
+                'build_date': pretty_datetime(properties.build_date),
                 'filename': properties.filename,
-                'installed_size': PackageDescription.size_to_str(properties.installed_size),
+                'installed_size': pretty_size(properties.installed_size),
                 'name': package,
                 'version': base.version
             } for base in packages for package, properties in base.packages.items()

src/ahriman/core/repository.py

@@ -79,6 +79,20 @@ class Repository:
         for package in os.listdir(self.paths.sources):
             shutil.rmtree(os.path.join(self.paths.sources, package))
 
+    def _clear_cache(self) -> None:
+        '''
+        clear cache directory
+        '''
+        for package in os.listdir(self.paths.cache):
+            shutil.rmtree(os.path.join(self.paths.cache, package))
+
+    def _clear_chroot(self) -> None:
+        '''
+        clear cache directory. Warning: this method is architecture independent and will clear every chroot
+        '''
+        for chroot in os.listdir(self.paths.chroot):
+            shutil.rmtree(os.path.join(self.paths.chroot, chroot))
+
     def _clear_manual(self) -> None:
         '''
         clear directory with manual package updates
@@ -130,7 +144,7 @@ class Repository:
         def build_single(package: Package) -> None:
             self.reporter.set_building(package.base)
             task = Task(package, self.architecture, self.config, self.paths)
-            task.clone()
+            task.init()
             built = task.build()
             for src in built:
                 dst = os.path.join(self.paths.packages, os.path.basename(src))
@@ -238,7 +252,7 @@ class Repository:
 
             try:
                 remote = Package.load(local.base, self.pacman, self.aur_url)
-                if local.is_outdated(remote):
+                if local.is_outdated(remote, self.paths):
                     result.append(remote)
                     self.reporter.set_pending(local.base)
             except Exception:

src/ahriman/core/util.py

@@ -17,11 +17,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import datetime
 import subprocess
 
 from logging import Logger
 from typing import Optional
 
+from ahriman.core.exceptions import InvalidOption
+
 
 def check_output(*args: str, exception: Optional[Exception],
                  cwd: Optional[str] = None, stderr: int = subprocess.STDOUT,
@@ -55,3 +58,37 @@ def package_like(filename: str) -> bool:
     :return: True in case if name contains `.pkg.` and not signature, False otherwise
     '''
     return '.pkg.' in filename and not filename.endswith('.sig')
+
+
+def pretty_datetime(timestamp: Optional[int]) -> str:
+    '''
+    convert datetime object to string
+    :param timestamp: datetime to convert
+    :return: pretty printable datetime as string
+    '''
+    return '' if timestamp is None else datetime.datetime.utcfromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
+
+
+def pretty_size(size: Optional[float], level: int = 0) -> str:
+    '''
+    convert size to string
+    :param size: size to convert
+    :param level: represents current units, 0 is B, 1 is KiB etc
+    :return: pretty printable size as string
+    '''
+    def str_level() -> str:
+        if level == 0:
+            return 'B'
+        elif level == 1:
+            return 'KiB'
+        elif level == 2:
+            return 'MiB'
+        elif level == 3:
+            return 'GiB'
+        raise InvalidOption(level)  # I hope it will not be more than 1024 GiB
+
+    if size is None:
+        return ''
+    elif size < 1024:
+        return f'{round(size, 2)} {str_level()}'
+    return pretty_size(size / 1024, level + 1)

src/ahriman/models/build_status.py

@@ -39,6 +39,21 @@ class BuildStatusEnum(Enum):
     Failed = 'failed'
     Success = 'success'
 
+    def badges_color(self) -> str:
+        '''
+        convert itself to shield.io badges color
+        :return: shields.io color
+        '''
+        if self == BuildStatusEnum.Pending:
+            return 'yellow'
+        elif self == BuildStatusEnum.Building:
+            return 'yellow'
+        elif self == BuildStatusEnum.Failed:
+            return 'critical'
+        elif self == BuildStatusEnum.Success:
+            return 'success'
+        return 'inactive'
+
 
 class BuildStatus:
     '''
@@ -48,18 +63,11 @@ class BuildStatus:
     '''
 
     def __init__(self, status: Union[BuildStatusEnum, str, None] = None,
-                 timestamp: Optional[datetime.datetime] = None) -> None:
+                 timestamp: Optional[int] = None) -> None:
         '''
         default constructor
         :param status: current build status if known. `BuildStatusEnum.Unknown` will be used if not set
         :param timestamp: build status timestamp. Current timestamp will be used if not set
         '''
         self.status = BuildStatusEnum(status) if status else BuildStatusEnum.Unknown
-        self._timestamp = timestamp or datetime.datetime.utcnow()
-
-    @property
-    def timestamp(self) -> str:
-        '''
-        :return: string representation of build status timestamp
-        '''
-        return self._timestamp.strftime('%Y-%m-%d %H:%M:%S')
+        self.timestamp = timestamp or int(datetime.datetime.utcnow().timestamp())

src/ahriman/models/package.py

@@ -19,10 +19,10 @@
 #
 from __future__ import annotations
 
+import logging
+
 import aur  # type: ignore
 import os
-import shutil
-import tempfile
 
 from dataclasses import dataclass
 from pyalpm import vercmp  # type: ignore
@@ -33,6 +33,7 @@ from ahriman.core.alpm.pacman import Pacman
 from ahriman.core.exceptions import InvalidPackageInfo
 from ahriman.core.util import check_output
 from ahriman.models.package_desciption import PackageDescription
+from ahriman.models.repository_paths import RepositoryPaths
 
 
 @dataclass
@@ -76,30 +77,30 @@ class Package:
         '''
         return f'{self.aur_url}/packages/{self.base}'
 
-    def actual_version(self) -> str:
+    def actual_version(self, paths: RepositoryPaths) -> str:
         '''
         additional method to handle VCS package versions
+        :param paths: repository paths instance
         :return: package version if package is not VCS and current version according to VCS otherwise
         '''
         if not self.is_vcs:
             return self.version
 
         from ahriman.core.build_tools.task import Task
-        clone_dir = tempfile.mkdtemp()
-        try:
-            Task.fetch(clone_dir, self.git_url)
-            # update pkgver first
-            check_output('makepkg', '--nodeps', '--nobuild',
-                         exception=None, cwd=clone_dir)
-            # generate new .SRCINFO and put it to parser
-            src_info_source = check_output('makepkg', '--printsrcinfo',
-                                           exception=None, cwd=clone_dir)
-            src_info, errors = parse_srcinfo(src_info_source)
-            if errors:
-                raise InvalidPackageInfo(errors)
-            return self.full_version(src_info.get('epoch'), src_info['pkgver'], src_info['pkgrel'])
-        finally:
-            shutil.rmtree(clone_dir, ignore_errors=True)
+
+        clone_dir = os.path.join(paths.cache, self.base)
+        logger = logging.getLogger('build_details')
+        Task.fetch(clone_dir, self.git_url)
+
+        # update pkgver first
+        check_output('makepkg', '--nodeps', '--nobuild', exception=None, cwd=clone_dir, logger=logger)
+        # generate new .SRCINFO and put it to parser
+        src_info_source = check_output('makepkg', '--printsrcinfo', exception=None, cwd=clone_dir, logger=logger)
+        src_info, errors = parse_srcinfo(src_info_source)
+        if errors:
+            raise InvalidPackageInfo(errors)
+
+        return self.full_version(src_info.get('epoch'), src_info['pkgver'], src_info['pkgrel'])
 
     @classmethod
     def from_archive(cls: Type[Package], path: str, pacman: Pacman, aur_url: str) -> Package:
@@ -111,7 +112,7 @@ class Package:
         :return: package properties
         '''
         package = pacman.handle.load_pkg(path)
-        properties = PackageDescription(os.path.basename(path), package.isize)
+        properties = PackageDescription(package.size, package.builddate, os.path.basename(path), package.isize)
         return cls(package.base, package.version, aur_url, {package.name: properties})
 
     @classmethod
@@ -196,12 +197,13 @@ class Package:
         except Exception as e:
             raise InvalidPackageInfo(str(e))
 
-    def is_outdated(self, remote: Package) -> bool:
+    def is_outdated(self, remote: Package, paths: RepositoryPaths) -> bool:
         '''
         check if package is out-of-dated
         :param remote: package properties from remote source
+        :param paths: repository paths instance. Required for VCS packages cache
         :return: True if the package is out-of-dated and False otherwise
         '''
-        remote_version = remote.actual_version()  # either normal version or updated VCS
+        remote_version = remote.actual_version(paths)  # either normal version or updated VCS
         result: int = vercmp(self.version, remote_version)
         return result < 0

src/ahriman/models/package_desciption.py

@@ -20,38 +20,18 @@
 from dataclasses import dataclass
 from typing import Optional
 
-from ahriman.core.exceptions import InvalidOption
-
 
 @dataclass
 class PackageDescription:
     '''
     package specific properties
+    :ivar archive_size: package archive size
+    :ivar build_date: package build date
+    :ivar filename: package archive name
+    :ivar installed_size: package installed size
     '''
+
+    archive_size: Optional[int] = None
+    build_date: Optional[int] = None
     filename: Optional[str] = None
     installed_size: Optional[int] = None
-
-    @staticmethod
-    def size_to_str(size: Optional[float], level: int = 0) -> str:
-        '''
-        convert size to string
-        :param size: size to convert
-        :param level: represents current units, 0 is B, 1 is KiB etc
-        :return: pretty printable size as string
-        '''
-        def str_level() -> str:
-            if level == 0:
-                return 'B'
-            elif level == 1:
-                return 'KiB'
-            elif level == 2:
-                return 'MiB'
-            elif level == 3:
-                return 'GiB'
-            raise InvalidOption(level)
-
-        if size is None:
-            return ''
-        elif size < 1024:
-            return f'{round(size, 2)} {str_level()}'
-        return PackageDescription.size_to_str(size / 1024, level + 1)

src/ahriman/models/report_settings.py

@@ -27,7 +27,7 @@ from ahriman.core.exceptions import InvalidOption
 class ReportSettings(Enum):
     '''
     report targets enumeration
-    :ivar HTML: html report generation
+    :cvar HTML: html report generation
     '''
 
     HTML = auto()

src/ahriman/models/repository_paths.py

@@ -25,7 +25,7 @@ from dataclasses import dataclass
 @dataclass
 class RepositoryPaths:
     '''
-    repository paths holder
+    repository paths holder. For the most operations with paths you want to use this object
     :ivar root: repository root (i.e. ahriman home)
     :ivar architecture: repository architecture
     '''
@@ -33,11 +33,19 @@ class RepositoryPaths:
     root: str
     architecture: str
 
+    @property
+    def cache(self) -> str:
+        '''
+        :return: directory for packages cache (mainly used for VCS packages)
+        '''
+        return os.path.join(self.root, 'cache')
+
     @property
     def chroot(self) -> str:
         '''
         :return: directory for devtools chroot
         '''
+        # for the chroot directory devtools will create own tree and we don't have to specify architecture here
         return os.path.join(self.root, 'chroot')
 
     @property
@@ -45,14 +53,14 @@ class RepositoryPaths:
         '''
         :return: directory for manual updates (i.e. from add command)
         '''
-        return os.path.join(self.root, 'manual')
+        return os.path.join(self.root, 'manual', self.architecture)
 
     @property
     def packages(self) -> str:
         '''
         :return: directory for built packages
         '''
-        return os.path.join(self.root, 'packages')
+        return os.path.join(self.root, 'packages', self.architecture)
 
     @property
     def repository(self) -> str:
@@ -66,12 +74,13 @@ class RepositoryPaths:
         '''
         :return: directory for downloaded PKGBUILDs for current build
         '''
-        return os.path.join(self.root, 'sources')
+        return os.path.join(self.root, 'sources', self.architecture)
 
     def create_tree(self) -> None:
         '''
         create ahriman working tree
         '''
+        os.makedirs(self.cache, mode=0o755, exist_ok=True)
         os.makedirs(self.chroot, mode=0o755, exist_ok=True)
         os.makedirs(self.manual, mode=0o755, exist_ok=True)
         os.makedirs(self.packages, mode=0o755, exist_ok=True)

src/ahriman/models/sign_settings.py

@@ -27,8 +27,8 @@ from ahriman.core.exceptions import InvalidOption
 class SignSettings(Enum):
     '''
     sign targets enumeration
-    :ivar SignPackages: sign each package
-    :ivar SignRepository: sign repository database file
+    :cvar SignPackages: sign each package
+    :cvar SignRepository: sign repository database file
     '''
 
     SignPackages = auto()

src/ahriman/models/upload_settings.py

@@ -27,8 +27,8 @@ from ahriman.core.exceptions import InvalidOption
 class UploadSettings(Enum):
     '''
     remote synchronization targets enumeration
-    :ivar Rsync: sync via rsync
-    :ivar S3: sync to Amazon S3
+    :cvar Rsync: sync via rsync
+    :cvar S3: sync to Amazon S3
     '''
 
     Rsync = auto()

src/ahriman/version.py

@@ -17,4 +17,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-__version__ = '0.12.1'
+__version__ = '0.14.0'

src/ahriman/web/views/index.py

@@ -23,6 +23,7 @@ from typing import Any, Dict
 
 import ahriman.version as version
 
+from ahriman.core.util import pretty_datetime
 from ahriman.web.views.base import BaseView
 
 
@@ -36,7 +37,7 @@ class IndexView(BaseView):
         packages - sorted list of packages properties: base, packages (sorted list), status,
                    timestamp, version, web_url. Required
         repository - repository name, string, required
-        service - service status properties: status, timestamp. Required
+        service - service status properties: status, status_color, timestamp. Required
         version - ahriman version, string, required
     '''
 
@@ -52,14 +53,15 @@ class IndexView(BaseView):
                 'base': package.base,
                 'packages': [p for p in sorted(package.packages)],
                 'status': status.status.value,
-                'timestamp': status.timestamp,
+                'timestamp': pretty_datetime(status.timestamp),
                 'version': package.version,
                 'web_url': package.web_url
             } for package, status in sorted(self.service.packages, key=lambda item: item[0].base)
         ]
         service = {
             'status': self.service.status.status.value,
-            'timestamp': self.service.status.timestamp
+            'status_color': self.service.status.status.badges_color(),
+            'timestamp': pretty_datetime(self.service.status.timestamp)
         }
 
         return {

src/ahriman/web/web.py

@@ -63,7 +63,8 @@ def run_server(application: web.Application, architecture: str) -> None:
     host = application['config'].get(section, 'host')
     port = application['config'].getint(section, 'port')
 
-    web.run_app(application, host=host, port=port, handle_signals=False)
+    web.run_app(application, host=host, port=port, handle_signals=False,
+                access_log=logging.getLogger('http'))
 
 
 def setup_service(architecture: str, config: Configuration) -> web.Application:
@@ -73,22 +74,22 @@ def setup_service(architecture: str, config: Configuration) -> web.Application:
     :param config: configuration instance
     :return: web application instance
     '''
-    app = web.Application(logger=logging.getLogger('http'))
-    app.on_shutdown.append(on_shutdown)
-    app.on_startup.append(on_startup)
+    application = web.Application(logger=logging.getLogger('http'))
+    application.on_shutdown.append(on_shutdown)
+    application.on_startup.append(on_startup)
 
-    app.middlewares.append(web.normalize_path_middleware(append_slash=False, remove_slash=True))
-    app.middlewares.append(exception_handler(app.logger))
+    application.middlewares.append(web.normalize_path_middleware(append_slash=False, remove_slash=True))
+    application.middlewares.append(exception_handler(application.logger))
 
-    app.logger.info('setup routes')
-    setup_routes(app)
-    app.logger.info('setup templates')
-    aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader(config.get('web', 'templates')))
+    application.logger.info('setup routes')
+    setup_routes(application)
+    application.logger.info('setup templates')
+    aiohttp_jinja2.setup(application, loader=jinja2.FileSystemLoader(config.get('web', 'templates')))
 
-    app.logger.info('setup configuration')
-    app['config'] = config
+    application.logger.info('setup configuration')
+    application['config'] = config
 
-    app.logger.info('setup watcher')
-    app['watcher'] = Watcher(architecture, config)
+    application.logger.info('setup watcher')
+    application['watcher'] = Watcher(architecture, config)
 
-    return app
+    return application