Release 1.2.0

* enable jinja autoescape by default for jinja raw generator
* allow to search by multiple strings (OR)
* replace test templates by symlink

.bandit-test.yml

@@ -0,0 +1 @@
+skips: ['B101', 'B404']

.bandit.yml

@@ -0,0 +1 @@
+skips: ['B404', 'B603']

.github/workflows/run-tests.yml

@@ -1,4 +1,4 @@
-name: tests 
+name: tests
 
 on:
   push:
@@ -21,5 +21,6 @@ jobs:
           /bin/bash -c "pacman --noconfirm -Syu base-devel python python-pip && \
           pip install -e .[web] && \
           pip install -e .[check] && \
+          pip install -e .[s3] && \
           pip install -e .[test] && \
           make check tests"

.pylintrc

@@ -22,7 +22,7 @@ ignore-patterns=
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 # number of processors available to use.
-jobs=1
+jobs=0
 
 # Control the amount of potential inferred values when inferring a single
 # object. This can help the performance when dealing with large functions or

CONFIGURING.md

@@ -89,10 +89,13 @@ Group name must refer to architecture, e.g. it should be `rsync:x86_64` for x86_
 
 ### `s3:*` groups
 
-Group name must refer to architecture, e.g. it should be `s3:x86_64` for x86_64 architecture. Requires `aws-cli` package to be installed. Do not forget to configure it for user `ahriman`.
+Group name must refer to architecture, e.g. it should be `s3:x86_64` for x86_64 architecture.
 
-* `command` - s3 command to run, space separated list of string, required.
-* `bucket` - bucket name (e.g. `s3://bucket/path`), string, required.
+* `access_key` - AWS access key ID, string, required.
+* `bucket` - bucket name (e.g. `bucket`), string, required.
+* `chunk_size` - chunk size for calculating entity tags, int, optional, default 8 * 1024 * 1024.
+* `region` - bucket region (e.g. `eu-central-1`), string, required.
+* `secret_key` - AWS secret access key, string, required.
 
 ## `web:*` groups
 

Makefile

@@ -23,10 +23,11 @@ archive_directory: $(TARGET_FILES)
 archlinux: archive
 	sed -i "s/pkgver=[0-9.]*/pkgver=$(VERSION)/" package/archlinux/PKGBUILD
 
-check: clean
-	cd src && mypy --implicit-reexport --strict -p "$(PROJECT)"
-	find "src/$(PROJECT)" "tests/$(PROJECT)" -name "*.py" -execdir autopep8 --exit-code --max-line-length 120 -aa -i {} +
-	cd src && pylint --rcfile=../.pylintrc "$(PROJECT)"
+check: clean mypy
+	autopep8 --exit-code --max-line-length 120 -aa -i -j 0 -r "src/$(PROJECT)" "tests/$(PROJECT)"
+	pylint --rcfile=.pylintrc "src/$(PROJECT)"
+	bandit -c .bandit.yml -r "src/$(PROJECT)"
+	bandit -c .bandit-test.yml -r "tests/$(PROJECT)"
 
 clean:
 	find . -type f -name "$(PROJECT)-*-src.tar.xz" -delete
@@ -35,6 +36,10 @@ clean:
 directory: clean
 	mkdir "$(PROJECT)"
 
+mypy:
+	cd src && mypy --implicit-reexport --strict -p "$(PROJECT)" --install-types --non-interactive || true
+	cd src && mypy --implicit-reexport --strict -p "$(PROJECT)"
+
 push: archlinux
 	git add package/archlinux/PKGBUILD src/ahriman/version.py
 	git commit -m "Release $(VERSION)"

package/archlinux/PKGBUILD

@@ -1,7 +1,7 @@
 # Maintainer: Evgeniy Alekseev
 
 pkgname='ahriman'
-pkgver=1.0.0
+pkgver=1.2.0
 pkgrel=1
 pkgdesc="ArcHlinux ReposItory MANager"
 arch=('any')
@@ -9,13 +9,13 @@ url="https://github.com/arcan1s/ahriman"
 license=('GPL3')
 depends=('devtools' 'git' 'pyalpm' 'python-aur' 'python-srcinfo')
 makedepends=('python-pip')
-optdepends=('aws-cli: sync to s3'
-            'breezy: -bzr packages support'
+optdepends=('breezy: -bzr packages support'
             'darcs: -darcs packages support'
             'gnupg: package and repository sign'
             'mercurial: -hg packages support'
             'python-aiohttp: web server'
             'python-aiohttp-jinja2: web server'
+            'python-boto3: sync to s3'
             'python-jinja: html report generation'
             'rsync: sync by using rsync'
             'subversion: -svn packages support')

package/etc/ahriman.ini

@@ -40,8 +40,8 @@ target =
 command = rsync --archive --compress --partial --delete
 
 [s3]
-command = aws s3 sync --quiet --delete
+chunk_size = 8388608
 
 [web]
-host = 0.0.0.0
+host = 127.0.0.1
 templates = /usr/share/ahriman

package/share/ahriman/build-status.jinja2

@@ -1,7 +1,7 @@
 <!doctype html>
 <html lang="en">
     <head>
-        <title>{{ repository|e }}</title>
+        <title>{{ repository }}</title>
 
         {% include "style.jinja2" %}
 
@@ -12,9 +12,9 @@
     <body>
         <div class="root">
             <h1>ahriman
-                <img src="https://img.shields.io/badge/version-{{ version|e }}-informational" alt="{{ version|e }}">
-                <img src="https://img.shields.io/badge/architecture-{{ architecture|e }}-informational" alt="{{ architecture|e }}">
-                <img src="https://img.shields.io/badge/service%20status-{{ service.status|e }}-{{ service.status_color|e }}" alt="{{ service.status|e }}" title="{{ service.timestamp|e }}">
+                <img src="https://img.shields.io/badge/version-{{ version }}-informational" alt="{{ version }}">
+                <img src="https://img.shields.io/badge/architecture-{{ architecture }}-informational" alt="{{ architecture }}">
+                <img src="https://img.shields.io/badge/service%20status-{{ service.status }}-{{ service.status_color }}" alt="{{ service.status }}" title="{{ service.timestamp }}">
             </h1>
 
             {% include "search-line.jinja2" %}
@@ -31,11 +31,11 @@
 
                     {% for package in packages %}
                         <tr class="package">
-                            <td class="include-search"><a href="{{ package.web_url|e }}" title="{{ package.base|e }}">{{ package.base|e }}</a></td>
+                            <td class="include-search"><a href="{{ package.web_url }}" title="{{ package.base }}">{{ package.base }}</a></td>
                             <td class="include-search">{{ package.packages|join("<br>"|safe) }}</td>
-                            <td>{{ package.version|e }}</td>
-                            <td>{{ package.timestamp|e }}</td>
-                            <td class="status package-{{ package.status|e }}">{{ package.status|e }}</td>
+                            <td>{{ package.version }}</td>
+                            <td>{{ package.timestamp }}</td>
+                            <td class="status package-{{ package.status }}">{{ package.status }}</td>
                         </tr>
                     {% endfor %}
                 </table>

package/share/ahriman/repo-index.jinja2

@@ -1,7 +1,7 @@
 <!doctype html>
 <html lang="en">
     <head>
-        <title>{{ repository|e }}</title>
+        <title>{{ repository }}</title>
 
         {% include "style.jinja2" %}
 
@@ -18,13 +18,13 @@
 
                 <section class="element">
                     {% if pgp_key is not none %}
-                        <p>This repository is signed with <a href="http://keys.gnupg.net/pks/lookup?search=0x{{ pgp_key|e }}&fingerprint=on&op=index" title="key search">{{ pgp_key|e }}</a> by default.</p>
+                        <p>This repository is signed with <a href="http://keys.gnupg.net/pks/lookup?search=0x{{ pgp_key }}&fingerprint=on&op=index" title="key search">{{ pgp_key }}</a> by default.</p>
                     {% endif %}
 
                     <code>
                         $ cat /etc/pacman.conf<br>
-                        [{{ repository|e }}]<br>
-                        Server = {{ link_path|e }}<br>
+                        [{{ repository }}]<br>
+                        Server = {{ link_path }}<br>
                         SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Package{% if has_package_signed %}Required{% else %}Never{% endif %} TrustedOnly
                     </code>
                 </section>
@@ -44,11 +44,11 @@
 
                     {% for package in packages %}
                         <tr class="package">
-                            <td class="include-search"><a href="{{ link_path|e }}/{{ package.filename|e }}" title="{{ package.name|e }}">{{ package.name|e }}</a></td>
-                            <td>{{ package.version|e }}</td>
-                            <td>{{ package.archive_size|e }}</td>
-                            <td>{{ package.installed_size|e }}</td>
-                            <td>{{ package.build_date|e }}</td>
+                            <td class="include-search"><a href="{{ link_path }}/{{ package.filename }}" title="{{ package.name }}">{{ package.name }}</a></td>
+                            <td>{{ package.version }}</td>
+                            <td>{{ package.archive_size }}</td>
+                            <td>{{ package.installed_size }}</td>
+                            <td>{{ package.build_date }}</td>
                         </tr>
                     {% endfor %}
                 </table>
@@ -58,7 +58,7 @@
                 <footer>
                     <ul class="navigation">
                         {% if homepage is not none %}
-                            <li><a href="{{ homepage|e }}" title="homepage">Homepage</a></li>
+                            <li><a href="{{ homepage }}" title="homepage">Homepage</a></li>
                         {% endif %}
                     </ul>
                 </footer>

package/share/ahriman/search-line.jinja2

@@ -1,3 +1,3 @@
 <section class="element">
     <input type="search" id="search" onkeyup="searchInTable()" placeholder="search for package" title="search for package"/>
-</section>
+</section>

package/share/ahriman/search.jinja2

@@ -5,21 +5,22 @@
         const tables = document.getElementsByClassName("search-table");
 
         for (let i = 0; i < tables.length; i++) {
-            const tr = tables[i].getElementsByTagName("tr");
+            const trs = tables[i].getElementsByTagName("tr");
             // from 1 coz of header
-            for (let i = 1; i < tr.length; i++) {
-                let td = tr[i].getElementsByClassName("include-search");
+            for (let i = 1; i < trs.length; i++) {
+                let tr = trs[i].getElementsByClassName("include-search");
                 let display = "none";
-                for (let j = 0; j < td.length; j++) {
-                    if (td[j].tagName.toLowerCase() === "td") {
-                        if (td[j].innerHTML.toLowerCase().indexOf(filter) > -1) {
+                for (let j = 0; j < tr.length; j++) {
+                    if (tr[j].tagName.toLowerCase() === "td") {
+                        let contains = (element) => tr[j].innerHTML.toLowerCase().indexOf(element) > -1
+                        if (filter.some(contains)) {
                             display = "";
                             break;
                         }
                     }
                 }
-                tr[i].style.display = display;
+                trs[i].style.display = display;
             }
         }
     }
-</script>
+</script>

package/share/ahriman/sorttable.jinja2

@@ -1 +1 @@
-<script src="https://www.kryogenix.org/code/browser/sorttable/sorttable.js"></script>
+<script src="https://www.kryogenix.org/code/browser/sorttable/sorttable.js"></script>

package/share/ahriman/style.jinja2

@@ -133,4 +133,4 @@
     ul.navigation li a:hover {
         background-color: rgba(var(--color-hover), 1.0);
     }
-</style>
+</style>

setup.py

@@ -74,9 +74,13 @@ setup(
     extras_require={
         "check": [
             "autopep8",
+            "bandit",
             "mypy",
             "pylint",
         ],
+        "s3": [
+            "boto3",
+        ],
         "test": [
             "pytest",
             "pytest-aiohttp",

src/ahriman/application/ahriman.py

@@ -19,6 +19,7 @@
 #
 import argparse
 import sys
+import tempfile
 
 from pathlib import Path
 
@@ -41,10 +42,17 @@ def _parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(prog="ahriman", description="ArcHlinux ReposItory MANager",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("-a", "--architecture", help="target architectures (can be used multiple times)",
-                        action="append", required=True)
+                        action="append")
     parser.add_argument("-c", "--configuration", help="configuration path", type=Path, default=Path("/etc/ahriman.ini"))
     parser.add_argument("--force", help="force run, remove file lock", action="store_true")
-    parser.add_argument("-l", "--lock", help="lock file", type=Path, default=Path("/tmp/ahriman.lock"))
+    parser.add_argument(
+        "-l",
+        "--lock",
+        help="lock file",
+        type=Path,
+        default=Path(
+            tempfile.gettempdir()) /
+        "ahriman.lock")
     parser.add_argument("--no-log", help="redirect all log messages to stderr", action="store_true")
     parser.add_argument("--no-report", help="force disable reporting to web service", action="store_true")
     parser.add_argument("--unsafe", help="allow to run ahriman as non-ahriman user", action="store_true")
@@ -56,6 +64,7 @@ def _parser() -> argparse.ArgumentParser:
     _set_check_parser(subparsers)
     _set_clean_parser(subparsers)
     _set_config_parser(subparsers)
+    _set_init_parser(subparsers)
     _set_key_import_parser(subparsers)
     _set_rebuild_parser(subparsers)
     _set_remove_parser(subparsers)
@@ -83,7 +92,7 @@ def _set_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser.add_argument("package", help="package base/name or archive path", nargs="+")
     parser.add_argument("--now", help="run update function after", action="store_true")
     parser.add_argument("--without-dependencies", help="do not add dependencies", action="store_true")
-    parser.set_defaults(handler=handlers.Add)
+    parser.set_defaults(handler=handlers.Add, architecture=[])
     return parser
 
 
@@ -98,7 +107,7 @@ def _set_check_parser(root: SubParserAction) -> argparse.ArgumentParser:
                              formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("package", help="filter check by package base", nargs="*")
     parser.add_argument("--no-vcs", help="do not check VCS packages", action="store_true")
-    parser.set_defaults(handler=handlers.Update, no_aur=False, no_manual=True, dry_run=True)
+    parser.set_defaults(handler=handlers.Update, architecture=[], no_aur=False, no_manual=True, dry_run=True)
     return parser
 
 
@@ -115,7 +124,7 @@ def _set_clean_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser.add_argument("--no-chroot", help="do not clear build chroot", action="store_true")
     parser.add_argument("--no-manual", help="do not clear directory with manually added packages", action="store_true")
     parser.add_argument("--no-packages", help="do not clear directory with built packages", action="store_true")
-    parser.set_defaults(handler=handlers.Clean, unsafe=True)
+    parser.set_defaults(handler=handlers.Clean, architecture=[], no_log=True, unsafe=True)
     return parser
 
 
@@ -128,7 +137,20 @@ def _set_config_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser = root.add_parser("config", help="dump configuration",
                              description="dump configuration for specified architecture",
                              formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.set_defaults(handler=handlers.Dump, lock=None, no_report=True, unsafe=True)
+    parser.set_defaults(handler=handlers.Dump, lock=None, no_log=True, no_report=True, unsafe=True)
+    return parser
+
+
+def _set_init_parser(root: SubParserAction) -> argparse.ArgumentParser:
+    """
+    add parser for init subcommand
+    :param root: subparsers for the commands
+    :return: created argument parser
+    """
+    parser = root.add_parser("init", help="create repository tree",
+                             description="create empty repository tree. Optional command for auto architecture support",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.set_defaults(handler=handlers.Init, no_report=True)
     return parser
 
 
@@ -143,7 +165,7 @@ def _set_key_import_parser(root: SubParserAction) -> argparse.ArgumentParser:
                              formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("--key-server", help="key server for key import", default="keys.gnupg.net")
     parser.add_argument("key", help="PGP key to import from public server")
-    parser.set_defaults(handler=handlers.KeyImport, lock=None, no_report=True)
+    parser.set_defaults(handler=handlers.KeyImport, architecture=[""], lock=None, no_report=True)
     return parser
 
 
@@ -155,8 +177,8 @@ def _set_rebuild_parser(root: SubParserAction) -> argparse.ArgumentParser:
     """
     parser = root.add_parser("rebuild", help="rebuild repository", description="rebuild whole repository",
                              formatter_class=argparse.ArgumentDefaultsHelpFormatter)
-    parser.add_argument("--depends-on", help="only rebuild packages that depend on specified package")
-    parser.set_defaults(handler=handlers.Rebuild)
+    parser.add_argument("--depends-on", help="only rebuild packages that depend on specified package", action="append")
+    parser.set_defaults(handler=handlers.Rebuild, architecture=[])
     return parser
 
 
@@ -169,7 +191,7 @@ def _set_remove_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser = root.add_parser("remove", help="remove package", description="remove package",
                              formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("package", help="package name or base", nargs="+")
-    parser.set_defaults(handler=handlers.Remove)
+    parser.set_defaults(handler=handlers.Remove, architecture=[])
     return parser
 
 
@@ -182,7 +204,7 @@ def _set_report_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser = root.add_parser("report", help="generate report", description="generate report",
                              formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("target", help="target to generate report", nargs="*")
-    parser.set_defaults(handler=handlers.Report)
+    parser.set_defaults(handler=handlers.Report, architecture=[])
     return parser
 
 
@@ -194,7 +216,7 @@ def _set_search_parser(root: SubParserAction) -> argparse.ArgumentParser:
     """
     parser = root.add_parser("search", help="search for package", description="search for package in AUR using API")
     parser.add_argument("search", help="search terms, can be specified multiple times", nargs="+")
-    parser.set_defaults(handler=handlers.Search, lock=None, no_report=True, unsafe=True)
+    parser.set_defaults(handler=handlers.Search, architecture=[""], lock=None, no_log=True, no_report=True, unsafe=True)
     return parser
 
 
@@ -215,9 +237,9 @@ def _set_setup_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser.add_argument("--repository", help="repository name", required=True)
     parser.add_argument("--sign-key", help="sign key id")
     parser.add_argument("--sign-target", help="sign options", type=SignSettings.from_option,
-                        choices=SignSettings, nargs="*")
+                        choices=SignSettings, action="append")
     parser.add_argument("--web-port", help="port of the web service", type=int)
-    parser.set_defaults(handler=handlers.Setup, lock=None, no_report=True, unsafe=True)
+    parser.set_defaults(handler=handlers.Setup, lock=None, no_log=True, no_report=True, unsafe=True)
     return parser
 
 
@@ -230,7 +252,7 @@ def _set_sign_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser = root.add_parser("sign", help="sign packages", description="(re-)sign packages and repository database",
                              formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("package", help="sign only specified packages", nargs="*")
-    parser.set_defaults(handler=handlers.Sign)
+    parser.set_defaults(handler=handlers.Sign, architecture=[])
     return parser
 
 
@@ -244,7 +266,7 @@ def _set_status_parser(root: SubParserAction) -> argparse.ArgumentParser:
                              formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("--ahriman", help="get service status itself", action="store_true")
     parser.add_argument("package", help="filter status by package base", nargs="*")
-    parser.set_defaults(handler=handlers.Status, lock=None, no_report=True, unsafe=True)
+    parser.set_defaults(handler=handlers.Status, lock=None, no_log=True, no_report=True, unsafe=True)
     return parser
 
 
@@ -263,7 +285,7 @@ def _set_status_update_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser.add_argument("--status", help="new status", choices=BuildStatusEnum,
                         type=BuildStatusEnum, default=BuildStatusEnum.Success)
     parser.add_argument("--remove", help="remove package status page", action="store_true")
-    parser.set_defaults(handler=handlers.StatusUpdate, lock=None, no_report=True, unsafe=True)
+    parser.set_defaults(handler=handlers.StatusUpdate, lock=None, no_log=True, no_report=True, unsafe=True)
     return parser
 
 
@@ -276,7 +298,7 @@ def _set_sync_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser = root.add_parser("sync", help="sync repository", description="sync packages to remote server",
                              formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("target", help="target to sync", nargs="*")
-    parser.set_defaults(handler=handlers.Sync)
+    parser.set_defaults(handler=handlers.Sync, architecture=[])
     return parser
 
 
@@ -293,7 +315,7 @@ def _set_update_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser.add_argument("--no-aur", help="do not check for AUR updates. Implies --no-vcs", action="store_true")
     parser.add_argument("--no-manual", help="do not include manual updates", action="store_true")
     parser.add_argument("--no-vcs", help="do not check VCS packages", action="store_true")
-    parser.set_defaults(handler=handlers.Update)
+    parser.set_defaults(handler=handlers.Update, architecture=[])
     return parser
 
 

src/ahriman/application/application.py

@@ -65,8 +65,10 @@ class Application:
         """
         known_packages: Set[str] = set()
         # local set
-        for package in self.repository.packages():
-            known_packages.update(package.packages.keys())
+        for base in self.repository.packages():
+            for package, properties in base.packages.items():
+                known_packages.add(package)
+                known_packages.update(properties.provides)
         known_packages.update(self.repository.pacman.all_packages())
         return known_packages
 
@@ -209,6 +211,8 @@ class Application:
         :param updates: list of packages to update
         """
         def process_update(paths: Iterable[Path]) -> None:
+            if not paths:
+                return  # don't need to process if no update supplied
             updated = [Package.load(path, self.repository.pacman, self.repository.aur_url) for path in paths]
             self.repository.process_update(paths)
             self._finalize(updated)

src/ahriman/application/handlers/__init__.py

@@ -22,6 +22,7 @@ from ahriman.application.handlers.handler import Handler
 from ahriman.application.handlers.add import Add
 from ahriman.application.handlers.clean import Clean
 from ahriman.application.handlers.dump import Dump
+from ahriman.application.handlers.init import Init
 from ahriman.application.handlers.key_import import KeyImport
 from ahriman.application.handlers.rebuild import Rebuild
 from ahriman.application.handlers.remove import Remove

src/ahriman/application/handlers/handler.py

@@ -23,10 +23,12 @@ import argparse
 import logging
 
 from multiprocessing import Pool
-from typing import Type
+from typing import Set, Type
 
 from ahriman.application.lock import Lock
 from ahriman.core.configuration import Configuration
+from ahriman.core.exceptions import MissingArchitecture
+from ahriman.models.repository_paths import RepositoryPaths
 
 
 class Handler:
@@ -58,11 +60,33 @@ class Handler:
         :param args: command line args
         :return: 0 on success, 1 otherwise
         """
-        with Pool(len(args.architecture)) as pool:
+        architectures = cls.extract_architectures(args)
+        with Pool(len(architectures)) as pool:
             result = pool.starmap(
-                cls._call, [(args, architecture) for architecture in set(args.architecture)])
+                cls._call, [(args, architecture) for architecture in architectures])
         return 0 if all(result) else 1
 
+    @classmethod
+    def extract_architectures(cls: Type[Handler], args: argparse.Namespace) -> Set[str]:
+        """
+        get known architectures
+        :param args: command line args
+        :return: list of architectures for which tree is created
+        """
+        if args.architecture is None:
+            raise MissingArchitecture(args.command)
+        if args.architecture:
+            return set(args.architecture)
+
+        config = Configuration()
+        config.load(args.configuration)
+        root = config.getpath("repository", "root")
+        architectures = RepositoryPaths.known_architectures(root)
+
+        if not architectures:
+            raise MissingArchitecture(args.command)
+        return architectures
+
     @classmethod
     def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, configuration: Configuration) -> None:
         """

src/ahriman/application/handlers/init.py

@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+
+from typing import Type
+
+from ahriman.application.application import Application
+from ahriman.application.handlers.handler import Handler
+from ahriman.core.configuration import Configuration
+
+
+class Init(Handler):
+    """
+    repository init handler
+    """
+
+    @classmethod
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, configuration: Configuration) -> None:
+        """
+        callback for command line
+        :param args: command line args
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        """
+        Application(architecture, configuration).repository.repo.init()

src/ahriman/application/handlers/rebuild.py

@@ -39,10 +39,12 @@ class Rebuild(Handler):
         :param architecture: repository architecture
         :param configuration: configuration instance
         """
+        depends_on = set(args.depends_on) if args.depends_on else None
+
         application = Application(architecture, configuration)
         packages = [
             package
             for package in application.repository.packages()
-            if args.depends_on is None or args.depends_on in package.depends
+            if depends_on is None or depends_on.intersection(package.depends)
         ]  # we have to use explicit list here for testing purpose
         application.update(packages)

src/ahriman/core/alpm/pacman.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from pyalpm import Handle  # type: ignore
-from typing import List, Set
+from typing import Set
 
 from ahriman.core.configuration import Configuration
 
@@ -40,13 +40,15 @@ class Pacman:
         for repository in configuration.getlist("alpm", "repositories"):
             self.handle.register_syncdb(repository, 0)  # 0 is pgp_level
 
-    def all_packages(self) -> List[str]:
+    def all_packages(self) -> Set[str]:
         """
         get list of packages known for alpm
         :return: list of package names
         """
         result: Set[str] = set()
         for database in self.handle.get_syncdbs():
-            result.update({package.name for package in database.pkgcache})
+            for package in database.pkgcache:
+                result.add(package.name)  # package itself
+                result.update(package.provides)  # provides list for meta-packages
 
-        return list(result)
+        return result

src/ahriman/core/alpm/repo.py

@@ -68,6 +68,16 @@ class Repo:
             cwd=self.paths.repository,
             logger=self.logger)
 
+    def init(self) -> None:
+        """
+        create empty repository database
+        """
+        Repo._check_output(
+            "repo-add", *self.sign_args, str(self.repo_path),
+            exception=None,
+            cwd=self.paths.repository,
+            logger=self.logger)
+
     def remove(self, package: str, filename: Path) -> None:
         """
         remove package from repository

src/ahriman/core/configuration.py

@@ -72,7 +72,8 @@ class Configuration(configparser.RawConfigParser):
         :return: configuration instance
         """
         config = cls()
-        config.load(path, architecture)
+        config.load(path)
+        config.merge_sections(architecture)
         config.load_logging(logfile)
         return config
 
@@ -120,16 +121,14 @@ class Configuration(configparser.RawConfigParser):
             return value
         return self.path.parent / value
 
-    def load(self, path: Path, architecture: str) -> None:
+    def load(self, path: Path) -> None:
         """
         fully load configuration
         :param path: path to root configuration file
-        :param architecture: repository architecture
         """
         self.path = path
         self.read(self.path)
         self.load_includes()
-        self.merge_sections(architecture)
 
     def load_includes(self) -> None:
         """

src/ahriman/core/exceptions.py

@@ -83,6 +83,19 @@ class InvalidPackageInfo(Exception):
         Exception.__init__(self, f"There are errors during reading package information: `{details}`")
 
 
+class MissingArchitecture(Exception):
+    """
+    exception which will be raised if architecture is required, but missing
+    """
+
+    def __init__(self, command: str) -> None:
+        """
+        default constructor
+        :param command: command name which throws exception
+        """
+        Exception.__init__(self, f"Architecture required for subcommand {command}, but missing")
+
+
 class ReportFailed(Exception):
     """
     report generation exception

src/ahriman/core/report/jinja_template.py

@@ -85,7 +85,7 @@ class JinjaTemplate:
         """
         # idea comes from https://stackoverflow.com/a/38642558
         loader = jinja2.FileSystemLoader(searchpath=self.template_path.parent)
-        environment = jinja2.Environment(loader=loader)
+        environment = jinja2.Environment(loader=loader, autoescape=True)
         template = environment.get_template(self.template_path.name)
 
         content = [

src/ahriman/core/upload/s3.py

@@ -17,24 +17,24 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import boto3  # type: ignore
+import hashlib
+
 from pathlib import Path
-from typing import Iterable
+from typing import Any, Dict, Generator, Iterable
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.upload.upload import Upload
-from ahriman.core.util import check_output
 from ahriman.models.package import Package
 
 
 class S3(Upload):
     """
     aws-cli wrapper
-    :ivar bucket: full bucket name
-    :ivar command: command arguments for sync
+    :ivar bucket: boto3 S3 bucket object
+    :ivar chunk_size: chunk size for calculating checksums
     """
 
-    _check_output = check_output
-
     def __init__(self, architecture: str, configuration: Configuration) -> None:
         """
         default constructor
@@ -42,8 +42,69 @@ class S3(Upload):
         :param configuration: configuration instance
         """
         Upload.__init__(self, architecture, configuration)
-        self.bucket = configuration.get("s3", "bucket")
-        self.command = configuration.getlist("s3", "command")
+        self.bucket = self.get_bucket(configuration)
+        self.chunk_size = configuration.getint("s3", "chunk_size", fallback=8 * 1024 * 1024)
+
+    @staticmethod
+    def calculate_etag(path: Path, chunk_size: int) -> str:
+        """
+        calculate amazon s3 etag
+        credits to https://teppen.io/2018/10/23/aws_s3_verify_etags/
+        For this method we have to define nosec because it is out of any security context and provided by AWS
+        :param path: path to local file
+        :param chunk_size: read chunk size, which depends on client settings
+        :return: calculated entity tag for local file
+        """
+        md5s = []
+        with path.open("rb") as local_file:
+            for chunk in iter(lambda: local_file.read(chunk_size), b""):
+                md5s.append(hashlib.md5(chunk))  # nosec
+
+        # in case if there is only one chunk it must be just this checksum
+        # and checksum of joined digest otherwise (including empty list)
+        checksum = md5s[0] if len(md5s) == 1 else hashlib.md5(b"".join(md5.digest() for md5 in md5s))  # nosec
+        # in case if there are more than one chunk it should be appended with amount of chunks
+        suffix = f"-{len(md5s)}" if len(md5s) > 1 else ""
+        return f"{checksum.hexdigest()}{suffix}"
+
+    @staticmethod
+    def get_bucket(configuration: Configuration) -> Any:
+        """
+        create resource client from configuration
+        :param configuration: configuration instance
+        :return: amazon client
+        """
+        client = boto3.resource(service_name="s3",
+                                region_name=configuration.get("s3", "region"),
+                                aws_access_key_id=configuration.get("s3", "access_key"),
+                                aws_secret_access_key=configuration.get("s3", "secret_key"))
+        return client.Bucket(configuration.get("s3", "bucket"))
+
+    def get_local_files(self, path: Path) -> Dict[Path, str]:
+        """
+        get all local files and their calculated checksums
+        :param path: local path to sync
+        :return: map of path object to its checksum
+        """
+        # credits to https://stackoverflow.com/a/64915960
+        def walk(directory_path: Path) -> Generator[Path, None, None]:
+            for element in directory_path.iterdir():
+                if element.is_dir():
+                    yield from walk(element)
+                    continue
+                yield element
+        return {
+            local_file.relative_to(path): self.calculate_etag(local_file, self.chunk_size)
+            for local_file in walk(path)
+        }
+
+    def get_remote_objects(self) -> Dict[Path, Any]:
+        """
+        get all remote objects and their checksums
+        :return: map of path object to the remote s3 object
+        """
+        objects = self.bucket.objects.filter(Prefix=self.architecture)
+        return {Path(item.key).relative_to(self.architecture): item for item in objects}
 
     def sync(self, path: Path, built_packages: Iterable[Package]) -> None:
         """
@@ -51,5 +112,21 @@ class S3(Upload):
         :param path: local path to sync
         :param built_packages: list of packages which has just been built
         """
-        # TODO rewrite to boto, but it is bullshit
-        S3._check_output(*self.command, str(path), self.bucket, exception=None, logger=self.logger)
+        remote_objects = self.get_remote_objects()
+        local_files = self.get_local_files(path)
+
+        # sync to remotes first
+        for local_file, checksum in local_files.items():
+            remote_object = remote_objects.get(local_file)
+            # 0 and -1 elements are " (double quote)
+            remote_checksum = remote_object.e_tag[1:-1] if remote_object is not None else None
+            if remote_checksum == checksum:
+                continue
+            remote_path = Path(self.architecture) / local_file
+            self.bucket.upload_file(str(path / local_file), str(remote_path))
+
+        # remove files which were removed locally
+        for local_file, remote_object in remote_objects.items():
+            if local_file in local_files:
+                continue
+            remote_object.delete()

src/ahriman/core/util.py

@@ -41,7 +41,7 @@ def check_output(*args: str, exception: Optional[Exception], cwd: Optional[Path]
     """
     try:
         # universal_newlines is required to read input from string
-        result: str = subprocess.check_output(args, cwd=cwd, input=input_data, stderr=subprocess.STDOUT,  # type: ignore
+        result: str = subprocess.check_output(args, cwd=cwd, input=input_data, stderr=subprocess.STDOUT,
                                               universal_newlines=True).strip()
         if logger is not None:
             for line in result.splitlines():

src/ahriman/models/package_description.py

@@ -38,6 +38,7 @@ class PackageDescription:
     :ivar groups: package groups
     :ivar installed_size: package installed size
     :ivar licenses: package licenses list
+    :ivar provides: list of provided packages
     :ivar url: package url
     """
 
@@ -50,6 +51,7 @@ class PackageDescription:
     groups: List[str] = field(default_factory=list)
     installed_size: Optional[int] = None
     licenses: List[str] = field(default_factory=list)
+    provides: List[str] = field(default_factory=list)
     url: Optional[str] = None
 
     @property
@@ -89,4 +91,5 @@ class PackageDescription:
             groups=package.groups,
             installed_size=package.isize,
             licenses=package.licenses,
+            provides=package.provides,
             url=package.url)

src/ahriman/models/repository_paths.py

@@ -17,9 +17,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from pathlib import Path
+from __future__ import annotations
 
 from dataclasses import dataclass
+from pathlib import Path
+from typing import Set, Type
 
 
 @dataclass
@@ -76,6 +78,20 @@ class RepositoryPaths:
         """
         return self.root / "sources" / self.architecture
 
+    @classmethod
+    def known_architectures(cls: Type[RepositoryPaths], root: Path) -> Set[str]:
+        """
+        get known architectures
+        :param root: repository root
+        :return: list of architectures for which tree is created
+        """
+        paths = cls(root, "")
+        return {
+            path.name
+            for path in paths.repository.iterdir()
+            if path.is_dir()
+        }
+
     def create_tree(self) -> None:
         """
         create ahriman working tree

src/ahriman/version.py

@@ -17,4 +17,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-__version__ = "1.0.0"
+__version__ = "1.2.0"

tests/ahriman/application/conftest.py

@@ -13,17 +13,32 @@ from ahriman.models.package import Package
 
 @pytest.fixture
 def application(configuration: Configuration, mocker: MockerFixture) -> Application:
+    """
+    fixture for application
+    :param configuration: configuration fixture
+    :param mocker: mocker object
+    :return: application test instance
+    """
     mocker.patch("pathlib.Path.mkdir")
     return Application("x86_64", configuration)
 
 
 @pytest.fixture
 def args() -> argparse.Namespace:
+    """
+    fixture for command line arguments
+    :return: command line arguments test instance
+    """
     return argparse.Namespace(lock=None, force=False, unsafe=False, no_report=True)
 
 
 @pytest.fixture
 def aur_package_ahriman(package_ahriman: Package) -> aur.Package:
+    """
+    fixture for AUR package
+    :param package_ahriman: package fixture
+    :return: AUR package test instance
+    """
     return aur.Package(
         num_votes=None,
         description=package_ahriman.packages[package_ahriman.base].description,
@@ -44,9 +59,19 @@ def aur_package_ahriman(package_ahriman: Package) -> aur.Package:
 
 @pytest.fixture
 def lock(args: argparse.Namespace, configuration: Configuration) -> Lock:
+    """
+    fixture for file lock
+    :param args: command line arguments fixture
+    :param configuration: configuration fixture
+    :return: file lock test instance
+    """
     return Lock(args, "x86_64", configuration)
 
 
 @pytest.fixture
 def parser() -> argparse.ArgumentParser:
+    """
+    fixture for command line arguments parser
+    :return: command line arguments parser test instance
+    """
     return _parser()

tests/ahriman/application/handlers/test_handler.py

@@ -6,6 +6,7 @@ from pytest_mock import MockerFixture
 
 from ahriman.application.handlers import Handler
 from ahriman.core.configuration import Configuration
+from ahriman.core.exceptions import MissingArchitecture
 
 
 def test_call(args: argparse.Namespace, mocker: MockerFixture) -> None:
@@ -43,7 +44,52 @@ def test_execute(args: argparse.Namespace, mocker: MockerFixture) -> None:
     starmap_mock.assert_called_once()
 
 
-def test_packages(args: argparse.Namespace, configuration: Configuration) -> None:
+def test_extract_architectures(args: argparse.Namespace, mocker: MockerFixture) -> None:
+    """
+    must generate list of available architectures
+    """
+    args.architecture = []
+    args.configuration = Path("")
+    mocker.patch("ahriman.core.configuration.Configuration.getpath")
+    known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures")
+
+    Handler.extract_architectures(args)
+    known_architectures_mock.assert_called_once()
+
+
+def test_extract_architectures_empty(args: argparse.Namespace, mocker: MockerFixture) -> None:
+    """
+    must raise exception if no available architectures found
+    """
+    args.architecture = []
+    args.command = "config"
+    args.configuration = Path("")
+    mocker.patch("ahriman.core.configuration.Configuration.getpath")
+    mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures", return_value=set())
+
+    with pytest.raises(MissingArchitecture):
+        Handler.extract_architectures(args)
+
+
+def test_extract_architectures_exception(args: argparse.Namespace) -> None:
+    """
+    must raise exception on missing architectures
+    """
+    args.command = "config"
+    args.architecture = None
+    with pytest.raises(MissingArchitecture):
+        Handler.extract_architectures(args)
+
+
+def test_extract_architectures_specified(args: argparse.Namespace) -> None:
+    """
+    must return architecture list if it has been specified
+    """
+    architectures = args.architecture = ["i686", "x86_64"]
+    assert Handler.extract_architectures(args) == set(architectures)
+
+
+def test_run(args: argparse.Namespace, configuration: Configuration) -> None:
     """
     must raise NotImplemented for missing method
     """

tests/ahriman/application/handlers/test_handler_add.py

@@ -7,6 +7,11 @@ from ahriman.core.configuration import Configuration
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
     args.package = []
     args.now = False
     args.without_dependencies = False

tests/ahriman/application/handlers/test_handler_clean.py

@@ -7,6 +7,11 @@ from ahriman.core.configuration import Configuration
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
     args.no_build = False
     args.no_cache = False
     args.no_chroot = False

tests/ahriman/application/handlers/test_handler_init.py

@@ -0,0 +1,18 @@
+import argparse
+
+from pytest_mock import MockerFixture
+
+from ahriman.application.handlers import Init
+from ahriman.core.configuration import Configuration
+
+
+def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must run command
+    """
+    create_tree_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree")
+    init_mock = mocker.patch("ahriman.core.alpm.repo.Repo.init")
+
+    Init.run(args, "x86_64", configuration)
+    create_tree_mock.assert_called_once()
+    init_mock.assert_called_once()

tests/ahriman/application/handlers/test_handler_key_import.py

@@ -7,6 +7,11 @@ from ahriman.core.configuration import Configuration
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
     args.key = "0xE989490C"
     args.key_server = "keys.gnupg.net"
     return args

tests/ahriman/application/handlers/test_handler_rebuild.py

@@ -8,7 +8,12 @@ from ahriman.models.package import Package
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
-    args.depends_on = None
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
+    args.depends_on = []
     return args
 
 
@@ -33,7 +38,7 @@ def test_run_filter(args: argparse.Namespace, configuration: Configuration,
     must run command with depends filter
     """
     args = _default_args(args)
-    args.depends_on = "python-aur"
+    args.depends_on = ["python-aur"]
     mocker.patch("pathlib.Path.mkdir")
     mocker.patch("ahriman.core.repository.repository.Repository.packages",
                  return_value=[package_ahriman, package_python_schedule])

tests/ahriman/application/handlers/test_handler_remove.py

@@ -7,6 +7,11 @@ from ahriman.core.configuration import Configuration
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
     args.package = []
     return args
 

tests/ahriman/application/handlers/test_handler_report.py

@@ -7,6 +7,11 @@ from ahriman.core.configuration import Configuration
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
     args.target = []
     return args
 

tests/ahriman/application/handlers/test_handler_search.py

@@ -8,6 +8,11 @@ from ahriman.core.configuration import Configuration
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
     args.search = ["ahriman"]
     return args
 

tests/ahriman/application/handlers/test_handler_setup.py

@@ -11,6 +11,11 @@ from ahriman.models.sign_settings import SignSettings
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
     args.build_command = "ahriman"
     args.from_configuration = Path("/usr/share/devtools/pacman-extra.conf")
     args.no_multilib = False

tests/ahriman/application/handlers/test_handler_sign.py

@@ -7,6 +7,11 @@ from ahriman.core.configuration import Configuration
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
     args.package = []
     return args
 

tests/ahriman/application/handlers/test_handler_status.py

@@ -9,6 +9,11 @@ from ahriman.models.package import Package
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
     args.ahriman = True
     args.package = []
     return args

tests/ahriman/application/handlers/test_handler_status_update.py

@@ -9,6 +9,11 @@ from ahriman.models.package import Package
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
     args.status = BuildStatusEnum.Success
     args.package = None
     args.remove = False

tests/ahriman/application/handlers/test_handler_sync.py

@@ -7,6 +7,11 @@ from ahriman.core.configuration import Configuration
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
     args.target = []
     return args
 

tests/ahriman/application/handlers/test_handler_update.py

@@ -8,6 +8,11 @@ from ahriman.core.configuration import Configuration
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
     args.package = []
     args.dry_run = False
     args.no_aur = False

tests/ahriman/application/test_ahriman.py

@@ -29,9 +29,9 @@ def test_parser_option_lock(parser: argparse.ArgumentParser) -> None:
     """
     must convert lock option to Path instance
     """
-    args = parser.parse_args(["-a", "x86_64", "update"])
+    args = parser.parse_args(["update"])
     assert isinstance(args.lock, Path)
-    args = parser.parse_args(["-a", "x86_64", "-l", "ahriman.lock", "update"])
+    args = parser.parse_args(["-l", "ahriman.lock", "update"])
     assert isinstance(args.lock, Path)
 
 
@@ -43,11 +43,20 @@ def test_multiple_architectures(parser: argparse.ArgumentParser) -> None:
     assert len(args.architecture) == 2
 
 
+def test_subparsers_add(parser: argparse.ArgumentParser) -> None:
+    """
+    add command must imply empty architectures list
+    """
+    args = parser.parse_args(["add", "ahriman"])
+    assert args.architecture == []
+
+
 def test_subparsers_check(parser: argparse.ArgumentParser) -> None:
     """
-    check command must imply no_aur, no_manual and dry_run
+    check command must imply empty architecture list, no-aur, no-manual and dry-run
     """
-    args = parser.parse_args(["-a", "x86_64", "check"])
+    args = parser.parse_args(["check"])
+    assert args.architecture == []
     assert not args.no_aur
     assert args.no_manual
     assert args.dry_run
@@ -55,48 +64,87 @@ def test_subparsers_check(parser: argparse.ArgumentParser) -> None:
 
 def test_subparsers_clean(parser: argparse.ArgumentParser) -> None:
     """
-    clean command must imply unsafe
+    clean command must imply empty architectures list, unsafe and no-log
     """
-    args = parser.parse_args(["-a", "x86_64", "clean"])
+    args = parser.parse_args(["clean"])
+    assert args.architecture == []
+    assert args.no_log
     assert args.unsafe
 
 
 def test_subparsers_config(parser: argparse.ArgumentParser) -> None:
     """
-    config command must imply lock, no_report and unsafe
+    config command must imply lock, no-log, no-report and unsafe
     """
-    args = parser.parse_args(["-a", "x86_64", "config"])
+    args = parser.parse_args(["config"])
     assert args.lock is None
+    assert args.no_log
     assert args.no_report
     assert args.unsafe
 
 
+def test_subparsers_init(parser: argparse.ArgumentParser) -> None:
+    """
+    init command must imply no_report
+    """
+    args = parser.parse_args(["init"])
+    assert args.no_report
+
+
 def test_subparsers_key_import(parser: argparse.ArgumentParser) -> None:
     """
-    key-import command must imply lock and no_report
+    key-import command must imply architecture list, lock and no-report
     """
-    args = parser.parse_args(["-a", "x86_64", "key-import", "key"])
+    args = parser.parse_args(["key-import", "key"])
+    assert args.architecture == [""]
     assert args.lock is None
     assert args.no_report
 
 
+def test_subparsers_rebuild(parser: argparse.ArgumentParser) -> None:
+    """
+    rebuild command must imply empty architectures list
+    """
+    args = parser.parse_args(["rebuild"])
+    assert args.architecture == []
+
+
+def test_subparsers_remove(parser: argparse.ArgumentParser) -> None:
+    """
+    remove command must imply empty architectures list
+    """
+    args = parser.parse_args(["remove", "ahriman"])
+    assert args.architecture == []
+
+
+def test_subparsers_report(parser: argparse.ArgumentParser) -> None:
+    """
+    report command must imply empty architectures list
+    """
+    args = parser.parse_args(["report"])
+    assert args.architecture == []
+
+
 def test_subparsers_search(parser: argparse.ArgumentParser) -> None:
     """
-    search command must imply lock, no_report and unsafe
+    search command must imply architecture list, lock, no-log, no-report and unsafe
     """
-    args = parser.parse_args(["-a", "x86_64", "search", "ahriman"])
+    args = parser.parse_args(["search", "ahriman"])
+    assert args.architecture == [""]
     assert args.lock is None
+    assert args.no_log
     assert args.no_report
     assert args.unsafe
 
 
 def test_subparsers_setup(parser: argparse.ArgumentParser) -> None:
     """
-    setup command must imply lock, no_report and unsafe
+    setup command must imply lock, no-log, no-report and unsafe
     """
     args = parser.parse_args(["-a", "x86_64", "setup", "--packager", "John Doe <john@doe.com>",
                               "--repository", "aur-clone"])
     assert args.lock is None
+    assert args.no_log
     assert args.no_report
     assert args.unsafe
 
@@ -123,22 +171,32 @@ def test_subparsers_setup_option_sign_target(parser: argparse.ArgumentParser) ->
     assert all(isinstance(target, SignSettings) for target in args.sign_target)
 
 
+def test_subparsers_sign(parser: argparse.ArgumentParser) -> None:
+    """
+    sign command must imply empty architectures list
+    """
+    args = parser.parse_args(["sign"])
+    assert args.architecture == []
+
+
 def test_subparsers_status(parser: argparse.ArgumentParser) -> None:
     """
-    status command must imply lock, no_report and unsafe
+    status command must imply lock, no-log, no-report and unsafe
     """
     args = parser.parse_args(["-a", "x86_64", "status"])
     assert args.lock is None
+    assert args.no_log
     assert args.no_report
     assert args.unsafe
 
 
 def test_subparsers_status_update(parser: argparse.ArgumentParser) -> None:
     """
-    status-update command must imply lock, no_report and unsafe
+    status-update command must imply lock, no-log, no-report and unsafe
     """
     args = parser.parse_args(["-a", "x86_64", "status-update"])
     assert args.lock is None
+    assert args.no_log
     assert args.no_report
     assert args.unsafe
 
@@ -153,6 +211,22 @@ def test_subparsers_status_update_option_status(parser: argparse.ArgumentParser)
     assert isinstance(args.status, BuildStatusEnum)
 
 
+def test_subparsers_sync(parser: argparse.ArgumentParser) -> None:
+    """
+    sync command must imply empty architectures list
+    """
+    args = parser.parse_args(["sync"])
+    assert args.architecture == []
+
+
+def test_subparsers_update(parser: argparse.ArgumentParser) -> None:
+    """
+    update command must imply empty architectures list
+    """
+    args = parser.parse_args(["update"])
+    assert args.architecture == []
+
+
 def test_subparsers_web(parser: argparse.ArgumentParser) -> None:
     """
     web command must imply lock and no_report

tests/ahriman/application/test_application.py

@@ -299,5 +299,5 @@ def test_update(application: Application, package_ahriman: Package, mocker: Mock
 
     application.update([package_ahriman])
     build_mock.assert_called_once()
-    update_mock.assert_has_calls([mock.call([]), mock.call(paths)])
-    finalize_mock.assert_has_calls([mock.call([]), mock.call([package_ahriman])])
+    update_mock.assert_called_with(paths)
+    finalize_mock.assert_called_with([package_ahriman])

tests/ahriman/application/test_lock.py

@@ -111,7 +111,7 @@ def test_clear(lock: Lock) -> None:
     """
     must remove lock file
     """
-    lock.path = Path(tempfile.mktemp())
+    lock.path = Path(tempfile.mktemp())  # nosec
     lock.path.touch()
 
     lock.clear()
@@ -122,7 +122,7 @@ def test_clear_missing(lock: Lock) -> None:
     """
     must not fail on lock removal if file is missing
     """
-    lock.path = Path(tempfile.mktemp())
+    lock.path = Path(tempfile.mktemp())  # nosec
     lock.clear()
 
 
@@ -139,7 +139,7 @@ def test_create(lock: Lock) -> None:
     """
     must create lock
     """
-    lock.path = Path(tempfile.mktemp())
+    lock.path = Path(tempfile.mktemp())  # nosec
 
     lock.create()
     assert lock.path.is_file()
@@ -150,7 +150,7 @@ def test_create_exception(lock: Lock) -> None:
     """
     must raise exception if file already exists
     """
-    lock.path = Path(tempfile.mktemp())
+    lock.path = Path(tempfile.mktemp())  # nosec
     lock.path.touch()
 
     with pytest.raises(DuplicateRun):
@@ -172,7 +172,7 @@ def test_create_unsafe(lock: Lock) -> None:
     must not raise exception if force flag set
     """
     lock.force = True
-    lock.path = Path(tempfile.mktemp())
+    lock.path = Path(tempfile.mktemp())  # nosec
     lock.path.touch()
 
     lock.create()

tests/ahriman/conftest.py

@@ -1,3 +1,5 @@
+from unittest.mock import MagicMock
+
 import pytest
 
 from pathlib import Path
@@ -10,6 +12,7 @@ from ahriman.models.package import Package
 from ahriman.models.package_description import PackageDescription
 from ahriman.models.repository_paths import RepositoryPaths
 
+
 T = TypeVar("T")
 
 
@@ -17,21 +20,63 @@ T = TypeVar("T")
 # https://stackoverflow.com/a/21611963
 @pytest.helpers.register
 def anyvar(cls: Type[T], strict: bool = False) -> T:
+    """
+    any value helper for mocker calls check
+    :param cls: type class
+    :param strict: if True then check type of supplied argument
+    :return: any wrapper
+    """
     class AnyVar(cls):
+        """
+        any value wrapper
+        """
+
         def __eq__(self, other: Any) -> bool:
+            """
+            compare object to other
+            :param other: other object to compare
+            :return: True in case if objects are equal
+            """
             return not strict or isinstance(other, cls)
+
     return AnyVar()
 
 
+@pytest.helpers.register
+class AsyncMock(MagicMock):
+    """
+    async magic mock object
+    """
+
+    async def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        """
+        async call function
+        :param args:
+        :param kwargs:
+        :return:
+        """
+        return MagicMock.__call__(self, *args, **kwargs)
+
+
 # generic fixtures
 @pytest.fixture
 def configuration(resource_path_root: Path) -> Configuration:
+    """
+    configuration fixture
+    :param resource_path_root: resource path root directory
+    :return: configuration test instance
+    """
     path = resource_path_root / "core" / "ahriman.ini"
     return Configuration.from_path(path=path, architecture="x86_64", logfile=False)
 
 
 @pytest.fixture
 def package_ahriman(package_description_ahriman: PackageDescription) -> Package:
+    """
+    package fixture
+    :param package_description_ahriman: description fixture
+    :return: package test instance
+    """
     packages = {"ahriman": package_description_ahriman}
     return Package(
         base="ahriman",
@@ -44,6 +89,12 @@ def package_ahriman(package_description_ahriman: PackageDescription) -> Package:
 def package_python_schedule(
         package_description_python_schedule: PackageDescription,
         package_description_python2_schedule: PackageDescription) -> Package:
+    """
+    multi package fixture
+    :param package_description_python_schedule: description fixture
+    :param package_description_python2_schedule: description fixture
+    :return: multi package test instance
+    """
     packages = {
         "python-schedule": package_description_python_schedule,
         "python2-schedule": package_description_python2_schedule
@@ -57,6 +108,10 @@ def package_python_schedule(
 
 @pytest.fixture
 def package_description_ahriman() -> PackageDescription:
+    """
+    package description fixture
+    :return: package description test instance
+    """
     return PackageDescription(
         architecture="x86_64",
         archive_size=4200,
@@ -72,6 +127,10 @@ def package_description_ahriman() -> PackageDescription:
 
 @pytest.fixture
 def package_description_python_schedule() -> PackageDescription:
+    """
+    package description fixture
+    :return: package description test instance
+    """
     return PackageDescription(
         architecture="x86_64",
         archive_size=4201,
@@ -87,6 +146,10 @@ def package_description_python_schedule() -> PackageDescription:
 
 @pytest.fixture
 def package_description_python2_schedule() -> PackageDescription:
+    """
+    package description fixture
+    :return: package description test instance
+    """
     return PackageDescription(
         architecture="x86_64",
         archive_size=4202,
@@ -102,6 +165,10 @@ def package_description_python2_schedule() -> PackageDescription:
 
 @pytest.fixture
 def repository_paths(configuration: Configuration) -> RepositoryPaths:
+    """
+    repository paths fixture
+    :return: repository paths test instance
+    """
     return RepositoryPaths(
         architecture="x86_64",
         root=configuration.getpath("repository", "root"))
@@ -109,5 +176,11 @@ def repository_paths(configuration: Configuration) -> RepositoryPaths:
 
 @pytest.fixture
 def watcher(configuration: Configuration, mocker: MockerFixture) -> Watcher:
+    """
+    package status watcher fixture
+    :param configuration: configuration fixture
+    :param mocker: mocker object
+    :return: package status watcher test instance
+    """
     mocker.patch("pathlib.Path.mkdir")
     return Watcher("x86_64", configuration)

tests/ahriman/core/alpm/test_pacman.py

@@ -8,3 +8,10 @@ def test_all_packages(pacman: Pacman) -> None:
     packages = pacman.all_packages()
     assert packages
     assert "pacman" in packages
+
+
+def test_all_packages_with_provides(pacman: Pacman) -> None:
+    """
+    package list must contain provides packages
+    """
+    assert "sh" in pacman.all_packages()

tests/ahriman/core/alpm/test_repo.py

@@ -24,6 +24,17 @@ def test_repo_add(repo: Repo, mocker: MockerFixture) -> None:
     assert check_output_mock.call_args[0][0] == "repo-add"
 
 
+def test_repo_init(repo: Repo, mocker: MockerFixture) -> None:
+    """
+    must call repo-add with empty package list on repo initializing
+    """
+    check_output_mock = mocker.patch("ahriman.core.alpm.repo.Repo._check_output")
+
+    repo.init()
+    check_output_mock.assert_called_once()
+    assert check_output_mock.call_args[0][0] == "repo-add"
+
+
 def test_repo_remove(repo: Repo, mocker: MockerFixture) -> None:
     """
     must call repo-remove on package addition

tests/ahriman/core/conftest.py

@@ -11,24 +11,52 @@ from ahriman.models.repository_paths import RepositoryPaths
 
 @pytest.fixture
 def leaf_ahriman(package_ahriman: Package) -> Leaf:
+    """
+    fixture for tree leaf with package
+    :param package_ahriman: package fixture
+    :return: tree leaf test instance
+    """
     return Leaf(package_ahriman, set())
 
 
 @pytest.fixture
 def leaf_python_schedule(package_python_schedule: Package) -> Leaf:
+    """
+    fixture for tree leaf with package
+    :param package_python_schedule: package fixture
+    :return: tree leaf test instance
+    """
     return Leaf(package_python_schedule, set())
 
 
 @pytest.fixture
 def pacman(configuration: Configuration) -> Pacman:
+    """
+    fixture for pacman wrapper
+    :param configuration: configuration fixture
+    :return: pacman wrapper test instance
+    """
     return Pacman(configuration)
 
 
 @pytest.fixture
 def repo(configuration: Configuration, repository_paths: RepositoryPaths) -> Repo:
+    """
+    fixture for repository wrapper
+    :param configuration: configuration fixture
+    :param repository_paths: repository paths fixture
+    :return: repository wrapper test instance
+    """
     return Repo(configuration.get("repository", "name"), repository_paths, [])
 
 
 @pytest.fixture
 def task_ahriman(package_ahriman: Package, configuration: Configuration, repository_paths: RepositoryPaths) -> Task:
+    """
+    fixture for built task
+    :param package_ahriman: package fixture
+    :param configuration: configuration fixture
+    :param repository_paths: repository paths fixture
+    :return: built task test instance
+    """
     return Task(package_ahriman, configuration, repository_paths)

tests/ahriman/core/repository/conftest.py

@@ -12,12 +12,24 @@ from ahriman.core.repository.update_handler import UpdateHandler
 
 @pytest.fixture
 def cleaner(configuration: Configuration, mocker: MockerFixture) -> Cleaner:
+    """
+    fixture for cleaner
+    :param configuration: configuration fixture
+    :param mocker: mocker object
+    :return: cleaner test instance
+    """
     mocker.patch("pathlib.Path.mkdir")
     return Cleaner("x86_64", configuration)
 
 
 @pytest.fixture
 def executor(configuration: Configuration, mocker: MockerFixture) -> Executor:
+    """
+    fixture for executor
+    :param configuration: configuration fixture
+    :param mocker: mocker object
+    :return: executor test instance
+    """
     mocker.patch("pathlib.Path.mkdir")
     mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_build")
     mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_cache")
@@ -29,17 +41,34 @@ def executor(configuration: Configuration, mocker: MockerFixture) -> Executor:
 
 @pytest.fixture
 def repository(configuration: Configuration, mocker: MockerFixture) -> Repository:
+    """
+    fixture for repository
+    :param configuration: configuration fixture
+    :param mocker: mocker object
+    :return: repository test instance
+    """
     mocker.patch("pathlib.Path.mkdir")
     return Repository("x86_64", configuration)
 
 
 @pytest.fixture
 def properties(configuration: Configuration) -> Properties:
+    """
+    fixture for properties
+    :param configuration: configuration fixture
+    :return: properties test instance
+    """
     return Properties("x86_64", configuration)
 
 
 @pytest.fixture
 def update_handler(configuration: Configuration, mocker: MockerFixture) -> UpdateHandler:
+    """
+    fixture for update handler
+    :param configuration: configuration fixture
+    :param mocker: mocker object
+    :return: update handler test instance
+    """
     mocker.patch("pathlib.Path.mkdir")
     mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_build")
     mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_cache")

tests/ahriman/core/repository/test_cleaner.py

@@ -9,11 +9,18 @@ from ahriman.core.repository.cleaner import Cleaner
 
 
 def _mock_clear(mocker: MockerFixture) -> None:
+    """
+    mocker helper for clear function
+    :param mocker: mocker object
+    """
     mocker.patch("pathlib.Path.iterdir", return_value=[Path("a"), Path("b"), Path("c")])
     mocker.patch("shutil.rmtree")
 
 
 def _mock_clear_check() -> None:
+    """
+    mocker helper for clear tests
+    """
     shutil.rmtree.assert_has_calls([
         mock.call(Path("a")),
         mock.call(Path("b")),

tests/ahriman/core/sign/conftest.py

@@ -6,10 +6,20 @@ from ahriman.core.sign.gpg import GPG
 
 @pytest.fixture
 def gpg(configuration: Configuration) -> GPG:
+    """
+    fixture for empty GPG
+    :param configuration: configuration fixture
+    :return: GPG test instance
+    """
     return GPG("x86_64", configuration)
 
 
 @pytest.fixture
 def gpg_with_key(gpg: GPG) -> GPG:
+    """
+    fixture for correct GPG
+    :param gpg: empty GPG fixture
+    :return: GPG test instance
+    """
     gpg.default_key = "key"
     return gpg

tests/ahriman/core/status/conftest.py

@@ -11,20 +11,38 @@ from ahriman.models.package import Package
 # helpers
 @pytest.helpers.register
 def get_package_status(package: Package) -> Dict[str, Any]:
+    """
+    helper to extract package status from package
+    :param package: package object
+    :return: simplified package status map (with only status and view)
+    """
     return {"status": BuildStatusEnum.Unknown.value, "package": package.view()}
 
 
 @pytest.helpers.register
 def get_package_status_extended(package: Package) -> Dict[str, Any]:
+    """
+    helper to extract package status from package
+    :param package: package object
+    :return: full package status map (with timestamped build status and view)
+    """
     return {"status": BuildStatus().view(), "package": package.view()}
 
 
 # fixtures
 @pytest.fixture
 def client() -> Client:
+    """
+    fixture for dummy client
+    :return: dummy client test instance
+    """
     return Client()
 
 
 @pytest.fixture
 def web_client() -> WebClient:
+    """
+    fixture for web client
+    :return: web client test instance
+    """
     return WebClient("localhost", 8080)

tests/ahriman/core/status/test_watcher.py

@@ -106,7 +106,7 @@ def test_cache_save_load(watcher: Watcher, package_ahriman: Package, mocker: Moc
     """
     must save state to cache which can be loaded later
     """
-    dump_file = Path(tempfile.mktemp())
+    dump_file = Path(tempfile.mktemp())   # nosec
     mocker.patch("ahriman.core.status.watcher.Watcher.cache_path",
                  new_callable=PropertyMock, return_value=dump_file)
     known_current = {package_ahriman.base: (package_ahriman, BuildStatus())}

tests/ahriman/core/upload/conftest.py

@@ -0,0 +1,31 @@
+import pytest
+
+from collections import namedtuple
+from typing import List
+from unittest.mock import MagicMock
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.upload.s3 import S3
+
+
+_s3_object = namedtuple("s3_object", ["key", "e_tag", "delete"])
+
+
+@pytest.fixture
+def s3(configuration: Configuration) -> S3:
+    """
+    fixture for S3 synchronization
+    :param configuration: configuration fixture
+    :return: S3 test instance
+    """
+    return S3("x86_64", configuration)
+
+
+@pytest.fixture
+def s3_remote_objects() -> List[_s3_object]:
+    """
+    fixture for boto3 like S3 objects
+    :return: boto3 like S3 objects test instance
+    """
+    delete_mock = MagicMock()
+    return list(map(lambda item: _s3_object(f"x86_64/{item}", f"\"{item}\"", delete_mock), ["a", "b", "c"]))

tests/ahriman/core/upload/test_s3.py

@@ -1,16 +1,98 @@
 from pathlib import Path
 from pytest_mock import MockerFixture
+from typing import Any, List
+from unittest import mock
+from unittest.mock import MagicMock
 
-from ahriman.core.configuration import Configuration
 from ahriman.core.upload.s3 import S3
 
 
-def test_sync(configuration: Configuration, mocker: MockerFixture) -> None:
+_chunk_size = 8 * 1024 * 1024
+
+
+def test_calculate_etag_big(resource_path_root: Path) -> None:
+    """
+    must calculate checksum for path which is more than one chunk
+    """
+    path = resource_path_root / "models" / "big_file_checksum"
+    assert S3.calculate_etag(path, _chunk_size) == "3b15154eaeed22ae19ae4667d4b98d28-2"
+
+
+def test_calculate_etag_empty(resource_path_root: Path) -> None:
+    """
+    must calculate checksum for empty file correctly
+    """
+    path = resource_path_root / "models" / "empty_file_checksum"
+    assert S3.calculate_etag(path, _chunk_size) == "d41d8cd98f00b204e9800998ecf8427e"
+
+
+def test_calculate_etag_small(resource_path_root: Path) -> None:
+    """
+    must calculate checksum for path which is single chunk
+    """
+    path = resource_path_root / "models" / "package_ahriman_srcinfo"
+    assert S3.calculate_etag(path, _chunk_size) == "04e75b4aa0fe6033e711e8ea98e059b2"
+
+
+def test_get_local_files(s3: S3, resource_path_root: Path) -> None:
+    """
+    must get all local files recursively
+    """
+    expected = sorted([
+        Path("core/ahriman.ini"),
+        Path("core/logging.ini"),
+        Path("models/big_file_checksum"),
+        Path("models/empty_file_checksum"),
+        Path("models/package_ahriman_srcinfo"),
+        Path("models/package_tpacpi-bat-git_srcinfo"),
+        Path("models/package_yay_srcinfo"),
+        Path("web/templates/search-line.jinja2"),
+        Path("web/templates/build-status.jinja2"),
+        Path("web/templates/repo-index.jinja2"),
+        Path("web/templates/sorttable.jinja2"),
+        Path("web/templates/style.jinja2"),
+        Path("web/templates/search.jinja2"),
+    ])
+
+    local_files = list(sorted(s3.get_local_files(resource_path_root).keys()))
+    assert local_files == expected
+
+
+def test_get_remote_objects(s3: S3, s3_remote_objects: List[Any]) -> None:
+    """
+    must generate list of remote objects by calling boto3 function
+    """
+    expected = {Path(item.key).relative_to(s3.architecture): item for item in s3_remote_objects}
+
+    s3.bucket = MagicMock()
+    s3.bucket.objects.filter.return_value = s3_remote_objects
+
+    assert s3.get_remote_objects() == expected
+
+
+def test_sync(s3: S3, s3_remote_objects: List[Any], mocker: MockerFixture) -> None:
     """
     must run sync command
     """
-    check_output_mock = mocker.patch("ahriman.core.upload.s3.S3._check_output")
+    root = Path("path")
+    local_files = {
+        Path(item.key.replace("a", "d")): item.e_tag.replace("b", "d").replace("\"", "")
+        for item in s3_remote_objects
+    }
+    remote_objects = {Path(item.key): item for item in s3_remote_objects}
+    print(local_files)
+    print(remote_objects)
 
-    upload = S3("x86_64", configuration)
-    upload.sync(Path("path"), [])
-    check_output_mock.assert_called_once()
+    local_files_mock = mocker.patch("ahriman.core.upload.s3.S3.get_local_files", return_value=local_files)
+    remote_objects_mock = mocker.patch("ahriman.core.upload.s3.S3.get_remote_objects", return_value=remote_objects)
+    upload_mock = s3.bucket = MagicMock()
+
+    s3.sync(root, [])
+
+    local_files_mock.assert_called_once()
+    remote_objects_mock.assert_called_once()
+    upload_mock.upload_file.assert_has_calls([
+        mock.call(str(root / s3.architecture / "b"), f"{s3.architecture}/{s3.architecture}/b"),
+        mock.call(str(root / s3.architecture / "d"), f"{s3.architecture}/{s3.architecture}/d"),
+    ], any_order=True)
+    remote_objects[Path("x86_64/a")].delete.assert_called_once()

tests/ahriman/models/conftest.py

@@ -12,11 +12,19 @@ from ahriman.models.package_description import PackageDescription
 
 @pytest.fixture
 def build_status_failed() -> BuildStatus:
+    """
+    build result fixture with failed status
+    :return: failed build status test instance
+    """
     return BuildStatus(BuildStatusEnum.Failed, 42)
 
 
 @pytest.fixture
 def counters() -> Counters:
+    """
+    counters fixture
+    :return: counters test instance
+    """
     return Counters(total=10,
                     unknown=1,
                     pending=2,
@@ -27,6 +35,11 @@ def counters() -> Counters:
 
 @pytest.fixture
 def internal_status(counters: Counters) -> InternalStatus:
+    """
+    internal status fixture
+    :param counters: counters fixture
+    :return: internal status test instance
+    """
     return InternalStatus(architecture="x86_64",
                           packages=counters,
                           version=version.__version__,
@@ -35,6 +48,10 @@ def internal_status(counters: Counters) -> InternalStatus:
 
 @pytest.fixture
 def package_tpacpi_bat_git() -> Package:
+    """
+    git package fixture
+    :return: git package test instance
+    """
     return Package(
         base="tpacpi-bat-git",
         version="3.1.r12.g4959b52-1",
@@ -44,6 +61,11 @@ def package_tpacpi_bat_git() -> Package:
 
 @pytest.fixture
 def pyalpm_handle(pyalpm_package_ahriman: MagicMock) -> MagicMock:
+    """
+    mock object for pyalpm
+    :param pyalpm_package_ahriman: mock object for pyalpm package
+    :return: pyalpm mock
+    """
     mock = MagicMock()
     mock.handle.load_pkg.return_value = pyalpm_package_ahriman
     return mock
@@ -51,6 +73,11 @@ def pyalpm_handle(pyalpm_package_ahriman: MagicMock) -> MagicMock:
 
 @pytest.fixture
 def pyalpm_package_ahriman(package_ahriman: Package) -> MagicMock:
+    """
+    mock object for pyalpm package
+    :param package_ahriman: package fixture
+    :return: pyalpm package mock
+    """
     mock = MagicMock()
     type(mock).base = PropertyMock(return_value=package_ahriman.base)
     type(mock).name = PropertyMock(return_value=package_ahriman.base)
@@ -60,6 +87,11 @@ def pyalpm_package_ahriman(package_ahriman: Package) -> MagicMock:
 
 @pytest.fixture
 def pyalpm_package_description_ahriman(package_description_ahriman: PackageDescription) -> MagicMock:
+    """
+    mock object for pyalpm package description
+    :param package_description_ahriman: package description fixture
+    :return: pyalpm package description mock
+    """
     mock = MagicMock()
     type(mock).arch = PropertyMock(return_value=package_description_ahriman.architecture)
     type(mock).builddate = PropertyMock(return_value=package_description_ahriman.build_date)
@@ -69,5 +101,6 @@ def pyalpm_package_description_ahriman(package_description_ahriman: PackageDescr
     type(mock).isize = PropertyMock(return_value=package_description_ahriman.installed_size)
     type(mock).licenses = PropertyMock(return_value=package_description_ahriman.licenses)
     type(mock).size = PropertyMock(return_value=package_description_ahriman.archive_size)
+    type(mock).provides = PropertyMock(return_value=package_description_ahriman.provides)
     type(mock).url = PropertyMock(return_value=package_description_ahriman.url)
     return mock

tests/ahriman/models/test_repository_paths.py

@@ -4,6 +4,15 @@ from unittest import mock
 from ahriman.models.repository_paths import RepositoryPaths
 
 
+def test_known_architectures(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
+    """
+    must list available directory paths
+    """
+    iterdir_mock = mocker.patch("pathlib.Path.iterdir")
+    repository_paths.known_architectures(repository_paths.root)
+    iterdir_mock.assert_called_once()
+
+
 def test_create_tree(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
     """
     must create whole tree
@@ -11,7 +20,7 @@ def test_create_tree(repository_paths: RepositoryPaths, mocker: MockerFixture) -
     paths = {
         prop
         for prop in dir(repository_paths)
-        if not prop.startswith("_") and prop not in ("architecture", "create_tree", "root")
+        if not prop.startswith("_") and prop not in ("architecture", "create_tree", "known_architectures", "root")
     }
     mkdir_mock = mocker.patch("pathlib.Path.mkdir")
 

tests/ahriman/web/conftest.py

@@ -9,5 +9,11 @@ from ahriman.web.web import setup_service
 
 @pytest.fixture
 def application(configuration: Configuration, mocker: MockerFixture) -> web.Application:
+    """
+    application fixture
+    :param configuration: configuration fixture
+    :param mocker: mocker object
+    :return: application test instance
+    """
     mocker.patch("pathlib.Path.mkdir")
     return setup_service("x86_64", configuration)

tests/ahriman/web/middlewares/conftest.py

@@ -0,0 +1,15 @@
+import pytest
+
+from collections import namedtuple
+
+
+_request = namedtuple("_request", ["path"])
+
+
+@pytest.fixture
+def aiohttp_request() -> _request:
+    """
+    fixture for aiohttp like object
+    :return: aiohttp like request test instance
+    """
+    return _request("path")

tests/ahriman/web/middlewares/test_exception_handler.py

@@ -0,0 +1,48 @@
+import logging
+import pytest
+
+from aiohttp.web_exceptions import HTTPBadRequest
+from pytest_mock import MockerFixture
+from typing import Any
+
+from ahriman.web.middlewares.exception_handler import exception_handler
+
+
+async def test_exception_handler(aiohttp_request: Any, mocker: MockerFixture) -> None:
+    """
+    must pass success response
+    """
+    request_handler = pytest.helpers.AsyncMock()
+    logging_mock = mocker.patch("logging.Logger.exception")
+
+    handler = exception_handler(logging.getLogger())
+    await handler(aiohttp_request, request_handler)
+    logging_mock.assert_not_called()
+
+
+async def test_exception_handler_client_error(aiohttp_request: Any, mocker: MockerFixture) -> None:
+    """
+    must pass client exception
+    """
+    request_handler = pytest.helpers.AsyncMock()
+    request_handler.side_effect = HTTPBadRequest()
+    logging_mock = mocker.patch("logging.Logger.exception")
+
+    handler = exception_handler(logging.getLogger())
+    with pytest.raises(HTTPBadRequest):
+        await handler(aiohttp_request, request_handler)
+    logging_mock.assert_not_called()
+
+
+async def test_exception_handler_server_error(aiohttp_request: Any, mocker: MockerFixture) -> None:
+    """
+    must log server exception and re-raise it
+    """
+    request_handler = pytest.helpers.AsyncMock()
+    request_handler.side_effect = Exception()
+    logging_mock = mocker.patch("logging.Logger.exception")
+
+    handler = exception_handler(logging.getLogger())
+    with pytest.raises(Exception):
+        await handler(aiohttp_request, request_handler)
+    logging_mock.assert_called_once()

tests/ahriman/web/test_routes.py

@@ -0,0 +1,11 @@
+from aiohttp import web
+
+from ahriman.web.routes import setup_routes
+
+
+def test_setup_routes(application: web.Application) -> None:
+    """
+    must generate non empty list of routes
+    """
+    setup_routes(application)
+    assert application.router.routes()

tests/ahriman/web/test_web.py

@@ -39,5 +39,5 @@ def test_run(application: web.Application, mocker: MockerFixture) -> None:
     run_application_mock = mocker.patch("aiohttp.web.run_app")
 
     run_server(application)
-    run_application_mock.assert_called_with(application, host="0.0.0.0", port=port,
+    run_application_mock.assert_called_with(application, host="127.0.0.1", port=port,
                                             handle_signals=False, access_log=pytest.helpers.anyvar(int))

tests/ahriman/web/views/conftest.py

@@ -10,5 +10,13 @@ from typing import Any
 @pytest.fixture
 def client(application: web.Application, loop: BaseEventLoop,
            aiohttp_client: Any, mocker: MockerFixture) -> TestClient:
+    """
+    web client fixture
+    :param application: application fixture
+    :param loop: context event loop
+    :param aiohttp_client: aiohttp client fixture
+    :param mocker: mocker object
+    :return: web client test instance
+    """
     mocker.patch("pathlib.Path.iterdir", return_value=[])
     return loop.run_until_complete(aiohttp_client(application))

tests/ahriman/web/views/test_view_status.py

@@ -8,7 +8,7 @@ from ahriman.models.package import Package
 
 async def test_get(client: TestClient, package_ahriman: Package) -> None:
     """
-    must generate web service status correctly)
+    must generate web service status correctly
     """
     await client.post(f"/api/v1/packages/{package_ahriman.base}",
                       json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()})

tests/testresources/core/ahriman.ini

@@ -26,7 +26,7 @@ target =
 target =
 
 [email]
-host = 0.0.0.0
+host = 127.0.0.1
 link_path =
 no_empty_report = no
 port = 587
@@ -48,9 +48,11 @@ command = rsync --archive --verbose --compress --partial --delete
 remote =
 
 [s3]
-bucket =
-command = aws s3 sync --quiet --delete
+access_key =
+bucket = bucket
+region = eu-central-1
+secret_key =
 
 [web]
-host = 0.0.0.0
+host = 127.0.0.1
 templates = ../web/templates

tests/testresources/web/templates

@@ -0,0 +1 @@
+../../../package/share/ahriman

tests/testresources/web/templates/build-status.jinja2

@@ -1,54 +0,0 @@
-<!doctype html>
-<html lang="en">
-    <head>
-        <title>{{ repository|e }}</title>
-
-        {% include "style.jinja2" %}
-
-        {% include "sorttable.jinja2" %}
-        {% include "search.jinja2" %}
-    </head>
-
-    <body>
-        <div class="root">
-            <h1>ahriman
-                <img src="https://img.shields.io/badge/version-{{ version|e }}-informational" alt="{{ version|e }}">
-                <img src="https://img.shields.io/badge/architecture-{{ architecture|e }}-informational" alt="{{ architecture|e }}">
-                <img src="https://img.shields.io/badge/service%20status-{{ service.status|e }}-{{ service.status_color|e }}" alt="{{ service.status|e }}" title="{{ service.timestamp|e }}">
-            </h1>
-
-            {% include "search-line.jinja2" %}
-
-            <section class="element">
-                <table class="sortable search-table">
-                    <tr class="header">
-                        <th>package base</th>
-                        <th>packages</th>
-                        <th>version</th>
-                        <th>last update</th>
-                        <th>status</th>
-                    </tr>
-
-                    {% for package in packages %}
-                        <tr class="package">
-                            <td class="include-search"><a href="{{ package.web_url|e }}" title="{{ package.base|e }}">{{ package.base|e }}</a></td>
-                            <td class="include-search">{{ package.packages|join("<br>"|safe) }}</td>
-                            <td>{{ package.version|e }}</td>
-                            <td>{{ package.timestamp|e }}</td>
-                            <td class="status package-{{ package.status|e }}">{{ package.status|e }}</td>
-                        </tr>
-                    {% endfor %}
-                </table>
-            </section>
-
-            <footer>
-                <ul class="navigation">
-                    <li><a href="https://github.com/arcan1s/ahriman" title="sources">ahriman</a></li>
-                    <li><a href="https://github.com/arcan1s/ahriman/releases" title="releases list">releases</a></li>
-                    <li><a href="https://github.com/arcan1s/ahriman/issues" title="issues tracker">report a bug</a></li>
-                </ul>
-            </footer>
-        </div>
-    </body>
-
-</html>

tests/testresources/web/templates/repo-index.jinja2

@@ -1,62 +0,0 @@
-<!doctype html>
-<html lang="en">
-    <head>
-        <title>{{ repository|e }}</title>
-
-        {% include "style.jinja2" %}
-
-        {% include "sorttable.jinja2" %}
-        {% include "search.jinja2" %}
-    </head>
-
-    <body>
-        <div class="root">
-            <h1>Archlinux user repository</h1>
-
-            <section class="element">
-                {% if pgp_key is not none %}
-                    <p>This repository is signed with <a href="http://keys.gnupg.net/pks/lookup?search=0x{{ pgp_key|e }}&fingerprint=on&op=index" title="key search">{{ pgp_key|e }}</a> by default.</p>
-                {% endif %}
-
-                <code>
-                    $ cat /etc/pacman.conf<br>
-                    [{{ repository|e }}]<br>
-                    Server = {{ link_path|e }}<br>
-                    SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Package{% if has_package_signed %}Required{% else %}Never{% endif %} TrustedOnly
-                </code>
-            </section>
-
-            {% include "search-line.jinja2" %}
-
-            <section class="element">
-                <table class="sortable search-table">
-                    <tr class="header">
-                        <th>package</th>
-                        <th>version</th>
-                        <th>archive size</th>
-                        <th>installed size</th>
-                        <th>build date</th>
-                    </tr>
-
-                    {% for package in packages %}
-                        <tr class="package">
-                            <td class="include-search"><a href="{{ link_path|e }}/{{ package.filename|e }}" title="{{ package.name|e }}">{{ package.name|e }}</a></td>
-                            <td>{{ package.version|e }}</td>
-                            <td>{{ package.archive_size|e }}</td>
-                            <td>{{ package.installed_size|e }}</td>
-                            <td>{{ package.build_date|e }}</td>
-                        </tr>
-                    {% endfor %}
-                </table>
-            </section>
-
-            <footer>
-                <ul class="navigation">
-                    {% if homepage is not none %}
-                        <li><a href="{{ homepage|e }}" title="homepage">Homepage</a></li>
-                    {% endif %}
-                </ul>
-            </footer>
-        </div>
-    </body>
-</html>

tests/testresources/web/templates/search-line.jinja2

@@ -1,3 +0,0 @@
-<section class="element">
-    <input type="search" id="search" onkeyup="searchInTable()" placeholder="search for package" title="search for package"/>
-</section>

tests/testresources/web/templates/search.jinja2

@@ -1,25 +0,0 @@
-<script type="text/javascript">
-    function searchInTable() {
-        const input = document.getElementById("search");
-        const filter = input.value.toLowerCase();
-        const tables = document.getElementsByClassName("search-table");
-
-        for (let i = 0; i < tables.length; i++) {
-            const tr = tables[i].getElementsByTagName("tr");
-            // from 1 coz of header
-            for (let i = 1; i < tr.length; i++) {
-                let td = tr[i].getElementsByClassName("include-search");
-                let display = "none";
-                for (let j = 0; j < td.length; j++) {
-                    if (td[j].tagName.toLowerCase() === "td") {
-                        if (td[j].innerHTML.toLowerCase().indexOf(filter) > -1) {
-                            display = "";
-                            break;
-                        }
-                    }
-                }
-                tr[i].style.display = display;
-            }
-        }
-    }
-</script>

tests/testresources/web/templates/sorttable.jinja2

@@ -1 +0,0 @@
-<script src="https://www.kryogenix.org/code/browser/sorttable/sorttable.js"></script>

tests/testresources/web/templates/style.jinja2

@@ -1,136 +0,0 @@
-<style>
-    :root {
-        --color-building: 255, 255, 146;
-        --color-failed: 255, 94, 94;
-        --color-pending: 255, 255, 146;
-        --color-success: 94, 255, 94;
-        --color-unknown: 225, 225, 225;
-
-        --color-header: 200, 200, 255;
-        --color-hover: 255, 255, 225;
-        --color-line-blue: 235, 235, 255;
-        --color-line-white: 255, 255, 255;
-    }
-
-    @keyframes blink-building {
-        0% { background-color: rgba(var(--color-building), 1.0); }
-        10% { background-color: rgba(var(--color-building), 0.9); }
-        20% { background-color: rgba(var(--color-building), 0.8); }
-        30% { background-color: rgba(var(--color-building), 0.7); }
-        40% { background-color: rgba(var(--color-building), 0.6); }
-        50% { background-color: rgba(var(--color-building), 0.5); }
-        60% { background-color: rgba(var(--color-building), 0.4); }
-        70% { background-color: rgba(var(--color-building), 0.3); }
-        80% { background-color: rgba(var(--color-building), 0.2); }
-        90% { background-color: rgba(var(--color-building), 0.1); }
-        100% { background-color: rgba(var(--color-building), 0.0); }
-    }
-
-    div.root {
-        width: 70%;
-        padding: 15px 15% 0;
-    }
-
-    section.element, footer {
-        width: 100%;
-        padding: 10px 0;
-    }
-
-    code, input, table {
-        width: inherit;
-    }
-
-    th, td {
-        padding: 5px;
-    }
-
-    tr.package:nth-child(odd) {
-        background-color: rgba(var(--color-line-white), 1.0);
-    }
-
-    tr.package:nth-child(even) {
-        background-color: rgba(var(--color-line-blue), 1.0);
-    }
-
-    tr.package:hover {
-        background-color: rgba(var(--color-hover), 1.0);
-    }
-
-    tr.header{
-        background-color: rgba(var(--color-header), 1.0);
-    }
-
-    td.status {
-        text-align: center;
-    }
-
-    td.package-unknown {
-        background-color: rgba(var(--color-unknown), 1.0);
-    }
-    td.package-pending {
-        background-color: rgba(var(--color-pending), 1.0);
-    }
-    td.package-building {
-        background-color: rgba(var(--color-building), 1.0);
-        animation-name: blink-building;
-        animation-duration: 1s;
-        animation-timing-function: linear;
-        animation-iteration-count: infinite;
-        animation-direction: alternate;
-    }
-    td.package-failed {
-        background-color: rgba(var(--color-failed), 1.0);
-    }
-    td.package-success {
-        background-color: rgba(var(--color-success), 1.0);
-    }
-
-    li.service-unknown {
-        background-color: rgba(var(--color-unknown), 1.0);
-    }
-    li.service-building {
-        background-color: rgba(var(--color-building), 1.0);
-        animation-name: blink-building;
-        animation-duration: 1s;
-        animation-timing-function: linear;
-        animation-iteration-count: infinite;
-        animation-direction: alternate;
-    }
-    li.service-failed {
-        background-color: rgba(var(--color-failed), 1.0);
-    }
-    li.service-success {
-        background-color: rgba(var(--color-success), 1.0);
-    }
-
-    ul.navigation {
-        list-style-type: none;
-        margin: 0;
-        padding: 0;
-        overflow: hidden;
-        background-color: rgba(var(--color-header), 1.0);
-    }
-
-    ul.navigation li {
-        float: left;
-    }
-
-    ul.navigation li.status {
-        display: block;
-        text-align: center;
-        text-decoration: none;
-        padding: 14px 16px;
-    }
-
-    ul.navigation li a {
-        display: block;
-        color: black;
-        text-align: center;
-        text-decoration: none;
-        padding: 14px 16px;
-    }
-
-    ul.navigation li a:hover {
-        background-color: rgba(var(--color-hover), 1.0);
-    }
-</style>