Release 1.3.0

also replace old user by new one before creation

.bandit-test.yml

@@ -0,0 +1 @@
+skips: ['B101', 'B404']

.bandit.yml

@@ -0,0 +1 @@
+skips: ['B404', 'B603']

.github/workflows/release.yml

@@ -0,0 +1,37 @@
+name: release
+
+on:
+  push:
+    tags:
+      - '*.*.*'
+
+jobs:
+  make-release:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: extract version
+        id: version
+        run: echo ::set-output name=VERSION::${GITHUB_REF#refs/tags/}
+      - name: create changelog
+        id: changelog
+        uses: jaywcjlove/changelog-generator@main
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          filter: 'Release \d+\.\d+\.\d+'
+      - name: create archive
+        run: make archive
+        env:
+          VERSION: ${{ steps.version.outputs.VERSION }}
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        with:
+          body: |
+            ${{ steps.changelog.outputs.compareurl }}
+            ${{ steps.changelog.outputs.changelog }}
+          files: ahriman-*-src.tar.xz
+          fail_on_unmatched_files: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/run-tests.yml

@@ -1,6 +1,4 @@
-# based on https://github.com/actions/starter-workflows/blob/main/ci/python-app.yml
+name: tests
-
-name: ahriman
 
 on:
   push:
@@ -9,19 +7,20 @@ on:
     branches: [ master ]
 
 jobs:
-  build:
+  run-tests:
 
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v2
-      - name: run check and tests in archlinux container
+      - name: run check and tests in arch linux container
         run: |
           docker run \
           -v  ${{ github.workspace }}:/build -w /build \
           archlinux:latest \
-          /bin/bash -c "pacman --noconfirm -Syu base-devel python python-pip && \
+          /bin/bash -c "pacman --noconfirm -Syu base-devel python-argparse-manpage python-pip && \
           pip install -e .[web] && \
           pip install -e .[check] && \
+          pip install -e .[s3] && \
           pip install -e .[test] && \
           make check tests"

.gitignore

@@ -94,3 +94,5 @@ ENV/
 .venv/
 
 *.tar.xz
+
+man/

.pylintrc

@@ -22,7 +22,7 @@ ignore-patterns=
 
 # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the
 # number of processors available to use.
-jobs=1
+jobs=0
 
 # Control the amount of potential inferred values when inferring a single
 # object. This can help the performance when dealing with large functions or
@@ -149,7 +149,6 @@ disable=print-statement,
         too-few-public-methods,
         too-many-instance-attributes,
         broad-except,
-        logging-fstring-interpolation,
         too-many-ancestors,
         fixme,
         too-many-arguments,

CONFIGURING.md

@@ -1,85 +0,0 @@
-# ahriman configuration
-
-Some groups can be specified for each architecture separately. E.g. if there are `build` and `build_x86_64` groups it will use the `build_x86_64` for the `x86_64` architecture and `build` for any other (architecture specific group has higher priority).
-
-## `settings` group
-
-Base configuration settings.
-
-* `include` - path to directory with configuration files overrides, string, required.
-* `logging` - path to logging configuration, string, required. Check `logging.ini` for reference.
-
-## `alpm` group
-
-libalpm and AUR related configuration.
-
-* `aur_url` - base url for AUR, string, required.
-* `database` - path to pacman local database cache, string, required.
-* `repositories` - list of pacman repositories, space separated list of strings, required.
-* `root` - root for alpm library, string, required.
-
-## `build_*` groups
-
-Build related configuration. Group name must refer to architecture, e.g. it should be `build_x86_64` for x86_64 architecture.
-
-* `archbuild_flags` - additional flags passed to `archbuild` command, space separated list of strings, optional.
-* `build_command` - default build command, string, required.
-* `ignore_packages` - list packages to ignore during a regular update (manual update will still work), space separated list of strings, optional.
-* `makepkg_flags` - additional flags passed to `makepkg` command, space separated list of strings, optional.
-* `makechrootpkg_flags` - additional flags passed to `makechrootpkg` command, space separated list of strings, optional.
-
-## `repository` group
-
-Base repository settings.
-
-* `name` - repository name, string, required.
-* `root` - root path for application, string, required.
-
-## `sign_*` groups
-
-Settings for signing packages or repository. Group name must refer to architecture, e.g. it should be `sign_x86_64` for x86_64 architecture.
-
-* `target` - configuration flag to enable signing, space separated list of strings, required. Allowed values are `package` (sign each package separately), `repository` (sign repository database file).
-* `key` - default PGP key, string, required. This key will also be used for database signing if enabled.
-* `key_*` settings - PGP key which will be used for specific packages, string, optional. For example, if there is `key_yay` option the specified key will be used for yay package and default key for others.
-
-## `report` group
-
-Report generation settings.
-
-* `target` - list of reports to be generated, space separated list of strings, optional. Allowed values are `html`.
-
-### `html_*` groups
-
-Group name must refer to architecture, e.g. it should be `html_x86_64` for x86_64 architecture.
-
-* `path` - path to html report file, string, required.
-* `homepage` - link to homepage, string, optional.
-* `link_path` - prefix for HTML links, string, required.
-* `template_path` - path to Jinja2 template, string, required.
-
-## `upload` group
-
-Remote synchronization settings.
-
-* `target` - list of synchronizations to be used, space separated list of strings, optional. Allowed values are `rsync`, `s3`.
-
-### `rsync_*` groups
-
-Group name must refer to architecture, e.g. it should be `rsync_x86_64` for x86_64 architecture. Requires `rsync` package to be installed. Do not forget to configure ssh for user `ahriman`.
-
-* `remote` - remote server to rsync (e.g. `1.2.3.4:5678:path/to/sync`), string, required.
-
-### `s3_*` groups
-
-Group name must refer to architecture, e.g. it should be `s3_x86_64` for x86_64 architecture. Requires `aws-cli` package to be installed. Do not forget to configure it for user `ahriman`.
-
-* `bucket` - bucket name (e.g. `s3://bucket/path`), string, required.
-
-## `web_*` groups
-
-Web server settings. If any of `host`/`port` is not set, web integration will be disabled. Group name must refer to architecture, e.g. it should be `web_x86_64` for x86_64 architecture.
-
-* `host` - host to bind, string, optional.
-* `port` - port to bind, int, optional.
-* `templates` - path to templates directory, string, required.

Makefile

@@ -1,15 +1,18 @@
-.PHONY: archive archive_directory archlinux check clean directory push tests version
+.PHONY: architecture archive archive_directory archlinux check clean directory push tests version
 .DEFAULT_GOAL := archlinux
 
 PROJECT := ahriman
 
-FILES := AUTHORS COPYING CONFIGURING.md README.md package src setup.py
+FILES := AUTHORS COPYING README.md docs package src setup.cfg setup.py
 TARGET_FILES := $(addprefix $(PROJECT)/, $(FILES))
 IGNORE_FILES := package/archlinux src/.mypy_cache
 
 $(TARGET_FILES) : $(addprefix $(PROJECT), %) : $(addprefix ., %) directory version
 	@cp -rp $< $@
 
+architecture:
+	cd src && pydeps ahriman -o ../docs/ahriman-architecture.svg --no-show --cluster
+
 archive: archive_directory
 	tar cJf "$(PROJECT)-$(VERSION)-src.tar.xz" "$(PROJECT)"
 	rm -rf "$(PROJECT)"
@@ -21,13 +24,13 @@ archive_directory: $(TARGET_FILES)
 	find "$(PROJECT)" -depth -type d -name "*.egg-info" -execdir rm -rf {} +
 
 archlinux: archive
-	sed -i "/sha512sums=('[0-9A-Fa-f]*/s/[^'][^)]*/sha512sums=('$$(sha512sum $(PROJECT)-$(VERSION)-src.tar.xz | awk '{print $$1}')'/" package/archlinux/PKGBUILD
 	sed -i "s/pkgver=[0-9.]*/pkgver=$(VERSION)/" package/archlinux/PKGBUILD
 
-check:
+check: clean mypy
-	cd src && mypy --implicit-reexport --strict -p "$(PROJECT)"
+	autopep8 --exit-code --max-line-length 120 -aa -i -j 0 -r "src/$(PROJECT)" "tests/$(PROJECT)"
-	find "src/$(PROJECT)" tests -name "*.py" -execdir autopep8 --exit-code --max-line-length 120 -aa -i {} +
+	pylint --rcfile=.pylintrc "src/$(PROJECT)"
-	cd src && pylint --rcfile=../.pylintrc "$(PROJECT)"
+	bandit -c .bandit.yml -r "src/$(PROJECT)"
+	bandit -c .bandit-test.yml -r "tests/$(PROJECT)"
 
 clean:
 	find . -type f -name "$(PROJECT)-*-src.tar.xz" -delete
@@ -36,14 +39,18 @@ clean:
 directory: clean
 	mkdir "$(PROJECT)"
 
+mypy:
+	cd src && mypy --implicit-reexport --strict -p "$(PROJECT)" --install-types --non-interactive || true
+	cd src && mypy --implicit-reexport --strict -p "$(PROJECT)"
+
 push: archlinux
 	git add package/archlinux/PKGBUILD src/ahriman/version.py
 	git commit -m "Release $(VERSION)"
-	git push
 	git tag "$(VERSION)"
+	git push
 	git push --tags
 
-tests:
+tests: clean
 	python setup.py test
 
 version:

README.md

@@ -1,70 +1,72 @@
-# ArcHlinux ReposItory MANager
+# ArcH Linux ReposItory MANager
+
+[![build status](https://github.com/arcan1s/ahriman/actions/workflows/run-tests.yml/badge.svg)](https://github.com/arcan1s/ahriman/actions/workflows/run-tests.yml)
+[![CodeFactor](https://www.codefactor.io/repository/github/arcan1s/ahriman/badge)](https://www.codefactor.io/repository/github/arcan1s/ahriman)
 
 Wrapper for managing custom repository inspired by [repo-scripts](https://github.com/arcan1s/repo-scripts).
 
 ## Features
 
-* Install-configure-forget manager for own repository
+* Install-configure-forget manager for own repository.
-* Multi-architecture support
+* Multi-architecture support.
-* VCS packages support
+* VCS packages support.
-* Sign support with gpg (repository, package, per package settings)
+* Sign support with gpg (repository, package, per package settings).
-* Synchronization to remote services (rsync, s3) and report generation (html)
+* Synchronization to remote services (rsync, s3) and report generation (html).
-* Dependency manager
+* Dependency manager.
-* Repository status interface
+* Repository status interface with optional authorization and control options:
+
+    ![web interface](web.png)
 
 ## Installation and run
 
-* Install package as usual.
+For installation details please refer to the [documentation](docs/setup.md). For command help, `--help` subcommand must be used, e.g.:
-* Change settings if required, see [CONFIGURING](CONFIGURING.md) for more details.
-* Create `/var/lib/ahriman/.makepkg.conf` with `makepkg.conf` overrides if required (at least you might want to set `PACKAGER`):
 
-    ```shell
+```shell
-    echo 'PACKAGER="John Doe <john@doe.com>"' | sudo -u ahriman tee -a /var/lib/ahriman/.makepkg.conf
+$ ahriman --help
-    ```
+usage: ahriman [-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--no-log] [--no-report] [--unsafe] [-v]
+               {add,check,clean,config,create-user,init,key-import,rebuild,remove,remove-unknown,report,search,setup,sign,status,status-update,sync,update,web} ...
 
-* Configure build tools (it is required for correct dependency management system):
+ArcH Linux ReposItory MANager
 
-    * create build command, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build` (you can choose any name for command, basically it should be `{name}-{arch}-build`);
+optional arguments:
-    * create configuration file, e.g. `cp /usr/share/devtools/pacman-{extra,ahriman}.conf` (same as previous `pacman-{name}.conf`);
+  -h, --help            show this help message and exit
-    * change configuration file, add your own repository, add multilib repository etc;
+  -a ARCHITECTURE, --architecture ARCHITECTURE
-    * set `build_command` option to point to your command;
+                        target architectures (can be used multiple times) (default: None)
-    * configure `/etc/sudoers.d/ahriman` to allow running command without a password.
+  -c CONFIGURATION, --configuration CONFIGURATION
+                        configuration path (default: /etc/ahriman.ini)
+  --force               force run, remove file lock (default: False)
+  -l LOCK, --lock LOCK  lock file (default: /tmp/ahriman.lock)
+  --no-log              redirect all log messages to stderr (default: False)
+  --no-report           force disable reporting to web service (default: False)
+  --unsafe              allow to run ahriman as non-ahriman user (default: False)
+  -v, --version         show program's version number and exit
 
-    ```shell
+command:
-    ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build
+  {add,check,clean,config,create-user,init,key-import,rebuild,remove,remove-unknown,report,search,setup,sign,status,status-update,sync,update,web}
-    cp /usr/share/devtools/pacman-{extra,ahriman}.conf
+                        command to run
+    add                 add package
+    check               check for updates
+    clean               clean local caches
+    config              dump configuration
+    create-user         create user for web services
+    init                create repository tree
+    key-import          import PGP key
+    rebuild             rebuild repository
+    remove              remove package
+    remove-unknown      remove unknown packages
+    report              generate report
+    search              search for package
+    setup               initial service configuration
+    sign                sign packages
+    status              get package status
+    status-update       update package status
+    sync                sync repository
+    update              update packages
+    web                 start web server
+```
 
-    echo '[multilib]' | tee -a /usr/share/devtools/pacman-ahriman.conf
+Subcommands have own help message as well.
-    echo 'Include = /etc/pacman.d/mirrorlist' | tee -a /usr/share/devtools/pacman-ahriman.conf
 
-    echo '[aur-clone]' | tee -a /usr/share/devtools/pacman-ahriman.conf
+## Configuration
-    echo 'SigLevel = Optional TrustAll' | tee -a /usr/share/devtools/pacman-ahriman.conf
-    echo 'Server = file:///var/lib/ahriman/repository/$arch' | tee -a /usr/share/devtools/pacman-ahriman.conf
 
-    echo '[build]' | tee -a /etc/ahriman.ini.d/build.ini
+Every available option is described in the [documentation](docs/configuration.md).
-    echo 'build_command = ahriman-x86_64-build' | tee -a /etc/ahriman.ini.d/build.ini
-
-    echo 'Cmnd_Alias CARCHBUILD_CMD = /usr/local/bin/ahriman-x86_64-build *' | tee -a /etc/sudoers.d/ahriman
-    echo 'ahriman ALL=(ALL) NOPASSWD: CARCHBUILD_CMD' | tee -a /etc/sudoers.d/ahriman
-    chmod 400 /etc/sudoers.d/ahriman
-    ```
-
-* Start and enable `ahriman@.timer` via `systemctl`:
-
-    ```shell
-    systemctl enable --now ahriman@x86_64.timer
-    ```
-
-* Start and enable status page:
-
-    ```shell
-    systemctl enable --now ahriman-web@x86_64
-    ```
-
-* Add packages by using `ahriman add {package}` command:
-
-    ```shell
-    sudo -u ahriman ahriman -a x86_64 add yay
-    ```
-
-Note that initial service configuration can be done by running `ahriman setup` with specific arguments.

docs/architecture.md

@@ -0,0 +1,184 @@
+# Package structure
+
+Packages have strict rules of importing:
+
+* `ahriman.application` package must not be used anywhere except for itself.
+* `ahriman.core` and `ahriman.models` packages don't have any import restriction. Actually we would like to totally restrict importing of `core` package from `models`, but it is impossible at the moment.
+* `ahriman.web` package is allowed to be imported from `ahriman.application` (web handler only, only `ahriman.web.web` methods). It also must not be imported globally, only local import is allowed. 
+
+Full dependency diagram:
+
+![architecture](ahriman-architecture.svg)
+
+## `ahriman.application` package
+
+This package contains application (aka executable) related classes and everything for that. It also contains package called `ahriman.application.handlers` in which all available subcommands are described as separated classes derived from base `ahriman.application.handlers.handler.Handler` class. `ahriman.application.ahriman` contains only command line parses and executes specified `Handler` on success, `ahriman.application.application.Application` is a god class which provides interfaces for all repository related actions. `ahriman.application.lock.Lock` is additional class which provides file-based lock and also performs some common checks.
+
+## `ahriman.core` package
+
+This package contains everything which is required for any time of application run and separated to several packages:
+
+* `ahriman.core.alpm` package controls pacman related functions. It provides wrappers for `pyalpm` library and safe calls for repository tools (`repo-add` and `repo-remove`).
+* `ahriman.core.auth` package provides classes for authorization methods used by web mostly. Base class is `ahriman.core.auth.auth.Auth` which must be called by `load` method.
+* `ahriman.core.build_tools` is a package which provides wrapper for `devtools` commands.
+* `ahriman.core.report` is a package with reporting classes. Usually it must be called by `ahriman.core.report.report.Report.load` method.
+* `ahriman.core.repository` contains several traits and base repository (`ahriman.core.repository.repository.Repository` class) implementation.
+* `ahriman.core.sign` package provides sign feature (only gpg calls are available).
+* `ahriman.core.status` contains helpers and watcher class which are required for web application. Reporter must be initialized by using `ahriman.core.status.client.Client.load` method.
+* `ahriman.core.upload` package provides sync feature, must be called by `ahriman.core.upload.upload.Upload.load` method.
+
+This package also provides some generic functions and classes which may be used by other packages:
+
+* `ahriman.core.configuration.Configuration` is an extension for standard `configparser` library.
+* `ahriman.core.exceptions` provides custom exceptions.
+* `ahriman.core.spawn.Spawn` is a tool which can spawn another `ahriman` process. This feature is used by web application.
+* `ahriman.core.tree` is a dependency tree implementation.
+
+## `ahriman.models` package
+
+It provides models for any other part of application. Unlike `ahriman.core` package classes from here provides only conversion methods (e.g. create class from another or convert to). Mostly case classes and enumerations.
+
+## `ahriman.web` package
+
+Web application. It is important that this package is isolated from any other to allow it to be optional feature (i.e. dependencies which are required by the package are optional).
+
+* `ahriman.web.middlewares` provides middlewares for request handlers.
+* `ahriman.web.views` contains web views derived from aiohttp view class.
+* `ahriman.web.routes` creates routes for web application.
+* `ahriman.web.web` provides main web application functions (e.g. start, initialization).
+
+# Application run
+
+* Parse command line arguments, find command and related handler which is set by parser.
+* Call `Handler.execute` method.
+* Define list of architectures to run. In case if there is more than one architecture specified run several subprocesses or process in current process otherwise. Class attribute `ALLOW_MULTI_ARCHITECTURE_RUN` controls whether application can be run in multiple processes or not - this feature is required for some handlers (e.g. `Web`) which should be able to spawn child process in daemon mode (it is impossible to do for daemonic processes).
+* In each child process call lock functions.
+* After success checks pass control to `Handler.run` method defined by specific handler class.
+* Return result (success or failure) of each subprocess and exit from application.
+
+In most cases handlers spawn god class `ahriman.application.application.Application` class and call required methods.
+
+Application is designed to run from `systemd` services and provides parametrized by architecture timer and service file for that.
+
+# Basic flows
+
+## Add new packages or rebuild existing
+
+Idea is to copy package to the directory from which it will be handled at the next update run. Different variants are supported:
+
+* If supplied argument is file then application moves the file to the directory with built packages. Same rule applies for directory, but in this case it copies every package-like file from the specified directory.
+* If supplied argument iis not file then application tries to lookup for the specified name in AUR and clones it into the directory with manual updates. This scenario can also handle package dependencies which are missing in repositories.
+
+## Rebuild packages
+
+Same as add function for every package in repository. Optional filter by reverse dependency can be supplied.
+
+## Remove packages
+
+This flow removes package from filesystem, updates repository database and also runs synchronization and reporting methods.
+
+## Update packages
+
+This feature is divided into to stages: check AUR for updates and run rebuild for required packages. Whereas check does not do anything except for check itself, update flow is the following:
+
+1. Process every built package first. Those packages are usually added manually.
+2. Run sync and report methods.
+3. Generate dependency tree for packages to be built.
+4. For each level of tree it does:
+   1. Download package data from AUR.
+   2. Build every package in clean chroot.
+   3. Sign packages if required.
+   4. Add packages to database and sign database if required.
+   5. Process sync and report methods.
+
+After any step any package data is being removed.
+
+# Core functions reference
+
+## Configuration 
+
+`ahriman.core.configuration.Configuration` class provides some additional methods (e.g. `getpath` and `getlist`) and also combines multiple files into single configuration dictionary using architecture overrides. It is recommended to read class related settings from the class, not outside.
+
+## Utils
+
+For every external command run (which is actually not recommended if possible) custom wrapper for `subprocess` is used. Additional functions `ahriman.core.auth.helpers` provide safe calls for `aiohttp_security` methods and are required to make this dependency optional.
+
+## Submodules
+
+Some packages provide different behaviour depending on configuration settings. In this cases inheritance is used and recommended way to deal with them is to call class method `load` from base classes.
+
+## Authorization
+
+The package provides several authorization methods: disabled, based on configuration and OAuth2. 
+
+Disabled (default) authorization provider just allows everything for everyone and does not have any specific configuration (it uses some default configuration parameters though). It also provides generic interface for derived classes.
+
+Mapping (aka configuration) provider uses hashed passwords with salt from configuration file in order to authenticate users. This provider also enables user permission checking (read/write) (authorization). Thus, it defines the following methods:
+
+* `check_credentials` - user password validation (authentication).
+* `verify_access` - user permission validation (authorization).
+
+Passwords must be stored in configuration as `hash(password + salt)`, where `password` is user defined password (taken from user input), `salt` is random string (any length) defined globally in configuration and `hash` is secure hash function. Thus, the following configuration
+
+```ini
+[auth:read]
+username = $6$rounds=656000$mWBiecMPrHAL1VgX$oU4Y5HH8HzlvMaxwkNEJjK13ozElyU1wAHBoO/WW5dAaE4YEfnB0X3FxbynKMl4FBdC3Ovap0jINz4LPkNADg0
+```
+
+means that there is user `username` with `read` access and password `password` hashed by `sha512` with salt `salt`.
+
+OAuth provider uses library definitions (`aioauth-client`) in order _authenticate_ users. It still requires user permission to be set in configuration, thus it inherits mapping provider without any changes. Whereas we could override `check_credentials` (authentication method) by something custom, OAuth flow is a bit more complex than just forward request, thus we have to implement the flow in login form.
+
+OAuth's implementation also allows authenticating users via username + password (in the same way as mapping does) though it is not recommended for end-users and password must be left blank. In particular this feature is used by service reporting (aka robots).
+
+In order to configure users there is special command.
+
+## Additional features
+
+Some features require optional dependencies to be installed:
+
+* Version control executables (e.g. `git`, `svn`) for VCS packages.
+* `gnupg` application for package and repository sign feature.
+* `rsync` application for rsync based repository sync.
+* `boto3` python package for `S3` sync.
+* `Jinja2` python package for HTML report generation (it is also used by web application).
+
+# Web application
+
+Web application requires the following python packages to be installed:
+
+* Core part requires `aiohttp` (application itself), `aiohttp_jinja2` and `Jinja2` (HTML generation from templates).
+* In addition, authorization feature requires `aiohttp_security`, `aiohttp_session` and `cryptography`.
+* In addition to base authorization dependencies, OAuth2 also requires `aioauth-client` library.
+
+## Middlewares
+
+Service provides some custom middlewares, e.g. logging every exception (except for user ones) and user authorization.
+
+## Web views
+
+All web views are defined in separated package and derived from `ahriman.web.views.base.Base` class which provides typed interfaces for web application. 
+
+REST API supports both form and JSON data, but the last one is recommended. 
+
+Different APIs are separated into different packages:
+
+* `ahriman.web.views.service` provides views for application controls.
+* `ahriman.web.views.status` package provides REST API for application reporting.
+* `ahriman.web.views.user` package provides login and logout methods which can be called without authorization.
+
+## Templating
+
+Package provides base jinja templates which can be overridden by settings. Vanilla templates are actively using bootstrap library.
+
+## Requests and scopes
+
+Service provides optional authorization which can be turned on in settings. In order to control user access there are two levels of authorization - read-only (only GET-like requests) and write (anything).
+
+If this feature is configured any request except for whitelisted will be prohibited without authentication. In addition, configuration flag `auth.allow_read_only` can be used in order to allow seeing main page without authorization (this page is in default white list).
+
+For authenticated users it uses encrypted session cookies to store tokens; encryption key is generated each time at the start of the application. It also stores expiration time of the session inside.
+
+## External calls
+
+Web application provides external calls to control main service. It spawns child process with specific arguments and waits for its termination. This feature must be used either with authorization or in safe (i.e. when status page is not available world-wide) environment.

docs/configuration.md

@@ -0,0 +1,135 @@
+# ahriman configuration
+
+Some groups can be specified for each architecture separately. E.g. if there are `build` and `build:x86_64` groups it will use the option from `build:x86_64` for the `x86_64` architecture and `build` for any other (architecture specific group has higher priority). In case if both groups are presented, architecture specific options will be merged into global ones overriding them. 
+
+## `settings` group
+
+Base configuration settings.
+
+* `include` - path to directory with configuration files overrides, string, required.
+* `logging` - path to logging configuration, string, required. Check `logging.ini` for reference.
+
+## `alpm` group
+
+libalpm and AUR related configuration.
+
+* `aur_url` - base url for AUR, string, required.
+* `database` - path to pacman local database cache, string, required.
+* `repositories` - list of pacman repositories, space separated list of strings, required.
+* `root` - root for alpm library, string, required.
+
+## `auth` group
+
+Base authorization settings. `OAuth` provider requires `aioauth-client` library to be installed.
+
+* `target` - specifies authorization provider, string, optional, default `disabled`. Allowed values are `disabled`, `configuration`, `oauth`.
+* `allow_read_only` - allow requesting read only pages without authorization, boolean, required.
+* `allowed_paths` - URI paths (exact match) which can be accessed without authorization, space separated list of strings, optional.
+* `allowed_paths_groups` - URI paths prefixes which can be accessed without authorization, space separated list of strings, optional.
+* `client_id` - OAuth2 application client ID, string, required in case if `oauth` is used.
+* `client_secret` - OAuth2 application client secret key, string, required in case if `oauth` is used.
+* `max_age` - parameter which controls both cookie expiration and token expiration inside the service, integer, optional, default is 7 days.
+* `oauth_provider` - OAuth2 provider class name as is in `aioauth-client` (e.g. `GoogleClient`, `GithubClient` etc), string, required in case if `oauth` is used.
+* `oauth_scopes` - scopes list for OAuth2 provider, which will allow retrieving user email (which is used for checking user permissions), e.g. `https://www.googleapis.com/auth/userinfo.email` for `GoogleClient` or `user:email` for `GithubClient`, space separated list of strings, required in case if `oauth` is used.
+* `salt` - password hash salt, string, required in case if authorization enabled (automatically generated by `create-user` subcommand).
+
+## `auth:*` groups
+
+Authorization mapping. Group name must refer to user access level, i.e. it should be one of `auth:read` (read hidden pages), `auth:write` (everything is allowed).
+
+Key is always username (case-insensitive), option value depends on authorization provider:
+
+* `OAuth` - by default requires only usernames and ignores values. But in case of direct login method call (via POST request) it will act as `Mapping` authorization method.
+* `Mapping` (default) - reads salted password hashes from values, uses SHA512 in order to hash passwords. Password can be set by using `create-user` subcommand.
+
+## `build:*` groups
+
+Build related configuration. Group name must refer to architecture, e.g. it should be `build:x86_64` for x86_64 architecture.
+
+* `archbuild_flags` - additional flags passed to `archbuild` command, space separated list of strings, optional.
+* `build_command` - default build command, string, required.
+* `ignore_packages` - list packages to ignore during a regular update (manual update will still work), space separated list of strings, optional.
+* `makepkg_flags` - additional flags passed to `makepkg` command, space separated list of strings, optional.
+* `makechrootpkg_flags` - additional flags passed to `makechrootpkg` command, space separated list of strings, optional.
+
+## `repository` group
+
+Base repository settings.
+
+* `name` - repository name, string, required.
+* `root` - root path for application, string, required.
+
+## `sign:*` groups
+
+Settings for signing packages or repository. Group name must refer to architecture, e.g. it should be `sign:x86_64` for x86_64 architecture.
+
+* `target` - configuration flag to enable signing, space separated list of strings, required. Allowed values are `package` (sign each package separately), `repository` (sign repository database file).
+* `key` - default PGP key, string, required. This key will also be used for database signing if enabled.
+* `key_*` settings - PGP key which will be used for specific packages, string, optional. For example, if there is `key_yay` option the specified key will be used for yay package and default key for others.
+
+## `report` group
+
+Report generation settings.
+
+* `target` - list of reports to be generated, space separated list of strings, required. Allowed values are `html`, `email`.
+
+### `email:*` groups
+
+Group name must refer to architecture, e.g. it should be `email:x86_64` for x86_64 architecture.
+
+* `full_template_path` - path to Jinja2 template for full package description index, string, optional.
+* `homepage` - link to homepage, string, optional.
+* `host` - SMTP host for sending emails, string, required.
+* `link_path` - prefix for HTML links, string, required.
+* `no_empty_report` - skip report generation for empty packages list, boolean, optional, default `yes`.
+* `password` - SMTP password to authenticate, string, optional.
+* `port` - SMTP port for sending emails, int, required.
+* `receivers` - SMTP receiver addresses, space separated list of strings, required.
+* `sender` - SMTP sender address, string, required.
+* `ssl` - SSL mode for SMTP connection, one of `ssl`, `starttls`, `disabled`, optional, default `disabled`.
+* `template_path` - path to Jinja2 template, string, required.
+* `user` - SMTP user to authenticate, string, optional.
+
+### `html:*` groups
+
+Group name must refer to architecture, e.g. it should be `html:x86_64` for x86_64 architecture.
+
+* `path` - path to html report file, string, required.
+* `homepage` - link to homepage, string, optional.
+* `link_path` - prefix for HTML links, string, required.
+* `template_path` - path to Jinja2 template, string, required.
+
+## `upload` group
+
+Remote synchronization settings.
+
+* `target` - list of synchronizations to be used, space separated list of strings, required. Allowed values are `rsync`, `s3`.
+
+### `rsync:*` groups
+
+Group name must refer to architecture, e.g. it should be `rsync:x86_64` for x86_64 architecture. Requires `rsync` package to be installed. Do not forget to configure ssh for user `ahriman`.
+
+* `command` - rsync command to run, space separated list of string, required.
+* `remote` - remote server to rsync (e.g. `1.2.3.4:5678:path/to/sync`), string, required.
+
+### `s3:*` groups
+
+Group name must refer to architecture, e.g. it should be `s3:x86_64` for x86_64 architecture.
+
+* `access_key` - AWS access key ID, string, required.
+* `bucket` - bucket name (e.g. `bucket`), string, required.
+* `chunk_size` - chunk size for calculating entity tags, int, optional, default 8 * 1024 * 1024.
+* `region` - bucket region (e.g. `eu-central-1`), string, required.
+* `secret_key` - AWS secret access key, string, required.
+
+## `web:*` groups
+
+Web server settings. If any of `host`/`port` is not set, web integration will be disabled. Group name must refer to architecture, e.g. it should be `web:x86_64` for x86_64 architecture.
+
+* `address` - optional address in form `proto://host:port` (`port` can be omitted in case of default `proto` ports), will be used instead of `http://{host}:{port}` in case if set, string, optional. This option is required in case if `OAuth` provider is used.
+* `host` - host to bind, string, optional.
+* `password` - password to authorize in web service in order to update service status, string, required in case if authorization enabled.  
+* `port` - port to bind, int, optional.
+* `static_path` - path to directory with static files, string, required.
+* `templates` - path to templates directory, string, required.
+* `username` - username to authorize in web service in order to update service status, string, required in case if authorization enabled.  

docs/setup.md

@@ -0,0 +1,60 @@
+# Setup instructions
+
+1. Install package as usual.
+2. Change settings if required, see [configuration reference](configuration.md) for more details.
+3. Create `/var/lib/ahriman/.makepkg.conf` with `makepkg.conf` overrides if required (at least you might want to set `PACKAGER`):
+
+    ```shell
+    echo 'PACKAGER="John Doe <john@doe.com>"' | sudo -u ahriman tee -a /var/lib/ahriman/.makepkg.conf
+    ```
+
+4. Configure build tools (it is required for correct dependency management system):
+
+    1. Create build command, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build` (you can choose any name for command, basically it should be `{name}-{arch}-build`).
+    2. Create configuration file, e.g. `cp /usr/share/devtools/pacman-{extra,ahriman}.conf` (same as previous `pacman-{name}.conf`).
+    3. Change configuration file, add your own repository, add multilib repository etc;
+    4. Set `build_command` option to point to your command.
+    5. Configure `/etc/sudoers.d/ahriman` to allow running command without a password.
+
+    ```shell
+    ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build
+    cp /usr/share/devtools/pacman-{extra,ahriman}.conf
+
+    echo '[multilib]' | tee -a /usr/share/devtools/pacman-ahriman.conf
+    echo 'Include = /etc/pacman.d/mirrorlist' | tee -a /usr/share/devtools/pacman-ahriman.conf
+
+    echo '[aur-clone]' | tee -a /usr/share/devtools/pacman-ahriman.conf
+    echo 'SigLevel = Optional TrustAll' | tee -a /usr/share/devtools/pacman-ahriman.conf
+    echo 'Server = file:///var/lib/ahriman/repository/$arch' | tee -a /usr/share/devtools/pacman-ahriman.conf
+
+    echo '[build]' | tee -a /etc/ahriman.ini.d/build.ini
+    echo 'build_command = ahriman-x86_64-build' | tee -a /etc/ahriman.ini.d/build.ini
+
+    echo 'Cmnd_Alias CARCHBUILD_CMD = /usr/local/bin/ahriman-x86_64-build *' | tee -a /etc/sudoers.d/ahriman
+    echo 'ahriman ALL=(ALL) NOPASSWD: CARCHBUILD_CMD' | tee -a /etc/sudoers.d/ahriman
+    chmod 400 /etc/sudoers.d/ahriman
+    ```
+
+5. Start and enable `ahriman@.timer` via `systemctl`:
+
+    ```shell
+    systemctl enable --now ahriman@x86_64.timer
+    ```
+
+6. Start and enable status page:
+
+    ```shell
+    systemctl enable --now ahriman-web@x86_64
+    ```
+
+7. Add packages by using `ahriman add {package}` command:
+
+    ```shell
+    sudo -u ahriman ahriman -a x86_64 add yay --now
+    ```
+
+Note that initial service configuration can be done by running `ahriman setup` with specific arguments.
+
+## User creation
+
+`create-user` subcommand is recommended for new user creation.

package/archlinux/PKGBUILD

@@ -1,31 +1,31 @@
 # Maintainer: Evgeniy Alekseev
 
 pkgname='ahriman'
-pkgver=0.18.0
+pkgver=1.3.0
 pkgrel=1
-pkgdesc="ArcHlinux ReposItory MANager"
+pkgdesc="ArcH Linux ReposItory MANager"
 arch=('any')
 url="https://github.com/arcan1s/ahriman"
 license=('GPL3')
-depends=('devtools' 'git' 'pyalpm' 'python-aur' 'python-srcinfo')
+depends=('devtools' 'git' 'pyalpm' 'python-aur' 'python-passlib' 'python-srcinfo')
 makedepends=('python-pip')
-optdepends=('aws-cli: sync to s3'
+optdepends=('breezy: -bzr packages support'
-            'breezy: -bzr packages support'
             'darcs: -darcs packages support'
             'gnupg: package and repository sign'
             'mercurial: -hg packages support'
+            'python-aioauth-client: web server with OAuth2 authorization'
             'python-aiohttp: web server'
             'python-aiohttp-jinja2: web server'
+            'python-aiohttp-security: web server with authorization'
+            'python-aiohttp-session: web server with authorization'
+            'python-boto3: sync to s3'
+            'python-cryptography: web server with authorization'
             'python-jinja: html report generation'
-            'python-requests: web server'
             'rsync: sync by using rsync'
             'subversion: -svn packages support')
 source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz"
         'ahriman.sysusers'
         'ahriman.tmpfiles')
-sha512sums=('8acc57f937d587ca665c29092cadddbaf3ba0b80e870b80d1551e283aba8f21306f9030a26fec8c71ab5863316f5f5f061b7ddc63cdff9e6d5a885f28ef1893d'
-            '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
-            '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')
 backup=('etc/ahriman.ini'
         'etc/ahriman.ini.d/logging.ini')
 
@@ -43,3 +43,7 @@ package() {
   install -Dm644 "$srcdir/$pkgname.sysusers" "$pkgdir/usr/lib/sysusers.d/$pkgname.conf"
   install -Dm644 "$srcdir/$pkgname.tmpfiles" "$pkgdir/usr/lib/tmpfiles.d/$pkgname.conf"
 }
+
+sha512sums=('6ab741bfb42f92ab00d1b6ecfc44426c00e5c433486e014efbdb585715d9a12dbbafc280e5a9f85b941c8681b13a9dad41327a3e3c44a9683ae30c1d6f017f50'
+            '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
+            '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')

package/archlinux/ahriman.sysusers

@@ -1 +1 @@
-u ahriman 643 "ArcHlinux ReposItory MANager" /var/lib/ahriman
+u ahriman 643 "ArcH Linux ReposItory MANager" /var/lib/ahriman

package/archlinux/ahriman.tmpfiles

@@ -1,2 +1 @@
-d /var/lib/ahriman 0775 ahriman log
 d /var/log/ahriman 0755 ahriman ahriman

package/etc/ahriman.ini

@@ -8,12 +8,19 @@ database = /var/lib/pacman
 repositories = core extra community multilib
 root = /
 
+[auth]
+target = disabled
+allow_read_only = yes
+max_age = 604800
+oauth_provider = GoogleClient
+oauth_scopes = https://www.googleapis.com/auth/userinfo.email
+
 [build]
 archbuild_flags =
 build_command = extra-x86_64-build
 ignore_packages =
 makechrootpkg_flags =
-makepkg_flags = --skippgpcheck
+makepkg_flags =
 
 [repository]
 name = aur-clone
@@ -21,25 +28,29 @@ root = /var/lib/ahriman
 
 [sign]
 target =
-key =
 
 [report]
 target =
 
+[email]
+full_template_path = /usr/share/ahriman/repo-index.jinja2
+no_empty_report = yes
+template_path = /usr/share/ahriman/email-index.jinja2
+ssl = disabled
+
 [html]
-path =
-homepage =
-link_path =
 template_path = /usr/share/ahriman/repo-index.jinja2
 
 [upload]
 target =
 
 [rsync]
-remote =
+command = rsync --archive --compress --partial --delete
 
 [s3]
-bucket =
+chunk_size = 8388608
 
 [web]
+host = 127.0.0.1
+static_path = /usr/share/ahriman/static
 templates = /usr/share/ahriman

package/etc/ahriman.ini.d/logging.ini

@@ -2,10 +2,10 @@
 keys = root,builder,build_details,http
 
 [handlers]
-keys = console_handler,build_file_handler,file_handler,http_handler
+keys = console_handler,syslog_handler
 
 [formatters]
-keys = generic_format
+keys = generic_format,syslog_format
 
 [handler_console_handler]
 class = StreamHandler
@@ -13,47 +13,39 @@ level = DEBUG
 formatter = generic_format
 args = (sys.stderr,)
 
-[handler_file_handler]
+[handler_syslog_handler]
-class = logging.handlers.RotatingFileHandler
+class = logging.handlers.SysLogHandler
 level = DEBUG
-formatter = generic_format
+formatter = syslog_format
-args = ("/var/log/ahriman/ahriman.log", "a", 20971520, 20)
+args = ("/dev/log",)
-
-[handler_build_file_handler]
-class = logging.handlers.RotatingFileHandler
-level = DEBUG
-formatter = generic_format
-args = ("/var/log/ahriman/build.log", "a", 20971520, 20)
-
-[handler_http_handler]
-class = logging.handlers.RotatingFileHandler
-level = DEBUG
-formatter = generic_format
-args = ("/var/log/ahriman/http.log", "a", 20971520, 20)
 
 [formatter_generic_format]
 format = [%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s
 datefmt =
 
+[formatter_syslog_format]
+format = [%(levelname)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s
+datefmt =
+
 [logger_root]
 level = DEBUG
-handlers = file_handler
+handlers = syslog_handler
 qualname = root
 
 [logger_builder]
 level = DEBUG
-handlers = file_handler
+handlers = syslog_handler
 qualname = builder
 propagate = 0
 
 [logger_build_details]
 level = DEBUG
-handlers = build_file_handler
+handlers = syslog_handler
 qualname = build_details
 propagate = 0
 
 [logger_http]
 level = DEBUG
-handlers = http_handler
+handlers = syslog_handler
 qualname = http
 propagate = 0

package/lib/systemd/system/ahriman-web@.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=ArcHlinux ReposItory MANager web server (%I architecture)
+Description=ArcH Linux ReposItory MANager web server (%I architecture)
 After=network.target
 
 [Service]
@@ -8,8 +8,5 @@ ExecStart=/usr/bin/ahriman --architecture %i web
 User=ahriman
 Group=ahriman
 
-KillSignal=SIGQUIT
-SuccessExitStatus=SIGQUIT
-
 [Install]
 WantedBy=multi-user.target

package/lib/systemd/system/ahriman@.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=ArcHlinux ReposItory MANager (%I architecture)
+Description=ArcH Linux ReposItory MANager (%I architecture)
 
 [Service]
 ExecStart=/usr/bin/ahriman --architecture %i update

package/lib/systemd/system/ahriman@.timer

@@ -1,5 +1,5 @@
 [Unit]
-Description=ArcHlinux ReposItory MANager timer (%I architecture)
+Description=ArcH Linux ReposItory MANager timer (%I architecture)
 
 [Timer]
 OnCalendar=daily

package/share/ahriman/build-status.jinja2

@@ -1,54 +1,127 @@
 <!doctype html>
 <html lang="en">
     <head>
-        <title>{{ repository|e }}</title>
+        <title>{{ repository }}</title>
 
-        {% include "style.jinja2" %}
+        <meta name="viewport" content="width=device-width, initial-scale=1">
 
-        {% include "sorttable.jinja2" %}
+        <link rel="shortcut icon" href="/static/favicon.ico">
-        {% include "search.jinja2" %}
+
+        {% include "utils/style.jinja2" %}
     </head>
 
     <body>
-        <div class="root">
+
+        <div class="container">
             <h1>ahriman
-                <img src="https://img.shields.io/badge/version-{{ version|e }}-informational" alt="{{ version|e }}">
+                {% if auth.authenticated %}
-                <img src="https://img.shields.io/badge/architecture-{{ architecture|e }}-informational" alt="{{ architecture|e }}">
+                    <img src="https://img.shields.io/badge/version-{{ version }}-informational" alt="{{ version }}">
-                <img src="https://img.shields.io/badge/service%20status-{{ service.status|e }}-{{ service.status_color|e }}" alt="{{ service.status|e }}" title="{{ service.timestamp|e }}">
+                    <img src="https://img.shields.io/badge/repository-{{ repository }}-informational" alt="{{ repository }}">
+                    <img src="https://img.shields.io/badge/architecture-{{ architecture }}-informational" alt="{{ architecture }}">
+                    <img src="https://img.shields.io/badge/service%20status-{{ service.status }}-{{ service.status_color }}" alt="{{ service.status }}" title="{{ service.timestamp }}">
+                {% endif %}
             </h1>
+        </div>
 
-            {% include "search-line.jinja2" %}
+        <div class="container">
+            <div id="toolbar">
+                {% if not auth.enabled or auth.username is not none %}
+                    <button id="add" class="btn btn-primary" data-bs-toggle="modal" data-bs-target="#addForm">
+                        <i class="fa fa-plus"></i> Add
+                    </button>
+                    <button id="update" class="btn btn-secondary" onclick="updatePackages()" disabled>
+                        <i class="fa fa-play"></i> Update
+                    </button>
+                    <button id="remove" class="btn btn-danger" onclick="removePackages()" disabled>
+                        <i class="fa fa-trash"></i> Remove
+                    </button>
+                {% endif %}
+            </div>
 
-            <section class="element">
+            <table id="packages" class="table table-striped table-hover"
-                <table class="sortable search-table">
+                   data-click-to-select="true"
-                    <tr class="header">
+                   data-export-options='{"fileName": "packages"}'
-                        <th>package base</th>
+                   data-page-list="[10, 25, 50, 100, all]"
-                        <th>packages</th>
+                   data-page-size="10"
-                        <th>version</th>
+                   data-pagination="true"
-                        <th>last update</th>
+                   data-resizable="true"
-                        <th>status</th>
+                   data-search="true"
+                   data-show-columns="true"
+                   data-show-columns-search="true"
+                   data-show-columns-toggle-all="true"
+                   data-show-export="true"
+                   data-show-fullscreen="true"
+                   data-show-search-clear-button="true"
+                   data-sortable="true"
+                   data-sort-reset="true"
+                   data-toggle="table"
+                   data-toolbar="#toolbar">
+                <thead class="table-primary">
+                    <tr>
+                        <th data-checkbox="true"></th>
+                        <th data-sortable="true" data-switchable="false">package base</th>
+                        <th data-sortable="true">version</th>
+                        <th data-sortable="true">packages</th>
+                        <th data-sortable="true" data-visible="false">groups</th>
+                        <th data-sortable="true" data-visible="false">licenses</th>
+                        <th data-sortable="true">last update</th>
+                        <th data-sortable="true">status</th>
                     </tr>
+                </thead>
 
+                <tbody>
+                {% if auth.authenticated %}
                     {% for package in packages %}
-                        <tr class="package">
+                        <tr data-package-base="{{ package.base }}">
-                            <td class="include-search"><a href="{{ package.web_url|e }}" title="{{ package.base|e }}">{{ package.base|e }}</a></td>
+                            <td data-checkbox="true"></td>
-                            <td class="include-search">{{ package.packages|join("<br>"|safe) }}</td>
+                            <td><a href="{{ package.web_url }}" title="{{ package.base }}">{{ package.base }}</a></td>
-                            <td>{{ package.version|e }}</td>
+                            <td>{{ package.version }}</td>
-                            <td>{{ package.timestamp|e }}</td>
+                            <td>{{ package.packages|join("<br>"|safe) }}</td>
-                            <td class="status package-{{ package.status|e }}">{{ package.status|e }}</td>
+                            <td>{{ package.groups|join("<br>"|safe) }}</td>
+                            <td>{{ package.licenses|join("<br>"|safe) }}</td>
+                            <td>{{ package.timestamp }}</td>
+                            <td class="table-{{ package.status_color }}">{{ package.status }}</td>
                         </tr>
                     {% endfor %}
-                </table>
+                {% else %}
-            </section>
+                    <tr>
+                        <td colspan="100%">In order to see statuses you must login first.</td>
+                    </tr>
+                {% endif %}
+                </tbody>
+            </table>
+        </div>
 
-            <footer>
+        <div class="container">
-                <ul class="navigation">
+            <footer class="d-flex flex-wrap justify-content-between align-items-center border-top">
-                    <li><a href="https://github.com/arcan1s/ahriman" title="sources">ahriman</a></li>
+                <ul class="nav">
-                    <li><a href="https://github.com/arcan1s/ahriman/releases" title="releases list">releases</a></li>
+                    <li><a class="nav-link" href="https://github.com/arcan1s/ahriman" title="sources">ahriman</a></li>
-                    <li><a href="https://github.com/arcan1s/ahriman/issues" title="issues tracker">report a bug</a></li>
+                    <li><a class="nav-link" href="https://github.com/arcan1s/ahriman/releases" title="releases list">releases</a></li>
+                    <li><a class="nav-link" href="https://github.com/arcan1s/ahriman/issues" title="issues tracker">report a bug</a></li>
                 </ul>
+
+                {% if auth.enabled %}
+                    {% if auth.username is none %}
+                        {{ auth.control|safe }}
+                    {% else %}
+                        <form action="/user-api/v1/logout" method="post">
+                            <button class="btn btn-link" style="text-decoration: none">logout ({{ auth.username }})</button>
+                        </form>
+                    {% endif %}
+                {% endif %}
             </footer>
         </div>
+
+        {% if auth.enabled %}
+            {% include "build-status/login-modal.jinja2" %}
+        {% endif %}
+
+        {% include "build-status/package-actions-modals.jinja2" %}
+
+        {% include "utils/bootstrap-scripts.jinja2" %}
+
+        {% include "build-status/package-actions-script.jinja2" %}
+
     </body>
 
 </html>

package/share/ahriman/build-status/login-modal.jinja2

@@ -0,0 +1,29 @@
+<div id="loginForm" tabindex="-1" role="dialog" class="modal fade">
+    <div class="modal-dialog" role="document">
+        <div class="modal-content">
+            <form action="/user-api/v1/login" method="post">
+                <div class="modal-header">
+                    <h4 class="modal-title">Login</h4>
+                    <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="close"></button>
+                </div>
+                <div class="modal-body">
+                    <div class="form-group row">
+                        <label for="username" class="col-sm-2 col-form-label">Username</label>
+                        <div class="col-sm-10">
+                            <input id="username" type="text" class="form-control" placeholder="enter username" name="username" required>
+                        </div>
+                    </div>
+                    <div class="form-group row">
+                        <label for="password" class="col-sm-2 col-form-label">Password</label>
+                        <div class="col-sm-10">
+                            <input id="password" type="password" class="form-control" placeholder="enter password" name="password" required>
+                        </div>
+                    </div>
+                </div>
+                <div class="modal-footer">
+                    <button class="btn btn-primary">Login</button>
+                </div>
+            </form>
+        </div>
+    </div>
+</div>

package/share/ahriman/build-status/package-actions-modals.jinja2

@@ -0,0 +1,59 @@
+<div id="addForm" tabindex="-1" role="dialog" class="modal fade">
+    <div class="modal-dialog" role="document">
+        <div class="modal-content">
+            <div class="modal-header">
+                <h4 class="modal-title">Add new packages</h4>
+                <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="close"></button>
+            </div>
+            <div class="modal-body">
+                <div class="form-group row">
+                    <label for="package" class="col-sm-2 col-form-label">Package</label>
+                    <div class="col-sm-10">
+                        <input id="package" type="text" list="knownPackages" class="form-control" placeholder="AUR package" name="package" required>
+                        <datalist id="knownPackages"></datalist>
+                    </div>
+                </div>
+            </div>
+            <div class="modal-footer">
+                <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+                <button type="button" class="btn btn-primary" data-bs-dismiss="modal" onclick="addPackages()">Add</button>
+            </div>
+        </div>
+    </div>
+</div>
+
+<div id="failedForm" tabindex="-1" role="dialog" class="modal fade">
+    <div class="modal-dialog" role="document">
+        <div class="modal-content">
+            <div class="modal-header bg-danger">
+                <h4 class="modal-title">Failed</h4>
+                <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="close"></button>
+            </div>
+            <div class="modal-body">
+                <p>Packages update has failed.</p>
+                <p id="errorDetails"></p>
+            </div>
+            <div class="modal-footer">
+                <button type="button" class="btn btn-primary" data-bs-dismiss="modal">Close</button>
+            </div>
+        </div>
+    </div>
+</div>
+
+<div id="successForm" tabindex="-1" role="dialog" class="modal fade">
+    <div class="modal-dialog" role="document">
+        <div class="modal-content">
+            <div class="modal-header bg-success">
+                <h4 class="modal-title">Success</h4>
+                <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="close"></button>
+            </div>
+            <div class="modal-body">
+                <p>Packages update has been run.</p>
+                <ul id="successDetails"></ul>
+            </div>
+            <div class="modal-footer">
+                <button type="button" class="btn btn-primary" data-bs-dismiss="modal">Close</button>
+            </div>
+        </div>
+    </div>
+</div>

package/share/ahriman/build-status/package-actions-script.jinja2

@@ -0,0 +1,89 @@
+<script>
+    const $remove = $("#remove");
+    const $update = $("#update");
+
+    const $table = $("#packages");
+    $table.on("check.bs.table uncheck.bs.table check-all.bs.table uncheck-all.bs.table",
+        function () {
+            $remove.prop("disabled", !$table.bootstrapTable("getSelections").length);
+            $update.prop("disabled", !$table.bootstrapTable("getSelections").length);
+        })
+
+    const $successForm = $("#successForm");
+    const $successDetails = $("#successDetails");
+    $successForm.on("hidden.bs.modal", function() { window.location.reload(); });
+
+    const $failedForm = $("#failedForm");
+    const $errorDetails = $("#errorDetails");
+    $failedForm.on("hidden.bs.modal", function() { window.location.reload(); });
+
+    const $package = $("#package");
+    const $knownPackages = $("#knownPackages");
+    $package.keyup(function () {
+        const $this = $(this);
+        clearTimeout($this.data("timeout"));
+
+        $this.data("timeout", setTimeout($.proxy(function () {
+            const $value = $package.val();
+
+            $.ajax({
+                url: "/service-api/v1/search",
+                data: {"for": $value},
+                type: "GET",
+                dataType: "json",
+                success: function (resp) {
+                    const $options = resp.map(function (pkg) {
+                        const $option = document.createElement("option");
+                        $option.value = `${pkg.package} (${pkg.description})`;
+                        return $option;
+                    });
+                    $knownPackages.empty().append($options);
+                    $this.focus();
+                },
+            })
+        }, this), 500));
+    })
+
+    function doPackageAction($uri, $packages) {
+        if ($packages.length === 0)
+            return;
+        $.ajax({
+            url: $uri,
+            data: JSON.stringify({packages: $packages}),
+            type: "POST",
+            contentType: "application/json",
+            success: function (_) {
+                const $details = $packages.map(function (pkg) {
+                    const $li = document.createElement("li");
+                    $li.innerText = pkg;
+                    return $li;
+                });
+                $successDetails.empty().append($details);
+                $successForm.modal("show");
+            },
+            error: function (jqXHR, textStatus, errorThrown) {
+                $errorDetails.text(errorThrown);
+                $failedForm.modal("show");
+            },
+        })
+    }
+
+    function getSelection() {
+        return $.map($table.bootstrapTable("getSelections"), function(row) {
+            return row._data["package-base"];
+        })
+    }
+
+    function addPackages() {
+        const $packages = [$package.val()]
+        doPackageAction("/service-api/v1/add", $packages);
+    }
+
+    function removePackages() { doPackageAction("/service-api/v1/remove", getSelection()); }
+
+    function updatePackages() { doPackageAction("/service-api/v1/add", getSelection()); }
+
+    $(function () {
+        $table.bootstrapTable("uncheckAll");
+    })
+</script>

package/share/ahriman/email-index.jinja2

@@ -0,0 +1,42 @@
+{#simplified version of full report#}
+<!doctype html>
+<html lang="en">
+    <head>
+        <title>{{ repository }}</title>
+
+        <meta name="viewport" content="width=device-width, initial-scale=1">
+
+        {% include "utils/style.jinja2" %}
+    </head>
+
+    <body>
+
+        <div class="container">
+            <table id="packages" class="table table-striped">
+                <thead class="table-primary">
+                    <tr>
+                        <th>package</th>
+                        <th>version</th>
+                        <th>archive size</th>
+                        <th>installed size</th>
+                        <th>build date</th>
+                    </tr>
+                </thead>
+
+                <tbody>
+                {% for package in packages %}
+                    <tr>
+                        <td><a href="{{ link_path }}/{{ package.filename }}" title="{{ package.name }}">{{ package.name }}</a></td>
+                        <td>{{ package.version }}</td>
+                        <td>{{ package.archive_size }}</td>
+                        <td>{{ package.installed_size }}</td>
+                        <td>{{ package.build_date }}</td>
+                    </tr>
+                {% endfor %}
+                </tbody>
+            </table>
+        </div>
+
+    </body>
+
+</html>

package/share/ahriman/repo-index.jinja2

@@ -1,62 +1,95 @@
 <!doctype html>
 <html lang="en">
     <head>
-        <title>{{ repository|e }}</title>
+        <title>{{ repository }}</title>
 
-        {% include "style.jinja2" %}
+        <meta name="viewport" content="width=device-width, initial-scale=1">
 
-        {% include "sorttable.jinja2" %}
+        {% include "utils/style.jinja2" %}
-        {% include "search.jinja2" %}
     </head>
 
     <body>
-        <div class="root">
-            <h1>Archlinux user repository</h1>
 
-            <section class="element">
+        <div class="container">
-                {% if pgp_key is not none %}
+            <h1>Arch Linux user repository</h1>
-                    <p>This repository is signed with <a href="http://keys.gnupg.net/pks/lookup?search=0x{{ pgp_key|e }}&fingerprint=on&op=index" title="key search">{{ pgp_key|e }}</a> by default.</p>
+        </div>
-                {% endif %}
 
-                <code>
+        <div class="container">
-                    $ cat /etc/pacman.conf<br>
+            {% if pgp_key is not none %}
-                    [{{ repository|e }}]<br>
+                <p>This repository is signed with <a href="https://pgp.mit.edu/pks/lookup?search=0x{{ pgp_key }}&fingerprint=on&op=index" title="key search">{{ pgp_key }}</a> by default.</p>
-                    Server = {{ link_path|e }}<br>
+            {% endif %}
-                    SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Package{% if has_package_signed %}Required{% else %}Never{% endif %} TrustedOnly
-                </code>
-            </section>
 
-            {% include "search-line.jinja2" %}
+<pre>$ cat /etc/pacman.conf
+[{{ repository }}]
+Server = {{ link_path }}
+SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Package{% if has_package_signed %}Required{% else %}Never{% endif %} TrustedOnly</pre>
+        </div>
 
-            <section class="element">
+        <div class="container">
-                <table class="sortable search-table">
+            <table id="packages" class="table table-striped table-hover"
-                    <tr class="header">
+                   data-export-options='{"fileName": "packages"}'
-                        <th>package</th>
+                   data-page-list="[10, 25, 50, 100, all]"
-                        <th>version</th>
+                   data-page-size="10"
-                        <th>archive size</th>
+                   data-pagination="true"
-                        <th>installed size</th>
+                   data-resizable="true"
-                        <th>build date</th>
+                   data-search="true"
+                   data-show-columns="true"
+                   data-show-columns-search="true"
+                   data-show-columns-toggle-all="true"
+                   data-show-export="true"
+                   data-show-fullscreen="true"
+                   data-show-search-clear-button="true"
+                   data-sortable="true"
+                   data-sort-reset="true"
+                   data-toggle="table">
+                <thead class="table-primary">
+                    <tr>
+                        <th data-sortable="true" data-switchable="false">package</th>
+                        <th data-sortable="true">version</th>
+                        <th data-sortable="true" data-visible="false">architecture</th>
+                        <th data-sortable="true" data-visible="false">description</th>
+                        <th data-sortable="true" data-visible="false">upstream url</th>
+                        <th data-sortable="true" data-visible="false">licenses</th>
+                        <th data-sortable="true" data-visible="false">groups</th>
+                        <th data-sortable="true" data-visible="false">depends</th>
+                        <th data-sortable="true">archive size</th>
+                        <th data-sortable="true">installed size</th>
+                        <th data-sortable="true">build date</th>
                     </tr>
+                </thead>
 
-                    {% for package in packages %}
+                <tbody>
-                        <tr class="package">
+                {% for package in packages %}
-                            <td class="include-search"><a href="{{ link_path|e }}/{{ package.filename|e }}" title="{{ package.name|e }}">{{ package.name|e }}</a></td>
+                    <tr>
-                            <td>{{ package.version|e }}</td>
+                        <td><a href="{{ link_path }}/{{ package.filename }}" title="{{ package.name }}">{{ package.name }}</a></td>
-                            <td>{{ package.archive_size|e }}</td>
+                        <td>{{ package.version }}</td>
-                            <td>{{ package.installed_size|e }}</td>
+                        <td>{{ package.architecture }}</td>
-                            <td>{{ package.build_date|e }}</td>
+                        <td>{{ package.description }}</td>
-                        </tr>
+                        <td><a href="{{ package.url }}" title="{{ package.name }} upstream url">{{ package.url }}</a></td>
-                    {% endfor %}
+                        <td>{{ package.licenses|join("<br>"|safe) }}</td>
-                </table>
+                        <td>{{ package.groups|join("<br>"|safe) }}</td>
-            </section>
+                        <td>{{ package.depends|join("<br>"|safe) }}</td>
+                        <td>{{ package.archive_size }}</td>
+                        <td>{{ package.installed_size }}</td>
+                        <td>{{ package.build_date }}</td>
+                    </tr>
+                {% endfor %}
+                </tbody>
+            </table>
+        </div>
 
-            <footer>
+        <div class="container">
-                <ul class="navigation">
+            <footer class="d-flex flex-wrap justify-content-between align-items-center border-top">
+                <ul class="nav">
                     {% if homepage is not none %}
-                        <li><a href="{{ homepage|e }}" title="homepage">Homepage</a></li>
+                        <li><a class="nav-link" href="{{ homepage }}" title="homepage">Homepage</a></li>
                     {% endif %}
                 </ul>
             </footer>
         </div>
+
+        {% include "utils/bootstrap-scripts.jinja2" %}
+
     </body>
+
 </html>

package/share/ahriman/search-line.jinja2

@@ -1,3 +0,0 @@
-<section class="element">
-    <input type="search" id="search" onkeyup="searchInTable()" placeholder="search for package" title="search for package"/>
-</section>

package/share/ahriman/search.jinja2

@@ -1,25 +0,0 @@
-<script type="text/javascript">
-    function searchInTable() {
-        const input = document.getElementById("search");
-        const filter = input.value.toLowerCase();
-        const tables = document.getElementsByClassName("search-table");
-
-        for (let i = 0; i < tables.length; i++) {
-            const tr = tables[i].getElementsByTagName("tr");
-            // from 1 coz of header
-            for (let i = 1; i < tr.length; i++) {
-                let td = tr[i].getElementsByClassName("include-search");
-                let display = "none";
-                for (let j = 0; j < td.length; j++) {
-                    if (td[j].tagName.toLowerCase() === "td") {
-                        if (td[j].innerHTML.toLowerCase().indexOf(filter) > -1) {
-                            display = "";
-                            break;
-                        }
-                    }
-                }
-                tr[i].style.display = display;
-            }
-        }
-    }
-</script>

package/share/ahriman/sorttable.jinja2

@@ -1 +0,0 @@
-<script src="https://www.kryogenix.org/code/browser/sorttable/sorttable.js"></script>

package/share/ahriman/style.jinja2

@@ -1,136 +0,0 @@
-<style>
-    :root {
-        --color-building: 255, 255, 146;
-        --color-failed: 255, 94, 94;
-        --color-pending: 255, 255, 146;
-        --color-success: 94, 255, 94;
-        --color-unknown: 225, 225, 225;
-
-        --color-header: 200, 200, 255;
-        --color-hover: 255, 255, 225;
-        --color-line-blue: 235, 235, 255;
-        --color-line-white: 255, 255, 255;
-    }
-
-    @keyframes blink-building {
-        0% { background-color: rgba(var(--color-building), 1.0); }
-        10% { background-color: rgba(var(--color-building), 0.9); }
-        20% { background-color: rgba(var(--color-building), 0.8); }
-        30% { background-color: rgba(var(--color-building), 0.7); }
-        40% { background-color: rgba(var(--color-building), 0.6); }
-        50% { background-color: rgba(var(--color-building), 0.5); }
-        60% { background-color: rgba(var(--color-building), 0.4); }
-        70% { background-color: rgba(var(--color-building), 0.3); }
-        80% { background-color: rgba(var(--color-building), 0.2); }
-        90% { background-color: rgba(var(--color-building), 0.1); }
-        100% { background-color: rgba(var(--color-building), 0.0); }
-    }
-
-    div.root {
-        width: 70%;
-        padding: 15px 15% 0;
-    }
-
-    section.element, footer {
-        width: 100%;
-        padding: 10px 0;
-    }
-
-    code, input, table {
-        width: inherit;
-    }
-
-    th, td {
-        padding: 5px;
-    }
-
-    tr.package:nth-child(odd) {
-        background-color: rgba(var(--color-line-white), 1.0);
-    }
-
-    tr.package:nth-child(even) {
-        background-color: rgba(var(--color-line-blue), 1.0);
-    }
-
-    tr.package:hover {
-        background-color: rgba(var(--color-hover), 1.0);
-    }
-
-    tr.header{
-        background-color: rgba(var(--color-header), 1.0);
-    }
-
-    td.status {
-        text-align: center;
-    }
-
-    td.package-unknown {
-        background-color: rgba(var(--color-unknown), 1.0);
-    }
-    td.package-pending {
-        background-color: rgba(var(--color-pending), 1.0);
-    }
-    td.package-building {
-        background-color: rgba(var(--color-building), 1.0);
-        animation-name: blink-building;
-        animation-duration: 1s;
-        animation-timing-function: linear;
-        animation-iteration-count: infinite;
-        animation-direction: alternate;
-    }
-    td.package-failed {
-        background-color: rgba(var(--color-failed), 1.0);
-    }
-    td.package-success {
-        background-color: rgba(var(--color-success), 1.0);
-    }
-
-    li.service-unknown {
-        background-color: rgba(var(--color-unknown), 1.0);
-    }
-    li.service-building {
-        background-color: rgba(var(--color-building), 1.0);
-        animation-name: blink-building;
-        animation-duration: 1s;
-        animation-timing-function: linear;
-        animation-iteration-count: infinite;
-        animation-direction: alternate;
-    }
-    li.service-failed {
-        background-color: rgba(var(--color-failed), 1.0);
-    }
-    li.service-success {
-        background-color: rgba(var(--color-success), 1.0);
-    }
-
-    ul.navigation {
-        list-style-type: none;
-        margin: 0;
-        padding: 0;
-        overflow: hidden;
-        background-color: rgba(var(--color-header), 1.0);
-    }
-
-    ul.navigation li {
-        float: left;
-    }
-
-    ul.navigation li.status {
-        display: block;
-        text-align: center;
-        text-decoration: none;
-        padding: 14px 16px;
-    }
-
-    ul.navigation li a {
-        display: block;
-        color: black;
-        text-align: center;
-        text-decoration: none;
-        padding: 14px 16px;
-    }
-
-    ul.navigation li a:hover {
-        background-color: rgba(var(--color-hover), 1.0);
-    }
-</style>

package/share/ahriman/utils/bootstrap-scripts.jinja2

@@ -0,0 +1,12 @@
+<script src="https://cdn.jsdelivr.net/npm/jquery/dist/jquery.min.js"></script>
+
+<script src="https://unpkg.com/tableexport.jquery.plugin/tableExport.min.js"></script>
+
+<script src="https://unpkg.com/jquery-resizable-columns@0.2.3/dist/jquery.resizableColumns.min.js"></script>
+
+<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.1.0/dist/js/bootstrap.bundle.min.js" integrity="sha384-U1DAWAznBHeqEIlVSCgzq+c9gqGAJn5c/t99JyeKa9xxaYpSvHU5awsuZVVFIhvj" crossorigin="anonymous"></script>
+<script src="https://unpkg.com/bootstrap-table@1.18.3/dist/bootstrap-table.min.js"></script>
+
+<script src="https://unpkg.com/bootstrap-table@1.18.3/dist/extensions/export/bootstrap-table-export.min.js"></script>
+
+<script src="https://unpkg.com/bootstrap-table@1.18.3/dist/extensions/resizable/bootstrap-table-resizable.js"></script>

package/share/ahriman/utils/style.jinja2

@@ -0,0 +1,9 @@
+<script src="https://kit.fontawesome.com/0d6d6d5226.js" crossorigin="anonymous"></script>
+<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.0/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-KyZXEAg3QhqLMpG8r+8fhAXLRk2vvoC2f3B09zVXn8CA5QIVfZOJ3BCsw2P0p/We" crossorigin="anonymous">
+
+<link href="https://unpkg.com/bootstrap-table@1.18.3/dist/bootstrap-table.min.css" rel="stylesheet">
+
+<link href="https://unpkg.com/jquery-resizable-columns@0.2.3/dist/jquery.resizableColumns.css" rel="stylesheet">
+
+<style>
+</style>

setup.cfg

@@ -2,4 +2,4 @@
 test = pytest
 
 [tool:pytest]
-addopts = --cov=ahriman --pspec
+addopts = --cov=ahriman --cov-report term-missing:skip-covered --pspec

setup.py

@@ -1,11 +1,13 @@
-from distutils.util import convert_path
+from pathlib import Path
 from setuptools import setup, find_packages
-from os import path
+from typing import Any, Dict
+
+
+metadata_path = Path(__file__).resolve().parent / "src/ahriman/version.py"
+metadata: Dict[str, Any] = {}
+with metadata_path.open() as metadata_file:
+    exec(metadata_file.read(), metadata)  # pylint: disable=exec-used
 
-here = path.abspath(path.dirname(__file__))
-metadata = dict()
-with open(convert_path("src/ahriman/version.py")) as metadata_file:
-    exec(metadata_file.read(), metadata)
 
 setup(
     name="ahriman",
@@ -13,7 +15,7 @@ setup(
     version=metadata["__version__"],
     zip_safe=False,
 
-    description="ArcHlinux ReposItory MANager",
+    description="ArcH Linux ReposItory MANager",
 
     author="arcanis",
     author_email="",
@@ -28,7 +30,9 @@ setup(
     ],
     install_requires=[
         "aur",
+        "passlib",
         "pyalpm",
+        "requests",
         "srcinfo",
     ],
     setup_requires=[
@@ -62,20 +66,33 @@ setup(
         ]),
         ("share/ahriman", [
             "package/share/ahriman/build-status.jinja2",
+            "package/share/ahriman/email-index.jinja2",
             "package/share/ahriman/repo-index.jinja2",
-            "package/share/ahriman/search.jinja2",
+        ]),
-            "package/share/ahriman/search-line.jinja2",
+        ("share/ahriman/build-status", [
-            "package/share/ahriman/sorttable.jinja2",
+            "package/share/ahriman/build-status/login-modal.jinja2",
-            "package/share/ahriman/style.jinja2",
+            "package/share/ahriman/build-status/package-actions-modals.jinja2",
+            "package/share/ahriman/build-status/package-actions-script.jinja2",
+        ]),
+        ("share/ahriman/static", [
+            "package/share/ahriman/static/favicon.ico",
+        ]),
+        ("share/ahriman/utils", [
+            "package/share/ahriman/utils/bootstrap-scripts.jinja2",
+            "package/share/ahriman/utils/style.jinja2",
         ]),
     ],
 
     extras_require={
         "check": [
             "autopep8",
+            "bandit",
             "mypy",
             "pylint",
         ],
+        "s3": [
+            "boto3",
+        ],
         "test": [
             "pytest",
             "pytest-aiohttp",
@@ -89,7 +106,10 @@ setup(
             "Jinja2",
             "aiohttp",
             "aiohttp_jinja2",
-            "requests",
+            "aioauth-client",
+            "aiohttp_session",
+            "aiohttp_security",
+            "cryptography",
         ],
     },
 )

src/ahriman/application/ahriman.py

@@ -19,16 +19,19 @@
 #
 import argparse
 import sys
+import tempfile
 
-import ahriman.application.handlers as handlers
+from pathlib import Path
-import ahriman.version as version
 
+from ahriman import version
+from ahriman.application import handlers
 from ahriman.models.build_status import BuildStatusEnum
+from ahriman.models.sign_settings import SignSettings
+from ahriman.models.user_access import UserAccess
 
 
 # pylint thinks it is bad idea, but get the fuck off
-# pylint: disable=protected-access
+SubParserAction = argparse._SubParsersAction  # pylint: disable=protected-access
-SubParserAction = argparse._SubParsersAction
 
 
 def _parser() -> argparse.ArgumentParser:
@@ -36,12 +39,18 @@ def _parser() -> argparse.ArgumentParser:
     command line parser generator
     :return: command line parser for the application
     """
-    parser = argparse.ArgumentParser(prog="ahriman", description="ArcHlinux ReposItory MANager")
+    parser = argparse.ArgumentParser(prog="ahriman", description="ArcH Linux ReposItory MANager",
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("-a", "--architecture", help="target architectures (can be used multiple times)",
-                        action="append", required=True)
+                        action="append")
-    parser.add_argument("-c", "--config", help="configuration path", default="/etc/ahriman.ini")
+    parser.add_argument("-c", "--configuration", help="configuration path", type=Path, default=Path("/etc/ahriman.ini"))
     parser.add_argument("--force", help="force run, remove file lock", action="store_true")
-    parser.add_argument("--lock", help="lock file", default="/tmp/ahriman.lock")
+    parser.add_argument(
+        "-l",
+        "--lock",
+        help="lock file",
+        type=Path,
+        default=Path(tempfile.gettempdir()) / "ahriman.lock")
     parser.add_argument("--no-log", help="redirect all log messages to stderr", action="store_true")
     parser.add_argument("--no-report", help="force disable reporting to web service", action="store_true")
     parser.add_argument("--unsafe", help="allow to run ahriman as non-ahriman user", action="store_true")
@@ -53,15 +62,20 @@ def _parser() -> argparse.ArgumentParser:
     _set_check_parser(subparsers)
     _set_clean_parser(subparsers)
     _set_config_parser(subparsers)
+    _set_init_parser(subparsers)
+    _set_key_import_parser(subparsers)
     _set_rebuild_parser(subparsers)
     _set_remove_parser(subparsers)
+    _set_remove_unknown_parser(subparsers)
     _set_report_parser(subparsers)
+    _set_search_parser(subparsers)
     _set_setup_parser(subparsers)
     _set_sign_parser(subparsers)
     _set_status_parser(subparsers)
     _set_status_update_parser(subparsers)
     _set_sync_parser(subparsers)
     _set_update_parser(subparsers)
+    _set_user_parser(subparsers)
     _set_web_parser(subparsers)
 
     return parser
@@ -73,10 +87,12 @@ def _set_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("add", description="add package")
+    parser = root.add_parser("add", help="add package", description="add package",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("package", help="package base/name or archive path", nargs="+")
+    parser.add_argument("--now", help="run update function after", action="store_true")
     parser.add_argument("--without-dependencies", help="do not add dependencies", action="store_true")
-    parser.set_defaults(handler=handlers.Add)
+    parser.set_defaults(handler=handlers.Add, architecture=[])
     return parser
 
 
@@ -86,10 +102,12 @@ def _set_check_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("check", description="check for updates. Same as update --dry-run --no-manual")
+    parser = root.add_parser("check", help="check for updates",
+                             description="check for updates. Same as update --dry-run --no-manual",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("package", help="filter check by package base", nargs="*")
     parser.add_argument("--no-vcs", help="do not check VCS packages", action="store_true")
-    parser.set_defaults(handler=handlers.Update, no_aur=False, no_manual=True, dry_run=True)
+    parser.set_defaults(handler=handlers.Update, architecture=[], no_aur=False, no_manual=True, dry_run=True)
     return parser
 
 
@@ -99,13 +117,14 @@ def _set_clean_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("clean", description="clear all local caches")
+    parser = root.add_parser("clean", help="clean local caches", description="clear local caches",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("--no-build", help="do not clear directory with package sources", action="store_true")
     parser.add_argument("--no-cache", help="do not clear directory with package caches", action="store_true")
     parser.add_argument("--no-chroot", help="do not clear build chroot", action="store_true")
     parser.add_argument("--no-manual", help="do not clear directory with manually added packages", action="store_true")
     parser.add_argument("--no-packages", help="do not clear directory with built packages", action="store_true")
-    parser.set_defaults(handler=handlers.Clean, unsafe=True)
+    parser.set_defaults(handler=handlers.Clean, architecture=[], no_log=True, unsafe=True)
     return parser
 
 
@@ -115,8 +134,38 @@ def _set_config_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("config", description="dump configuration for specified architecture")
+    parser = root.add_parser("config", help="dump configuration",
-    parser.set_defaults(handler=handlers.Dump, lock=None, no_report=True, unsafe=True)
+                             description="dump configuration for specified architecture",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.set_defaults(handler=handlers.Dump, lock=None, no_log=True, no_report=True, unsafe=True)
+    return parser
+
+
+def _set_init_parser(root: SubParserAction) -> argparse.ArgumentParser:
+    """
+    add parser for init subcommand
+    :param root: subparsers for the commands
+    :return: created argument parser
+    """
+    parser = root.add_parser("init", help="create repository tree",
+                             description="create empty repository tree. Optional command for auto architecture support",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.set_defaults(handler=handlers.Init, no_report=True)
+    return parser
+
+
+def _set_key_import_parser(root: SubParserAction) -> argparse.ArgumentParser:
+    """
+    add parser for key import subcommand
+    :param root: subparsers for the commands
+    :return: created argument parser
+    """
+    parser = root.add_parser("key-import", help="import PGP key",
+                             description="import PGP key from public sources to repository user",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("--key-server", help="key server for key import", default="pgp.mit.edu")
+    parser.add_argument("key", help="PGP key to import from public server")
+    parser.set_defaults(handler=handlers.KeyImport, architecture=[""], lock=None, no_report=True)
     return parser
 
 
@@ -126,8 +175,10 @@ def _set_rebuild_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("rebuild", description="rebuild whole repository")
+    parser = root.add_parser("rebuild", help="rebuild repository", description="rebuild whole repository",
-    parser.set_defaults(handler=handlers.Rebuild)
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("--depends-on", help="only rebuild packages that depend on specified package", action="append")
+    parser.set_defaults(handler=handlers.Rebuild, architecture=[])
     return parser
 
 
@@ -137,9 +188,24 @@ def _set_remove_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("remove", description="remove package")
+    parser = root.add_parser("remove", help="remove package", description="remove package",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("package", help="package name or base", nargs="+")
-    parser.set_defaults(handler=handlers.Remove)
+    parser.set_defaults(handler=handlers.Remove, architecture=[])
+    return parser
+
+
+def _set_remove_unknown_parser(root: SubParserAction) -> argparse.ArgumentParser:
+    """
+    add parser for remove unknown packages subcommand
+    :param root: subparsers for the commands
+    :return: created argument parser
+    """
+    parser = root.add_parser("remove-unknown", help="remove unknown packages",
+                             description="remove packages which are missing in AUR",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("--dry-run", help="just perform check for packages without removal", action="store_true")
+    parser.set_defaults(handler=handlers.RemoveUnknown, architecture=[])
     return parser
 
 
@@ -149,9 +215,22 @@ def _set_report_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("report", description="generate report")
+    parser = root.add_parser("report", help="generate report", description="generate report",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("target", help="target to generate report", nargs="*")
-    parser.set_defaults(handler=handlers.Report)
+    parser.set_defaults(handler=handlers.Report, architecture=[])
+    return parser
+
+
+def _set_search_parser(root: SubParserAction) -> argparse.ArgumentParser:
+    """
+    add parser for search subcommand
+    :param root: subparsers for the commands
+    :return: created argument parser
+    """
+    parser = root.add_parser("search", help="search for package", description="search for package in AUR using API")
+    parser.add_argument("search", help="search terms, can be specified multiple times", nargs="+")
+    parser.set_defaults(handler=handlers.Search, architecture=[""], lock=None, no_log=True, no_report=True, unsafe=True)
     return parser
 
 
@@ -161,14 +240,20 @@ def _set_setup_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("setup", description="create initial service configuration, requires root")
+    parser = root.add_parser("setup", help="initial service configuration",
+                             description="create initial service configuration, requires root",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("--build-command", help="build command prefix", default="ahriman")
-    parser.add_argument("--from-config", help="path to default devtools pacman configuration",
+    parser.add_argument("--from-configuration", help="path to default devtools pacman configuration",
-                        default="/usr/share/devtools/pacman-extra.conf")
+                        type=Path, default=Path("/usr/share/devtools/pacman-extra.conf"))
     parser.add_argument("--no-multilib", help="do not add multilib repository", action="store_true")
     parser.add_argument("--packager", help="packager name and email", required=True)
-    parser.add_argument("--repository", help="repository name", default="aur-clone")
+    parser.add_argument("--repository", help="repository name", required=True)
-    parser.set_defaults(handler=handlers.Setup, lock=None, no_report=True, unsafe=True)
+    parser.add_argument("--sign-key", help="sign key id")
+    parser.add_argument("--sign-target", help="sign options", type=SignSettings.from_option,
+                        choices=SignSettings, action="append")
+    parser.add_argument("--web-port", help="port of the web service", type=int)
+    parser.set_defaults(handler=handlers.Setup, lock=None, no_log=True, no_report=True, unsafe=True)
     return parser
 
 
@@ -178,9 +263,10 @@ def _set_sign_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("sign", description="(re-)sign packages and repository database")
+    parser = root.add_parser("sign", help="sign packages", description="(re-)sign packages and repository database",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("package", help="sign only specified packages", nargs="*")
-    parser.set_defaults(handler=handlers.Sign)
+    parser.set_defaults(handler=handlers.Sign, architecture=[])
     return parser
 
 
@@ -190,10 +276,12 @@ def _set_status_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("status", description="request status of the package")
+    parser = root.add_parser("status", help="get package status", description="request status of the package",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("--ahriman", help="get service status itself", action="store_true")
+    parser.add_argument("--status", help="filter packages by status", choices=BuildStatusEnum, type=BuildStatusEnum)
     parser.add_argument("package", help="filter status by package base", nargs="*")
-    parser.set_defaults(handler=handlers.Status, lock=None, no_report=True, unsafe=True)
+    parser.set_defaults(handler=handlers.Status, lock=None, no_log=True, no_report=True, unsafe=True)
     return parser
 
 
@@ -203,14 +291,16 @@ def _set_status_update_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("status-update", description="request status of the package")
+    parser = root.add_parser("status-update", help="update package status", description="request status of the package",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument(
         "package",
         help="set status for specified packages. If no packages supplied, service status will be updated",
         nargs="*")
-    parser.add_argument("--status", help="new status", choices=[value.value for value in BuildStatusEnum],
+    parser.add_argument("--status", help="new status", choices=BuildStatusEnum,
-                        default="success")
+                        type=BuildStatusEnum, default=BuildStatusEnum.Success)
-    parser.set_defaults(handler=handlers.StatusUpdate, lock=None, no_report=True, unsafe=True)
+    parser.add_argument("--remove", help="remove package status page", action="store_true")
+    parser.set_defaults(handler=handlers.StatusUpdate, lock=None, no_log=True, no_report=True, unsafe=True)
     return parser
 
 
@@ -220,9 +310,10 @@ def _set_sync_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("sync", description="sync packages to remote server")
+    parser = root.add_parser("sync", help="sync repository", description="sync packages to remote server",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("target", help="target to sync", nargs="*")
-    parser.set_defaults(handler=handlers.Sync)
+    parser.set_defaults(handler=handlers.Sync, architecture=[])
     return parser
 
 
@@ -232,13 +323,41 @@ def _set_update_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("update", description="run updates")
+    parser = root.add_parser("update", help="update packages", description="run updates",
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument("package", help="filter check by package base", nargs="*")
     parser.add_argument("--dry-run", help="just perform check for updates, same as check command", action="store_true")
     parser.add_argument("--no-aur", help="do not check for AUR updates. Implies --no-vcs", action="store_true")
     parser.add_argument("--no-manual", help="do not include manual updates", action="store_true")
     parser.add_argument("--no-vcs", help="do not check VCS packages", action="store_true")
-    parser.set_defaults(handler=handlers.Update)
+    parser.set_defaults(handler=handlers.Update, architecture=[])
+    return parser
+
+
+def _set_user_parser(root: SubParserAction) -> argparse.ArgumentParser:
+    """
+    add parser for create user subcommand
+    :param root: subparsers for the commands
+    :return: created argument parser
+    """
+    parser = root.add_parser(
+        "user",
+        help="manage users for web services",
+        description="manage users for web services with password and role. In case if password was not entered it will be asked interactively",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("username", help="username for web service")
+    parser.add_argument("--as-service", help="add user as service user", action="store_true")
+    parser.add_argument(
+        "-a",
+        "--access",
+        help="user access level",
+        type=UserAccess,
+        choices=UserAccess,
+        default=UserAccess.Read)
+    parser.add_argument("--no-reload", help="do not reload authentication module", action="store_true")
+    parser.add_argument("-p", "--password", help="user password")
+    parser.add_argument("-r", "--remove", help="remove user from configuration", action="store_true")
+    parser.set_defaults(handler=handlers.User, architecture=[""], lock=None, no_log=True, no_report=True, unsafe=True)
     return parser
 
 
@@ -248,16 +367,24 @@ def _set_web_parser(root: SubParserAction) -> argparse.ArgumentParser:
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("web", description="start web server")
+    parser = root.add_parser("web", help="start web server", description="start web server",
-    parser.set_defaults(handler=handlers.Web, lock=None, no_report=True)
+                             formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.set_defaults(handler=handlers.Web, lock=None, no_report=True, parser=_parser)
     return parser
 
 
-if __name__ == "__main__":
+def run() -> None:
-    args_parser = _parser()
+    """
-    args = args_parser.parse_args()
+    run application instance
+    """
+    if __name__ == "__main__":
+        args_parser = _parser()
+        args = args_parser.parse_args()
 
-    handler: handlers.Handler = args.handler
+        handler: handlers.Handler = args.handler
-    status = handler.execute(args)
+        status = handler.execute(args)
 
-    sys.exit(status)
+        sys.exit(status)
+
+
+run()

src/ahriman/application/application.py

@@ -35,21 +35,29 @@ class Application:
     """
     base application class
     :ivar architecture: repository architecture
-    :ivar config: configuration instance
+    :ivar configuration: configuration instance
     :ivar logger: application logger
     :ivar repository: repository instance
     """
 
-    def __init__(self, architecture: str, config: Configuration) -> None:
+    def __init__(self, architecture: str, configuration: Configuration, no_report: bool) -> None:
         """
         default constructor
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
         self.logger = logging.getLogger("root")
-        self.config = config
+        self.configuration = configuration
         self.architecture = architecture
-        self.repository = Repository(architecture, config)
+        self.repository = Repository(architecture, configuration, no_report)
+
+    def _finalize(self, built_packages: Iterable[Package]) -> None:
+        """
+        generate report and sync to remote server
+        """
+        self.report([], built_packages)
+        self.sync([], built_packages)
 
     def _known_packages(self) -> Set[str]:
         """
@@ -58,18 +66,13 @@ class Application:
         """
         known_packages: Set[str] = set()
         # local set
-        for package in self.repository.packages():
+        for base in self.repository.packages():
-            known_packages.update(package.packages.keys())
+            for package, properties in base.packages.items():
+                known_packages.add(package)
+                known_packages.update(properties.provides)
         known_packages.update(self.repository.pacman.all_packages())
         return known_packages
 
-    def _finalize(self) -> None:
-        """
-        generate report and sync to remote server
-        """
-        self.report([])
-        self.sync([])
-
     def get_updates(self, filter_packages: List[str], no_aur: bool, no_manual: bool, no_vcs: bool,
                     log_fn: Callable[[str], None]) -> List[Package]:
         """
@@ -106,7 +109,7 @@ class Application:
                 add_archive(full_path)
 
         def add_manual(src: str) -> Path:
-            package = Package.load(src, self.repository.pacman, self.config.get("alpm", "aur_url"))
+            package = Package.load(src, self.repository.pacman, self.configuration.get("alpm", "aur_url"))
             path = self.repository.paths.manual / package.base
             Task.fetch(path, package.git_url)
             return path
@@ -160,15 +163,16 @@ class Application:
         :param names: list of packages (either base or name) to remove
         """
         self.repository.process_remove(names)
-        self._finalize()
+        self._finalize([])
 
-    def report(self, target: Iterable[str]) -> None:
+    def report(self, target: Iterable[str], built_packages: Iterable[Package]) -> None:
         """
         generate report
         :param target: list of targets to run (e.g. html)
+        :param built_packages: list of packages which has just been built
         """
         targets = target or None
-        self.repository.process_report(targets)
+        self.repository.process_report(targets, built_packages)
 
     def sign(self, packages: Iterable[str]) -> None:
         """
@@ -182,6 +186,7 @@ class Application:
                 continue
             for archive in package.packages.values():
                 if archive.filepath is None:
+                    self.logger.warning("filepath is empty for %s", package.base)
                     continue  # avoid mypy warning
                 src = self.repository.paths.repository / archive.filepath
                 dst = self.repository.paths.packages / archive.filepath
@@ -190,15 +195,29 @@ class Application:
         self.update([])
         # sign repository database if set
         self.repository.sign.sign_repository(self.repository.repo.repo_path)
-        self._finalize()
+        self._finalize([])
 
-    def sync(self, target: Iterable[str]) -> None:
+    def sync(self, target: Iterable[str], built_packages: Iterable[Package]) -> None:
         """
         sync to remote server
         :param target: list of targets to run (e.g. s3)
+        :param built_packages: list of packages which has just been built
         """
         targets = target or None
-        self.repository.process_sync(targets)
+        self.repository.process_sync(targets, built_packages)
+
+    def unknown(self) -> List[Package]:
+        """
+        get packages which were not found in AUR
+        :return: unknown package list
+        """
+        packages = []
+        for base in self.repository.packages():
+            try:
+                _ = Package.from_aur(base.base, base.aur_url)
+            except Exception:
+                packages.append(base)
+        return packages
 
     def update(self, updates: Iterable[Package]) -> None:
         """
@@ -206,8 +225,11 @@ class Application:
         :param updates: list of packages to update
         """
         def process_update(paths: Iterable[Path]) -> None:
+            if not paths:
+                return  # don't need to process if no update supplied
+            updated = [Package.load(path, self.repository.pacman, self.repository.aur_url) for path in paths]
             self.repository.process_update(paths)
-            self._finalize()
+            self._finalize(updated)
 
         # process built packages
         packages = self.repository.packages_built()
@@ -216,6 +238,6 @@ class Application:
         # process manual packages
         tree = Tree.load(updates)
         for num, level in enumerate(tree.levels()):
-            self.logger.info(f"processing level #{num} {[package.base for package in level]}")
+            self.logger.info("processing level #%i %s", num, [package.base for package in level])
             packages = self.repository.process_build(level)
             process_update(packages)

src/ahriman/application/handlers/__init__.py

@@ -22,13 +22,18 @@ from ahriman.application.handlers.handler import Handler
 from ahriman.application.handlers.add import Add
 from ahriman.application.handlers.clean import Clean
 from ahriman.application.handlers.dump import Dump
+from ahriman.application.handlers.init import Init
+from ahriman.application.handlers.key_import import KeyImport
 from ahriman.application.handlers.rebuild import Rebuild
 from ahriman.application.handlers.remove import Remove
+from ahriman.application.handlers.remove_unknown import RemoveUnknown
 from ahriman.application.handlers.report import Report
+from ahriman.application.handlers.search import Search
 from ahriman.application.handlers.setup import Setup
 from ahriman.application.handlers.sign import Sign
 from ahriman.application.handlers.status import Status
 from ahriman.application.handlers.status_update import StatusUpdate
 from ahriman.application.handlers.sync import Sync
 from ahriman.application.handlers.update import Update
+from ahriman.application.handlers.user import User
 from ahriman.application.handlers.web import Web

src/ahriman/application/handlers/add.py

@@ -32,11 +32,19 @@ class Add(Handler):
     """
 
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
-        Application(architecture, config).add(args.package, args.without_dependencies)
+        application = Application(architecture, configuration, no_report)
+        application.add(args.package, args.without_dependencies)
+        if not args.now:
+            return
+
+        packages = application.get_updates(args.package, True, False, True, application.logger.info)
+        application.update(packages)

src/ahriman/application/handlers/clean.py

@@ -32,12 +32,14 @@ class Clean(Handler):
     """
 
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
-        Application(architecture, config).clean(args.no_build, args.no_cache, args.no_chroot,
+        Application(architecture, configuration, no_report).clean(args.no_build, args.no_cache, args.no_chroot,
-                                                args.no_manual, args.no_packages)
+                                                                  args.no_manual, args.no_packages)

src/ahriman/application/handlers/dump.py

@@ -27,20 +27,24 @@ from ahriman.core.configuration import Configuration
 
 class Dump(Handler):
     """
-    dump config handler
+    dump configuration handler
     """
 
+    _print = print
+
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
-        config_dump = config.dump(architecture)
+        dump = configuration.dump()
-        for section, values in sorted(config_dump.items()):
+        for section, values in sorted(dump.items()):
-            print(f"[{section}]")
+            Dump._print(f"[{section}]")
             for key, value in sorted(values.items()):
-                print(f"{key} = {value}")
+                Dump._print(f"{key} = {value}")
-            print()
+            Dump._print()

src/ahriman/application/handlers/handler.py

@@ -23,29 +23,34 @@ import argparse
 import logging
 
 from multiprocessing import Pool
-from typing import Type
+from typing import Set, Type
 
 from ahriman.application.lock import Lock
 from ahriman.core.configuration import Configuration
+from ahriman.core.exceptions import MissingArchitecture, MultipleArchitecture
+from ahriman.models.repository_paths import RepositoryPaths
 
 
 class Handler:
     """
     base handler class for command callbacks
+    :cvar ALLOW_MULTI_ARCHITECTURE_RUN: allow to run with multiple architectures
     """
 
+    ALLOW_MULTI_ARCHITECTURE_RUN = True
+
     @classmethod
-    def _call(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> bool:
+    def call(cls: Type[Handler], args: argparse.Namespace, architecture: str) -> bool:
         """
         additional function to wrap all calls for multiprocessing library
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
         :return: True on success, False otherwise
         """
         try:
-            with Lock(args, architecture, config):
+            configuration = Configuration.from_path(args.configuration, architecture, not args.no_log)
-                cls.run(args, architecture, config)
+            with Lock(args, architecture, configuration):
+                cls.run(args, architecture, configuration, args.no_report)
             return True
         except Exception:
             logging.getLogger("root").exception("process exception")
@@ -58,18 +63,51 @@ class Handler:
         :param args: command line args
         :return: 0 on success, 1 otherwise
         """
-        configuration = Configuration.from_path(args.config, not args.no_log)
+        architectures = cls.extract_architectures(args)
-        with Pool(len(args.architecture)) as pool:
+
-            result = pool.starmap(
+        # actually we do not have to spawn another process if it is single-process application, do we?
-                cls._call, [(args, architecture, configuration) for architecture in args.architecture])
+        if len(architectures) > 1:
+            if not cls.ALLOW_MULTI_ARCHITECTURE_RUN:
+                raise MultipleArchitecture(args.command)
+
+            with Pool(len(architectures)) as pool:
+                result = pool.starmap(
+                    cls.call, [(args, architecture) for architecture in architectures])
+        else:
+            result = [cls.call(args, architectures.pop())]
+
         return 0 if all(result) else 1
 
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def extract_architectures(cls: Type[Handler], args: argparse.Namespace) -> Set[str]:
+        """
+        get known architectures
+        :param args: command line args
+        :return: list of architectures for which tree is created
+        """
+        if args.architecture is None:
+            raise MissingArchitecture(args.command)
+        if args.architecture:
+            return set(args.architecture)
+
+        config = Configuration()
+        config.load(args.configuration)
+        # wtf???
+        root = config.getpath("repository", "root")  # pylint: disable=assignment-from-no-return
+        architectures = RepositoryPaths.known_architectures(root)
+
+        if not architectures:
+            raise MissingArchitecture(args.command)
+        return architectures
+
+    @classmethod
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
         raise NotImplementedError

src/ahriman/application/handlers/init.py

@@ -0,0 +1,44 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+
+from typing import Type
+
+from ahriman.application.application import Application
+from ahriman.application.handlers.handler import Handler
+from ahriman.core.configuration import Configuration
+
+
+class Init(Handler):
+    """
+    repository init handler
+    """
+
+    @classmethod
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
+        """
+        callback for command line
+        :param args: command line args
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
+        """
+        Application(architecture, configuration, no_report).repository.repo.init()

src/ahriman/application/handlers/key_import.py

@@ -0,0 +1,44 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+
+from typing import Type
+
+from ahriman.application.application import Application
+from ahriman.application.handlers.handler import Handler
+from ahriman.core.configuration import Configuration
+
+
+class KeyImport(Handler):
+    """
+    key import packages handler
+    """
+
+    @classmethod
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
+        """
+        callback for command line
+        :param args: command line args
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
+        """
+        Application(architecture, configuration, no_report).repository.sign.import_key(args.key_server, args.key)

src/ahriman/application/handlers/rebuild.py

@@ -32,13 +32,21 @@ class Rebuild(Handler):
     """
 
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
-        application = Application(architecture, config)
+        depends_on = set(args.depends_on) if args.depends_on else None
-        packages = application.repository.packages()
+
+        application = Application(architecture, configuration, no_report)
+        packages = [
+            package
+            for package in application.repository.packages()
+            if depends_on is None or depends_on.intersection(package.depends)
+        ]  # we have to use explicit list here for testing purpose
         application.update(packages)

src/ahriman/application/handlers/remove.py

@@ -32,11 +32,13 @@ class Remove(Handler):
     """
 
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
-        Application(architecture, config).remove(args.package)
+        Application(architecture, configuration, no_report).remove(args.package)

src/ahriman/application/handlers/remove_unknown.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+
+from typing import Type
+
+from ahriman.application.application import Application
+from ahriman.application.handlers.handler import Handler
+from ahriman.core.configuration import Configuration
+from ahriman.models.package import Package
+
+
+class RemoveUnknown(Handler):
+    """
+    remove unknown packages handler
+    """
+
+    @classmethod
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
+        """
+        callback for command line
+        :param args: command line args
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
+        """
+        application = Application(architecture, configuration, no_report)
+        unknown_packages = application.unknown()
+        if args.dry_run:
+            for package in unknown_packages:
+                RemoveUnknown.log_fn(package)
+            return
+
+        application.remove(package.base for package in unknown_packages)
+
+    @staticmethod
+    def log_fn(package: Package) -> None:
+        """
+        log package information
+        :param package: package object to log
+        """
+        print(f"=> {package.base} {package.version}")
+        print(f"   {package.web_url}")

src/ahriman/application/handlers/report.py

@@ -32,11 +32,13 @@ class Report(Handler):
     """
 
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
-        Application(architecture, config).report(args.target)
+        Application(architecture, configuration, no_report).report(args.target, [])

src/ahriman/application/handlers/search.py

@@ -0,0 +1,60 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+import aur  # type: ignore
+
+from typing import Callable, Type
+
+from ahriman.application.handlers.handler import Handler
+from ahriman.core.configuration import Configuration
+
+
+class Search(Handler):
+    """
+    packages search handler
+    """
+
+    @classmethod
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
+        """
+        callback for command line
+        :param args: command line args
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
+        """
+        search = " ".join(args.search)
+        packages = aur.search(search)
+
+        # it actually always should return string
+        # explicit cast to string just to avoid mypy warning for untyped library
+        comparator: Callable[[aur.Package], str] = lambda item: str(item.package_base)
+        for package in sorted(packages, key=comparator):
+            Search.log_fn(package)
+
+    @staticmethod
+    def log_fn(package: aur.Package) -> None:
+        """
+        log package information
+        :param package: package object as from AUR
+        """
+        print(f"=> {package.package_base} {package.version}")
+        print(f"   {package.description}")

src/ahriman/application/handlers/setup.py

@@ -18,7 +18,6 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import argparse
-import configparser
 
 from pathlib import Path
 from typing import Type
@@ -44,19 +43,21 @@ class Setup(Handler):
     SUDOERS_PATH = Path("/etc/sudoers.d/ahriman")
 
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
-        application = Application(architecture, config)
+        application = Application(architecture, configuration, no_report)
         Setup.create_makepkg_configuration(args.packager, application.repository.paths)
         Setup.create_executable(args.build_command, architecture)
-        Setup.create_devtools_configuration(args.build_command, architecture, Path(args.from_config), args.no_multilib,
+        Setup.create_devtools_configuration(args.build_command, architecture, args.from_configuration,
-                                            args.repository, application.repository.paths)
+                                            args.no_multilib, args.repository, application.repository.paths)
-        Setup.create_ahriman_configuration(args.build_command, architecture, args.repository, config.include)
+        Setup.create_ahriman_configuration(args, architecture, args.repository, configuration.include)
         Setup.create_sudo_configuration(args.build_command, architecture)
 
     @staticmethod
@@ -70,25 +71,33 @@ class Setup(Handler):
         return Setup.BIN_DIR_PATH / f"{prefix}-{architecture}-build"
 
     @staticmethod
-    def create_ahriman_configuration(prefix: str, architecture: str, repository: str, include_path: Path) -> None:
+    def create_ahriman_configuration(args: argparse.Namespace, architecture: str, repository: str,
+                                     include_path: Path) -> None:
         """
         create service specific configuration
-        :param prefix: command prefix in {prefix}-{architecture}-build
+        :param args: command line args
         :param architecture: repository architecture
         :param repository: repository name
         :param include_path: path to directory with configuration includes
         """
-        config = configparser.ConfigParser()
+        configuration = Configuration()
 
-        config.add_section("build")
+        section = Configuration.section_name("build", architecture)
-        config.set("build", "build_command", str(Setup.build_command(prefix, architecture)))
+        configuration.set_option(section, "build_command", str(Setup.build_command(args.build_command, architecture)))
+        configuration.set_option("repository", "name", repository)
 
-        config.add_section("repository")
+        if args.sign_key is not None:
-        config.set("repository", "name", repository)
+            section = Configuration.section_name("sign", architecture)
+            configuration.set_option(section, "target", " ".join([target.name.lower() for target in args.sign_target]))
+            configuration.set_option(section, "key", args.sign_key)
 
-        target = include_path / "build-overrides.ini"
+        if args.web_port is not None:
-        with target.open("w") as ahriman_config:
+            section = Configuration.section_name("web", architecture)
-            config.write(ahriman_config)
+            configuration.set_option(section, "port", str(args.web_port))
+
+        target = include_path / "setup-overrides.ini"
+        with target.open("w") as ahriman_configuration:
+            configuration.write(ahriman_configuration)
 
     @staticmethod
     def create_devtools_configuration(prefix: str, architecture: str, source: Path,
@@ -102,31 +111,29 @@ class Setup(Handler):
         :param repository: repository name
         :param paths: repository paths instance
         """
-        config = configparser.ConfigParser()
+        configuration = Configuration()
         # preserve case
         # stupid mypy thinks that it is impossible
-        config.optionxform = lambda key: key  # type: ignore
+        configuration.optionxform = lambda key: key  # type: ignore
 
         # load default configuration first
         # we cannot use Include here because it will be copied to new chroot, thus no includes there
-        config.read(source)
+        configuration.read(source)
 
         # set our architecture now
-        config.set("options", "Architecture", architecture)
+        configuration.set_option("options", "Architecture", architecture)
 
         # add multilib
         if not no_multilib:
-            config.add_section("multilib")
+            configuration.set_option("multilib", "Include", str(Setup.MIRRORLIST_PATH))
-            config.set("multilib", "Include", str(Setup.MIRRORLIST_PATH))
 
         # add repository itself
-        config.add_section(repository)
+        configuration.set_option(repository, "SigLevel", "Optional TrustAll")  # we don't care
-        config.set(repository, "SigLevel", "Optional TrustAll")  # we don't care
+        configuration.set_option(repository, "Server", f"file://{paths.repository}")
-        config.set(repository, "Server", f"file://{paths.repository}")
 
         target = source.parent / f"pacman-{prefix}-{architecture}.conf"
-        with target.open("w") as devtools_config:
+        with target.open("w") as devtools_configuration:
-            config.write(devtools_config)
+            configuration.write(devtools_configuration)
 
     @staticmethod
     def create_makepkg_configuration(packager: str, paths: RepositoryPaths) -> None:
@@ -145,7 +152,7 @@ class Setup(Handler):
         :param architecture: repository architecture
         """
         command = Setup.build_command(prefix, architecture)
-        Setup.SUDOERS_PATH.write_text(f"ahriman ALL=(ALL) NOPASSWD: {command} *\n")
+        Setup.SUDOERS_PATH.write_text(f"ahriman ALL=(ALL) NOPASSWD: {command} *\n", encoding="utf8")
         Setup.SUDOERS_PATH.chmod(0o400)  # security!
 
     @staticmethod

src/ahriman/application/handlers/sign.py

@@ -32,11 +32,13 @@ class Sign(Handler):
     """
 
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
-        Application(architecture, config).sign(args.package)
+        Application(architecture, configuration, no_report).sign(args.package)

src/ahriman/application/handlers/status.py

@@ -19,7 +19,7 @@
 #
 import argparse
 
-from typing import Iterable, Tuple, Type
+from typing import Callable, Iterable, Tuple, Type
 
 from ahriman.application.application import Application
 from ahriman.application.handlers.handler import Handler
@@ -34,25 +34,32 @@ class Status(Handler):
     """
 
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
-        application = Application(architecture, config)
+        # we are using reporter here
+        client = Application(architecture, configuration, no_report=False).repository.reporter
         if args.ahriman:
-            ahriman = application.repository.reporter.get_self()
+            ahriman = client.get_self()
             print(ahriman.pretty_print())
             print()
         if args.package:
             packages: Iterable[Tuple[Package, BuildStatus]] = sum(
-                [application.repository.reporter.get(base) for base in args.package],
+                [client.get(base) for base in args.package],
                 start=[])
         else:
-            packages = application.repository.reporter.get(None)
+            packages = client.get(None)
-        for package, package_status in sorted(packages, key=lambda item: item[0].base):
+
+        comparator: Callable[[Tuple[Package, BuildStatus]], str] = lambda item: item[0].base
+        filter_fn: Callable[[Tuple[Package, BuildStatus]], bool] =\
+            lambda item: args.status is None or item[1].status == args.status
+        for package, package_status in sorted(filter(filter_fn, packages), key=comparator):
             print(package.pretty_print())
             print(f"\t{package.version}")
             print(f"\t{package_status.pretty_print()}")

src/ahriman/application/handlers/status_update.py

@@ -19,12 +19,12 @@
 #
 import argparse
 
-from typing import Type
+from typing import Callable, Type
 
 from ahriman.application.application import Application
 from ahriman.application.handlers.handler import Handler
 from ahriman.core.configuration import Configuration
-from ahriman.models.build_status import BuildStatusEnum
+from ahriman.core.exceptions import InvalidCommand
 
 
 class StatusUpdate(Handler):
@@ -33,19 +33,24 @@ class StatusUpdate(Handler):
     """
 
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
-        client = Application(architecture, config).repository.reporter
+        # we are using reporter here
-        status = BuildStatusEnum(args.status)
+        client = Application(architecture, configuration, no_report=False).repository.reporter
+        callback: Callable[[str], None] = lambda p: client.remove(p) if args.remove else client.update(p, args.status)
         if args.package:
             # update packages statuses
             for package in args.package:
-                client.update(package, status)
+                callback(package)
+        elif args.remove:
+            raise InvalidCommand("Remove option is supplied, but no packages set")
         else:
             # update service status
-            client.update_self(status)
+            client.update_self(args.status)

src/ahriman/application/handlers/sync.py

@@ -32,11 +32,13 @@ class Sync(Handler):
     """
 
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
-        Application(architecture, config).sync(args.target)
+        Application(architecture, configuration, no_report).sync(args.target, [])

src/ahriman/application/handlers/update.py

@@ -19,7 +19,7 @@
 #
 import argparse
 
-from typing import Type
+from typing import Callable, Type
 
 from ahriman.application.application import Application
 from ahriman.application.handlers.handler import Handler
@@ -32,20 +32,31 @@ class Update(Handler):
     """
 
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
-        # typing workaround
+        application = Application(architecture, configuration, no_report)
-        def log_fn(line: str) -> None:
+        packages = application.get_updates(args.package, args.no_aur, args.no_manual, args.no_vcs,
-            return print(line) if args.dry_run else application.logger.info(line)
+                                           Update.log_fn(application, args.dry_run))
-
-        application = Application(architecture, config)
-        packages = application.get_updates(args.package, args.no_aur, args.no_manual, args.no_vcs, log_fn)
         if args.dry_run:
             return
 
         application.update(packages)
+
+    @staticmethod
+    def log_fn(application: Application, dry_run: bool) -> Callable[[str], None]:
+        """
+        package updates log function
+        :param application: application instance
+        :param dry_run: do not perform update itself
+        :return: in case if dry_run is set it will return print, logger otherwise
+        """
+        def inner(line: str) -> None:
+            return print(line) if dry_run else application.logger.info(line)
+        return inner

src/ahriman/application/handlers/user.py

@@ -0,0 +1,139 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+import getpass
+
+from pathlib import Path
+from typing import Type
+
+from ahriman.application.application import Application
+from ahriman.application.handlers.handler import Handler
+from ahriman.core.configuration import Configuration
+from ahriman.models.user import User as MUser
+from ahriman.models.user_access import UserAccess
+
+
+class User(Handler):
+    """
+    user management handler
+    """
+
+    @classmethod
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
+        """
+        callback for command line
+        :param args: command line args
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
+        """
+        salt = User.get_salt(configuration)
+        user = User.create_user(args)
+        auth_configuration = User.get_auth_configuration(configuration.include)
+
+        User.clear_user(auth_configuration, user)
+        if not args.remove:
+            User.create_configuration(auth_configuration, user, salt, args.as_service)
+        User.write_configuration(auth_configuration)
+
+        if not args.no_reload:
+            client = Application(architecture, configuration, no_report=False).repository.reporter
+            client.reload_auth()
+
+    @staticmethod
+    def clear_user(configuration: Configuration, user: MUser) -> None:
+        """
+        remove user user from configuration file in case if it exists
+        :param configuration: configuration instance
+        :param user: user descriptor
+        """
+        for role in UserAccess:
+            section = Configuration.section_name("auth", role.value)
+            if not configuration.has_option(section, user.username):
+                continue
+            configuration.remove_option(section, user.username)
+
+    @staticmethod
+    def create_configuration(configuration: Configuration, user: MUser, salt: str, as_service_user: bool) -> None:
+        """
+        put new user to configuration
+        :param configuration: configuration instance
+        :param user: user descriptor
+        :param salt: password hash salt
+        :param as_service_user: add user as service user, also set password and user to configuration
+        """
+        section = Configuration.section_name("auth", user.access.value)
+        configuration.set_option("auth", "salt", salt)
+        configuration.set_option(section, user.username, user.hash_password(salt))
+
+        if as_service_user:
+            configuration.set_option("web", "username", user.username)
+            configuration.set_option("web", "password", user.password)
+
+    @staticmethod
+    def create_user(args: argparse.Namespace) -> MUser:
+        """
+        create user descriptor from arguments
+        :param args: command line args
+        :return: built user descriptor
+        """
+        user = MUser(args.username, args.password, args.access)
+        if user.password is None:
+            user.password = getpass.getpass()
+        return user
+
+    @staticmethod
+    def get_auth_configuration(include_path: Path) -> Configuration:
+        """
+        create configuration instance
+        :param include_path: path to directory with configuration includes
+        :return: configuration instance. In case if there are local settings they will be loaded
+        """
+        target = include_path / "auth.ini"
+        configuration = Configuration()
+        configuration.load(target)
+
+        return configuration
+
+    @staticmethod
+    def get_salt(configuration: Configuration, salt_length: int = 20) -> str:
+        """
+        get salt from configuration or create new string
+        :param configuration: configuration instance
+        :param salt_length: salt length
+        :return: current salt
+        """
+        salt = configuration.get("auth", "salt", fallback=None)
+        if salt:
+            return salt
+        return MUser.generate_password(salt_length)
+
+    @staticmethod
+    def write_configuration(configuration: Configuration) -> None:
+        """
+        write configuration file
+        :param configuration: configuration instance
+        """
+        if configuration.path is None:
+            return  # should never happen actually
+        with configuration.path.open("w") as ahriman_configuration:
+            configuration.write(ahriman_configuration)
+        configuration.path.chmod(0o600)

src/ahriman/application/handlers/web.py

@@ -23,6 +23,7 @@ from typing import Type
 
 from ahriman.application.handlers.handler import Handler
 from ahriman.core.configuration import Configuration
+from ahriman.core.spawn import Spawn
 
 
 class Web(Handler):
@@ -30,14 +31,23 @@ class Web(Handler):
     web server handler
     """
 
+    ALLOW_MULTI_ARCHITECTURE_RUN = False  # required to be able to spawn external processes
+
     @classmethod
-    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool) -> None:
         """
         callback for command line
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
         """
+        # we are using local import for optional dependencies
         from ahriman.web.web import run_server, setup_service
-        application = setup_service(architecture, config)
+
+        spawner = Spawn(args.parser(), architecture, configuration)
+        spawner.start()
+
+        application = setup_service(architecture, configuration, spawner)
         run_server(application)

src/ahriman/application/lock.py

@@ -20,12 +20,14 @@
 from __future__ import annotations
 
 import argparse
+import logging
 import os
 
 from pathlib import Path
 from types import TracebackType
 from typing import Literal, Optional, Type
 
+from ahriman import version
 from ahriman.core.configuration import Configuration
 from ahriman.core.exceptions import DuplicateRun, UnsafeRun
 from ahriman.core.status.client import Client
@@ -42,31 +44,32 @@ class Lock:
     :ivar unsafe: skip user check
     """
 
-    def __init__(self, args: argparse.Namespace, architecture: str, config: Configuration) -> None:
+    def __init__(self, args: argparse.Namespace, architecture: str, configuration: Configuration) -> None:
         """
         default constructor
         :param args: command line args
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
         """
         self.path = Path(f"{args.lock}_{architecture}") if args.lock is not None else None
         self.force = args.force
         self.unsafe = args.unsafe
 
-        self.root = Path(config.get("repository", "root"))
+        self.root = Path(configuration.get("repository", "root"))
-        self.reporter = Client() if args.no_report else Client.load(architecture, config)
+        self.reporter = Client() if args.no_report else Client.load(configuration)
 
     def __enter__(self) -> Lock:
         """
         default workflow is the following:
 
             check user UID
-            remove lock file if force flag is set
             check if there is lock file
+            check web status watcher status
             create lock file
             report to web if enabled
         """
         self.check_user()
+        self.check_version()
         self.create()
         self.reporter.update_self(BuildStatusEnum.Building)
         return self
@@ -80,11 +83,22 @@ class Lock:
         :param exc_tb: exception traceback if any
         :return: always False (do not suppress any exception)
         """
-        self.remove()
+        self.clear()
         status = BuildStatusEnum.Success if exc_val is None else BuildStatusEnum.Failed
         self.reporter.update_self(status)
         return False
 
+    def check_version(self) -> None:
+        """
+        check web server version
+        """
+        status = self.reporter.get_internal()
+        if status.version is not None and status.version != version.__version__:
+            logging.getLogger("root").warning(
+                "status watcher version mismatch, our %s, their %s",
+                version.__version__,
+                status.version)
+
     def check_user(self) -> None:
         """
         check if current user is actually owner of ahriman root
@@ -96,6 +110,14 @@ class Lock:
         if current_uid != root_uid:
             raise UnsafeRun(current_uid, root_uid)
 
+    def clear(self) -> None:
+        """
+        remove lock file
+        """
+        if self.path is None:
+            return
+        self.path.unlink(missing_ok=True)
+
     def create(self) -> None:
         """
         create lock file
@@ -106,11 +128,3 @@ class Lock:
             self.path.touch(exist_ok=self.force)
         except FileExistsError:
             raise DuplicateRun()
-
-    def remove(self) -> None:
-        """
-        remove lock file
-        """
-        if self.path is None:
-            return
-        self.path.unlink(missing_ok=True)

src/ahriman/core/alpm/pacman.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from pyalpm import Handle  # type: ignore
-from typing import List, Set
+from typing import Set
 
 from ahriman.core.configuration import Configuration
 
@@ -29,24 +29,26 @@ class Pacman:
     :ivar handle: pyalpm root `Handle`
     """
 
-    def __init__(self, config: Configuration) -> None:
+    def __init__(self, configuration: Configuration) -> None:
         """
         default constructor
-        :param config: configuration instance
+        :param configuration: configuration instance
         """
-        root = config.get("alpm", "root")
+        root = configuration.get("alpm", "root")
-        pacman_root = config.getpath("alpm", "database")
+        pacman_root = configuration.getpath("alpm", "database")
         self.handle = Handle(root, str(pacman_root))
-        for repository in config.getlist("alpm", "repositories"):
+        for repository in configuration.getlist("alpm", "repositories"):
             self.handle.register_syncdb(repository, 0)  # 0 is pgp_level
 
-    def all_packages(self) -> List[str]:
+    def all_packages(self) -> Set[str]:
         """
         get list of packages known for alpm
         :return: list of package names
         """
         result: Set[str] = set()
         for database in self.handle.get_syncdbs():
-            result.update({package.name for package in database.pkgcache})
+            for package in database.pkgcache:
+                result.add(package.name)  # package itself
+                result.update(package.provides)  # provides list for meta-packages
 
-        return list(result)
+        return result

src/ahriman/core/alpm/repo.py

@@ -68,6 +68,16 @@ class Repo:
             cwd=self.paths.repository,
             logger=self.logger)
 
+    def init(self) -> None:
+        """
+        create empty repository database
+        """
+        Repo._check_output(
+            "repo-add", *self.sign_args, str(self.repo_path),
+            exception=None,
+            cwd=self.paths.repository,
+            logger=self.logger)
+
     def remove(self, package: str, filename: Path) -> None:
         """
         remove package from repository

src/ahriman/core/auth/__init__.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#

src/ahriman/core/auth/auth.py

@@ -0,0 +1,149 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+import logging
+
+from typing import Dict, Optional, Type
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.exceptions import DuplicateUser
+from ahriman.models.auth_settings import AuthSettings
+from ahriman.models.user import User
+from ahriman.models.user_access import UserAccess
+
+
+class Auth:
+    """
+    helper to deal with user authorization
+    :ivar allowed_paths: URI paths which can be accessed without authorization
+    :ivar allowed_paths_groups: URI paths prefixes which can be accessed without authorization
+    :ivar enabled: indicates if authorization is enabled
+    :cvar ALLOWED_PATHS: URI paths which can be accessed without authorization, predefined
+    :cvar ALLOWED_PATHS_GROUPS: URI paths prefixes which can be accessed without authorization, predefined
+    """
+
+    ALLOWED_PATHS = {"/", "/index.html"}
+    ALLOWED_PATHS_GROUPS = {"/static", "/user-api"}
+
+    def __init__(self, configuration: Configuration, provider: AuthSettings = AuthSettings.Disabled) -> None:
+        """
+        default constructor
+        :param configuration: configuration instance
+        :param provider: authorization type definition
+        """
+        self.logger = logging.getLogger("http")
+
+        self.allow_read_only = configuration.getboolean("auth", "allow_read_only")
+        self.allowed_paths = set(configuration.getlist("auth", "allowed_paths", fallback=[]))
+        self.allowed_paths.update(self.ALLOWED_PATHS)
+        self.allowed_paths_groups = set(configuration.getlist("auth", "allowed_paths_groups", fallback=[]))
+        self.allowed_paths_groups.update(self.ALLOWED_PATHS_GROUPS)
+        self.enabled = provider.is_enabled
+        self.max_age = configuration.getint("auth", "max_age", fallback=7 * 24 * 3600)
+
+    @property
+    def auth_control(self) -> str:
+        """
+        This workaround is required to make different behaviour for login interface.
+        In case of internal authentication it must provide an interface (modal form) to login with button sends POST
+        request. But for an external providers behaviour can be different: e.g. OAuth provider requires sending GET
+        request to external resource
+        :return: login control as html code to insert
+        """
+        return """<button type="button" class="btn btn-link" data-bs-toggle="modal" data-bs-target="#loginForm" style="text-decoration: none">login</button>"""
+
+    @classmethod
+    def load(cls: Type[Auth], configuration: Configuration) -> Auth:
+        """
+        load authorization module from settings
+        :param configuration: configuration instance
+        :return: authorization module according to current settings
+        """
+        provider = AuthSettings.from_option(configuration.get("auth", "target", fallback="disabled"))
+        if provider == AuthSettings.Configuration:
+            from ahriman.core.auth.mapping import Mapping
+            return Mapping(configuration)
+        if provider == AuthSettings.OAuth:
+            from ahriman.core.auth.oauth import OAuth
+            return OAuth(configuration)
+        return cls(configuration)
+
+    @staticmethod
+    def get_users(configuration: Configuration) -> Dict[str, User]:
+        """
+        load users from settings
+        :param configuration: configuration instance
+        :return: map of username to its descriptor
+        """
+        users: Dict[str, User] = {}
+        for role in UserAccess:
+            section = configuration.section_name("auth", role.value)
+            if not configuration.has_section(section):
+                continue
+            for user, password in configuration[section].items():
+                normalized_user = user.lower()
+                if normalized_user in users:
+                    raise DuplicateUser(normalized_user)
+                users[normalized_user] = User(normalized_user, password, role)
+        return users
+
+    async def check_credentials(self, username: Optional[str], password: Optional[str]) -> bool:  # pylint: disable=no-self-use
+        """
+        validate user password
+        :param username: username
+        :param password: entered password
+        :return: True in case if password matches, False otherwise
+        """
+        del username, password
+        return True
+
+    async def is_safe_request(self, uri: Optional[str], required: UserAccess) -> bool:
+        """
+        check if requested path are allowed without authorization
+        :param uri: request uri
+        :param required: required access level
+        :return: True in case if this URI can be requested without authorization and False otherwise
+        """
+        if required == UserAccess.Read and self.allow_read_only:
+            return True  # in case if read right requested and allowed in options
+        if not uri:
+            return False  # request without context is not allowed
+        return uri in self.allowed_paths or any(uri.startswith(path) for path in self.allowed_paths_groups)
+
+    async def known_username(self, username: Optional[str]) -> bool:  # pylint: disable=no-self-use
+        """
+        check if user is known
+        :param username: username
+        :return: True in case if user is known and can be authorized and False otherwise
+        """
+        del username
+        return True
+
+    async def verify_access(self, username: str, required: UserAccess, context: Optional[str]) -> bool:  # pylint: disable=no-self-use
+        """
+        validate if user has access to requested resource
+        :param username: username
+        :param required: required access level
+        :param context: URI request path
+        :return: True in case if user is allowed to do this request and False otherwise
+        """
+        del username, required, context
+        return True

src/ahriman/core/auth/helpers.py

@@ -0,0 +1,70 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from typing import Any
+
+try:
+    import aiohttp_security  # type: ignore
+    _has_aiohttp_security = True
+except ImportError:
+    _has_aiohttp_security = False
+
+
+async def authorized_userid(*args: Any) -> Any:
+    """
+    handle aiohttp security methods
+    :param args: argument list as provided by authorized_userid function
+    :return: None in case if no aiohttp_security module found and function call otherwise
+    """
+    if _has_aiohttp_security:
+        return await aiohttp_security.authorized_userid(*args)  # pylint: disable=no-value-for-parameter
+    return None
+
+
+async def check_authorized(*args: Any) -> Any:
+    """
+    handle aiohttp security methods
+    :param args: argument list as provided by check_authorized function
+    :return: None in case if no aiohttp_security module found and function call otherwise
+    """
+    if _has_aiohttp_security:
+        return await aiohttp_security.check_authorized(*args)  # pylint: disable=no-value-for-parameter
+    return None
+
+
+async def forget(*args: Any) -> Any:
+    """
+    handle aiohttp security methods
+    :param args: argument list as provided by forget function
+    :return: None in case if no aiohttp_security module found and function call otherwise
+    """
+    if _has_aiohttp_security:
+        return await aiohttp_security.forget(*args)  # pylint: disable=no-value-for-parameter
+    return None
+
+
+async def remember(*args: Any) -> Any:
+    """
+    handle disabled auth
+    :param args: argument list as provided by remember function
+    :return: None in case if no aiohttp_security module found and function call otherwise
+    """
+    if _has_aiohttp_security:
+        return await aiohttp_security.remember(*args)  # pylint: disable=no-value-for-parameter
+    return None

src/ahriman/core/auth/mapping.py

@@ -0,0 +1,84 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from typing import Optional
+
+from ahriman.core.auth.auth import Auth
+from ahriman.core.configuration import Configuration
+from ahriman.models.auth_settings import AuthSettings
+from ahriman.models.user import User
+from ahriman.models.user_access import UserAccess
+
+
+class Mapping(Auth):
+    """
+    user authorization based on mapping from configuration file
+    :ivar salt: random generated string to salt passwords
+    :ivar _users: map of username to its descriptor
+    """
+
+    def __init__(self, configuration: Configuration, provider: AuthSettings = AuthSettings.Configuration) -> None:
+        """
+        default constructor
+        :param configuration: configuration instance
+        :param provider: authorization type definition
+        """
+        Auth.__init__(self, configuration, provider)
+        self.salt = configuration.get("auth", "salt")
+        self._users = self.get_users(configuration)
+
+    async def check_credentials(self, username: Optional[str], password: Optional[str]) -> bool:
+        """
+        validate user password
+        :param username: username
+        :param password: entered password
+        :return: True in case if password matches, False otherwise
+        """
+        if username is None or password is None:
+            return False  # invalid data supplied
+        user = self.get_user(username)
+        return user is not None and user.check_credentials(password, self.salt)
+
+    def get_user(self, username: str) -> Optional[User]:
+        """
+        retrieve user from in-memory mapping
+        :param username: username
+        :return: user descriptor if username is known and None otherwise
+        """
+        normalized_user = username.lower()
+        return self._users.get(normalized_user)
+
+    async def known_username(self, username: Optional[str]) -> bool:
+        """
+        check if user is known
+        :param username: username
+        :return: True in case if user is known and can be authorized and False otherwise
+        """
+        return username is not None and self.get_user(username) is not None
+
+    async def verify_access(self, username: str, required: UserAccess, context: Optional[str]) -> bool:
+        """
+        validate if user has access to requested resource
+        :param username: username
+        :param required: required access level
+        :param context: URI request path
+        :return: True in case if user is allowed to do this request and False otherwise
+        """
+        user = self.get_user(username)
+        return user is not None and user.verify_access(required)

src/ahriman/core/auth/oauth.py

@@ -0,0 +1,113 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import aioauth_client
+
+from typing import Optional, Type
+
+from ahriman.core.auth.mapping import Mapping
+from ahriman.core.configuration import Configuration
+from ahriman.core.exceptions import InvalidOption
+from ahriman.models.auth_settings import AuthSettings
+
+
+class OAuth(Mapping):
+    """
+    OAuth user authorization.
+    It is required to create application first and put application credentials.
+    :ivar client_id: application client id
+    :ivar client_secret: application client secret key
+    :ivar provider: provider class, should be one of aiohttp-client provided classes
+    :ivar redirect_uri: redirect URI registered in provider
+    :ivar scopes: list of scopes required by the application
+    """
+
+    def __init__(self, configuration: Configuration, provider: AuthSettings = AuthSettings.OAuth) -> None:
+        """
+        default constructor
+        :param configuration: configuration instance
+        :param provider: authorization type definition
+        """
+        Mapping.__init__(self, configuration, provider)
+        self.client_id = configuration.get("auth", "client_id")
+        self.client_secret = configuration.get("auth", "client_secret")
+        # in order to use OAuth feature the service must be publicity available
+        # thus we expect that address is set
+        self.redirect_uri = f"""{configuration.get("web", "address")}/user-api/v1/login"""
+        self.provider = self.get_provider(configuration.get("auth", "oauth_provider"))
+        # it is list but we will have to convert to string it anyway
+        self.scopes = configuration.get("auth", "oauth_scopes")
+
+    @property
+    def auth_control(self) -> str:
+        """
+        :return: login control as html code to insert
+        """
+        return """<a class="nav-link" href="/user-api/v1/login" title="login via OAuth2">login</a>"""
+
+    @staticmethod
+    def get_provider(name: str) -> Type[aioauth_client.OAuth2Client]:
+        """
+        load OAuth2 provider by name
+        :param name: name of the provider. Must be valid class defined in aioauth-client library
+        :return: loaded provider type
+        """
+        provider: Type[aioauth_client.OAuth2Client] = getattr(aioauth_client, name)
+        try:
+            is_oauth2_client = issubclass(provider, aioauth_client.OAuth2Client)
+        except TypeError:  # what if it is random string?
+            is_oauth2_client = False
+        if not is_oauth2_client:
+            raise InvalidOption(name)
+        return provider
+
+    def get_client(self) -> aioauth_client.OAuth2Client:
+        """
+        load client from parameters
+        :return: generated client according to current settings
+        """
+        return self.provider(client_id=self.client_id, client_secret=self.client_secret)
+
+    def get_oauth_url(self) -> str:
+        """
+        get authorization URI for the specified settings
+        :return: authorization URI as a string
+        """
+        client = self.get_client()
+        uri: str = client.get_authorize_url(scope=self.scopes, redirect_uri=self.redirect_uri)
+        return uri
+
+    async def get_oauth_username(self, code: str) -> Optional[str]:
+        """
+        extract OAuth username from remote
+        :param code: authorization code provided by external service
+        :return: username as is in OAuth provider
+        """
+        try:
+            client = self.get_client()
+            access_token, _ = await client.get_access_token(code, redirect_uri=self.redirect_uri)
+            client.access_token = access_token
+
+            print(f"HEEELOOOO {client}")
+            user, _ = await client.user_info()
+            username: str = user.email  # type: ignore
+            return username
+        except Exception:
+            self.logger.exception("got exception while performing request")
+            return None

src/ahriman/core/build_tools/task.py

@@ -41,12 +41,11 @@ class Task:
 
     _check_output = check_output
 
-    def __init__(self, package: Package, architecture: str, config: Configuration, paths: RepositoryPaths) -> None:
+    def __init__(self, package: Package, configuration: Configuration, paths: RepositoryPaths) -> None:
         """
         default constructor
         :param package: package definitions
-        :param architecture: repository architecture
+        :param configuration: configuration instance
-        :param config: configuration instance
         :param paths: repository paths instance
         """
         self.logger = logging.getLogger("builder")
@@ -54,11 +53,10 @@ class Task:
         self.package = package
         self.paths = paths
 
-        section = config.get_section_name("build", architecture)
+        self.archbuild_flags = configuration.getlist("build", "archbuild_flags", fallback=[])
-        self.archbuild_flags = config.getlist(section, "archbuild_flags")
+        self.build_command = configuration.get("build", "build_command")
-        self.build_command = config.get(section, "build_command")
+        self.makepkg_flags = configuration.getlist("build", "makepkg_flags", fallback=[])
-        self.makepkg_flags = config.getlist(section, "makepkg_flags")
+        self.makechrootpkg_flags = configuration.getlist("build", "makechrootpkg_flags", fallback=[])
-        self.makechrootpkg_flags = config.getlist(section, "makechrootpkg_flags")
 
     @property
     def cache_path(self) -> Path:
@@ -97,14 +95,14 @@ class Task:
         run package build
         :return: paths of produced packages
         """
-        cmd = [self.build_command, "-r", str(self.paths.chroot)]
+        command = [self.build_command, "-r", str(self.paths.chroot)]
-        cmd.extend(self.archbuild_flags)
+        command.extend(self.archbuild_flags)
-        cmd.extend(["--"] + self.makechrootpkg_flags)
+        command.extend(["--"] + self.makechrootpkg_flags)
-        cmd.extend(["--"] + self.makepkg_flags)
+        command.extend(["--"] + self.makepkg_flags)
-        self.logger.info(f"using {cmd} for {self.package.base}")
+        self.logger.info("using %s for %s", command, self.package.base)
 
         Task._check_output(
-            *cmd,
+            *command,
             exception=BuildFailed(self.package.base),
             cwd=self.git_path,
             logger=self.build_logger)
@@ -125,4 +123,4 @@ class Task:
         if self.cache_path.is_dir():
             # no need to clone whole repository, just copy from cache first
             shutil.copytree(self.cache_path, git_path)
-        return Task.fetch(git_path, self.package.git_url)
+        return self.fetch(git_path, self.package.git_url)

src/ahriman/core/configuration.py

@@ -24,30 +24,35 @@ import logging
 
 from logging.config import fileConfig
 from pathlib import Path
-from typing import Dict, List, Optional, Type
+from typing import Any, Dict, List, Optional, Type
+
+from ahriman.core.exceptions import InitializeException
 
 
 class Configuration(configparser.RawConfigParser):
     """
     extension for built-in configuration parser
+    :ivar architecture: repository architecture
     :ivar path: path to root configuration file
     :cvar ARCHITECTURE_SPECIFIC_SECTIONS: known sections which can be architecture specific (required by dump)
     :cvar DEFAULT_LOG_FORMAT: default log format (in case of fallback)
     :cvar DEFAULT_LOG_LEVEL: default log level (in case of fallback)
-    :cvar STATIC_SECTIONS: known sections which are not architecture specific (required by dump)
     """
 
     DEFAULT_LOG_FORMAT = "[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s"
     DEFAULT_LOG_LEVEL = logging.DEBUG
 
-    STATIC_SECTIONS = ["alpm", "report", "repository", "settings", "upload"]
     ARCHITECTURE_SPECIFIC_SECTIONS = ["build", "html", "rsync", "s3", "sign", "web"]
 
     def __init__(self) -> None:
         """
         default constructor. In the most cases must not be called directly
         """
-        configparser.RawConfigParser.__init__(self, allow_no_value=True)
+        configparser.RawConfigParser.__init__(self, allow_no_value=True, converters={
+            "list": lambda value: value.split(),
+            "path": self.__convert_path,
+        })
+        self.architecture: Optional[str] = None
         self.path: Optional[Path] = None
 
     @property
@@ -57,71 +62,64 @@ class Configuration(configparser.RawConfigParser):
         """
         return self.getpath("settings", "include")
 
+    @property
+    def logging_path(self) -> Path:
+        """
+        :return: path to logging configuration
+        """
+        return self.getpath("settings", "logging")
+
     @classmethod
-    def from_path(cls: Type[Configuration], path: Path, logfile: bool) -> Configuration:
+    def from_path(cls: Type[Configuration], path: Path, architecture: str, logfile: bool) -> Configuration:
         """
         constructor with full object initialization
         :param path: path to root configuration file
+        :param architecture: repository architecture
         :param logfile: use log file to output messages
         :return: configuration instance
         """
         config = cls()
         config.load(path)
+        config.merge_sections(architecture)
         config.load_logging(logfile)
         return config
 
-    def dump(self, architecture: str) -> Dict[str, Dict[str, str]]:
+    @staticmethod
+    def section_name(section: str, suffix: str) -> str:
+        """
+        generate section name for sections which depends on context
+        :param section: section name
+        :param suffix: session suffix, e.g. repository architecture
+        :return: correct section name for repository specific section
+        """
+        return f"{section}:{suffix}"
+
+    def __convert_path(self, value: str) -> Path:
+        """
+        convert string value to path object
+        :param value: string configuration value
+        :return: path object which represents the configuration value
+        """
+        path = Path(value)
+        if self.path is None or path.is_absolute():
+            return path
+        return self.path.parent / path
+
+    def dump(self) -> Dict[str, Dict[str, str]]:
         """
         dump configuration to dictionary
-        :param architecture: repository architecture
         :return: configuration dump for specific architecture
         """
-        result: Dict[str, Dict[str, str]] = {}
+        return {
-        for section in Configuration.STATIC_SECTIONS:
+            section: dict(self[section])
-            if not self.has_section(section):
+            for section in self.sections()
-                continue
+        }
-            result[section] = dict(self[section])
-        for group in Configuration.ARCHITECTURE_SPECIFIC_SECTIONS:
-            section = self.get_section_name(group, architecture)
-            if not self.has_section(section):
-                continue
-            result[section] = dict(self[section])
 
-        return result
+    # pylint and mypy are too stupid to find these methods
+    # pylint: disable=missing-function-docstring,multiple-statements,unused-argument,no-self-use
+    def getlist(self, *args: Any, **kwargs: Any) -> List[str]: ...
 
-    def getlist(self, section: str, key: str) -> List[str]:
+    def getpath(self, *args: Any, **kwargs: Any) -> Path: ...
-        """
-        get space separated string list option
-        :param section: section name
-        :param key: key name
-        :return: list of string if option is set, empty list otherwise
-        """
-        raw = self.get(section, key, fallback=None)
-        if not raw:  # empty string or none
-            return []
-        return raw.split()
-
-    def getpath(self, section: str, key: str) -> Path:
-        """
-        helper to generate absolute configuration path for relative settings value
-        :param section: section name
-        :param key: key name
-        :return: absolute path according to current path configuration
-        """
-        value = Path(self.get(section, key))
-        if self.path is None or value.is_absolute():
-            return value
-        return self.path.parent / value
-
-    def get_section_name(self, prefix: str, suffix: str) -> str:
-        """
-        check if there is `prefix`_`suffix` section and return it on success. Return `prefix` otherwise
-        :param prefix: section name prefix
-        :param suffix: section name suffix (e.g. architecture name)
-        :return: found section name
-        """
-        probe = f"{prefix}_{suffix}"
-        return probe if self.has_section(probe) else prefix
 
     def load(self, path: Path) -> None:
         """
@@ -138,8 +136,10 @@ class Configuration(configparser.RawConfigParser):
         """
         try:
             for path in sorted(self.include.glob("*.ini")):
+                if path == self.logging_path:
+                    continue  # we don't want to load logging explicitly
                 self.read(path)
-        except (FileNotFoundError, configparser.NoOptionError):
+        except (FileNotFoundError, configparser.NoOptionError, configparser.NoSectionError):
             pass
 
     def load_logging(self, logfile: bool) -> None:
@@ -149,17 +149,58 @@ class Configuration(configparser.RawConfigParser):
         """
         def file_logger() -> None:
             try:
-                config_path = self.getpath("settings", "logging")
+                path = self.logging_path
-                fileConfig(config_path)
+                fileConfig(path)
             except (FileNotFoundError, PermissionError):
                 console_logger()
                 logging.exception("could not create logfile, fallback to stderr")
 
         def console_logger() -> None:
-            logging.basicConfig(filename=None, format=Configuration.DEFAULT_LOG_FORMAT,
+            logging.basicConfig(filename=None, format=self.DEFAULT_LOG_FORMAT,
-                                level=Configuration.DEFAULT_LOG_LEVEL)
+                                level=self.DEFAULT_LOG_LEVEL)
 
         if logfile:
             file_logger()
         else:
             console_logger()
+
+    def merge_sections(self, architecture: str) -> None:
+        """
+        merge architecture specific sections into main configuration
+        :param architecture: repository architecture
+        """
+        self.architecture = architecture
+        for section in self.ARCHITECTURE_SPECIFIC_SECTIONS:
+            # get overrides
+            specific = self.section_name(section, architecture)
+            if self.has_section(specific):
+                # if there is no such section it means that there is no overrides for this arch
+                # but we anyway will have to delete sections for others archs
+                for key, value in self[specific].items():
+                    self.set_option(section, key, value)
+            # remove any arch specific section
+            for foreign in self.sections():
+                # we would like to use lambda filter here, but pylint is too dumb
+                if not foreign.startswith(f"{section}:"):
+                    continue
+                self.remove_section(foreign)
+
+    def reload(self) -> None:
+        """
+        reload configuration if possible or raise exception otherwise
+        """
+        if self.path is None or self.architecture is None:
+            raise InitializeException("Configuration path and/or architecture are not set")
+        self.load(self.path)
+        self.merge_sections(self.architecture)
+
+    def set_option(self, section: str, option: str, value: Optional[str]) -> None:
+        """
+        set option. Unlike default `configparser.RawConfigParser.set` it also creates section if it does not exist
+        :param section: section name
+        :param option: option name
+        :param value: option value as string in parsable format
+        """
+        if not self.has_section(section):
+            self.add_section(section)
+        self.set(section, option, value)

src/ahriman/core/exceptions.py

@@ -20,7 +20,7 @@
 from typing import Any
 
 
-class BuildFailed(Exception):
+class BuildFailed(RuntimeError):
     """
     base exception for failed builds
     """
@@ -30,10 +30,10 @@ class BuildFailed(Exception):
         default constructor
         :param package: package base raised exception
         """
-        Exception.__init__(self, f"Package {package} build failed, check logs for details")
+        RuntimeError.__init__(self, f"Package {package} build failed, check logs for details")
 
 
-class DuplicateRun(Exception):
+class DuplicateRun(RuntimeError):
     """
     exception which will be raised if there is another application instance
     """
@@ -42,22 +42,49 @@ class DuplicateRun(Exception):
         """
         default constructor
         """
-        Exception.__init__(self, "Another application instance is run")
+        RuntimeError.__init__(self, "Another application instance is run")
 
 
-class InitializeException(Exception):
+class DuplicateUser(ValueError):
+    """
+    exception which will be thrown in case if there are two users with different settings
+    """
+
+    def __init__(self, username: str) -> None:
+        """
+        default constructor
+        :param username: username with duplicates
+        """
+        ValueError.__init__(self, f"Found duplicate user with username {username}")
+
+
+class InitializeException(RuntimeError):
     """
     base service initialization exception
     """
 
-    def __init__(self) -> None:
+    def __init__(self, details: str) -> None:
         """
         default constructor
+        :param details: details of the exception
         """
-        Exception.__init__(self, "Could not load service")
+        RuntimeError.__init__(self, f"Could not load service: {details}")
 
 
-class InvalidOption(Exception):
+class InvalidCommand(ValueError):
+    """
+    exception raised on invalid command line options
+    """
+
+    def __init__(self, details: Any) -> None:
+        """
+        default constructor
+        :param details" error details
+        """
+        ValueError.__init__(self, details)
+
+
+class InvalidOption(ValueError):
     """
     exception which will be raised on configuration errors
     """
@@ -67,10 +94,10 @@ class InvalidOption(Exception):
         default constructor
         :param value: option value
         """
-        Exception.__init__(self, f"Invalid or unknown option value `{value}`")
+        ValueError.__init__(self, f"Invalid or unknown option value `{value}`")
 
 
-class InvalidPackageInfo(Exception):
+class InvalidPackageInfo(RuntimeError):
     """
     exception which will be raised on package load errors
     """
@@ -80,10 +107,36 @@ class InvalidPackageInfo(Exception):
         default constructor
         :param details: error details
         """
-        Exception.__init__(self, f"There are errors during reading package information: `{details}`")
+        RuntimeError.__init__(self, f"There are errors during reading package information: `{details}`")
 
 
-class ReportFailed(Exception):
+class MissingArchitecture(ValueError):
+    """
+    exception which will be raised if architecture is required, but missing
+    """
+
+    def __init__(self, command: str) -> None:
+        """
+        default constructor
+        :param command: command name which throws exception
+        """
+        ValueError.__init__(self, f"Architecture required for subcommand {command}, but missing")
+
+
+class MultipleArchitecture(ValueError):
+    """
+    exception which will be raised if multiple architectures are not supported by the handler
+    """
+
+    def __init__(self, command: str) -> None:
+        """
+        default constructor
+        :param command: command name which throws exception
+        """
+        ValueError.__init__(self, f"Multiple architectures are not supported by subcommand {command}")
+
+
+class ReportFailed(RuntimeError):
     """
     report generation exception
     """
@@ -92,10 +145,10 @@ class ReportFailed(Exception):
         """
         default constructor
         """
-        Exception.__init__(self, "Report failed")
+        RuntimeError.__init__(self, "Report failed")
 
 
-class SyncFailed(Exception):
+class SyncFailed(RuntimeError):
     """
     remote synchronization exception
     """
@@ -104,19 +157,19 @@ class SyncFailed(Exception):
         """
         default constructor
         """
-        Exception.__init__(self, "Sync failed")
+        RuntimeError.__init__(self, "Sync failed")
 
 
-class UnknownPackage(Exception):
+class UnknownPackage(ValueError):
     """
     exception for status watcher which will be thrown on unknown package
     """
 
     def __init__(self, base: str) -> None:
-        Exception.__init__(self, f"Package base {base} is unknown")
+        ValueError.__init__(self, f"Package base {base} is unknown")
 
 
-class UnsafeRun(Exception):
+class UnsafeRun(RuntimeError):
     """
     exception which will be raised in case if user is not owner of repository
     """
@@ -125,7 +178,7 @@ class UnsafeRun(Exception):
         """
         default constructor
         """
-        Exception.__init__(
+        RuntimeError.__init__(
             self,
             f"""Current UID {current_uid} differs from root owner {root_uid}.
 Note that for the most actions it is unsafe to run application as different user.

src/ahriman/core/report/email.py

@@ -0,0 +1,111 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import datetime
+import smtplib
+
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from typing import Dict, Iterable
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.report.jinja_template import JinjaTemplate
+from ahriman.core.report.report import Report
+from ahriman.core.util import pretty_datetime
+from ahriman.models.package import Package
+from ahriman.models.smtp_ssl_settings import SmtpSSLSettings
+
+
+class Email(Report, JinjaTemplate):
+    """
+    email report generator
+    :ivar host: SMTP host to connect
+    :ivar no_empty_report: skip empty report generation
+    :ivar password: password to authenticate via SMTP
+    :ivar port: SMTP port to connect
+    :ivar receivers: list of receivers emails
+    :ivar sender: sender email address
+    :ivar ssl: SSL mode for SMTP connection
+    :ivar user: username to authenticate via SMTP
+    """
+
+    def __init__(self, architecture: str, configuration: Configuration) -> None:
+        """
+        default constructor
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        """
+        Report.__init__(self, architecture, configuration)
+        JinjaTemplate.__init__(self, "email", configuration)
+
+        self.full_template_path = configuration.getpath("email", "full_template_path", fallback=None)
+        self.template_path = configuration.getpath("email", "template_path")
+
+        # base smtp settings
+        self.host = configuration.get("email", "host")
+        self.no_empty_report = configuration.getboolean("email", "no_empty_report", fallback=True)
+        self.password = configuration.get("email", "password", fallback=None)
+        self.port = configuration.getint("email", "port")
+        self.receivers = configuration.getlist("email", "receivers")
+        self.sender = configuration.get("email", "sender")
+        self.ssl = SmtpSSLSettings.from_option(configuration.get("email", "ssl", fallback="disabled"))
+        self.user = configuration.get("email", "user", fallback=None)
+
+    def _send(self, text: str, attachment: Dict[str, str]) -> None:
+        """
+        send email callback
+        :param text: email body text
+        :param attachment: map of attachment filename to attachment content
+        """
+        message = MIMEMultipart()
+        message["From"] = self.sender
+        message["To"] = ", ".join(self.receivers)
+        message["Subject"] = f"{self.name} build report at {pretty_datetime(datetime.datetime.utcnow().timestamp())}"
+
+        message.attach(MIMEText(text, "html"))
+        for filename, content in attachment.items():
+            attach = MIMEText(content, "html")
+            attach.add_header("Content-Disposition", "attachment", filename=filename)
+            message.attach(attach)
+
+        if self.ssl != SmtpSSLSettings.SSL:
+            session = smtplib.SMTP(self.host, self.port)
+            if self.ssl == SmtpSSLSettings.STARTTLS:
+                session.starttls()
+        else:
+            session = smtplib.SMTP_SSL(self.host, self.port)
+        if self.user is not None and self.password is not None:
+            session.login(self.user, self.password)
+        session.sendmail(self.sender, self.receivers, message.as_string())
+        session.quit()
+
+    def generate(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None:
+        """
+        generate report for the specified packages
+        :param packages: list of packages to generate report
+        :param built_packages: list of packages which has just been built
+        """
+        if self.no_empty_report and not built_packages:
+            return
+        text = self.make_html(built_packages, self.template_path)
+        if self.full_template_path is not None:
+            attachments = {"index.html": self.make_html(packages, self.full_template_path)}
+        else:
+            attachments = {}
+        self._send(text, attachments)

src/ahriman/core/report/html.py

@@ -17,105 +17,37 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-import jinja2
+from typing import Iterable
-
-from typing import Callable, Dict, Iterable
 
 from ahriman.core.configuration import Configuration
+from ahriman.core.report.jinja_template import JinjaTemplate
 from ahriman.core.report.report import Report
-from ahriman.core.util import pretty_datetime, pretty_size
 from ahriman.models.package import Package
-from ahriman.models.sign_settings import SignSettings
 
 
-class HTML(Report):
+class HTML(Report, JinjaTemplate):
     """
     html report generator
-
-    It uses jinja2 templates for report generation, the following variables are allowed:
-
-        homepage - link to homepage, string, optional
-        link_path - prefix fo packages to download, string, required
-        has_package_signed - True in case if package sign enabled, False otherwise, required
-        has_repo_signed - True in case if repository database sign enabled, False otherwise, required
-        packages - sorted list of packages properties, required
-                   * architecture, string
-                   * archive_size, pretty printed size, string
-                   * build_date, pretty printed datetime, string
-                   * description, string
-                   * filename, string,
-                   * groups, sorted list of strings
-                   * installed_size, pretty printed datetime, string
-                   * licenses, sorted list of strings
-                   * name, string
-                   * url, string
-                   * version, string
-        pgp_key - default PGP key ID, string, optional
-        repository - repository name, string, required
-
-    :ivar homepage: homepage link if any (for footer)
-    :ivar link_path: prefix fo packages to download
-    :ivar name: repository name
-    :ivar pgp_key: default PGP key
     :ivar report_path: output path to html report
-    :ivar sign_targets: targets to sign enabled in configuration
-    :ivar template_path: path to directory with jinja templates
     """
 
-    def __init__(self, architecture: str, config: Configuration) -> None:
+    def __init__(self, architecture: str, configuration: Configuration) -> None:
         """
         default constructor
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
         """
-        Report.__init__(self, architecture, config)
+        Report.__init__(self, architecture, configuration)
-        section = config.get_section_name("html", architecture)
+        JinjaTemplate.__init__(self, "html", configuration)
-        self.report_path = config.getpath(section, "path")
-        self.link_path = config.get(section, "link_path")
-        self.template_path = config.getpath(section, "template_path")
 
-        # base template vars
+        self.report_path = configuration.getpath("html", "path")
-        self.homepage = config.get(section, "homepage", fallback=None)
+        self.template_path = configuration.getpath("html", "template_path")
-        self.name = config.get("repository", "name")
 
-        sign_section = config.get_section_name("sign", architecture)
+    def generate(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None:
-        self.sign_targets = [SignSettings.from_option(opt) for opt in config.getlist(sign_section, "target")]
-        self.pgp_key = config.get(sign_section, "key") if self.sign_targets else None
-
-    def generate(self, packages: Iterable[Package]) -> None:
         """
         generate report for the specified packages
         :param packages: list of packages to generate report
+        :param built_packages: list of packages which has just been built
         """
-        # idea comes from https://stackoverflow.com/a/38642558
+        html = self.make_html(packages, self.template_path)
-        loader = jinja2.FileSystemLoader(searchpath=self.template_path.parent)
-        environment = jinja2.Environment(loader=loader)
-        template = environment.get_template(self.template_path.name)
-
-        content = [
-            {
-                "architecture": properties.architecture or "",
-                "archive_size": pretty_size(properties.archive_size),
-                "build_date": pretty_datetime(properties.build_date),
-                "description": properties.description or "",
-                "filename": properties.filename,
-                "groups": properties.groups,
-                "installed_size": pretty_size(properties.installed_size),
-                "licenses": properties.licenses,
-                "name": package,
-                "url": properties.url or "",
-                "version": base.version
-            } for base in packages for package, properties in base.packages.items()
-        ]
-        comparator: Callable[[Dict[str, str]], str] = lambda item: item["filename"]
-
-        html = template.render(
-            homepage=self.homepage,
-            link_path=self.link_path,
-            has_package_signed=SignSettings.SignPackages in self.sign_targets,
-            has_repo_signed=SignSettings.SignRepository in self.sign_targets,
-            packages=sorted(content, key=comparator),
-            pgp_key=self.pgp_key,
-            repository=self.name)
-
         self.report_path.write_text(html)

src/ahriman/core/report/jinja_template.py

@@ -0,0 +1,115 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import jinja2
+
+from pathlib import Path
+from typing import Callable, Dict, Iterable
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.sign.gpg import GPG
+from ahriman.core.util import pretty_datetime, pretty_size
+from ahriman.models.package import Package
+from ahriman.models.sign_settings import SignSettings
+
+
+class JinjaTemplate:
+    """
+    jinja based report generator
+
+    It uses jinja2 templates for report generation, the following variables are allowed:
+
+        homepage - link to homepage, string, optional
+        link_path - prefix fo packages to download, string, required
+        has_package_signed - True in case if package sign enabled, False otherwise, required
+        has_repo_signed - True in case if repository database sign enabled, False otherwise, required
+        packages - sorted list of packages properties, required
+                   * architecture, string
+                   * archive_size, pretty printed size, string
+                   * build_date, pretty printed datetime, string
+                   * depends, sorted list of strings
+                   * description, string
+                   * filename, string,
+                   * groups, sorted list of strings
+                   * installed_size, pretty printed datetime, string
+                   * licenses, sorted list of strings
+                   * name, string
+                   * url, string
+                   * version, string
+        pgp_key - default PGP key ID, string, optional
+        repository - repository name, string, required
+
+    :ivar homepage: homepage link if any (for footer)
+    :ivar link_path: prefix fo packages to download
+    :ivar name: repository name
+    :ivar default_pgp_key: default PGP key
+    :ivar sign_targets: targets to sign enabled in configuration
+    """
+
+    def __init__(self, section: str, configuration: Configuration) -> None:
+        """
+        default constructor
+        :param section: settings section name
+        :param configuration: configuration instance
+        """
+        self.link_path = configuration.get(section, "link_path")
+
+        # base template vars
+        self.homepage = configuration.get(section, "homepage", fallback=None)
+        self.name = configuration.get("repository", "name")
+
+        self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration)
+
+    def make_html(self, packages: Iterable[Package], template_path: Path) -> str:
+        """
+        generate report for the specified packages
+        :param packages: list of packages to generate report
+        :param template_path: path to jinja template
+        """
+        # idea comes from https://stackoverflow.com/a/38642558
+        loader = jinja2.FileSystemLoader(searchpath=template_path.parent)
+        environment = jinja2.Environment(loader=loader, autoescape=True)
+        template = environment.get_template(template_path.name)
+
+        content = [
+            {
+                "architecture": properties.architecture or "",
+                "archive_size": pretty_size(properties.archive_size),
+                "build_date": pretty_datetime(properties.build_date),
+                "depends": properties.depends,
+                "description": properties.description or "",
+                "filename": properties.filename,
+                "groups": properties.groups,
+                "installed_size": pretty_size(properties.installed_size),
+                "licenses": properties.licenses,
+                "name": package,
+                "url": properties.url or "",
+                "version": base.version
+            } for base in packages for package, properties in base.packages.items()
+        ]
+        comparator: Callable[[Dict[str, str]], str] = lambda item: item["filename"]
+
+        return template.render(
+            homepage=self.homepage,
+            link_path=self.link_path,
+            has_package_signed=SignSettings.Packages in self.sign_targets,
+            has_repo_signed=SignSettings.Repository in self.sign_targets,
+            packages=sorted(content, key=comparator),
+            pgp_key=self.default_pgp_key,
+            repository=self.name)

src/ahriman/core/report/report.py

@@ -17,9 +17,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+from __future__ import annotations
+
 import logging
 
-from typing import Iterable
+from typing import Iterable, Type
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.exceptions import ReportFailed
@@ -31,44 +33,53 @@ class Report:
     """
     base report generator
     :ivar architecture: repository architecture
-    :ivar config: configuration instance
+    :ivar configuration: configuration instance
     :ivar logger: class logger
     """
 
-    def __init__(self, architecture: str, config: Configuration) -> None:
+    def __init__(self, architecture: str, configuration: Configuration) -> None:
         """
         default constructor
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
         """
         self.logger = logging.getLogger("builder")
         self.architecture = architecture
-        self.config = config
+        self.configuration = configuration
 
-    @staticmethod
+    @classmethod
-    def run(architecture: str, config: Configuration, target: str, packages: Iterable[Package]) -> None:
+    def load(cls: Type[Report], architecture: str, configuration: Configuration, target: str) -> Report:
         """
-        run report generation
+        load client from settings
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
         :param target: target to generate report (e.g. html)
-        :param packages: list of packages to generate report
+        :return: client according to current settings
         """
         provider = ReportSettings.from_option(target)
         if provider == ReportSettings.HTML:
             from ahriman.core.report.html import HTML
-            report: Report = HTML(architecture, config)
+            return HTML(architecture, configuration)
-        else:
+        if provider == ReportSettings.Email:
-            report = Report(architecture, config)
+            from ahriman.core.report.email import Email
+            return Email(architecture, configuration)
+        return cls(architecture, configuration)  # should never happen
 
-        try:
+    def generate(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None:
-            report.generate(packages)
-        except Exception:
-            report.logger.exception(f"report generation failed for target {provider.name}")
-            raise ReportFailed()
-
-    def generate(self, packages: Iterable[Package]) -> None:
         """
         generate report for the specified packages
         :param packages: list of packages to generate report
+        :param built_packages: list of packages which has just been built
         """
+
+    def run(self, packages: Iterable[Package], built_packages: Iterable[Package]) -> None:
+        """
+        run report generation
+        :param packages: list of packages to generate report
+        :param built_packages: list of packages which has just been built
+        """
+        try:
+            self.generate(packages, built_packages)
+        except Exception:
+            self.logger.exception("report generation failed")
+            raise ReportFailed()

src/ahriman/core/repository/executor.py

@@ -25,7 +25,7 @@ from typing import Dict, Iterable, List, Optional
 from ahriman.core.build_tools.task import Task
 from ahriman.core.report.report import Report
 from ahriman.core.repository.cleaner import Cleaner
-from ahriman.core.upload.uploader import Uploader
+from ahriman.core.upload.upload import Upload
 from ahriman.models.package import Package
 
 
@@ -49,7 +49,7 @@ class Executor(Cleaner):
         """
         def build_single(package: Package) -> None:
             self.reporter.set_building(package.base)
-            task = Task(package, self.architecture, self.config, self.paths)
+            task = Task(package, self.configuration, self.paths)
             task.init()
             built = task.build()
             for src in built:
@@ -61,7 +61,7 @@ class Executor(Cleaner):
                 build_single(single)
             except Exception:
                 self.reporter.set_failed(single.base)
-                self.logger.exception(f"{single.base} ({self.architecture}) build exception")
+                self.logger.exception("%s (%s) build exception", single.base, self.architecture)
         self.clear_build()
 
         return self.packages_built()
@@ -76,7 +76,7 @@ class Executor(Cleaner):
             try:
                 self.repo.remove(package, fn)
             except Exception:
-                self.logger.exception(f"could not remove {package}")
+                self.logger.exception("could not remove %s", package)
 
         requested = set(packages)
         for local in self.packages():
@@ -94,31 +94,35 @@ class Executor(Cleaner):
                     if package in requested and properties.filename is not None
                 }
             else:
-                to_remove = dict()
+                to_remove = {}
             for package, filename in to_remove.items():
                 remove_single(package, filename)
 
         return self.repo.repo_path
 
-    def process_report(self, targets: Optional[Iterable[str]]) -> None:
+    def process_report(self, targets: Optional[Iterable[str]], built_packages: Iterable[Package]) -> None:
         """
         generate reports
         :param targets: list of targets to generate reports. Configuration option will be used if it is not set
+        :param built_packages: list of packages which has just been built
         """
         if targets is None:
-            targets = self.config.getlist("report", "target")
+            targets = self.configuration.getlist("report", "target")
         for target in targets:
-            Report.run(self.architecture, self.config, target, self.packages())
+            runner = Report.load(self.architecture, self.configuration, target)
+            runner.run(self.packages(), built_packages)
 
-    def process_sync(self, targets: Optional[Iterable[str]]) -> None:
+    def process_sync(self, targets: Optional[Iterable[str]], built_packages: Iterable[Package]) -> None:
         """
         process synchronization to remote servers
         :param targets: list of targets to sync. Configuration option will be used if it is not set
+        :param built_packages: list of packages which has just been built
         """
         if targets is None:
-            targets = self.config.getlist("upload", "target")
+            targets = self.configuration.getlist("upload", "target")
         for target in targets:
-            Uploader.run(self.architecture, self.config, target, self.paths.repository)
+            runner = Upload.load(self.architecture, self.configuration, target)
+            runner.run(self.paths.repository, built_packages)
 
     def process_update(self, packages: Iterable[Path]) -> Path:
         """
@@ -128,7 +132,7 @@ class Executor(Cleaner):
         """
         def update_single(fn: Optional[str], base: str) -> None:
             if fn is None:
-                self.logger.warning(f"received empty package name for base {base}")
+                self.logger.warning("received empty package name for base %s", base)
                 return  # suppress type checking, it never can be none actually
             # in theory it might be NOT packages directory, but we suppose it is
             full_path = self.paths.packages / fn
@@ -146,7 +150,7 @@ class Executor(Cleaner):
                 local = Package.load(filename, self.pacman, self.aur_url)
                 updates.setdefault(local.base, local).packages.update(local.packages)
             except Exception:
-                self.logger.exception(f"could not load package from {filename}")
+                self.logger.exception("could not load package from %s", filename)
 
         for local in updates.values():
             try:
@@ -155,7 +159,7 @@ class Executor(Cleaner):
                 self.reporter.set_success(local)
             except Exception:
                 self.reporter.set_failed(local.base)
-                self.logger.exception(f"could not process {local.base}")
+                self.logger.exception("could not process %s", local.base)
         self.clear_packages()
 
         return self.repo.repo_path

src/ahriman/core/repository/properties.py

@@ -32,7 +32,8 @@ class Properties:
     repository internal objects holder
     :ivar architecture: repository architecture
     :ivar aur_url: base AUR url
-    :ivar config: configuration instance
+    :ivar configuration: configuration instance
+    :ivar ignore_list: package bases which will be ignored during auto updates
     :ivar logger: class logger
     :ivar name: repository name
     :ivar pacman: alpm wrapper instance
@@ -42,18 +43,25 @@ class Properties:
     :ivar sign: GPG wrapper instance
     """
 
-    def __init__(self, architecture: str, config: Configuration) -> None:
+    def __init__(self, architecture: str, configuration: Configuration, no_report: bool) -> None:
+        """
+        default constructor
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
+        """
         self.logger = logging.getLogger("builder")
         self.architecture = architecture
-        self.config = config
+        self.configuration = configuration
 
-        self.aur_url = config.get("alpm", "aur_url")
+        self.aur_url = configuration.get("alpm", "aur_url")
-        self.name = config.get("repository", "name")
+        self.name = configuration.get("repository", "name")
 
-        self.paths = RepositoryPaths(config.getpath("repository", "root"), architecture)
+        self.paths = RepositoryPaths(configuration.getpath("repository", "root"), architecture)
         self.paths.create_tree()
 
-        self.pacman = Pacman(config)
+        self.ignore_list = configuration.getlist("build", "ignore_packages", fallback=[])
-        self.sign = GPG(architecture, config)
+        self.pacman = Pacman(configuration)
+        self.sign = GPG(architecture, configuration)
         self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
-        self.reporter = Client.load(architecture, config)
+        self.reporter = Client() if no_report else Client.load(configuration)

src/ahriman/core/repository/repository.py

@@ -37,14 +37,12 @@ class Repository(Executor, UpdateHandler):
         :return: list of packages properties
         """
         result: Dict[str, Package] = {}
-        for full_path in self.paths.repository.iterdir():
+        for full_path in filter(package_like, self.paths.repository.iterdir()):
-            if not package_like(full_path):
-                continue
             try:
                 local = Package.load(full_path, self.pacman, self.aur_url)
                 result.setdefault(local.base, local).packages.update(local.packages)
             except Exception:
-                self.logger.exception(f"could not load package from {full_path}")
+                self.logger.exception("could not load package from %s", full_path)
                 continue
         return list(result.values())
 
@@ -53,4 +51,4 @@ class Repository(Executor, UpdateHandler):
         get list of files in built packages directory
         :return: list of filenames from the directory
         """
-        return list(self.paths.packages.iterdir())
+        return list(filter(package_like, self.paths.packages.iterdir()))

src/ahriman/core/repository/update_handler.py

@@ -44,11 +44,8 @@ class UpdateHandler(Cleaner):
         """
         result: List[Package] = []
 
-        build_section = self.config.get_section_name("build", self.architecture)
-        ignore_list = self.config.getlist(build_section, "ignore_packages")
-
         for local in self.packages():
-            if local.base in ignore_list:
+            if local.base in self.ignore_list:
                 continue
             if local.is_vcs and no_vcs:
                 continue
@@ -62,7 +59,7 @@ class UpdateHandler(Cleaner):
                     result.append(remote)
             except Exception:
                 self.reporter.set_failed(local.base)
-                self.logger.exception(f"could not load remote package {local.base}")
+                self.logger.exception("could not load remote package %s", local.base)
                 continue
 
         return result
@@ -84,7 +81,7 @@ class UpdateHandler(Cleaner):
                 else:
                     self.reporter.set_pending(local.base)
             except Exception:
-                self.logger.exception(f"could not add package from {fn}")
+                self.logger.exception("could not add package from %s", fn)
         self.clear_manual()
 
         return result

src/ahriman/core/sign/gpg.py

@@ -18,13 +18,14 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import logging
+import requests
 
 from pathlib import Path
-from typing import List
+from typing import List, Optional, Set, Tuple
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.exceptions import BuildFailed
-from ahriman.core.util import check_output
+from ahriman.core.util import check_output, exception_response_text
 from ahriman.models.sign_settings import SignSettings
 
 
@@ -32,37 +33,39 @@ class GPG:
     """
     gnupg wrapper
     :ivar architecture: repository architecture
-    :ivar config: configuration instance
+    :ivar configuration: configuration instance
     :ivar default_key: default PGP key ID to use
     :ivar logger: class logger
-    :ivar target: list of targets to sign (repository, package etc)
+    :ivar targets: list of targets to sign (repository, package etc)
     """
 
     _check_output = check_output
 
-    def __init__(self, architecture: str, config: Configuration) -> None:
+    def __init__(self, architecture: str, configuration: Configuration) -> None:
         """
         default constructor
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
         """
         self.logger = logging.getLogger("build_details")
-        self.config = config
+        self.architecture = architecture
-        self.section = config.get_section_name("sign", architecture)
+        self.configuration = configuration
-        self.target = {SignSettings.from_option(opt) for opt in config.getlist(self.section, "target")}
+        self.targets, self.default_key = self.sign_options(configuration)
-        self.default_key = config.get(self.section, "key") if self.target else ""
 
     @property
     def repository_sign_args(self) -> List[str]:
         """
         :return: command line arguments for repo-add command to sign database
         """
-        if SignSettings.SignRepository not in self.target:
+        if SignSettings.Repository not in self.targets:
+            return []
+        if self.default_key is None:
+            self.logger.error("no default key set, skip repository sign")
             return []
         return ["--sign", "--key", self.default_key]
 
     @staticmethod
-    def sign_cmd(path: Path, key: str) -> List[str]:
+    def sign_command(path: Path, key: str) -> List[str]:
         """
         gpg command to run
         :param path: path to file to sign
@@ -71,6 +74,50 @@ class GPG:
         """
         return ["gpg", "-u", key, "-b", str(path)]
 
+    @staticmethod
+    def sign_options(configuration: Configuration) -> Tuple[Set[SignSettings], Optional[str]]:
+        """
+        extract default sign options from configuration
+        :param configuration: configuration instance
+        :return: tuple of sign targets and default PGP key
+        """
+        targets = {
+            SignSettings.from_option(option)
+            for option in configuration.getlist("sign", "target")
+        }
+        default_key = configuration.get("sign", "key") if targets else None
+        return targets, default_key
+
+    def download_key(self, server: str, key: str) -> str:
+        """
+        download key from public PGP server
+        :param server: public PGP server which will be used to download the key
+        :param key: key ID to download
+        :return: key as plain text
+        """
+        key = key if key.startswith("0x") else f"0x{key}"
+        try:
+            response = requests.get(f"http://{server}/pks/lookup", params={
+                "op": "get",
+                "options": "mr",
+                "search": key
+            })
+            response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception("could not download key %s from %s: %s", key, server, exception_response_text(e))
+            raise
+        return response.text
+
+    def import_key(self, server: str, key: str) -> None:
+        """
+        import key to current user and sign it locally
+        :param server: public PGP server which will be used to download the key
+        :param key: key ID to import
+        """
+        key_body = self.download_key(server, key)
+        GPG._check_output("gpg", "--import", input_data=key_body, exception=None, logger=self.logger)
+        GPG._check_output("gpg", "--quick-lsign-key", key, exception=None, logger=self.logger)
+
     def process(self, path: Path, key: str) -> List[Path]:
         """
         gpg command wrapper
@@ -79,7 +126,7 @@ class GPG:
         :return: list of generated files including original file
         """
         GPG._check_output(
-            *GPG.sign_cmd(path, key),
+            *GPG.sign_command(path, key),
             exception=BuildFailed(path.name),
             logger=self.logger)
         return [path, path.parent / f"{path.name}.sig"]
@@ -91,9 +138,12 @@ class GPG:
         :param base: package base required to check for key overrides
         :return: list of generated files including original file
         """
-        if SignSettings.SignPackages not in self.target:
+        if SignSettings.Packages not in self.targets:
+            return [path]
+        key = self.configuration.get("sign", f"key_{base}", fallback=self.default_key)
+        if key is None:
+            self.logger.error("no default key set, skip package %s sign", path)
             return [path]
-        key = self.config.get(self.section, f"key_{base}", fallback=self.default_key)
         return self.process(path, key)
 
     def sign_repository(self, path: Path) -> List[Path]:
@@ -103,6 +153,9 @@ class GPG:
         :param path: path to repository database
         :return: list of generated files including original file
         """
-        if SignSettings.SignRepository not in self.target:
+        if SignSettings.Repository not in self.targets:
+            return [path]
+        if self.default_key is None:
+            self.logger.error("no default key set, skip repository sign")
             return [path]
         return self.process(path, self.default_key)

src/ahriman/core/spawn.py

@@ -0,0 +1,137 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+import argparse
+import logging
+import uuid
+
+from multiprocessing import Process, Queue
+from threading import Lock, Thread
+from typing import Callable, Dict, Iterable, Tuple
+
+from ahriman.core.configuration import Configuration
+
+
+class Spawn(Thread):
+    """
+    helper to spawn external ahriman process
+    MUST NOT be used directly, the only one usage allowed is to spawn process from web services
+    :ivar active: map of active child processes required to avoid zombies
+    :ivar architecture: repository architecture
+    :ivar configuration: configuration instance
+    :ivar logger: spawner logger
+    :ivar queue: multiprocessing queue to read updates from processes
+    """
+
+    def __init__(self, args_parser: argparse.ArgumentParser, architecture: str, configuration: Configuration) -> None:
+        """
+        default constructor
+        :param args_parser: command line parser for the application
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        """
+        Thread.__init__(self, name="spawn")
+        self.architecture = architecture
+        self.args_parser = args_parser
+        self.configuration = configuration
+        self.logger = logging.getLogger("http")
+
+        self.lock = Lock()
+        self.active: Dict[str, Process] = {}
+        # stupid pylint does not know that it is possible
+        self.queue: Queue[Tuple[str, bool]] = Queue()  # pylint: disable=unsubscriptable-object
+
+    @staticmethod
+    def process(callback: Callable[[argparse.Namespace, str], bool], args: argparse.Namespace, architecture: str,
+                process_id: str, queue: Queue[Tuple[str, bool]]) -> None:  # pylint: disable=unsubscriptable-object
+        """
+        helper to run external process
+        :param callback: application run function (i.e. Handler.run method)
+        :param args: command line arguments
+        :param architecture: repository architecture
+        :param process_id: process unique identifier
+        :param queue: output queue
+        """
+        result = callback(args, architecture)
+        queue.put((process_id, result))
+
+    def packages_add(self, packages: Iterable[str], now: bool) -> None:
+        """
+        add packages
+        :param packages: packages list to add
+        :param now: build packages now
+        """
+        kwargs = {"now": ""} if now else {}
+        self.spawn_process("add", *packages, **kwargs)
+
+    def packages_remove(self, packages: Iterable[str]) -> None:
+        """
+        remove packages
+        :param packages: packages list to remove
+        """
+        self.spawn_process("remove", *packages)
+
+    def spawn_process(self, command: str, *args: str, **kwargs: str) -> None:
+        """
+        spawn external ahriman process with supplied arguments
+        :param command: subcommand to run
+        :param args: positional command arguments
+        :param kwargs: named command arguments
+        """
+        # default arguments
+        arguments = ["--architecture", self.architecture]
+        if self.configuration.path is not None:
+            arguments.extend(["--configuration", str(self.configuration.path)])
+        # positional command arguments
+        arguments.append(command)
+        arguments.extend(args)
+        # named command arguments
+        for argument, value in kwargs.items():
+            arguments.append(f"--{argument}")
+            if value:
+                arguments.append(value)
+
+        process_id = str(uuid.uuid4())
+        self.logger.info("full command line arguments of %s are %s", process_id, arguments)
+        parsed = self.args_parser.parse_args(arguments)
+
+        callback = parsed.handler.call
+        process = Process(target=self.process,
+                          args=(callback, parsed, self.architecture, process_id, self.queue),
+                          daemon=True)
+        process.start()
+
+        with self.lock:
+            self.active[process_id] = process
+
+    def run(self) -> None:
+        """
+        thread run method
+        """
+        for process_id, status in iter(self.queue.get, None):
+            self.logger.info("process %s has been terminated with status %s", process_id, status)
+
+            with self.lock:
+                process = self.active.pop(process_id, None)
+
+            if process is not None:
+                process.terminate()  # make sure lol
+                process.join()

src/ahriman/core/status/client.py

@@ -19,10 +19,11 @@
 #
 from __future__ import annotations
 
-from typing import List, Optional, Tuple
+from typing import List, Optional, Tuple, Type
 
 from ahriman.core.configuration import Configuration
 from ahriman.models.build_status import BuildStatus, BuildStatusEnum
+from ahriman.models.internal_status import InternalStatus
 from ahriman.models.package import Package
 
 
@@ -31,6 +32,21 @@ class Client:
     base build status reporter client
     """
 
+    @classmethod
+    def load(cls: Type[Client], configuration: Configuration) -> Client:
+        """
+        load client from settings
+        :param configuration: configuration instance
+        :return: client according to current settings
+        """
+        address = configuration.get("web", "address", fallback=None)
+        host = configuration.get("web", "host", fallback=None)
+        port = configuration.getint("web", "port", fallback=None)
+        if address or (host and port):
+            from ahriman.core.status.web_client import WebClient
+            return WebClient(configuration)
+        return cls()
+
     def add(self, package: Package, status: BuildStatusEnum) -> None:
         """
         add new package with status
@@ -38,8 +54,7 @@ class Client:
         :param status: current package build status
         """
 
-    # pylint: disable=no-self-use
+    def get(self, base: Optional[str]) -> List[Tuple[Package, BuildStatus]]:  # pylint: disable=no-self-use
-    def get(self, base: Optional[str]) -> List[Tuple[Package, BuildStatus]]:
         """
         get package status
         :param base: package base to get
@@ -48,14 +63,25 @@ class Client:
         del base
         return []
 
-    # pylint: disable=no-self-use
+    def get_internal(self) -> InternalStatus:  # pylint: disable=no-self-use
-    def get_self(self) -> BuildStatus:
+        """
+        get internal service status
+        :return: current internal (web) service status
+        """
+        return InternalStatus()
+
+    def get_self(self) -> BuildStatus:  # pylint: disable=no-self-use
         """
         get ahriman status itself
         :return: current ahriman status
         """
         return BuildStatus()
 
+    def reload_auth(self) -> None:
+        """
+        reload authentication module call
+        """
+
     def remove(self, base: str) -> None:
         """
         remove packages from watcher
@@ -109,20 +135,3 @@ class Client:
         :param package: current package properties
         """
         return self.add(package, BuildStatusEnum.Unknown)
-
-    @staticmethod
-    def load(architecture: str, config: Configuration) -> Client:
-        """
-        load client from settings
-        :param architecture: repository architecture
-        :param config: configuration instance
-        :return: client according to current settings
-        """
-        section = config.get_section_name("web", architecture)
-        host = config.get(section, "host", fallback=None)
-        port = config.getint(section, "port", fallback=None)
-        if host is None or port is None:
-            return Client()
-
-        from ahriman.core.status.web_client import WebClient
-        return WebClient(host, port)

src/ahriman/core/status/watcher.py

@@ -40,16 +40,16 @@ class Watcher:
     :ivar status: daemon status
     """
 
-    def __init__(self, architecture: str, config: Configuration) -> None:
+    def __init__(self, architecture: str, configuration: Configuration) -> None:
         """
         default constructor
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
         """
         self.logger = logging.getLogger("http")
 
         self.architecture = architecture
-        self.repository = Repository(architecture, config)
+        self.repository = Repository(architecture, configuration, no_report=True)
 
         self.known: Dict[str, Tuple[Package, BuildStatus]] = {}
         self.status = BuildStatus()
@@ -90,7 +90,7 @@ class Watcher:
             try:
                 parse_single(item)
             except Exception:
-                self.logger.exception(f"cannot parse item f{item} to package")
+                self.logger.exception("cannot parse item %s to package", item)
 
     def _cache_save(self) -> None:
         """

src/ahriman/core/status/web_client.py

@@ -22,35 +22,99 @@ import requests
 
 from typing import List, Optional, Tuple
 
+from ahriman.core.configuration import Configuration
 from ahriman.core.status.client import Client
+from ahriman.core.util import exception_response_text
 from ahriman.models.build_status import BuildStatusEnum, BuildStatus
+from ahriman.models.internal_status import InternalStatus
 from ahriman.models.package import Package
+from ahriman.models.user import User
 
 
 class WebClient(Client):
     """
     build status reporter web client
-    :ivar host: host of web service
+    :ivar address: address of the web service
     :ivar logger: class logger
-    :ivar port: port of web service
+    :ivar user: web service user descriptor
     """
 
-    def __init__(self, host: str, port: int) -> None:
+    def __init__(self, configuration: Configuration) -> None:
         """
         default constructor
-        :param host: host of web service
+        :param configuration: configuration instance
-        :param port: port of web service
         """
         self.logger = logging.getLogger("http")
-        self.host = host
+        self.address = self.parse_address(configuration)
-        self.port = port
+        self.user = User.from_option(
+            configuration.get("web", "username", fallback=None),
+            configuration.get("web", "password", fallback=None))
 
+        self.__session = requests.session()
+        self._login()
+
+    @property
     def _ahriman_url(self) -> str:
         """
-        url generator
         :return: full url for web service for ahriman service itself
         """
-        return f"http://{self.host}:{self.port}/api/v1/ahriman"
+        return f"{self.address}/status-api/v1/ahriman"
+
+    @property
+    def _login_url(self) -> str:
+        """
+        :return: full url for web service to login
+        """
+        return f"{self.address}/user-api/v1/login"
+
+    @property
+    def _reload_auth_url(self) -> str:
+        """
+        :return: full url for web service to reload authentication module
+        """
+        return f"{self.address}/service-api/v1/reload-auth"
+
+    @property
+    def _status_url(self) -> str:
+        """
+        :return: full url for web service for status
+        """
+        return f"{self.address}/status-api/v1/status"
+
+    @staticmethod
+    def parse_address(configuration: Configuration) -> str:
+        """
+        parse address from configuration
+        :param configuration: configuration instance
+        :return: valid http address
+        """
+        address = configuration.get("web", "address", fallback=None)
+        if not address:
+            # build address from host and port directly
+            host = configuration.get("web", "host")
+            port = configuration.getint("web", "port")
+            address = f"http://{host}:{port}"
+        return address
+
+    def _login(self) -> None:
+        """
+        process login to the service
+        """
+        if self.user is None:
+            return  # no auth configured
+
+        payload = {
+            "username": self.user.username,
+            "password": self.user.password
+        }
+
+        try:
+            response = self.__session.post(self._login_url, json=payload)
+            response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception("could not login as %s: %s", self.user, exception_response_text(e))
+        except Exception:
+            self.logger.exception("could not login as %s", self.user)
 
     def _package_url(self, base: str = "") -> str:
         """
@@ -58,7 +122,7 @@ class WebClient(Client):
         :param base: package base to generate url
         :return: full url of web service for specific package base
         """
-        return f"http://{self.host}:{self.port}/api/v1/packages/{base}"
+        return f"{self.address}/status-api/v1/packages/{base}"
 
     def add(self, package: Package, status: BuildStatusEnum) -> None:
         """
@@ -72,12 +136,12 @@ class WebClient(Client):
         }
 
         try:
-            response = requests.post(self._package_url(package.base), json=payload)
+            response = self.__session.post(self._package_url(package.base), json=payload)
             response.raise_for_status()
         except requests.exceptions.HTTPError as e:
-            self.logger.exception(f"could not add {package.base}: {e.response.text}")
+            self.logger.exception("could not add %s: %s", package.base, exception_response_text(e))
         except Exception:
-            self.logger.exception(f"could not add {package.base}")
+            self.logger.exception("could not add %s", package.base)
 
     def get(self, base: Optional[str]) -> List[Tuple[Package, BuildStatus]]:
         """
@@ -86,7 +150,7 @@ class WebClient(Client):
         :return: list of current package description and status if it has been found
         """
         try:
-            response = requests.get(self._package_url(base or ""))
+            response = self.__session.get(self._package_url(base or ""))
             response.raise_for_status()
 
             status_json = response.json()
@@ -95,40 +159,69 @@ class WebClient(Client):
                 for package in status_json
             ]
         except requests.exceptions.HTTPError as e:
-            self.logger.exception(f"could not get {base}: {e.response.text}")
+            self.logger.exception("could not get %s: %s", base, exception_response_text(e))
         except Exception:
-            self.logger.exception(f"could not get {base}")
+            self.logger.exception("could not get %s", base)
         return []
 
+    def get_internal(self) -> InternalStatus:
+        """
+        get internal service status
+        :return: current internal (web) service status
+        """
+        try:
+            response = self.__session.get(self._status_url)
+            response.raise_for_status()
+
+            status_json = response.json()
+            return InternalStatus.from_json(status_json)
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception("could not get web service status: %s", exception_response_text(e))
+        except Exception:
+            self.logger.exception("could not get web service status")
+        return InternalStatus()
+
     def get_self(self) -> BuildStatus:
         """
         get ahriman status itself
         :return: current ahriman status
         """
         try:
-            response = requests.get(self._ahriman_url())
+            response = self.__session.get(self._ahriman_url)
             response.raise_for_status()
 
             status_json = response.json()
             return BuildStatus.from_json(status_json)
         except requests.exceptions.HTTPError as e:
-            self.logger.exception(f"could not get service status: {e.response.text}")
+            self.logger.exception("could not get service status: %s", exception_response_text(e))
         except Exception:
             self.logger.exception("could not get service status")
         return BuildStatus()
 
+    def reload_auth(self) -> None:
+        """
+        reload authentication module call
+        """
+        try:
+            response = self.__session.post(self._reload_auth_url)
+            response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            self.logger.exception("could not reload auth module: %s", exception_response_text(e))
+        except Exception:
+            self.logger.exception("could not reload auth module")
+
     def remove(self, base: str) -> None:
         """
         remove packages from watcher
         :param base: basename to remove
         """
         try:
-            response = requests.delete(self._package_url(base))
+            response = self.__session.delete(self._package_url(base))
             response.raise_for_status()
         except requests.exceptions.HTTPError as e:
-            self.logger.exception(f"could not delete {base}: {e.response.text}")
+            self.logger.exception("could not delete %s: %s", base, exception_response_text(e))
         except Exception:
-            self.logger.exception(f"could not delete {base}")
+            self.logger.exception("could not delete %s", base)
 
     def update(self, base: str, status: BuildStatusEnum) -> None:
         """
@@ -139,12 +232,12 @@ class WebClient(Client):
         payload = {"status": status.value}
 
         try:
-            response = requests.post(self._package_url(base), json=payload)
+            response = self.__session.post(self._package_url(base), json=payload)
             response.raise_for_status()
         except requests.exceptions.HTTPError as e:
-            self.logger.exception(f"could not update {base}: {e.response.text}")
+            self.logger.exception("could not update %s: %s", base, exception_response_text(e))
         except Exception:
-            self.logger.exception(f"could not update {base}")
+            self.logger.exception("could not update %s", base)
 
     def update_self(self, status: BuildStatusEnum) -> None:
         """
@@ -154,9 +247,9 @@ class WebClient(Client):
         payload = {"status": status.value}
 
         try:
-            response = requests.post(self._ahriman_url(), json=payload)
+            response = self.__session.post(self._ahriman_url, json=payload)
             response.raise_for_status()
         except requests.exceptions.HTTPError as e:
-            self.logger.exception(f"could not update service status: {e.response.text}")
+            self.logger.exception("could not update service status: %s", exception_response_text(e))
         except Exception:
             self.logger.exception("could not update service status")

src/ahriman/core/upload/rsync.py

@@ -18,43 +18,37 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from pathlib import Path
+from typing import Iterable
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.upload.uploader import Uploader
+from ahriman.core.upload.upload import Upload
 from ahriman.core.util import check_output
+from ahriman.models.package import Package
 
 
-class Rsync(Uploader):
+class Rsync(Upload):
     """
     rsync wrapper
+    :ivar command: command arguments for sync
     :ivar remote: remote address to sync
     """
 
     _check_output = check_output
 
-    def __init__(self, architecture: str, config: Configuration) -> None:
+    def __init__(self, architecture: str, configuration: Configuration) -> None:
         """
         default constructor
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
         """
-        Uploader.__init__(self, architecture, config)
+        Upload.__init__(self, architecture, configuration)
-        section = config.get_section_name("rsync", architecture)
+        self.command = configuration.getlist("rsync", "command")
-        self.remote = config.get(section, "remote")
+        self.remote = configuration.get("rsync", "remote")
 
-    def sync(self, path: Path) -> None:
+    def sync(self, path: Path, built_packages: Iterable[Package]) -> None:
         """
         sync data to remote server
         :param path: local path to sync
+        :param built_packages: list of packages which has just been built
         """
-        Rsync._check_output(
+        Rsync._check_output(*self.command, str(path), self.remote, exception=None, logger=self.logger)
-            "rsync",
-            "--archive",
-            "--verbose",
-            "--compress",
-            "--partial",
-            "--delete",
-            str(path),
-            self.remote,
-            exception=None,
-            logger=self.logger)

src/ahriman/core/upload/s3.py

@@ -17,37 +17,137 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import boto3  # type: ignore
+import hashlib
+import mimetypes
+
 from pathlib import Path
+from typing import Any, Dict, Generator, Iterable
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.upload.uploader import Uploader
+from ahriman.core.upload.upload import Upload
-from ahriman.core.util import check_output
+from ahriman.models.package import Package
 
 
-class S3(Uploader):
+class S3(Upload):
     """
     aws-cli wrapper
-    :ivar bucket: full bucket name
+    :ivar bucket: boto3 S3 bucket object
+    :ivar chunk_size: chunk size for calculating checksums
     """
 
-    _check_output = check_output
+    def __init__(self, architecture: str, configuration: Configuration) -> None:
-
-    def __init__(self, architecture: str, config: Configuration) -> None:
         """
         default constructor
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
         """
-        Uploader.__init__(self, architecture, config)
+        Upload.__init__(self, architecture, configuration)
-        section = config.get_section_name("s3", architecture)
+        self.bucket = self.get_bucket(configuration)
-        self.bucket = config.get(section, "bucket")
+        self.chunk_size = configuration.getint("s3", "chunk_size", fallback=8 * 1024 * 1024)
 
-    def sync(self, path: Path) -> None:
+    @staticmethod
+    def calculate_etag(path: Path, chunk_size: int) -> str:
+        """
+        calculate amazon s3 etag
+        credits to https://teppen.io/2018/10/23/aws_s3_verify_etags/
+        For this method we have to define nosec because it is out of any security context and provided by AWS
+        :param path: path to local file
+        :param chunk_size: read chunk size, which depends on client settings
+        :return: calculated entity tag for local file
+        """
+        md5s = []
+        with path.open("rb") as local_file:
+            for chunk in iter(lambda: local_file.read(chunk_size), b""):
+                md5s.append(hashlib.md5(chunk))  # nosec
+
+        # in case if there is only one chunk it must be just this checksum
+        # and checksum of joined digest otherwise (including empty list)
+        checksum = md5s[0] if len(md5s) == 1 else hashlib.md5(b"".join(md5.digest() for md5 in md5s))  # nosec
+        # in case if there are more than one chunk it should be appended with amount of chunks
+        suffix = f"-{len(md5s)}" if len(md5s) > 1 else ""
+        return f"{checksum.hexdigest()}{suffix}"
+
+    @staticmethod
+    def get_bucket(configuration: Configuration) -> Any:
+        """
+        create resource client from configuration
+        :param configuration: configuration instance
+        :return: amazon client
+        """
+        client = boto3.resource(service_name="s3",
+                                region_name=configuration.get("s3", "region"),
+                                aws_access_key_id=configuration.get("s3", "access_key"),
+                                aws_secret_access_key=configuration.get("s3", "secret_key"))
+        return client.Bucket(configuration.get("s3", "bucket"))
+
+    @staticmethod
+    def remove_files(local_files: Dict[Path, str], remote_objects: Dict[Path, Any]) -> None:
+        """
+        remove files which have been removed locally
+        :param local_files: map of local path object to its checksum
+        :param remote_objects: map of remote path object to the remote s3 object
+        """
+        for local_file, remote_object in remote_objects.items():
+            if local_file in local_files:
+                continue
+            remote_object.delete()
+
+    def get_local_files(self, path: Path) -> Dict[Path, str]:
+        """
+        get all local files and their calculated checksums
+        :param path: local path to sync
+        :return: map of path object to its checksum
+        """
+        # credits to https://stackoverflow.com/a/64915960
+        def walk(directory_path: Path) -> Generator[Path, None, None]:
+            for element in directory_path.iterdir():
+                if element.is_dir():
+                    yield from walk(element)
+                    continue
+                yield element
+        return {
+            local_file.relative_to(path): self.calculate_etag(local_file, self.chunk_size)
+            for local_file in walk(path)
+        }
+
+    def get_remote_objects(self) -> Dict[Path, Any]:
+        """
+        get all remote objects and their checksums
+        :return: map of path object to the remote s3 object
+        """
+        objects = self.bucket.objects.filter(Prefix=self.architecture)
+        return {Path(item.key).relative_to(self.architecture): item for item in objects}
+
+    def sync(self, path: Path, built_packages: Iterable[Package]) -> None:
         """
         sync data to remote server
         :param path: local path to sync
+        :param built_packages: list of packages which has just been built
         """
-        # TODO rewrite to boto, but it is bullshit
+        remote_objects = self.get_remote_objects()
-        S3._check_output("aws", "s3", "sync", "--quiet", "--delete", str(path), self.bucket,
+        local_files = self.get_local_files(path)
-                         exception=None,
+
-                         logger=self.logger)
+        self.upload_files(path, local_files, remote_objects)
+        self.remove_files(local_files, remote_objects)
+
+    def upload_files(self, path: Path, local_files: Dict[Path, str], remote_objects: Dict[Path, Any]) -> None:
+        """
+        upload changed files to s3
+        :param path: local path to sync
+        :param local_files: map of local path object to its checksum
+        :param remote_objects: map of remote path object to the remote s3 object
+        """
+        for local_file, checksum in local_files.items():
+            remote_object = remote_objects.get(local_file)
+            # 0 and -1 elements are " (double quote)
+            remote_checksum = remote_object.e_tag[1:-1] if remote_object is not None else None
+            if remote_checksum == checksum:
+                continue
+
+            local_path = path / local_file
+            remote_path = Path(self.architecture) / local_file
+            (mime, _) = mimetypes.guess_type(local_path)
+            extra_args = {"ContentType": mime} if mime is not None else None
+
+            self.bucket.upload_file(Filename=str(local_path), Key=str(remote_path), ExtraArgs=extra_args)

src/ahriman/core/upload/upload.py

@@ -17,60 +17,70 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+from __future__ import annotations
+
 import logging
 
 from pathlib import Path
+from typing import Iterable, Type
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.exceptions import SyncFailed
+from ahriman.models.package import Package
 from ahriman.models.upload_settings import UploadSettings
 
 
-class Uploader:
+class Upload:
     """
     base remote sync class
     :ivar architecture: repository architecture
-    :ivar config: configuration instance
+    :ivar configuration: configuration instance
     :ivar logger: application logger
     """
 
-    def __init__(self, architecture: str, config: Configuration) -> None:
+    def __init__(self, architecture: str, configuration: Configuration) -> None:
         """
         default constructor
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
         """
         self.logger = logging.getLogger("builder")
         self.architecture = architecture
-        self.config = config
+        self.config = configuration
 
-    @staticmethod
+    @classmethod
-    def run(architecture: str, config: Configuration, target: str, path: Path) -> None:
+    def load(cls: Type[Upload], architecture: str, configuration: Configuration, target: str) -> Upload:
         """
-        run remote sync
+        load client from settings
         :param architecture: repository architecture
-        :param config: configuration instance
+        :param configuration: configuration instance
         :param target: target to run sync (e.g. s3)
-        :param path: local path to sync
+        :return: client according to current settings
         """
         provider = UploadSettings.from_option(target)
         if provider == UploadSettings.Rsync:
             from ahriman.core.upload.rsync import Rsync
-            uploader: Uploader = Rsync(architecture, config)
+            return Rsync(architecture, configuration)
-        elif provider == UploadSettings.S3:
+        if provider == UploadSettings.S3:
             from ahriman.core.upload.s3 import S3
-            uploader = S3(architecture, config)
+            return S3(architecture, configuration)
-        else:
+        return cls(architecture, configuration)  # should never happen
-            uploader = Uploader(architecture, config)
 
+    def run(self, path: Path, built_packages: Iterable[Package]) -> None:
+        """
+        run remote sync
+        :param path: local path to sync
+        :param built_packages: list of packages which has just been built
+        """
         try:
-            uploader.sync(path)
+            self.sync(path, built_packages)
         except Exception:
-            uploader.logger.exception(f"remote sync failed for {provider.name}")
+            self.logger.exception("remote sync failed")
             raise SyncFailed()
 
-    def sync(self, path: Path) -> None:
+    def sync(self, path: Path, built_packages: Iterable[Package]) -> None:
         """
         sync data to remote server
         :param path: local path to sync
+        :param built_packages: list of packages which has just been built
         """

src/ahriman/core/util.py

@@ -19,34 +19,48 @@
 #
 import datetime
 import subprocess
+import requests
 
 from logging import Logger
 from pathlib import Path
-from typing import Optional
+from typing import Optional, Union
 
 from ahriman.core.exceptions import InvalidOption
 
 
-def check_output(*args: str, exception: Optional[Exception],
+def check_output(*args: str, exception: Optional[Exception], cwd: Optional[Path] = None,
-                 cwd: Optional[Path] = None, logger: Optional[Logger] = None) -> str:
+                 input_data: Optional[str] = None, logger: Optional[Logger] = None) -> str:
     """
     subprocess wrapper
     :param args: command line arguments
     :param exception: exception which has to be reraised instead of default subprocess exception
     :param cwd: current working directory
+    :param input_data: data which will be written to command stdin
     :param logger: logger to log command result if required
     :return: command output
     """
     try:
-        result = subprocess.check_output(args, cwd=cwd, stderr=subprocess.STDOUT).decode("utf8").strip()
+        # universal_newlines is required to read input from string
+        result: str = subprocess.check_output(args, cwd=cwd, input=input_data, stderr=subprocess.STDOUT,
+                                              universal_newlines=True).strip()
         if logger is not None:
             for line in result.splitlines():
                 logger.debug(line)
+        return result
     except subprocess.CalledProcessError as e:
         if e.output is not None and logger is not None:
-            for line in e.output.decode("utf8").splitlines():
+            for line in e.output.splitlines():
                 logger.debug(line)
         raise exception or e
+
+
+def exception_response_text(exception: requests.exceptions.HTTPError) -> str:
+    """
+    safe response exception text generation
+    :param exception: exception raised
+    :return: text of the response if it is not None and empty string otherwise
+    """
+    result: str = exception.response.text if exception.response is not None else ""
     return result
 
 
@@ -60,7 +74,7 @@ def package_like(filename: Path) -> bool:
     return ".pkg." in name and not name.endswith(".sig")
 
 
-def pretty_datetime(timestamp: Optional[int]) -> str:
+def pretty_datetime(timestamp: Optional[Union[float, int]]) -> str:
     """
     convert datetime object to string
     :param timestamp: datetime to convert
@@ -89,6 +103,6 @@ def pretty_size(size: Optional[float], level: int = 0) -> str:
 
     if size is None:
         return ""
-    if size < 1024 or level == 3:
+    if size < 1024 or level >= 3:
         return f"{size:.1f} {str_level()}"
     return pretty_size(size / 1024, level + 1)

src/ahriman/models/auth_settings.py

@@ -0,0 +1,62 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+from enum import Enum, auto
+from typing import Type
+
+from ahriman.core.exceptions import InvalidOption
+
+
+class AuthSettings(Enum):
+    """
+    web authorization type
+    :cvar Disabled: authorization is disabled
+    :cvar Configuration: configuration based authorization
+    :cvar OAuth: OAuth based provider
+    """
+
+    Disabled = auto()
+    Configuration = auto()
+    OAuth = auto()
+
+    @classmethod
+    def from_option(cls: Type[AuthSettings], value: str) -> AuthSettings:
+        """
+        construct value from configuration
+        :param value: configuration value
+        :return: parsed value
+        """
+        if value.lower() in ("disabled", "no"):
+            return cls.Disabled
+        if value.lower() in ("configuration", "mapping"):
+            return cls.Configuration
+        if value.lower() in ('oauth', 'oauth2'):
+            return cls.OAuth
+        raise InvalidOption(value)
+
+    @property
+    def is_enabled(self) -> bool:
+        """
+        :return: False in case if authorization is disabled and True otherwise
+        """
+        if self == AuthSettings.Disabled:
+            return False
+        return True

src/ahriman/models/build_status.py

@@ -58,12 +58,27 @@ class BuildStatusEnum(Enum):
             return "success"
         return "inactive"
 
+    def bootstrap_color(self) -> str:
+        """
+        converts itself to bootstrap color
+        :return: bootstrap color
+        """
+        if self == BuildStatusEnum.Pending:
+            return "warning"
+        if self == BuildStatusEnum.Building:
+            return "warning"
+        if self == BuildStatusEnum.Failed:
+            return "danger"
+        if self == BuildStatusEnum.Success:
+            return "success"
+        return "secondary"
+
 
 class BuildStatus:
     """
     build status holder
     :ivar status: build status
-    :ivar _timestamp: build status update time
+    :ivar timestamp: build status update time
     """
 
     def __init__(self, status: Union[BuildStatusEnum, str, None] = None,

src/ahriman/models/counters.py

@@ -0,0 +1,72 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+from dataclasses import dataclass, fields
+from typing import Any, Dict, List, Tuple, Type
+
+from ahriman.models.build_status import BuildStatus
+from ahriman.models.package import Package
+
+
+@dataclass
+class Counters:
+    """
+    package counters
+    :ivar total: total packages count
+    :ivar unknown: packages in unknown status count
+    :ivar pending: packages in pending status count
+    :ivar building: packages in building status count
+    :ivar failed: packages in failed status count
+    :ivar success: packages in success status count
+    """
+
+    total: int
+    unknown: int = 0
+    pending: int = 0
+    building: int = 0
+    failed: int = 0
+    success: int = 0
+
+    @classmethod
+    def from_json(cls: Type[Counters], dump: Dict[str, Any]) -> Counters:
+        """
+        construct counters from json dump
+        :param dump: json dump body
+        :return: status counters
+        """
+        # filter to only known fields
+        known_fields = [pair.name for pair in fields(cls)]
+        dump = {key: value for key, value in dump.items() if key in known_fields}
+        return cls(**dump)
+
+    @classmethod
+    def from_packages(cls: Type[Counters], packages: List[Tuple[Package, BuildStatus]]) -> Counters:
+        """
+        construct counters from packages statuses
+        :param packages: list of package and their status as per watcher property
+        :return: status counters
+        """
+        per_status = {"total": len(packages)}
+        for _, status in packages:
+            key = status.status.name.lower()
+            per_status.setdefault(key, 0)
+            per_status[key] += 1
+        return cls(**per_status)

src/ahriman/models/internal_status.py

@@ -0,0 +1,61 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, Optional, Type
+
+from ahriman.models.counters import Counters
+
+
+@dataclass
+class InternalStatus:
+    """
+    internal server status
+    :ivar architecture: repository architecture
+    :ivar packages: packages statuses counter object
+    :ivar repository: repository name
+    :ivar version: service version
+    """
+
+    architecture: Optional[str] = None
+    packages: Counters = field(default=Counters(total=0))
+    repository: Optional[str] = None
+    version: Optional[str] = None
+
+    @classmethod
+    def from_json(cls: Type[InternalStatus], dump: Dict[str, Any]) -> InternalStatus:
+        """
+        construct internal status from json dump
+        :param dump: json dump body
+        :return: internal status
+        """
+        counters = Counters.from_json(dump["packages"]) if "packages" in dump else Counters(total=0)
+        return cls(architecture=dump.get("architecture"),
+                   packages=counters,
+                   repository=dump.get("repository"),
+                   version=dump.get("version"))
+
+    def view(self) -> Dict[str, Any]:
+        """
+        generate json status view
+        :return: json-friendly dictionary
+        """
+        return asdict(self)

src/ahriman/models/package.py

@@ -31,7 +31,7 @@ from typing import Any, Dict, List, Optional, Set, Type, Union
 from ahriman.core.alpm.pacman import Pacman
 from ahriman.core.exceptions import InvalidPackageInfo
 from ahriman.core.util import check_output
-from ahriman.models.package_desciption import PackageDescription
+from ahriman.models.package_description import PackageDescription
 from ahriman.models.repository_paths import RepositoryPaths
 
 
@@ -52,6 +52,13 @@ class Package:
 
     _check_output = check_output
 
+    @property
+    def depends(self) -> List[str]:
+        """
+        :return: sum of dependencies per arch package
+        """
+        return sorted(set(sum([package.depends for package in self.packages.values()], start=[])))
+
     @property
     def git_url(self) -> str:
         """
@@ -147,7 +154,7 @@ class Package:
         :return: package properties
         """
         packages = {
-            key: PackageDescription(**value)
+            key: PackageDescription.from_json(value)
             for key, value in dump.get("packages", {}).items()
         }
         return Package(
@@ -156,6 +163,27 @@ class Package:
             aur_url=dump["aur_url"],
             packages=packages)
 
+    @classmethod
+    def load(cls: Type[Package], path: Union[Path, str], pacman: Pacman, aur_url: str) -> Package:
+        """
+        package constructor from available sources
+        :param path: one of path to sources directory, path to archive or package name/base
+        :param pacman: alpm wrapper instance (required to load from archive)
+        :param aur_url: AUR root url
+        :return: package properties
+        """
+        try:
+            maybe_path = Path(path)
+            if maybe_path.is_dir():
+                return cls.from_build(maybe_path, aur_url)
+            if maybe_path.is_file():
+                return cls.from_archive(maybe_path, pacman, aur_url)
+            return cls.from_aur(str(path), aur_url)
+        except InvalidPackageInfo:
+            raise
+        except Exception as e:
+            raise InvalidPackageInfo(str(e))
+
     @staticmethod
     def dependencies(path: Path) -> Set[str]:
         """
@@ -188,35 +216,12 @@ class Package:
         generate full version from components
         :param epoch: package epoch if any
         :param pkgver: package version
-        :param pkgrel: package release version (archlinux specific)
+        :param pkgrel: package release version (arch linux specific)
         :return: generated version
         """
         prefix = f"{epoch}:" if epoch else ""
         return f"{prefix}{pkgver}-{pkgrel}"
 
-    @staticmethod
-    def load(path: Union[Path, str], pacman: Pacman, aur_url: str) -> Package:
-        """
-        package constructor from available sources
-        :param path: one of path to sources directory, path to archive or package name/base
-        :param pacman: alpm wrapper instance (required to load from archive)
-        :param aur_url: AUR root url
-        :return: package properties
-        """
-        try:
-            maybe_path = Path(path)
-            if maybe_path.is_dir():
-                package: Package = Package.from_build(maybe_path, aur_url)
-            elif maybe_path.is_file():
-                package = Package.from_archive(maybe_path, pacman, aur_url)
-            else:
-                package = Package.from_aur(str(path), aur_url)
-            return package
-        except InvalidPackageInfo:
-            raise
-        except Exception as e:
-            raise InvalidPackageInfo(str(e))
-
     def actual_version(self, paths: RepositoryPaths) -> str:
         """
         additional method to handle VCS package versions

src/ahriman/models/package_description.py

@@ -19,10 +19,10 @@
 #
 from __future__ import annotations
 
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, fields
 from pathlib import Path
 from pyalpm import Package  # type: ignore
-from typing import List, Optional, Type
+from typing import Any, Dict, List, Optional, Type
 
 
 @dataclass
@@ -32,22 +32,26 @@ class PackageDescription:
     :ivar architecture: package architecture
     :ivar archive_size: package archive size
     :ivar build_date: package build date
+    :ivar depends: package dependencies list
     :ivar description: package description
     :ivar filename: package archive name
     :ivar groups: package groups
     :ivar installed_size: package installed size
     :ivar licenses: package licenses list
+    :ivar provides: list of provided packages
     :ivar url: package url
     """
 
     architecture: Optional[str] = None
     archive_size: Optional[int] = None
     build_date: Optional[int] = None
+    depends: List[str] = field(default_factory=list)
     description: Optional[str] = None
     filename: Optional[str] = None
     groups: List[str] = field(default_factory=list)
     installed_size: Optional[int] = None
     licenses: List[str] = field(default_factory=list)
+    provides: List[str] = field(default_factory=list)
     url: Optional[str] = None
 
     @property
@@ -57,6 +61,18 @@ class PackageDescription:
         """
         return Path(self.filename) if self.filename is not None else None
 
+    @classmethod
+    def from_json(cls: Type[PackageDescription], dump: Dict[str, Any]) -> PackageDescription:
+        """
+        construct package properties from json dump
+        :param dump: json dump body
+        :return: package properties
+        """
+        # filter to only known fields
+        known_fields = [pair.name for pair in fields(cls)]
+        dump = {key: value for key, value in dump.items() if key in known_fields}
+        return cls(**dump)
+
     @classmethod
     def from_package(cls: Type[PackageDescription], package: Package, path: Path) -> PackageDescription:
         """
@@ -65,13 +81,15 @@ class PackageDescription:
         :param path: path to package archive
         :return: package properties based on tarball
         """
-        return PackageDescription(
+        return cls(
             architecture=package.arch,
             archive_size=package.size,
             build_date=package.builddate,
+            depends=package.depends,
             description=package.desc,
             filename=path.name,
             groups=package.groups,
             installed_size=package.isize,
             licenses=package.licenses,
+            provides=package.provides,
             url=package.url)

src/ahriman/models/report_settings.py

@@ -20,6 +20,7 @@
 from __future__ import annotations
 
 from enum import Enum, auto
+from typing import Type
 
 from ahriman.core.exceptions import InvalidOption
 
@@ -27,18 +28,24 @@ from ahriman.core.exceptions import InvalidOption
 class ReportSettings(Enum):
     """
     report targets enumeration
+    :cvar Disabled: option which generates no report for testing purpose
     :cvar HTML: html report generation
+    :cvar Email: email report generation
     """
 
+    Disabled = auto()  # for testing purpose
     HTML = auto()
+    Email = auto()
 
-    @staticmethod
+    @classmethod
-    def from_option(value: str) -> ReportSettings:
+    def from_option(cls: Type[ReportSettings], value: str) -> ReportSettings:
         """
         construct value from configuration
         :param value: configuration value
         :return: parsed value
         """
         if value.lower() in ("html",):
-            return ReportSettings.HTML
+            return cls.HTML
+        if value.lower() in ("email",):
+            return cls.Email
         raise InvalidOption(value)

src/ahriman/models/repository_paths.py

@@ -17,9 +17,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from pathlib import Path
+from __future__ import annotations
 
 from dataclasses import dataclass
+from pathlib import Path
+from typing import Set, Type
 
 
 @dataclass
@@ -76,6 +78,20 @@ class RepositoryPaths:
         """
         return self.root / "sources" / self.architecture
 
+    @classmethod
+    def known_architectures(cls: Type[RepositoryPaths], root: Path) -> Set[str]:
+        """
+        get known architectures
+        :param root: repository root
+        :return: list of architectures for which tree is created
+        """
+        paths = cls(root, "")
+        return {
+            path.name
+            for path in paths.repository.iterdir()
+            if path.is_dir()
+        }
+
     def create_tree(self) -> None:
         """
         create ahriman working tree

src/ahriman/models/sign_settings.py

@@ -20,6 +20,7 @@
 from __future__ import annotations
 
 from enum import Enum, auto
+from typing import Type
 
 from ahriman.core.exceptions import InvalidOption
 
@@ -27,22 +28,22 @@ from ahriman.core.exceptions import InvalidOption
 class SignSettings(Enum):
     """
     sign targets enumeration
-    :cvar SignPackages: sign each package
+    :cvar Packages: sign each package
-    :cvar SignRepository: sign repository database file
+    :cvar Repository: sign repository database file
     """
 
-    SignPackages = auto()
+    Packages = auto()
-    SignRepository = auto()
+    Repository = auto()
 
-    @staticmethod
+    @classmethod
-    def from_option(value: str) -> SignSettings:
+    def from_option(cls: Type[SignSettings], value: str) -> SignSettings:
         """
         construct value from configuration
         :param value: configuration value
         :return: parsed value
         """
         if value.lower() in ("package", "packages", "sign-package"):
-            return SignSettings.SignPackages
+            return cls.Packages
         if value.lower() in ("repository", "sign-repository"):
-            return SignSettings.SignRepository
+            return cls.Repository
         raise InvalidOption(value)

src/ahriman/models/smtp_ssl_settings.py

@@ -0,0 +1,49 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+from enum import Enum, auto
+from typing import Type
+
+
+class SmtpSSLSettings(Enum):
+    """
+    SMTP SSL mode enumeration
+    :cvar Disabled: no SSL enabled
+    :cvar SSL: use SMTP_SSL instead of normal SMTP client
+    :cvar STARTTLS: use STARTTLS in normal SMTP client
+    """
+
+    Disabled = auto()
+    SSL = auto()
+    STARTTLS = auto()
+
+    @classmethod
+    def from_option(cls: Type[SmtpSSLSettings], value: str) -> SmtpSSLSettings:
+        """
+        construct value from configuration
+        :param value: configuration value
+        :return: parsed value
+        """
+        if value.lower() in ("ssl", "ssl/tls"):
+            return cls.SSL
+        if value.lower() in ("starttls",):
+            return cls.STARTTLS
+        return cls.Disabled

src/ahriman/models/upload_settings.py

@@ -20,6 +20,7 @@
 from __future__ import annotations
 
 from enum import Enum, auto
+from typing import Type
 
 from ahriman.core.exceptions import InvalidOption
 
@@ -27,22 +28,24 @@ from ahriman.core.exceptions import InvalidOption
 class UploadSettings(Enum):
     """
     remote synchronization targets enumeration
+    :cvar Disabled: no sync will be performed, required for testing purpose
     :cvar Rsync: sync via rsync
     :cvar S3: sync to Amazon S3
     """
 
+    Disabled = auto()  # for testing purpose
     Rsync = auto()
     S3 = auto()
 
-    @staticmethod
+    @classmethod
-    def from_option(value: str) -> UploadSettings:
+    def from_option(cls: Type[UploadSettings], value: str) -> UploadSettings:
         """
         construct value from configuration
         :param value: configuration value
         :return: parsed value
         """
         if value.lower() in ("rsync",):
-            return UploadSettings.Rsync
+            return cls.Rsync
         if value.lower() in ("s3",):
-            return UploadSettings.S3
+            return cls.S3
         raise InvalidOption(value)

src/ahriman/models/user.py

@@ -0,0 +1,110 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Optional, Type
+from passlib.pwd import genword as generate_password  # type: ignore
+from passlib.handlers.sha2_crypt import sha512_crypt  # type: ignore
+
+from ahriman.models.user_access import UserAccess
+
+
+@dataclass
+class User:
+    """
+    authorized web user model
+    :ivar username: username
+    :ivar password: hashed user password with salt
+    :ivar access: user role
+    """
+
+    username: str
+    password: str
+    access: UserAccess
+
+    _HASHER = sha512_crypt
+
+    @classmethod
+    def from_option(cls: Type[User], username: Optional[str], password: Optional[str],
+                    access: UserAccess = UserAccess.Read) -> Optional[User]:
+        """
+        build user descriptor from configuration options
+        :param username: username
+        :param password: password as string
+        :param access: optional user access
+        :return: generated user descriptor if all options are supplied and None otherwise
+        """
+        if username is None or password is None:
+            return None
+        return cls(username, password, access)
+
+    @staticmethod
+    def generate_password(length: int) -> str:
+        """
+        generate password with specified length
+        :param length: password length
+        :return: random string which contains letters and numbers
+        """
+        password: str = generate_password(length=length)
+        return password
+
+    def check_credentials(self, password: str, salt: str) -> bool:
+        """
+        validate user password
+        :param password: entered password
+        :param salt: salt for hashed password
+        :return: True in case if password matches, False otherwise
+        """
+        try:
+            verified: bool = self._HASHER.verify(password + salt, self.password)
+        except ValueError:
+            verified = False  # the absence of evidence is not the evidence of absence (c) Gin Rummy
+        return verified
+
+    def hash_password(self, salt: str) -> str:
+        """
+        generate hashed password from plain text
+        :param salt: salt for hashed password
+        :return: hashed string to store in configuration
+        """
+        if not self.password:
+            # in case of empty password we leave it empty. This feature is used by any external (like OAuth) provider
+            # when we do not store any password here
+            return ""
+        password_hash: str = self._HASHER.hash(self.password + salt)
+        return password_hash
+
+    def verify_access(self, required: UserAccess) -> bool:
+        """
+        validate if user has access to requested resource
+        :param required: required access level
+        :return: True in case if user is allowed to do this request and False otherwise
+        """
+        if self.access == UserAccess.Write:
+            return True  # everything is allowed
+        return self.access == required
+
+    def __repr__(self) -> str:
+        """
+        generate string representation of object
+        :return: unique string representation
+        """
+        return f"User(username={self.username}, access={self.access})"