Release 2.8.0

manually

.bandit-test.yml

@@ -1 +1 @@
-skips: ['B101', 'B105', 'B106', 'B404']
+skips: ['B101', 'B104', 'B105', 'B106', 'B404']

.github/workflows/setup.sh

@@ -18,7 +18,7 @@ if [[ -z $MINIMAL_INSTALL ]]; then
     # VCS support
     pacman --noconfirm -Sy breezy darcs mercurial subversion
     # web server
-    pacman --noconfirm -Sy python-aioauth-client python-aiohttp python-aiohttp-debugtoolbar python-aiohttp-jinja2 python-aiohttp-security python-aiohttp-session python-cryptography python-jinja
+    pacman --noconfirm -Sy python-aioauth-client python-aiohttp python-aiohttp-apispec-git python-aiohttp-cors python-aiohttp-debugtoolbar python-aiohttp-jinja2 python-aiohttp-security python-aiohttp-session python-cryptography python-jinja
     # additional features
     pacman --noconfirm -Sy gnupg python-boto3 rsync
 fi

.pylintrc

@@ -533,5 +533,5 @@ valid-metaclass-classmethod-first-arg=cls
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "BaseException, Exception".
-overgeneral-exceptions=BaseException,
-                       Exception
+overgeneral-exceptions=builtins.BaseException,
+                       builtins.Exception

CONTRIBUTING.md

@@ -156,6 +156,52 @@ Again, the most checks can be performed by `make check` command, though some add
 * No global variable is allowed outside of `ahriman.version` module. `ahriman.core.context` is also special case.
 * Single quotes are not allowed. The reason behind this restriction is the fact that docstrings must be written by using double quotes only, and we would like to make style consistent.
 * If your class writes anything to log, the `ahriman.core.log.LazyLogging` trait must be used.
+* Web API methods must be documented by using `aiohttp_apispec` library. Schema testing mostly should be implemented in related view class tests. Recommended example for documentation (excluding comments):
+
+    ```python
+    import aiohttp_apispec
+
+    from marshmallow import Schema, fields  
+
+    from ahriman.web.schemas.auth_schema import AuthSchema
+    from ahriman.web.schemas.error_schema import ErrorSchema
+    from ahriman.web.schemas.package_name_schema import PackageNameSchema
+    from ahriman.web.views.base import BaseView
+
+
+    class RequestSchema(Schema):
+
+        field = fields.String(metadata={"description": "Field description", "example": "foo"})
+
+
+    class ResponseSchema(Schema):
+
+        field = fields.String(required=True, metadata={"description": "Field description"})
+
+
+    class Foo(BaseView):
+
+        POST_PERMISSION = ...
+
+        @aiohttp_apispec.docs(
+            tags=["Tag"],
+            summary="Do foo",
+            description="Extended description of the method which does foo",
+            responses={
+                200: {"description": "Success response", "schema": ResponseSchema},
+                204: {"description": "Success response"},  # example without json schema response
+                400: {"description": "Bad data is supplied", "schema": ErrorSchema},  # exception raised by this method
+                401: {"description": "Authorization required", "schema": ErrorSchema},  # should be always presented
+                403: {"description": "Access is forbidden", "schema": ErrorSchema},  # should be always presented
+                500: {"description": "Internal server error", "schema": ErrorSchema},  # should be always presented
+            },
+            security=[{"token": [POST_PERMISSION]}],
+        )
+        @aiohttp_apispec.cookies_schema(AuthSchema)  # should be always presented
+        @aiohttp_apispec.match_info_schema(PackageNameSchema)
+        @aiohttp_apispec.json_schema(RequestSchema(many=True))
+        async def post(self) -> None: ...
+    ```
 
 ### Other checks
 

Dockerfile

@@ -30,9 +30,9 @@ COPY "docker/install-aur-package.sh" "/usr/local/bin/install-aur-package"
 ## darcs is not installed by reasons, because it requires a lot haskell packages which dramatically increase image size
 RUN pacman --noconfirm -Sy devtools git pyalpm python-cerberus python-inflection python-passlib python-requests python-setuptools python-srcinfo && \
     pacman --noconfirm -Sy python-build python-installer python-wheel && \
-    pacman --noconfirm -Sy breezy mercurial python-aiohttp python-boto3 python-cryptography python-jinja python-requests-unixsocket rsync subversion && \
-    runuser -u build -- install-aur-package python-aioauth-client python-aiohttp-jinja2 python-aiohttp-debugtoolbar \
-                                            python-aiohttp-session python-aiohttp-security
+    pacman --noconfirm -Sy breezy mercurial python-aiohttp python-aiohttp-cors python-boto3 python-cryptography python-jinja python-requests-unixsocket rsync subversion && \
+    runuser -u build -- install-aur-package python-aioauth-client python-aiohttp-apispec-git python-aiohttp-jinja2  \
+                                            python-aiohttp-debugtoolbar python-aiohttp-session python-aiohttp-security
 
 # cleanup unused
 RUN find "/var/cache/pacman/pkg" -type f -delete

README.md

@@ -36,6 +36,6 @@ The application provides reasonable defaults which allow to use it out-of-box; h
 
 ## Live demos
 
-* [Build status page](https://ahriman-demo.arcanis.me). You can log in as `demo` user by using `demo` password. However, you will not be able to run tasks.
+* [Build status page](https://ahriman-demo.arcanis.me). You can log in as `demo` user by using `demo` password. However, you will not be able to run tasks. [HTTP API documentation](https://ahriman-demo.arcanis.me/api-docs) is also available.
 * [Repository index](http://repo.arcanis.me/x86_64/index.html).
 * [Telegram feed](https://t.me/arcanisrepo).

docs/ahriman.1

@@ -1,4 +1,4 @@
-.TH AHRIMAN "1" "2023\-01\-25" "ahriman" "Generated Python Manual"
+.TH AHRIMAN "1" "2023\-04\-09" "ahriman" "Generated Python Manual"
 .SH NAME
 ahriman
 .SH SYNOPSIS
@@ -156,7 +156,7 @@ web server
 
 .SH COMMAND \fI\,'ahriman aur\-search'\/\fR
 usage: ahriman aur\-search [\-h] [\-e] [\-\-info | \-\-no\-info]
-                          [\-\-sort\-by {description,first_submitted,id,last_modified,maintainer,name,num_votes,out_of_date,package_base,package_base_id,popularity,repository,url,url_path,version}]
+                          [\-\-sort\-by {description,first_submitted,id,last_modified,maintainer,name,num_votes,out_of_date,package_base,package_base_id,popularity,repository,submitter,url,url_path,version}]
                           search [search ...]
 
 search for package in AUR using API
@@ -175,7 +175,7 @@ return non\-zero exit status if result is empty
 show additional package information (default: False)
 
 .TP
-\fB\-\-sort\-by\fR \fI\,{description,first_submitted,id,last_modified,maintainer,name,num_votes,out_of_date,package_base,package_base_id,popularity,repository,url,url_path,version}\/\fR
+\fB\-\-sort\-by\fR \fI\,{description,first_submitted,id,last_modified,maintainer,name,num_votes,out_of_date,package_base,package_base_id,popularity,repository,submitter,url,url_path,version}\/\fR
 sort field by this field. In case if two packages have the same value of the specified field, they will be always sorted
 by name
 
@@ -215,8 +215,8 @@ usage: ahriman help\-version [\-h]
 print application and its dependencies versions
 
 .SH COMMAND \fI\,'ahriman package\-add'\/\fR
-usage: ahriman package\-add [\-h] [\-e] [\-n] [\-y] [\-s {auto,archive,aur,directory,local,remote,repository}]
-                           [\-\-without\-dependencies]
+usage: ahriman package\-add [\-h] [\-\-dependencies | \-\-no\-dependencies] [\-e] [\-n] [\-y]
+                           [\-s {auto,archive,aur,directory,local,remote,repository}]
                            package [package ...]
 
 add existing or new package to the build queue
@@ -226,6 +226,10 @@ add existing or new package to the build queue
 package source (base name, path to local files, remote URL)
 
 .SH OPTIONS \fI\,'ahriman package\-add'\/\fR
+.TP
+\fB\-\-dependencies\fR, \fB\-\-no\-dependencies\fR
+process missing package dependencies (default: True)
+
 .TP
 \fB\-e\fR, \fB\-\-exit\-code\fR
 return non\-zero exit status if result is empty
@@ -242,10 +246,6 @@ download fresh package databases from the mirror before actions, \-yy to force r
 \fB\-s\fR \fI\,{auto,archive,aur,directory,local,remote,repository}\/\fR, \fB\-\-source\fR \fI\,{auto,archive,aur,directory,local,remote,repository}\/\fR
 explicitly specify the package source for this command
 
-.TP
-\fB\-\-without\-dependencies\fR
-do not add dependencies
-
 .SH COMMAND \fI\,'ahriman package\-remove'\/\fR
 usage: ahriman package\-remove [\-h] package [package ...]
 
@@ -401,8 +401,8 @@ fetch actual version of VCS packages (default: True)
 download fresh package databases from the mirror before actions, \-yy to force refresh even if up to date
 
 .SH COMMAND \fI\,'ahriman repo\-daemon'\/\fR
-usage: ahriman repo\-daemon [\-h] [\-i INTERVAL] [\-\-aur | \-\-no\-aur] [\-\-local | \-\-no\-local] [\-\-manual | \-\-no\-manual]
-                           [\-\-vcs | \-\-no\-vcs] [\-y]
+usage: ahriman repo\-daemon [\-h] [\-i INTERVAL] [\-\-aur | \-\-no\-aur] [\-\-dependencies | \-\-no\-dependencies]
+                           [\-\-local | \-\-no\-local] [\-\-manual | \-\-no\-manual] [\-\-vcs | \-\-no\-vcs] [\-y]
 
 start process which periodically will run update process
 
@@ -415,6 +415,10 @@ interval between runs in seconds
 \fB\-\-aur\fR, \fB\-\-no\-aur\fR
 enable or disable checking for AUR updates (default: True)
 
+.TP
+\fB\-\-dependencies\fR, \fB\-\-no\-dependencies\fR
+process missing package dependencies (default: True)
+
 .TP
 \fB\-\-local\fR, \fB\-\-no\-local\fR
 enable or disable checking of local packages for updates (default: True)
@@ -523,8 +527,8 @@ run triggers on empty build result as configured by settings
 instead of running all triggers as set by configuration, just process specified ones in order of mention
 
 .SH COMMAND \fI\,'ahriman repo\-update'\/\fR
-usage: ahriman repo\-update [\-h] [\-\-dry\-run] [\-e] [\-\-aur | \-\-no\-aur] [\-\-local | \-\-no\-local] [\-\-manual | \-\-no\-manual]
-                           [\-\-vcs | \-\-no\-vcs] [\-y]
+usage: ahriman repo\-update [\-h] [\-\-aur | \-\-no\-aur] [\-\-dependencies | \-\-no\-dependencies] [\-\-dry\-run] [\-e]
+                           [\-\-local | \-\-no\-local] [\-\-manual | \-\-no\-manual] [\-\-vcs | \-\-no\-vcs] [\-y]
                            [package ...]
 
 check for packages updates and run build process if requested
@@ -534,6 +538,14 @@ check for packages updates and run build process if requested
 filter check by package base
 
 .SH OPTIONS \fI\,'ahriman repo\-update'\/\fR
+.TP
+\fB\-\-aur\fR, \fB\-\-no\-aur\fR
+enable or disable checking for AUR updates (default: True)
+
+.TP
+\fB\-\-dependencies\fR, \fB\-\-no\-dependencies\fR
+process missing package dependencies (default: True)
+
 .TP
 \fB\-\-dry\-run\fR
 just perform check for updates, same as check command
@@ -542,10 +554,6 @@ just perform check for updates, same as check command
 \fB\-e\fR, \fB\-\-exit\-code\fR
 return non\-zero exit status if result is empty
 
-.TP
-\fB\-\-aur\fR, \fB\-\-no\-aur\fR
-enable or disable checking for AUR updates (default: True)
-
 .TP
 \fB\-\-local\fR, \fB\-\-no\-local\fR
 enable or disable checking of local packages for updates (default: True)
@@ -590,10 +598,15 @@ clear directory with built packages (default: False)
 clear directory with pacman local database cache (default: False)
 
 .SH COMMAND \fI\,'ahriman service\-config'\/\fR
-usage: ahriman service\-config [\-h]
+usage: ahriman service\-config [\-h] [\-\-secure | \-\-no\-secure]
 
 dump configuration for the specified architecture
 
+.SH OPTIONS \fI\,'ahriman service\-config'\/\fR
+.TP
+\fB\-\-secure\fR, \fB\-\-no\-secure\fR
+hide passwords and secrets from output (default: True)
+
 .SH COMMAND \fI\,'ahriman service\-config\-validate'\/\fR
 usage: ahriman service\-config\-validate [\-h] [\-e]
 

docs/ahriman.core.database.data.rst

@@ -1,45 +0,0 @@
-ahriman.core.database.data package
-==================================
-
-Submodules
-----------
-
-ahriman.core.database.data.package\_remotes module
---------------------------------------------------
-
-.. automodule:: ahriman.core.database.data.package_remotes
-   :members:
-   :no-undoc-members:
-   :show-inheritance:
-
-ahriman.core.database.data.package\_statuses module
----------------------------------------------------
-
-.. automodule:: ahriman.core.database.data.package_statuses
-   :members:
-   :no-undoc-members:
-   :show-inheritance:
-
-ahriman.core.database.data.patches module
------------------------------------------
-
-.. automodule:: ahriman.core.database.data.patches
-   :members:
-   :no-undoc-members:
-   :show-inheritance:
-
-ahriman.core.database.data.users module
----------------------------------------
-
-.. automodule:: ahriman.core.database.data.users
-   :members:
-   :no-undoc-members:
-   :show-inheritance:
-
-Module contents
----------------
-
-.. automodule:: ahriman.core.database.data
-   :members:
-   :no-undoc-members:
-   :show-inheritance:

docs/ahriman.core.database.migrations.rst

@@ -44,6 +44,22 @@ ahriman.core.database.migrations.m004\_logs module
    :no-undoc-members:
    :show-inheritance:
 
+ahriman.core.database.migrations.m005\_make\_opt\_depends module
+----------------------------------------------------------------
+
+.. automodule:: ahriman.core.database.migrations.m005_make_opt_depends
+   :members:
+   :no-undoc-members:
+   :show-inheritance:
+
+ahriman.core.database.migrations.m006\_packages\_architecture\_required module
+------------------------------------------------------------------------------
+
+.. automodule:: ahriman.core.database.migrations.m006_packages_architecture_required
+   :members:
+   :no-undoc-members:
+   :show-inheritance:
+
 Module contents
 ---------------
 

docs/ahriman.core.database.rst

@@ -7,7 +7,6 @@ Subpackages
 .. toctree::
    :maxdepth: 4
 
-   ahriman.core.database.data
    ahriman.core.database.migrations
    ahriman.core.database.operations
 

docs/ahriman.models.rst

@@ -196,14 +196,6 @@ ahriman.models.user\_access module
    :no-undoc-members:
    :show-inheritance:
 
-ahriman.models.user\_identity module
-------------------------------------
-
-.. automodule:: ahriman.models.user_identity
-   :members:
-   :no-undoc-members:
-   :show-inheritance:
-
 Module contents
 ---------------
 

docs/architecture.rst

@@ -114,7 +114,7 @@ Schema and data migrations
 
 The schema migration are applied according to current ``pragma user_info`` values, located at ``ahriman.core.database.migrations`` package and named as ``m000_migration_name.py`` (the preceding ``m`` is required in order to import migration content for tests). Additional class ``ahriman.core.database.migrations.Migrations`` reads all migrations automatically and applies them in alphabetical order.
 
-There are also data migrations which are located at ``ahriman.core.database.data`` package and move data from old-style (e.g. json files in filesystem, directory trees, etc) to the database. They are also part of migration and (unlike schema migrations) are applied only at specific version breakpoints (e.g. if ``user_version`` is more than 0 no initial migration will be applied).
+These migrations also contain data migrations. Though the recommended way is to migrate data directly from SQL requests, sometimes it is required to have external data (like packages list) in order to set correct data. To do so, special method `migrate_data` is used.
 
 Type conversions
 ^^^^^^^^^^^^^^^^
@@ -269,6 +269,7 @@ Web application
 Web application requires the following python packages to be installed:
 
 * Core part requires ``aiohttp`` (application itself), ``aiohttp_jinja2`` and ``Jinja2`` (HTML generation from templates).
+* Additional web features also require ``aiohttp-apispec`` (autogenerated documentation), ``aiohttp_cors`` (CORS support, required by documentation)
 * In addition, ``aiohttp_debugtoolbar`` is required for debug panel. Please note that this option does not work together with authorization and basically must not be used in production.
 * In addition, authorization feature requires ``aiohttp_security``, ``aiohttp_session`` and ``cryptography``.
 * In addition to base authorization dependencies, OAuth2 also requires ``aioauth-client`` library.
@@ -279,6 +280,13 @@ Middlewares
 
 Service provides some custom middlewares, e.g. logging every exception (except for user ones) and user authorization.
 
+HEAD and OPTIONS requests
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``HEAD`` request is automatically generated by ``ahriman.web.views.base.BaseView`` class. It just calls ``GET`` method, removes any data from body and returns the result. In case if no ``GET`` method available for this view, the ``aiohttp.web.HTTPMethodNotAllowed`` exception will be raised.
+
+On the other side, ``OPTIONS`` method is implemented in the ``ahriman.web.middlewares.exception_handler.exception_handler`` middleware. In case if ``aiohttp.web.HTTPMethodNotAllowed`` exception is raised and original method was ``OPTIONS``, the middleware handles it, converts to valid request and returns response to user.
+
 Web views
 ^^^^^^^^^
 
@@ -288,6 +296,7 @@ REST API supports both form and JSON data, but the last one is recommended.
 
 Different APIs are separated into different packages:
 
+* ``ahriman.web.views.api`` not a real API, but some views which provide OpenAPI support.
 * ``ahriman.web.views.service`` provides views for application controls.
 * ``ahriman.web.views.status`` package provides REST API for application reporting.
 * ``ahriman.web.views.user`` package provides login and logout methods which can be called without authorization.

docs/completions/bash/_ahriman

@@ -10,9 +10,9 @@ _shtab_ahriman_help_commands_unsafe_option_strings=('-h' '--help' '--command')
 _shtab_ahriman_help_updates_option_strings=('-h' '--help' '-e' '--exit-code')
 _shtab_ahriman_help_version_option_strings=('-h' '--help')
 _shtab_ahriman_version_option_strings=('-h' '--help')
-_shtab_ahriman_package_add_option_strings=('-h' '--help' '-e' '--exit-code' '-n' '--now' '-y' '--refresh' '-s' '--source' '--without-dependencies')
-_shtab_ahriman_add_option_strings=('-h' '--help' '-e' '--exit-code' '-n' '--now' '-y' '--refresh' '-s' '--source' '--without-dependencies')
-_shtab_ahriman_package_update_option_strings=('-h' '--help' '-e' '--exit-code' '-n' '--now' '-y' '--refresh' '-s' '--source' '--without-dependencies')
+_shtab_ahriman_package_add_option_strings=('-h' '--help' '--dependencies' '--no-dependencies' '-e' '--exit-code' '-n' '--now' '-y' '--refresh' '-s' '--source')
+_shtab_ahriman_add_option_strings=('-h' '--help' '--dependencies' '--no-dependencies' '-e' '--exit-code' '-n' '--now' '-y' '--refresh' '-s' '--source')
+_shtab_ahriman_package_update_option_strings=('-h' '--help' '--dependencies' '--no-dependencies' '-e' '--exit-code' '-n' '--now' '-y' '--refresh' '-s' '--source')
 _shtab_ahriman_package_remove_option_strings=('-h' '--help')
 _shtab_ahriman_remove_option_strings=('-h' '--help')
 _shtab_ahriman_package_status_option_strings=('-h' '--help' '--ahriman' '-e' '--exit-code' '--info' '--no-info' '-s' '--status')
@@ -27,8 +27,8 @@ _shtab_ahriman_patch_set_add_option_strings=('-h' '--help' '-t' '--track')
 _shtab_ahriman_repo_backup_option_strings=('-h' '--help')
 _shtab_ahriman_repo_check_option_strings=('-h' '--help' '-e' '--exit-code' '--vcs' '--no-vcs' '-y' '--refresh')
 _shtab_ahriman_check_option_strings=('-h' '--help' '-e' '--exit-code' '--vcs' '--no-vcs' '-y' '--refresh')
-_shtab_ahriman_repo_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh')
-_shtab_ahriman_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh')
+_shtab_ahriman_repo_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--dependencies' '--no-dependencies' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh')
+_shtab_ahriman_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--dependencies' '--no-dependencies' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh')
 _shtab_ahriman_repo_rebuild_option_strings=('-h' '--help' '--depends-on' '--dry-run' '--from-database' '-e' '--exit-code')
 _shtab_ahriman_rebuild_option_strings=('-h' '--help' '--depends-on' '--dry-run' '--from-database' '-e' '--exit-code')
 _shtab_ahriman_repo_remove_unknown_option_strings=('-h' '--help' '--dry-run')
@@ -43,14 +43,14 @@ _shtab_ahriman_repo_sync_option_strings=('-h' '--help')
 _shtab_ahriman_sync_option_strings=('-h' '--help')
 _shtab_ahriman_repo_tree_option_strings=('-h' '--help')
 _shtab_ahriman_repo_triggers_option_strings=('-h' '--help')
-_shtab_ahriman_repo_update_option_strings=('-h' '--help' '--dry-run' '-e' '--exit-code' '--aur' '--no-aur' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh')
-_shtab_ahriman_update_option_strings=('-h' '--help' '--dry-run' '-e' '--exit-code' '--aur' '--no-aur' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh')
+_shtab_ahriman_repo_update_option_strings=('-h' '--help' '--aur' '--no-aur' '--dependencies' '--no-dependencies' '--dry-run' '-e' '--exit-code' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh')
+_shtab_ahriman_update_option_strings=('-h' '--help' '--aur' '--no-aur' '--dependencies' '--no-dependencies' '--dry-run' '-e' '--exit-code' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh')
 _shtab_ahriman_service_clean_option_strings=('-h' '--help' '--cache' '--no-cache' '--chroot' '--no-chroot' '--manual' '--no-manual' '--packages' '--no-packages' '--pacman' '--no-pacman')
 _shtab_ahriman_clean_option_strings=('-h' '--help' '--cache' '--no-cache' '--chroot' '--no-chroot' '--manual' '--no-manual' '--packages' '--no-packages' '--pacman' '--no-pacman')
 _shtab_ahriman_repo_clean_option_strings=('-h' '--help' '--cache' '--no-cache' '--chroot' '--no-chroot' '--manual' '--no-manual' '--packages' '--no-packages' '--pacman' '--no-pacman')
-_shtab_ahriman_service_config_option_strings=('-h' '--help')
-_shtab_ahriman_config_option_strings=('-h' '--help')
-_shtab_ahriman_repo_config_option_strings=('-h' '--help')
+_shtab_ahriman_service_config_option_strings=('-h' '--help' '--secure' '--no-secure')
+_shtab_ahriman_config_option_strings=('-h' '--help' '--secure' '--no-secure')
+_shtab_ahriman_repo_config_option_strings=('-h' '--help' '--secure' '--no-secure')
 _shtab_ahriman_service_config_validate_option_strings=('-h' '--help' '-e' '--exit-code')
 _shtab_ahriman_config_validate_option_strings=('-h' '--help' '-e' '--exit-code')
 _shtab_ahriman_repo_config_validate_option_strings=('-h' '--help' '-e' '--exit-code')
@@ -71,8 +71,8 @@ _shtab_ahriman_web_option_strings=('-h' '--help')
 
 
 _shtab_ahriman_pos_0_choices=('aur-search' 'search' 'help' 'help-commands-unsafe' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'user-add' 'user-list' 'user-remove' 'web')
-_shtab_ahriman_aur_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'url' 'url_path' 'version')
-_shtab_ahriman_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'url' 'url_path' 'version')
+_shtab_ahriman_aur_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'submitter' 'url' 'url_path' 'version')
+_shtab_ahriman_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'submitter' 'url' 'url_path' 'version')
 _shtab_ahriman_package_add__s_choices=('auto' 'archive' 'aur' 'directory' 'local' 'remote' 'repository')
 _shtab_ahriman_package_add___source_choices=('auto' 'archive' 'aur' 'directory' 'local' 'remote' 'repository')
 _shtab_ahriman_add__s_choices=('auto' 'archive' 'aur' 'directory' 'local' 'remote' 'repository')
@@ -139,33 +139,36 @@ _shtab_ahriman_version___help_nargs=0
 _shtab_ahriman_package_add_pos_0_nargs=+
 _shtab_ahriman_package_add__h_nargs=0
 _shtab_ahriman_package_add___help_nargs=0
+_shtab_ahriman_package_add___dependencies_nargs=0
+_shtab_ahriman_package_add___no_dependencies_nargs=0
 _shtab_ahriman_package_add__e_nargs=0
 _shtab_ahriman_package_add___exit_code_nargs=0
 _shtab_ahriman_package_add__n_nargs=0
 _shtab_ahriman_package_add___now_nargs=0
 _shtab_ahriman_package_add__y_nargs=0
 _shtab_ahriman_package_add___refresh_nargs=0
-_shtab_ahriman_package_add___without_dependencies_nargs=0
 _shtab_ahriman_add_pos_0_nargs=+
 _shtab_ahriman_add__h_nargs=0
 _shtab_ahriman_add___help_nargs=0
+_shtab_ahriman_add___dependencies_nargs=0
+_shtab_ahriman_add___no_dependencies_nargs=0
 _shtab_ahriman_add__e_nargs=0
 _shtab_ahriman_add___exit_code_nargs=0
 _shtab_ahriman_add__n_nargs=0
 _shtab_ahriman_add___now_nargs=0
 _shtab_ahriman_add__y_nargs=0
 _shtab_ahriman_add___refresh_nargs=0
-_shtab_ahriman_add___without_dependencies_nargs=0
 _shtab_ahriman_package_update_pos_0_nargs=+
 _shtab_ahriman_package_update__h_nargs=0
 _shtab_ahriman_package_update___help_nargs=0
+_shtab_ahriman_package_update___dependencies_nargs=0
+_shtab_ahriman_package_update___no_dependencies_nargs=0
 _shtab_ahriman_package_update__e_nargs=0
 _shtab_ahriman_package_update___exit_code_nargs=0
 _shtab_ahriman_package_update__n_nargs=0
 _shtab_ahriman_package_update___now_nargs=0
 _shtab_ahriman_package_update__y_nargs=0
 _shtab_ahriman_package_update___refresh_nargs=0
-_shtab_ahriman_package_update___without_dependencies_nargs=0
 _shtab_ahriman_package_remove_pos_0_nargs=+
 _shtab_ahriman_package_remove__h_nargs=0
 _shtab_ahriman_package_remove___help_nargs=0
@@ -231,6 +234,8 @@ _shtab_ahriman_repo_daemon__h_nargs=0
 _shtab_ahriman_repo_daemon___help_nargs=0
 _shtab_ahriman_repo_daemon___aur_nargs=0
 _shtab_ahriman_repo_daemon___no_aur_nargs=0
+_shtab_ahriman_repo_daemon___dependencies_nargs=0
+_shtab_ahriman_repo_daemon___no_dependencies_nargs=0
 _shtab_ahriman_repo_daemon___local_nargs=0
 _shtab_ahriman_repo_daemon___no_local_nargs=0
 _shtab_ahriman_repo_daemon___manual_nargs=0
@@ -243,6 +248,8 @@ _shtab_ahriman_daemon__h_nargs=0
 _shtab_ahriman_daemon___help_nargs=0
 _shtab_ahriman_daemon___aur_nargs=0
 _shtab_ahriman_daemon___no_aur_nargs=0
+_shtab_ahriman_daemon___dependencies_nargs=0
+_shtab_ahriman_daemon___no_dependencies_nargs=0
 _shtab_ahriman_daemon___local_nargs=0
 _shtab_ahriman_daemon___no_local_nargs=0
 _shtab_ahriman_daemon___manual_nargs=0
@@ -295,11 +302,13 @@ _shtab_ahriman_repo_triggers___help_nargs=0
 _shtab_ahriman_repo_update_pos_0_nargs=*
 _shtab_ahriman_repo_update__h_nargs=0
 _shtab_ahriman_repo_update___help_nargs=0
+_shtab_ahriman_repo_update___aur_nargs=0
+_shtab_ahriman_repo_update___no_aur_nargs=0
+_shtab_ahriman_repo_update___dependencies_nargs=0
+_shtab_ahriman_repo_update___no_dependencies_nargs=0
 _shtab_ahriman_repo_update___dry_run_nargs=0
 _shtab_ahriman_repo_update__e_nargs=0
 _shtab_ahriman_repo_update___exit_code_nargs=0
-_shtab_ahriman_repo_update___aur_nargs=0
-_shtab_ahriman_repo_update___no_aur_nargs=0
 _shtab_ahriman_repo_update___local_nargs=0
 _shtab_ahriman_repo_update___no_local_nargs=0
 _shtab_ahriman_repo_update___manual_nargs=0
@@ -311,11 +320,13 @@ _shtab_ahriman_repo_update___refresh_nargs=0
 _shtab_ahriman_update_pos_0_nargs=*
 _shtab_ahriman_update__h_nargs=0
 _shtab_ahriman_update___help_nargs=0
+_shtab_ahriman_update___aur_nargs=0
+_shtab_ahriman_update___no_aur_nargs=0
+_shtab_ahriman_update___dependencies_nargs=0
+_shtab_ahriman_update___no_dependencies_nargs=0
 _shtab_ahriman_update___dry_run_nargs=0
 _shtab_ahriman_update__e_nargs=0
 _shtab_ahriman_update___exit_code_nargs=0
-_shtab_ahriman_update___aur_nargs=0
-_shtab_ahriman_update___no_aur_nargs=0
 _shtab_ahriman_update___local_nargs=0
 _shtab_ahriman_update___no_local_nargs=0
 _shtab_ahriman_update___manual_nargs=0
@@ -362,10 +373,16 @@ _shtab_ahriman_repo_clean___pacman_nargs=0
 _shtab_ahriman_repo_clean___no_pacman_nargs=0
 _shtab_ahriman_service_config__h_nargs=0
 _shtab_ahriman_service_config___help_nargs=0
+_shtab_ahriman_service_config___secure_nargs=0
+_shtab_ahriman_service_config___no_secure_nargs=0
 _shtab_ahriman_config__h_nargs=0
 _shtab_ahriman_config___help_nargs=0
+_shtab_ahriman_config___secure_nargs=0
+_shtab_ahriman_config___no_secure_nargs=0
 _shtab_ahriman_repo_config__h_nargs=0
 _shtab_ahriman_repo_config___help_nargs=0
+_shtab_ahriman_repo_config___secure_nargs=0
+_shtab_ahriman_repo_config___no_secure_nargs=0
 _shtab_ahriman_service_config_validate__h_nargs=0
 _shtab_ahriman_service_config_validate___help_nargs=0
 _shtab_ahriman_service_config_validate__e_nargs=0

docs/completions/zsh/_ahriman

@@ -87,11 +87,11 @@ _shtab_ahriman_options=(
 
 _shtab_ahriman_add_options=(
   "(- : *)"{-h,--help}"[show this help message and exit]"
+  {--dependencies,--no-dependencies}"[process missing package dependencies (default\: \%(default)s)]:dependencies:"
   {-e,--exit-code}"[return non-zero exit status if result is empty]"
   {-n,--now}"[run update function after]"
   "*"{-y,--refresh}"[download fresh package databases from the mirror before actions, -yy to force refresh even if up to date]"
   {-s,--source}"[explicitly specify the package source for this command]:source:(auto archive aur directory local remote repository)"
-  "--without-dependencies[do not add dependencies]"
   "(*):package source (base name, path to local files, remote URL):"
 )
 
@@ -99,7 +99,7 @@ _shtab_ahriman_aur_search_options=(
   "(- : *)"{-h,--help}"[show this help message and exit]"
   {-e,--exit-code}"[return non-zero exit status if result is empty]"
   {--info,--no-info}"[show additional package information (default\: \%(default)s)]:info:"
-  "--sort-by[sort field by this field. In case if two packages have the same value of the specified field, they will be always sorted by name]:sort_by:(description first_submitted id last_modified maintainer name num_votes out_of_date package_base package_base_id popularity repository url url_path version)"
+  "--sort-by[sort field by this field. In case if two packages have the same value of the specified field, they will be always sorted by name]:sort_by:(description first_submitted id last_modified maintainer name num_votes out_of_date package_base package_base_id popularity repository submitter url url_path version)"
   "(*):search terms, can be specified multiple times, the result will match all terms:"
 )
 
@@ -122,6 +122,7 @@ _shtab_ahriman_clean_options=(
 
 _shtab_ahriman_config_options=(
   "(- : *)"{-h,--help}"[show this help message and exit]"
+  {--secure,--no-secure}"[hide passwords and secrets from output (default\: \%(default)s)]:secure:"
 )
 
 _shtab_ahriman_config_validate_options=(
@@ -133,6 +134,7 @@ _shtab_ahriman_daemon_options=(
   "(- : *)"{-h,--help}"[show this help message and exit]"
   {-i,--interval}"[interval between runs in seconds]:interval:"
   {--aur,--no-aur}"[enable or disable checking for AUR updates (default\: \%(default)s)]:aur:"
+  {--dependencies,--no-dependencies}"[process missing package dependencies (default\: \%(default)s)]:dependencies:"
   {--local,--no-local}"[enable or disable checking of local packages for updates (default\: \%(default)s)]:local:"
   {--manual,--no-manual}"[include or exclude manual updates (default\: \%(default)s)]:manual:"
   {--vcs,--no-vcs}"[fetch actual version of VCS packages (default\: \%(default)s)]:vcs:"
@@ -182,11 +184,11 @@ _shtab_ahriman_key_import_options=(
 
 _shtab_ahriman_package_add_options=(
   "(- : *)"{-h,--help}"[show this help message and exit]"
+  {--dependencies,--no-dependencies}"[process missing package dependencies (default\: \%(default)s)]:dependencies:"
   {-e,--exit-code}"[return non-zero exit status if result is empty]"
   {-n,--now}"[run update function after]"
   "*"{-y,--refresh}"[download fresh package databases from the mirror before actions, -yy to force refresh even if up to date]"
   {-s,--source}"[explicitly specify the package source for this command]:source:(auto archive aur directory local remote repository)"
-  "--without-dependencies[do not add dependencies]"
   "(*):package source (base name, path to local files, remote URL):"
 )
 
@@ -217,11 +219,11 @@ _shtab_ahriman_package_status_update_options=(
 
 _shtab_ahriman_package_update_options=(
   "(- : *)"{-h,--help}"[show this help message and exit]"
+  {--dependencies,--no-dependencies}"[process missing package dependencies (default\: \%(default)s)]:dependencies:"
   {-e,--exit-code}"[return non-zero exit status if result is empty]"
   {-n,--now}"[run update function after]"
   "*"{-y,--refresh}"[download fresh package databases from the mirror before actions, -yy to force refresh even if up to date]"
   {-s,--source}"[explicitly specify the package source for this command]:source:(auto archive aur directory local remote repository)"
-  "--without-dependencies[do not add dependencies]"
   "(*):package source (base name, path to local files, remote URL):"
 )
 
@@ -293,6 +295,7 @@ _shtab_ahriman_repo_clean_options=(
 
 _shtab_ahriman_repo_config_options=(
   "(- : *)"{-h,--help}"[show this help message and exit]"
+  {--secure,--no-secure}"[hide passwords and secrets from output (default\: \%(default)s)]:secure:"
 )
 
 _shtab_ahriman_repo_config_validate_options=(
@@ -304,6 +307,7 @@ _shtab_ahriman_repo_daemon_options=(
   "(- : *)"{-h,--help}"[show this help message and exit]"
   {-i,--interval}"[interval between runs in seconds]:interval:"
   {--aur,--no-aur}"[enable or disable checking for AUR updates (default\: \%(default)s)]:aur:"
+  {--dependencies,--no-dependencies}"[process missing package dependencies (default\: \%(default)s)]:dependencies:"
   {--local,--no-local}"[enable or disable checking of local packages for updates (default\: \%(default)s)]:local:"
   {--manual,--no-manual}"[include or exclude manual updates (default\: \%(default)s)]:manual:"
   {--vcs,--no-vcs}"[fetch actual version of VCS packages (default\: \%(default)s)]:vcs:"
@@ -390,9 +394,10 @@ _shtab_ahriman_repo_triggers_options=(
 
 _shtab_ahriman_repo_update_options=(
   "(- : *)"{-h,--help}"[show this help message and exit]"
+  {--aur,--no-aur}"[enable or disable checking for AUR updates (default\: \%(default)s)]:aur:"
+  {--dependencies,--no-dependencies}"[process missing package dependencies (default\: \%(default)s)]:dependencies:"
   "--dry-run[just perform check for updates, same as check command]"
   {-e,--exit-code}"[return non-zero exit status if result is empty]"
-  {--aur,--no-aur}"[enable or disable checking for AUR updates (default\: \%(default)s)]:aur:"
   {--local,--no-local}"[enable or disable checking of local packages for updates (default\: \%(default)s)]:local:"
   {--manual,--no-manual}"[include or exclude manual updates (default\: \%(default)s)]:manual:"
   {--vcs,--no-vcs}"[fetch actual version of VCS packages (default\: \%(default)s)]:vcs:"
@@ -408,7 +413,7 @@ _shtab_ahriman_search_options=(
   "(- : *)"{-h,--help}"[show this help message and exit]"
   {-e,--exit-code}"[return non-zero exit status if result is empty]"
   {--info,--no-info}"[show additional package information (default\: \%(default)s)]:info:"
-  "--sort-by[sort field by this field. In case if two packages have the same value of the specified field, they will be always sorted by name]:sort_by:(description first_submitted id last_modified maintainer name num_votes out_of_date package_base package_base_id popularity repository url url_path version)"
+  "--sort-by[sort field by this field. In case if two packages have the same value of the specified field, they will be always sorted by name]:sort_by:(description first_submitted id last_modified maintainer name num_votes out_of_date package_base package_base_id popularity repository submitter url url_path version)"
   "(*):search terms, can be specified multiple times, the result will match all terms:"
 )
 
@@ -423,6 +428,7 @@ _shtab_ahriman_service_clean_options=(
 
 _shtab_ahriman_service_config_options=(
   "(- : *)"{-h,--help}"[show this help message and exit]"
+  {--secure,--no-secure}"[hide passwords and secrets from output (default\: \%(default)s)]:secure:"
 )
 
 _shtab_ahriman_service_config_validate_options=(
@@ -504,9 +510,10 @@ _shtab_ahriman_sync_options=(
 
 _shtab_ahriman_update_options=(
   "(- : *)"{-h,--help}"[show this help message and exit]"
+  {--aur,--no-aur}"[enable or disable checking for AUR updates (default\: \%(default)s)]:aur:"
+  {--dependencies,--no-dependencies}"[process missing package dependencies (default\: \%(default)s)]:dependencies:"
   "--dry-run[just perform check for updates, same as check command]"
   {-e,--exit-code}"[return non-zero exit status if result is empty]"
-  {--aur,--no-aur}"[enable or disable checking for AUR updates (default\: \%(default)s)]:aur:"
   {--local,--no-local}"[enable or disable checking of local packages for updates (default\: \%(default)s)]:local:"
   {--manual,--no-manual}"[include or exclude manual updates (default\: \%(default)s)]:manual:"
   {--vcs,--no-vcs}"[fetch actual version of VCS packages (default\: \%(default)s)]:vcs:"

docs/configuration.rst

@@ -28,6 +28,7 @@ Base configuration settings.
 * ``include`` - path to directory with configuration files overrides, string, required.
 * ``database`` - path to SQLite database, string, required.
 * ``logging`` - path to logging configuration, string, required. Check ``logging.ini`` for reference.
+* ``suppress_http_log_errors`` - suppress http log errors, boolean, optional, default ``no``. If set to ``yes``, any http log errors (e.g. if web server is not available, but http logging is enabled) will be suppressed.
 
 ``alpm`` group
 --------------
@@ -49,6 +50,7 @@ Base authorization settings. ``OAuth`` provider requires ``aioauth-client`` libr
 * ``allow_read_only`` - allow requesting status APIs without authorization, boolean, required.
 * ``client_id`` - OAuth2 application client ID, string, required in case if ``oauth`` is used.
 * ``client_secret`` - OAuth2 application client secret key, string, required in case if ``oauth`` is used.
+* ``cookie_secret_key`` - secret key which will be used for cookies encryption, string, optional. It must be 32 url-safe base64-encoded bytes and can be generated as following ``base64.urlsafe_b64encode(os.urandom(32)).decode("utf8")``. If not set, it will be generated automatically; note, however, that in this case, all sessions will be automatically expired during restart.
 * ``max_age`` - parameter which controls both cookie expiration and token expiration inside the service, integer, optional, default is 7 days.
 * ``oauth_provider`` - OAuth2 provider class name as is in ``aioauth-client`` (e.g. ``GoogleClient``, ``GithubClient`` etc), string, required in case if ``oauth`` is used.
 * ``oauth_scopes`` - scopes list for OAuth2 provider, which will allow retrieving user email (which is used for checking user permissions), e.g. ``https://www.googleapis.com/auth/userinfo.email`` for ``GoogleClient`` or ``user:email`` for ``GithubClient``, space separated list of strings, required in case if ``oauth`` is used.
@@ -67,7 +69,7 @@ Build related configuration. Group name can refer to architecture, e.g. ``build:
 * ``makepkg_flags`` - additional flags passed to ``makepkg`` command, space separated list of strings, optional.
 * ``makechrootpkg_flags`` - additional flags passed to ``makechrootpkg`` command, space separated list of strings, optional.
 * ``triggers`` - list of ``ahriman.core.triggers.Trigger`` class implementation (e.g. ``ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger``) which will be loaded and run at the end of processing, space separated list of strings, optional. You can also specify triggers by their paths, e.g. ``/usr/lib/python3.10/site-packages/ahriman/core/report/report.py.ReportTrigger``. Triggers are run in the order of mention.
-* ``vcs_allowed_age`` - maximal age in seconds of the VCS packages before their version will be updated with its remote source, int, optional, default ``0``.
+* ``vcs_allowed_age`` - maximal age in seconds of the VCS packages before their version will be updated with its remote source, int, optional, default ``604800``.
 
 ``repository`` group
 --------------------

docs/faq.rst

@@ -878,6 +878,11 @@ How to enable OAuth authorization
 #.
    Restart web service ``systemctl restart ahriman-web@x86_64``.
 
+How to implement own interface
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+You can write your own interface by using API which is provided by the web service. Full autogenerated API documentation is available at ``http://localhost:8080/api-docs``.
+
 Backup and restore
 ------------------
 

docs/setup.rst

@@ -88,4 +88,6 @@ Initial setup
 
    .. code-block:: shell
 
-       sudo -u ahriman ahriman -a x86_64 package-add ahriman --now
+       sudo -u ahriman ahriman -a x86_64 package-add ahriman --now --refresh
+
+   The ``--refresh`` flag is required in order to handle local database update.

package/archlinux/PKGBUILD

@@ -1,7 +1,7 @@
 # Maintainer: Evgeniy Alekseev
 
 pkgname='ahriman'
-pkgver=2.6.1
+pkgver=2.8.0
 pkgrel=1
 pkgdesc="ArcH linux ReposItory MANager"
 arch=('any')
@@ -14,6 +14,8 @@ optdepends=('breezy: -bzr packages support'
             'mercurial: -hg packages support'
             'python-aioauth-client: web server with OAuth2 authorization'
             'python-aiohttp: web server'
+            'python-aiohttp-apispec>=3.0.0: web server'
+            'python-aiohttp-cors: web server'
             'python-aiohttp-debugtoolbar: web server with enabled debug panel'
             'python-aiohttp-jinja2: web server'
             'python-aiohttp-security: web server with authorization'

package/share/ahriman/settings/ahriman.ini

@@ -2,6 +2,7 @@
 include = ahriman.ini.d
 logging = ahriman.ini.d/logging.ini
 database = /var/lib/ahriman/ahriman.db
+suppress_http_log_errors = yes
 
 [alpm]
 database = /var/lib/pacman

package/share/ahriman/templates/api.jinja2

@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>ahriman API</title>
+
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <!-- Embed elements Elements via Web Component -->
+    <script src="https://unpkg.com/@stoplight/elements/web-components.min.js"></script>
+    <link rel="stylesheet" href="https://unpkg.com/@stoplight/elements/styles.min.css" type="text/css">
+
+    <link rel="shortcut icon" href="/static/favicon.ico">
+</head>
+<body>
+
+    <elements-api
+            apiDescriptionUrl="/api-docs/swagger.json"
+            router="hash"
+            layout="sidebar"
+    />
+
+</body>
+</html>

setup.py

@@ -68,6 +68,7 @@ setup(
         ]),
         # templates
         ("share/ahriman/templates", [
+            "package/share/ahriman/templates/api.jinja2",
             "package/share/ahriman/templates/build-status.jinja2",
             "package/share/ahriman/templates/email-index.jinja2",
             "package/share/ahriman/templates/error.jinja2",
@@ -140,9 +141,11 @@ setup(
         ],
         "web": [
             "Jinja2",
-            "aiohttp",
-            "aiohttp_jinja2",
             "aioauth-client",
+            "aiohttp",
+            "aiohttp-apispec",
+            "aiohttp_cors",
+            "aiohttp_jinja2",
             "aiohttp_debugtoolbar",
             "aiohttp_session",
             "aiohttp_security",

src/ahriman/application/ahriman.py

@@ -245,6 +245,8 @@ def _set_package_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
                                     "5) and finally you can add package from AUR.",
                              formatter_class=_formatter)
     parser.add_argument("package", help="package source (base name, path to local files, remote URL)", nargs="+")
+    parser.add_argument("--dependencies", help="process missing package dependencies",
+                        action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
     parser.add_argument("-n", "--now", help="run update function after", action="store_true")
     parser.add_argument("-y", "--refresh", help="download fresh package databases from the mirror before actions, "
@@ -252,7 +254,6 @@ def _set_package_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
                         action="count", default=False)
     parser.add_argument("-s", "--source", help="explicitly specify the package source for this command",
                         type=PackageSource, choices=enum_values(PackageSource), default=PackageSource.Auto)
-    parser.add_argument("--without-dependencies", help="do not add dependencies", action="store_true")
     parser.set_defaults(handler=handlers.Add)
     return parser
 
@@ -472,7 +473,7 @@ def _set_repo_check_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser.add_argument("-y", "--refresh", help="download fresh package databases from the mirror before actions, "
                                                 "-yy to force refresh even if up to date",
                         action="count", default=False)
-    parser.set_defaults(handler=handlers.Update, dry_run=True, aur=True, local=True, manual=False)
+    parser.set_defaults(handler=handlers.Update, dependencies=False, dry_run=True, aur=True, local=True, manual=False)
     return parser
 
 
@@ -492,6 +493,8 @@ def _set_repo_daemon_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser.add_argument("-i", "--interval", help="interval between runs in seconds", type=int, default=60 * 60 * 12)
     parser.add_argument("--aur", help="enable or disable checking for AUR updates",
                         action=argparse.BooleanOptionalAction, default=True)
+    parser.add_argument("--dependencies", help="process missing package dependencies",
+                        action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("--local", help="enable or disable checking of local packages for updates",
                         action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("--manual", help="include or exclude manual updates",
@@ -691,10 +694,12 @@ def _set_repo_update_parser(root: SubParserAction) -> argparse.ArgumentParser:
                              description="check for packages updates and run build process if requested",
                              formatter_class=_formatter)
     parser.add_argument("package", help="filter check by package base", nargs="*")
-    parser.add_argument("--dry-run", help="just perform check for updates, same as check command", action="store_true")
-    parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
     parser.add_argument("--aur", help="enable or disable checking for AUR updates",
                         action=argparse.BooleanOptionalAction, default=True)
+    parser.add_argument("--dependencies", help="process missing package dependencies",
+                        action=argparse.BooleanOptionalAction, default=True)
+    parser.add_argument("--dry-run", help="just perform check for updates, same as check command", action="store_true")
+    parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
     parser.add_argument("--local", help="enable or disable checking of local packages for updates",
                         action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("--manual", help="include or exclude manual updates",
@@ -750,6 +755,8 @@ def _set_service_config_parser(root: SubParserAction) -> argparse.ArgumentParser
     parser = root.add_parser("service-config", aliases=["config", "repo-config"], help="dump configuration",
                              description="dump configuration for the specified architecture",
                              formatter_class=_formatter)
+    parser.add_argument("--secure", help="hide passwords and secrets from output",
+                        action=argparse.BooleanOptionalAction, default=True)
     parser.set_defaults(handler=handlers.Dump, lock=None, report=False, quiet=True, unsafe=True)
     return parser
 

src/ahriman/application/application/application.py

@@ -17,10 +17,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Set
+from typing import Iterable, List, Set
 
 from ahriman.application.application.application_packages import ApplicationPackages
 from ahriman.application.application.application_repository import ApplicationRepository
+from ahriman.models.package import Package
 from ahriman.models.result import Result
 
 
@@ -87,3 +88,39 @@ class Application(ApplicationPackages, ApplicationRepository):
         directly as it will be called after on_start action
         """
         self.repository.triggers.on_stop()
+
+    def with_dependencies(self, packages: List[Package], *, process_dependencies: bool) -> List[Package]:
+        """
+        add missing dependencies to list of packages
+
+        Args:
+            packages(List[Package]): list of source packages of which dependencies have to be processed
+            process_dependencies(bool): if no set, dependencies will not be processed
+        """
+        def missing_dependencies(source: Iterable[Package]) -> Set[str]:
+            # build initial list of dependencies
+            result = set()
+            for package in source:
+                result.update(package.depends_build)
+
+            # remove ones which are already well-known
+            result = result.difference(known_packages)
+
+            # remove ones which are in this list already
+            for package in source:
+                result = result.difference(package.packages_full)
+
+            return result
+
+        if not process_dependencies or not packages:
+            return packages
+
+        known_packages = self._known_packages()
+        with_dependencies = {package.base: package for package in packages}
+
+        while missing := missing_dependencies(with_dependencies.values()):
+            for package_name in missing:
+                package = Package.from_aur(package_name, self.repository.pacman)
+                with_dependencies[package.base] = package
+
+        return list(with_dependencies.values())

src/ahriman/application/application/application_packages.py

@@ -21,8 +21,7 @@ import requests
 import shutil
 
 from pathlib import Path
-from tempfile import TemporaryDirectory
-from typing import Any, Iterable, Set
+from typing import Any, Iterable
 
 from ahriman.application.application.application_properties import ApplicationProperties
 from ahriman.core.build_tools.sources import Sources
@@ -48,24 +47,18 @@ class ApplicationPackages(ApplicationProperties):
         dst = self.repository.paths.packages / local_path.name
         shutil.copy(local_path, dst)
 
-    def _add_aur(self, source: str, known_packages: Set[str], without_dependencies: bool) -> None:
+    def _add_aur(self, source: str) -> None:
         """
         add package from AUR
 
         Args:
             source(str): package base name
-            known_packages(Set[str]): list of packages which are known by the service
-            without_dependencies(bool): if set, dependency check will be disabled
         """
         package = Package.from_aur(source, self.repository.pacman)
 
         self.database.build_queue_insert(package)
         self.database.remote_update(package)
 
-        with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (local_dir := Path(dir_name)):
-            Sources.load(local_dir, package, self.database.patches_get(package.base), self.repository.paths)
-            self._process_dependencies(local_dir, known_packages, without_dependencies)
-
     def _add_directory(self, source: str, *_: Any) -> None:
         """
         add packages from directory
@@ -77,25 +70,21 @@ class ApplicationPackages(ApplicationProperties):
         for full_path in filter(package_like, local_dir.iterdir()):
             self._add_archive(str(full_path))
 
-    def _add_local(self, source: str, known_packages: Set[str], without_dependencies: bool) -> None:
+    def _add_local(self, source: str) -> None:
         """
         add package from local PKGBUILDs
 
         Args:
             source(str): path to directory with local source files
-            known_packages(Set[str]): list of packages which are known by the service
-            without_dependencies(bool): if set, dependency check will be disabled
         """
         source_dir = Path(source)
-        package = Package.from_build(source_dir)
+        package = Package.from_build(source_dir, self.architecture)
         cache_dir = self.repository.paths.cache_for(package.base)
         shutil.copytree(source_dir, cache_dir)  # copy package to store in caches
         Sources.init(cache_dir)  # we need to run init command in directory where we do have permissions
 
         self.database.build_queue_insert(package)
 
-        self._process_dependencies(cache_dir, known_packages, without_dependencies)
-
     def _add_remote(self, source: str, *_: Any) -> None:
         """
         add package from remote sources (e.g. HTTP)
@@ -104,7 +93,8 @@ class ApplicationPackages(ApplicationProperties):
             source(str): remote URL of the package archive
         """
         dst = self.repository.paths.packages / Path(source).name  # URL is path, is not it?
-        response = requests.get(source, stream=True, timeout=None)  # timeout=None to suppress pylint warns
+        # timeout=None to suppress pylint warns. Also suppress bandit warnings
+        response = requests.get(source, stream=True, timeout=None)  # nosec
         response.raise_for_status()
 
         with dst.open("wb") as local_file:
@@ -121,50 +111,19 @@ class ApplicationPackages(ApplicationProperties):
         package = Package.from_official(source, self.repository.pacman)
         self.database.build_queue_insert(package)
         self.database.remote_update(package)
-        # repository packages must not depend on unknown packages, thus we are not going to process dependencies
 
-    def _known_packages(self) -> Set[str]:
-        """
-        load packages from repository and pacman repositories
-
-        Returns:
-            Set[str]: list of known packages
-
-        Raises:
-            NotImplementedError: not implemented method
-        """
-        raise NotImplementedError
-
-    def _process_dependencies(self, local_dir: Path, known_packages: Set[str], without_dependencies: bool) -> None:
-        """
-        process package dependencies
-
-        Args:
-            local_dir(Path): path to local package sources (i.e. cloned AUR repository)
-            known_packages(Set[str]): list of packages which are known by the service
-            without_dependencies(bool): if set, dependency check will be disabled
-        """
-        if without_dependencies:
-            return
-
-        dependencies = Package.dependencies(local_dir)
-        self.add(dependencies.difference(known_packages), PackageSource.AUR, without_dependencies)
-
-    def add(self, names: Iterable[str], source: PackageSource, without_dependencies: bool) -> None:
+    def add(self, names: Iterable[str], source: PackageSource) -> None:
         """
         add packages for the next build
 
         Args:
             names(Iterable[str]): list of package bases to add
             source(PackageSource): package source to add
-            without_dependencies(bool): if set, dependency check will be disabled
         """
-        known_packages = self._known_packages()  # speedup dependencies processing
-
         for name in names:
             resolved_source = source.resolve(name)
             fn = getattr(self, f"_add_{resolved_source.value}")
-            fn(name, known_packages, without_dependencies)
+            fn(name)
 
     def on_result(self, result: Result) -> None:
         """

src/ahriman/application/application/application_repository.py

@@ -111,7 +111,7 @@ class ApplicationRepository(ApplicationProperties):
 
         def unknown_local(probe: Package) -> List[str]:
             cache_dir = self.repository.paths.cache_for(probe.base)
-            local = Package.from_build(cache_dir)
+            local = Package.from_build(cache_dir, self.architecture)
             packages = set(probe.packages.keys()).difference(local.packages.keys())
             return list(packages)
 
@@ -145,7 +145,7 @@ class ApplicationRepository(ApplicationProperties):
         process_update(packages, build_result)
 
         # process manual packages
-        tree = Tree.resolve(updates, self.repository.paths, self.database)
+        tree = Tree.resolve(updates)
         for num, level in enumerate(tree):
             self.logger.info("processing level #%i %s", num, [package.base for package in level])
             build_result = self.repository.process_build(level)
@@ -183,7 +183,7 @@ class ApplicationRepository(ApplicationProperties):
         updated_packages = [package for _, package in sorted(updates.items())]
 
         # reorder updates according to the dependency tree
-        tree = Tree.resolve(updated_packages, self.repository.paths, self.database)
+        tree = Tree.resolve(updated_packages)
         for level in tree:
             for package in level:
                 UpdatePrinter(package, local_versions.get(package.base)).print(

src/ahriman/application/handlers/add.py

@@ -47,11 +47,12 @@ class Add(Handler):
         application = Application(architecture, configuration,
                                   report=report, unsafe=unsafe, refresh_pacman_database=args.refresh)
         application.on_start()
-        application.add(args.package, args.source, args.without_dependencies)
+        application.add(args.package, args.source)
         if not args.now:
             return
 
         packages = application.updates(args.package, aur=False, local=False, manual=True, vcs=False,
                                        log_fn=application.logger.info)
+        packages = application.with_dependencies(packages, process_dependencies=args.dependencies)
         result = application.update(packages)
         Add.check_if_empty(args.exit_code, result.is_empty)

src/ahriman/application/handlers/dump.py

@@ -48,4 +48,4 @@ class Dump(Handler):
         """
         dump = configuration.dump()
         for section, values in sorted(dump.items()):
-            ConfigurationPrinter(section, values).print(verbose=False, separator=" = ")
+            ConfigurationPrinter(section, values).print(verbose=not args.secure, separator=" = ")

src/ahriman/application/handlers/patch.py

@@ -58,7 +58,7 @@ class Patch(Handler):
             patch = Patch.patch_create_from_function(args.variable, args.patch)
             Patch.patch_set_create(application, args.package, patch)
         elif args.action == Action.Update and args.variable is None:
-            package_base, patch = Patch.patch_create_from_diff(args.package, args.track)
+            package_base, patch = Patch.patch_create_from_diff(args.package, architecture, args.track)
             Patch.patch_set_create(application, package_base, patch)
         elif args.action == Action.List:
             Patch.patch_set_list(application, args.package, args.variable, args.exit_code)
@@ -66,19 +66,20 @@ class Patch(Handler):
             Patch.patch_set_remove(application, args.package, args.variable)
 
     @staticmethod
-    def patch_create_from_diff(sources_dir: Path, track: List[str]) -> Tuple[str, PkgbuildPatch]:
+    def patch_create_from_diff(sources_dir: Path, architecture: str, track: List[str]) -> Tuple[str, PkgbuildPatch]:
         """
         create PKGBUILD plain diff patches from sources directory
 
         Args:
             sources_dir(Path): path to directory with the package sources
+            architecture(str): repository architecture
             track(List[str]): track files which match the glob before creating the patch
 
         Returns:
             Tuple[str, PkgbuildPatch]: package base and created PKGBUILD patch based on the diff from master HEAD
                 to current files
         """
-        package = Package.from_build(sources_dir)
+        package = Package.from_build(sources_dir, architecture)
         patch = Sources.patch_create(sources_dir, *track)
         return package.base, PkgbuildPatch(None, patch)
 

src/ahriman/application/handlers/remove_unknown.py

@@ -51,7 +51,7 @@ class RemoveUnknown(Handler):
 
         if args.dry_run:
             for package in sorted(unknown_packages):
-                StringPrinter(package).print(False)
+                StringPrinter(package).print(verbose=False)
             return
 
         application.remove(unknown_packages)

src/ahriman/application/handlers/search.py

@@ -40,7 +40,7 @@ class Search(Handler):
     """
 
     ALLOW_AUTO_ARCHITECTURE_RUN = False  # it should be called only as "no-architecture"
-    SORT_FIELDS = {field.name for field in fields(AURPackage) if field.default_factory is not list}
+    SORT_FIELDS = {field.name for field in fields(AURPackage) if field.default_factory is not list}  # type: ignore
 
     @classmethod
     def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, configuration: Configuration, *,
@@ -64,7 +64,7 @@ class Search(Handler):
         for packages_list in (official_packages_list, aur_packages_list):
             # keep sorting by packages source
             for package in Search.sort(packages_list, args.sort_by):
-                AurPrinter(package).print(args.info)
+                AurPrinter(package).print(verbose=args.info)
 
     @staticmethod
     def sort(packages: Iterable[AURPackage], sort_by: str) -> List[AURPackage]:

src/ahriman/application/handlers/status.py

@@ -53,7 +53,7 @@ class Status(Handler):
         client = Application(architecture, configuration, report=True, unsafe=unsafe).repository.reporter
         if args.ahriman:
             service_status = client.get_internal()
-            StatusPrinter(service_status.status).print(args.info)
+            StatusPrinter(service_status.status).print(verbose=args.info)
         if args.package:
             packages: Iterable[Tuple[Package, BuildStatus]] = sum(
                 (client.get(base) for base in args.package),
@@ -67,4 +67,4 @@ class Status(Handler):
         filter_fn: Callable[[Tuple[Package, BuildStatus]], bool] =\
             lambda item: args.status is None or item[1].status == args.status
         for package, package_status in sorted(filter(filter_fn, packages), key=comparator):
-            PackagePrinter(package, package_status).print(args.info)
+            PackagePrinter(package, package_status).print(verbose=args.info)

src/ahriman/application/handlers/structure.py

@@ -51,6 +51,6 @@ class Structure(Handler):
         application = Application(architecture, configuration, report=report, unsafe=unsafe)
         packages = application.repository.packages()
 
-        tree = Tree.resolve(packages, application.repository.paths, application.database)
+        tree = Tree.resolve(packages)
         for num, level in enumerate(tree):
             TreePrinter(num, level).print(verbose=True, separator=" ")

src/ahriman/application/handlers/update.py

@@ -53,6 +53,7 @@ class Update(Handler):
         if args.dry_run:
             return
 
+        packages = application.with_dependencies(packages, process_dependencies=args.dependencies)
         result = application.update(packages)
         Update.check_if_empty(args.exit_code, result.is_empty)
 

src/ahriman/application/handlers/validate.py

@@ -52,7 +52,7 @@ class Validate(Handler):
             unsafe(bool): if set no user check will be performed before path creation
         """
         schema = Validate.schema(architecture, configuration)
-        validator = Validator(instance=configuration, schema=schema)
+        validator = Validator(configuration=configuration, schema=schema)
 
         if validator.validate(configuration.dump()):
             return  # no errors found

src/ahriman/application/handlers/web.py

@@ -55,3 +55,7 @@ class Web(Handler):
 
         application = setup_service(architecture, configuration, spawner)
         run_server(application)
+
+        # terminate spawn process at the last
+        spawner.stop()
+        spawner.join()

src/ahriman/application/lock.py

@@ -68,7 +68,6 @@ class Lock(LazyLogging):
             configuration(Configuration): configuration instance
         """
         self.path = args.lock.with_stem(f"{args.lock.stem}_{architecture}") if args.lock is not None else None
-        print(self.path)
         self.force = args.force
         self.unsafe = args.unsafe
 

src/ahriman/core/alpm/pacman.py

@@ -25,6 +25,7 @@ from typing import Any, Callable, Generator, Set
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.log import LazyLogging
+from ahriman.core.util import trim_package
 from ahriman.models.repository_paths import RepositoryPaths
 
 
@@ -175,8 +176,10 @@ class Pacman(LazyLogging):
         result: Set[str] = set()
         for database in self.handle.get_syncdbs():
             for package in database.pkgcache:
-                result.add(package.name)  # package itself
-                result.update(package.provides)  # provides list for meta-packages
+                # package itself
+                result.add(package.name)
+                # provides list for meta-packages
+                result.update(trim_package(provides) for provides in package.provides)
 
         return result
 

src/ahriman/core/configuration/configuration.py

@@ -56,7 +56,7 @@ class Configuration(configparser.RawConfigParser):
         architecture according to the merge rules. Moreover, the architecture names will be removed from section names.
 
         In order to get current settings, the ``check_loaded`` method can be used. This method will raise an
-        ``InitializeException`` in case if configuration was not yet loaded::
+        ``InitializeError`` in case if configuration was not yet loaded::
 
             >>> path, architecture = configuration.check_loaded()
     """
@@ -165,7 +165,7 @@ class Configuration(configparser.RawConfigParser):
             Tuple[Path, str]: configuration root path and architecture if loaded
 
         Raises:
-            InitializeException: in case if architecture and/or path are not set
+            InitializeError: in case if architecture and/or path are not set
         """
         if self.path is None or self.architecture is None:
             raise InitializeError("Configuration path and/or architecture are not set")
@@ -213,9 +213,9 @@ class Configuration(configparser.RawConfigParser):
         if self.has_section(full_section):
             return full_section, section
         # okay lets just use section as type
-        if not self.has_section(section):
-            raise configparser.NoSectionError(section)
-        return section, section
+        if self.has_section(section):
+            return section, section
+        raise configparser.NoSectionError(section)
 
     def load(self, path: Path) -> None:
         """

src/ahriman/core/configuration/schema.py

@@ -47,6 +47,10 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
                 "required": True,
                 "path_exists": True,
             },
+            "suppress_http_log_errors": {
+                "type": "boolean",
+                "coerce": "boolean",
+            }
         },
     },
     "alpm": {
@@ -60,6 +64,7 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
             "mirror": {
                 "type": "string",
                 "required": True,
+                "is_url": [],
             },
             "repositories": {
                 "type": "list",
@@ -105,9 +110,15 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
             "client_secret": {
                 "type": "string",
             },
+            "cookie_secret_key": {
+                "type": "string",
+                "minlength": 32,
+                "maxlength": 64,  # we cannot verify maxlength, because base64 representation might be longer than bytes
+            },
             "max_age": {
                 "type": "integer",
                 "coerce": "integer",
+                "min": 0,
             },
             "oauth_provider": {
                 "type": "string",
@@ -155,6 +166,7 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
             "vcs_allowed_age": {
                 "type": "integer",
                 "coerce": "integer",
+                "min": 0,
             },
         },
     },
@@ -197,6 +209,7 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
         "schema": {
             "address": {
                 "type": "string",
+                "is_url": ["http", "https"],
             },
             "debug": {
                 "type": "boolean",
@@ -213,9 +226,11 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
             },
             "host": {
                 "type": "string",
+                "is_ip_address": ["localhost"],
             },
             "index_url": {
                 "type": "string",
+                "is_url": ["http", "https"],
             },
             "password": {
                 "type": "string",
@@ -251,44 +266,4 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
             },
         },
     },
-    "remote-pull": {
-        "type": "dict",
-        "schema": {
-            "target": {
-                "type": "list",
-                "coerce": "list",
-                "schema": {"type": "string"},
-            },
-        },
-    },
-    "remote-push": {
-        "type": "dict",
-        "schema": {
-            "target": {
-                "type": "list",
-                "coerce": "list",
-                "schema": {"type": "string"},
-            },
-        },
-    },
-    "report": {
-        "type": "dict",
-        "schema": {
-            "target": {
-                "type": "list",
-                "coerce": "list",
-                "schema": {"type": "string"},
-            },
-        },
-    },
-    "upload": {
-        "type": "dict",
-        "schema": {
-            "target": {
-                "type": "list",
-                "coerce": "list",
-                "schema": {"type": "string"},
-            },
-        },
-    },
 }

src/ahriman/core/configuration/validator.py

@@ -17,19 +17,22 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import ipaddress
+
 from cerberus import TypeDefinition, Validator as RootValidator  # type: ignore
 from pathlib import Path
 from typing import Any, List
+from urllib.parse import urlparse
 
 from ahriman.core.configuration import Configuration
 
 
-class Validator(RootValidator):  # type: ignore
+class Validator(RootValidator):
     """
     class which defines custom validation methods for the service configuration
 
     Attributes:
-        instance(Configuration): configuration instance
+        configuration(Configuration): configuration instance
     """
 
     types_mapping = RootValidator.types_mapping.copy()
@@ -40,12 +43,12 @@ class Validator(RootValidator):  # type: ignore
         default constructor
 
         Args:
-            instance(Configuration): configuration instance used for extraction
+            configuration(Configuration): configuration instance used for extraction
             *args(Any): positional arguments to be passed to base validator
             **kwargs(): keyword arguments to be passed to base validator
         """
         RootValidator.__init__(self, *args, **kwargs)
-        self.instance: Configuration = kwargs["instance"]
+        self.configuration: Configuration = kwargs["configuration"]
 
     def _normalize_coerce_absolute_path(self, value: str) -> Path:
         """
@@ -57,7 +60,7 @@ class Validator(RootValidator):  # type: ignore
         Returns:
             Path: value converted to path instance according to configuration rules
         """
-        converted: Path = self.instance.converters["path"](value)
+        converted: Path = self.configuration.converters["path"](value)
         return converted
 
     def _normalize_coerce_boolean(self, value: str) -> bool:
@@ -71,7 +74,7 @@ class Validator(RootValidator):  # type: ignore
             bool: value converted to boolean according to configuration rules
         """
         # pylint: disable=protected-access
-        converted: bool = self.instance._convert_to_boolean(value)  # type: ignore
+        converted: bool = self.configuration._convert_to_boolean(value)  # type: ignore
         return converted
 
     def _normalize_coerce_integer(self, value: str) -> int:
@@ -97,9 +100,50 @@ class Validator(RootValidator):  # type: ignore
         Returns:
             List[str]: value converted to string list instance according to configuration rules
         """
-        converted: List[str] = self.instance.converters["list"](value)
+        converted: List[str] = self.configuration.converters["list"](value)
         return converted
 
+    def _validate_is_ip_address(self, constraint: List[str], field: str, value: str) -> None:
+        """
+        check if the specified value is valid ip address
+
+        Args:
+            constraint(List[str]): optional list of allowed special words (e.g. ``localhost``)
+            field(str): field name to be checked
+            value(Path): value to be checked
+
+        Examples:
+            The rule's arguments are validated against this schema:
+            {"type": "list", "schema": {"type": "string"}}
+        """
+        if value in constraint:
+            return
+        try:
+            ipaddress.ip_address(value)
+        except ValueError:
+            self._error(field, f"Value {value} must be valid IP address")
+
+    def _validate_is_url(self, constraint: List[str], field: str, value: str) -> None:
+        """
+        check if the specified value is a valid url
+
+        Args:
+            constraint(List[str]): optional list of supported schemas. If empty, no schema validation will be performed
+            field(str): field name to be checked
+            value(str): value to be checked
+
+        Examples:
+            The rule's arguments are validated against this schema:
+            {"type": "list", "schema": {"type": "string"}}
+        """
+        url = urlparse(value)  # it probably will never rise exceptions on parse
+        if not url.scheme:
+            self._error(field, f"Url scheme is not set for {value}")
+        if not url.netloc and url.scheme not in ("file",):
+            self._error(field, f"Location must be set for url {value} of scheme {url.scheme}")
+        if constraint and url.scheme not in constraint:
+            self._error(field, f"Url {value} scheme must be one of {constraint}")
+
     def _validate_path_exists(self, constraint: bool, field: str, value: Path) -> None:
         """
         check if paths exists

src/ahriman/core/database/data/__init__.py

@@ -1,47 +0,0 @@
-#
-# Copyright (c) 2021-2023 ahriman team.
-#
-# This file is part of ahriman
-# (see https://github.com/arcan1s/ahriman).
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from sqlite3 import Connection
-
-from ahriman.core.configuration import Configuration
-from ahriman.core.database.data.package_remotes import migrate_package_remotes
-from ahriman.core.database.data.package_statuses import migrate_package_statuses
-from ahriman.core.database.data.patches import migrate_patches
-from ahriman.core.database.data.users import migrate_users_data
-from ahriman.models.migration_result import MigrationResult
-
-
-def migrate_data(result: MigrationResult, connection: Connection, configuration: Configuration) -> None:
-    """
-    perform data migration
-
-    Args:
-        result(MigrationResult): result of the schema migration
-        connection(Connection): database connection
-        configuration(Configuration): configuration instance
-    """
-    # initial data migration
-    repository_paths = configuration.repository_paths
-
-    if result.old_version <= 0:
-        migrate_package_statuses(connection, repository_paths)
-        migrate_patches(connection, repository_paths)
-        migrate_users_data(connection, configuration)
-    if result.old_version <= 1:
-        migrate_package_remotes(connection, repository_paths)

src/ahriman/core/database/data/package_remotes.py

@@ -1,64 +0,0 @@
-#
-# Copyright (c) 2021-2023 ahriman team.
-#
-# This file is part of ahriman
-# (see https://github.com/arcan1s/ahriman).
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from sqlite3 import Connection
-
-from ahriman.models.package_source import PackageSource
-from ahriman.models.remote_source import RemoteSource
-from ahriman.models.repository_paths import RepositoryPaths
-
-
-__all__ = ["migrate_package_remotes"]
-
-
-# pylint: disable=protected-access
-def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> None:
-    """
-    perform migration for package remote sources
-
-    Args:
-        connection(Connection): database connection
-        paths(RepositoryPaths): repository paths instance
-    """
-    from ahriman.core.database.operations import PackageOperations
-
-    def insert_remote(base: str, remote: RemoteSource) -> None:
-        connection.execute(
-            """
-            update package_bases set
-            branch = :branch, git_url = :git_url, path = :path,
-            web_url = :web_url, source = :source
-            where package_base = :package_base
-            """,
-            dict(
-                package_base=base,
-                branch=remote.branch, git_url=remote.git_url, path=remote.path,
-                web_url=remote.web_url, source=remote.source
-            )
-        )
-
-    packages = PackageOperations._packages_get_select_package_bases(connection)
-    for package_base, package in packages.items():
-        local_cache = paths.cache_for(package_base)
-        if local_cache.exists() and not package.is_vcs:
-            continue  # skip packages which are not VCS and with local cache
-        remote_source = RemoteSource.from_source(PackageSource.AUR, package_base, "aur")
-        if remote_source is None:
-            continue  # should never happen
-        insert_remote(package_base, remote_source)

src/ahriman/core/database/data/package_statuses.py

@@ -1,82 +0,0 @@
-#
-# Copyright (c) 2021-2023 ahriman team.
-#
-# This file is part of ahriman
-# (see https://github.com/arcan1s/ahriman).
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-import json
-
-from sqlite3 import Connection
-
-from ahriman.models.build_status import BuildStatus
-from ahriman.models.package import Package
-from ahriman.models.repository_paths import RepositoryPaths
-
-
-__all__ = ["migrate_package_statuses"]
-
-
-def migrate_package_statuses(connection: Connection, paths: RepositoryPaths) -> None:
-    """
-    perform migration for package statuses
-
-    Args:
-        connection(Connection): database connection
-        paths(RepositoryPaths): repository paths instance
-    """
-    def insert_base(metadata: Package, last_status: BuildStatus) -> None:
-        connection.execute(
-            """
-            insert into package_bases
-            (package_base, version, aur_url)
-            values
-            (:package_base, :version, :aur_url)
-            """,
-            dict(package_base=metadata.base, version=metadata.version, aur_url=""))
-        connection.execute(
-            """
-            insert into package_statuses
-            (package_base, status, last_updated)
-            values
-            (:package_base, :status, :last_updated)""",
-            dict(package_base=metadata.base, status=last_status.status.value, last_updated=last_status.timestamp))
-
-    def insert_packages(metadata: Package) -> None:
-        package_list = []
-        for name, description in metadata.packages.items():
-            package_list.append(dict(package=name, package_base=metadata.base, **description.view()))
-        connection.executemany(
-            """
-            insert into packages
-            (package, package_base, architecture, archive_size, build_date, depends, description,
-            filename, "groups", installed_size, licenses, provides, url)
-            values
-            (:package, :package_base, :architecture, :archive_size, :build_date, :depends, :description,
-            :filename, :groups, :installed_size, :licenses, :provides, :url)
-            """,
-            package_list)
-
-    cache_path = paths.root / "status_cache.json"
-    if not cache_path.is_file():
-        return  # no file found
-    with cache_path.open() as cache:
-        dump = json.load(cache)
-
-    for item in dump.get("packages", []):
-        package = Package.from_json(item["package"])
-        status = BuildStatus.from_json(item["status"])
-        insert_base(package, status)
-        insert_packages(package)

src/ahriman/core/database/data/patches.py

@@ -1,47 +0,0 @@
-#
-# Copyright (c) 2021-2023 ahriman team.
-#
-# This file is part of ahriman
-# (see https://github.com/arcan1s/ahriman).
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from sqlite3 import Connection
-
-from ahriman.models.repository_paths import RepositoryPaths
-
-
-__all__ = ["migrate_patches"]
-
-
-def migrate_patches(connection: Connection, paths: RepositoryPaths) -> None:
-    """
-    perform migration for patches
-
-    Args:
-        connection(Connection): database connection
-        paths(RepositoryPaths): repository paths instance
-    """
-    root = paths.root / "patches"
-    if not root.is_dir():
-        return  # no directory found
-
-    for package in root.iterdir():
-        patch_path = package / "00-main.patch"
-        if not patch_path.is_file():
-            continue  # not exist
-        content = patch_path.read_text(encoding="utf8")
-        connection.execute(
-            """insert into patches (package_base, patch) values (:package_base, :patch)""",
-            {"package_base": package.name, "patch": content})

src/ahriman/core/database/migrations/__init__.py

@@ -17,16 +17,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from __future__ import annotations
-
 from importlib import import_module
 from pathlib import Path
 from pkgutil import iter_modules
-from sqlite3 import Connection
-from typing import List, Type
+from sqlite3 import Connection, Cursor
+from typing import Callable, List
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.database.data import migrate_data
 from ahriman.core.log import LazyLogging
 from ahriman.models.migration import Migration
 from ahriman.models.migration_result import MigrationResult
@@ -53,8 +50,8 @@ class Migrations(LazyLogging):
         self.connection = connection
         self.configuration = configuration
 
-    @classmethod
-    def migrate(cls: Type[Migrations], connection: Connection, configuration: Configuration) -> MigrationResult:
+    @staticmethod
+    def migrate(connection: Connection, configuration: Configuration) -> MigrationResult:
         """
         perform migrations implicitly
 
@@ -65,7 +62,26 @@ class Migrations(LazyLogging):
         Returns:
             MigrationResult: current schema version
         """
-        return cls(connection, configuration).run()
+        return Migrations(connection, configuration).run()
+
+    def migration(self, cursor: Cursor, migration: Migration) -> None:
+        """
+        perform single migration
+
+        Args:
+            cursor(Cursor): connection cursor
+            migration(Migration): single migration to perform
+        """
+        self.logger.info("applying table migration %s at index %s", migration.name, migration.index)
+        for statement in migration.steps:
+            cursor.execute(statement)
+        self.logger.info("table migration %s at index %s has been applied", migration.name, migration.index)
+
+        self.logger.info("perform data migration %s at index %s", migration.name, migration.index)
+        migration.migrate_data(self.connection, self.configuration)
+        self.logger.info(
+            "data migration %s at index %s has been performed",
+            migration.name, migration.index)
 
     def migrations(self) -> List[Migration]:
         """
@@ -81,9 +97,21 @@ class Migrations(LazyLogging):
 
         for index, module_name in enumerate(sorted(modules)):
             module = import_module(f"{__name__}.{module_name}")
+
             steps: List[str] = getattr(module, "steps", [])
             self.logger.debug("found migration %s at index %s with steps count %s", module_name, index, len(steps))
-            migrations.append(Migration(index=index, name=module_name, steps=steps))
+
+            migrate_data: Callable[[Connection, Configuration], None] = \
+                getattr(module, "migrate_data", lambda *args: None)
+
+            migrations.append(
+                Migration(
+                    index=index,
+                    name=module_name,
+                    steps=steps,
+                    migrate_data=migrate_data
+                )
+            )
 
         return migrations
 
@@ -110,13 +138,7 @@ class Migrations(LazyLogging):
             try:
                 cursor.execute("begin exclusive")
                 for migration in migrations[current_version:]:
-                    self.logger.info("applying migration %s at index %s", migration.name, migration.index)
-                    for statement in migration.steps:
-                        cursor.execute(statement)
-                    self.logger.info("migration %s at index %s has been applied", migration.name, migration.index)
-
-                migrate_data(result, self.connection, self.configuration)
-
+                    self.migration(cursor, migration)
                 cursor.execute(f"pragma user_version = {expected_version}")  # no support for ? placeholders
             except Exception:
                 self.logger.exception("migration failed with exception")

src/ahriman/core/database/migrations/m000_initial.py

@@ -17,7 +17,17 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-__all__ = ["steps"]
+import json
+
+from sqlite3 import Connection
+
+from ahriman.core.configuration import Configuration
+from ahriman.models.build_status import BuildStatus
+from ahriman.models.package import Package
+from ahriman.models.repository_paths import RepositoryPaths
+
+
+__all__ = ["migrate_data", "steps"]
 
 
 steps = [
@@ -73,3 +83,109 @@ steps = [
     )
     """,
 ]
+
+
+def migrate_data(connection: Connection, configuration: Configuration) -> None:
+    """
+    perform data migration
+
+    Args:
+        connection(Connection): database connection
+        configuration(Configuration): configuration instance
+    """
+    migrate_package_statuses(connection, configuration.repository_paths)
+    migrate_patches(connection, configuration.repository_paths)
+    migrate_users_data(connection, configuration)
+
+
+def migrate_package_statuses(connection: Connection, paths: RepositoryPaths) -> None:
+    """
+    perform migration for package statuses
+
+    Args:
+        connection(Connection): database connection
+        paths(RepositoryPaths): repository paths instance
+    """
+    def insert_base(metadata: Package, last_status: BuildStatus) -> None:
+        connection.execute(
+            """
+            insert into package_bases
+            (package_base, version, aur_url)
+            values
+            (:package_base, :version, :aur_url)
+            """,
+            {"package_base": metadata.base, "version": metadata.version, "aur_url": ""})
+        connection.execute(
+            """
+            insert into package_statuses
+            (package_base, status, last_updated)
+            values
+            (:package_base, :status, :last_updated)""",
+            {"package_base": metadata.base, "status": last_status.status.value, "last_updated": last_status.timestamp})
+
+    def insert_packages(metadata: Package) -> None:
+        package_list = []
+        for name, description in metadata.packages.items():
+            package_list.append({"package": name, "package_base": metadata.base, **description.view()})
+        connection.executemany(
+            """
+            insert into packages
+            (package, package_base, architecture, archive_size, build_date, depends, description,
+            filename, "groups", installed_size, licenses, provides, url)
+            values
+            (:package, :package_base, :architecture, :archive_size, :build_date, :depends, :description,
+            :filename, :groups, :installed_size, :licenses, :provides, :url)
+            """,
+            package_list)
+
+    cache_path = paths.root / "status_cache.json"
+    if not cache_path.is_file():
+        return  # no file found
+    with cache_path.open() as cache:
+        dump = json.load(cache)
+
+    for item in dump.get("packages", []):
+        package = Package.from_json(item["package"])
+        status = BuildStatus.from_json(item["status"])
+        insert_base(package, status)
+        insert_packages(package)
+
+
+def migrate_patches(connection: Connection, paths: RepositoryPaths) -> None:
+    """
+    perform migration for patches
+
+    Args:
+        connection(Connection): database connection
+        paths(RepositoryPaths): repository paths instance
+    """
+    root = paths.root / "patches"
+    if not root.is_dir():
+        return  # no directory found
+
+    for package in root.iterdir():
+        patch_path = package / "00-main.patch"
+        if not patch_path.is_file():
+            continue  # not exist
+        content = patch_path.read_text(encoding="utf8")
+        connection.execute(
+            """insert into patches (package_base, patch) values (:package_base, :patch)""",
+            {"package_base": package.name, "patch": content})
+
+
+def migrate_users_data(connection: Connection, configuration: Configuration) -> None:
+    """
+    perform migration for users
+
+    Args:
+        connection(Connection): database connection
+        configuration(Configuration): configuration instance
+    """
+    for section in configuration.sections():
+        for option, value in configuration[section].items():
+            if not section.startswith("auth:"):
+                continue
+            access = section[5:]
+            connection.execute(
+                """insert into users (username, access, password) values (:username, :access, :password)""",
+                {"username": option.lower(), "access": access, "password": value})

src/ahriman/core/database/migrations/m001_package_source.py

@@ -17,7 +17,15 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-__all__ = ["steps"]
+from sqlite3 import Connection
+
+from ahriman.core.configuration import Configuration
+from ahriman.models.package_source import PackageSource
+from ahriman.models.remote_source import RemoteSource
+from ahriman.models.repository_paths import RepositoryPaths
+
+
+__all__ = ["migrate_data", "steps"]
 
 
 steps = [
@@ -40,3 +48,51 @@ steps = [
     alter table package_bases drop column aur_url
     """,
 ]
+
+
+def migrate_data(connection: Connection, configuration: Configuration) -> None:
+    """
+    perform data migration
+
+    Args:
+        connection(Connection): database connection
+        configuration(Configuration): configuration instance
+    """
+    migrate_package_remotes(connection, configuration.repository_paths)
+
+
+# pylint: disable=protected-access
+def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> None:
+    """
+    perform migration for package remote sources
+
+    Args:
+        connection(Connection): database connection
+        paths(RepositoryPaths): repository paths instance
+    """
+    from ahriman.core.database.operations import PackageOperations
+
+    def insert_remote(base: str, remote: RemoteSource) -> None:
+        connection.execute(
+            """
+            update package_bases set
+            branch = :branch, git_url = :git_url, path = :path,
+            web_url = :web_url, source = :source
+            where package_base = :package_base
+            """,
+            {
+                "package_base": base,
+                "branch": remote.branch, "git_url": remote.git_url, "path": remote.path,
+                "web_url": remote.web_url, "source": remote.source
+            }
+        )
+
+    packages = PackageOperations._packages_get_select_package_bases(connection)
+    for package_base, package in packages.items():
+        local_cache = paths.cache_for(package_base)
+        if local_cache.exists() and not package.is_vcs:
+            continue  # skip packages which are not VCS and with local cache
+        remote_source = RemoteSource.from_source(PackageSource.AUR, package_base, "aur")
+        if remote_source is None:
+            continue  # should never happen
+        insert_remote(package_base, remote_source)

src/ahriman/core/database/migrations/m005_make_opt_depends.py

@@ -0,0 +1,83 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from sqlite3 import Connection
+
+from ahriman.core.alpm.pacman import Pacman
+from ahriman.core.configuration import Configuration
+from ahriman.core.util import package_like
+from ahriman.models.package import Package
+
+
+__all__ = ["migrate_data", "steps"]
+
+
+steps = [
+    """
+    alter table packages add column make_depends json
+    """,
+    """
+    alter table packages add column opt_depends json
+    """,
+]
+
+
+def migrate_data(connection: Connection, configuration: Configuration) -> None:
+    """
+    perform data migration
+
+    Args:
+        connection(Connection): database connection
+        configuration(Configuration): configuration instance
+    """
+    migrate_package_depends(connection, configuration)
+
+
+def migrate_package_depends(connection: Connection, configuration: Configuration) -> None:
+    """
+    migrate package opt and make depends fields
+
+    Args:
+        connection(Connection): database connection
+        configuration(Configuration): configuration instance
+    """
+    if not configuration.repository_paths.repository.is_dir():
+        return
+
+    _, architecture = configuration.check_loaded()
+    pacman = Pacman(architecture, configuration, refresh_database=False)
+
+    package_list = []
+    for full_path in filter(package_like, configuration.repository_paths.repository.iterdir()):
+        base = Package.from_archive(full_path, pacman, remote=None)
+        for package, description in base.packages.items():
+            package_list.append({
+                "make_depends": description.make_depends,
+                "opt_depends": description.opt_depends,
+                "package": package,
+            })
+
+    connection.executemany(
+        """
+        update packages set
+        make_depends = :make_depends, opt_depends = :opt_depends
+        where package = :package
+        """,
+        package_list
+    )

src/ahriman/core/database/migrations/m006_packages_architecture_required.py

@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+__all__ = ["steps"]
+
+
+steps = [
+    """
+    alter table packages rename to packages_
+    """,
+    """
+    create table packages (
+        package text not null,
+        package_base text not null,
+        architecture text not null,
+        archive_size integer,
+        build_date integer,
+        depends json,
+        description text,
+        filename text,
+        "groups" json,
+        installed_size integer,
+        licenses json,
+        provides json,
+        url text,
+        make_depends json,
+        opt_depends json,
+        unique (package, architecture)
+    )
+    """,
+    """
+    insert into packages select * from packages_ where architecture is not null;
+    """,
+    """
+    drop table packages_;
+    """,
+]

src/ahriman/core/database/operations/logs_operations.py

@@ -71,12 +71,12 @@ class LogsOperations(Operations):
                 values
                 (:package_base, :process_id, :created, :record)
                 """,
-                dict(
-                    package_base=log_record_id.package_base,
-                    process_id=log_record_id.process_id,
-                    created=created,
-                    record=record
-                )
+                {
+                    "package_base": log_record_id.package_base,
+                    "process_id": log_record_id.process_id,
+                    "created": created,
+                    "record": record,
+                }
             )
 
         return self.with_connection(run, commit=True)

src/ahriman/core/database/operations/operations.py

@@ -20,7 +20,7 @@
 import sqlite3
 
 from pathlib import Path
-from typing import Any, Dict, Tuple, TypeVar, Callable
+from typing import Any, Callable, Dict, Tuple, TypeVar
 
 from ahriman.core.log import LazyLogging
 

src/ahriman/core/database/operations/package_operations.py

@@ -82,15 +82,15 @@ class PackageOperations(Operations):
             on conflict (package_base) do update set
             version = :version, branch = :branch, git_url = :git_url, path = :path, web_url = :web_url, source = :source
             """,
-            dict(
-                package_base=package.base,
-                version=package.version,
-                branch=package.remote.branch if package.remote is not None else None,
-                git_url=package.remote.git_url if package.remote is not None else None,
-                path=package.remote.path if package.remote is not None else None,
-                web_url=package.remote.web_url if package.remote is not None else None,
-                source=package.remote.source.value if package.remote is not None else None,
-            )
+            {
+                "package_base": package.base,
+                "version": package.version,
+                "branch": package.remote.branch if package.remote is not None else None,
+                "git_url": package.remote.git_url if package.remote is not None else None,
+                "path": package.remote.path if package.remote is not None else None,
+                "web_url": package.remote.web_url if package.remote is not None else None,
+                "source": package.remote.source.value if package.remote is not None else None,
+            }
         )
 
     @staticmethod
@@ -104,21 +104,26 @@ class PackageOperations(Operations):
         """
         package_list = []
         for name, description in package.packages.items():
-            package_list.append(dict(package=name, package_base=package.base, **description.view()))
+            if description.architecture is None:
+                continue  # architecture is required
+            package_list.append({"package": name, "package_base": package.base, **description.view()})
         connection.executemany(
             """
             insert into packages
             (package, package_base, architecture, archive_size,
             build_date, depends, description, filename,
-            "groups", installed_size, licenses, provides, url)
+            "groups", installed_size, licenses, provides,
+            url, make_depends, opt_depends)
             values
             (:package, :package_base, :architecture, :archive_size,
             :build_date, :depends, :description, :filename,
-            :groups, :installed_size, :licenses, :provides, :url)
+            :groups, :installed_size, :licenses, :provides,
+            :url, :make_depends, :opt_depends)
             on conflict (package, architecture) do update set
             package_base = :package_base, archive_size = :archive_size,
             build_date = :build_date, depends = :depends, description = :description, filename = :filename,
-            "groups" = :groups, installed_size = :installed_size, licenses = :licenses, provides = :provides, url = :url
+            "groups" = :groups, installed_size = :installed_size, licenses = :licenses, provides = :provides,
+            url = :url, make_depends = :make_depends, opt_depends = :opt_depends
             """,
             package_list)
 
@@ -140,7 +145,7 @@ class PackageOperations(Operations):
             on conflict (package_base) do update set
             status = :status, last_updated = :last_updated
             """,
-            dict(package_base=package_base, status=status.status.value, last_updated=status.timestamp))
+            {"package_base": package_base, "status": status.status.value, "last_updated": status.timestamp})
 
     @staticmethod
     def _packages_get_select_package_bases(connection: Connection) -> Dict[str, Package]:

src/ahriman/core/formatters/__init__.py

@@ -27,7 +27,7 @@ from ahriman.core.formatters.package_printer import PackagePrinter
 from ahriman.core.formatters.patch_printer import PatchPrinter
 from ahriman.core.formatters.status_printer import StatusPrinter
 from ahriman.core.formatters.tree_printer import TreePrinter
-from ahriman.core.formatters.validation_printer import ValidationPrinter
 from ahriman.core.formatters.update_printer import UpdatePrinter
 from ahriman.core.formatters.user_printer import UserPrinter
+from ahriman.core.formatters.validation_printer import ValidationPrinter
 from ahriman.core.formatters.version_printer import VersionPrinter

src/ahriman/core/formatters/configuration_printer.py

@@ -28,9 +28,20 @@ class ConfigurationPrinter(StringPrinter):
     print content of the configuration section
 
     Attributes:
+        HIDE_KEYS(List[str]): (class attribute) hide values for mentioned keys. This list must be used in order to hide
+            passwords from outputs
         values(Dict[str, str]): configuration values dictionary
     """
 
+    HIDE_KEYS = [
+        "api_key",  # telegram key
+        "client_secret",  # oauth secret
+        "cookie_secret_key",  # cookie secret key
+        "password",  # generic password (github, email, web server, etc)
+        "salt",  # password default salt
+        "secret_key",  # aws secret key
+    ]
+
     def __init__(self, section: str, values: Dict[str, str]) -> None:
         """
         default constructor
@@ -50,6 +61,6 @@ class ConfigurationPrinter(StringPrinter):
             List[Property]: list of content properties
         """
         return [
-            Property(key, value, is_required=True)
+            Property(key, value, is_required=key not in self.HIDE_KEYS)
             for key, value in sorted(self.values.items())
         ]

src/ahriman/core/formatters/printer.py

@@ -27,13 +27,13 @@ class Printer:
     base class for formatters
     """
 
-    def print(self, verbose: bool, log_fn: Callable[[str], None] = print, separator: str = ": ") -> None:
+    def print(self, *, verbose: bool, log_fn: Callable[[str], None] = print, separator: str = ": ") -> None:
         """
         print content
 
         Args:
             verbose(bool): print all fields
-            log_fn(Callable[[str], None]): logger function to log data (Default value = print)
+            log_fn(Callable[[str], None], optional): logger function to log data (Default value = print)
             separator(str, optional): separator for property name and property value (Default value = ": ")
         """
         if (title := self.title()) is not None:

src/ahriman/core/gitremote/remote_pull_trigger.py

@@ -33,6 +33,16 @@ class RemotePullTrigger(Trigger):
     """
 
     CONFIGURATION_SCHEMA = {
+        "remote-pull": {
+            "type": "dict",
+            "schema": {
+                "target": {
+                    "type": "list",
+                    "coerce": "list",
+                    "schema": {"type": "string"},
+                },
+            },
+        },
         "gitremote": {
             "type": "dict",
             "schema": {

src/ahriman/core/gitremote/remote_push.py

@@ -82,7 +82,10 @@ class RemotePush(LazyLogging):
         Sources.fetch(package_target_dir, package.remote)
         # ...and last, but not least, we remove the dot-git directory...
         for git_file in package_target_dir.glob(".git*"):
-            shutil.rmtree(package_target_dir / git_file)
+            if git_file.is_file():
+                git_file.unlink()
+            else:
+                shutil.rmtree(git_file)
         # ...copy all patches...
         for patch in self.database.patches_get(package.base):
             filename = f"ahriman-{package.base}.patch" if patch.key is None else f"ahriman-{patch.key}.patch"

src/ahriman/core/gitremote/remote_push_trigger.py

@@ -38,6 +38,16 @@ class RemotePushTrigger(Trigger):
     """
 
     CONFIGURATION_SCHEMA = {
+        "remote-push": {
+            "type": "dict",
+            "schema": {
+                "target": {
+                    "type": "list",
+                    "coerce": "list",
+                    "schema": {"type": "string"},
+                },
+            },
+        },
         "gitremote": {
             "type": "dict",
             "schema": {

src/ahriman/core/log/http_log_handler.py

@@ -31,22 +31,25 @@ class HttpLogHandler(logging.Handler):
 
     Attributes:
         reporter(Client): build status reporter instance
+        suppress_errors(bool): suppress logging errors (e.g. if no web server available)
     """
 
-    def __init__(self, configuration: Configuration, *, report: bool) -> None:
+    def __init__(self, configuration: Configuration, *, report: bool, suppress_errors: bool) -> None:
         """
         default constructor
 
         Args:
             configuration(Configuration): configuration instance
             report(bool): force enable or disable reporting
+            suppress_errors(bool): suppress logging errors (e.g. if no web server available)
         """
         # we don't really care about those parameters because they will be handled by the reporter
         logging.Handler.__init__(self)
 
-        # client has to be importer here because of circular imports
+        # client has to be imported here because of circular imports
         from ahriman.core.status.client import Client
         self.reporter = Client.load(configuration, report=report)
+        self.suppress_errors = suppress_errors
 
     @classmethod
     def load(cls, configuration: Configuration, *, report: bool) -> HttpLogHandler:
@@ -62,7 +65,8 @@ class HttpLogHandler(logging.Handler):
         if (handler := next((handler for handler in root.handlers if isinstance(handler, cls)), None)) is not None:
             return handler  # there is already registered instance
 
-        handler = cls(configuration, report=report)
+        suppress_errors = configuration.getboolean("settings", "suppress_http_log_errors", fallback=False)
+        handler = cls(configuration, report=report, suppress_errors=suppress_errors)
         root.addHandler(handler)
 
         return handler
@@ -81,4 +85,6 @@ class HttpLogHandler(logging.Handler):
         try:
             self.reporter.logs(package_base, record)
         except Exception:
+            if self.suppress_errors:
+                return
             self.handleError(record)

src/ahriman/core/report/report_trigger.py

@@ -35,6 +35,16 @@ class ReportTrigger(Trigger):
     """
 
     CONFIGURATION_SCHEMA = {
+        "report": {
+            "type": "dict",
+            "schema": {
+                "target": {
+                    "type": "list",
+                    "coerce": "list",
+                    "schema": {"type": "string"},
+                },
+            },
+        },
         "console": {
             "type": "dict",
             "schema": {
@@ -62,6 +72,7 @@ class ReportTrigger(Trigger):
                 },
                 "homepage": {
                     "type": "string",
+                    "is_url": ["http", "https"],
                 },
                 "host": {
                     "type": "string",
@@ -70,6 +81,7 @@ class ReportTrigger(Trigger):
                 "link_path": {
                     "type": "string",
                     "required": True,
+                    "is_url": [],
                 },
                 "no_empty_report": {
                     "type": "boolean",
@@ -82,6 +94,8 @@ class ReportTrigger(Trigger):
                     "type": "integer",
                     "coerce": "integer",
                     "required": True,
+                    "min": 0,
+                    "max": 65535,
                 },
                 "receivers": {
                     "type": "list",
@@ -118,10 +132,12 @@ class ReportTrigger(Trigger):
                 },
                 "homepage": {
                     "type": "string",
+                    "is_url": ["http", "https"],
                 },
                 "link_path": {
                     "type": "string",
                     "required": True,
+                    "is_url": [],
                 },
                 "path": {
                     "type": "path",
@@ -153,10 +169,12 @@ class ReportTrigger(Trigger):
                 },
                 "homepage": {
                     "type": "string",
+                    "is_url": ["http", "https"],
                 },
                 "link_path": {
                     "type": "string",
                     "required": True,
+                    "is_url": [],
                 },
                 "template_path": {
                     "type": "path",
@@ -171,6 +189,7 @@ class ReportTrigger(Trigger):
                 "timeout": {
                     "type": "integer",
                     "coerce": "integer",
+                    "min": 0,
                 },
             },
         },

src/ahriman/core/repository/executor.py

@@ -21,7 +21,7 @@ import shutil
 
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import Iterable, List, Optional
+from typing import Dict, Iterable, List, Optional
 
 from ahriman.core.build_tools.task import Task
 from ahriman.core.repository.cleaner import Cleaner
@@ -122,26 +122,40 @@ class Executor(Cleaner):
             except Exception:
                 self.logger.exception("could not remove %s", package)
 
+        packages_to_remove: Dict[str, Path] = {}
+        bases_to_remove: List[str] = []
+
+        # build package list based on user input
         requested = set(packages)
         for local in self.packages():
             if local.base in packages or all(package in requested for package in local.packages):
-                to_remove = {
+                packages_to_remove.update({
                     package: properties.filepath
                     for package, properties in local.packages.items()
                     if properties.filepath is not None
-                }
-                remove_base(local.base)
+                })
+                bases_to_remove.append(local.base)
             elif requested.intersection(local.packages.keys()):
-                to_remove = {
+                packages_to_remove.update({
                     package: properties.filepath
                     for package, properties in local.packages.items()
                     if package in requested and properties.filepath is not None
-                }
-            else:
-                to_remove = {}
+                })
 
-            for package, filename in to_remove.items():
-                remove_package(package, filename)
+        # check for packages which were requested to remove, but weren't found locally
+        # it might happen for example, if there were no success build before
+        for unknown in requested:
+            if unknown in packages_to_remove or unknown in bases_to_remove:
+                continue
+            bases_to_remove.append(unknown)
+
+        # remove packages from repository files
+        for package, filename in packages_to_remove.items():
+            remove_package(package, filename)
+
+        # remove bases from registered
+        for package in bases_to_remove:
+            remove_base(package)
 
         return self.repo.repo_path
 

src/ahriman/core/repository/update_handler.py

@@ -98,7 +98,7 @@ class UpdateHandler(Cleaner):
             with self.in_package_context(cache_dir.name):
                 try:
                     Sources.fetch(cache_dir, remote=None)
-                    remote = Package.from_build(cache_dir)
+                    remote = Package.from_build(cache_dir, self.architecture)
 
                     local = packages.get(remote.base)
                     if local is None:

src/ahriman/core/spawn.py

@@ -60,7 +60,7 @@ class Spawn(Thread, LazyLogging):
         self.lock = Lock()
         self.active: Dict[str, Process] = {}
         # stupid pylint does not know that it is possible
-        self.queue: Queue[Tuple[str, bool]] = Queue()  # pylint: disable=unsubscriptable-object
+        self.queue: Queue[Tuple[str, bool] | None] = Queue()  # pylint: disable=unsubscriptable-object
 
     @staticmethod
     def process(callback: Callable[[argparse.Namespace, str], bool], args: argparse.Namespace, architecture: str,
@@ -78,55 +78,7 @@ class Spawn(Thread, LazyLogging):
         result = callback(args, architecture)
         queue.put((process_id, result))
 
-    def key_import(self, key: str, server: Optional[str]) -> None:
-        """
-        import key to service cache
-
-        Args:
-            key(str): key to import
-            server(str): PGP key server
-        """
-        kwargs = {} if server is None else {"key-server": server}
-        self.spawn_process("service-key-import", key, **kwargs)
-
-    def packages_add(self, packages: Iterable[str], *, now: bool) -> None:
-        """
-        add packages
-
-        Args:
-            packages(Iterable[str]): packages list to add
-            now(bool): build packages now
-        """
-        kwargs = {"source": PackageSource.AUR.value}  # avoid abusing by building non-aur packages
-        if now:
-            kwargs["now"] = ""
-        self.spawn_process("package-add", *packages, **kwargs)
-
-    def packages_rebuild(self, depends_on: str) -> None:
-        """
-        rebuild packages which depend on the specified package
-
-        Args:
-            depends_on(str): packages dependency
-        """
-        self.spawn_process("repo-rebuild", **{"depends-on": depends_on})
-
-    def packages_remove(self, packages: Iterable[str]) -> None:
-        """
-        remove packages
-
-        Args:
-            packages(Iterable[str]): packages list to remove
-        """
-        self.spawn_process("package-remove", *packages)
-
-    def packages_update(self, ) -> None:
-        """
-        run full repository update
-        """
-        self.spawn_process("repo-update")
-
-    def spawn_process(self, command: str, *args: str, **kwargs: str) -> None:
+    def _spawn_process(self, command: str, *args: str, **kwargs: str) -> None:
         """
         spawn external ahriman process with supplied arguments
 
@@ -161,6 +113,54 @@ class Spawn(Thread, LazyLogging):
         with self.lock:
             self.active[process_id] = process
 
+    def key_import(self, key: str, server: Optional[str]) -> None:
+        """
+        import key to service cache
+
+        Args:
+            key(str): key to import
+            server(str): PGP key server
+        """
+        kwargs = {} if server is None else {"key-server": server}
+        self._spawn_process("service-key-import", key, **kwargs)
+
+    def packages_add(self, packages: Iterable[str], *, now: bool) -> None:
+        """
+        add packages
+
+        Args:
+            packages(Iterable[str]): packages list to add
+            now(bool): build packages now
+        """
+        kwargs = {"source": PackageSource.AUR.value}  # avoid abusing by building non-aur packages
+        if now:
+            kwargs["now"] = ""
+        self._spawn_process("package-add", *packages, **kwargs)
+
+    def packages_rebuild(self, depends_on: str) -> None:
+        """
+        rebuild packages which depend on the specified package
+
+        Args:
+            depends_on(str): packages dependency
+        """
+        self._spawn_process("repo-rebuild", **{"depends-on": depends_on})
+
+    def packages_remove(self, packages: Iterable[str]) -> None:
+        """
+        remove packages
+
+        Args:
+            packages(Iterable[str]): packages list to remove
+        """
+        self._spawn_process("package-remove", *packages)
+
+    def packages_update(self, ) -> None:
+        """
+        run full repository update
+        """
+        self._spawn_process("repo-update")
+
     def run(self) -> None:
         """
         thread run method
@@ -174,3 +174,9 @@ class Spawn(Thread, LazyLogging):
             if process is not None:
                 process.terminate()  # make sure lol
                 process.join()
+
+    def stop(self) -> None:
+        """
+        gracefully terminate thread
+        """
+        self.queue.put(None)

src/ahriman/core/status/web_client.py

@@ -17,17 +17,18 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import contextlib
 import logging
 import requests
 
-from typing import List, Optional, Tuple
+from typing import Generator, List, Optional, Tuple
 from urllib.parse import quote_plus as urlencode
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.log import LazyLogging
 from ahriman.core.status.client import Client
 from ahriman.core.util import exception_response_text
-from ahriman.models.build_status import BuildStatusEnum, BuildStatus
+from ahriman.models.build_status import BuildStatus, BuildStatusEnum
 from ahriman.models.internal_status import InternalStatus
 from ahriman.models.package import Package
 from ahriman.models.user import User
@@ -98,6 +99,18 @@ class WebClient(Client, LazyLogging):
             address = f"http://{host}:{port}"
         return address, False
 
+    @contextlib.contextmanager
+    def __execute_request(self) -> Generator[None, None, None]:
+        """
+        execute request and handle exceptions
+        """
+        try:
+            yield
+        except requests.HTTPError as e:
+            self.logger.exception("could not perform http request: %s", exception_response_text(e))
+        except Exception:
+            self.logger.exception("could not perform http request")
+
     def _create_session(self, *, use_unix_socket: bool) -> requests.Session:
         """
         generate new request session
@@ -130,13 +143,9 @@ class WebClient(Client, LazyLogging):
             "password": self.user.password
         }
 
-        try:
+        with self.__execute_request():
             response = self.__session.post(self._login_url, json=payload)
             response.raise_for_status()
-        except requests.HTTPError as e:
-            self.logger.exception("could not login as %s: %s", self.user, exception_response_text(e))
-        except Exception:
-            self.logger.exception("could not login as %s", self.user)
 
     def _logs_url(self, package_base: str) -> str:
         """
@@ -177,13 +186,9 @@ class WebClient(Client, LazyLogging):
             "package": package.view()
         }
 
-        try:
+        with self.__execute_request():
             response = self.__session.post(self._package_url(package.base), json=payload)
             response.raise_for_status()
-        except requests.HTTPError as e:
-            self.logger.exception("could not add %s: %s", package.base, exception_response_text(e))
-        except Exception:
-            self.logger.exception("could not add %s", package.base)
 
     def get(self, package_base: Optional[str]) -> List[Tuple[Package, BuildStatus]]:
         """
@@ -195,7 +200,7 @@ class WebClient(Client, LazyLogging):
         Returns:
             List[Tuple[Package, BuildStatus]]: list of current package description and status if it has been found
         """
-        try:
+        with self.__execute_request():
             response = self.__session.get(self._package_url(package_base or ""))
             response.raise_for_status()
 
@@ -204,10 +209,8 @@ class WebClient(Client, LazyLogging):
                 (Package.from_json(package["package"]), BuildStatus.from_json(package["status"]))
                 for package in status_json
             ]
-        except requests.HTTPError as e:
-            self.logger.exception("could not get %s: %s", package_base, exception_response_text(e))
-        except Exception:
-            self.logger.exception("could not get %s", package_base)
+
+        # noinspection PyUnreachableCode
         return []
 
     def get_internal(self) -> InternalStatus:
@@ -217,16 +220,14 @@ class WebClient(Client, LazyLogging):
         Returns:
             InternalStatus: current internal (web) service status
         """
-        try:
+        with self.__execute_request():
             response = self.__session.get(self._status_url)
             response.raise_for_status()
 
             status_json = response.json()
             return InternalStatus.from_json(status_json)
-        except requests.HTTPError as e:
-            self.logger.exception("could not get web service status: %s", exception_response_text(e))
-        except Exception:
-            self.logger.exception("could not get web service status")
+
+        # noinspection PyUnreachableCode
         return InternalStatus(status=BuildStatus())
 
     def logs(self, package_base: str, record: logging.LogRecord) -> None:
@@ -254,13 +255,9 @@ class WebClient(Client, LazyLogging):
         Args:
             package_base(str): basename to remove
         """
-        try:
+        with self.__execute_request():
             response = self.__session.delete(self._package_url(package_base))
             response.raise_for_status()
-        except requests.HTTPError as e:
-            self.logger.exception("could not delete %s: %s", package_base, exception_response_text(e))
-        except Exception:
-            self.logger.exception("could not delete %s", package_base)
 
     def update(self, package_base: str, status: BuildStatusEnum) -> None:
         """
@@ -272,13 +269,9 @@ class WebClient(Client, LazyLogging):
         """
         payload = {"status": status.value}
 
-        try:
+        with self.__execute_request():
             response = self.__session.post(self._package_url(package_base), json=payload)
             response.raise_for_status()
-        except requests.HTTPError as e:
-            self.logger.exception("could not update %s: %s", package_base, exception_response_text(e))
-        except Exception:
-            self.logger.exception("could not update %s", package_base)
 
     def update_self(self, status: BuildStatusEnum) -> None:
         """
@@ -289,10 +282,6 @@ class WebClient(Client, LazyLogging):
         """
         payload = {"status": status.value}
 
-        try:
+        with self.__execute_request():
             response = self.__session.post(self._status_url, json=payload)
             response.raise_for_status()
-        except requests.HTTPError as e:
-            self.logger.exception("could not update service status: %s", exception_response_text(e))
-        except Exception:
-            self.logger.exception("could not update service status")

src/ahriman/core/tree.py

@@ -19,16 +19,12 @@
 #
 from __future__ import annotations
 
-import itertools
+import functools
 
-from pathlib import Path
-from tempfile import TemporaryDirectory
-from typing import Callable, Iterable, List, Set, Tuple, Type
+from typing import Callable, Iterable, List
 
-from ahriman.core.build_tools.sources import Sources
-from ahriman.core.database import SQLite
+from ahriman.core.util import partition
 from ahriman.models.package import Package
-from ahriman.models.repository_paths import RepositoryPaths
 
 
 class Leaf:
@@ -40,16 +36,15 @@ class Leaf:
         package(Package): leaf package properties
     """
 
-    def __init__(self, package: Package, dependencies: Set[str]) -> None:
+    def __init__(self, package: Package) -> None:
         """
         default constructor
 
         Args:
             package(Package): package properties
-            dependencies(Set[str]): package dependencies
         """
         self.package = package
-        self.dependencies = dependencies
+        self.dependencies = package.depends_build
 
     @property
     def items(self) -> Iterable[str]:
@@ -61,24 +56,6 @@ class Leaf:
         """
         return self.package.packages.keys()
 
-    @classmethod
-    def load(cls: Type[Leaf], package: Package, paths: RepositoryPaths, database: SQLite) -> Leaf:
-        """
-        load leaf from package with dependencies
-
-        Args:
-            package(Package): package properties
-            paths(RepositoryPaths): repository paths instance
-            database(SQLite): database instance
-
-        Returns:
-            Leaf: loaded class
-        """
-        with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (clone_dir := Path(dir_name)):
-            Sources.load(clone_dir, package, database.patches_get(package.base), paths)
-            dependencies = Package.dependencies(clone_dir)
-        return cls(package, dependencies)
-
     def is_dependency(self, packages: Iterable[Leaf]) -> bool:
         """
         check if the package is dependency of any other package from list or not
@@ -130,7 +107,7 @@ class Tree:
             >>> repository = Repository.load("x86_64", configuration, database, report=True, unsafe=False)
             >>> packages = repository.packages()
             >>>
-            >>> tree = Tree.resolve(packages, configuration.repository_paths, database)
+            >>> tree = Tree.resolve(packages)
             >>> for tree_level in tree:
             >>>     for package in tree_level:
             >>>         print(package.base)
@@ -138,14 +115,8 @@ class Tree:
 
         The direct constructor call is also possible but requires tree leaves to be instantioned in advance, e.g.::
 
-            >>> leaves = [Leaf.load(package, database) for package in packages]
+            >>> leaves = [Leaf(package) for package in packages]
             >>> tree = Tree(leaves)
-
-        Using the default ``Leaf()`` method is possible, but not really recommended because it requires from the user to
-        build the dependency list by himself::
-
-            >>> leaf = Leaf(package, dependecies)
-            >>> tree = Tree([leaf])
     """
 
     def __init__(self, leaves: List[Leaf]) -> None:
@@ -157,23 +128,20 @@ class Tree:
         """
         self.leaves = leaves
 
-    @classmethod
-    def resolve(cls: Type[Tree], packages: Iterable[Package], paths: RepositoryPaths,
-                database: SQLite) -> List[List[Package]]:
+    @staticmethod
+    def resolve(packages: Iterable[Package]) -> List[List[Package]]:
         """
         resolve dependency tree
 
         Args:
             packages(Iterable[Package]): packages list
-            paths(RepositoryPaths): repository paths instance
-            database(SQLite): database instance
 
         Returns:
             List[List[Package]]: list of packages lists based on their dependencies
         """
-        leaves = [Leaf.load(package, paths, database) for package in packages]
-        tree = cls(leaves)
-        return tree.levels()
+        leaves = [Leaf(package) for package in packages]
+        instance = Tree(leaves)
+        return instance.levels()
 
     def levels(self) -> List[List[Package]]:
         """
@@ -182,13 +150,6 @@ class Tree:
         Returns:
             List[List[Package]]: sorted list of packages lists based on their dependencies
         """
-        # https://docs.python.org/dev/library/itertools.html#itertools-recipes
-        def partition(source: List[Leaf]) -> Tuple[List[Leaf], Iterable[Leaf]]:
-            first_iter, second_iter = itertools.tee(source)
-            filter_fn: Callable[[Leaf], bool] = lambda leaf: leaf.is_dependency(next_level)
-            # materialize first list and leave second as iterator
-            return list(filter(filter_fn, first_iter)), itertools.filterfalse(filter_fn, second_iter)
-
         unsorted: List[List[Leaf]] = []
 
         # build initial tree
@@ -203,7 +164,9 @@ class Tree:
             next_level = unsorted[next_num]
 
             # change lists inside the collection
-            unsorted[current_num], to_be_moved = partition(current_level)
+            # additional workaround with partial in order to hide cell-var-from-loop pylint warning
+            predicate = functools.partial(Leaf.is_dependency, packages=next_level)
+            unsorted[current_num], to_be_moved = partition(current_level, predicate)
             unsorted[next_num].extend(to_be_moved)
 
         comparator: Callable[[Package], str] = lambda package: package.base

src/ahriman/core/upload/upload_trigger.py

@@ -35,6 +35,16 @@ class UploadTrigger(Trigger):
     """
 
     CONFIGURATION_SCHEMA = {
+        "upload": {
+            "type": "dict",
+            "schema": {
+                "target": {
+                    "type": "list",
+                    "coerce": "list",
+                    "schema": {"type": "string"},
+                },
+            },
+        },
         "github": {
             "type": "dict",
             "schema": {
@@ -57,6 +67,7 @@ class UploadTrigger(Trigger):
                 "timeout": {
                     "type": "integer",
                     "coerce": "integer",
+                    "min": 0,
                 },
                 "username": {
                     "type": "string",
@@ -101,6 +112,7 @@ class UploadTrigger(Trigger):
                 "chunk_size": {
                     "type": "integer",
                     "coerce": "integer",
+                    "min": 0,
                 },
                 "region": {
                     "type": "string",

src/ahriman/core/util.py

@@ -19,6 +19,7 @@
 #
 import datetime
 import io
+import itertools
 import logging
 import os
 import re
@@ -28,14 +29,31 @@ import subprocess
 from enum import Enum
 from pathlib import Path
 from pwd import getpwuid
-from typing import Any, Dict, Generator, IO, Iterable, List, Optional, Type, Union
+from typing import Any, Callable, Dict, Generator, IO, Iterable, List, Optional, Tuple, Type, TypeVar, Union
 
 from ahriman.core.exceptions import OptionError, UnsafeRunError
 from ahriman.models.repository_paths import RepositoryPaths
 
 
-__all__ = ["check_output", "check_user", "enum_values", "exception_response_text", "filter_json", "full_version",
-           "package_like", "pretty_datetime", "pretty_size", "safe_filename", "utcnow", "walk"]
+__all__ = [
+    "check_output",
+    "check_user",
+    "enum_values",
+    "exception_response_text",
+    "filter_json",
+    "full_version",
+    "package_like",
+    "partition",
+    "pretty_datetime",
+    "pretty_size",
+    "safe_filename",
+    "trim_package",
+    "utcnow",
+    "walk",
+]
+
+
+T = TypeVar("T")
 
 
 def check_output(*args: str, exception: Optional[Exception] = None, cwd: Optional[Path] = None,
@@ -225,6 +243,21 @@ def package_like(filename: Path) -> bool:
     return ".pkg." in name and not name.endswith(".sig")
 
 
+def partition(source: List[T], predicate: Callable[[T], bool]) -> Tuple[List[T], List[T]]:
+    """
+    partition list into two based on predicate, based on # https://docs.python.org/dev/library/itertools.html#itertools-recipes
+
+    Args:
+        source(List[T]): source list to be partitioned
+        predicate(Callable[[T], bool]): filter function
+
+    Returns:
+        Tuple[List[T], List[T]]: two lists, first is which ``predicate`` is ``True``, second is ``False``
+    """
+    first_iter, second_iter = itertools.tee(source)
+    return list(filter(predicate, first_iter)), list(itertools.filterfalse(predicate, second_iter))
+
+
 def pretty_datetime(timestamp: Optional[Union[datetime.datetime, float, int]]) -> str:
     """
     convert datetime object to string
@@ -295,6 +328,23 @@ def safe_filename(source: str) -> str:
     return re.sub(r"[^A-Za-z\d\-._~:\[\]@]", "-", source)
 
 
+def trim_package(package_name: str) -> str:
+    """
+    remove version bound and description from package name. Pacman allows to specify version bound (=, <=, >= etc) for
+    packages in dependencies and also allows to specify description (via :); this function removes trailing parts and
+    return exact package name
+
+    Args:
+        package_name(str): source package name
+
+    Returns:
+        str: package name without description or version bound
+    """
+    for symbol in ("<", "=", ">", ":"):
+        package_name = package_name.partition(symbol)[0]
+    return package_name
+
+
 def utcnow() -> datetime.datetime:
     """
     get current time

src/ahriman/models/aur_package.py

@@ -45,6 +45,7 @@ class AURPackage:
         popularity(float): package popularity
         out_of_date(Optional[datetime.datetime]): package out of date timestamp if any
         maintainer(Optional[str]): package maintainer
+        submitter(Optional[str]): package first submitter
         first_submitted(datetime.datetime): timestamp of the first package submission
         last_modified(datetime.datetime): timestamp of the last package submission
         url_path(str): AUR package path
@@ -89,6 +90,7 @@ class AURPackage:
     url: Optional[str] = None
     out_of_date: Optional[datetime.datetime] = None
     maintainer: Optional[str] = None
+    submitter: Optional[str] = None
     repository: str = "aur"
     depends: List[str] = field(default_factory=list)
     make_depends: List[str] = field(default_factory=list)
@@ -140,6 +142,7 @@ class AURPackage:
             url=package.url,
             out_of_date=None,
             maintainer=None,
+            submitter=None,
             repository=package.db.name,
             depends=package.depends,
             make_depends=package.makedepends,
@@ -178,6 +181,7 @@ class AURPackage:
                 dump["flag_date"],
                 "%Y-%m-%dT%H:%M:%S.%fZ") if dump["flag_date"] is not None else None,
             maintainer=next(iter(dump["maintainers"]), None),
+            submitter=None,
             repository=dump["repo"],
             depends=dump["depends"],
             make_depends=dump["makedepends"],

src/ahriman/models/internal_status.py

@@ -57,7 +57,8 @@ class InternalStatus:
             InternalStatus: internal status
         """
         counters = Counters.from_json(dump["packages"]) if "packages" in dump else Counters(total=0)
-        return cls(status=BuildStatus.from_json(dump.get("status", {})),
+        build_status = dump.get("status") or {}
+        return cls(status=BuildStatus.from_json(build_status),
                    architecture=dump.get("architecture"),
                    packages=counters,
                    repository=dump.get("repository"),

src/ahriman/models/migration.py

@@ -18,7 +18,10 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from dataclasses import dataclass
-from typing import List
+from sqlite3 import Connection
+from typing import Callable, List
+
+from ahriman.core.configuration import Configuration
 
 
 @dataclass(frozen=True, kw_only=True)
@@ -30,8 +33,10 @@ class Migration:
         index(int): migration position
         name(str): migration name
         steps(List[str]): migration steps
+        migrate_data(Callable[[Connection, Configuration], None]): data migration callback
     """
 
     index: int
     name: str
     steps: List[str]
+    migrate_data: Callable[[Connection, Configuration], None]

src/ahriman/models/package.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-# pylint: disable=too-many-lines
+# pylint: disable=too-many-lines,too-many-public-methods
 from __future__ import annotations
 
 import copy
@@ -88,6 +88,36 @@ class Package(LazyLogging):
         """
         return sorted(set(sum((package.depends for package in self.packages.values()), start=[])))
 
+    @property
+    def depends_build(self) -> Set[str]:
+        """
+        get full list of external dependencies which has to be installed for build process
+
+        Returns:
+            Set[str]: full dependencies list used by devtools
+        """
+        return (set(self.depends) | set(self.depends_make)).difference(self.packages_full)
+
+    @property
+    def depends_make(self) -> List[str]:
+        """
+        get package make dependencies
+
+        Returns:
+            List[str]: sum of make dependencies per each package
+        """
+        return sorted(set(sum((package.make_depends for package in self.packages.values()), start=[])))
+
+    @property
+    def depends_opt(self) -> List[str]:
+        """
+        get package optional dependencies
+
+        Returns:
+            List[str]: sum of optional dependencies per each package
+        """
+        return sorted(set(sum((package.opt_depends for package in self.packages.values()), start=[])))
+
     @property
     def groups(self) -> List[str]:
         """
@@ -133,6 +163,20 @@ class Package(LazyLogging):
         """
         return sorted(set(sum((package.licenses for package in self.packages.values()), start=[])))
 
+    @property
+    def packages_full(self) -> List[str]:
+        """
+        get full packages list including provides
+
+        Returns:
+            List[str]: full list of packages which this base contains
+        """
+        packages = set()
+        for package, properties in self.packages.items():
+            packages.add(package)
+            packages.update(properties.provides)
+        return sorted(packages)
+
     @classmethod
     def from_archive(cls: Type[Package], path: Path, pacman: Pacman, remote: Optional[RemoteSource]) -> Package:
         """
@@ -168,15 +212,16 @@ class Package(LazyLogging):
             base=package.package_base,
             version=package.version,
             remote=remote,
-            packages={package.name: PackageDescription()})
+            packages={package.name: PackageDescription.from_aur(package)})
 
     @classmethod
-    def from_build(cls: Type[Package], path: Path) -> Package:
+    def from_build(cls: Type[Package], path: Path, architecture: str) -> Package:
         """
         construct package properties from sources directory
 
         Args:
             path(Path): path to package sources directory
+            architecture(str): load package for specific architecture
 
         Returns:
             Package: package properties
@@ -188,7 +233,21 @@ class Package(LazyLogging):
         srcinfo, errors = parse_srcinfo(srcinfo_source)
         if errors:
             raise PackageInfoError(errors)
-        packages = {key: PackageDescription() for key in srcinfo["packages"]}
+
+        def get_property(key: str, properties: Dict[str, Any], default: Any) -> Any:
+            return properties.get(key) or srcinfo.get(key) or default
+
+        def get_list(key: str, properties: Dict[str, Any]) -> Any:
+            return get_property(key, properties, []) + get_property(f"{key}_{architecture}", properties, [])
+
+        packages = {
+            package: PackageDescription(
+                depends=get_list("depends", properties),
+                make_depends=get_list("makedepends", properties),
+                opt_depends=get_list("optdepends", properties),
+            )
+            for package, properties in srcinfo["packages"].items()
+        }
         version = full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"])
 
         return cls(base=srcinfo["pkgbase"], version=version, remote=None, packages=packages)
@@ -204,11 +263,12 @@ class Package(LazyLogging):
         Returns:
             Package: package properties
         """
+        packages_json = dump.get("packages") or {}
         packages = {
             key: PackageDescription.from_json(value)
-            for key, value in dump.get("packages", {}).items()
+            for key, value in packages_json.items()
         }
-        remote = dump.get("remote", {})
+        remote = dump.get("remote") or {}
         return cls(base=dump["base"], version=dump["version"], remote=RemoteSource.from_json(remote), packages=packages)
 
     @classmethod
@@ -230,43 +290,7 @@ class Package(LazyLogging):
             base=package.package_base,
             version=package.version,
             remote=remote,
-            packages={package.name: PackageDescription()})
-
-    @staticmethod
-    def dependencies(path: Path) -> Set[str]:
-        """
-        load dependencies from package sources
-
-        Args:
-            path(Path): path to package sources directory
-
-        Returns:
-            Set[str]: list of package dependencies including makedepends array, but excluding packages from this base
-
-        Raises:
-            InvalidPackageInfo: if there are parsing errors
-        """
-        # additional function to remove versions from dependencies
-        def extract_packages(raw_packages_list: List[str]) -> Set[str]:
-            return {trim_version(package_name) for package_name in raw_packages_list}
-
-        def trim_version(package_name: str) -> str:
-            for symbol in ("<", "=", ">"):
-                package_name = package_name.split(symbol)[0]
-            return package_name
-
-        srcinfo_source = Package._check_output("makepkg", "--printsrcinfo", cwd=path)
-        srcinfo, errors = parse_srcinfo(srcinfo_source)
-        if errors:
-            raise PackageInfoError(errors)
-        makedepends = extract_packages(srcinfo.get("makedepends", []))
-        # sum over each package
-        depends = extract_packages(srcinfo.get("depends", []))
-        for package in srcinfo["packages"].values():
-            depends |= extract_packages(package.get("depends", []))
-        # we are not interested in dependencies inside pkgbase
-        packages = set(srcinfo["packages"].keys())
-        return (depends | makedepends) - packages
+            packages={package.name: PackageDescription.from_aur(package)})
 
     @staticmethod
     def supported_architectures(path: Path) -> Set[str]:

src/ahriman/models/package_description.py

@@ -24,7 +24,8 @@ from pathlib import Path
 from pyalpm import Package  # type: ignore
 from typing import Any, Dict, List, Optional, Type
 
-from ahriman.core.util import filter_json
+from ahriman.core.util import filter_json, trim_package
+from ahriman.models.aur_package import AURPackage
 
 
 @dataclass(kw_only=True)
@@ -37,6 +38,8 @@ class PackageDescription:
         archive_size(Optional[int]): package archive size
         build_date(Optional[int]): package build date
         depends(List[str]): package dependencies list
+        opt_depends(List[str]): optional package dependencies list
+        make_depends(List[str]): package dependencies list used for building
         description(Optional[str]): package description
         filename(Optional[str]): package archive name
         groups(List[str]): package groups
@@ -67,6 +70,8 @@ class PackageDescription:
     archive_size: Optional[int] = None
     build_date: Optional[int] = None
     depends: List[str] = field(default_factory=list)
+    make_depends: List[str] = field(default_factory=list)
+    opt_depends: List[str] = field(default_factory=list)
     description: Optional[str] = None
     filename: Optional[str] = None
     groups: List[str] = field(default_factory=list)
@@ -75,6 +80,14 @@ class PackageDescription:
     provides: List[str] = field(default_factory=list)
     url: Optional[str] = None
 
+    def __post_init__(self) -> None:
+        """
+        update dependencies list accordingly
+        """
+        self.depends = [trim_package(package) for package in self.depends]
+        self.opt_depends = [trim_package(package) for package in self.opt_depends]
+        self.make_depends = [trim_package(package) for package in self.make_depends]
+
     @property
     def filepath(self) -> Optional[Path]:
         """
@@ -85,6 +98,27 @@ class PackageDescription:
         """
         return Path(self.filename) if self.filename is not None else None
 
+    @classmethod
+    def from_aur(cls: Type[PackageDescription], package: AURPackage) -> PackageDescription:
+        """
+        construct properties from AUR package model
+
+        Args:
+            package(AURPackage): AUR package model
+
+        Returns:
+            PackageDescription: package properties based on source AUR package
+        """
+        return cls(
+            depends=package.depends,
+            make_depends=package.make_depends,
+            opt_depends=package.opt_depends,
+            description=package.description,
+            licenses=package.license,
+            provides=package.provides,
+            url=package.url,
+        )
+
     @classmethod
     def from_json(cls: Type[PackageDescription], dump: Dict[str, Any]) -> PackageDescription:
         """
@@ -117,13 +151,16 @@ class PackageDescription:
             archive_size=package.size,
             build_date=package.builddate,
             depends=package.depends,
+            make_depends=package.makedepends,
+            opt_depends=package.optdepends,
             description=package.desc,
             filename=path.name,
             groups=package.groups,
             installed_size=package.isize,
             licenses=package.licenses,
             provides=package.provides,
-            url=package.url)
+            url=package.url,
+        )
 
     def view(self) -> Dict[str, Any]:
         """

src/ahriman/models/result.py

@@ -19,7 +19,7 @@
 #
 from __future__ import annotations
 
-from typing import Any, List, Optional, Iterable
+from typing import Any, Iterable, List, Optional
 
 from ahriman.core.exceptions import UnprocessedPackageStatusError
 from ahriman.models.package import Package

src/ahriman/models/user_identity.py

@@ -1,102 +0,0 @@
-#
-# Copyright (c) 2021-2023 ahriman team.
-#
-# This file is part of ahriman
-# (see https://github.com/arcan1s/ahriman).
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-#
-from __future__ import annotations
-
-import time
-
-from dataclasses import dataclass
-from typing import Optional, Type
-
-
-@dataclass(frozen=True)
-class UserIdentity:
-    """
-    user identity used inside web service
-
-    Attributes:
-        username(str): username
-        expire_at(int): identity expiration timestamp
-    """
-
-    username: str
-    expire_at: int
-
-    @classmethod
-    def from_identity(cls: Type[UserIdentity], identity: str) -> Optional[UserIdentity]:
-        """
-        parse identity into object
-
-        Args:
-            identity(str): identity from session data
-
-        Returns:
-            Optional[UserIdentity]: user identity object if it can be parsed and not expired and None otherwise
-        """
-        try:
-            username, expire_at = identity.split()
-            user = cls(username, int(expire_at))
-            return None if user.is_expired() else user
-        except ValueError:
-            return None
-
-    @classmethod
-    def from_username(cls: Type[UserIdentity], username: Optional[str], max_age: int) -> Optional[UserIdentity]:
-        """
-        generate identity from username
-
-        Args:
-            username(Optional[str]): username
-            max_age(int): time to expire, seconds
-
-        Returns:
-            Optional[UserIdentity]: constructed identity object
-        """
-        return cls(username, cls.expire_when(max_age)) if username is not None else None
-
-    @staticmethod
-    def expire_when(max_age: int) -> int:
-        """
-        generate expiration time using delta
-
-        Args:
-            max_age(int): time delta to generate. Must be usually TTE
-
-        Returns:
-            int: expiration timestamp
-        """
-        return int(time.time()) + max_age
-
-    def is_expired(self) -> bool:
-        """
-        compare timestamp with current timestamp and return True in case if identity is expired
-
-        Returns:
-            bool: True in case if identity is expired and False otherwise
-        """
-        return self.expire_when(0) > self.expire_at
-
-    def to_identity(self) -> str:
-        """
-        convert object to identity representation
-
-        Returns:
-            str: web service identity
-        """
-        return f"{self.username} {self.expire_at}"

src/ahriman/version.py

@@ -17,4 +17,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-__version__ = "2.6.1"
+__version__ = "2.8.0"

src/ahriman/web/apispec.py

@@ -0,0 +1,120 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import aiohttp_apispec  # type: ignore
+
+from aiohttp.web import Application
+from typing import Any, Dict, List
+
+from ahriman import version
+from ahriman.core.configuration import Configuration
+
+
+__all__ = ["setup_apispec"]
+
+
+def _info() -> Dict[str, Any]:
+    """
+    create info object for swagger docs
+
+    Returns:
+        Dict[str, Any]: info object as per openapi specification
+    """
+    return {
+        "title": "ahriman",
+        "description": """Wrapper for managing custom repository inspired by [repo-scripts](https://github.com/arcan1s/repo-scripts).
+
+## Features
+
+* Install-configure-forget manager for the very own repository.
+* Multi-architecture support.
+* Dependency manager.
+* VCS packages support.
+* Official repository support.
+* Ability to patch AUR packages and even create package from local PKGBUILDs.
+* Sign support with gpg (repository, package, per package settings).
+* Triggers for repository updates, e.g. synchronization to remote services (rsync, s3 and github) and report generation (email, html, telegram).
+* Repository status interface with optional authorization and control options
+
+<security-definitions />
+""",
+        "license": {
+            "name": "GPL3",
+            "url": "https://raw.githubusercontent.com/arcan1s/ahriman/master/COPYING",
+        },
+        "version": version.__version__,
+    }
+
+
+def _security() -> List[Dict[str, Any]]:
+    """
+    get security definitions
+
+    Returns:
+        List[Dict[str, Any]]: generated security definition
+    """
+    return [{
+        "token": {
+            "type": "apiKey",  # as per specification we are using api key
+            "name": "API_SESSION",
+            "in": "cookie",
+        }
+    }]
+
+
+def _servers(application: Application) -> List[Dict[str, Any]]:
+    """
+    get list of defined addresses for server
+
+    Args:
+        application(Application): web application instance
+
+    Returns:
+        List[Dict[str, Any]]: list (actually only one) of defined web urls
+    """
+    configuration: Configuration = application["configuration"]
+    address = configuration.get("web", "address", fallback=None)
+    if not address:
+        host = configuration.get("web", "host")
+        port = configuration.getint("web", "port")
+        address = f"http://{host}:{port}"
+
+    return [{
+        "url": address,
+    }]
+
+
+def setup_apispec(application: Application) -> aiohttp_apispec.AiohttpApiSpec:
+    """
+    setup swagger api specification
+
+    Args:
+        application(Application): web application instance
+
+    Returns:
+        aiohttp_apispec.AiohttpApiSpec: created specification instance
+    """
+    return aiohttp_apispec.setup_aiohttp_apispec(
+        application,
+        url="/api-docs/swagger.json",
+        openapi_version="3.0.2",
+        info=_info(),
+        servers=_servers(application),
+        security=_security(),
+    )

src/ahriman/web/cors.py

@@ -17,27 +17,32 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from sqlite3 import Connection
+import aiohttp_cors  # type: ignore
 
-from ahriman.core.configuration import Configuration
+from aiohttp.web import Application
 
 
-__all__ = ["migrate_users_data"]
+__all__ = ["setup_cors"]
 
 
-def migrate_users_data(connection: Connection, configuration: Configuration) -> None:
+def setup_cors(application: Application) -> aiohttp_cors.CorsConfig:
     """
-    perform migration for users
+    setup CORS for the web application
 
     Args:
-        connection(Connection): database connection
-        configuration(Configuration): configuration instance
+        application(Application): web application instance
+
+    Returns:
+        aiohttp_cors.CorsConfig: generated CORS configuration
     """
-    for section in configuration.sections():
-        for option, value in configuration[section].items():
-            if not section.startswith("auth:"):
-                continue
-            access = section[5:]
-            connection.execute(
-                """insert into users (username, access, password) values (:username, :access, :password)""",
-                {"username": option.lower(), "access": access, "password": value})
+    cors = aiohttp_cors.setup(application, defaults={
+        "*": aiohttp_cors.ResourceOptions(
+            expose_headers="*",
+            allow_headers="*",
+            allow_methods="*",
+        )
+    })
+    for route in application.router.routes():
+        cors.add(route)
+
+    return cors

src/ahriman/web/middlewares/__init__.py

@@ -17,8 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from aiohttp.web import Request
-from aiohttp.web_response import StreamResponse
+from aiohttp.web import Request, StreamResponse
 from typing import Awaitable, Callable
 
 

src/ahriman/web/middlewares/auth_handler.py

@@ -18,29 +18,25 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import aiohttp_security  # type: ignore
-import base64
 import socket
 import types
 
-from aiohttp import web
-from aiohttp.web import middleware, Request
-from aiohttp.web_response import StreamResponse
-from aiohttp.web_urldispatcher import StaticResource
+from aiohttp.web import Application, Request, StaticResource, StreamResponse, middleware
 from aiohttp_session import setup as setup_session
 from aiohttp_session.cookie_storage import EncryptedCookieStorage
 from cryptography import fernet
 from typing import Optional
 
 from ahriman.core.auth import Auth
+from ahriman.core.configuration import Configuration
 from ahriman.models.user_access import UserAccess
-from ahriman.models.user_identity import UserIdentity
 from ahriman.web.middlewares import HandlerType, MiddlewareType
 
 
-__all__ = ["AuthorizationPolicy", "auth_handler", "setup_auth"]
+__all__ = ["setup_auth"]
 
 
-class AuthorizationPolicy(aiohttp_security.AbstractAuthorizationPolicy):  # type: ignore
+class _AuthorizationPolicy(aiohttp_security.AbstractAuthorizationPolicy):
     """
     authorization policy implementation
 
@@ -67,10 +63,7 @@ class AuthorizationPolicy(aiohttp_security.AbstractAuthorizationPolicy):  # type
         Returns:
             Optional[str]: user identity (username) in case if user exists and None otherwise
         """
-        user = UserIdentity.from_identity(identity)
-        if user is None:
-            return None
-        return user.username if await self.validator.known_username(user.username) else None
+        return identity if await self.validator.known_username(identity) else None
 
     async def permits(self, identity: str, permission: UserAccess, context: Optional[str] = None) -> bool:
         """
@@ -84,13 +77,10 @@ class AuthorizationPolicy(aiohttp_security.AbstractAuthorizationPolicy):  # type
         Returns:
             bool: True in case if user is allowed to perform this request and False otherwise
         """
-        user = UserIdentity.from_identity(identity)
-        if user is None:
-            return False
-        return await self.validator.verify_access(user.username, permission, context)
+        return await self.validator.verify_access(identity, permission, context)
 
 
-def auth_handler(allow_read_only: bool) -> MiddlewareType:
+def _auth_handler(allow_read_only: bool) -> MiddlewareType:
     """
     authorization and authentication middleware
 
@@ -125,26 +115,43 @@ def auth_handler(allow_read_only: bool) -> MiddlewareType:
     return handle
 
 
-def setup_auth(application: web.Application, validator: Auth) -> web.Application:
+def _cookie_secret_key(configuration: Configuration) -> fernet.Fernet:
+    """
+    extract cookie secret key from configuration if set or generate new one
+
+    Args:
+        configuration(Configuration): configuration instance
+
+    Returns:
+        fernet.Fernet: fernet key instance
+    """
+    if (secret_key := configuration.get("auth", "cookie_secret_key", fallback=None)) is not None:
+        return fernet.Fernet(secret_key)
+
+    secret_key = fernet.Fernet.generate_key()
+    return fernet.Fernet(secret_key)
+
+
+def setup_auth(application: Application, configuration: Configuration, validator: Auth) -> Application:
     """
     setup authorization policies for the application
 
     Args:
-        application(web.Application): web application instance
+        application(Application): web application instance
+        configuration(Configuration): configuration instance
         validator(Auth): authorization module instance
 
     Returns:
-        web.Application: configured web application
+        Application: configured web application
     """
-    fernet_key = fernet.Fernet.generate_key()
-    secret_key = base64.urlsafe_b64decode(fernet_key)
+    secret_key = _cookie_secret_key(configuration)
     storage = EncryptedCookieStorage(secret_key, cookie_name="API_SESSION", max_age=validator.max_age)
     setup_session(application, storage)
 
-    authorization_policy = AuthorizationPolicy(validator)
+    authorization_policy = _AuthorizationPolicy(validator)
     identity_policy = aiohttp_security.SessionIdentityPolicy()
 
     aiohttp_security.setup(application, identity_policy, authorization_policy)
-    application.middlewares.append(auth_handler(validator.allow_read_only))
+    application.middlewares.append(_auth_handler(validator.allow_read_only))
 
     return application

src/ahriman/web/middlewares/exception_handler.py

@@ -20,8 +20,8 @@
 import aiohttp_jinja2
 import logging
 
-from aiohttp.web import HTTPClientError, HTTPException, HTTPServerError, HTTPUnauthorized, Request, StreamResponse, \
-    json_response, middleware
+from aiohttp.web import HTTPClientError, HTTPException, HTTPMethodNotAllowed, HTTPNoContent, HTTPServerError, \
+    HTTPUnauthorized, Request, StreamResponse, json_response, middleware
 
 from ahriman.web.middlewares import HandlerType, MiddlewareType
 
@@ -29,6 +29,20 @@ from ahriman.web.middlewares import HandlerType, MiddlewareType
 __all__ = ["exception_handler"]
 
 
+def _is_templated_unauthorized(request: Request) -> bool:
+    """
+    check if the request is eligible for rendering html template
+
+    Args:
+        request(Request): source request to check
+
+    Returns:
+        bool: True in case if response should be rendered as html and False otherwise
+    """
+    return request.path in ("/api/v1/login", "/api/v1/logout") \
+        and "application/json" not in request.headers.getall("accept", [])
+
+
 def exception_handler(logger: logging.Logger) -> MiddlewareType:
     """
     exception handler middleware. Just log any exception (except for client ones)
@@ -44,10 +58,21 @@ def exception_handler(logger: logging.Logger) -> MiddlewareType:
         try:
             return await handler(request)
         except HTTPUnauthorized as e:
-            if is_templated_unauthorized(request):
+            if _is_templated_unauthorized(request):
                 context = {"code": e.status_code, "reason": e.reason}
                 return aiohttp_jinja2.render_template("error.jinja2", request, context, status=e.status_code)
             return json_response(data={"error": e.reason}, status=e.status_code)
+        except HTTPMethodNotAllowed as e:
+            if e.method == "OPTIONS":
+                # automatically handle OPTIONS method, idea comes from
+                # https://github.com/arcan1s/ffxivbis/blob/master/src/main/scala/me/arcanis/ffxivbis/http/api/v1/HttpHandler.scala#L32
+                raise HTTPNoContent(headers={"Allow": ",".join(sorted(e.allowed_methods))})
+            if e.method == "HEAD":
+                # since we have special autogenerated HEAD method, we need to remove it from list of available
+                e.allowed_methods = {method for method in e.allowed_methods if method != "HEAD"}
+                e.headers["Allow"] = ",".join(sorted(e.allowed_methods))
+                raise e
+            raise
         except HTTPClientError as e:
             return json_response(data={"error": e.reason}, status=e.status_code)
         except HTTPServerError as e:
@@ -60,17 +85,3 @@ def exception_handler(logger: logging.Logger) -> MiddlewareType:
             return json_response(data={"error": str(e)}, status=500)
 
     return handle
-
-
-def is_templated_unauthorized(request: Request) -> bool:
-    """
-    check if the request is eligible for rendering html template
-
-    Args:
-        request(Request): source request to check
-
-    Returns:
-        bool: True in case if response should be rendered as html and False otherwise
-    """
-    return request.path in ("/api/v1/login", "/api/v1/logout") \
-        and "application/json" not in request.headers.getall("accept", [])

src/ahriman/web/routes.py

@@ -20,6 +20,8 @@
 from aiohttp.web import Application
 from pathlib import Path
 
+from ahriman.web.views.api.docs import DocsView
+from ahriman.web.views.api.swagger import SwaggerView
 from ahriman.web.views.index import IndexView
 from ahriman.web.views.service.add import AddView
 from ahriman.web.views.service.pgp import PGPView
@@ -43,82 +45,31 @@ def setup_routes(application: Application, static_path: Path) -> None:
     """
     setup all defined routes
 
-    Available routes are:
-
-        * ``GET /`` get build status page
-        * ``GET /index.html`` same as above
-
-        * ``POST /api/v1/service/add`` add new packages to repository
-
-        * ``GET /api/v1/service/pgp`` fetch PGP key from the keyserver
-        * ``POST /api/v1/service/pgp`` import PGP key from the keyserver
-
-        * ``POST /api/v1/service/rebuild`` rebuild packages based on their dependency list
-
-        * ``POST /api/v1/service/remove`` remove existing package from repository
-
-        * ``POST /api/v1/service/request`` request to add new packages to repository
-
-        * ``GET /api/v1/service/search`` search for substring in AUR
-
-        * ``POST /api/v1/service/update`` update all packages in repository
-
-        * ``GET /api/v1/packages`` get all known packages
-        * ``POST /api/v1/packages`` force update every package from repository
-
-        * ``DELETE /api/v1/package/:base`` delete package base from status page
-        * ``GET /api/v1/package/:base`` get package base status
-        * ``POST /api/v1/package/:base`` update package base status
-
-        * ``DELETE /api/v1/packages/{package}/logs`` delete package related logs
-        * ``GET /api/v1/packages/{package}/logs`` create log record for the package
-        * ``POST /api/v1/packages/{package}/logs`` get last package logs
-
-        * ``GET /api/v1/status`` get service status itself
-        * ``POST /api/v1/status`` update service status itself
-
-        * ``GET /api/v1/login`` OAuth2 handler for login
-        * ``POST /api/v1/login`` login to service
-        * ``POST /api/v1/logout`` logout from service
-
     Args:
         application(Application): web application instance
         static_path(Path): path to static files directory
     """
-    application.router.add_get("/", IndexView, allow_head=True)
-    application.router.add_get("/index.html", IndexView, allow_head=True)
+    application.router.add_view("/", IndexView)
+    application.router.add_view("/index.html", IndexView)
+
+    application.router.add_view("/api-docs", DocsView)
+    application.router.add_view("/api-docs/swagger.json", SwaggerView)
 
     application.router.add_static("/static", static_path, follow_symlinks=True)
 
-    application.router.add_post("/api/v1/service/add", AddView)
+    application.router.add_view("/api/v1/service/add", AddView)
+    application.router.add_view("/api/v1/service/pgp", PGPView)
+    application.router.add_view("/api/v1/service/rebuild", RebuildView)
+    application.router.add_view("/api/v1/service/remove", RemoveView)
+    application.router.add_view("/api/v1/service/request", RequestView)
+    application.router.add_view("/api/v1/service/search", SearchView)
+    application.router.add_view("/api/v1/service/update", UpdateView)
 
-    application.router.add_get("/api/v1/service/pgp", PGPView, allow_head=True)
-    application.router.add_post("/api/v1/service/pgp", PGPView)
+    application.router.add_view("/api/v1/packages", PackagesView)
+    application.router.add_view("/api/v1/packages/{package}", PackageView)
+    application.router.add_view("/api/v1/packages/{package}/logs", LogsView)
 
-    application.router.add_post("/api/v1/service/rebuild", RebuildView)
+    application.router.add_view("/api/v1/status", StatusView)
 
-    application.router.add_post("/api/v1/service/remove", RemoveView)
-
-    application.router.add_post("/api/v1/service/request", RequestView)
-
-    application.router.add_get("/api/v1/service/search", SearchView, allow_head=False)
-
-    application.router.add_post("/api/v1/service/update", UpdateView)
-
-    application.router.add_get("/api/v1/packages", PackagesView, allow_head=True)
-    application.router.add_post("/api/v1/packages", PackagesView)
-
-    application.router.add_delete("/api/v1/packages/{package}", PackageView)
-    application.router.add_get("/api/v1/packages/{package}", PackageView, allow_head=True)
-    application.router.add_post("/api/v1/packages/{package}", PackageView)
-
-    application.router.add_delete("/api/v1/packages/{package}/logs", LogsView)
-    application.router.add_get("/api/v1/packages/{package}/logs", LogsView, allow_head=True)
-    application.router.add_post("/api/v1/packages/{package}/logs", LogsView)
-
-    application.router.add_get("/api/v1/status", StatusView, allow_head=True)
-    application.router.add_post("/api/v1/status", StatusView)
-
-    application.router.add_get("/api/v1/login", LoginView)
-    application.router.add_post("/api/v1/login", LoginView)
-    application.router.add_post("/api/v1/logout", LogoutView)
+    application.router.add_view("/api/v1/login", LoginView)
+    application.router.add_view("/api/v1/logout", LogoutView)

src/ahriman/web/schemas/__init__.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#

src/ahriman/web/schemas/aur_package_schema.py

@@ -0,0 +1,35 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class AURPackageSchema(Schema):
+    """
+    response AUR package schema
+    """
+
+    package = fields.String(required=True, metadata={
+        "description": "Package base",
+        "example": "ahriman",
+    })
+    description = fields.String(required=True, metadata={
+        "description": "Package description",
+        "example": "ArcH linux ReposItory MANager",
+    })

src/ahriman/web/schemas/auth_schema.py

@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class AuthSchema(Schema):
+    """
+    request cookie authorization schema
+    """
+
+    API_SESSION = fields.String(required=True, metadata={
+        "description": "API session key as returned from authorization",
+    })

src/ahriman/web/schemas/counters_schema.py

@@ -0,0 +1,51 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class CountersSchema(Schema):
+    """
+    response package counters schema
+    """
+
+    total = fields.Integer(required=True, metadata={
+        "description": "Total amount of packages",
+        "example": 6,
+    })
+    _unknown = fields.Integer(data_key="unknown", required=True, metadata={
+        "description": "Amount of packages in unknown state",
+        "example": 0,
+    })
+    pending = fields.Integer(required=True, metadata={
+        "description": "Amount of packages in pending state",
+        "example": 2,
+    })
+    building = fields.Integer(required=True, metadata={
+        "description": "Amount of packages in building state",
+        "example": 1,
+    })
+    failed = fields.Integer(required=True, metadata={
+        "description": "Amount of packages in failed state",
+        "example": 1,
+    })
+    success = fields.Integer(required=True, metadata={
+        "description": "Amount of packages in success state",
+        "example": 3,
+    })

src/ahriman/web/schemas/error_schema.py

@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class ErrorSchema(Schema):
+    """
+    response error schema
+    """
+
+    error = fields.String(required=True, metadata={
+        "description": "Error description",
+    })

src/ahriman/web/schemas/internal_status_schema.py

@@ -0,0 +1,49 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+from ahriman import version
+from ahriman.web.schemas.counters_schema import CountersSchema
+from ahriman.web.schemas.status_schema import StatusSchema
+
+
+class InternalStatusSchema(Schema):
+    """
+    response service status schema
+    """
+
+    architecture = fields.String(required=True, metadata={
+        "description": "Repository architecture",
+        "example": "x86_64",
+    })
+    packages = fields.Nested(CountersSchema, required=True, metadata={
+        "description": "Repository package counters",
+    })
+    repository = fields.String(required=True, metadata={
+        "description": "Repository name",
+        "example": "repo-clone",
+    })
+    status = fields.Nested(StatusSchema, required=True, metadata={
+        "description": "Repository status as stored by web service",
+    })
+    version = fields.String(required=True, metadata={
+        "description": "Repository version",
+        "example": version.__version__,
+    })

src/ahriman/web/schemas/log_schema.py

@@ -0,0 +1,38 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class LogSchema(Schema):
+    """
+    request package log schema
+    """
+
+    created = fields.Float(required=True, metadata={
+        "description": "Log record timestamp",
+        "example": 1680537091.233495,
+    })
+    process_id = fields.Integer(required=True, metadata={
+        "description": "Current process id",
+        "example": 42,
+    })
+    message = fields.String(required=True, metadata={
+        "description": "Log message",
+    })

src/ahriman/web/schemas/login_schema.py

@@ -0,0 +1,35 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class LoginSchema(Schema):
+    """
+    request login schema
+    """
+
+    username = fields.String(required=True, metadata={
+        "description": "Login username",
+        "example": "user",
+    })
+    password = fields.String(required=True, metadata={
+        "description": "Login password",
+        "example": "pa55w0rd",
+    })

src/ahriman/web/schemas/logs_schema.py

@@ -0,0 +1,39 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+from ahriman.web.schemas.status_schema import StatusSchema
+
+
+class LogsSchema(Schema):
+    """
+    response package logs schema
+    """
+
+    package_base = fields.String(required=True, metadata={
+        "description": "Package base name",
+        "example": "ahriman",
+    })
+    status = fields.Nested(StatusSchema, required=True, metadata={
+        "description": "Last package status",
+    })
+    logs = fields.String(required=True, metadata={
+        "description": "Full package log from the last build",
+    })

src/ahriman/web/schemas/oauth2_schema.py

@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class OAuth2Schema(Schema):
+    """
+    request OAuth2 authorization schema
+    """
+
+    code = fields.String(metadata={
+        "description": "OAuth2 authorization code. In case if not set, the redirect to provider will be initiated",
+    })

src/ahriman/web/schemas/package_name_schema.py

@@ -0,0 +1,31 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class PackageNameSchema(Schema):
+    """
+    request package name schema
+    """
+
+    package = fields.String(required=True, metadata={
+        "description": "Package name",
+        "example": "ahriman",
+    })

src/ahriman/web/schemas/package_names_schema.py

@@ -0,0 +1,31 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class PackageNamesSchema(Schema):
+    """
+    request package names schema
+    """
+
+    packages = fields.List(fields.String(), required=True, metadata={
+        "description": "Package names",
+        "example": ["ahriman"],
+    })

src/ahriman/web/schemas/package_properties_schema.py

@@ -0,0 +1,79 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class PackagePropertiesSchema(Schema):
+    """
+    request and response package properties schema
+    """
+
+    architecture = fields.String(metadata={
+        "description": "Package architecture",
+        "example": "x86_64",
+    })
+    archive_size = fields.Integer(metadata={
+        "description": "Archive size in bytes",
+        "example": 287989,
+    })
+    build_date = fields.Integer(metadata={
+        "description": "Package build timestamp",
+        "example": 1680537091,
+    })
+    depends = fields.List(fields.String(), metadata={
+        "description": "Package dependencies list",
+        "example": ["devtools"],
+    })
+    make_depends = fields.List(fields.String(), metadata={
+        "description": "Package make dependencies list",
+        "example": ["python-build"],
+    })
+    opt_depends = fields.List(fields.String(), metadata={
+        "description": "Package optional dependencies list",
+        "example": ["python-aiohttp"],
+    })
+    description = fields.String(metadata={
+        "description": "Package description",
+        "example": "ArcH linux ReposItory MANager",
+    })
+    filename = fields.String(metadata={
+        "description": "Package file name",
+        "example": "ahriman-2.7.1-1-any.pkg.tar.zst",
+    })
+    groups = fields.List(fields.String(), metadata={
+        "description": "Package groups",
+        "example": ["base-devel"],
+    })
+    installed_size = fields.Integer(metadata={
+        "description": "Installed package size in bytes",
+        "example": 2047658,
+    })
+    licenses = fields.List(fields.String(), metadata={
+        "description": "Package licenses",
+        "example": ["GPL3"],
+    })
+    provides = fields.List(fields.String(), metadata={
+        "description": "Package provides list",
+        "example": ["ahriman-git"],
+    })
+    url = fields.String(metadata={
+        "description": "Upstream url",
+        "example": "https://github.com/arcan1s/ahriman",
+    })

src/ahriman/web/schemas/package_schema.py

@@ -0,0 +1,46 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+from ahriman import version
+from ahriman.web.schemas.package_properties_schema import PackagePropertiesSchema
+from ahriman.web.schemas.remote_schema import RemoteSchema
+
+
+class PackageSchema(Schema):
+    """
+    request and response package schema
+    """
+
+    base = fields.String(required=True, metadata={
+        "description": "Package base",
+        "example": "ahriman",
+    })
+    version = fields.String(required=True, metadata={
+        "description": "Package version",
+        "example": version.__version__,
+    })
+    remote = fields.Nested(RemoteSchema, required=True, metadata={
+        "description": "Package remote properties",
+    })
+    packages = fields.Dict(
+        keys=fields.String(), values=fields.Nested(PackagePropertiesSchema), required=True, metadata={
+            "description": "Packages which belong to this base",
+        })

src/ahriman/web/schemas/package_status_schema.py

@@ -0,0 +1,50 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+from ahriman.models.build_status import BuildStatusEnum
+from ahriman.web.schemas.package_schema import PackageSchema
+from ahriman.web.schemas.status_schema import StatusSchema
+
+
+class PackageStatusSimplifiedSchema(Schema):
+    """
+    special request package status schema
+    """
+
+    package = fields.Nested(PackageSchema, metadata={
+        "description": "Package description",
+    })
+    status = fields.Enum(BuildStatusEnum, by_value=True, required=True, metadata={
+        "description": "Current status",
+    })
+
+
+class PackageStatusSchema(Schema):
+    """
+    response package status schema
+    """
+
+    package = fields.Nested(PackageSchema, required=True, metadata={
+        "description": "Package description",
+    })
+    status = fields.Nested(StatusSchema, required=True, metadata={
+        "description": "Last package status",
+    })

src/ahriman/web/schemas/pgp_key_id_schema.py

@@ -0,0 +1,35 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class PGPKeyIdSchema(Schema):
+    """
+    request PGP key ID schema
+    """
+
+    key = fields.String(required=True, metadata={
+        "description": "PGP key ID",
+        "example": "0xE989490C",
+    })
+    server = fields.String(required=True, metadata={
+        "description": "PGP key server",
+        "example": "keyserver.ubuntu.com",
+    })

src/ahriman/web/schemas/pgp_key_schema.py

@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class PGPKeySchema(Schema):
+    """
+    response PGP key schema
+    """
+
+    key = fields.String(required=True, metadata={
+        "description": "PGP key body",
+    })

src/ahriman/web/schemas/remote_schema.py

@@ -0,0 +1,48 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+from ahriman.models.package_source import PackageSource
+
+
+class RemoteSchema(Schema):
+    """
+    request and response package remote schema
+    """
+
+    branch = fields.String(required=True, metadata={
+        "description": "Repository branch",
+        "example": "master",
+    })
+    git_url = fields.String(required=True, metadata={
+        "description": "Package git url",
+        "example": "https://aur.archlinux.org/ahriman.git",
+    })
+    path = fields.String(required=True, metadata={
+        "description": "Path to package sources in git repository",
+        "example": ".",
+    })
+    source = fields.Enum(PackageSource, by_value=True, required=True, metadata={
+        "description": "Pacakge source",
+    })
+    web_url = fields.String(required=True, metadata={
+        "description": "Package repository page",
+        "example": "https://aur.archlinux.org/packages/ahriman",
+    })