mirror of
https://github.com/arcan1s/ahriman.git
synced 2025-11-15 21:13:41 +00:00
Compare commits
4 Commits
2.15.3
...
60b09f646b
| Author | SHA1 | Date | |
|---|---|---|---|
| 60b09f646b | |||
| dfc33bf501 | |||
| 783edfd3b2 | |||
| c58fd3a4b9 |
@ -108,6 +108,14 @@ ahriman.core.database.migrations.m012\_last\_commit\_sha module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.database.migrations.m013\_workers module
|
||||
-----------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.database.migrations.m013_workers
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
Module contents
|
||||
---------------
|
||||
|
||||
|
||||
@ -60,6 +60,14 @@ ahriman.core.database.operations.patch\_operations module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.database.operations.workers\_operations module
|
||||
-----------------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.database.operations.workers_operations
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
Module contents
|
||||
---------------
|
||||
|
||||
|
||||
53
docs/ahriman.core.distributed.rst
Normal file
53
docs/ahriman.core.distributed.rst
Normal file
@ -0,0 +1,53 @@
|
||||
ahriman.core.distributed package
|
||||
================================
|
||||
|
||||
Submodules
|
||||
----------
|
||||
|
||||
ahriman.core.distributed.distributed\_system module
|
||||
---------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.distributed.distributed_system
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.distributed.worker\_loader\_trigger module
|
||||
-------------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.distributed.worker_loader_trigger
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.distributed.worker\_register\_trigger module
|
||||
---------------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.distributed.worker_register_trigger
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.distributed.worker\_trigger module
|
||||
-----------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.distributed.worker_trigger
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.distributed.worker\_unregister\_trigger module
|
||||
-----------------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.distributed.worker_unregister_trigger
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
Module contents
|
||||
---------------
|
||||
|
||||
.. automodule:: ahriman.core.distributed
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
@ -12,6 +12,7 @@ Subpackages
|
||||
ahriman.core.build_tools
|
||||
ahriman.core.configuration
|
||||
ahriman.core.database
|
||||
ahriman.core.distributed
|
||||
ahriman.core.formatters
|
||||
ahriman.core.gitremote
|
||||
ahriman.core.http
|
||||
|
||||
@ -260,6 +260,22 @@ ahriman.web.schemas.versioned\_log\_schema module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.web.schemas.worker\_id\_schema module
|
||||
---------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.web.schemas.worker_id_schema
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.web.schemas.worker\_schema module
|
||||
-----------------------------------------
|
||||
|
||||
.. automodule:: ahriman.web.schemas.worker_schema
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
Module contents
|
||||
---------------
|
||||
|
||||
|
||||
29
docs/ahriman.web.views.v1.distributed.rst
Normal file
29
docs/ahriman.web.views.v1.distributed.rst
Normal file
@ -0,0 +1,29 @@
|
||||
ahriman.web.views.v1.distributed package
|
||||
========================================
|
||||
|
||||
Submodules
|
||||
----------
|
||||
|
||||
ahriman.web.views.v1.distributed.worker module
|
||||
----------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.web.views.v1.distributed.worker
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.web.views.v1.distributed.workers module
|
||||
-----------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.web.views.v1.distributed.workers
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
Module contents
|
||||
---------------
|
||||
|
||||
.. automodule:: ahriman.web.views.v1.distributed
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
@ -7,6 +7,7 @@ Subpackages
|
||||
.. toctree::
|
||||
:maxdepth: 4
|
||||
|
||||
ahriman.web.views.v1.distributed
|
||||
ahriman.web.views.v1.service
|
||||
ahriman.web.views.v1.status
|
||||
ahriman.web.views.v1.user
|
||||
|
||||
@ -86,7 +86,7 @@ Build related configuration. Group name can refer to architecture, e.g. ``build:
|
||||
* ``triggers`` - list of ``ahriman.core.triggers.Trigger`` class implementation (e.g. ``ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger``) which will be loaded and run at the end of processing, space separated list of strings, optional. You can also specify triggers by their paths, e.g. ``/usr/lib/python3.10/site-packages/ahriman/core/report/report.py.ReportTrigger``. Triggers are run in the order of definition.
|
||||
* ``triggers_known`` - optional list of ``ahriman.core.triggers.Trigger`` class implementations which are not run automatically and used only for trigger discovery and configuration validation.
|
||||
* ``vcs_allowed_age`` - maximal age in seconds of the VCS packages before their version will be updated with its remote source, integer, optional, default is 7 days.
|
||||
* ``workers`` - list of worker nodes addresses used for build process, space separated list of strings, optional. Each worker address must be valid and reachable url, e.g. ``https://10.0.0.1:8080``. If none set, the build process will be run on the current node.
|
||||
* ``workers`` - list of worker nodes addresses used for build process, space separated list of strings, optional. Each worker address must be valid and reachable url, e.g. ``https://10.0.0.1:8080``. If none set, the build process will be run on the current node. There is also special trigger which loads this value based on the list of the discovered nodes.
|
||||
|
||||
``repository`` group
|
||||
--------------------
|
||||
@ -351,4 +351,13 @@ Requires ``boto3`` library to be installed. Section name must be either ``s3`` (
|
||||
* ``chunk_size`` - chunk size for calculating entity tags, integer, optional, default 8 * 1024 * 1024.
|
||||
* ``object_path`` - path prefix for stored objects, string, optional. If none set, the prefix as in repository tree will be used.
|
||||
* ``region`` - bucket region (e.g. ``eu-central-1``), string, required.
|
||||
* ``secret_key`` - AWS secret access key, string, required.
|
||||
* ``secret_key`` - AWS secret access key, string, required.
|
||||
|
||||
``worker`` group
|
||||
----------------
|
||||
|
||||
This section controls settings for ``ahriman.core.distributed.WorkerTrigger`` plugin.
|
||||
|
||||
* ``address`` - address of the instance, string, required. Must be reachable for the master instance.
|
||||
* ``identifier`` - unique identifier of the instance, string, optional. If none set, the random uuid will be generated on each run automatically.
|
||||
* ``identifier_path`` - path to lock file, string, optional, default is ``/tmp/ahriman-worker-identifier``.
|
||||
|
||||
@ -1168,6 +1168,11 @@ Addition of new package, package removal, repository update
|
||||
|
||||
In all scenarios, update process must be run only on ``master`` node. Unlike the manually distributed packages described above, automatic update must be enabled only for ``master`` node.
|
||||
|
||||
Automatic worker nodes discovery
|
||||
""""""""""""""""""""""""""""""""
|
||||
|
||||
Instead of setting ``build.workers`` option it is also possible to configure services to load worker list dynamically. To do so, the ``ahriman.core.distributed.WorkerLoaderTrigger`` and ``ahriman.core.distributed.WorkerTrigger`` must be used for ``master`` and ``worker`` nodes repsectively. See recipes for more details.
|
||||
|
||||
Known limitations
|
||||
"""""""""""""""""
|
||||
|
||||
|
||||
@ -14,6 +14,18 @@ Built-in triggers
|
||||
|
||||
For the configuration details and settings explanation kindly refer to the :doc:`documentation <configuration>`.
|
||||
|
||||
``ahriman.core.distributed.WorkerLoaderTrigger``
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Special trigger to be used to load workers from database on the start of the application rather than configuration. If the option is already set, it will skip processing.
|
||||
|
||||
``ahriman.core.distributed.WorkerTrigger``
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Another trigger for the distributed system, which registers itself as remote worker. It calls the remote server on start (if no lock file found) and, later, it deregister itself before the stop.
|
||||
|
||||
There are also two triggers which performs only registration and removal (``ahriman.core.distributed.WorkerRegisterTrigger`` and ``ahriman.core.distributed.WorkerUnregisterTrigger`` respectively), but they are not meant to be called directly.
|
||||
|
||||
``ahriman.core.gitremote.RemotePullTrigger``
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
|
||||
@ -24,7 +24,7 @@ ignore_packages =
|
||||
makechrootpkg_flags =
|
||||
makepkg_flags = --nocolor --ignorearch
|
||||
triggers = ahriman.core.gitremote.RemotePullTrigger ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger ahriman.core.gitremote.RemotePushTrigger
|
||||
triggers_known = ahriman.core.gitremote.RemotePullTrigger ahriman.core.gitremote.RemotePushTrigger ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger ahriman.core.support.KeyringTrigger ahriman.core.support.MirrorlistTrigger
|
||||
triggers_known = ahriman.core.distributed.WorkerLoaderTrigger ahriman.core.distributed.WorkerRegisterTrigger ahriman.core.distributed.WorkerTrigger ahriman.core.distributed.WorkerUnregisterTrigger ahriman.core.gitremote.RemotePullTrigger ahriman.core.gitremote.RemotePushTrigger ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger ahriman.core.support.KeyringTrigger ahriman.core.support.MirrorlistTrigger
|
||||
vcs_allowed_age = 604800
|
||||
|
||||
[repository]
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# AUTOMATICALLY GENERATED by `shtab`
|
||||
|
||||
_shtab_ahriman_subparsers=('aur-search' 'search' 'help-commands-unsafe' 'help' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-changes' 'package-changes-remove' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-repositories' 'service-run' 'run' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'service-tree-migrate' 'user-add' 'user-list' 'user-remove' 'web')
|
||||
_shtab_ahriman_subparsers=('aur-search' 'search' 'help-commands-unsafe' 'help' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-changes' 'package-changes-remove' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-repositories' 'service-run' 'run' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'service-tree-migrate' 'service-worker-register' 'service-worker-unregister' 'user-add' 'user-list' 'user-remove' 'web')
|
||||
|
||||
_shtab_ahriman_option_strings=('-h' '--help' '-a' '--architecture' '-c' '--configuration' '--force' '-l' '--lock' '--log-handler' '-q' '--quiet' '--report' '--no-report' '-r' '--repository' '--unsafe' '-V' '--version' '--wait-timeout')
|
||||
_shtab_ahriman_aur_search_option_strings=('-h' '--help' '-e' '--exit-code' '--info' '--no-info' '--sort-by')
|
||||
@ -71,6 +71,8 @@ _shtab_ahriman_setup_option_strings=('-h' '--help' '--build-as-user' '--from-con
|
||||
_shtab_ahriman_service_shell_option_strings=('-h' '--help')
|
||||
_shtab_ahriman_shell_option_strings=('-h' '--help')
|
||||
_shtab_ahriman_service_tree_migrate_option_strings=('-h' '--help')
|
||||
_shtab_ahriman_service_worker_register_option_strings=('-h' '--help')
|
||||
_shtab_ahriman_service_worker_unregister_option_strings=('-h' '--help')
|
||||
_shtab_ahriman_user_add_option_strings=('-h' '--help' '--key' '--packager' '-p' '--password' '-R' '--role')
|
||||
_shtab_ahriman_user_list_option_strings=('-h' '--help' '-e' '--exit-code' '-R' '--role')
|
||||
_shtab_ahriman_user_remove_option_strings=('-h' '--help')
|
||||
@ -78,7 +80,7 @@ _shtab_ahriman_web_option_strings=('-h' '--help')
|
||||
|
||||
|
||||
|
||||
_shtab_ahriman_pos_0_choices=('aur-search' 'search' 'help-commands-unsafe' 'help' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-changes' 'package-changes-remove' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-repositories' 'service-run' 'run' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'service-tree-migrate' 'user-add' 'user-list' 'user-remove' 'web')
|
||||
_shtab_ahriman_pos_0_choices=('aur-search' 'search' 'help-commands-unsafe' 'help' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-changes' 'package-changes-remove' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-repositories' 'service-run' 'run' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'service-tree-migrate' 'service-worker-register' 'service-worker-unregister' 'user-add' 'user-list' 'user-remove' 'web')
|
||||
_shtab_ahriman___log_handler_choices=('console' 'syslog' 'journald')
|
||||
_shtab_ahriman_aur_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'submitter' 'url' 'url_path' 'version')
|
||||
_shtab_ahriman_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'submitter' 'url' 'url_path' 'version')
|
||||
@ -525,6 +527,10 @@ _shtab_ahriman_shell__v_nargs=0
|
||||
_shtab_ahriman_shell___verbose_nargs=0
|
||||
_shtab_ahriman_service_tree_migrate__h_nargs=0
|
||||
_shtab_ahriman_service_tree_migrate___help_nargs=0
|
||||
_shtab_ahriman_service_worker_register__h_nargs=0
|
||||
_shtab_ahriman_service_worker_register___help_nargs=0
|
||||
_shtab_ahriman_service_worker_unregister__h_nargs=0
|
||||
_shtab_ahriman_service_worker_unregister___help_nargs=0
|
||||
_shtab_ahriman_user_add__h_nargs=0
|
||||
_shtab_ahriman_user_add___help_nargs=0
|
||||
_shtab_ahriman_user_list__h_nargs=0
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
.TH AHRIMAN "1" "2023\-12\-27" "ahriman" "Generated Python Manual"
|
||||
.TH AHRIMAN "1" "2023\-12\-31" "ahriman" "Generated Python Manual"
|
||||
.SH NAME
|
||||
ahriman
|
||||
.SH SYNOPSIS
|
||||
.B ahriman
|
||||
[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--log-handler {console,syslog,journald}] [-q] [--report | --no-report] [-r REPOSITORY] [--unsafe] [-V] [--wait-timeout WAIT_TIMEOUT] {aur-search,search,help-commands-unsafe,help,help-updates,help-version,version,package-add,add,package-update,package-changes,package-changes-remove,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,patch-set-add,repo-backup,repo-check,check,repo-create-keyring,repo-create-mirrorlist,repo-daemon,daemon,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-sign,sign,repo-status-update,repo-sync,sync,repo-tree,repo-triggers,repo-update,update,service-clean,clean,repo-clean,service-config,config,repo-config,service-config-validate,config-validate,repo-config-validate,service-key-import,key-import,service-repositories,service-run,run,service-setup,init,repo-init,repo-setup,setup,service-shell,shell,service-tree-migrate,user-add,user-list,user-remove,web} ...
|
||||
[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--log-handler {console,syslog,journald}] [-q] [--report | --no-report] [-r REPOSITORY] [--unsafe] [-V] [--wait-timeout WAIT_TIMEOUT] {aur-search,search,help-commands-unsafe,help,help-updates,help-version,version,package-add,add,package-update,package-changes,package-changes-remove,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,patch-set-add,repo-backup,repo-check,check,repo-create-keyring,repo-create-mirrorlist,repo-daemon,daemon,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-sign,sign,repo-status-update,repo-sync,sync,repo-tree,repo-triggers,repo-update,update,service-clean,clean,repo-clean,service-config,config,repo-config,service-config-validate,config-validate,repo-config-validate,service-key-import,key-import,service-repositories,service-run,run,service-setup,init,repo-init,repo-setup,setup,service-shell,shell,service-tree-migrate,service-worker-register,service-worker-unregister,user-add,user-list,user-remove,web} ...
|
||||
.SH DESCRIPTION
|
||||
ArcH linux ReposItory MANager
|
||||
|
||||
@ -176,6 +176,12 @@ invoke python shell
|
||||
\fBahriman\fR \fI\,service\-tree\-migrate\/\fR
|
||||
migrate repository tree
|
||||
.TP
|
||||
\fBahriman\fR \fI\,service\-worker\-register\/\fR
|
||||
register itself as worker
|
||||
.TP
|
||||
\fBahriman\fR \fI\,service\-worker\-unregister\/\fR
|
||||
unregister itself as worker
|
||||
.TP
|
||||
\fBahriman\fR \fI\,user\-add\/\fR
|
||||
create or update user
|
||||
.TP
|
||||
@ -869,6 +875,16 @@ usage: ahriman service\-tree\-migrate [\-h]
|
||||
|
||||
migrate repository tree between versions
|
||||
|
||||
.SH COMMAND \fI\,'ahriman service\-worker\-register'\/\fR
|
||||
usage: ahriman service\-worker\-register [\-h]
|
||||
|
||||
call remote service registering itself as available worker
|
||||
|
||||
.SH COMMAND \fI\,'ahriman service\-worker\-unregister'\/\fR
|
||||
usage: ahriman service\-worker\-unregister [\-h]
|
||||
|
||||
call remote service removing itself from list of available workers
|
||||
|
||||
.SH COMMAND \fI\,'ahriman user\-add'\/\fR
|
||||
usage: ahriman user\-add [\-h] [\-\-key KEY] [\-\-packager PACKAGER] [\-p PASSWORD] [\-R {unauthorized,read,reporter,full}]
|
||||
username
|
||||
|
||||
@ -65,6 +65,8 @@ _shtab_ahriman_commands() {
|
||||
"service-setup:create initial service configuration, requires root"
|
||||
"service-shell:drop into python shell"
|
||||
"service-tree-migrate:migrate repository tree between versions"
|
||||
"service-worker-register:call remote service registering itself as available worker"
|
||||
"service-worker-unregister:call remote service removing itself from list of available workers"
|
||||
"setup:create initial service configuration, requires root"
|
||||
"shell:drop into python shell"
|
||||
"sign:(re-)sign packages and repository database according to current settings"
|
||||
@ -552,6 +554,14 @@ _shtab_ahriman_service_tree_migrate_options=(
|
||||
"(- : *)"{-h,--help}"[show this help message and exit]"
|
||||
)
|
||||
|
||||
_shtab_ahriman_service_worker_register_options=(
|
||||
"(- : *)"{-h,--help}"[show this help message and exit]"
|
||||
)
|
||||
|
||||
_shtab_ahriman_service_worker_unregister_options=(
|
||||
"(- : *)"{-h,--help}"[show this help message and exit]"
|
||||
)
|
||||
|
||||
_shtab_ahriman_setup_options=(
|
||||
"(- : *)"{-h,--help}"[show this help message and exit]"
|
||||
"--build-as-user[force makepkg user to the specific one (default\: None)]:build_as_user:"
|
||||
@ -717,6 +727,8 @@ _shtab_ahriman() {
|
||||
service-setup) _arguments -C -s $_shtab_ahriman_service_setup_options ;;
|
||||
service-shell) _arguments -C -s $_shtab_ahriman_service_shell_options ;;
|
||||
service-tree-migrate) _arguments -C -s $_shtab_ahriman_service_tree_migrate_options ;;
|
||||
service-worker-register) _arguments -C -s $_shtab_ahriman_service_worker_register_options ;;
|
||||
service-worker-unregister) _arguments -C -s $_shtab_ahriman_service_worker_unregister_options ;;
|
||||
setup) _arguments -C -s $_shtab_ahriman_setup_options ;;
|
||||
shell) _arguments -C -s $_shtab_ahriman_shell_options ;;
|
||||
sign) _arguments -C -s $_shtab_ahriman_sign_options ;;
|
||||
|
||||
@ -8,4 +8,4 @@
|
||||
6. All updates from worker instances are uploaded to the web service.
|
||||
7. Repository is available at `http://localhost:8080/repo`.
|
||||
|
||||
Note, in this configuration, workers are spawned in replicated mode, thus the backend accesses them in round-robin-like manner.
|
||||
In this example, worker list is automatically defined based on the addresses they reported.
|
||||
|
||||
@ -77,6 +77,10 @@ services:
|
||||
interval: 10s
|
||||
start_period: 30s
|
||||
|
||||
depends_on:
|
||||
backend:
|
||||
condition: service_healthy
|
||||
|
||||
command: web
|
||||
|
||||
configs:
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
target = configuration
|
||||
|
||||
[build]
|
||||
workers = http://worker:8080 http://worker:8080
|
||||
triggers = ahriman.core.distributed.WorkerLoaderTrigger ahriman.core.gitremote.RemotePullTrigger ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger ahriman.core.gitremote.RemotePushTrigger
|
||||
|
||||
[status]
|
||||
username = demo
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
target = configuration
|
||||
|
||||
[build]
|
||||
triggers = ahriman.core.upload.UploadTrigger ahriman.core.report.ReportTrigger
|
||||
triggers = ahriman.core.distributed.WorkerTrigger ahriman.core.upload.UploadTrigger ahriman.core.report.ReportTrigger
|
||||
|
||||
[status]
|
||||
address = http://backend:8080
|
||||
@ -20,3 +20,6 @@ wait_timeout = 0
|
||||
target = remote-service
|
||||
|
||||
[remote-service]
|
||||
|
||||
[worker]
|
||||
address = http://$HOSTNAME:8080
|
||||
|
||||
@ -135,6 +135,8 @@ def _parser() -> argparse.ArgumentParser:
|
||||
_set_service_setup_parser(subparsers)
|
||||
_set_service_shell_parser(subparsers)
|
||||
_set_service_tree_migrate_parser(subparsers)
|
||||
_set_service_worker_register_parser(subparsers)
|
||||
_set_service_worker_unregister_parser(subparsers)
|
||||
_set_user_add_parser(subparsers)
|
||||
_set_user_list_parser(subparsers)
|
||||
_set_user_remove_parser(subparsers)
|
||||
@ -1052,6 +1054,40 @@ def _set_service_tree_migrate_parser(root: SubParserAction) -> argparse.Argument
|
||||
return parser
|
||||
|
||||
|
||||
def _set_service_worker_register_parser(root: SubParserAction) -> argparse.ArgumentParser:
|
||||
"""
|
||||
add parser for remote worker registration subcommand
|
||||
|
||||
Args:
|
||||
root(SubParserAction): subparsers for the commands
|
||||
|
||||
Returns:
|
||||
argparse.ArgumentParser: created argument parser
|
||||
"""
|
||||
parser = root.add_parser("service-worker-register", help="register itself as worker",
|
||||
description="call remote service registering itself as available worker",
|
||||
formatter_class=_formatter)
|
||||
parser.set_defaults(handler=handlers.Triggers, trigger=["ahriman.core.distributed.WorkerRegisterTrigger"])
|
||||
return parser
|
||||
|
||||
|
||||
def _set_service_worker_unregister_parser(root: SubParserAction) -> argparse.ArgumentParser:
|
||||
"""
|
||||
add parser for remote worker removal subcommand
|
||||
|
||||
Args:
|
||||
root(SubParserAction): subparsers for the commands
|
||||
|
||||
Returns:
|
||||
argparse.ArgumentParser: created argument parser
|
||||
"""
|
||||
parser = root.add_parser("service-worker-unregister", help="unregister itself as worker",
|
||||
description="call remote service removing itself from list of available workers",
|
||||
formatter_class=_formatter)
|
||||
parser.set_defaults(handler=handlers.Triggers, trigger=["ahriman.core.distributed.WorkerUnregisterTrigger"])
|
||||
return parser
|
||||
|
||||
|
||||
def _set_user_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
|
||||
"""
|
||||
add parser for create user subcommand
|
||||
|
||||
@ -24,6 +24,7 @@ from collections.abc import Generator
|
||||
from ahriman.application.handlers import Handler
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.spawn import Spawn
|
||||
from ahriman.core.triggers import TriggerLoader
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
|
||||
|
||||
@ -53,13 +54,16 @@ class Web(Handler):
|
||||
spawner = Spawn(args.parser(), list(spawner_args))
|
||||
spawner.start()
|
||||
|
||||
triggers = TriggerLoader.load(repository_id, configuration)
|
||||
triggers.on_start()
|
||||
|
||||
dummy_args = argparse.Namespace(
|
||||
architecture=None,
|
||||
configuration=args.configuration,
|
||||
repository=None,
|
||||
repository_id=None,
|
||||
)
|
||||
repositories = cls.repositories_extract(dummy_args)
|
||||
repositories = Web.repositories_extract(dummy_args)
|
||||
application = setup_server(configuration, spawner, repositories)
|
||||
run_server(application)
|
||||
|
||||
|
||||
31
src/ahriman/core/database/migrations/m013_workers.py
Normal file
31
src/ahriman/core/database/migrations/m013_workers.py
Normal file
@ -0,0 +1,31 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
__all__ = ["steps"]
|
||||
|
||||
|
||||
steps = [
|
||||
"""
|
||||
create table workers (
|
||||
identifier text not null,
|
||||
address text not null,
|
||||
unique (identifier)
|
||||
)
|
||||
"""
|
||||
]
|
||||
@ -25,3 +25,4 @@ from ahriman.core.database.operations.changes_operations import ChangesOperation
|
||||
from ahriman.core.database.operations.logs_operations import LogsOperations
|
||||
from ahriman.core.database.operations.package_operations import PackageOperations
|
||||
from ahriman.core.database.operations.patch_operations import PatchOperations
|
||||
from ahriman.core.database.operations.workers_operations import WorkersOperations
|
||||
|
||||
83
src/ahriman/core/database/operations/workers_operations.py
Normal file
83
src/ahriman/core/database/operations/workers_operations.py
Normal file
@ -0,0 +1,83 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from sqlite3 import Connection
|
||||
|
||||
from ahriman.core.database.operations import Operations
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
class WorkersOperations(Operations):
|
||||
"""
|
||||
operations for remote workers
|
||||
"""
|
||||
|
||||
def workers_get(self) -> list[Worker]:
|
||||
"""
|
||||
retrieve registered workers
|
||||
|
||||
Returns:
|
||||
list[Worker]: list of available workers
|
||||
"""
|
||||
def run(connection: Connection) -> list[Worker]:
|
||||
return [
|
||||
Worker(row["address"], identifier=row["identifier"])
|
||||
for row in connection.execute("""select * from workers""")
|
||||
]
|
||||
|
||||
return self.with_connection(run)
|
||||
|
||||
def workers_insert(self, worker: Worker) -> None:
|
||||
"""
|
||||
insert or update worker in database
|
||||
|
||||
Args:
|
||||
worker(Worker): remote worker descriptor
|
||||
"""
|
||||
def run(connection: Connection) -> None:
|
||||
connection.execute(
|
||||
"""
|
||||
insert into workers
|
||||
(identifier, address)
|
||||
values
|
||||
(:identifier, :address)
|
||||
on conflict (identifier) do update set
|
||||
address = :address
|
||||
""",
|
||||
worker.view()
|
||||
)
|
||||
|
||||
return self.with_connection(run, commit=True)
|
||||
|
||||
def workers_remove(self, identifier: str | None = None) -> None:
|
||||
"""
|
||||
unregister remote worker
|
||||
|
||||
Args:
|
||||
identifier(str | None, optional): remote worker identifier. If none set it will clear all workers
|
||||
(Default value = None)
|
||||
"""
|
||||
def run(connection: Connection) -> None:
|
||||
connection.execute(
|
||||
"""
|
||||
delete from workers where (:identifier is null or identifier = :identifier)
|
||||
""",
|
||||
{"identifier": identifier})
|
||||
|
||||
return self.with_connection(run, commit=True)
|
||||
@ -26,11 +26,12 @@ from typing import Self
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.database.migrations import Migrations
|
||||
from ahriman.core.database.operations import AuthOperations, BuildOperations, ChangesOperations, LogsOperations, \
|
||||
PackageOperations, PatchOperations
|
||||
PackageOperations, PatchOperations, WorkersOperations
|
||||
|
||||
|
||||
# pylint: disable=too-many-ancestors
|
||||
class SQLite(AuthOperations, BuildOperations, ChangesOperations, LogsOperations, PackageOperations, PatchOperations):
|
||||
class SQLite(AuthOperations, BuildOperations, ChangesOperations, LogsOperations, PackageOperations, PatchOperations,
|
||||
WorkersOperations):
|
||||
"""
|
||||
wrapper for sqlite3 database
|
||||
|
||||
|
||||
23
src/ahriman/core/distributed/__init__.py
Normal file
23
src/ahriman/core/distributed/__init__.py
Normal file
@ -0,0 +1,23 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from ahriman.core.distributed.worker_loader_trigger import WorkerLoaderTrigger
|
||||
from ahriman.core.distributed.worker_register_trigger import WorkerRegisterTrigger
|
||||
from ahriman.core.distributed.worker_trigger import WorkerTrigger
|
||||
from ahriman.core.distributed.worker_unregister_trigger import WorkerUnregisterTrigger
|
||||
183
src/ahriman/core/distributed/distributed_system.py
Normal file
183
src/ahriman/core/distributed/distributed_system.py
Normal file
@ -0,0 +1,183 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import contextlib
|
||||
import tempfile
|
||||
import uuid
|
||||
|
||||
from pathlib import Path
|
||||
from functools import cached_property
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.configuration.schema import ConfigurationSchema
|
||||
from ahriman.core.status.web_client import WebClient
|
||||
from ahriman.core.triggers import Trigger
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
class DistributedSystem(Trigger, WebClient):
|
||||
"""
|
||||
simple class to (un)register itself as a distributed worker
|
||||
|
||||
Attributes:
|
||||
identifier_path(Path): path to cached worker identifier
|
||||
"""
|
||||
|
||||
CONFIGURATION_SCHEMA: ConfigurationSchema = {
|
||||
"worker": {
|
||||
"type": "dict",
|
||||
"schema": {
|
||||
"address": {
|
||||
"type": "string",
|
||||
"required": True,
|
||||
"empty": False,
|
||||
"is_url": [],
|
||||
},
|
||||
"identifier": {
|
||||
"type": "string",
|
||||
"empty": False,
|
||||
},
|
||||
"identifier_path": {
|
||||
"type": "path",
|
||||
"coerce": "absolute_path",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
|
||||
"""
|
||||
default constructor
|
||||
|
||||
Args:
|
||||
repository_id(RepositoryId): repository unique identifier
|
||||
configuration(Configuration): configuration instance
|
||||
"""
|
||||
Trigger.__init__(self, repository_id, configuration)
|
||||
WebClient.__init__(self, repository_id, configuration)
|
||||
|
||||
section = next(iter(self.configuration_sections(configuration)))
|
||||
self.identifier_path = configuration.getpath(
|
||||
section, "identifier_path", fallback=Path(tempfile.gettempdir()) / "ahriman-worker-identifier")
|
||||
self._owe_identifier = False
|
||||
|
||||
@cached_property
|
||||
def worker(self) -> Worker:
|
||||
"""
|
||||
load and set worker. Lazy property loaded because it is not always required
|
||||
|
||||
Returns:
|
||||
Worker: unique self worker identifier
|
||||
"""
|
||||
section = next(iter(self.configuration_sections(self.configuration)))
|
||||
identifier = self.load_identifier(self.configuration, section)
|
||||
return Worker(self.configuration.get(section, "address"), identifier=identifier)
|
||||
|
||||
@classmethod
|
||||
def configuration_sections(cls, configuration: Configuration) -> list[str]:
|
||||
"""
|
||||
extract configuration sections from configuration
|
||||
|
||||
Args:
|
||||
configuration(Configuration): configuration instance
|
||||
|
||||
Returns:
|
||||
list[str]: read configuration sections belong to this trigger
|
||||
"""
|
||||
return list(cls.CONFIGURATION_SCHEMA.keys())
|
||||
|
||||
def _workers_url(self, identifier: str = "") -> str:
|
||||
"""
|
||||
workers url generator
|
||||
|
||||
Args:
|
||||
identifier(str, optional): worker identifier (Default value = "")
|
||||
|
||||
Returns:
|
||||
str: full url of web service for specific worker
|
||||
"""
|
||||
suffix = f"/{identifier}" if identifier else ""
|
||||
return f"{self.address}/api/v1/distributed{suffix}"
|
||||
|
||||
def load_identifier(self, configuration: Configuration, section: str) -> str:
|
||||
"""
|
||||
load identifier from filesystem if available or from configuration otherwise. If cache file is available,
|
||||
the method will read from it. Otherwise, it will try to read it from configuration. And, finally, if no
|
||||
identifier set, it will generate uuid
|
||||
|
||||
Args:
|
||||
configuration(Configuration): configuration instance
|
||||
section(str): settings section name
|
||||
|
||||
Returns:
|
||||
str: unique worker identifier
|
||||
"""
|
||||
if self.identifier_path.is_file(): # load cached value
|
||||
return self.identifier_path.read_text(encoding="utf8")
|
||||
return configuration.get(section, "identifier", fallback=str(uuid.uuid4()))
|
||||
|
||||
def register(self, force: bool = False) -> None:
|
||||
"""
|
||||
register itself in remote system
|
||||
|
||||
Args:
|
||||
force(bool, optional): register worker even if it has been already registered before (Default value = False)
|
||||
"""
|
||||
if self.identifier_path.is_file() and not force:
|
||||
return # there is already registered identifier
|
||||
|
||||
self.make_request("POST", self._workers_url(), json=self.worker.view())
|
||||
# save identifier
|
||||
self.identifier_path.write_text(self.worker.identifier, encoding="utf8")
|
||||
self._owe_identifier = True
|
||||
self.logger.info("registered instance %s at %s", self.worker, self.address)
|
||||
|
||||
def unregister(self, force: bool = False) -> None:
|
||||
"""
|
||||
unregister itself in remote system
|
||||
|
||||
Args:
|
||||
force(bool, optional): unregister worker even if it has been registered in another process
|
||||
(Default value = False)
|
||||
"""
|
||||
if not self._owe_identifier and not force:
|
||||
return # we do not owe this identifier
|
||||
|
||||
self.make_request("DELETE", self._workers_url(self.worker.identifier))
|
||||
self.identifier_path.unlink(missing_ok=True)
|
||||
self.logger.info("unregistered instance %s at %s", self.worker, self.address)
|
||||
|
||||
def workers(self) -> list[Worker]:
|
||||
"""
|
||||
retrieve list of available remote workers
|
||||
|
||||
Returns:
|
||||
list[Worker]: currently registered workers
|
||||
"""
|
||||
with contextlib.suppress(Exception):
|
||||
response = self.make_request("GET", self._workers_url())
|
||||
response_json = response.json()
|
||||
|
||||
return [
|
||||
Worker(worker["address"], identifier=worker["identifier"])
|
||||
for worker in response_json
|
||||
]
|
||||
|
||||
return []
|
||||
40
src/ahriman/core/distributed/worker_loader_trigger.py
Normal file
40
src/ahriman/core/distributed/worker_loader_trigger.py
Normal file
@ -0,0 +1,40 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from ahriman.core.distributed.distributed_system import DistributedSystem
|
||||
|
||||
|
||||
class WorkerLoaderTrigger(DistributedSystem):
|
||||
"""
|
||||
remote worker processor trigger (server side)
|
||||
"""
|
||||
|
||||
def on_start(self) -> None:
|
||||
"""
|
||||
trigger action which will be called at the start of the application
|
||||
"""
|
||||
if self.configuration.has_option("build", "workers"):
|
||||
return # there is manually set option
|
||||
|
||||
workers = [worker.address for worker in self.workers()]
|
||||
if not workers:
|
||||
return
|
||||
|
||||
self.logger.info("load workers %s", workers)
|
||||
self.configuration.set_option("build", "workers", " ".join(workers))
|
||||
32
src/ahriman/core/distributed/worker_register_trigger.py
Normal file
32
src/ahriman/core/distributed/worker_register_trigger.py
Normal file
@ -0,0 +1,32 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from ahriman.core.distributed.distributed_system import DistributedSystem
|
||||
|
||||
|
||||
class WorkerRegisterTrigger(DistributedSystem):
|
||||
"""
|
||||
remote worker registration trigger
|
||||
"""
|
||||
|
||||
def on_start(self) -> None:
|
||||
"""
|
||||
trigger action which will be called at the start of the application
|
||||
"""
|
||||
self.register(force=True)
|
||||
38
src/ahriman/core/distributed/worker_trigger.py
Normal file
38
src/ahriman/core/distributed/worker_trigger.py
Normal file
@ -0,0 +1,38 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from ahriman.core.distributed.distributed_system import DistributedSystem
|
||||
|
||||
|
||||
class WorkerTrigger(DistributedSystem):
|
||||
"""
|
||||
remote worker processor trigger (client side)
|
||||
"""
|
||||
|
||||
def on_start(self) -> None:
|
||||
"""
|
||||
trigger action which will be called at the start of the application
|
||||
"""
|
||||
self.register()
|
||||
|
||||
def on_stop(self) -> None:
|
||||
"""
|
||||
trigger action which will be called before the stop of the application
|
||||
"""
|
||||
self.unregister()
|
||||
32
src/ahriman/core/distributed/worker_unregister_trigger.py
Normal file
32
src/ahriman/core/distributed/worker_unregister_trigger.py
Normal file
@ -0,0 +1,32 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from ahriman.core.distributed.distributed_system import DistributedSystem
|
||||
|
||||
|
||||
class WorkerUnregisterTrigger(DistributedSystem):
|
||||
"""
|
||||
remote worker registration trigger
|
||||
"""
|
||||
|
||||
def on_start(self) -> None:
|
||||
"""
|
||||
trigger action which will be called at the start of the application
|
||||
"""
|
||||
self.unregister(force=True)
|
||||
@ -47,8 +47,8 @@ class SyncHttpClient(LazyLogging):
|
||||
default constructor
|
||||
|
||||
Args:
|
||||
configuration(Configuration | None): configuration instance (Default value = None)
|
||||
section(str, optional): settings section name (Default value = None)
|
||||
configuration(Configuration | None, optional): configuration instance (Default value = None)
|
||||
section(str | None, optional): settings section name (Default value = None)
|
||||
suppress_errors(bool, optional): suppress logging of request errors (Default value = False)
|
||||
"""
|
||||
if configuration is None:
|
||||
|
||||
@ -26,6 +26,7 @@ from ahriman.models.log_record_id import LogRecordId
|
||||
from ahriman.models.package import Package
|
||||
from ahriman.models.pkgbuild_patch import PkgbuildPatch
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
class Watcher(LazyLogging):
|
||||
@ -223,3 +224,30 @@ class Watcher(LazyLogging):
|
||||
status(BuildStatusEnum): new service status
|
||||
"""
|
||||
self.status = BuildStatus(status)
|
||||
|
||||
def workers_get(self) -> list[Worker]:
|
||||
"""
|
||||
retrieve registered remote workers
|
||||
|
||||
Returns:
|
||||
list[Worker]: list of currently available workers
|
||||
"""
|
||||
return self.database.workers_get()
|
||||
|
||||
def workers_remove(self, identifier: str | None = None) -> None:
|
||||
"""
|
||||
unregister remote worker
|
||||
|
||||
Args:
|
||||
identifier(str | None, optional): remote worker identifier if any (Default value = None)
|
||||
"""
|
||||
self.database.workers_remove(identifier)
|
||||
|
||||
def workers_update(self, worker: Worker) -> None:
|
||||
"""
|
||||
register or update remote worker
|
||||
|
||||
Args:
|
||||
worker(Worker): worker to register
|
||||
"""
|
||||
self.database.workers_insert(worker)
|
||||
|
||||
@ -118,7 +118,6 @@ class WebClient(Client, SyncAhrimanClient):
|
||||
Returns:
|
||||
str: full url of web service for specific package base
|
||||
"""
|
||||
# in case if unix socket is used we need to normalize url
|
||||
suffix = f"/{package_base}" if package_base else ""
|
||||
return f"{self.address}/api/v1/packages{suffix}"
|
||||
|
||||
|
||||
@ -262,6 +262,6 @@ class TriggerLoader(LazyLogging):
|
||||
run triggers before the application exit
|
||||
"""
|
||||
self.logger.debug("executing triggers on stop")
|
||||
for trigger in self.triggers:
|
||||
for trigger in reversed(self.triggers):
|
||||
with self.__execute_trigger(trigger):
|
||||
trigger.on_stop()
|
||||
|
||||
@ -18,8 +18,11 @@
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ahriman.core.util import dataclass_view
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Worker:
|
||||
@ -39,3 +42,12 @@ class Worker:
|
||||
update identifier based on settings
|
||||
"""
|
||||
object.__setattr__(self, "identifier", self.identifier or urlparse(self.address).netloc)
|
||||
|
||||
def view(self) -> dict[str, Any]:
|
||||
"""
|
||||
generate json patch view
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: json-friendly dictionary
|
||||
"""
|
||||
return dataclass_view(self)
|
||||
|
||||
@ -47,5 +47,7 @@ from ahriman.web.schemas.remote_schema import RemoteSchema
|
||||
from ahriman.web.schemas.repository_id_schema import RepositoryIdSchema
|
||||
from ahriman.web.schemas.search_schema import SearchSchema
|
||||
from ahriman.web.schemas.status_schema import StatusSchema
|
||||
from ahriman.web.schemas.versioned_log_schema import VersionedLogSchema
|
||||
from ahriman.web.schemas.update_flags_schema import UpdateFlagsSchema
|
||||
from ahriman.web.schemas.versioned_log_schema import VersionedLogSchema
|
||||
from ahriman.web.schemas.worker_id_schema import WorkerIdSchema
|
||||
from ahriman.web.schemas.worker_schema import WorkerSchema
|
||||
|
||||
31
src/ahriman/web/schemas/worker_id_schema.py
Normal file
31
src/ahriman/web/schemas/worker_id_schema.py
Normal file
@ -0,0 +1,31 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from marshmallow import Schema, fields
|
||||
|
||||
|
||||
class WorkerIdSchema(Schema):
|
||||
"""
|
||||
request and response schema for workers
|
||||
"""
|
||||
|
||||
identifier = fields.String(required=True, metadata={
|
||||
"description": "Worker unique identifier",
|
||||
"example": "42f03a62-48f7-46b7-af40-dacc720e92fa",
|
||||
})
|
||||
33
src/ahriman/web/schemas/worker_schema.py
Normal file
33
src/ahriman/web/schemas/worker_schema.py
Normal file
@ -0,0 +1,33 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from marshmallow import fields
|
||||
|
||||
from ahriman.web.schemas.worker_id_schema import WorkerIdSchema
|
||||
|
||||
|
||||
class WorkerSchema(WorkerIdSchema):
|
||||
"""
|
||||
request and response schema for workers
|
||||
"""
|
||||
|
||||
address = fields.String(required=True, metadata={
|
||||
"description": "Worker address",
|
||||
"example": "http://localhost:8081",
|
||||
})
|
||||
19
src/ahriman/web/views/v1/distributed/__init__.py
Normal file
19
src/ahriman/web/views/v1/distributed/__init__.py
Normal file
@ -0,0 +1,19 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
99
src/ahriman/web/views/v1/distributed/worker.py
Normal file
99
src/ahriman/web/views/v1/distributed/worker.py
Normal file
@ -0,0 +1,99 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import aiohttp_apispec # type: ignore[import-untyped]
|
||||
|
||||
from aiohttp.web import HTTPNoContent, HTTPNotFound, Response, json_response
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, WorkerIdSchema, WorkerSchema
|
||||
from ahriman.web.views.base import BaseView
|
||||
|
||||
|
||||
class WorkerView(BaseView):
|
||||
"""
|
||||
distributed worker view
|
||||
|
||||
Attributes:
|
||||
DELETE_PERMISSION(UserAccess): (class attribute) delete permissions of self
|
||||
GET_PERMISSION(UserAccess): (class attribute) get permissions of self
|
||||
"""
|
||||
|
||||
DELETE_PERMISSION = GET_PERMISSION = UserAccess.Full
|
||||
ROUTES = ["/api/v1/distributed/{identifier}"]
|
||||
|
||||
@aiohttp_apispec.docs(
|
||||
tags=["Distributed"],
|
||||
summary="Unregister worker",
|
||||
description="Unregister worker and remove it from the service",
|
||||
responses={
|
||||
204: {"description": "Success response"},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
500: {"description": "Internal server error", "schema": ErrorSchema},
|
||||
},
|
||||
security=[{"token": [DELETE_PERMISSION]}],
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
@aiohttp_apispec.match_info_schema(WorkerIdSchema)
|
||||
async def delete(self) -> None:
|
||||
"""
|
||||
unregister worker
|
||||
|
||||
Raises:
|
||||
HTTPNoContent: on success response
|
||||
"""
|
||||
identifier = self.request.match_info["identifier"]
|
||||
self.service().workers_remove(identifier)
|
||||
|
||||
raise HTTPNoContent
|
||||
|
||||
@aiohttp_apispec.docs(
|
||||
tags=["Distributed"],
|
||||
summary="Get worker",
|
||||
description="Retrieve registered worker by its identifier",
|
||||
responses={
|
||||
200: {"description": "Success response", "schema": WorkerSchema(many=True)},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
404: {"description": "Worker is unknown", "schema": ErrorSchema},
|
||||
500: {"description": "Internal server error", "schema": ErrorSchema},
|
||||
},
|
||||
security=[{"token": [GET_PERMISSION]}],
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
@aiohttp_apispec.match_info_schema(WorkerIdSchema)
|
||||
async def get(self) -> Response:
|
||||
"""
|
||||
get worker by identifier
|
||||
|
||||
Returns:
|
||||
Response: 200 with workers list on success
|
||||
|
||||
Raises:
|
||||
HTTPNotFound: if no worker was found
|
||||
"""
|
||||
identifier = self.request.match_info["identifier"]
|
||||
|
||||
try:
|
||||
worker = next(worker for worker in self.service().workers_get() if worker.identifier == identifier)
|
||||
except StopIteration:
|
||||
raise HTTPNotFound(reason=f"Worker {identifier} not found")
|
||||
|
||||
return json_response([worker.view()])
|
||||
126
src/ahriman/web/views/v1/distributed/workers.py
Normal file
126
src/ahriman/web/views/v1/distributed/workers.py
Normal file
@ -0,0 +1,126 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import aiohttp_apispec # type: ignore[import-untyped]
|
||||
|
||||
from collections.abc import Callable
|
||||
from aiohttp.web import HTTPBadRequest, HTTPNoContent, Response, json_response
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.models.worker import Worker
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, WorkerSchema
|
||||
from ahriman.web.views.base import BaseView
|
||||
|
||||
|
||||
class WorkersView(BaseView):
|
||||
"""
|
||||
distributed workers view
|
||||
|
||||
Attributes:
|
||||
DELETE_PERMISSION(UserAccess): (class attribute) delete permissions of self
|
||||
GET_PERMISSION(UserAccess): (class attribute) get permissions of self
|
||||
POST_PERMISSION(UserAccess): (class attribute) post permissions of self
|
||||
"""
|
||||
|
||||
DELETE_PERMISSION = GET_PERMISSION = POST_PERMISSION = UserAccess.Full
|
||||
ROUTES = ["/api/v1/distributed"]
|
||||
|
||||
@aiohttp_apispec.docs(
|
||||
tags=["Distributed"],
|
||||
summary="Unregister all workers",
|
||||
description="Unregister and remove all known workers from the service",
|
||||
responses={
|
||||
204: {"description": "Success response"},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
500: {"description": "Internal server error", "schema": ErrorSchema},
|
||||
},
|
||||
security=[{"token": [DELETE_PERMISSION]}],
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
async def delete(self) -> None:
|
||||
"""
|
||||
unregister worker
|
||||
|
||||
Raises:
|
||||
HTTPNoContent: on success response
|
||||
"""
|
||||
self.service().workers_remove()
|
||||
|
||||
raise HTTPNoContent
|
||||
|
||||
@aiohttp_apispec.docs(
|
||||
tags=["Distributed"],
|
||||
summary="Get workers",
|
||||
description="Retrieve registered workers",
|
||||
responses={
|
||||
200: {"description": "Success response", "schema": WorkerSchema(many=True)},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
500: {"description": "Internal server error", "schema": ErrorSchema},
|
||||
},
|
||||
security=[{"token": [GET_PERMISSION]}],
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
async def get(self) -> Response:
|
||||
"""
|
||||
get workers list
|
||||
|
||||
Returns:
|
||||
Response: 200 with workers list on success
|
||||
"""
|
||||
workers = self.service().workers_get()
|
||||
|
||||
comparator: Callable[[Worker], str] = lambda item: item.identifier
|
||||
response = [worker.view() for worker in sorted(workers, key=comparator)]
|
||||
|
||||
return json_response(response)
|
||||
|
||||
@aiohttp_apispec.docs(
|
||||
tags=["Distributed"],
|
||||
summary="Register worker",
|
||||
description="Register or update remote worker",
|
||||
responses={
|
||||
204: {"description": "Success response"},
|
||||
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
500: {"description": "Internal server error", "schema": ErrorSchema},
|
||||
},
|
||||
security=[{"token": [POST_PERMISSION]}],
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
@aiohttp_apispec.json_schema(WorkerSchema)
|
||||
async def post(self) -> None:
|
||||
"""
|
||||
register remote worker
|
||||
|
||||
Raises:
|
||||
HTTPBadRequest: if bad data is supplied
|
||||
HTTPNoContent: in case of success response
|
||||
"""
|
||||
try:
|
||||
data = await self.request.json()
|
||||
worker = Worker(data["address"], identifier=data["identifier"])
|
||||
except Exception as ex:
|
||||
raise HTTPBadRequest(reason=str(ex))
|
||||
|
||||
self.service().workers_update(worker)
|
||||
|
||||
raise HTTPNoContent
|
||||
@ -79,6 +79,7 @@ def test_schema(configuration: Configuration) -> None:
|
||||
assert schema.pop("s3")
|
||||
assert schema.pop("telegram")
|
||||
assert schema.pop("upload")
|
||||
assert schema.pop("worker")
|
||||
|
||||
assert schema == CONFIGURATION_SCHEMA
|
||||
|
||||
|
||||
@ -39,6 +39,7 @@ def test_run(args: argparse.Namespace, configuration: Configuration, repository:
|
||||
setup_mock = mocker.patch("ahriman.web.web.setup_server")
|
||||
run_mock = mocker.patch("ahriman.web.web.run_server")
|
||||
start_mock = mocker.patch("ahriman.core.spawn.Spawn.start")
|
||||
trigger_mock = mocker.patch("ahriman.core.triggers.TriggerLoader.load")
|
||||
stop_mock = mocker.patch("ahriman.core.spawn.Spawn.stop")
|
||||
join_mock = mocker.patch("ahriman.core.spawn.Spawn.join")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
@ -48,6 +49,8 @@ def test_run(args: argparse.Namespace, configuration: Configuration, repository:
|
||||
setup_mock.assert_called_once_with(configuration, pytest.helpers.anyvar(int), [repository_id])
|
||||
run_mock.assert_called_once_with(pytest.helpers.anyvar(int))
|
||||
start_mock.assert_called_once_with()
|
||||
trigger_mock.assert_called_once_with(repository_id, configuration)
|
||||
trigger_mock().on_start.assert_called_once_with()
|
||||
stop_mock.assert_called_once_with()
|
||||
join_mock.assert_called_once_with()
|
||||
|
||||
|
||||
@ -1320,6 +1320,80 @@ def test_subparsers_service_tree_migrate(parser: argparse.ArgumentParser) -> Non
|
||||
assert not args.report
|
||||
|
||||
|
||||
def test_subparsers_service_worker_register(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-register command must imply trigger
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-register"])
|
||||
assert args.trigger == ["ahriman.core.distributed.WorkerRegisterTrigger"]
|
||||
|
||||
|
||||
def test_subparsers_service_worker_register_option_architecture(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-register command must correctly parse architecture list
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-register"])
|
||||
assert args.architecture is None
|
||||
args = parser.parse_args(["-a", "x86_64", "service-worker-register"])
|
||||
assert args.architecture == "x86_64"
|
||||
|
||||
|
||||
def test_subparsers_service_worker_register_option_repository(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-register command must correctly parse repository list
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-register"])
|
||||
assert args.repository is None
|
||||
args = parser.parse_args(["-r", "repo", "service-worker-register"])
|
||||
assert args.repository == "repo"
|
||||
|
||||
|
||||
def test_subparsers_service_worker_register_repo_triggers(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-register must have same keys as repo-triggers
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-register"])
|
||||
reference_args = parser.parse_args(["repo-triggers"])
|
||||
assert dir(args) == dir(reference_args)
|
||||
|
||||
|
||||
def test_subparsers_service_worker_unregister(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-unregister command must imply trigger
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-unregister"])
|
||||
assert args.trigger == ["ahriman.core.distributed.WorkerUnregisterTrigger"]
|
||||
|
||||
|
||||
def test_subparsers_service_worker_unregister_option_architecture(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-unregister command must correctly parse architecture list
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-unregister"])
|
||||
assert args.architecture is None
|
||||
args = parser.parse_args(["-a", "x86_64", "service-worker-unregister"])
|
||||
assert args.architecture == "x86_64"
|
||||
|
||||
|
||||
def test_subparsers_service_worker_unregister_option_repository(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-unregister command must correctly parse repository list
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-unregister"])
|
||||
assert args.repository is None
|
||||
args = parser.parse_args(["-r", "repo", "service-worker-unregister"])
|
||||
assert args.repository == "repo"
|
||||
|
||||
|
||||
def test_subparsers_service_worker_unregister_repo_triggers(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-unregister must have same keys as repo-triggers
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-unregister"])
|
||||
reference_args = parser.parse_args(["repo-triggers"])
|
||||
assert dir(args) == dir(reference_args)
|
||||
|
||||
|
||||
def test_subparsers_user_add(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
user-add command must imply action, architecture, exit code, lock, quiet, report and repository
|
||||
|
||||
@ -0,0 +1,8 @@
|
||||
from ahriman.core.database.migrations.m013_workers import steps
|
||||
|
||||
|
||||
def test_migration_workers() -> None:
|
||||
"""
|
||||
migration must not be empty
|
||||
"""
|
||||
assert steps
|
||||
@ -0,0 +1,46 @@
|
||||
from ahriman.core.database import SQLite
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
def test_workers_get_insert(database: SQLite) -> None:
|
||||
"""
|
||||
must insert workers to database
|
||||
"""
|
||||
database.workers_insert(Worker("address1", identifier="1"))
|
||||
database.workers_insert(Worker("address2", identifier="2"))
|
||||
assert database.workers_get() == [
|
||||
Worker("address1", identifier="1"), Worker("address2", identifier="2")
|
||||
]
|
||||
|
||||
|
||||
def test_workers_insert_remove(database: SQLite) -> None:
|
||||
"""
|
||||
must remove worker from database
|
||||
"""
|
||||
database.workers_insert(Worker("address1", identifier="1"))
|
||||
database.workers_insert(Worker("address2", identifier="2"))
|
||||
database.workers_remove("1")
|
||||
|
||||
assert database.workers_get() == [Worker("address2", identifier="2")]
|
||||
|
||||
|
||||
def test_workers_insert_remove_all(database: SQLite) -> None:
|
||||
"""
|
||||
must remove all workers
|
||||
"""
|
||||
database.workers_insert(Worker("address1", identifier="1"))
|
||||
database.workers_insert(Worker("address2", identifier="2"))
|
||||
database.workers_remove()
|
||||
|
||||
assert database.workers_get() == []
|
||||
|
||||
|
||||
def test_workers_insert_insert(database: SQLite) -> None:
|
||||
"""
|
||||
must update worker in database
|
||||
"""
|
||||
database.workers_insert(Worker("address1", identifier="1"))
|
||||
assert database.workers_get() == [Worker("address1", identifier="1")]
|
||||
|
||||
database.workers_insert(Worker("address2", identifier="1"))
|
||||
assert database.workers_get() == [Worker("address2", identifier="1")]
|
||||
20
tests/ahriman/core/distributed/conftest.py
Normal file
20
tests/ahriman/core/distributed/conftest.py
Normal file
@ -0,0 +1,20 @@
|
||||
import pytest
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.distributed.distributed_system import DistributedSystem
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def distributed_system(configuration: Configuration) -> DistributedSystem:
|
||||
"""
|
||||
distributed system fixture
|
||||
|
||||
Args:
|
||||
configuration(Configuration): configuration fixture
|
||||
|
||||
Returns:
|
||||
DistributedSystem: distributed system test instance
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
return DistributedSystem(repository_id, configuration)
|
||||
198
tests/ahriman/core/distributed/test_distributed_system.py
Normal file
198
tests/ahriman/core/distributed/test_distributed_system.py
Normal file
@ -0,0 +1,198 @@
|
||||
import json
|
||||
import requests
|
||||
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.distributed.distributed_system import DistributedSystem
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
def test_identifier_path(configuration: Configuration) -> None:
|
||||
"""
|
||||
must correctly set default identifier path
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
assert DistributedSystem(repository_id, configuration).identifier_path
|
||||
|
||||
|
||||
def test_configuration_sections(configuration: Configuration) -> None:
|
||||
"""
|
||||
must correctly parse target list
|
||||
"""
|
||||
assert DistributedSystem.configuration_sections(configuration) == ["worker"]
|
||||
|
||||
|
||||
def test_workers_url(distributed_system: DistributedSystem) -> None:
|
||||
"""
|
||||
must generate workers url correctly
|
||||
"""
|
||||
assert distributed_system._workers_url().startswith(distributed_system.address)
|
||||
assert distributed_system._workers_url().endswith("/api/v1/distributed")
|
||||
|
||||
assert distributed_system._workers_url("id").startswith(distributed_system.address)
|
||||
assert distributed_system._workers_url("id").endswith("/api/v1/distributed/id")
|
||||
|
||||
|
||||
def test_load_identifier(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must generate identifier
|
||||
"""
|
||||
mocker.patch("pathlib.Path.is_file", return_value=False)
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
system = DistributedSystem(repository_id, configuration)
|
||||
|
||||
assert system.load_identifier(configuration, "worker")
|
||||
|
||||
|
||||
def test_load_identifier_configuration(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must load identifier from configuration
|
||||
"""
|
||||
identifier = "id"
|
||||
mocker.patch("pathlib.Path.is_file", return_value=False)
|
||||
configuration.set_option("worker", "identifier", identifier)
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
system = DistributedSystem(repository_id, configuration)
|
||||
|
||||
assert system.worker.identifier == identifier
|
||||
|
||||
|
||||
def test_load_identifier_filesystem(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must load identifier from filesystem
|
||||
"""
|
||||
identifier = "id"
|
||||
mocker.patch("pathlib.Path.is_file", return_value=True)
|
||||
read_mock = mocker.patch("pathlib.Path.read_text", return_value=identifier)
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
system = DistributedSystem(repository_id, configuration)
|
||||
|
||||
assert system.worker.identifier == identifier
|
||||
read_mock.assert_called_once_with(encoding="utf8")
|
||||
|
||||
|
||||
def test_register(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must register service
|
||||
"""
|
||||
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
|
||||
mocker.patch("pathlib.Path.is_file", return_value=False)
|
||||
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
|
||||
write_mock = mocker.patch("pathlib.Path.write_text")
|
||||
|
||||
distributed_system.register()
|
||||
run_mock.assert_called_once_with("POST", f"{distributed_system.address}/api/v1/distributed",
|
||||
json=distributed_system.worker.view())
|
||||
write_mock.assert_called_once_with(distributed_system.worker.identifier, encoding="utf8")
|
||||
assert distributed_system._owe_identifier
|
||||
|
||||
|
||||
def test_register_skip(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must skip service registration if it doesn't owe the identifier
|
||||
"""
|
||||
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
|
||||
mocker.patch("pathlib.Path.is_file", return_value=True)
|
||||
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
|
||||
write_mock = mocker.patch("pathlib.Path.write_text")
|
||||
|
||||
distributed_system.register()
|
||||
run_mock.assert_not_called()
|
||||
write_mock.assert_not_called()
|
||||
assert not distributed_system._owe_identifier
|
||||
|
||||
|
||||
def test_register_force(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must register service even if it doesn't owe the identifier if force is supplied
|
||||
"""
|
||||
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
|
||||
mocker.patch("pathlib.Path.is_file", return_value=True)
|
||||
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
|
||||
write_mock = mocker.patch("pathlib.Path.write_text")
|
||||
|
||||
distributed_system.register(force=True)
|
||||
run_mock.assert_called_once_with("POST", f"{distributed_system.address}/api/v1/distributed",
|
||||
json=distributed_system.worker.view())
|
||||
write_mock.assert_called_once_with(distributed_system.worker.identifier, encoding="utf8")
|
||||
assert distributed_system._owe_identifier
|
||||
|
||||
|
||||
def test_unregister(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must unregister service
|
||||
"""
|
||||
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
|
||||
remove_mock = mocker.patch("pathlib.Path.unlink")
|
||||
distributed_system._owe_identifier = True
|
||||
|
||||
distributed_system.unregister()
|
||||
run_mock.assert_called_once_with(
|
||||
"DELETE", f"{distributed_system.address}/api/v1/distributed/{distributed_system.worker.identifier}")
|
||||
remove_mock.assert_called_once_with(missing_ok=True)
|
||||
|
||||
|
||||
def test_unregister_skip(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must skip service removal if it doesn't owe the identifier
|
||||
"""
|
||||
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
|
||||
remove_mock = mocker.patch("pathlib.Path.unlink")
|
||||
|
||||
distributed_system.unregister()
|
||||
run_mock.assert_not_called()
|
||||
remove_mock.assert_not_called()
|
||||
|
||||
|
||||
def test_unregister_force(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must remove service even if it doesn't owe the identifier if force is supplied
|
||||
"""
|
||||
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
|
||||
remove_mock = mocker.patch("pathlib.Path.unlink")
|
||||
|
||||
distributed_system.unregister(force=True)
|
||||
run_mock.assert_called_once_with(
|
||||
"DELETE", f"{distributed_system.address}/api/v1/distributed/{distributed_system.worker.identifier}")
|
||||
remove_mock.assert_called_once_with(missing_ok=True)
|
||||
|
||||
|
||||
def test_workers(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must return available remote workers
|
||||
"""
|
||||
worker = Worker("remote")
|
||||
response_obj = requests.Response()
|
||||
response_obj._content = json.dumps([worker.view()]).encode("utf8")
|
||||
response_obj.status_code = 200
|
||||
|
||||
requests_mock = mocker.patch("ahriman.core.status.web_client.WebClient.make_request",
|
||||
return_value=response_obj)
|
||||
|
||||
result = distributed_system.workers()
|
||||
requests_mock.assert_called_once_with("GET", distributed_system._workers_url())
|
||||
assert result == [worker]
|
||||
|
||||
|
||||
def test_workers_failed(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must suppress any exception happened during worker extraction
|
||||
"""
|
||||
mocker.patch("requests.Session.request", side_effect=Exception())
|
||||
distributed_system.workers()
|
||||
|
||||
|
||||
def test_workers_failed_http_error(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must suppress HTTP exception happened during worker extraction
|
||||
"""
|
||||
mocker.patch("requests.Session.request", side_effect=requests.HTTPError())
|
||||
distributed_system.workers()
|
||||
47
tests/ahriman/core/distributed/test_worker_loader_trigger.py
Normal file
47
tests/ahriman/core/distributed/test_worker_loader_trigger.py
Normal file
@ -0,0 +1,47 @@
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.distributed import WorkerLoaderTrigger
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must load workers from remote
|
||||
"""
|
||||
worker = Worker("address")
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.WorkerLoaderTrigger.workers", return_value=[worker])
|
||||
_, repository_id = configuration.check_loaded()
|
||||
|
||||
trigger = WorkerLoaderTrigger(repository_id, configuration)
|
||||
trigger.on_start()
|
||||
run_mock.assert_called_once_with()
|
||||
assert configuration.getlist("build", "workers") == [worker.address]
|
||||
|
||||
|
||||
def test_on_start_skip(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must skip loading if option is already set
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
configuration.set_option("build", "workers", "address")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.WorkerLoaderTrigger.workers")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
|
||||
trigger = WorkerLoaderTrigger(repository_id, configuration)
|
||||
trigger.on_start()
|
||||
run_mock.assert_not_called()
|
||||
|
||||
|
||||
def test_on_start_empty_list(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must do not set anything if workers are not available
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
mocker.patch("ahriman.core.distributed.WorkerLoaderTrigger.workers", return_value=[])
|
||||
_, repository_id = configuration.check_loaded()
|
||||
|
||||
trigger = WorkerLoaderTrigger(repository_id, configuration)
|
||||
trigger.on_start()
|
||||
assert not configuration.has_option("build", "workers")
|
||||
@ -0,0 +1,17 @@
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.distributed import WorkerRegisterTrigger
|
||||
|
||||
|
||||
def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must register itself as worker
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.WorkerRegisterTrigger.register")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
|
||||
trigger = WorkerRegisterTrigger(repository_id, configuration)
|
||||
trigger.on_start()
|
||||
run_mock.assert_called_once_with(force=True)
|
||||
30
tests/ahriman/core/distributed/test_worker_trigger.py
Normal file
30
tests/ahriman/core/distributed/test_worker_trigger.py
Normal file
@ -0,0 +1,30 @@
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.distributed import WorkerTrigger
|
||||
|
||||
|
||||
def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must register itself as worker
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.WorkerTrigger.register")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
|
||||
trigger = WorkerTrigger(repository_id, configuration)
|
||||
trigger.on_start()
|
||||
run_mock.assert_called_once_with()
|
||||
|
||||
|
||||
def test_on_stop(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must unregister itself as worker
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.WorkerTrigger.unregister")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
|
||||
trigger = WorkerTrigger(repository_id, configuration)
|
||||
trigger.on_stop()
|
||||
run_mock.assert_called_once_with()
|
||||
@ -0,0 +1,17 @@
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.distributed import WorkerUnregisterTrigger
|
||||
|
||||
|
||||
def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must unregister itself as worker
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.WorkerUnregisterTrigger.unregister")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
|
||||
trigger = WorkerUnregisterTrigger(repository_id, configuration)
|
||||
trigger.on_start()
|
||||
run_mock.assert_called_once_with(force=True)
|
||||
@ -10,6 +10,7 @@ from ahriman.models.changes import Changes
|
||||
from ahriman.models.log_record_id import LogRecordId
|
||||
from ahriman.models.package import Package
|
||||
from ahriman.models.pkgbuild_patch import PkgbuildPatch
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
def test_load(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
|
||||
@ -227,3 +228,40 @@ def test_status_update(watcher: Watcher) -> None:
|
||||
"""
|
||||
watcher.status_update(BuildStatusEnum.Success)
|
||||
assert watcher.status.status == BuildStatusEnum.Success
|
||||
|
||||
|
||||
def test_workers_get(watcher: Watcher, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must retrieve workers
|
||||
"""
|
||||
worker = Worker("remote")
|
||||
worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_get", return_value=[worker])
|
||||
|
||||
assert watcher.workers_get() == [worker]
|
||||
worker_mock.assert_called_once_with()
|
||||
|
||||
|
||||
def test_workers_remove(watcher: Watcher, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must remove workers
|
||||
"""
|
||||
identifier = "identifier"
|
||||
worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_remove")
|
||||
|
||||
watcher.workers_remove(identifier)
|
||||
watcher.workers_remove()
|
||||
worker_mock.assert_has_calls([
|
||||
MockCall(identifier),
|
||||
MockCall(None),
|
||||
])
|
||||
|
||||
|
||||
def test_workers_update(watcher: Watcher, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must update workers
|
||||
"""
|
||||
worker = Worker("remote")
|
||||
worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_insert")
|
||||
|
||||
watcher.workers_update(worker)
|
||||
worker_mock.assert_called_once_with(worker)
|
||||
|
||||
@ -12,6 +12,7 @@ from ahriman.models.changes import Changes
|
||||
from ahriman.models.internal_status import InternalStatus
|
||||
from ahriman.models.log_record_id import LogRecordId
|
||||
from ahriman.models.package import Package
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
def test_parse_address(configuration: Configuration) -> None:
|
||||
@ -32,14 +33,6 @@ def test_parse_address(configuration: Configuration) -> None:
|
||||
assert WebClient.parse_address(configuration) == ("status", "http://localhost:8082")
|
||||
|
||||
|
||||
def test_status_url(web_client: WebClient) -> None:
|
||||
"""
|
||||
must generate package status url correctly
|
||||
"""
|
||||
assert web_client._status_url().startswith(web_client.address)
|
||||
assert web_client._status_url().endswith("/api/v1/status")
|
||||
|
||||
|
||||
def test_changes_url(web_client: WebClient, package_ahriman: Package) -> None:
|
||||
"""
|
||||
must generate changes url correctly
|
||||
@ -67,6 +60,14 @@ def test_package_url(web_client: WebClient, package_ahriman: Package) -> None:
|
||||
assert web_client._package_url(package_ahriman.base).endswith(f"/api/v1/packages/{package_ahriman.base}")
|
||||
|
||||
|
||||
def test_status_url(web_client: WebClient) -> None:
|
||||
"""
|
||||
must generate package status url correctly
|
||||
"""
|
||||
assert web_client._status_url().startswith(web_client.address)
|
||||
assert web_client._status_url().endswith("/api/v1/status")
|
||||
|
||||
|
||||
def test_package_add(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must process package addition
|
||||
|
||||
@ -8,3 +8,17 @@ def test_post_init() -> None:
|
||||
assert Worker("http://localhost:8080").identifier == "localhost:8080"
|
||||
assert Worker("remote").identifier == "" # not a valid url
|
||||
assert Worker("remote", identifier="id").identifier == "id"
|
||||
|
||||
|
||||
def test_view() -> None:
|
||||
"""
|
||||
must generate json view
|
||||
"""
|
||||
worker = Worker("address")
|
||||
assert worker.view() == {"address": worker.address, "identifier": worker.identifier}
|
||||
|
||||
worker = Worker("http://localhost:8080")
|
||||
assert worker.view() == {"address": worker.address, "identifier": worker.identifier}
|
||||
|
||||
worker = Worker("http://localhost:8080", identifier="abc")
|
||||
assert worker.view() == {"address": worker.address, "identifier": worker.identifier}
|
||||
|
||||
1
tests/ahriman/web/schemas/test_worker_id_schema.py
Normal file
1
tests/ahriman/web/schemas/test_worker_id_schema.py
Normal file
@ -0,0 +1 @@
|
||||
# schema testing goes in view class tests
|
||||
1
tests/ahriman/web/schemas/test_worker_schema.py
Normal file
1
tests/ahriman/web/schemas/test_worker_schema.py
Normal file
@ -0,0 +1 @@
|
||||
# schema testing goes in view class tests
|
||||
@ -0,0 +1,70 @@
|
||||
import pytest
|
||||
|
||||
from aiohttp.test_utils import TestClient
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.models.worker import Worker
|
||||
from ahriman.web.views.v1.distributed.worker import WorkerView
|
||||
|
||||
|
||||
async def test_get_permission() -> None:
|
||||
"""
|
||||
must return correct permission for the request
|
||||
"""
|
||||
for method in ("DELETE", "GET"):
|
||||
request = pytest.helpers.request("", "", method)
|
||||
assert await WorkerView.get_permission(request) == UserAccess.Full
|
||||
|
||||
|
||||
def test_routes() -> None:
|
||||
"""
|
||||
must return correct routes
|
||||
"""
|
||||
assert WorkerView.ROUTES == ["/api/v1/distributed/{identifier}"]
|
||||
|
||||
|
||||
async def test_delete(client: TestClient) -> None:
|
||||
"""
|
||||
must delete single worker
|
||||
"""
|
||||
await client.post("/api/v1/distributed", json={"address": "address1", "identifier": "1"})
|
||||
await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"})
|
||||
|
||||
response = await client.delete("/api/v1/distributed/1")
|
||||
assert response.status == 204
|
||||
|
||||
response = await client.get("/api/v1/distributed/1")
|
||||
assert response.status == 404
|
||||
|
||||
response = await client.get("/api/v1/distributed/2")
|
||||
assert response.ok
|
||||
|
||||
|
||||
async def test_get(client: TestClient) -> None:
|
||||
"""
|
||||
must return specific worker
|
||||
"""
|
||||
worker = Worker("address1", identifier="1")
|
||||
|
||||
await client.post("/api/v1/distributed", json=worker.view())
|
||||
await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"})
|
||||
response_schema = pytest.helpers.schema_response(WorkerView.get)
|
||||
|
||||
response = await client.get(f"/api/v1/distributed/{worker.identifier}")
|
||||
assert response.ok
|
||||
json = await response.json()
|
||||
assert not response_schema.validate(json, many=True)
|
||||
|
||||
workers = [Worker(item["address"], identifier=item["identifier"]) for item in json]
|
||||
assert workers == [worker]
|
||||
|
||||
|
||||
async def test_get_not_found(client: TestClient) -> None:
|
||||
"""
|
||||
must return Not Found for unknown package
|
||||
"""
|
||||
response_schema = pytest.helpers.schema_response(WorkerView.get, code=404)
|
||||
|
||||
response = await client.get("/api/v1/distributed/1")
|
||||
assert response.status == 404
|
||||
assert not response_schema.validate(await response.json())
|
||||
@ -0,0 +1,83 @@
|
||||
import pytest
|
||||
|
||||
from aiohttp.test_utils import TestClient
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.models.worker import Worker
|
||||
from ahriman.web.views.v1.distributed.workers import WorkersView
|
||||
|
||||
|
||||
async def test_get_permission() -> None:
|
||||
"""
|
||||
must return correct permission for the request
|
||||
"""
|
||||
for method in ("DELETE", "GET", "POST"):
|
||||
request = pytest.helpers.request("", "", method)
|
||||
assert await WorkersView.get_permission(request) == UserAccess.Full
|
||||
|
||||
|
||||
def test_routes() -> None:
|
||||
"""
|
||||
must return correct routes
|
||||
"""
|
||||
assert WorkersView.ROUTES == ["/api/v1/distributed"]
|
||||
|
||||
|
||||
async def test_delete(client: TestClient) -> None:
|
||||
"""
|
||||
must delete all workers
|
||||
"""
|
||||
await client.post("/api/v1/distributed", json={"address": "address1", "identifier": "1"})
|
||||
await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"})
|
||||
|
||||
response = await client.delete("/api/v1/distributed")
|
||||
assert response.status == 204
|
||||
|
||||
response = await client.get("/api/v1/distributed")
|
||||
json = await response.json()
|
||||
assert not json
|
||||
|
||||
|
||||
async def test_get(client: TestClient) -> None:
|
||||
"""
|
||||
must return all workers
|
||||
"""
|
||||
await client.post("/api/v1/distributed", json={"address": "address1", "identifier": "1"})
|
||||
await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"})
|
||||
response_schema = pytest.helpers.schema_response(WorkersView.get)
|
||||
|
||||
response = await client.get("/api/v1/distributed")
|
||||
assert response.ok
|
||||
json = await response.json()
|
||||
assert not response_schema.validate(json, many=True)
|
||||
|
||||
workers = [Worker(item["address"], identifier=item["identifier"]) for item in json]
|
||||
assert workers == [Worker("address1", identifier="1"), Worker("address2", identifier="2")]
|
||||
|
||||
|
||||
async def test_post(client: TestClient) -> None:
|
||||
"""
|
||||
must update worker
|
||||
"""
|
||||
worker = Worker("address1", identifier="1")
|
||||
request_schema = pytest.helpers.schema_request(WorkersView.post)
|
||||
|
||||
payload = worker.view()
|
||||
assert not request_schema.validate(payload)
|
||||
|
||||
response = await client.post("/api/v1/distributed", json=payload)
|
||||
assert response.status == 204
|
||||
|
||||
response = await client.get(f"/api/v1/distributed/{worker.identifier}")
|
||||
assert response.ok
|
||||
|
||||
|
||||
async def test_post_exception(client: TestClient) -> None:
|
||||
"""
|
||||
must raise exception on invalid payload
|
||||
"""
|
||||
response_schema = pytest.helpers.schema_response(WorkersView.post, code=400)
|
||||
|
||||
response = await client.post("/api/v1/distributed", json={})
|
||||
assert response.status == 400
|
||||
assert not response_schema.validate(await response.json())
|
||||
@ -25,7 +25,7 @@ ignore_packages =
|
||||
makechrootpkg_flags =
|
||||
makepkg_flags = --skippgpcheck
|
||||
triggers = ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger
|
||||
triggers_known = ahriman.core.gitremote.RemotePullTrigger ahriman.core.gitremote.RemotePushTrigger ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger ahriman.core.support.KeyringTrigger ahriman.core.support.MirrorlistTrigger
|
||||
triggers_known = ahriman.core.distributed.WorkerLoaderTrigger ahriman.core.distributed.WorkerRegisterTrigger ahriman.core.distributed.WorkerTrigger ahriman.core.distributed.WorkerUnregisterTrigger ahriman.core.gitremote.RemotePullTrigger ahriman.core.gitremote.RemotePushTrigger ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger ahriman.core.support.KeyringTrigger ahriman.core.support.MirrorlistTrigger
|
||||
|
||||
[repository]
|
||||
name = aur-clone
|
||||
@ -115,4 +115,7 @@ username = arcan1s
|
||||
enable_archive_upload = yes
|
||||
host = 127.0.0.1
|
||||
static_path = ../web/templates/static
|
||||
templates = ../web/templates
|
||||
templates = ../web/templates
|
||||
|
||||
[worker]
|
||||
address = http://localhost:8081
|
||||
|
||||
Reference in New Issue
Block a user