mirror of
https://github.com/arcan1s/ahriman.git
synced 2025-04-24 07:17:17 +00:00
ping based workers
This commit is contained in:
parent
ee806f6d44
commit
ac1ed209d4
@ -108,14 +108,6 @@ ahriman.core.database.migrations.m012\_last\_commit\_sha module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.database.migrations.m013\_workers module
|
||||
-----------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.database.migrations.m013_workers
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
Module contents
|
||||
---------------
|
||||
|
||||
|
||||
@ -60,14 +60,6 @@ ahriman.core.database.operations.patch\_operations module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.database.operations.workers\_operations module
|
||||
-----------------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.database.operations.workers_operations
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
Module contents
|
||||
---------------
|
||||
|
||||
|
||||
@ -20,14 +20,6 @@ ahriman.core.distributed.worker\_loader\_trigger module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.distributed.worker\_register\_trigger module
|
||||
---------------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.distributed.worker_register_trigger
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.distributed.worker\_trigger module
|
||||
-----------------------------------------------
|
||||
|
||||
@ -36,10 +28,10 @@ ahriman.core.distributed.worker\_trigger module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.distributed.worker\_unregister\_trigger module
|
||||
-----------------------------------------------------------
|
||||
ahriman.core.distributed.workers\_cache module
|
||||
----------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.distributed.worker_unregister_trigger
|
||||
.. automodule:: ahriman.core.distributed.workers_cache
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
@ -260,14 +260,6 @@ ahriman.web.schemas.versioned\_log\_schema module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.web.schemas.worker\_id\_schema module
|
||||
---------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.web.schemas.worker_id_schema
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.web.schemas.worker\_schema module
|
||||
-----------------------------------------
|
||||
|
||||
|
||||
@ -4,14 +4,6 @@ ahriman.web.views.v1.distributed package
|
||||
Submodules
|
||||
----------
|
||||
|
||||
ahriman.web.views.v1.distributed.worker module
|
||||
----------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.web.views.v1.distributed.worker
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.web.views.v1.distributed.workers module
|
||||
-----------------------------------------------
|
||||
|
||||
|
||||
@ -37,6 +37,7 @@ This package contains everything required for the most of application actions an
|
||||
* ``ahriman.core.build_tools`` is a package which provides wrapper for ``devtools`` commands.
|
||||
* ``ahriman.core.configuration`` contains extension for standard ``configparser`` library and some validation related classes.
|
||||
* ``ahriman.core.database`` is everything for database, including data and schema migrations.
|
||||
* ``ahriman.core.distributed`` package with triggers and helpers for distributed build system.
|
||||
* ``ahriman.core.formatters`` package provides ``Printer`` sub-classes for printing data (e.g. package properties) to stdout which are used by some handlers.
|
||||
* ``ahriman.core.gitremote`` is a package with remote PKGBUILD triggers. Should not be called directly.
|
||||
* ``ahriman.core.http`` package provides HTTP clients which can be used later by other classes.
|
||||
|
||||
@ -359,5 +359,5 @@ Requires ``boto3`` library to be installed. Section name must be either ``s3`` (
|
||||
This section controls settings for ``ahriman.core.distributed.WorkerTrigger`` plugin.
|
||||
|
||||
* ``address`` - address of the instance, string, required. Must be reachable for the master instance.
|
||||
* ``identifier`` - unique identifier of the instance, string, optional. If none set, the random uuid will be generated on each run automatically.
|
||||
* ``identifier_path`` - path to lock file, string, optional, default is ``/tmp/ahriman-worker-identifier``.
|
||||
* ``identifier`` - unique identifier of the instance, string, optional.
|
||||
* ``time_to_live`` - amount of time which remote worker will be considered alive in seconds, integer, optional, default is ``60``. The ping interval will be set automatically equal this value divided by 4.
|
||||
|
||||
@ -22,9 +22,7 @@ Special trigger to be used to load workers from database on the start of the app
|
||||
``ahriman.core.distributed.WorkerTrigger``
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Another trigger for the distributed system, which registers itself as remote worker. It calls the remote server on start (if no lock file found) and, later, it deregister itself before the stop.
|
||||
|
||||
There are also two triggers which performs only registration and removal (``ahriman.core.distributed.WorkerRegisterTrigger`` and ``ahriman.core.distributed.WorkerUnregisterTrigger`` respectively), but they are not meant to be called directly.
|
||||
Another trigger for the distributed system, which registers itself as remote worker, calling remote service periodically.
|
||||
|
||||
``ahriman.core.gitremote.RemotePullTrigger``
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# AUTOMATICALLY GENERATED by `shtab`
|
||||
|
||||
_shtab_ahriman_subparsers=('aur-search' 'search' 'help-commands-unsafe' 'help' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-changes' 'package-changes-remove' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-repositories' 'service-run' 'run' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'service-tree-migrate' 'service-worker-register' 'service-worker-unregister' 'user-add' 'user-list' 'user-remove' 'web')
|
||||
_shtab_ahriman_subparsers=('aur-search' 'search' 'help-commands-unsafe' 'help' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-changes' 'package-changes-remove' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-repositories' 'service-run' 'run' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'service-tree-migrate' 'user-add' 'user-list' 'user-remove' 'web')
|
||||
|
||||
_shtab_ahriman_option_strings=('-h' '--help' '-a' '--architecture' '-c' '--configuration' '--force' '-l' '--lock' '--log-handler' '-q' '--quiet' '--report' '--no-report' '-r' '--repository' '--unsafe' '-V' '--version' '--wait-timeout')
|
||||
_shtab_ahriman_aur_search_option_strings=('-h' '--help' '-e' '--exit-code' '--info' '--no-info' '--sort-by')
|
||||
@ -71,8 +71,6 @@ _shtab_ahriman_setup_option_strings=('-h' '--help' '--build-as-user' '--from-con
|
||||
_shtab_ahriman_service_shell_option_strings=('-h' '--help')
|
||||
_shtab_ahriman_shell_option_strings=('-h' '--help')
|
||||
_shtab_ahriman_service_tree_migrate_option_strings=('-h' '--help')
|
||||
_shtab_ahriman_service_worker_register_option_strings=('-h' '--help')
|
||||
_shtab_ahriman_service_worker_unregister_option_strings=('-h' '--help')
|
||||
_shtab_ahriman_user_add_option_strings=('-h' '--help' '--key' '--packager' '-p' '--password' '-R' '--role')
|
||||
_shtab_ahriman_user_list_option_strings=('-h' '--help' '-e' '--exit-code' '-R' '--role')
|
||||
_shtab_ahriman_user_remove_option_strings=('-h' '--help')
|
||||
@ -80,7 +78,7 @@ _shtab_ahriman_web_option_strings=('-h' '--help')
|
||||
|
||||
|
||||
|
||||
_shtab_ahriman_pos_0_choices=('aur-search' 'search' 'help-commands-unsafe' 'help' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-changes' 'package-changes-remove' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-repositories' 'service-run' 'run' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'service-tree-migrate' 'service-worker-register' 'service-worker-unregister' 'user-add' 'user-list' 'user-remove' 'web')
|
||||
_shtab_ahriman_pos_0_choices=('aur-search' 'search' 'help-commands-unsafe' 'help' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-changes' 'package-changes-remove' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-repositories' 'service-run' 'run' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'service-tree-migrate' 'user-add' 'user-list' 'user-remove' 'web')
|
||||
_shtab_ahriman___log_handler_choices=('console' 'syslog' 'journald')
|
||||
_shtab_ahriman_aur_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'submitter' 'url' 'url_path' 'version')
|
||||
_shtab_ahriman_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'submitter' 'url' 'url_path' 'version')
|
||||
@ -527,10 +525,6 @@ _shtab_ahriman_shell__v_nargs=0
|
||||
_shtab_ahriman_shell___verbose_nargs=0
|
||||
_shtab_ahriman_service_tree_migrate__h_nargs=0
|
||||
_shtab_ahriman_service_tree_migrate___help_nargs=0
|
||||
_shtab_ahriman_service_worker_register__h_nargs=0
|
||||
_shtab_ahriman_service_worker_register___help_nargs=0
|
||||
_shtab_ahriman_service_worker_unregister__h_nargs=0
|
||||
_shtab_ahriman_service_worker_unregister___help_nargs=0
|
||||
_shtab_ahriman_user_add__h_nargs=0
|
||||
_shtab_ahriman_user_add___help_nargs=0
|
||||
_shtab_ahriman_user_list__h_nargs=0
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
.TH AHRIMAN "1" "2023\-12\-31" "ahriman" "Generated Python Manual"
|
||||
.TH AHRIMAN "1" "2024\-01\-02" "ahriman" "Generated Python Manual"
|
||||
.SH NAME
|
||||
ahriman
|
||||
.SH SYNOPSIS
|
||||
.B ahriman
|
||||
[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--log-handler {console,syslog,journald}] [-q] [--report | --no-report] [-r REPOSITORY] [--unsafe] [-V] [--wait-timeout WAIT_TIMEOUT] {aur-search,search,help-commands-unsafe,help,help-updates,help-version,version,package-add,add,package-update,package-changes,package-changes-remove,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,patch-set-add,repo-backup,repo-check,check,repo-create-keyring,repo-create-mirrorlist,repo-daemon,daemon,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-sign,sign,repo-status-update,repo-sync,sync,repo-tree,repo-triggers,repo-update,update,service-clean,clean,repo-clean,service-config,config,repo-config,service-config-validate,config-validate,repo-config-validate,service-key-import,key-import,service-repositories,service-run,run,service-setup,init,repo-init,repo-setup,setup,service-shell,shell,service-tree-migrate,service-worker-register,service-worker-unregister,user-add,user-list,user-remove,web} ...
|
||||
[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--log-handler {console,syslog,journald}] [-q] [--report | --no-report] [-r REPOSITORY] [--unsafe] [-V] [--wait-timeout WAIT_TIMEOUT] {aur-search,search,help-commands-unsafe,help,help-updates,help-version,version,package-add,add,package-update,package-changes,package-changes-remove,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,patch-set-add,repo-backup,repo-check,check,repo-create-keyring,repo-create-mirrorlist,repo-daemon,daemon,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-sign,sign,repo-status-update,repo-sync,sync,repo-tree,repo-triggers,repo-update,update,service-clean,clean,repo-clean,service-config,config,repo-config,service-config-validate,config-validate,repo-config-validate,service-key-import,key-import,service-repositories,service-run,run,service-setup,init,repo-init,repo-setup,setup,service-shell,shell,service-tree-migrate,user-add,user-list,user-remove,web} ...
|
||||
.SH DESCRIPTION
|
||||
ArcH linux ReposItory MANager
|
||||
|
||||
@ -176,12 +176,6 @@ invoke python shell
|
||||
\fBahriman\fR \fI\,service\-tree\-migrate\/\fR
|
||||
migrate repository tree
|
||||
.TP
|
||||
\fBahriman\fR \fI\,service\-worker\-register\/\fR
|
||||
register itself as worker
|
||||
.TP
|
||||
\fBahriman\fR \fI\,service\-worker\-unregister\/\fR
|
||||
unregister itself as worker
|
||||
.TP
|
||||
\fBahriman\fR \fI\,user\-add\/\fR
|
||||
create or update user
|
||||
.TP
|
||||
@ -875,16 +869,6 @@ usage: ahriman service\-tree\-migrate [\-h]
|
||||
|
||||
migrate repository tree between versions
|
||||
|
||||
.SH COMMAND \fI\,'ahriman service\-worker\-register'\/\fR
|
||||
usage: ahriman service\-worker\-register [\-h]
|
||||
|
||||
call remote service registering itself as available worker
|
||||
|
||||
.SH COMMAND \fI\,'ahriman service\-worker\-unregister'\/\fR
|
||||
usage: ahriman service\-worker\-unregister [\-h]
|
||||
|
||||
call remote service removing itself from list of available workers
|
||||
|
||||
.SH COMMAND \fI\,'ahriman user\-add'\/\fR
|
||||
usage: ahriman user\-add [\-h] [\-\-key KEY] [\-\-packager PACKAGER] [\-p PASSWORD] [\-R {unauthorized,read,reporter,full}]
|
||||
username
|
||||
|
||||
@ -65,8 +65,6 @@ _shtab_ahriman_commands() {
|
||||
"service-setup:create initial service configuration, requires root"
|
||||
"service-shell:drop into python shell"
|
||||
"service-tree-migrate:migrate repository tree between versions"
|
||||
"service-worker-register:call remote service registering itself as available worker"
|
||||
"service-worker-unregister:call remote service removing itself from list of available workers"
|
||||
"setup:create initial service configuration, requires root"
|
||||
"shell:drop into python shell"
|
||||
"sign:(re-)sign packages and repository database according to current settings"
|
||||
@ -554,14 +552,6 @@ _shtab_ahriman_service_tree_migrate_options=(
|
||||
"(- : *)"{-h,--help}"[show this help message and exit]"
|
||||
)
|
||||
|
||||
_shtab_ahriman_service_worker_register_options=(
|
||||
"(- : *)"{-h,--help}"[show this help message and exit]"
|
||||
)
|
||||
|
||||
_shtab_ahriman_service_worker_unregister_options=(
|
||||
"(- : *)"{-h,--help}"[show this help message and exit]"
|
||||
)
|
||||
|
||||
_shtab_ahriman_setup_options=(
|
||||
"(- : *)"{-h,--help}"[show this help message and exit]"
|
||||
"--build-as-user[force makepkg user to the specific one (default\: None)]:build_as_user:"
|
||||
@ -727,8 +717,6 @@ _shtab_ahriman() {
|
||||
service-setup) _arguments -C -s $_shtab_ahriman_service_setup_options ;;
|
||||
service-shell) _arguments -C -s $_shtab_ahriman_service_shell_options ;;
|
||||
service-tree-migrate) _arguments -C -s $_shtab_ahriman_service_tree_migrate_options ;;
|
||||
service-worker-register) _arguments -C -s $_shtab_ahriman_service_worker_register_options ;;
|
||||
service-worker-unregister) _arguments -C -s $_shtab_ahriman_service_worker_unregister_options ;;
|
||||
setup) _arguments -C -s $_shtab_ahriman_setup_options ;;
|
||||
shell) _arguments -C -s $_shtab_ahriman_shell_options ;;
|
||||
sign) _arguments -C -s $_shtab_ahriman_sign_options ;;
|
||||
|
||||
@ -135,8 +135,6 @@ def _parser() -> argparse.ArgumentParser:
|
||||
_set_service_setup_parser(subparsers)
|
||||
_set_service_shell_parser(subparsers)
|
||||
_set_service_tree_migrate_parser(subparsers)
|
||||
_set_service_worker_register_parser(subparsers)
|
||||
_set_service_worker_unregister_parser(subparsers)
|
||||
_set_user_add_parser(subparsers)
|
||||
_set_user_list_parser(subparsers)
|
||||
_set_user_remove_parser(subparsers)
|
||||
@ -1054,40 +1052,6 @@ def _set_service_tree_migrate_parser(root: SubParserAction) -> argparse.Argument
|
||||
return parser
|
||||
|
||||
|
||||
def _set_service_worker_register_parser(root: SubParserAction) -> argparse.ArgumentParser:
|
||||
"""
|
||||
add parser for remote worker registration subcommand
|
||||
|
||||
Args:
|
||||
root(SubParserAction): subparsers for the commands
|
||||
|
||||
Returns:
|
||||
argparse.ArgumentParser: created argument parser
|
||||
"""
|
||||
parser = root.add_parser("service-worker-register", help="register itself as worker",
|
||||
description="call remote service registering itself as available worker",
|
||||
formatter_class=_formatter)
|
||||
parser.set_defaults(handler=handlers.Triggers, trigger=["ahriman.core.distributed.WorkerRegisterTrigger"])
|
||||
return parser
|
||||
|
||||
|
||||
def _set_service_worker_unregister_parser(root: SubParserAction) -> argparse.ArgumentParser:
|
||||
"""
|
||||
add parser for remote worker removal subcommand
|
||||
|
||||
Args:
|
||||
root(SubParserAction): subparsers for the commands
|
||||
|
||||
Returns:
|
||||
argparse.ArgumentParser: created argument parser
|
||||
"""
|
||||
parser = root.add_parser("service-worker-unregister", help="unregister itself as worker",
|
||||
description="call remote service removing itself from list of available workers",
|
||||
formatter_class=_formatter)
|
||||
parser.set_defaults(handler=handlers.Triggers, trigger=["ahriman.core.distributed.WorkerUnregisterTrigger"])
|
||||
return parser
|
||||
|
||||
|
||||
def _set_user_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
|
||||
"""
|
||||
add parser for create user subcommand
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
__all__ = ["steps"]
|
||||
|
||||
|
||||
steps = [
|
||||
"""
|
||||
create table workers (
|
||||
identifier text not null,
|
||||
address text not null,
|
||||
unique (identifier)
|
||||
)
|
||||
"""
|
||||
]
|
||||
@ -25,4 +25,3 @@ from ahriman.core.database.operations.changes_operations import ChangesOperation
|
||||
from ahriman.core.database.operations.logs_operations import LogsOperations
|
||||
from ahriman.core.database.operations.package_operations import PackageOperations
|
||||
from ahriman.core.database.operations.patch_operations import PatchOperations
|
||||
from ahriman.core.database.operations.workers_operations import WorkersOperations
|
||||
|
||||
@ -1,83 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from sqlite3 import Connection
|
||||
|
||||
from ahriman.core.database.operations import Operations
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
class WorkersOperations(Operations):
|
||||
"""
|
||||
operations for remote workers
|
||||
"""
|
||||
|
||||
def workers_get(self) -> list[Worker]:
|
||||
"""
|
||||
retrieve registered workers
|
||||
|
||||
Returns:
|
||||
list[Worker]: list of available workers
|
||||
"""
|
||||
def run(connection: Connection) -> list[Worker]:
|
||||
return [
|
||||
Worker(row["address"], identifier=row["identifier"])
|
||||
for row in connection.execute("""select * from workers""")
|
||||
]
|
||||
|
||||
return self.with_connection(run)
|
||||
|
||||
def workers_insert(self, worker: Worker) -> None:
|
||||
"""
|
||||
insert or update worker in database
|
||||
|
||||
Args:
|
||||
worker(Worker): remote worker descriptor
|
||||
"""
|
||||
def run(connection: Connection) -> None:
|
||||
connection.execute(
|
||||
"""
|
||||
insert into workers
|
||||
(identifier, address)
|
||||
values
|
||||
(:identifier, :address)
|
||||
on conflict (identifier) do update set
|
||||
address = :address
|
||||
""",
|
||||
worker.view()
|
||||
)
|
||||
|
||||
return self.with_connection(run, commit=True)
|
||||
|
||||
def workers_remove(self, identifier: str | None = None) -> None:
|
||||
"""
|
||||
unregister remote worker
|
||||
|
||||
Args:
|
||||
identifier(str | None, optional): remote worker identifier. If none set it will clear all workers
|
||||
(Default value = None)
|
||||
"""
|
||||
def run(connection: Connection) -> None:
|
||||
connection.execute(
|
||||
"""
|
||||
delete from workers where (:identifier is null or identifier = :identifier)
|
||||
""",
|
||||
{"identifier": identifier})
|
||||
|
||||
return self.with_connection(run, commit=True)
|
||||
@ -26,12 +26,11 @@ from typing import Self
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.database.migrations import Migrations
|
||||
from ahriman.core.database.operations import AuthOperations, BuildOperations, ChangesOperations, LogsOperations, \
|
||||
PackageOperations, PatchOperations, WorkersOperations
|
||||
PackageOperations, PatchOperations
|
||||
|
||||
|
||||
# pylint: disable=too-many-ancestors
|
||||
class SQLite(AuthOperations, BuildOperations, ChangesOperations, LogsOperations, PackageOperations, PatchOperations,
|
||||
WorkersOperations):
|
||||
class SQLite(AuthOperations, BuildOperations, ChangesOperations, LogsOperations, PackageOperations, PatchOperations):
|
||||
"""
|
||||
wrapper for sqlite3 database
|
||||
|
||||
|
||||
@ -18,6 +18,5 @@
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from ahriman.core.distributed.worker_loader_trigger import WorkerLoaderTrigger
|
||||
from ahriman.core.distributed.worker_register_trigger import WorkerRegisterTrigger
|
||||
from ahriman.core.distributed.worker_trigger import WorkerTrigger
|
||||
from ahriman.core.distributed.worker_unregister_trigger import WorkerUnregisterTrigger
|
||||
from ahriman.core.distributed.workers_cache import WorkersCache
|
||||
|
||||
@ -18,10 +18,7 @@
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import contextlib
|
||||
import tempfile
|
||||
import uuid
|
||||
|
||||
from pathlib import Path
|
||||
from functools import cached_property
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
@ -35,9 +32,6 @@ from ahriman.models.worker import Worker
|
||||
class DistributedSystem(Trigger, WebClient):
|
||||
"""
|
||||
simple class to (un)register itself as a distributed worker
|
||||
|
||||
Attributes:
|
||||
identifier_path(Path): path to cached worker identifier
|
||||
"""
|
||||
|
||||
CONFIGURATION_SCHEMA: ConfigurationSchema = {
|
||||
@ -54,9 +48,10 @@ class DistributedSystem(Trigger, WebClient):
|
||||
"type": "string",
|
||||
"empty": False,
|
||||
},
|
||||
"identifier_path": {
|
||||
"type": "path",
|
||||
"coerce": "absolute_path",
|
||||
"time_to_live": {
|
||||
"type": "integer",
|
||||
"coerce": "integer",
|
||||
"min": 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -73,11 +68,6 @@ class DistributedSystem(Trigger, WebClient):
|
||||
Trigger.__init__(self, repository_id, configuration)
|
||||
WebClient.__init__(self, repository_id, configuration)
|
||||
|
||||
section = next(iter(self.configuration_sections(configuration)))
|
||||
self.identifier_path = configuration.getpath(
|
||||
section, "identifier_path", fallback=Path(tempfile.gettempdir()) / "ahriman-worker-identifier")
|
||||
self._owe_identifier = False
|
||||
|
||||
@cached_property
|
||||
def worker(self) -> Worker:
|
||||
"""
|
||||
@ -87,8 +77,10 @@ class DistributedSystem(Trigger, WebClient):
|
||||
Worker: unique self worker identifier
|
||||
"""
|
||||
section = next(iter(self.configuration_sections(self.configuration)))
|
||||
identifier = self.load_identifier(self.configuration, section)
|
||||
return Worker(self.configuration.get(section, "address"), identifier=identifier)
|
||||
|
||||
address = self.configuration.get(section, "address")
|
||||
identifier = self.configuration.get(section, "identifier", fallback="")
|
||||
return Worker(address, identifier=identifier)
|
||||
|
||||
@classmethod
|
||||
def configuration_sections(cls, configuration: Configuration) -> list[str]:
|
||||
@ -103,66 +95,21 @@ class DistributedSystem(Trigger, WebClient):
|
||||
"""
|
||||
return list(cls.CONFIGURATION_SCHEMA.keys())
|
||||
|
||||
def _workers_url(self, identifier: str = "") -> str:
|
||||
def _workers_url(self) -> str:
|
||||
"""
|
||||
workers url generator
|
||||
|
||||
Args:
|
||||
identifier(str, optional): worker identifier (Default value = "")
|
||||
|
||||
Returns:
|
||||
str: full url of web service for specific worker
|
||||
str: full url of web service for workers
|
||||
"""
|
||||
suffix = f"/{identifier}" if identifier else ""
|
||||
return f"{self.address}/api/v1/distributed{suffix}"
|
||||
return f"{self.address}/api/v1/distributed"
|
||||
|
||||
def load_identifier(self, configuration: Configuration, section: str) -> str:
|
||||
"""
|
||||
load identifier from filesystem if available or from configuration otherwise. If cache file is available,
|
||||
the method will read from it. Otherwise, it will try to read it from configuration. And, finally, if no
|
||||
identifier set, it will generate uuid
|
||||
|
||||
Args:
|
||||
configuration(Configuration): configuration instance
|
||||
section(str): settings section name
|
||||
|
||||
Returns:
|
||||
str: unique worker identifier
|
||||
"""
|
||||
if self.identifier_path.is_file(): # load cached value
|
||||
return self.identifier_path.read_text(encoding="utf8")
|
||||
return configuration.get(section, "identifier", fallback=str(uuid.uuid4()))
|
||||
|
||||
def register(self, force: bool = False) -> None:
|
||||
def register(self) -> None:
|
||||
"""
|
||||
register itself in remote system
|
||||
|
||||
Args:
|
||||
force(bool, optional): register worker even if it has been already registered before (Default value = False)
|
||||
"""
|
||||
if self.identifier_path.is_file() and not force:
|
||||
return # there is already registered identifier
|
||||
|
||||
self.make_request("POST", self._workers_url(), json=self.worker.view())
|
||||
# save identifier
|
||||
self.identifier_path.write_text(self.worker.identifier, encoding="utf8")
|
||||
self._owe_identifier = True
|
||||
self.logger.info("registered instance %s at %s", self.worker, self.address)
|
||||
|
||||
def unregister(self, force: bool = False) -> None:
|
||||
"""
|
||||
unregister itself in remote system
|
||||
|
||||
Args:
|
||||
force(bool, optional): unregister worker even if it has been registered in another process
|
||||
(Default value = False)
|
||||
"""
|
||||
if not self._owe_identifier and not force:
|
||||
return # we do not owe this identifier
|
||||
|
||||
self.make_request("DELETE", self._workers_url(self.worker.identifier))
|
||||
self.identifier_path.unlink(missing_ok=True)
|
||||
self.logger.info("unregistered instance %s at %s", self.worker, self.address)
|
||||
with contextlib.suppress(Exception):
|
||||
self.make_request("POST", self._workers_url(), json=self.worker.view())
|
||||
|
||||
def workers(self) -> list[Worker]:
|
||||
"""
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from ahriman.core.distributed.distributed_system import DistributedSystem
|
||||
|
||||
|
||||
class WorkerRegisterTrigger(DistributedSystem):
|
||||
"""
|
||||
remote worker registration trigger
|
||||
"""
|
||||
|
||||
def on_start(self) -> None:
|
||||
"""
|
||||
trigger action which will be called at the start of the application
|
||||
"""
|
||||
self.register(force=True)
|
||||
@ -17,22 +17,54 @@
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from threading import Timer
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.distributed.distributed_system import DistributedSystem
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
|
||||
|
||||
class WorkerTrigger(DistributedSystem):
|
||||
"""
|
||||
remote worker processor trigger (client side)
|
||||
|
||||
Attributes:
|
||||
ping_interval(float): interval to call remote service in seconds, defined as ``worker.time_to_live / 4``
|
||||
timer(Timer): timer object
|
||||
"""
|
||||
|
||||
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
|
||||
"""
|
||||
default constructor
|
||||
|
||||
Args:
|
||||
repository_id(RepositoryId): repository unique identifier
|
||||
configuration(Configuration): configuration instance
|
||||
"""
|
||||
DistributedSystem.__init__(self, repository_id, configuration)
|
||||
|
||||
section = next(iter(self.configuration_sections(configuration)))
|
||||
self.ping_interval = configuration.getint(section, "time_to_live", fallback=60) / 4.0
|
||||
self.timer = Timer(self.ping_interval, self.ping)
|
||||
|
||||
def on_start(self) -> None:
|
||||
"""
|
||||
trigger action which will be called at the start of the application
|
||||
"""
|
||||
self.register()
|
||||
self.logger.info("registering instance %s at %s", self.worker, self.address)
|
||||
self.timer.start()
|
||||
|
||||
def on_stop(self) -> None:
|
||||
"""
|
||||
trigger action which will be called before the stop of the application
|
||||
"""
|
||||
self.unregister()
|
||||
self.logger.info("removing instance %s at %s", self.worker, self.address)
|
||||
self.timer.cancel()
|
||||
|
||||
def ping(self) -> None:
|
||||
"""
|
||||
register itself as alive worker and update the timer
|
||||
"""
|
||||
self.register()
|
||||
self.timer = Timer(self.ping_interval, self.ping)
|
||||
self.timer.start()
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from ahriman.core.distributed.distributed_system import DistributedSystem
|
||||
|
||||
|
||||
class WorkerUnregisterTrigger(DistributedSystem):
|
||||
"""
|
||||
remote worker registration trigger
|
||||
"""
|
||||
|
||||
def on_start(self) -> None:
|
||||
"""
|
||||
trigger action which will be called at the start of the application
|
||||
"""
|
||||
self.unregister(force=True)
|
||||
73
src/ahriman/core/distributed/workers_cache.py
Normal file
73
src/ahriman/core/distributed/workers_cache.py
Normal file
@ -0,0 +1,73 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import time
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.log import LazyLogging
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
class WorkersCache(LazyLogging):
|
||||
"""
|
||||
cached storage for healthy workers
|
||||
|
||||
Attributes:
|
||||
time_to_live(int): maximal amount of time in seconds to keep worker alive
|
||||
"""
|
||||
|
||||
def __init__(self, configuration: Configuration) -> None:
|
||||
"""
|
||||
default constructor
|
||||
|
||||
Args:
|
||||
configuration(Configuration): configuration instance
|
||||
"""
|
||||
self.time_to_live = configuration.getint("worker", "time_to_live", fallback=60)
|
||||
self._workers: dict[str, tuple[Worker, float]] = {}
|
||||
|
||||
@property
|
||||
def workers(self) -> list[Worker]:
|
||||
"""
|
||||
extract currently healthy workers
|
||||
|
||||
Returns:
|
||||
list[Worker]: list of currently registered workers which have been seen not earlier than :attr:`time_to_live`
|
||||
"""
|
||||
valid_from = time.monotonic() - self.time_to_live
|
||||
return [
|
||||
worker
|
||||
for worker, last_seen in self._workers.values()
|
||||
if last_seen > valid_from
|
||||
]
|
||||
|
||||
def workers_remove(self) -> None:
|
||||
"""
|
||||
remove all workers from the cache
|
||||
"""
|
||||
self._workers = {}
|
||||
|
||||
def workers_update(self, worker: Worker) -> None:
|
||||
"""
|
||||
register or update remote worker
|
||||
|
||||
Args:
|
||||
worker(Worker): worker to register
|
||||
"""
|
||||
self._workers[worker.identifier] = (worker, time.monotonic())
|
||||
@ -27,15 +27,31 @@ class FilteredAccessLogger(AccessLogger):
|
||||
access logger implementation with log filter enabled
|
||||
|
||||
Attributes:
|
||||
DISTRIBUTED_PATH_REGEX(str): (class attribute) regex used for distributed system uri
|
||||
HEALTH_PATH_REGEX(re.Pattern): (class attribute) regex for health check endpoint
|
||||
LOG_PATH_REGEX(re.Pattern): (class attribute) regex for logs uri
|
||||
LOG_PATPROCESS_PATH_REGEXH_REGEX(re.Pattern): (class attribute) regex for process uri
|
||||
PROCESS_PATH_REGEX(re.Pattern): (class attribute) regex for process uri
|
||||
"""
|
||||
|
||||
DISTRIBUTED_PATH_REGEX = "/api/v1/distributed"
|
||||
HEALTH_PATH_REGEX = "/api/v1/info"
|
||||
LOG_PATH_REGEX = re.compile(r"^/api/v1/packages/[^/]+/logs$")
|
||||
# technically process id is uuid, but we might change it later
|
||||
PROCESS_PATH_REGEX = re.compile(r"^/api/v1/service/process/[^/]+$")
|
||||
|
||||
@staticmethod
|
||||
def is_distributed_post(request: BaseRequest) -> bool:
|
||||
"""
|
||||
check if the request is for distributed services ping
|
||||
|
||||
Args:
|
||||
request(BaseRequest): http reqeust descriptor
|
||||
|
||||
Returns:
|
||||
bool: True in case if request is distributed service ping endpoint and False otherwise
|
||||
"""
|
||||
return request.method == "POST" and FilteredAccessLogger.DISTRIBUTED_PATH_REGEX == request.path
|
||||
|
||||
@staticmethod
|
||||
def is_info_get(request: BaseRequest) -> bool:
|
||||
"""
|
||||
@ -84,7 +100,8 @@ class FilteredAccessLogger(AccessLogger):
|
||||
response(StreamResponse): streaming response object
|
||||
time(float): log record timestamp
|
||||
"""
|
||||
if self.is_info_get(request) \
|
||||
if self.is_distributed_post(request) \
|
||||
or self.is_info_get(request) \
|
||||
or self.is_logs_post(request) \
|
||||
or self.is_process_get(request):
|
||||
return
|
||||
|
||||
@ -26,7 +26,6 @@ from ahriman.models.log_record_id import LogRecordId
|
||||
from ahriman.models.package import Package
|
||||
from ahriman.models.pkgbuild_patch import PkgbuildPatch
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
class Watcher(LazyLogging):
|
||||
@ -224,30 +223,3 @@ class Watcher(LazyLogging):
|
||||
status(BuildStatusEnum): new service status
|
||||
"""
|
||||
self.status = BuildStatus(status)
|
||||
|
||||
def workers_get(self) -> list[Worker]:
|
||||
"""
|
||||
retrieve registered remote workers
|
||||
|
||||
Returns:
|
||||
list[Worker]: list of currently available workers
|
||||
"""
|
||||
return self.database.workers_get()
|
||||
|
||||
def workers_remove(self, identifier: str | None = None) -> None:
|
||||
"""
|
||||
unregister remote worker
|
||||
|
||||
Args:
|
||||
identifier(str | None, optional): remote worker identifier if any (Default value = None)
|
||||
"""
|
||||
self.database.workers_remove(identifier)
|
||||
|
||||
def workers_update(self, worker: Worker) -> None:
|
||||
"""
|
||||
register or update remote worker
|
||||
|
||||
Args:
|
||||
worker(Worker): worker to register
|
||||
"""
|
||||
self.database.workers_insert(worker)
|
||||
|
||||
@ -21,6 +21,7 @@ from aiohttp.web import AppKey
|
||||
|
||||
from ahriman.core.auth import Auth
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.distributed import WorkersCache
|
||||
from ahriman.core.spawn import Spawn
|
||||
from ahriman.core.status.watcher import Watcher
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
@ -31,6 +32,7 @@ __all__ = [
|
||||
"ConfigurationKey",
|
||||
"SpawnKey",
|
||||
"WatcherKey",
|
||||
"WorkersKey",
|
||||
]
|
||||
|
||||
|
||||
@ -38,3 +40,4 @@ AuthKey = AppKey("validator", Auth)
|
||||
ConfigurationKey = AppKey("configuration", Configuration)
|
||||
SpawnKey = AppKey("spawn", Spawn)
|
||||
WatcherKey = AppKey("watcher", dict[RepositoryId, Watcher])
|
||||
WorkersKey = AppKey("workers", WorkersCache)
|
||||
|
||||
@ -49,5 +49,4 @@ from ahriman.web.schemas.search_schema import SearchSchema
|
||||
from ahriman.web.schemas.status_schema import StatusSchema
|
||||
from ahriman.web.schemas.update_flags_schema import UpdateFlagsSchema
|
||||
from ahriman.web.schemas.versioned_log_schema import VersionedLogSchema
|
||||
from ahriman.web.schemas.worker_id_schema import WorkerIdSchema
|
||||
from ahriman.web.schemas.worker_schema import WorkerSchema
|
||||
|
||||
@ -1,31 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from marshmallow import Schema, fields
|
||||
|
||||
|
||||
class WorkerIdSchema(Schema):
|
||||
"""
|
||||
request and response schema for workers
|
||||
"""
|
||||
|
||||
identifier = fields.String(required=True, metadata={
|
||||
"description": "Worker unique identifier",
|
||||
"example": "42f03a62-48f7-46b7-af40-dacc720e92fa",
|
||||
})
|
||||
@ -17,12 +17,10 @@
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from marshmallow import fields
|
||||
|
||||
from ahriman.web.schemas.worker_id_schema import WorkerIdSchema
|
||||
from marshmallow import Schema, fields
|
||||
|
||||
|
||||
class WorkerSchema(WorkerIdSchema):
|
||||
class WorkerSchema(Schema):
|
||||
"""
|
||||
request and response schema for workers
|
||||
"""
|
||||
@ -31,3 +29,7 @@ class WorkerSchema(WorkerIdSchema):
|
||||
"description": "Worker address",
|
||||
"example": "http://localhost:8081",
|
||||
})
|
||||
identifier = fields.String(required=True, metadata={
|
||||
"description": "Worker unique identifier",
|
||||
"example": "42f03a62-48f7-46b7-af40-dacc720e92fa",
|
||||
})
|
||||
|
||||
@ -24,12 +24,13 @@ from typing import TypeVar
|
||||
|
||||
from ahriman.core.auth import Auth
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.distributed import WorkersCache
|
||||
from ahriman.core.sign.gpg import GPG
|
||||
from ahriman.core.spawn import Spawn
|
||||
from ahriman.core.status.watcher import Watcher
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.web.keys import AuthKey, ConfigurationKey, SpawnKey, WatcherKey
|
||||
from ahriman.web.keys import AuthKey, ConfigurationKey, SpawnKey, WatcherKey, WorkersKey
|
||||
|
||||
|
||||
T = TypeVar("T", str, list[str])
|
||||
@ -97,6 +98,16 @@ class BaseView(View, CorsViewMixin):
|
||||
"""
|
||||
return self.request.app[AuthKey]
|
||||
|
||||
@property
|
||||
def workers(self) -> WorkersCache:
|
||||
"""
|
||||
get workers cache instance
|
||||
|
||||
Returns:
|
||||
WorkersCache: workers service
|
||||
"""
|
||||
return self.request.app[WorkersKey]
|
||||
|
||||
@classmethod
|
||||
async def get_permission(cls, request: Request) -> UserAccess:
|
||||
"""
|
||||
|
||||
@ -1,99 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import aiohttp_apispec # type: ignore[import-untyped]
|
||||
|
||||
from aiohttp.web import HTTPNoContent, HTTPNotFound, Response, json_response
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, WorkerIdSchema, WorkerSchema
|
||||
from ahriman.web.views.base import BaseView
|
||||
|
||||
|
||||
class WorkerView(BaseView):
|
||||
"""
|
||||
distributed worker view
|
||||
|
||||
Attributes:
|
||||
DELETE_PERMISSION(UserAccess): (class attribute) delete permissions of self
|
||||
GET_PERMISSION(UserAccess): (class attribute) get permissions of self
|
||||
"""
|
||||
|
||||
DELETE_PERMISSION = GET_PERMISSION = UserAccess.Full
|
||||
ROUTES = ["/api/v1/distributed/{identifier}"]
|
||||
|
||||
@aiohttp_apispec.docs(
|
||||
tags=["Distributed"],
|
||||
summary="Unregister worker",
|
||||
description="Unregister worker and remove it from the service",
|
||||
responses={
|
||||
204: {"description": "Success response"},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
500: {"description": "Internal server error", "schema": ErrorSchema},
|
||||
},
|
||||
security=[{"token": [DELETE_PERMISSION]}],
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
@aiohttp_apispec.match_info_schema(WorkerIdSchema)
|
||||
async def delete(self) -> None:
|
||||
"""
|
||||
unregister worker
|
||||
|
||||
Raises:
|
||||
HTTPNoContent: on success response
|
||||
"""
|
||||
identifier = self.request.match_info["identifier"]
|
||||
self.service().workers_remove(identifier)
|
||||
|
||||
raise HTTPNoContent
|
||||
|
||||
@aiohttp_apispec.docs(
|
||||
tags=["Distributed"],
|
||||
summary="Get worker",
|
||||
description="Retrieve registered worker by its identifier",
|
||||
responses={
|
||||
200: {"description": "Success response", "schema": WorkerSchema(many=True)},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
404: {"description": "Worker is unknown", "schema": ErrorSchema},
|
||||
500: {"description": "Internal server error", "schema": ErrorSchema},
|
||||
},
|
||||
security=[{"token": [GET_PERMISSION]}],
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
@aiohttp_apispec.match_info_schema(WorkerIdSchema)
|
||||
async def get(self) -> Response:
|
||||
"""
|
||||
get worker by identifier
|
||||
|
||||
Returns:
|
||||
Response: 200 with workers list on success
|
||||
|
||||
Raises:
|
||||
HTTPNotFound: if no worker was found
|
||||
"""
|
||||
identifier = self.request.match_info["identifier"]
|
||||
|
||||
try:
|
||||
worker = next(worker for worker in self.service().workers_get() if worker.identifier == identifier)
|
||||
except StopIteration:
|
||||
raise HTTPNotFound(reason=f"Worker {identifier} not found")
|
||||
|
||||
return json_response([worker.view()])
|
||||
@ -61,7 +61,7 @@ class WorkersView(BaseView):
|
||||
Raises:
|
||||
HTTPNoContent: on success response
|
||||
"""
|
||||
self.service().workers_remove()
|
||||
self.workers.workers_remove()
|
||||
|
||||
raise HTTPNoContent
|
||||
|
||||
@ -85,7 +85,7 @@ class WorkersView(BaseView):
|
||||
Returns:
|
||||
Response: 200 with workers list on success
|
||||
"""
|
||||
workers = self.service().workers_get()
|
||||
workers = self.workers.workers
|
||||
|
||||
comparator: Callable[[Worker], str] = lambda item: item.identifier
|
||||
response = [worker.view() for worker in sorted(workers, key=comparator)]
|
||||
@ -121,6 +121,6 @@ class WorkersView(BaseView):
|
||||
except Exception as ex:
|
||||
raise HTTPBadRequest(reason=str(ex))
|
||||
|
||||
self.service().workers_update(worker)
|
||||
self.workers.workers_update(worker)
|
||||
|
||||
raise HTTPNoContent
|
||||
|
||||
@ -27,6 +27,7 @@ from aiohttp.web import Application, normalize_path_middleware, run_app
|
||||
from ahriman.core.auth import Auth
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.database import SQLite
|
||||
from ahriman.core.distributed import WorkersCache
|
||||
from ahriman.core.exceptions import InitializeError
|
||||
from ahriman.core.log.filtered_access_logger import FilteredAccessLogger
|
||||
from ahriman.core.spawn import Spawn
|
||||
@ -34,7 +35,7 @@ from ahriman.core.status.watcher import Watcher
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
from ahriman.web.apispec import setup_apispec
|
||||
from ahriman.web.cors import setup_cors
|
||||
from ahriman.web.keys import AuthKey, ConfigurationKey, SpawnKey, WatcherKey
|
||||
from ahriman.web.keys import AuthKey, ConfigurationKey, SpawnKey, WatcherKey, WorkersKey
|
||||
from ahriman.web.middlewares.exception_handler import exception_handler
|
||||
from ahriman.web.routes import setup_routes
|
||||
|
||||
@ -159,7 +160,8 @@ def setup_server(configuration: Configuration, spawner: Spawn, repositories: lis
|
||||
application.logger.info("setup configuration")
|
||||
application[ConfigurationKey] = configuration
|
||||
|
||||
application.logger.info("setup watchers")
|
||||
application.logger.info("setup services")
|
||||
# package cache
|
||||
if not repositories:
|
||||
raise InitializeError("No repositories configured, exiting")
|
||||
database = SQLite.load(configuration)
|
||||
@ -168,8 +170,9 @@ def setup_server(configuration: Configuration, spawner: Spawn, repositories: lis
|
||||
application.logger.info("load repository %s", repository_id)
|
||||
watchers[repository_id] = Watcher(repository_id, database)
|
||||
application[WatcherKey] = watchers
|
||||
|
||||
application.logger.info("setup process spawner")
|
||||
# workers cache
|
||||
application[WorkersKey] = WorkersCache(configuration)
|
||||
# process spawner
|
||||
application[SpawnKey] = spawner
|
||||
|
||||
application.logger.info("setup authorization")
|
||||
|
||||
@ -1320,80 +1320,6 @@ def test_subparsers_service_tree_migrate(parser: argparse.ArgumentParser) -> Non
|
||||
assert not args.report
|
||||
|
||||
|
||||
def test_subparsers_service_worker_register(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-register command must imply trigger
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-register"])
|
||||
assert args.trigger == ["ahriman.core.distributed.WorkerRegisterTrigger"]
|
||||
|
||||
|
||||
def test_subparsers_service_worker_register_option_architecture(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-register command must correctly parse architecture list
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-register"])
|
||||
assert args.architecture is None
|
||||
args = parser.parse_args(["-a", "x86_64", "service-worker-register"])
|
||||
assert args.architecture == "x86_64"
|
||||
|
||||
|
||||
def test_subparsers_service_worker_register_option_repository(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-register command must correctly parse repository list
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-register"])
|
||||
assert args.repository is None
|
||||
args = parser.parse_args(["-r", "repo", "service-worker-register"])
|
||||
assert args.repository == "repo"
|
||||
|
||||
|
||||
def test_subparsers_service_worker_register_repo_triggers(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-register must have same keys as repo-triggers
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-register"])
|
||||
reference_args = parser.parse_args(["repo-triggers"])
|
||||
assert dir(args) == dir(reference_args)
|
||||
|
||||
|
||||
def test_subparsers_service_worker_unregister(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-unregister command must imply trigger
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-unregister"])
|
||||
assert args.trigger == ["ahriman.core.distributed.WorkerUnregisterTrigger"]
|
||||
|
||||
|
||||
def test_subparsers_service_worker_unregister_option_architecture(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-unregister command must correctly parse architecture list
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-unregister"])
|
||||
assert args.architecture is None
|
||||
args = parser.parse_args(["-a", "x86_64", "service-worker-unregister"])
|
||||
assert args.architecture == "x86_64"
|
||||
|
||||
|
||||
def test_subparsers_service_worker_unregister_option_repository(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-unregister command must correctly parse repository list
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-unregister"])
|
||||
assert args.repository is None
|
||||
args = parser.parse_args(["-r", "repo", "service-worker-unregister"])
|
||||
assert args.repository == "repo"
|
||||
|
||||
|
||||
def test_subparsers_service_worker_unregister_repo_triggers(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
service-worker-unregister must have same keys as repo-triggers
|
||||
"""
|
||||
args = parser.parse_args(["service-worker-unregister"])
|
||||
reference_args = parser.parse_args(["repo-triggers"])
|
||||
assert dir(args) == dir(reference_args)
|
||||
|
||||
|
||||
def test_subparsers_user_add(parser: argparse.ArgumentParser) -> None:
|
||||
"""
|
||||
user-add command must imply action, architecture, exit code, lock, quiet, report and repository
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
from ahriman.core.database.migrations.m013_workers import steps
|
||||
|
||||
|
||||
def test_migration_workers() -> None:
|
||||
"""
|
||||
migration must not be empty
|
||||
"""
|
||||
assert steps
|
||||
@ -1,46 +0,0 @@
|
||||
from ahriman.core.database import SQLite
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
def test_workers_get_insert(database: SQLite) -> None:
|
||||
"""
|
||||
must insert workers to database
|
||||
"""
|
||||
database.workers_insert(Worker("address1", identifier="1"))
|
||||
database.workers_insert(Worker("address2", identifier="2"))
|
||||
assert database.workers_get() == [
|
||||
Worker("address1", identifier="1"), Worker("address2", identifier="2")
|
||||
]
|
||||
|
||||
|
||||
def test_workers_insert_remove(database: SQLite) -> None:
|
||||
"""
|
||||
must remove worker from database
|
||||
"""
|
||||
database.workers_insert(Worker("address1", identifier="1"))
|
||||
database.workers_insert(Worker("address2", identifier="2"))
|
||||
database.workers_remove("1")
|
||||
|
||||
assert database.workers_get() == [Worker("address2", identifier="2")]
|
||||
|
||||
|
||||
def test_workers_insert_remove_all(database: SQLite) -> None:
|
||||
"""
|
||||
must remove all workers
|
||||
"""
|
||||
database.workers_insert(Worker("address1", identifier="1"))
|
||||
database.workers_insert(Worker("address2", identifier="2"))
|
||||
database.workers_remove()
|
||||
|
||||
assert database.workers_get() == []
|
||||
|
||||
|
||||
def test_workers_insert_insert(database: SQLite) -> None:
|
||||
"""
|
||||
must update worker in database
|
||||
"""
|
||||
database.workers_insert(Worker("address1", identifier="1"))
|
||||
assert database.workers_get() == [Worker("address1", identifier="1")]
|
||||
|
||||
database.workers_insert(Worker("address2", identifier="1"))
|
||||
assert database.workers_get() == [Worker("address2", identifier="1")]
|
||||
@ -1,6 +1,7 @@
|
||||
import pytest
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.distributed import WorkersCache
|
||||
from ahriman.core.distributed.distributed_system import DistributedSystem
|
||||
|
||||
|
||||
@ -18,3 +19,17 @@ def distributed_system(configuration: Configuration) -> DistributedSystem:
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
return DistributedSystem(repository_id, configuration)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def workers_cache(configuration: Configuration) -> WorkersCache:
|
||||
"""
|
||||
workers cache fixture
|
||||
|
||||
Args:
|
||||
configuration(Configuration): configuration fixture
|
||||
|
||||
Returns:
|
||||
WorkersCache: workers cache test instance
|
||||
"""
|
||||
return WorkersCache(configuration)
|
||||
|
||||
@ -8,15 +8,6 @@ from ahriman.core.distributed.distributed_system import DistributedSystem
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
def test_identifier_path(configuration: Configuration) -> None:
|
||||
"""
|
||||
must correctly set default identifier path
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
assert DistributedSystem(repository_id, configuration).identifier_path
|
||||
|
||||
|
||||
def test_configuration_sections(configuration: Configuration) -> None:
|
||||
"""
|
||||
must correctly parse target list
|
||||
@ -31,138 +22,31 @@ def test_workers_url(distributed_system: DistributedSystem) -> None:
|
||||
assert distributed_system._workers_url().startswith(distributed_system.address)
|
||||
assert distributed_system._workers_url().endswith("/api/v1/distributed")
|
||||
|
||||
assert distributed_system._workers_url("id").startswith(distributed_system.address)
|
||||
assert distributed_system._workers_url("id").endswith("/api/v1/distributed/id")
|
||||
|
||||
|
||||
def test_load_identifier(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must generate identifier
|
||||
"""
|
||||
mocker.patch("pathlib.Path.is_file", return_value=False)
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
system = DistributedSystem(repository_id, configuration)
|
||||
|
||||
assert system.load_identifier(configuration, "worker")
|
||||
|
||||
|
||||
def test_load_identifier_configuration(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must load identifier from configuration
|
||||
"""
|
||||
identifier = "id"
|
||||
mocker.patch("pathlib.Path.is_file", return_value=False)
|
||||
configuration.set_option("worker", "identifier", identifier)
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
system = DistributedSystem(repository_id, configuration)
|
||||
|
||||
assert system.worker.identifier == identifier
|
||||
|
||||
|
||||
def test_load_identifier_filesystem(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must load identifier from filesystem
|
||||
"""
|
||||
identifier = "id"
|
||||
mocker.patch("pathlib.Path.is_file", return_value=True)
|
||||
read_mock = mocker.patch("pathlib.Path.read_text", return_value=identifier)
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
system = DistributedSystem(repository_id, configuration)
|
||||
|
||||
assert system.worker.identifier == identifier
|
||||
read_mock.assert_called_once_with(encoding="utf8")
|
||||
|
||||
|
||||
def test_register(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must register service
|
||||
"""
|
||||
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
|
||||
mocker.patch("pathlib.Path.is_file", return_value=False)
|
||||
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
|
||||
write_mock = mocker.patch("pathlib.Path.write_text")
|
||||
|
||||
distributed_system.register()
|
||||
run_mock.assert_called_once_with("POST", f"{distributed_system.address}/api/v1/distributed",
|
||||
json=distributed_system.worker.view())
|
||||
write_mock.assert_called_once_with(distributed_system.worker.identifier, encoding="utf8")
|
||||
assert distributed_system._owe_identifier
|
||||
|
||||
|
||||
def test_register_skip(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
def test_register_failed(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must skip service registration if it doesn't owe the identifier
|
||||
must suppress any exception happened during worker registration
|
||||
"""
|
||||
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
|
||||
mocker.patch("pathlib.Path.is_file", return_value=True)
|
||||
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
|
||||
write_mock = mocker.patch("pathlib.Path.write_text")
|
||||
|
||||
mocker.patch("requests.Session.request", side_effect=Exception())
|
||||
distributed_system.register()
|
||||
run_mock.assert_not_called()
|
||||
write_mock.assert_not_called()
|
||||
assert not distributed_system._owe_identifier
|
||||
|
||||
|
||||
def test_register_force(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
def test_register_failed_http_error(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must register service even if it doesn't owe the identifier if force is supplied
|
||||
must suppress HTTP exception happened during worker registration
|
||||
"""
|
||||
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
|
||||
mocker.patch("pathlib.Path.is_file", return_value=True)
|
||||
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
|
||||
write_mock = mocker.patch("pathlib.Path.write_text")
|
||||
|
||||
distributed_system.register(force=True)
|
||||
run_mock.assert_called_once_with("POST", f"{distributed_system.address}/api/v1/distributed",
|
||||
json=distributed_system.worker.view())
|
||||
write_mock.assert_called_once_with(distributed_system.worker.identifier, encoding="utf8")
|
||||
assert distributed_system._owe_identifier
|
||||
|
||||
|
||||
def test_unregister(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must unregister service
|
||||
"""
|
||||
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
|
||||
remove_mock = mocker.patch("pathlib.Path.unlink")
|
||||
distributed_system._owe_identifier = True
|
||||
|
||||
distributed_system.unregister()
|
||||
run_mock.assert_called_once_with(
|
||||
"DELETE", f"{distributed_system.address}/api/v1/distributed/{distributed_system.worker.identifier}")
|
||||
remove_mock.assert_called_once_with(missing_ok=True)
|
||||
|
||||
|
||||
def test_unregister_skip(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must skip service removal if it doesn't owe the identifier
|
||||
"""
|
||||
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
|
||||
remove_mock = mocker.patch("pathlib.Path.unlink")
|
||||
|
||||
distributed_system.unregister()
|
||||
run_mock.assert_not_called()
|
||||
remove_mock.assert_not_called()
|
||||
|
||||
|
||||
def test_unregister_force(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must remove service even if it doesn't owe the identifier if force is supplied
|
||||
"""
|
||||
mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request")
|
||||
remove_mock = mocker.patch("pathlib.Path.unlink")
|
||||
|
||||
distributed_system.unregister(force=True)
|
||||
run_mock.assert_called_once_with(
|
||||
"DELETE", f"{distributed_system.address}/api/v1/distributed/{distributed_system.worker.identifier}")
|
||||
remove_mock.assert_called_once_with(missing_ok=True)
|
||||
mocker.patch("requests.Session.request", side_effect=requests.HTTPError())
|
||||
distributed_system.register()
|
||||
|
||||
|
||||
def test_workers(distributed_system: DistributedSystem, mocker: MockerFixture) -> None:
|
||||
|
||||
@ -1,17 +0,0 @@
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.distributed import WorkerRegisterTrigger
|
||||
|
||||
|
||||
def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must register itself as worker
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.WorkerRegisterTrigger.register")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
|
||||
trigger = WorkerRegisterTrigger(repository_id, configuration)
|
||||
trigger.on_start()
|
||||
run_mock.assert_called_once_with(force=True)
|
||||
@ -9,11 +9,10 @@ def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
must register itself as worker
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.WorkerTrigger.register")
|
||||
run_mock = mocker.patch("threading.Timer.start")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
|
||||
trigger = WorkerTrigger(repository_id, configuration)
|
||||
trigger.on_start()
|
||||
WorkerTrigger(repository_id, configuration).on_start()
|
||||
run_mock.assert_called_once_with()
|
||||
|
||||
|
||||
@ -22,9 +21,32 @@ def test_on_stop(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
must unregister itself as worker
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.WorkerTrigger.unregister")
|
||||
run_mock = mocker.patch("threading.Timer.cancel")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
|
||||
trigger = WorkerTrigger(repository_id, configuration)
|
||||
trigger.on_stop()
|
||||
WorkerTrigger(repository_id, configuration).on_stop()
|
||||
run_mock.assert_called_once_with()
|
||||
|
||||
|
||||
def test_on_stop_empty_timer(configuration: Configuration) -> None:
|
||||
"""
|
||||
must do not fail if no timer was started
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
|
||||
WorkerTrigger(repository_id, configuration).on_stop()
|
||||
|
||||
|
||||
def test_ping(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must correctly process timer action
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.WorkerTrigger.register")
|
||||
timer_mock = mocker.patch("threading.Timer.start")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
|
||||
WorkerTrigger(repository_id, configuration).ping()
|
||||
run_mock.assert_called_once_with()
|
||||
timer_mock.assert_called_once_with()
|
||||
|
||||
@ -1,17 +0,0 @@
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.distributed import WorkerUnregisterTrigger
|
||||
|
||||
|
||||
def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must unregister itself as worker
|
||||
"""
|
||||
configuration.set_option("status", "address", "http://localhost:8081")
|
||||
run_mock = mocker.patch("ahriman.core.distributed.WorkerUnregisterTrigger.unregister")
|
||||
_, repository_id = configuration.check_loaded()
|
||||
|
||||
trigger = WorkerUnregisterTrigger(repository_id, configuration)
|
||||
trigger.on_start()
|
||||
run_mock.assert_called_once_with(force=True)
|
||||
43
tests/ahriman/core/distributed/test_workers_cache.py
Normal file
43
tests/ahriman/core/distributed/test_workers_cache.py
Normal file
@ -0,0 +1,43 @@
|
||||
import time
|
||||
|
||||
from ahriman.core.distributed import WorkersCache
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
def test_workers(workers_cache: WorkersCache) -> None:
|
||||
"""
|
||||
must return alive workers
|
||||
"""
|
||||
workers_cache._workers = {
|
||||
str(index): (Worker(f"address{index}"), index)
|
||||
for index in range(2)
|
||||
}
|
||||
workers_cache.time_to_live = time.monotonic()
|
||||
|
||||
assert workers_cache.workers == [Worker("address1")]
|
||||
|
||||
|
||||
def test_workers_remove(workers_cache: WorkersCache) -> None:
|
||||
"""
|
||||
must remove all workers
|
||||
"""
|
||||
workers_cache.workers_update(Worker("address"))
|
||||
assert workers_cache.workers
|
||||
|
||||
workers_cache.workers_remove()
|
||||
assert not workers_cache.workers
|
||||
|
||||
|
||||
def test_workers_update(workers_cache: WorkersCache) -> None:
|
||||
"""
|
||||
must update worker
|
||||
"""
|
||||
worker = Worker("address")
|
||||
|
||||
workers_cache.workers_update(worker)
|
||||
assert workers_cache.workers == [worker]
|
||||
_, first_last_seen = workers_cache._workers[worker.identifier]
|
||||
|
||||
workers_cache.workers_update(worker)
|
||||
_, second_last_seen = workers_cache._workers[worker.identifier]
|
||||
assert first_last_seen < second_last_seen
|
||||
@ -4,7 +4,26 @@ from unittest.mock import MagicMock
|
||||
from ahriman.core.log.filtered_access_logger import FilteredAccessLogger
|
||||
|
||||
|
||||
def is_info_get() -> None:
|
||||
def test_is_distributed_post() -> None:
|
||||
"""
|
||||
must correctly define distributed services ping request
|
||||
"""
|
||||
request = MagicMock()
|
||||
|
||||
request.method = "POST"
|
||||
request.path = "/api/v1/distributed"
|
||||
assert FilteredAccessLogger.is_distributed_post(request)
|
||||
|
||||
request.method = "GET"
|
||||
request.path = "/api/v1/distributed"
|
||||
assert not FilteredAccessLogger.is_distributed_post(request)
|
||||
|
||||
request.method = "POST"
|
||||
request.path = "/api/v1/distributed/path"
|
||||
assert not FilteredAccessLogger.is_distributed_post(request)
|
||||
|
||||
|
||||
def test_is_info_get() -> None:
|
||||
"""
|
||||
must correctly define health check request
|
||||
"""
|
||||
|
||||
@ -10,7 +10,6 @@ from ahriman.models.changes import Changes
|
||||
from ahriman.models.log_record_id import LogRecordId
|
||||
from ahriman.models.package import Package
|
||||
from ahriman.models.pkgbuild_patch import PkgbuildPatch
|
||||
from ahriman.models.worker import Worker
|
||||
|
||||
|
||||
def test_load(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
|
||||
@ -228,40 +227,3 @@ def test_status_update(watcher: Watcher) -> None:
|
||||
"""
|
||||
watcher.status_update(BuildStatusEnum.Success)
|
||||
assert watcher.status.status == BuildStatusEnum.Success
|
||||
|
||||
|
||||
def test_workers_get(watcher: Watcher, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must retrieve workers
|
||||
"""
|
||||
worker = Worker("remote")
|
||||
worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_get", return_value=[worker])
|
||||
|
||||
assert watcher.workers_get() == [worker]
|
||||
worker_mock.assert_called_once_with()
|
||||
|
||||
|
||||
def test_workers_remove(watcher: Watcher, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must remove workers
|
||||
"""
|
||||
identifier = "identifier"
|
||||
worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_remove")
|
||||
|
||||
watcher.workers_remove(identifier)
|
||||
watcher.workers_remove()
|
||||
worker_mock.assert_has_calls([
|
||||
MockCall(identifier),
|
||||
MockCall(None),
|
||||
])
|
||||
|
||||
|
||||
def test_workers_update(watcher: Watcher, mocker: MockerFixture) -> None:
|
||||
"""
|
||||
must update workers
|
||||
"""
|
||||
worker = Worker("remote")
|
||||
worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_insert")
|
||||
|
||||
watcher.workers_update(worker)
|
||||
worker_mock.assert_called_once_with(worker)
|
||||
|
||||
@ -1 +0,0 @@
|
||||
# schema testing goes in view class tests
|
||||
@ -50,11 +50,18 @@ def test_spawn(base: BaseView) -> None:
|
||||
|
||||
def test_validator(base: BaseView) -> None:
|
||||
"""
|
||||
must return service
|
||||
must return validator service
|
||||
"""
|
||||
assert base.validator
|
||||
|
||||
|
||||
def test_workers(base: BaseView) -> None:
|
||||
"""
|
||||
must return worker service
|
||||
"""
|
||||
assert base.workers
|
||||
|
||||
|
||||
async def test_get_permission(base: BaseView) -> None:
|
||||
"""
|
||||
must search for permission attribute in class
|
||||
|
||||
@ -1,70 +0,0 @@
|
||||
import pytest
|
||||
|
||||
from aiohttp.test_utils import TestClient
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.models.worker import Worker
|
||||
from ahriman.web.views.v1.distributed.worker import WorkerView
|
||||
|
||||
|
||||
async def test_get_permission() -> None:
|
||||
"""
|
||||
must return correct permission for the request
|
||||
"""
|
||||
for method in ("DELETE", "GET"):
|
||||
request = pytest.helpers.request("", "", method)
|
||||
assert await WorkerView.get_permission(request) == UserAccess.Full
|
||||
|
||||
|
||||
def test_routes() -> None:
|
||||
"""
|
||||
must return correct routes
|
||||
"""
|
||||
assert WorkerView.ROUTES == ["/api/v1/distributed/{identifier}"]
|
||||
|
||||
|
||||
async def test_delete(client: TestClient) -> None:
|
||||
"""
|
||||
must delete single worker
|
||||
"""
|
||||
await client.post("/api/v1/distributed", json={"address": "address1", "identifier": "1"})
|
||||
await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"})
|
||||
|
||||
response = await client.delete("/api/v1/distributed/1")
|
||||
assert response.status == 204
|
||||
|
||||
response = await client.get("/api/v1/distributed/1")
|
||||
assert response.status == 404
|
||||
|
||||
response = await client.get("/api/v1/distributed/2")
|
||||
assert response.ok
|
||||
|
||||
|
||||
async def test_get(client: TestClient) -> None:
|
||||
"""
|
||||
must return specific worker
|
||||
"""
|
||||
worker = Worker("address1", identifier="1")
|
||||
|
||||
await client.post("/api/v1/distributed", json=worker.view())
|
||||
await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"})
|
||||
response_schema = pytest.helpers.schema_response(WorkerView.get)
|
||||
|
||||
response = await client.get(f"/api/v1/distributed/{worker.identifier}")
|
||||
assert response.ok
|
||||
json = await response.json()
|
||||
assert not response_schema.validate(json, many=True)
|
||||
|
||||
workers = [Worker(item["address"], identifier=item["identifier"]) for item in json]
|
||||
assert workers == [worker]
|
||||
|
||||
|
||||
async def test_get_not_found(client: TestClient) -> None:
|
||||
"""
|
||||
must return Not Found for unknown package
|
||||
"""
|
||||
response_schema = pytest.helpers.schema_response(WorkerView.get, code=404)
|
||||
|
||||
response = await client.get("/api/v1/distributed/1")
|
||||
assert response.status == 404
|
||||
assert not response_schema.validate(await response.json())
|
||||
@ -68,9 +68,6 @@ async def test_post(client: TestClient) -> None:
|
||||
response = await client.post("/api/v1/distributed", json=payload)
|
||||
assert response.status == 204
|
||||
|
||||
response = await client.get(f"/api/v1/distributed/{worker.identifier}")
|
||||
assert response.ok
|
||||
|
||||
|
||||
async def test_post_exception(client: TestClient) -> None:
|
||||
"""
|
||||
|
||||
Loading…
Reference in New Issue
Block a user