diff --git a/docs/ahriman.core.database.migrations.rst b/docs/ahriman.core.database.migrations.rst index b21bec7f..30cb7d28 100644 --- a/docs/ahriman.core.database.migrations.rst +++ b/docs/ahriman.core.database.migrations.rst @@ -108,14 +108,6 @@ ahriman.core.database.migrations.m012\_last\_commit\_sha module :no-undoc-members: :show-inheritance: -ahriman.core.database.migrations.m013\_workers module ------------------------------------------------------ - -.. automodule:: ahriman.core.database.migrations.m013_workers - :members: - :no-undoc-members: - :show-inheritance: - Module contents --------------- diff --git a/docs/ahriman.core.database.operations.rst b/docs/ahriman.core.database.operations.rst index afa23802..c2d38a94 100644 --- a/docs/ahriman.core.database.operations.rst +++ b/docs/ahriman.core.database.operations.rst @@ -60,14 +60,6 @@ ahriman.core.database.operations.patch\_operations module :no-undoc-members: :show-inheritance: -ahriman.core.database.operations.workers\_operations module ------------------------------------------------------------ - -.. automodule:: ahriman.core.database.operations.workers_operations - :members: - :no-undoc-members: - :show-inheritance: - Module contents --------------- diff --git a/docs/ahriman.core.distributed.rst b/docs/ahriman.core.distributed.rst index 7158f788..b22cf82e 100644 --- a/docs/ahriman.core.distributed.rst +++ b/docs/ahriman.core.distributed.rst @@ -20,14 +20,6 @@ ahriman.core.distributed.worker\_loader\_trigger module :no-undoc-members: :show-inheritance: -ahriman.core.distributed.worker\_register\_trigger module ---------------------------------------------------------- - -.. automodule:: ahriman.core.distributed.worker_register_trigger - :members: - :no-undoc-members: - :show-inheritance: - ahriman.core.distributed.worker\_trigger module ----------------------------------------------- @@ -36,10 +28,10 @@ ahriman.core.distributed.worker\_trigger module :no-undoc-members: :show-inheritance: -ahriman.core.distributed.worker\_unregister\_trigger module ------------------------------------------------------------ +ahriman.core.distributed.workers\_cache module +---------------------------------------------- -.. automodule:: ahriman.core.distributed.worker_unregister_trigger +.. automodule:: ahriman.core.distributed.workers_cache :members: :no-undoc-members: :show-inheritance: diff --git a/docs/ahriman.web.schemas.rst b/docs/ahriman.web.schemas.rst index f6924214..ef02c15a 100644 --- a/docs/ahriman.web.schemas.rst +++ b/docs/ahriman.web.schemas.rst @@ -260,14 +260,6 @@ ahriman.web.schemas.versioned\_log\_schema module :no-undoc-members: :show-inheritance: -ahriman.web.schemas.worker\_id\_schema module ---------------------------------------------- - -.. automodule:: ahriman.web.schemas.worker_id_schema - :members: - :no-undoc-members: - :show-inheritance: - ahriman.web.schemas.worker\_schema module ----------------------------------------- diff --git a/docs/ahriman.web.views.v1.distributed.rst b/docs/ahriman.web.views.v1.distributed.rst index 452b46eb..eed37ee1 100644 --- a/docs/ahriman.web.views.v1.distributed.rst +++ b/docs/ahriman.web.views.v1.distributed.rst @@ -4,14 +4,6 @@ ahriman.web.views.v1.distributed package Submodules ---------- -ahriman.web.views.v1.distributed.worker module ----------------------------------------------- - -.. automodule:: ahriman.web.views.v1.distributed.worker - :members: - :no-undoc-members: - :show-inheritance: - ahriman.web.views.v1.distributed.workers module ----------------------------------------------- diff --git a/docs/architecture.rst b/docs/architecture.rst index cece384a..67d8af34 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -37,6 +37,7 @@ This package contains everything required for the most of application actions an * ``ahriman.core.build_tools`` is a package which provides wrapper for ``devtools`` commands. * ``ahriman.core.configuration`` contains extension for standard ``configparser`` library and some validation related classes. * ``ahriman.core.database`` is everything for database, including data and schema migrations. +* ``ahriman.core.distributed`` package with triggers and helpers for distributed build system. * ``ahriman.core.formatters`` package provides ``Printer`` sub-classes for printing data (e.g. package properties) to stdout which are used by some handlers. * ``ahriman.core.gitremote`` is a package with remote PKGBUILD triggers. Should not be called directly. * ``ahriman.core.http`` package provides HTTP clients which can be used later by other classes. diff --git a/docs/configuration.rst b/docs/configuration.rst index 65d96fee..fbe99e66 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -359,5 +359,5 @@ Requires ``boto3`` library to be installed. Section name must be either ``s3`` ( This section controls settings for ``ahriman.core.distributed.WorkerTrigger`` plugin. * ``address`` - address of the instance, string, required. Must be reachable for the master instance. -* ``identifier`` - unique identifier of the instance, string, optional. If none set, the random uuid will be generated on each run automatically. -* ``identifier_path`` - path to lock file, string, optional, default is ``/tmp/ahriman-worker-identifier``. +* ``identifier`` - unique identifier of the instance, string, optional. +* ``time_to_live`` - amount of time which remote worker will be considered alive in seconds, integer, optional, default is ``60``. The ping interval will be set automatically equal this value divided by 4. diff --git a/docs/triggers.rst b/docs/triggers.rst index c276c098..e1c3d814 100644 --- a/docs/triggers.rst +++ b/docs/triggers.rst @@ -22,9 +22,7 @@ Special trigger to be used to load workers from database on the start of the app ``ahriman.core.distributed.WorkerTrigger`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Another trigger for the distributed system, which registers itself as remote worker. It calls the remote server on start (if no lock file found) and, later, it deregister itself before the stop. - -There are also two triggers which performs only registration and removal (``ahriman.core.distributed.WorkerRegisterTrigger`` and ``ahriman.core.distributed.WorkerUnregisterTrigger`` respectively), but they are not meant to be called directly. +Another trigger for the distributed system, which registers itself as remote worker, calling remote service periodically. ``ahriman.core.gitremote.RemotePullTrigger`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/package/share/bash-completion/completions/_ahriman b/package/share/bash-completion/completions/_ahriman index d802c4df..e7de6870 100644 --- a/package/share/bash-completion/completions/_ahriman +++ b/package/share/bash-completion/completions/_ahriman @@ -1,6 +1,6 @@ # AUTOMATICALLY GENERATED by `shtab` -_shtab_ahriman_subparsers=('aur-search' 'search' 'help-commands-unsafe' 'help' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-changes' 'package-changes-remove' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-repositories' 'service-run' 'run' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'service-tree-migrate' 'service-worker-register' 'service-worker-unregister' 'user-add' 'user-list' 'user-remove' 'web') +_shtab_ahriman_subparsers=('aur-search' 'search' 'help-commands-unsafe' 'help' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-changes' 'package-changes-remove' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-repositories' 'service-run' 'run' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'service-tree-migrate' 'user-add' 'user-list' 'user-remove' 'web') _shtab_ahriman_option_strings=('-h' '--help' '-a' '--architecture' '-c' '--configuration' '--force' '-l' '--lock' '--log-handler' '-q' '--quiet' '--report' '--no-report' '-r' '--repository' '--unsafe' '-V' '--version' '--wait-timeout') _shtab_ahriman_aur_search_option_strings=('-h' '--help' '-e' '--exit-code' '--info' '--no-info' '--sort-by') @@ -71,8 +71,6 @@ _shtab_ahriman_setup_option_strings=('-h' '--help' '--build-as-user' '--from-con _shtab_ahriman_service_shell_option_strings=('-h' '--help') _shtab_ahriman_shell_option_strings=('-h' '--help') _shtab_ahriman_service_tree_migrate_option_strings=('-h' '--help') -_shtab_ahriman_service_worker_register_option_strings=('-h' '--help') -_shtab_ahriman_service_worker_unregister_option_strings=('-h' '--help') _shtab_ahriman_user_add_option_strings=('-h' '--help' '--key' '--packager' '-p' '--password' '-R' '--role') _shtab_ahriman_user_list_option_strings=('-h' '--help' '-e' '--exit-code' '-R' '--role') _shtab_ahriman_user_remove_option_strings=('-h' '--help') @@ -80,7 +78,7 @@ _shtab_ahriman_web_option_strings=('-h' '--help') -_shtab_ahriman_pos_0_choices=('aur-search' 'search' 'help-commands-unsafe' 'help' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-changes' 'package-changes-remove' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-repositories' 'service-run' 'run' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'service-tree-migrate' 'service-worker-register' 'service-worker-unregister' 'user-add' 'user-list' 'user-remove' 'web') +_shtab_ahriman_pos_0_choices=('aur-search' 'search' 'help-commands-unsafe' 'help' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-changes' 'package-changes-remove' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-repositories' 'service-run' 'run' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'service-tree-migrate' 'user-add' 'user-list' 'user-remove' 'web') _shtab_ahriman___log_handler_choices=('console' 'syslog' 'journald') _shtab_ahriman_aur_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'submitter' 'url' 'url_path' 'version') _shtab_ahriman_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'submitter' 'url' 'url_path' 'version') @@ -527,10 +525,6 @@ _shtab_ahriman_shell__v_nargs=0 _shtab_ahriman_shell___verbose_nargs=0 _shtab_ahriman_service_tree_migrate__h_nargs=0 _shtab_ahriman_service_tree_migrate___help_nargs=0 -_shtab_ahriman_service_worker_register__h_nargs=0 -_shtab_ahriman_service_worker_register___help_nargs=0 -_shtab_ahriman_service_worker_unregister__h_nargs=0 -_shtab_ahriman_service_worker_unregister___help_nargs=0 _shtab_ahriman_user_add__h_nargs=0 _shtab_ahriman_user_add___help_nargs=0 _shtab_ahriman_user_list__h_nargs=0 diff --git a/package/share/man/man1/ahriman.1 b/package/share/man/man1/ahriman.1 index 033e5e97..7d8a86b4 100644 --- a/package/share/man/man1/ahriman.1 +++ b/package/share/man/man1/ahriman.1 @@ -1,9 +1,9 @@ -.TH AHRIMAN "1" "2023\-12\-31" "ahriman" "Generated Python Manual" +.TH AHRIMAN "1" "2024\-01\-02" "ahriman" "Generated Python Manual" .SH NAME ahriman .SH SYNOPSIS .B ahriman -[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--log-handler {console,syslog,journald}] [-q] [--report | --no-report] [-r REPOSITORY] [--unsafe] [-V] [--wait-timeout WAIT_TIMEOUT] {aur-search,search,help-commands-unsafe,help,help-updates,help-version,version,package-add,add,package-update,package-changes,package-changes-remove,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,patch-set-add,repo-backup,repo-check,check,repo-create-keyring,repo-create-mirrorlist,repo-daemon,daemon,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-sign,sign,repo-status-update,repo-sync,sync,repo-tree,repo-triggers,repo-update,update,service-clean,clean,repo-clean,service-config,config,repo-config,service-config-validate,config-validate,repo-config-validate,service-key-import,key-import,service-repositories,service-run,run,service-setup,init,repo-init,repo-setup,setup,service-shell,shell,service-tree-migrate,service-worker-register,service-worker-unregister,user-add,user-list,user-remove,web} ... +[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--log-handler {console,syslog,journald}] [-q] [--report | --no-report] [-r REPOSITORY] [--unsafe] [-V] [--wait-timeout WAIT_TIMEOUT] {aur-search,search,help-commands-unsafe,help,help-updates,help-version,version,package-add,add,package-update,package-changes,package-changes-remove,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,patch-set-add,repo-backup,repo-check,check,repo-create-keyring,repo-create-mirrorlist,repo-daemon,daemon,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-sign,sign,repo-status-update,repo-sync,sync,repo-tree,repo-triggers,repo-update,update,service-clean,clean,repo-clean,service-config,config,repo-config,service-config-validate,config-validate,repo-config-validate,service-key-import,key-import,service-repositories,service-run,run,service-setup,init,repo-init,repo-setup,setup,service-shell,shell,service-tree-migrate,user-add,user-list,user-remove,web} ... .SH DESCRIPTION ArcH linux ReposItory MANager @@ -176,12 +176,6 @@ invoke python shell \fBahriman\fR \fI\,service\-tree\-migrate\/\fR migrate repository tree .TP -\fBahriman\fR \fI\,service\-worker\-register\/\fR -register itself as worker -.TP -\fBahriman\fR \fI\,service\-worker\-unregister\/\fR -unregister itself as worker -.TP \fBahriman\fR \fI\,user\-add\/\fR create or update user .TP @@ -875,16 +869,6 @@ usage: ahriman service\-tree\-migrate [\-h] migrate repository tree between versions -.SH COMMAND \fI\,'ahriman service\-worker\-register'\/\fR -usage: ahriman service\-worker\-register [\-h] - -call remote service registering itself as available worker - -.SH COMMAND \fI\,'ahriman service\-worker\-unregister'\/\fR -usage: ahriman service\-worker\-unregister [\-h] - -call remote service removing itself from list of available workers - .SH COMMAND \fI\,'ahriman user\-add'\/\fR usage: ahriman user\-add [\-h] [\-\-key KEY] [\-\-packager PACKAGER] [\-p PASSWORD] [\-R {unauthorized,read,reporter,full}] username diff --git a/package/share/zsh/site-functions/_ahriman b/package/share/zsh/site-functions/_ahriman index 6aba1c0c..5ebe47c9 100644 --- a/package/share/zsh/site-functions/_ahriman +++ b/package/share/zsh/site-functions/_ahriman @@ -65,8 +65,6 @@ _shtab_ahriman_commands() { "service-setup:create initial service configuration, requires root" "service-shell:drop into python shell" "service-tree-migrate:migrate repository tree between versions" - "service-worker-register:call remote service registering itself as available worker" - "service-worker-unregister:call remote service removing itself from list of available workers" "setup:create initial service configuration, requires root" "shell:drop into python shell" "sign:(re-)sign packages and repository database according to current settings" @@ -554,14 +552,6 @@ _shtab_ahriman_service_tree_migrate_options=( "(- : *)"{-h,--help}"[show this help message and exit]" ) -_shtab_ahriman_service_worker_register_options=( - "(- : *)"{-h,--help}"[show this help message and exit]" -) - -_shtab_ahriman_service_worker_unregister_options=( - "(- : *)"{-h,--help}"[show this help message and exit]" -) - _shtab_ahriman_setup_options=( "(- : *)"{-h,--help}"[show this help message and exit]" "--build-as-user[force makepkg user to the specific one (default\: None)]:build_as_user:" @@ -727,8 +717,6 @@ _shtab_ahriman() { service-setup) _arguments -C -s $_shtab_ahriman_service_setup_options ;; service-shell) _arguments -C -s $_shtab_ahriman_service_shell_options ;; service-tree-migrate) _arguments -C -s $_shtab_ahriman_service_tree_migrate_options ;; - service-worker-register) _arguments -C -s $_shtab_ahriman_service_worker_register_options ;; - service-worker-unregister) _arguments -C -s $_shtab_ahriman_service_worker_unregister_options ;; setup) _arguments -C -s $_shtab_ahriman_setup_options ;; shell) _arguments -C -s $_shtab_ahriman_shell_options ;; sign) _arguments -C -s $_shtab_ahriman_sign_options ;; diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 5bddaf9e..24ace99d 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -135,8 +135,6 @@ def _parser() -> argparse.ArgumentParser: _set_service_setup_parser(subparsers) _set_service_shell_parser(subparsers) _set_service_tree_migrate_parser(subparsers) - _set_service_worker_register_parser(subparsers) - _set_service_worker_unregister_parser(subparsers) _set_user_add_parser(subparsers) _set_user_list_parser(subparsers) _set_user_remove_parser(subparsers) @@ -1054,40 +1052,6 @@ def _set_service_tree_migrate_parser(root: SubParserAction) -> argparse.Argument return parser -def _set_service_worker_register_parser(root: SubParserAction) -> argparse.ArgumentParser: - """ - add parser for remote worker registration subcommand - - Args: - root(SubParserAction): subparsers for the commands - - Returns: - argparse.ArgumentParser: created argument parser - """ - parser = root.add_parser("service-worker-register", help="register itself as worker", - description="call remote service registering itself as available worker", - formatter_class=_formatter) - parser.set_defaults(handler=handlers.Triggers, trigger=["ahriman.core.distributed.WorkerRegisterTrigger"]) - return parser - - -def _set_service_worker_unregister_parser(root: SubParserAction) -> argparse.ArgumentParser: - """ - add parser for remote worker removal subcommand - - Args: - root(SubParserAction): subparsers for the commands - - Returns: - argparse.ArgumentParser: created argument parser - """ - parser = root.add_parser("service-worker-unregister", help="unregister itself as worker", - description="call remote service removing itself from list of available workers", - formatter_class=_formatter) - parser.set_defaults(handler=handlers.Triggers, trigger=["ahriman.core.distributed.WorkerUnregisterTrigger"]) - return parser - - def _set_user_add_parser(root: SubParserAction) -> argparse.ArgumentParser: """ add parser for create user subcommand diff --git a/src/ahriman/core/database/migrations/m013_workers.py b/src/ahriman/core/database/migrations/m013_workers.py deleted file mode 100644 index a5a1098c..00000000 --- a/src/ahriman/core/database/migrations/m013_workers.py +++ /dev/null @@ -1,31 +0,0 @@ -# -# Copyright (c) 2021-2023 ahriman team. -# -# This file is part of ahriman -# (see https://github.com/arcan1s/ahriman). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -__all__ = ["steps"] - - -steps = [ - """ - create table workers ( - identifier text not null, - address text not null, - unique (identifier) - ) - """ -] diff --git a/src/ahriman/core/database/operations/__init__.py b/src/ahriman/core/database/operations/__init__.py index 0fa35c84..13ca0404 100644 --- a/src/ahriman/core/database/operations/__init__.py +++ b/src/ahriman/core/database/operations/__init__.py @@ -25,4 +25,3 @@ from ahriman.core.database.operations.changes_operations import ChangesOperation from ahriman.core.database.operations.logs_operations import LogsOperations from ahriman.core.database.operations.package_operations import PackageOperations from ahriman.core.database.operations.patch_operations import PatchOperations -from ahriman.core.database.operations.workers_operations import WorkersOperations diff --git a/src/ahriman/core/database/operations/workers_operations.py b/src/ahriman/core/database/operations/workers_operations.py deleted file mode 100644 index 8703515f..00000000 --- a/src/ahriman/core/database/operations/workers_operations.py +++ /dev/null @@ -1,83 +0,0 @@ -# -# Copyright (c) 2021-2023 ahriman team. -# -# This file is part of ahriman -# (see https://github.com/arcan1s/ahriman). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -from sqlite3 import Connection - -from ahriman.core.database.operations import Operations -from ahriman.models.worker import Worker - - -class WorkersOperations(Operations): - """ - operations for remote workers - """ - - def workers_get(self) -> list[Worker]: - """ - retrieve registered workers - - Returns: - list[Worker]: list of available workers - """ - def run(connection: Connection) -> list[Worker]: - return [ - Worker(row["address"], identifier=row["identifier"]) - for row in connection.execute("""select * from workers""") - ] - - return self.with_connection(run) - - def workers_insert(self, worker: Worker) -> None: - """ - insert or update worker in database - - Args: - worker(Worker): remote worker descriptor - """ - def run(connection: Connection) -> None: - connection.execute( - """ - insert into workers - (identifier, address) - values - (:identifier, :address) - on conflict (identifier) do update set - address = :address - """, - worker.view() - ) - - return self.with_connection(run, commit=True) - - def workers_remove(self, identifier: str | None = None) -> None: - """ - unregister remote worker - - Args: - identifier(str | None, optional): remote worker identifier. If none set it will clear all workers - (Default value = None) - """ - def run(connection: Connection) -> None: - connection.execute( - """ - delete from workers where (:identifier is null or identifier = :identifier) - """, - {"identifier": identifier}) - - return self.with_connection(run, commit=True) diff --git a/src/ahriman/core/database/sqlite.py b/src/ahriman/core/database/sqlite.py index 2a7c992f..663729ec 100644 --- a/src/ahriman/core/database/sqlite.py +++ b/src/ahriman/core/database/sqlite.py @@ -26,12 +26,11 @@ from typing import Self from ahriman.core.configuration import Configuration from ahriman.core.database.migrations import Migrations from ahriman.core.database.operations import AuthOperations, BuildOperations, ChangesOperations, LogsOperations, \ - PackageOperations, PatchOperations, WorkersOperations + PackageOperations, PatchOperations # pylint: disable=too-many-ancestors -class SQLite(AuthOperations, BuildOperations, ChangesOperations, LogsOperations, PackageOperations, PatchOperations, - WorkersOperations): +class SQLite(AuthOperations, BuildOperations, ChangesOperations, LogsOperations, PackageOperations, PatchOperations): """ wrapper for sqlite3 database diff --git a/src/ahriman/core/distributed/__init__.py b/src/ahriman/core/distributed/__init__.py index 15de7396..15a7251e 100644 --- a/src/ahriman/core/distributed/__init__.py +++ b/src/ahriman/core/distributed/__init__.py @@ -18,6 +18,5 @@ # along with this program. If not, see . # from ahriman.core.distributed.worker_loader_trigger import WorkerLoaderTrigger -from ahriman.core.distributed.worker_register_trigger import WorkerRegisterTrigger from ahriman.core.distributed.worker_trigger import WorkerTrigger -from ahriman.core.distributed.worker_unregister_trigger import WorkerUnregisterTrigger +from ahriman.core.distributed.workers_cache import WorkersCache diff --git a/src/ahriman/core/distributed/distributed_system.py b/src/ahriman/core/distributed/distributed_system.py index b5bf9b50..a85a09ff 100644 --- a/src/ahriman/core/distributed/distributed_system.py +++ b/src/ahriman/core/distributed/distributed_system.py @@ -18,10 +18,7 @@ # along with this program. If not, see . # import contextlib -import tempfile -import uuid -from pathlib import Path from functools import cached_property from ahriman.core.configuration import Configuration @@ -35,9 +32,6 @@ from ahriman.models.worker import Worker class DistributedSystem(Trigger, WebClient): """ simple class to (un)register itself as a distributed worker - - Attributes: - identifier_path(Path): path to cached worker identifier """ CONFIGURATION_SCHEMA: ConfigurationSchema = { @@ -54,9 +48,10 @@ class DistributedSystem(Trigger, WebClient): "type": "string", "empty": False, }, - "identifier_path": { - "type": "path", - "coerce": "absolute_path", + "time_to_live": { + "type": "integer", + "coerce": "integer", + "min": 0, }, }, }, @@ -73,11 +68,6 @@ class DistributedSystem(Trigger, WebClient): Trigger.__init__(self, repository_id, configuration) WebClient.__init__(self, repository_id, configuration) - section = next(iter(self.configuration_sections(configuration))) - self.identifier_path = configuration.getpath( - section, "identifier_path", fallback=Path(tempfile.gettempdir()) / "ahriman-worker-identifier") - self._owe_identifier = False - @cached_property def worker(self) -> Worker: """ @@ -87,8 +77,10 @@ class DistributedSystem(Trigger, WebClient): Worker: unique self worker identifier """ section = next(iter(self.configuration_sections(self.configuration))) - identifier = self.load_identifier(self.configuration, section) - return Worker(self.configuration.get(section, "address"), identifier=identifier) + + address = self.configuration.get(section, "address") + identifier = self.configuration.get(section, "identifier", fallback="") + return Worker(address, identifier=identifier) @classmethod def configuration_sections(cls, configuration: Configuration) -> list[str]: @@ -103,66 +95,21 @@ class DistributedSystem(Trigger, WebClient): """ return list(cls.CONFIGURATION_SCHEMA.keys()) - def _workers_url(self, identifier: str = "") -> str: + def _workers_url(self) -> str: """ workers url generator - Args: - identifier(str, optional): worker identifier (Default value = "") - Returns: - str: full url of web service for specific worker + str: full url of web service for workers """ - suffix = f"/{identifier}" if identifier else "" - return f"{self.address}/api/v1/distributed{suffix}" + return f"{self.address}/api/v1/distributed" - def load_identifier(self, configuration: Configuration, section: str) -> str: - """ - load identifier from filesystem if available or from configuration otherwise. If cache file is available, - the method will read from it. Otherwise, it will try to read it from configuration. And, finally, if no - identifier set, it will generate uuid - - Args: - configuration(Configuration): configuration instance - section(str): settings section name - - Returns: - str: unique worker identifier - """ - if self.identifier_path.is_file(): # load cached value - return self.identifier_path.read_text(encoding="utf8") - return configuration.get(section, "identifier", fallback=str(uuid.uuid4())) - - def register(self, force: bool = False) -> None: + def register(self) -> None: """ register itself in remote system - - Args: - force(bool, optional): register worker even if it has been already registered before (Default value = False) """ - if self.identifier_path.is_file() and not force: - return # there is already registered identifier - - self.make_request("POST", self._workers_url(), json=self.worker.view()) - # save identifier - self.identifier_path.write_text(self.worker.identifier, encoding="utf8") - self._owe_identifier = True - self.logger.info("registered instance %s at %s", self.worker, self.address) - - def unregister(self, force: bool = False) -> None: - """ - unregister itself in remote system - - Args: - force(bool, optional): unregister worker even if it has been registered in another process - (Default value = False) - """ - if not self._owe_identifier and not force: - return # we do not owe this identifier - - self.make_request("DELETE", self._workers_url(self.worker.identifier)) - self.identifier_path.unlink(missing_ok=True) - self.logger.info("unregistered instance %s at %s", self.worker, self.address) + with contextlib.suppress(Exception): + self.make_request("POST", self._workers_url(), json=self.worker.view()) def workers(self) -> list[Worker]: """ diff --git a/src/ahriman/core/distributed/worker_register_trigger.py b/src/ahriman/core/distributed/worker_register_trigger.py deleted file mode 100644 index 03d94b37..00000000 --- a/src/ahriman/core/distributed/worker_register_trigger.py +++ /dev/null @@ -1,32 +0,0 @@ -# -# Copyright (c) 2021-2023 ahriman team. -# -# This file is part of ahriman -# (see https://github.com/arcan1s/ahriman). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -from ahriman.core.distributed.distributed_system import DistributedSystem - - -class WorkerRegisterTrigger(DistributedSystem): - """ - remote worker registration trigger - """ - - def on_start(self) -> None: - """ - trigger action which will be called at the start of the application - """ - self.register(force=True) diff --git a/src/ahriman/core/distributed/worker_trigger.py b/src/ahriman/core/distributed/worker_trigger.py index 87905b5c..8f0898e2 100644 --- a/src/ahriman/core/distributed/worker_trigger.py +++ b/src/ahriman/core/distributed/worker_trigger.py @@ -17,22 +17,54 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +from threading import Timer + +from ahriman.core.configuration import Configuration from ahriman.core.distributed.distributed_system import DistributedSystem +from ahriman.models.repository_id import RepositoryId class WorkerTrigger(DistributedSystem): """ remote worker processor trigger (client side) + + Attributes: + ping_interval(float): interval to call remote service in seconds, defined as ``worker.time_to_live / 4`` + timer(Timer): timer object """ + def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: + """ + default constructor + + Args: + repository_id(RepositoryId): repository unique identifier + configuration(Configuration): configuration instance + """ + DistributedSystem.__init__(self, repository_id, configuration) + + section = next(iter(self.configuration_sections(configuration))) + self.ping_interval = configuration.getint(section, "time_to_live", fallback=60) / 4.0 + self.timer = Timer(self.ping_interval, self.ping) + def on_start(self) -> None: """ trigger action which will be called at the start of the application """ - self.register() + self.logger.info("registering instance %s at %s", self.worker, self.address) + self.timer.start() def on_stop(self) -> None: """ trigger action which will be called before the stop of the application """ - self.unregister() + self.logger.info("removing instance %s at %s", self.worker, self.address) + self.timer.cancel() + + def ping(self) -> None: + """ + register itself as alive worker and update the timer + """ + self.register() + self.timer = Timer(self.ping_interval, self.ping) + self.timer.start() diff --git a/src/ahriman/core/distributed/worker_unregister_trigger.py b/src/ahriman/core/distributed/worker_unregister_trigger.py deleted file mode 100644 index 33cac3fc..00000000 --- a/src/ahriman/core/distributed/worker_unregister_trigger.py +++ /dev/null @@ -1,32 +0,0 @@ -# -# Copyright (c) 2021-2023 ahriman team. -# -# This file is part of ahriman -# (see https://github.com/arcan1s/ahriman). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -from ahriman.core.distributed.distributed_system import DistributedSystem - - -class WorkerUnregisterTrigger(DistributedSystem): - """ - remote worker registration trigger - """ - - def on_start(self) -> None: - """ - trigger action which will be called at the start of the application - """ - self.unregister(force=True) diff --git a/src/ahriman/core/distributed/workers_cache.py b/src/ahriman/core/distributed/workers_cache.py new file mode 100644 index 00000000..7a981fea --- /dev/null +++ b/src/ahriman/core/distributed/workers_cache.py @@ -0,0 +1,73 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import time + +from ahriman.core.configuration import Configuration +from ahriman.core.log import LazyLogging +from ahriman.models.worker import Worker + + +class WorkersCache(LazyLogging): + """ + cached storage for healthy workers + + Attributes: + time_to_live(int): maximal amount of time in seconds to keep worker alive + """ + + def __init__(self, configuration: Configuration) -> None: + """ + default constructor + + Args: + configuration(Configuration): configuration instance + """ + self.time_to_live = configuration.getint("worker", "time_to_live", fallback=60) + self._workers: dict[str, tuple[Worker, float]] = {} + + @property + def workers(self) -> list[Worker]: + """ + extract currently healthy workers + + Returns: + list[Worker]: list of currently registered workers which have been seen not earlier than :attr:`time_to_live` + """ + valid_from = time.monotonic() - self.time_to_live + return [ + worker + for worker, last_seen in self._workers.values() + if last_seen > valid_from + ] + + def workers_remove(self) -> None: + """ + remove all workers from the cache + """ + self._workers = {} + + def workers_update(self, worker: Worker) -> None: + """ + register or update remote worker + + Args: + worker(Worker): worker to register + """ + self._workers[worker.identifier] = (worker, time.monotonic()) diff --git a/src/ahriman/core/log/filtered_access_logger.py b/src/ahriman/core/log/filtered_access_logger.py index b0fdd3bd..57c76111 100644 --- a/src/ahriman/core/log/filtered_access_logger.py +++ b/src/ahriman/core/log/filtered_access_logger.py @@ -27,15 +27,31 @@ class FilteredAccessLogger(AccessLogger): access logger implementation with log filter enabled Attributes: + DISTRIBUTED_PATH_REGEX(str): (class attribute) regex used for distributed system uri + HEALTH_PATH_REGEX(re.Pattern): (class attribute) regex for health check endpoint LOG_PATH_REGEX(re.Pattern): (class attribute) regex for logs uri - LOG_PATPROCESS_PATH_REGEXH_REGEX(re.Pattern): (class attribute) regex for process uri + PROCESS_PATH_REGEX(re.Pattern): (class attribute) regex for process uri """ + DISTRIBUTED_PATH_REGEX = "/api/v1/distributed" HEALTH_PATH_REGEX = "/api/v1/info" LOG_PATH_REGEX = re.compile(r"^/api/v1/packages/[^/]+/logs$") # technically process id is uuid, but we might change it later PROCESS_PATH_REGEX = re.compile(r"^/api/v1/service/process/[^/]+$") + @staticmethod + def is_distributed_post(request: BaseRequest) -> bool: + """ + check if the request is for distributed services ping + + Args: + request(BaseRequest): http reqeust descriptor + + Returns: + bool: True in case if request is distributed service ping endpoint and False otherwise + """ + return request.method == "POST" and FilteredAccessLogger.DISTRIBUTED_PATH_REGEX == request.path + @staticmethod def is_info_get(request: BaseRequest) -> bool: """ @@ -84,7 +100,8 @@ class FilteredAccessLogger(AccessLogger): response(StreamResponse): streaming response object time(float): log record timestamp """ - if self.is_info_get(request) \ + if self.is_distributed_post(request) \ + or self.is_info_get(request) \ or self.is_logs_post(request) \ or self.is_process_get(request): return diff --git a/src/ahriman/core/status/watcher.py b/src/ahriman/core/status/watcher.py index 81bc5903..aada9e29 100644 --- a/src/ahriman/core/status/watcher.py +++ b/src/ahriman/core/status/watcher.py @@ -26,7 +26,6 @@ from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.repository_id import RepositoryId -from ahriman.models.worker import Worker class Watcher(LazyLogging): @@ -224,30 +223,3 @@ class Watcher(LazyLogging): status(BuildStatusEnum): new service status """ self.status = BuildStatus(status) - - def workers_get(self) -> list[Worker]: - """ - retrieve registered remote workers - - Returns: - list[Worker]: list of currently available workers - """ - return self.database.workers_get() - - def workers_remove(self, identifier: str | None = None) -> None: - """ - unregister remote worker - - Args: - identifier(str | None, optional): remote worker identifier if any (Default value = None) - """ - self.database.workers_remove(identifier) - - def workers_update(self, worker: Worker) -> None: - """ - register or update remote worker - - Args: - worker(Worker): worker to register - """ - self.database.workers_insert(worker) diff --git a/src/ahriman/web/keys.py b/src/ahriman/web/keys.py index 04e5a323..06402568 100644 --- a/src/ahriman/web/keys.py +++ b/src/ahriman/web/keys.py @@ -21,6 +21,7 @@ from aiohttp.web import AppKey from ahriman.core.auth import Auth from ahriman.core.configuration import Configuration +from ahriman.core.distributed import WorkersCache from ahriman.core.spawn import Spawn from ahriman.core.status.watcher import Watcher from ahriman.models.repository_id import RepositoryId @@ -31,6 +32,7 @@ __all__ = [ "ConfigurationKey", "SpawnKey", "WatcherKey", + "WorkersKey", ] @@ -38,3 +40,4 @@ AuthKey = AppKey("validator", Auth) ConfigurationKey = AppKey("configuration", Configuration) SpawnKey = AppKey("spawn", Spawn) WatcherKey = AppKey("watcher", dict[RepositoryId, Watcher]) +WorkersKey = AppKey("workers", WorkersCache) diff --git a/src/ahriman/web/schemas/__init__.py b/src/ahriman/web/schemas/__init__.py index de61f3cb..ebae2233 100644 --- a/src/ahriman/web/schemas/__init__.py +++ b/src/ahriman/web/schemas/__init__.py @@ -49,5 +49,4 @@ from ahriman.web.schemas.search_schema import SearchSchema from ahriman.web.schemas.status_schema import StatusSchema from ahriman.web.schemas.update_flags_schema import UpdateFlagsSchema from ahriman.web.schemas.versioned_log_schema import VersionedLogSchema -from ahriman.web.schemas.worker_id_schema import WorkerIdSchema from ahriman.web.schemas.worker_schema import WorkerSchema diff --git a/src/ahriman/web/schemas/worker_id_schema.py b/src/ahriman/web/schemas/worker_id_schema.py deleted file mode 100644 index fb14b7c7..00000000 --- a/src/ahriman/web/schemas/worker_id_schema.py +++ /dev/null @@ -1,31 +0,0 @@ -# -# Copyright (c) 2021-2023 ahriman team. -# -# This file is part of ahriman -# (see https://github.com/arcan1s/ahriman). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -from marshmallow import Schema, fields - - -class WorkerIdSchema(Schema): - """ - request and response schema for workers - """ - - identifier = fields.String(required=True, metadata={ - "description": "Worker unique identifier", - "example": "42f03a62-48f7-46b7-af40-dacc720e92fa", - }) diff --git a/src/ahriman/web/schemas/worker_schema.py b/src/ahriman/web/schemas/worker_schema.py index 1051f99d..3f4c31ac 100644 --- a/src/ahriman/web/schemas/worker_schema.py +++ b/src/ahriman/web/schemas/worker_schema.py @@ -17,12 +17,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from marshmallow import fields - -from ahriman.web.schemas.worker_id_schema import WorkerIdSchema +from marshmallow import Schema, fields -class WorkerSchema(WorkerIdSchema): +class WorkerSchema(Schema): """ request and response schema for workers """ @@ -31,3 +29,7 @@ class WorkerSchema(WorkerIdSchema): "description": "Worker address", "example": "http://localhost:8081", }) + identifier = fields.String(required=True, metadata={ + "description": "Worker unique identifier", + "example": "42f03a62-48f7-46b7-af40-dacc720e92fa", + }) diff --git a/src/ahriman/web/views/base.py b/src/ahriman/web/views/base.py index 87905731..72b733a9 100644 --- a/src/ahriman/web/views/base.py +++ b/src/ahriman/web/views/base.py @@ -24,12 +24,13 @@ from typing import TypeVar from ahriman.core.auth import Auth from ahriman.core.configuration import Configuration +from ahriman.core.distributed import WorkersCache from ahriman.core.sign.gpg import GPG from ahriman.core.spawn import Spawn from ahriman.core.status.watcher import Watcher from ahriman.models.repository_id import RepositoryId from ahriman.models.user_access import UserAccess -from ahriman.web.keys import AuthKey, ConfigurationKey, SpawnKey, WatcherKey +from ahriman.web.keys import AuthKey, ConfigurationKey, SpawnKey, WatcherKey, WorkersKey T = TypeVar("T", str, list[str]) @@ -97,6 +98,16 @@ class BaseView(View, CorsViewMixin): """ return self.request.app[AuthKey] + @property + def workers(self) -> WorkersCache: + """ + get workers cache instance + + Returns: + WorkersCache: workers service + """ + return self.request.app[WorkersKey] + @classmethod async def get_permission(cls, request: Request) -> UserAccess: """ diff --git a/src/ahriman/web/views/v1/distributed/worker.py b/src/ahriman/web/views/v1/distributed/worker.py deleted file mode 100644 index fa3a23ff..00000000 --- a/src/ahriman/web/views/v1/distributed/worker.py +++ /dev/null @@ -1,99 +0,0 @@ -# -# Copyright (c) 2021-2023 ahriman team. -# -# This file is part of ahriman -# (see https://github.com/arcan1s/ahriman). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -import aiohttp_apispec # type: ignore[import-untyped] - -from aiohttp.web import HTTPNoContent, HTTPNotFound, Response, json_response - -from ahriman.models.user_access import UserAccess -from ahriman.web.schemas import AuthSchema, ErrorSchema, WorkerIdSchema, WorkerSchema -from ahriman.web.views.base import BaseView - - -class WorkerView(BaseView): - """ - distributed worker view - - Attributes: - DELETE_PERMISSION(UserAccess): (class attribute) delete permissions of self - GET_PERMISSION(UserAccess): (class attribute) get permissions of self - """ - - DELETE_PERMISSION = GET_PERMISSION = UserAccess.Full - ROUTES = ["/api/v1/distributed/{identifier}"] - - @aiohttp_apispec.docs( - tags=["Distributed"], - summary="Unregister worker", - description="Unregister worker and remove it from the service", - responses={ - 204: {"description": "Success response"}, - 401: {"description": "Authorization required", "schema": ErrorSchema}, - 403: {"description": "Access is forbidden", "schema": ErrorSchema}, - 500: {"description": "Internal server error", "schema": ErrorSchema}, - }, - security=[{"token": [DELETE_PERMISSION]}], - ) - @aiohttp_apispec.cookies_schema(AuthSchema) - @aiohttp_apispec.match_info_schema(WorkerIdSchema) - async def delete(self) -> None: - """ - unregister worker - - Raises: - HTTPNoContent: on success response - """ - identifier = self.request.match_info["identifier"] - self.service().workers_remove(identifier) - - raise HTTPNoContent - - @aiohttp_apispec.docs( - tags=["Distributed"], - summary="Get worker", - description="Retrieve registered worker by its identifier", - responses={ - 200: {"description": "Success response", "schema": WorkerSchema(many=True)}, - 401: {"description": "Authorization required", "schema": ErrorSchema}, - 403: {"description": "Access is forbidden", "schema": ErrorSchema}, - 404: {"description": "Worker is unknown", "schema": ErrorSchema}, - 500: {"description": "Internal server error", "schema": ErrorSchema}, - }, - security=[{"token": [GET_PERMISSION]}], - ) - @aiohttp_apispec.cookies_schema(AuthSchema) - @aiohttp_apispec.match_info_schema(WorkerIdSchema) - async def get(self) -> Response: - """ - get worker by identifier - - Returns: - Response: 200 with workers list on success - - Raises: - HTTPNotFound: if no worker was found - """ - identifier = self.request.match_info["identifier"] - - try: - worker = next(worker for worker in self.service().workers_get() if worker.identifier == identifier) - except StopIteration: - raise HTTPNotFound(reason=f"Worker {identifier} not found") - - return json_response([worker.view()]) diff --git a/src/ahriman/web/views/v1/distributed/workers.py b/src/ahriman/web/views/v1/distributed/workers.py index fe5b5f0a..a5171db2 100644 --- a/src/ahriman/web/views/v1/distributed/workers.py +++ b/src/ahriman/web/views/v1/distributed/workers.py @@ -61,7 +61,7 @@ class WorkersView(BaseView): Raises: HTTPNoContent: on success response """ - self.service().workers_remove() + self.workers.workers_remove() raise HTTPNoContent @@ -85,7 +85,7 @@ class WorkersView(BaseView): Returns: Response: 200 with workers list on success """ - workers = self.service().workers_get() + workers = self.workers.workers comparator: Callable[[Worker], str] = lambda item: item.identifier response = [worker.view() for worker in sorted(workers, key=comparator)] @@ -121,6 +121,6 @@ class WorkersView(BaseView): except Exception as ex: raise HTTPBadRequest(reason=str(ex)) - self.service().workers_update(worker) + self.workers.workers_update(worker) raise HTTPNoContent diff --git a/src/ahriman/web/web.py b/src/ahriman/web/web.py index 64a48f44..c4809730 100644 --- a/src/ahriman/web/web.py +++ b/src/ahriman/web/web.py @@ -27,6 +27,7 @@ from aiohttp.web import Application, normalize_path_middleware, run_app from ahriman.core.auth import Auth from ahriman.core.configuration import Configuration from ahriman.core.database import SQLite +from ahriman.core.distributed import WorkersCache from ahriman.core.exceptions import InitializeError from ahriman.core.log.filtered_access_logger import FilteredAccessLogger from ahriman.core.spawn import Spawn @@ -34,7 +35,7 @@ from ahriman.core.status.watcher import Watcher from ahriman.models.repository_id import RepositoryId from ahriman.web.apispec import setup_apispec from ahriman.web.cors import setup_cors -from ahriman.web.keys import AuthKey, ConfigurationKey, SpawnKey, WatcherKey +from ahriman.web.keys import AuthKey, ConfigurationKey, SpawnKey, WatcherKey, WorkersKey from ahriman.web.middlewares.exception_handler import exception_handler from ahriman.web.routes import setup_routes @@ -159,7 +160,8 @@ def setup_server(configuration: Configuration, spawner: Spawn, repositories: lis application.logger.info("setup configuration") application[ConfigurationKey] = configuration - application.logger.info("setup watchers") + application.logger.info("setup services") + # package cache if not repositories: raise InitializeError("No repositories configured, exiting") database = SQLite.load(configuration) @@ -168,8 +170,9 @@ def setup_server(configuration: Configuration, spawner: Spawn, repositories: lis application.logger.info("load repository %s", repository_id) watchers[repository_id] = Watcher(repository_id, database) application[WatcherKey] = watchers - - application.logger.info("setup process spawner") + # workers cache + application[WorkersKey] = WorkersCache(configuration) + # process spawner application[SpawnKey] = spawner application.logger.info("setup authorization") diff --git a/tests/ahriman/application/test_ahriman.py b/tests/ahriman/application/test_ahriman.py index 428137be..d4121150 100644 --- a/tests/ahriman/application/test_ahriman.py +++ b/tests/ahriman/application/test_ahriman.py @@ -1320,80 +1320,6 @@ def test_subparsers_service_tree_migrate(parser: argparse.ArgumentParser) -> Non assert not args.report -def test_subparsers_service_worker_register(parser: argparse.ArgumentParser) -> None: - """ - service-worker-register command must imply trigger - """ - args = parser.parse_args(["service-worker-register"]) - assert args.trigger == ["ahriman.core.distributed.WorkerRegisterTrigger"] - - -def test_subparsers_service_worker_register_option_architecture(parser: argparse.ArgumentParser) -> None: - """ - service-worker-register command must correctly parse architecture list - """ - args = parser.parse_args(["service-worker-register"]) - assert args.architecture is None - args = parser.parse_args(["-a", "x86_64", "service-worker-register"]) - assert args.architecture == "x86_64" - - -def test_subparsers_service_worker_register_option_repository(parser: argparse.ArgumentParser) -> None: - """ - service-worker-register command must correctly parse repository list - """ - args = parser.parse_args(["service-worker-register"]) - assert args.repository is None - args = parser.parse_args(["-r", "repo", "service-worker-register"]) - assert args.repository == "repo" - - -def test_subparsers_service_worker_register_repo_triggers(parser: argparse.ArgumentParser) -> None: - """ - service-worker-register must have same keys as repo-triggers - """ - args = parser.parse_args(["service-worker-register"]) - reference_args = parser.parse_args(["repo-triggers"]) - assert dir(args) == dir(reference_args) - - -def test_subparsers_service_worker_unregister(parser: argparse.ArgumentParser) -> None: - """ - service-worker-unregister command must imply trigger - """ - args = parser.parse_args(["service-worker-unregister"]) - assert args.trigger == ["ahriman.core.distributed.WorkerUnregisterTrigger"] - - -def test_subparsers_service_worker_unregister_option_architecture(parser: argparse.ArgumentParser) -> None: - """ - service-worker-unregister command must correctly parse architecture list - """ - args = parser.parse_args(["service-worker-unregister"]) - assert args.architecture is None - args = parser.parse_args(["-a", "x86_64", "service-worker-unregister"]) - assert args.architecture == "x86_64" - - -def test_subparsers_service_worker_unregister_option_repository(parser: argparse.ArgumentParser) -> None: - """ - service-worker-unregister command must correctly parse repository list - """ - args = parser.parse_args(["service-worker-unregister"]) - assert args.repository is None - args = parser.parse_args(["-r", "repo", "service-worker-unregister"]) - assert args.repository == "repo" - - -def test_subparsers_service_worker_unregister_repo_triggers(parser: argparse.ArgumentParser) -> None: - """ - service-worker-unregister must have same keys as repo-triggers - """ - args = parser.parse_args(["service-worker-unregister"]) - reference_args = parser.parse_args(["repo-triggers"]) - assert dir(args) == dir(reference_args) - - def test_subparsers_user_add(parser: argparse.ArgumentParser) -> None: """ user-add command must imply action, architecture, exit code, lock, quiet, report and repository diff --git a/tests/ahriman/core/database/migrations/test_m013_workers.py b/tests/ahriman/core/database/migrations/test_m013_workers.py deleted file mode 100644 index ed1a3ab6..00000000 --- a/tests/ahriman/core/database/migrations/test_m013_workers.py +++ /dev/null @@ -1,8 +0,0 @@ -from ahriman.core.database.migrations.m013_workers import steps - - -def test_migration_workers() -> None: - """ - migration must not be empty - """ - assert steps diff --git a/tests/ahriman/core/database/operations/test_workers_operations.py b/tests/ahriman/core/database/operations/test_workers_operations.py deleted file mode 100644 index 0835a5c9..00000000 --- a/tests/ahriman/core/database/operations/test_workers_operations.py +++ /dev/null @@ -1,46 +0,0 @@ -from ahriman.core.database import SQLite -from ahriman.models.worker import Worker - - -def test_workers_get_insert(database: SQLite) -> None: - """ - must insert workers to database - """ - database.workers_insert(Worker("address1", identifier="1")) - database.workers_insert(Worker("address2", identifier="2")) - assert database.workers_get() == [ - Worker("address1", identifier="1"), Worker("address2", identifier="2") - ] - - -def test_workers_insert_remove(database: SQLite) -> None: - """ - must remove worker from database - """ - database.workers_insert(Worker("address1", identifier="1")) - database.workers_insert(Worker("address2", identifier="2")) - database.workers_remove("1") - - assert database.workers_get() == [Worker("address2", identifier="2")] - - -def test_workers_insert_remove_all(database: SQLite) -> None: - """ - must remove all workers - """ - database.workers_insert(Worker("address1", identifier="1")) - database.workers_insert(Worker("address2", identifier="2")) - database.workers_remove() - - assert database.workers_get() == [] - - -def test_workers_insert_insert(database: SQLite) -> None: - """ - must update worker in database - """ - database.workers_insert(Worker("address1", identifier="1")) - assert database.workers_get() == [Worker("address1", identifier="1")] - - database.workers_insert(Worker("address2", identifier="1")) - assert database.workers_get() == [Worker("address2", identifier="1")] diff --git a/tests/ahriman/core/distributed/conftest.py b/tests/ahriman/core/distributed/conftest.py index 9f7ddb87..d40a7476 100644 --- a/tests/ahriman/core/distributed/conftest.py +++ b/tests/ahriman/core/distributed/conftest.py @@ -1,6 +1,7 @@ import pytest from ahriman.core.configuration import Configuration +from ahriman.core.distributed import WorkersCache from ahriman.core.distributed.distributed_system import DistributedSystem @@ -18,3 +19,17 @@ def distributed_system(configuration: Configuration) -> DistributedSystem: configuration.set_option("status", "address", "http://localhost:8081") _, repository_id = configuration.check_loaded() return DistributedSystem(repository_id, configuration) + + +@pytest.fixture +def workers_cache(configuration: Configuration) -> WorkersCache: + """ + workers cache fixture + + Args: + configuration(Configuration): configuration fixture + + Returns: + WorkersCache: workers cache test instance + """ + return WorkersCache(configuration) diff --git a/tests/ahriman/core/distributed/test_distributed_system.py b/tests/ahriman/core/distributed/test_distributed_system.py index ec27d925..8965b0f5 100644 --- a/tests/ahriman/core/distributed/test_distributed_system.py +++ b/tests/ahriman/core/distributed/test_distributed_system.py @@ -8,15 +8,6 @@ from ahriman.core.distributed.distributed_system import DistributedSystem from ahriman.models.worker import Worker -def test_identifier_path(configuration: Configuration) -> None: - """ - must correctly set default identifier path - """ - configuration.set_option("status", "address", "http://localhost:8081") - _, repository_id = configuration.check_loaded() - assert DistributedSystem(repository_id, configuration).identifier_path - - def test_configuration_sections(configuration: Configuration) -> None: """ must correctly parse target list @@ -31,138 +22,31 @@ def test_workers_url(distributed_system: DistributedSystem) -> None: assert distributed_system._workers_url().startswith(distributed_system.address) assert distributed_system._workers_url().endswith("/api/v1/distributed") - assert distributed_system._workers_url("id").startswith(distributed_system.address) - assert distributed_system._workers_url("id").endswith("/api/v1/distributed/id") - - -def test_load_identifier(configuration: Configuration, mocker: MockerFixture) -> None: - """ - must generate identifier - """ - mocker.patch("pathlib.Path.is_file", return_value=False) - configuration.set_option("status", "address", "http://localhost:8081") - _, repository_id = configuration.check_loaded() - system = DistributedSystem(repository_id, configuration) - - assert system.load_identifier(configuration, "worker") - - -def test_load_identifier_configuration(configuration: Configuration, mocker: MockerFixture) -> None: - """ - must load identifier from configuration - """ - identifier = "id" - mocker.patch("pathlib.Path.is_file", return_value=False) - configuration.set_option("worker", "identifier", identifier) - configuration.set_option("status", "address", "http://localhost:8081") - _, repository_id = configuration.check_loaded() - system = DistributedSystem(repository_id, configuration) - - assert system.worker.identifier == identifier - - -def test_load_identifier_filesystem(configuration: Configuration, mocker: MockerFixture) -> None: - """ - must load identifier from filesystem - """ - identifier = "id" - mocker.patch("pathlib.Path.is_file", return_value=True) - read_mock = mocker.patch("pathlib.Path.read_text", return_value=identifier) - configuration.set_option("status", "address", "http://localhost:8081") - _, repository_id = configuration.check_loaded() - system = DistributedSystem(repository_id, configuration) - - assert system.worker.identifier == identifier - read_mock.assert_called_once_with(encoding="utf8") - def test_register(distributed_system: DistributedSystem, mocker: MockerFixture) -> None: """ must register service """ - mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id") - mocker.patch("pathlib.Path.is_file", return_value=False) run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request") - write_mock = mocker.patch("pathlib.Path.write_text") - distributed_system.register() run_mock.assert_called_once_with("POST", f"{distributed_system.address}/api/v1/distributed", json=distributed_system.worker.view()) - write_mock.assert_called_once_with(distributed_system.worker.identifier, encoding="utf8") - assert distributed_system._owe_identifier -def test_register_skip(distributed_system: DistributedSystem, mocker: MockerFixture) -> None: +def test_register_failed(distributed_system: DistributedSystem, mocker: MockerFixture) -> None: """ - must skip service registration if it doesn't owe the identifier + must suppress any exception happened during worker registration """ - mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id") - mocker.patch("pathlib.Path.is_file", return_value=True) - run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request") - write_mock = mocker.patch("pathlib.Path.write_text") - + mocker.patch("requests.Session.request", side_effect=Exception()) distributed_system.register() - run_mock.assert_not_called() - write_mock.assert_not_called() - assert not distributed_system._owe_identifier -def test_register_force(distributed_system: DistributedSystem, mocker: MockerFixture) -> None: +def test_register_failed_http_error(distributed_system: DistributedSystem, mocker: MockerFixture) -> None: """ - must register service even if it doesn't owe the identifier if force is supplied + must suppress HTTP exception happened during worker registration """ - mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id") - mocker.patch("pathlib.Path.is_file", return_value=True) - run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request") - write_mock = mocker.patch("pathlib.Path.write_text") - - distributed_system.register(force=True) - run_mock.assert_called_once_with("POST", f"{distributed_system.address}/api/v1/distributed", - json=distributed_system.worker.view()) - write_mock.assert_called_once_with(distributed_system.worker.identifier, encoding="utf8") - assert distributed_system._owe_identifier - - -def test_unregister(distributed_system: DistributedSystem, mocker: MockerFixture) -> None: - """ - must unregister service - """ - mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id") - run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request") - remove_mock = mocker.patch("pathlib.Path.unlink") - distributed_system._owe_identifier = True - - distributed_system.unregister() - run_mock.assert_called_once_with( - "DELETE", f"{distributed_system.address}/api/v1/distributed/{distributed_system.worker.identifier}") - remove_mock.assert_called_once_with(missing_ok=True) - - -def test_unregister_skip(distributed_system: DistributedSystem, mocker: MockerFixture) -> None: - """ - must skip service removal if it doesn't owe the identifier - """ - mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id") - run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request") - remove_mock = mocker.patch("pathlib.Path.unlink") - - distributed_system.unregister() - run_mock.assert_not_called() - remove_mock.assert_not_called() - - -def test_unregister_force(distributed_system: DistributedSystem, mocker: MockerFixture) -> None: - """ - must remove service even if it doesn't owe the identifier if force is supplied - """ - mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.load_identifier", return_value="id") - run_mock = mocker.patch("ahriman.core.distributed.distributed_system.DistributedSystem.make_request") - remove_mock = mocker.patch("pathlib.Path.unlink") - - distributed_system.unregister(force=True) - run_mock.assert_called_once_with( - "DELETE", f"{distributed_system.address}/api/v1/distributed/{distributed_system.worker.identifier}") - remove_mock.assert_called_once_with(missing_ok=True) + mocker.patch("requests.Session.request", side_effect=requests.HTTPError()) + distributed_system.register() def test_workers(distributed_system: DistributedSystem, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/core/distributed/test_worker_register_trigger.py b/tests/ahriman/core/distributed/test_worker_register_trigger.py deleted file mode 100644 index c8f1beb8..00000000 --- a/tests/ahriman/core/distributed/test_worker_register_trigger.py +++ /dev/null @@ -1,17 +0,0 @@ -from pytest_mock import MockerFixture - -from ahriman.core.configuration import Configuration -from ahriman.core.distributed import WorkerRegisterTrigger - - -def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None: - """ - must register itself as worker - """ - configuration.set_option("status", "address", "http://localhost:8081") - run_mock = mocker.patch("ahriman.core.distributed.WorkerRegisterTrigger.register") - _, repository_id = configuration.check_loaded() - - trigger = WorkerRegisterTrigger(repository_id, configuration) - trigger.on_start() - run_mock.assert_called_once_with(force=True) diff --git a/tests/ahriman/core/distributed/test_worker_trigger.py b/tests/ahriman/core/distributed/test_worker_trigger.py index bdbd904b..b0eb4418 100644 --- a/tests/ahriman/core/distributed/test_worker_trigger.py +++ b/tests/ahriman/core/distributed/test_worker_trigger.py @@ -9,11 +9,10 @@ def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None: must register itself as worker """ configuration.set_option("status", "address", "http://localhost:8081") - run_mock = mocker.patch("ahriman.core.distributed.WorkerTrigger.register") + run_mock = mocker.patch("threading.Timer.start") _, repository_id = configuration.check_loaded() - trigger = WorkerTrigger(repository_id, configuration) - trigger.on_start() + WorkerTrigger(repository_id, configuration).on_start() run_mock.assert_called_once_with() @@ -22,9 +21,32 @@ def test_on_stop(configuration: Configuration, mocker: MockerFixture) -> None: must unregister itself as worker """ configuration.set_option("status", "address", "http://localhost:8081") - run_mock = mocker.patch("ahriman.core.distributed.WorkerTrigger.unregister") + run_mock = mocker.patch("threading.Timer.cancel") _, repository_id = configuration.check_loaded() - trigger = WorkerTrigger(repository_id, configuration) - trigger.on_stop() + WorkerTrigger(repository_id, configuration).on_stop() run_mock.assert_called_once_with() + + +def test_on_stop_empty_timer(configuration: Configuration) -> None: + """ + must do not fail if no timer was started + """ + configuration.set_option("status", "address", "http://localhost:8081") + _, repository_id = configuration.check_loaded() + + WorkerTrigger(repository_id, configuration).on_stop() + + +def test_ping(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must correctly process timer action + """ + configuration.set_option("status", "address", "http://localhost:8081") + run_mock = mocker.patch("ahriman.core.distributed.WorkerTrigger.register") + timer_mock = mocker.patch("threading.Timer.start") + _, repository_id = configuration.check_loaded() + + WorkerTrigger(repository_id, configuration).ping() + run_mock.assert_called_once_with() + timer_mock.assert_called_once_with() diff --git a/tests/ahriman/core/distributed/test_worker_unregister_trigger.py b/tests/ahriman/core/distributed/test_worker_unregister_trigger.py deleted file mode 100644 index 3a35f359..00000000 --- a/tests/ahriman/core/distributed/test_worker_unregister_trigger.py +++ /dev/null @@ -1,17 +0,0 @@ -from pytest_mock import MockerFixture - -from ahriman.core.configuration import Configuration -from ahriman.core.distributed import WorkerUnregisterTrigger - - -def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None: - """ - must unregister itself as worker - """ - configuration.set_option("status", "address", "http://localhost:8081") - run_mock = mocker.patch("ahriman.core.distributed.WorkerUnregisterTrigger.unregister") - _, repository_id = configuration.check_loaded() - - trigger = WorkerUnregisterTrigger(repository_id, configuration) - trigger.on_start() - run_mock.assert_called_once_with(force=True) diff --git a/tests/ahriman/core/distributed/test_workers_cache.py b/tests/ahriman/core/distributed/test_workers_cache.py new file mode 100644 index 00000000..77b8ea78 --- /dev/null +++ b/tests/ahriman/core/distributed/test_workers_cache.py @@ -0,0 +1,43 @@ +import time + +from ahriman.core.distributed import WorkersCache +from ahriman.models.worker import Worker + + +def test_workers(workers_cache: WorkersCache) -> None: + """ + must return alive workers + """ + workers_cache._workers = { + str(index): (Worker(f"address{index}"), index) + for index in range(2) + } + workers_cache.time_to_live = time.monotonic() + + assert workers_cache.workers == [Worker("address1")] + + +def test_workers_remove(workers_cache: WorkersCache) -> None: + """ + must remove all workers + """ + workers_cache.workers_update(Worker("address")) + assert workers_cache.workers + + workers_cache.workers_remove() + assert not workers_cache.workers + + +def test_workers_update(workers_cache: WorkersCache) -> None: + """ + must update worker + """ + worker = Worker("address") + + workers_cache.workers_update(worker) + assert workers_cache.workers == [worker] + _, first_last_seen = workers_cache._workers[worker.identifier] + + workers_cache.workers_update(worker) + _, second_last_seen = workers_cache._workers[worker.identifier] + assert first_last_seen < second_last_seen diff --git a/tests/ahriman/core/log/test_filtered_access_logger.py b/tests/ahriman/core/log/test_filtered_access_logger.py index 923cdc81..f2b397e8 100644 --- a/tests/ahriman/core/log/test_filtered_access_logger.py +++ b/tests/ahriman/core/log/test_filtered_access_logger.py @@ -4,7 +4,26 @@ from unittest.mock import MagicMock from ahriman.core.log.filtered_access_logger import FilteredAccessLogger -def is_info_get() -> None: +def test_is_distributed_post() -> None: + """ + must correctly define distributed services ping request + """ + request = MagicMock() + + request.method = "POST" + request.path = "/api/v1/distributed" + assert FilteredAccessLogger.is_distributed_post(request) + + request.method = "GET" + request.path = "/api/v1/distributed" + assert not FilteredAccessLogger.is_distributed_post(request) + + request.method = "POST" + request.path = "/api/v1/distributed/path" + assert not FilteredAccessLogger.is_distributed_post(request) + + +def test_is_info_get() -> None: """ must correctly define health check request """ diff --git a/tests/ahriman/core/status/test_watcher.py b/tests/ahriman/core/status/test_watcher.py index fa86f939..ba36cfcb 100644 --- a/tests/ahriman/core/status/test_watcher.py +++ b/tests/ahriman/core/status/test_watcher.py @@ -10,7 +10,6 @@ from ahriman.models.changes import Changes from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch -from ahriman.models.worker import Worker def test_load(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: @@ -228,40 +227,3 @@ def test_status_update(watcher: Watcher) -> None: """ watcher.status_update(BuildStatusEnum.Success) assert watcher.status.status == BuildStatusEnum.Success - - -def test_workers_get(watcher: Watcher, mocker: MockerFixture) -> None: - """ - must retrieve workers - """ - worker = Worker("remote") - worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_get", return_value=[worker]) - - assert watcher.workers_get() == [worker] - worker_mock.assert_called_once_with() - - -def test_workers_remove(watcher: Watcher, mocker: MockerFixture) -> None: - """ - must remove workers - """ - identifier = "identifier" - worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_remove") - - watcher.workers_remove(identifier) - watcher.workers_remove() - worker_mock.assert_has_calls([ - MockCall(identifier), - MockCall(None), - ]) - - -def test_workers_update(watcher: Watcher, mocker: MockerFixture) -> None: - """ - must update workers - """ - worker = Worker("remote") - worker_mock = mocker.patch("ahriman.core.database.SQLite.workers_insert") - - watcher.workers_update(worker) - worker_mock.assert_called_once_with(worker) diff --git a/tests/ahriman/web/schemas/test_worker_id_schema.py b/tests/ahriman/web/schemas/test_worker_id_schema.py deleted file mode 100644 index 1982fb6b..00000000 --- a/tests/ahriman/web/schemas/test_worker_id_schema.py +++ /dev/null @@ -1 +0,0 @@ -# schema testing goes in view class tests diff --git a/tests/ahriman/web/views/test_view_base.py b/tests/ahriman/web/views/test_view_base.py index fce32fbc..acf691c6 100644 --- a/tests/ahriman/web/views/test_view_base.py +++ b/tests/ahriman/web/views/test_view_base.py @@ -50,11 +50,18 @@ def test_spawn(base: BaseView) -> None: def test_validator(base: BaseView) -> None: """ - must return service + must return validator service """ assert base.validator +def test_workers(base: BaseView) -> None: + """ + must return worker service + """ + assert base.workers + + async def test_get_permission(base: BaseView) -> None: """ must search for permission attribute in class diff --git a/tests/ahriman/web/views/v1/distributed/test_view_v1_distributed_worker.py b/tests/ahriman/web/views/v1/distributed/test_view_v1_distributed_worker.py deleted file mode 100644 index d3e2134d..00000000 --- a/tests/ahriman/web/views/v1/distributed/test_view_v1_distributed_worker.py +++ /dev/null @@ -1,70 +0,0 @@ -import pytest - -from aiohttp.test_utils import TestClient - -from ahriman.models.user_access import UserAccess -from ahriman.models.worker import Worker -from ahriman.web.views.v1.distributed.worker import WorkerView - - -async def test_get_permission() -> None: - """ - must return correct permission for the request - """ - for method in ("DELETE", "GET"): - request = pytest.helpers.request("", "", method) - assert await WorkerView.get_permission(request) == UserAccess.Full - - -def test_routes() -> None: - """ - must return correct routes - """ - assert WorkerView.ROUTES == ["/api/v1/distributed/{identifier}"] - - -async def test_delete(client: TestClient) -> None: - """ - must delete single worker - """ - await client.post("/api/v1/distributed", json={"address": "address1", "identifier": "1"}) - await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"}) - - response = await client.delete("/api/v1/distributed/1") - assert response.status == 204 - - response = await client.get("/api/v1/distributed/1") - assert response.status == 404 - - response = await client.get("/api/v1/distributed/2") - assert response.ok - - -async def test_get(client: TestClient) -> None: - """ - must return specific worker - """ - worker = Worker("address1", identifier="1") - - await client.post("/api/v1/distributed", json=worker.view()) - await client.post("/api/v1/distributed", json={"address": "address2", "identifier": "2"}) - response_schema = pytest.helpers.schema_response(WorkerView.get) - - response = await client.get(f"/api/v1/distributed/{worker.identifier}") - assert response.ok - json = await response.json() - assert not response_schema.validate(json, many=True) - - workers = [Worker(item["address"], identifier=item["identifier"]) for item in json] - assert workers == [worker] - - -async def test_get_not_found(client: TestClient) -> None: - """ - must return Not Found for unknown package - """ - response_schema = pytest.helpers.schema_response(WorkerView.get, code=404) - - response = await client.get("/api/v1/distributed/1") - assert response.status == 404 - assert not response_schema.validate(await response.json()) diff --git a/tests/ahriman/web/views/v1/distributed/test_view_v1_distributed_workers.py b/tests/ahriman/web/views/v1/distributed/test_view_v1_distributed_workers.py index 397e37b2..5365f212 100644 --- a/tests/ahriman/web/views/v1/distributed/test_view_v1_distributed_workers.py +++ b/tests/ahriman/web/views/v1/distributed/test_view_v1_distributed_workers.py @@ -68,9 +68,6 @@ async def test_post(client: TestClient) -> None: response = await client.post("/api/v1/distributed", json=payload) assert response.status == 204 - response = await client.get(f"/api/v1/distributed/{worker.identifier}") - assert response.ok - async def test_post_exception(client: TestClient) -> None: """