From d942a7027232cd696130ea7a1b4cc0df6331a5c8 Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Mon, 9 Jan 2023 17:22:29 +0200 Subject: [PATCH] add config validator subcommand (#80) * add config validator subcommand * add --exit-code flag * docs & faq update --- .github/workflows/setup.sh | 4 +- CONTRIBUTING.md | 6 +- Dockerfile | 3 +- docker/entrypoint.sh | 3 + docs/ahriman.1 | 17 +- docs/ahriman.application.handlers.rst | 8 + docs/ahriman.core.configuration.rst | 37 ++ docs/ahriman.core.formatters.rst | 8 + docs/ahriman.core.rst | 9 +- docs/completions/bash/_ahriman | 14 +- docs/completions/zsh/_ahriman | 14 + docs/configuration.rst | 48 +- docs/faq.rst | 30 +- package/archlinux/PKGBUILD | 2 +- setup.py | 1 + src/ahriman/application/ahriman.py | 20 + src/ahriman/application/handlers/__init__.py | 1 + src/ahriman/application/handlers/validate.py | 150 +++++ src/ahriman/core/configuration/__init__.py | 20 + .../core/{ => configuration}/configuration.py | 7 +- src/ahriman/core/configuration/schema.py | 554 ++++++++++++++++++ src/ahriman/core/configuration/validator.py | 116 ++++ src/ahriman/core/formatters/__init__.py | 1 + src/ahriman/core/formatters/printer.py | 3 +- .../core/formatters/validation_printer.py | 77 +++ src/ahriman/models/property.py | 2 + .../application/handlers/test_handler_dump.py | 5 +- .../handlers/test_handler_validate.py | 104 ++++ tests/ahriman/application/test_ahriman.py | 12 + tests/ahriman/core/configuration/conftest.py | 19 + .../{ => configuration}/test_configuration.py | 0 .../ahriman/core/configuration/test_schema.py | 0 .../core/configuration/test_validator.py | 70 +++ tests/ahriman/core/formatters/conftest.py | 25 +- tests/ahriman/core/formatters/test_printer.py | 22 +- .../formatters/test_validation_printer.py | 28 + 36 files changed, 1393 insertions(+), 47 deletions(-) create mode 100644 docs/ahriman.core.configuration.rst create mode 100644 src/ahriman/application/handlers/validate.py create mode 100644 src/ahriman/core/configuration/__init__.py rename src/ahriman/core/{ => configuration}/configuration.py (97%) create mode 100644 src/ahriman/core/configuration/schema.py create mode 100644 src/ahriman/core/configuration/validator.py create mode 100644 src/ahriman/core/formatters/validation_printer.py create mode 100644 tests/ahriman/application/handlers/test_handler_validate.py create mode 100644 tests/ahriman/core/configuration/conftest.py rename tests/ahriman/core/{ => configuration}/test_configuration.py (100%) create mode 100644 tests/ahriman/core/configuration/test_schema.py create mode 100644 tests/ahriman/core/configuration/test_validator.py create mode 100644 tests/ahriman/core/formatters/test_validation_printer.py diff --git a/.github/workflows/setup.sh b/.github/workflows/setup.sh index 25692d41..c8d064ba 100755 --- a/.github/workflows/setup.sh +++ b/.github/workflows/setup.sh @@ -10,7 +10,7 @@ echo -e '[arcanisrepo]\nServer = http://repo.arcanis.me/$arch\nSigLevel = Never' # refresh the image pacman --noconfirm -Syu # main dependencies -pacman --noconfirm -Sy base-devel devtools git pyalpm python-aur python-passlib python-setuptools python-srcinfo sudo +pacman --noconfirm -Sy base-devel devtools git pyalpm python-cerberus python-inflection python-passlib python-requests python-setuptools python-srcinfo sudo # make dependencies pacman --noconfirm -Sy python-build python-installer python-wheel # optional dependencies @@ -39,6 +39,8 @@ sed -i "s/handlers = syslog_handler/handlers = console_handler/g" /etc/ahriman.i # initial setup command as root [[ -z $MINIMAL_INSTALL ]] && WEB_ARGS=("--web-port" "8080") ahriman -a x86_64 repo-setup --packager "ahriman bot " --repository "github" "${WEB_ARGS[@]}" +# validate configuration +ahriman -a x86_64 repo-config-validate --exit-code # enable services systemctl enable ahriman-web@x86_64 systemctl enable ahriman@x86_64.timer diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 88ab1fcd..b559bb67 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,9 +77,13 @@ Again, the most checks can be performed by `make check` command, though some add CLAZZ_ATTRIBUTE = 42 - def __init__(self) -> None: + def __init__(self, *args: Any, **kwargs: Any) -> None: """ default constructor + + Args: + *args(Any): positional arguments + **kwargs(Any): keyword arguments """ self.instance_attribute = "" ``` diff --git a/Dockerfile b/Dockerfile index c811e85d..a2d1ec12 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ ENV AHRIMAN_REPOSITORY="aur-clone" ENV AHRIMAN_REPOSITORY_ROOT="/var/lib/ahriman/ahriman" ENV AHRIMAN_UNIX_SOCKET="" ENV AHRIMAN_USER="ahriman" +ENV AHRIMAN_VALIDATE_CONFIGURATION="" # install environment ## update pacman.conf with multilib @@ -25,7 +26,7 @@ RUN useradd -m -d "/home/build" -s "/usr/bin/nologin" build && \ COPY "docker/install-aur-package.sh" "/usr/local/bin/install-aur-package" ## install package dependencies ## darcs is not installed by reasons, because it requires a lot haskell packages which dramatically increase image size -RUN pacman --noconfirm -Sy devtools git pyalpm python-inflection python-passlib python-requests python-setuptools python-srcinfo && \ +RUN pacman --noconfirm -Sy devtools git pyalpm python-cerberus python-inflection python-passlib python-requests python-setuptools python-srcinfo && \ pacman --noconfirm -Sy python-build python-installer python-wheel && \ pacman --noconfirm -Sy breezy mercurial python-aiohttp python-boto3 python-cryptography python-jinja python-requests-unixsocket rsync subversion && \ runuser -u build -- install-aur-package python-aioauth-client python-aiohttp-jinja2 python-aiohttp-debugtoolbar \ diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index cd4f7192..1bfec206 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -47,6 +47,9 @@ if [ -n "$AHRIMAN_UNIX_SOCKET" ]; then fi ahriman "${AHRIMAN_DEFAULT_ARGS[@]}" repo-setup "${AHRIMAN_SETUP_ARGS[@]}" +# validate configuration if set +[ -n "$AHRIMAN_VALIDATE_CONFIGURATION" ] && ahriman "${AHRIMAN_DEFAULT_ARGS[@]}" repo-config-validate --exit-code + # create machine-id which is required by build tools systemd-machine-id-setup &> /dev/null diff --git a/docs/ahriman.1 b/docs/ahriman.1 index f44f7cc7..baae1399 100644 --- a/docs/ahriman.1 +++ b/docs/ahriman.1 @@ -1,9 +1,9 @@ -.TH AHRIMAN "1" "2023\-01\-03" "ahriman" "Generated Python Manual" +.TH AHRIMAN "1" "2023\-01\-09" "ahriman" "Generated Python Manual" .SH NAME ahriman .SH SYNOPSIS .B ahriman -[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--report | --no-report] [-q] [--unsafe] [-V] {aur-search,search,daemon,help,help-commands-unsafe,key-import,package-add,add,package-update,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,patch-set-add,repo-backup,repo-check,check,repo-clean,clean,repo-config,config,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-setup,init,repo-init,setup,repo-sign,sign,repo-status-update,repo-sync,sync,repo-tree,repo-triggers,repo-update,update,shell,user-add,user-list,user-remove,version,web} ... +[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--report | --no-report] [-q] [--unsafe] [-V] {aur-search,search,daemon,help,help-commands-unsafe,key-import,package-add,add,package-update,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,patch-set-add,repo-backup,repo-check,check,repo-clean,clean,repo-config,config,repo-config-validate,config-validate,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-setup,init,repo-init,setup,repo-sign,sign,repo-status-update,repo-sync,sync,repo-tree,repo-triggers,repo-update,update,shell,user-add,user-list,user-remove,version,web} ... .SH DESCRIPTION ArcH linux ReposItory MANager @@ -97,6 +97,9 @@ clean local caches \fBahriman\fR \fI\,repo\-config\/\fR dump configuration .TP +\fBahriman\fR \fI\,repo\-config\-validate\/\fR +validate system configuration +.TP \fBahriman\fR \fI\,repo\-rebuild\/\fR rebuild repository .TP @@ -456,6 +459,16 @@ usage: ahriman repo\-config [\-h] dump configuration for the specified architecture +.SH COMMAND \fI\,'ahriman repo\-config\-validate'\/\fR +usage: ahriman repo\-config\-validate [\-h] [\-e] + +validate configuration and print found errors + +.SH OPTIONS \fI\,'ahriman repo\-config\-validate'\/\fR +.TP +\fB\-e\fR, \fB\-\-exit\-code\fR +return non\-zero exit status if configuration is invalid + .SH COMMAND \fI\,'ahriman repo\-rebuild'\/\fR usage: ahriman repo\-rebuild [\-h] [\-\-depends\-on DEPENDS_ON] [\-\-dry\-run] [\-\-from\-database] [\-e] diff --git a/docs/ahriman.application.handlers.rst b/docs/ahriman.application.handlers.rst index be647e54..952f912c 100644 --- a/docs/ahriman.application.handlers.rst +++ b/docs/ahriman.application.handlers.rst @@ -196,6 +196,14 @@ ahriman.application.handlers.users module :no-undoc-members: :show-inheritance: +ahriman.application.handlers.validate module +-------------------------------------------- + +.. automodule:: ahriman.application.handlers.validate + :members: + :no-undoc-members: + :show-inheritance: + ahriman.application.handlers.versions module -------------------------------------------- diff --git a/docs/ahriman.core.configuration.rst b/docs/ahriman.core.configuration.rst new file mode 100644 index 00000000..f33f01f4 --- /dev/null +++ b/docs/ahriman.core.configuration.rst @@ -0,0 +1,37 @@ +ahriman.core.configuration package +================================== + +Submodules +---------- + +ahriman.core.configuration.configuration module +----------------------------------------------- + +.. automodule:: ahriman.core.configuration.configuration + :members: + :no-undoc-members: + :show-inheritance: + +ahriman.core.configuration.schema module +---------------------------------------- + +.. automodule:: ahriman.core.configuration.schema + :members: + :no-undoc-members: + :show-inheritance: + +ahriman.core.configuration.validator module +------------------------------------------- + +.. automodule:: ahriman.core.configuration.validator + :members: + :no-undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: ahriman.core.configuration + :members: + :no-undoc-members: + :show-inheritance: diff --git a/docs/ahriman.core.formatters.rst b/docs/ahriman.core.formatters.rst index 59821916..974ce7cd 100644 --- a/docs/ahriman.core.formatters.rst +++ b/docs/ahriman.core.formatters.rst @@ -92,6 +92,14 @@ ahriman.core.formatters.user\_printer module :no-undoc-members: :show-inheritance: +ahriman.core.formatters.validation\_printer module +-------------------------------------------------- + +.. automodule:: ahriman.core.formatters.validation_printer + :members: + :no-undoc-members: + :show-inheritance: + ahriman.core.formatters.version\_printer module ----------------------------------------------- diff --git a/docs/ahriman.core.rst b/docs/ahriman.core.rst index 67795328..580b2315 100644 --- a/docs/ahriman.core.rst +++ b/docs/ahriman.core.rst @@ -10,6 +10,7 @@ Subpackages ahriman.core.alpm ahriman.core.auth ahriman.core.build_tools + ahriman.core.configuration ahriman.core.database ahriman.core.formatters ahriman.core.gitremote @@ -24,14 +25,6 @@ Subpackages Submodules ---------- -ahriman.core.configuration module ---------------------------------- - -.. automodule:: ahriman.core.configuration - :members: - :no-undoc-members: - :show-inheritance: - ahriman.core.exceptions module ------------------------------ diff --git a/docs/completions/bash/_ahriman b/docs/completions/bash/_ahriman index 2174a21b..ef5eb435 100644 --- a/docs/completions/bash/_ahriman +++ b/docs/completions/bash/_ahriman @@ -1,6 +1,6 @@ # AUTOMATICALLY GENERATED by `shtab` -_shtab_ahriman_subparsers=('aur-search' 'search' 'daemon' 'help' 'help-commands-unsafe' 'key-import' 'package-add' 'add' 'package-update' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-clean' 'clean' 'repo-config' 'config' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-setup' 'init' 'repo-init' 'setup' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'shell' 'user-add' 'user-list' 'user-remove' 'version' 'web') +_shtab_ahriman_subparsers=('aur-search' 'search' 'daemon' 'help' 'help-commands-unsafe' 'key-import' 'package-add' 'add' 'package-update' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-clean' 'clean' 'repo-config' 'config' 'repo-config-validate' 'config-validate' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-setup' 'init' 'repo-init' 'setup' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'shell' 'user-add' 'user-list' 'user-remove' 'version' 'web') _shtab_ahriman_option_strings=('-h' '--help' '-a' '--architecture' '-c' '--configuration' '--force' '-l' '--lock' '--report' '--no-report' '-q' '--quiet' '--unsafe' '-V' '--version') _shtab_ahriman_aur_search_option_strings=('-h' '--help' '-e' '--exit-code' '--info' '--no-info' '--sort-by') @@ -30,6 +30,8 @@ _shtab_ahriman_repo_clean_option_strings=('-h' '--help' '--cache' '--no-cache' ' _shtab_ahriman_clean_option_strings=('-h' '--help' '--cache' '--no-cache' '--chroot' '--no-chroot' '--manual' '--no-manual' '--packages' '--no-packages' '--pacman' '--no-pacman') _shtab_ahriman_repo_config_option_strings=('-h' '--help') _shtab_ahriman_config_option_strings=('-h' '--help') +_shtab_ahriman_repo_config_validate_option_strings=('-h' '--help' '-e' '--exit-code') +_shtab_ahriman_config_validate_option_strings=('-h' '--help' '-e' '--exit-code') _shtab_ahriman_repo_rebuild_option_strings=('-h' '--help' '--depends-on' '--dry-run' '--from-database' '-e' '--exit-code') _shtab_ahriman_rebuild_option_strings=('-h' '--help' '--depends-on' '--dry-run' '--from-database' '-e' '--exit-code') _shtab_ahriman_repo_remove_unknown_option_strings=('-h' '--help' '--dry-run') @@ -59,7 +61,7 @@ _shtab_ahriman_web_option_strings=('-h' '--help') -_shtab_ahriman_pos_0_choices=('aur-search' 'search' 'daemon' 'help' 'help-commands-unsafe' 'key-import' 'package-add' 'add' 'package-update' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-clean' 'clean' 'repo-config' 'config' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-setup' 'init' 'repo-init' 'setup' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'shell' 'user-add' 'user-list' 'user-remove' 'version' 'web') +_shtab_ahriman_pos_0_choices=('aur-search' 'search' 'daemon' 'help' 'help-commands-unsafe' 'key-import' 'package-add' 'add' 'package-update' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-clean' 'clean' 'repo-config' 'config' 'repo-config-validate' 'config-validate' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-setup' 'init' 'repo-init' 'setup' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'shell' 'user-add' 'user-list' 'user-remove' 'version' 'web') _shtab_ahriman_aur_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'url' 'url_path' 'version') _shtab_ahriman_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'url' 'url_path' 'version') _shtab_ahriman_package_add__s_choices=('auto' 'archive' 'aur' 'directory' 'local' 'remote' 'repository') @@ -249,6 +251,14 @@ _shtab_ahriman_repo_config__h_nargs=0 _shtab_ahriman_repo_config___help_nargs=0 _shtab_ahriman_config__h_nargs=0 _shtab_ahriman_config___help_nargs=0 +_shtab_ahriman_repo_config_validate__h_nargs=0 +_shtab_ahriman_repo_config_validate___help_nargs=0 +_shtab_ahriman_repo_config_validate__e_nargs=0 +_shtab_ahriman_repo_config_validate___exit_code_nargs=0 +_shtab_ahriman_config_validate__h_nargs=0 +_shtab_ahriman_config_validate___help_nargs=0 +_shtab_ahriman_config_validate__e_nargs=0 +_shtab_ahriman_config_validate___exit_code_nargs=0 _shtab_ahriman_repo_rebuild__h_nargs=0 _shtab_ahriman_repo_rebuild___help_nargs=0 _shtab_ahriman_repo_rebuild___dry_run_nargs=0 diff --git a/docs/completions/zsh/_ahriman b/docs/completions/zsh/_ahriman index 5da5846e..8e1aa3c6 100644 --- a/docs/completions/zsh/_ahriman +++ b/docs/completions/zsh/_ahriman @@ -10,6 +10,7 @@ _shtab_ahriman_commands() { "check:check for packages updates. Same as repo-update --dry-run --no-manual" "clean:remove local caches" "config:dump configuration for the specified architecture" + "config-validate:validate configuration and print found errors" "daemon:start process which periodically will run update process" "help:show help message for application or command and exit" "help-commands-unsafe:list unsafe commands as defined in default args" @@ -32,6 +33,7 @@ _shtab_ahriman_commands() { "repo-check:check for packages updates. Same as repo-update --dry-run --no-manual" "repo-clean:remove local caches" "repo-config:dump configuration for the specified architecture" + "repo-config-validate:validate configuration and print found errors" "repo-init:create initial service configuration, requires root" "repo-rebuild:force rebuild whole repository" "repo-remove-unknown:remove packages which are missing in AUR and do not have local PKGBUILDs" @@ -113,6 +115,11 @@ _shtab_ahriman_config_options=( "(- : *)"{-h,--help}"[show this help message and exit]" ) +_shtab_ahriman_config_validate_options=( + "(- : *)"{-h,--help}"[show this help message and exit]" + {-e,--exit-code}"[return non-zero exit status if configuration is invalid]" +) + _shtab_ahriman_daemon_options=( "(- : *)"{-h,--help}"[show this help message and exit]" {-i,--interval}"[interval between runs in seconds]:interval:" @@ -269,6 +276,11 @@ _shtab_ahriman_repo_config_options=( "(- : *)"{-h,--help}"[show this help message and exit]" ) +_shtab_ahriman_repo_config_validate_options=( + "(- : *)"{-h,--help}"[show this help message and exit]" + {-e,--exit-code}"[return non-zero exit status if configuration is invalid]" +) + _shtab_ahriman_repo_init_options=( "(- : *)"{-h,--help}"[show this help message and exit]" "--build-as-user[force makepkg user to the specific one]:build_as_user:" @@ -473,6 +485,7 @@ _shtab_ahriman() { check) _arguments -C $_shtab_ahriman_check_options ;; clean) _arguments -C $_shtab_ahriman_clean_options ;; config) _arguments -C $_shtab_ahriman_config_options ;; + config-validate) _arguments -C $_shtab_ahriman_config_validate_options ;; daemon) _arguments -C $_shtab_ahriman_daemon_options ;; help) _arguments -C $_shtab_ahriman_help_options ;; help-commands-unsafe) _arguments -C $_shtab_ahriman_help_commands_unsafe_options ;; @@ -495,6 +508,7 @@ _shtab_ahriman() { repo-check) _arguments -C $_shtab_ahriman_repo_check_options ;; repo-clean) _arguments -C $_shtab_ahriman_repo_clean_options ;; repo-config) _arguments -C $_shtab_ahriman_repo_config_options ;; + repo-config-validate) _arguments -C $_shtab_ahriman_repo_config_validate_options ;; repo-init) _arguments -C $_shtab_ahriman_repo_init_options ;; repo-rebuild) _arguments -C $_shtab_ahriman_repo_rebuild_options ;; repo-remove-unknown) _arguments -C $_shtab_ahriman_repo_remove_unknown_options ;; diff --git a/docs/configuration.rst b/docs/configuration.rst index b013feb6..5e4977c5 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -12,6 +12,14 @@ There are two variable types which have been added to default ones, they are pat Path values, except for casting to ``pathlib.Path`` type, will be also expanded to absolute paths relative to the configuration path. E.g. if path is set to ``ahriman.ini.d/logging.ini`` and root configuration path is ``/etc/ahriman.ini``, the value will be expanded to ``/etc/ahriman.ini.d/logging.ini``. In order to disable path expand, use the full path, e.g. ``/etc/ahriman.ini.d/logging.ini``. +There is also additional subcommand which will allow to validate configuration and print found errors. In order to do so, run ``repo-config-validate`` subcommand, e.g.: + +.. code-block:: shell + + ahriman -a x86_64 repo-config-validate + +It will check current settings on common errors and compare configuration with known schema. + ``settings`` group ------------------ @@ -78,6 +86,25 @@ Settings for signing packages or repository. Group name can refer to architectur * ``key`` - default PGP key, string, required. This key will also be used for database signing if enabled. * ``key_*`` settings - PGP key which will be used for specific packages, string, optional. For example, if there is ``key_yay`` option the specified key will be used for yay package and default key for others. +``web:*`` groups +---------------- + +Web server settings. If any of ``host``/``port`` is not set, web integration will be disabled. Group name can refer to architecture, e.g. ``web:x86_64`` can be used for x86_64 architecture specific settings. This feature requires ``aiohttp`` libraries to be installed. + +* ``address`` - optional address in form ``proto://host:port`` (``port`` can be omitted in case of default ``proto`` ports), will be used instead of ``http://{host}:{port}`` in case if set, string, optional. This option is required in case if ``OAuth`` provider is used. +* ``debug`` - enable debug toolbar, boolean, optional, default ``no``. +* ``debug_check_host`` - check hosts to access debug toolbar, boolean, optional, default ``no``. +* ``debug_allowed_hosts`` - allowed hosts to get access to debug toolbar, space separated list of string, optional. +* ``host`` - host to bind, string, optional. +* ``index_url`` - full url of the repository index page, string, optional. +* ``password`` - password to authorize in web service in order to update service status, string, required in case if authorization enabled. +* ``port`` - port to bind, int, optional. +* ``static_path`` - path to directory with static files, string, required. +* ``templates`` - path to templates directory, string, required. +* ``unix_socket`` - path to the listening unix socket, string, optional. If set, server will create the socket on the specified address which can (and will) be used by application. Note, that unlike usual host/port configuration, unix socket allows to perform requests without authorization. +* ``unix_socket_unsafe`` - set unsafe (o+w) permissions to unix socket, boolean, optional, default ``yes``. This option is enabled by default, because it is supposed that unix socket is created in safe environment (only web service is supposed to be used in unsafe), but it can be disabled by configuration. +* ``username`` - username to authorize in web service in order to update service status, string, required in case if authorization enabled. + ``remote-pull`` group --------------------- @@ -224,23 +251,4 @@ Requires ``boto3`` library to be installed. Section name must be either ``s3`` ( * ``bucket`` - bucket name (e.g. ``bucket``), string, required. * ``chunk_size`` - chunk size for calculating entity tags, int, optional, default 8 * 1024 * 1024. * ``region`` - bucket region (e.g. ``eu-central-1``), string, required. -* ``secret_key`` - AWS secret access key, string, required. - -``web:*`` groups ----------------- - -Web server settings. If any of ``host``/``port`` is not set, web integration will be disabled. Group name can refer to architecture, e.g. ``web:x86_64`` can be used for x86_64 architecture specific settings. This feature requires ``aiohttp`` libraries to be installed. - -* ``address`` - optional address in form ``proto://host:port`` (``port`` can be omitted in case of default ``proto`` ports), will be used instead of ``http://{host}:{port}`` in case if set, string, optional. This option is required in case if ``OAuth`` provider is used. -* ``debug`` - enable debug toolbar, boolean, optional, default ``no``. -* ``debug_check_host`` - check hosts to access debug toolbar, boolean, optional, default ``no``. -* ``debug_allowed_hosts`` - allowed hosts to get access to debug toolbar, space separated list of string, optional. -* ``host`` - host to bind, string, optional. -* ``index_url`` - full url of the repository index page, string, optional. -* ``password`` - password to authorize in web service in order to update service status, string, required in case if authorization enabled. -* ``port`` - port to bind, int, optional. -* ``static_path`` - path to directory with static files, string, required. -* ``templates`` - path to templates directory, string, required. -* ``unix_socket`` - path to the listening unix socket, string, optional. If set, server will create the socket on the specified address which can (and will) be used by application. Note, that unlike usual host/port configuration, unix socket allows to perform requests without authorization. -* ``unix_socket_unsafe`` - set unsafe (o+w) permissions to unix socket, boolean, optional, default ``yes``. This option is enabled by default, because it is supposed that unix socket is created in safe environment (only web service is supposed to be used in unsafe), but it can be disabled by configuration. -* ``username`` - username to authorize in web service in order to update service status, string, required in case if authorization enabled. +* ``secret_key`` - AWS secret access key, string, required. \ No newline at end of file diff --git a/docs/faq.rst b/docs/faq.rst index 63ef35bf..1530ac23 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -26,6 +26,33 @@ Long answer The idea is to install the package as usual, create working directory tree, create configuration for ``sudo`` and ``devtools``. Detailed description of the setup instruction can be found :doc:`here `. +How to validate settings +^^^^^^^^^^^^^^^^^^^^^^^^ + +There is special command which can be used in order to validate current configuration: + +.. code-block:: shell + + ahriman -a x86_64 repo-config-validate --exit-code + +This command will print found errors, based on `cerberus `_, e.g.: + +.. code-block:: shell + + auth + ssalt: unknown field + target: none or more than one rule validate + oneof definition 0: unallowed value mapping + oneof definition 1: field 'salt' is required + oneof definition 2: unallowed value mapping + oneof definition 2: field 'salt' is required + oneof definition 2: field 'client_id' is required + oneof definition 2: field 'client_secret' is required + gitremote + pull_url: unknown field + +If an additional flag ``--exit-code`` is supplied, the application will return non-zero exit code, which can be used partially in scripts. + What does "architecture specific" mean / How to configure for different architectures ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -392,7 +419,8 @@ The following environment variables are supported: * ``AHRIMAN_REPOSITORY`` - repository name, default is ``aur-clone``. * ``AHRIMAN_REPOSITORY_ROOT`` - repository root. Because of filesystem rights it is required to override default repository root. By default, it uses ``ahriman`` directory inside ahriman's home, which can be passed as mount volume. * ``AHRIMAN_UNIX_SOCKET`` - full path to unix socket which is used by web server, default is empty. Note that more likely you would like to put it inside ``AHRIMAN_REPOSITORY_ROOT`` directory (e.g. ``/var/lib/ahriman/ahriman/ahriman-web.sock``) or to ``/tmp``. -* ``AHRIMAN_USER`` - ahriman user, usually must not be overwritten, default is ``ahriman``. +* ``AHRIMAN_USER`` - ahriman user, usually must not be overwritten, default is ``ahriman``. +* ``AHRIMAN_VALIDATE_CONFIGURATION`` - if set validate service configuration You can pass any of these variables by using ``-e`` argument, e.g.: diff --git a/package/archlinux/PKGBUILD b/package/archlinux/PKGBUILD index a137674c..158140d8 100644 --- a/package/archlinux/PKGBUILD +++ b/package/archlinux/PKGBUILD @@ -7,7 +7,7 @@ pkgdesc="ArcH linux ReposItory MANager" arch=('any') url="https://github.com/arcan1s/ahriman" license=('GPL3') -depends=('devtools' 'git' 'pyalpm' 'python-inflection' 'python-passlib' 'python-requests' 'python-setuptools' 'python-srcinfo') +depends=('devtools' 'git' 'pyalpm' 'python-cerberus' 'python-inflection' 'python-passlib' 'python-requests' 'python-setuptools' 'python-srcinfo') makedepends=('python-build' 'python-installer' 'python-wheel') optdepends=('breezy: -bzr packages support' 'darcs: -darcs packages support' diff --git a/setup.py b/setup.py index ec8afac9..420bfe55 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ setup( dependency_links=[ ], install_requires=[ + "cerberus", "inflection", "passlib", "requests", diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 175ec91f..cf609737 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -101,6 +101,7 @@ def _parser() -> argparse.ArgumentParser: _set_repo_check_parser(subparsers) _set_repo_clean_parser(subparsers) _set_repo_config_parser(subparsers) + _set_repo_config_validate_parser(subparsers) _set_repo_rebuild_parser(subparsers) _set_repo_remove_unknown_parser(subparsers) _set_repo_report_parser(subparsers) @@ -537,6 +538,25 @@ def _set_repo_config_parser(root: SubParserAction) -> argparse.ArgumentParser: return parser +def _set_repo_config_validate_parser(root: SubParserAction) -> argparse.ArgumentParser: + """ + add parser for config validation subcommand + + Args: + root(SubParserAction): subparsers for the commands + + Returns: + argparse.ArgumentParser: created argument parser + """ + parser = root.add_parser("repo-config-validate", aliases=["config-validate"], help="validate system configuration", + description="validate configuration and print found errors", + formatter_class=_formatter) + parser.add_argument("-e", "--exit-code", help="return non-zero exit status if configuration is invalid", + action="store_true") + parser.set_defaults(handler=handlers.Validate, lock=None, report=False, quiet=True, unsafe=True) + return parser + + def _set_repo_rebuild_parser(root: SubParserAction) -> argparse.ArgumentParser: """ add parser for repository rebuild subcommand diff --git a/src/ahriman/application/handlers/__init__.py b/src/ahriman/application/handlers/__init__.py index 8bbb63ac..2b19f62e 100644 --- a/src/ahriman/application/handlers/__init__.py +++ b/src/ahriman/application/handlers/__init__.py @@ -42,5 +42,6 @@ from ahriman.application.handlers.triggers import Triggers from ahriman.application.handlers.unsafe_commands import UnsafeCommands from ahriman.application.handlers.update import Update from ahriman.application.handlers.users import Users +from ahriman.application.handlers.validate import Validate from ahriman.application.handlers.versions import Versions from ahriman.application.handlers.web import Web diff --git a/src/ahriman/application/handlers/validate.py b/src/ahriman/application/handlers/validate.py new file mode 100644 index 00000000..f3f48f36 --- /dev/null +++ b/src/ahriman/application/handlers/validate.py @@ -0,0 +1,150 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import argparse +import copy + +from typing import Any, Callable, Dict, Optional, Type + +from ahriman.application.handlers import Handler +from ahriman.core.configuration import Configuration +from ahriman.core.configuration.schema import CONFIGURATION_SCHEMA, \ + GITREMOTE_REMOTE_PULL_SCHEMA, GITREMOTE_REMOTE_PUSH_SCHEMA, \ + REPORT_CONSOLE_SCHEMA, REPORT_EMAIL_SCHEMA, REPORT_HTML_SCHEMA, REPORT_TELEGRAM_SCHEMA,\ + UPLOAD_GITHUB_SCHEMA, UPLOAD_RSYNC_SCHEMA, UPLOAD_S3_SCHEMA +from ahriman.core.configuration.validator import Validator +from ahriman.core.formatters import ValidationPrinter + + +class Validate(Handler): + """ + configuration validator handler + """ + + ALLOW_AUTO_ARCHITECTURE_RUN = False + + @classmethod + def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, configuration: Configuration, *, + report: bool, unsafe: bool) -> None: + """ + callback for command line + + Args: + args(argparse.Namespace): command line args + architecture(str): repository architecture + configuration(Configuration): configuration instance + report(bool): force enable or disable reporting + unsafe(bool): if set no user check will be performed before path creation + """ + schema = Validate.schema(architecture, configuration) + validator = Validator(instance=configuration, schema=schema) + + if validator.validate(configuration.dump()): + return # no errors found + for node, errors in validator.errors.items(): + ValidationPrinter(node, errors).print(verbose=True) + + # as we reach this part it means that we always have errors + Validate.check_if_empty(args.exit_code, True) + + @staticmethod + def schema(architecture: str, configuration: Configuration) -> Dict[str, Any]: + """ + get schema with triggers + + Args: + architecture(str): repository architecture + configuration(Configuration): configuration instance + + Returns: + Dict[str, Any]: configuration validation schema + """ + root = copy.deepcopy(CONFIGURATION_SCHEMA) + + # that's actually bad design, but in order to validate built-in triggers we need to know which are set + Validate.schema_insert(architecture, configuration, root, "remote-pull", lambda _: GITREMOTE_REMOTE_PULL_SCHEMA) + Validate.schema_insert(architecture, configuration, root, "remote-push", lambda _: GITREMOTE_REMOTE_PUSH_SCHEMA) + + report_schemas = { + "console": REPORT_CONSOLE_SCHEMA, + "email": REPORT_EMAIL_SCHEMA, + "html": REPORT_HTML_SCHEMA, + "telegram": REPORT_TELEGRAM_SCHEMA, + } + for schema_name, schema in report_schemas.items(): + root[schema_name] = Validate.schema_erase_required(copy.deepcopy(schema)) + Validate.schema_insert(architecture, configuration, root, "report", report_schemas.get) + + upload_schemas = { + "github": UPLOAD_GITHUB_SCHEMA, + "rsync": UPLOAD_RSYNC_SCHEMA, + "s3": UPLOAD_S3_SCHEMA, + } + for schema_name, schema in upload_schemas.items(): + root[schema_name] = Validate.schema_erase_required(copy.deepcopy(schema)) + Validate.schema_insert(architecture, configuration, root, "upload", upload_schemas.get) + + return root + + @staticmethod + def schema_erase_required(schema: Dict[str, Any]) -> Dict[str, Any]: + """ + recursively remove required field from supplied cerberus schema + + Args: + schema(Dict[str, Any]): source schema from which required field must be removed + + Returns: + Dict[str, Any]: schema without required fields + """ + schema.pop("required", None) + for value in filter(lambda v: isinstance(v, dict), schema.values()): + Validate.schema_erase_required(value) + return schema + + @staticmethod + def schema_insert(architecture: str, configuration: Configuration, root: Dict[str, Any], root_section: str, + schema_mapping: Callable[[str], Optional[Dict[str, Any]]]) -> Dict[str, Any]: + """ + insert child schema into the root schema based on mapping rules + + Notes: + Actually it is a bad design, because we are reading triggers configuration from parsers which (basically) + don't know anything about triggers. But in order to validate built-in triggers we need to know which are set + + Args: + architecture(str): repository architecture + configuration(Configuration): configuration instance + root(Dict[str, Any]): root schema in which child schema will be inserted + root_section(str): section name in root schema + schema_mapping(Callable[[str], Optional[Dict[str, Any]]]): extractor for child schema based on trigger type + + Returns: + Dict[str, Any]: modified root schema. Note, however, that schema will be modified in place + """ + if not configuration.has_section(root_section): + return root + + targets = configuration.getlist(root_section, "target", fallback=[]) + for target in targets: + section, schema_name = configuration.gettype(target, architecture) + if (schema := schema_mapping(schema_name)) is not None: + root[section] = copy.deepcopy(schema) + + return root diff --git a/src/ahriman/core/configuration/__init__.py b/src/ahriman/core/configuration/__init__.py new file mode 100644 index 00000000..3abc5d5b --- /dev/null +++ b/src/ahriman/core/configuration/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from ahriman.core.configuration.configuration import Configuration diff --git a/src/ahriman/core/configuration.py b/src/ahriman/core/configuration/configuration.py similarity index 97% rename from src/ahriman/core/configuration.py rename to src/ahriman/core/configuration/configuration.py index 38f204fc..21fb744b 100644 --- a/src/ahriman/core/configuration.py +++ b/src/ahriman/core/configuration/configuration.py @@ -24,7 +24,7 @@ import shlex import sys from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Type +from typing import Any, Callable, Dict, List, Optional, Tuple, Type from ahriman.core.exceptions import InitializeError from ahriman.models.repository_paths import RepositoryPaths @@ -63,6 +63,7 @@ class Configuration(configparser.RawConfigParser): ARCHITECTURE_SPECIFIC_SECTIONS = ["build", "sign", "web"] SYSTEM_CONFIGURATION_PATH = Path(sys.prefix) / "share" / "ahriman" / "settings" / "ahriman.ini" + converters: Dict[str, Callable[[str], Any]] # typing guard def __init__(self, allow_no_value: bool = False) -> None: """ @@ -74,7 +75,7 @@ class Configuration(configparser.RawConfigParser): """ configparser.RawConfigParser.__init__(self, allow_no_value=allow_no_value, converters={ "list": shlex.split, - "path": self.__convert_path, + "path": self._convert_path, }) self.architecture: Optional[str] = None self.path: Optional[Path] = None @@ -141,7 +142,7 @@ class Configuration(configparser.RawConfigParser): """ return f"{section}:{suffix}" - def __convert_path(self, value: str) -> Path: + def _convert_path(self, value: str) -> Path: """ convert string value to path object diff --git a/src/ahriman/core/configuration/schema.py b/src/ahriman/core/configuration/schema.py new file mode 100644 index 00000000..34fc7e5e --- /dev/null +++ b/src/ahriman/core/configuration/schema.py @@ -0,0 +1,554 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# pylint: disable=too-many-lines +__all__ = [ + "CONFIGURATION_SCHEMA", + "GITREMOTE_REMOTE_PULL_SCHEMA", "GITREMOTE_REMOTE_PUSH_SCHEMA", + "REPORT_CONSOLE_SCHEMA", "REPORT_EMAIL_SCHEMA", "REPORT_HTML_SCHEMA", "REPORT_TELEGRAM_SCHEMA", + "UPLOAD_GITHUB_SCHEMA", "UPLOAD_RSYNC_SCHEMA", "UPLOAD_S3_SCHEMA", +] + + +CONFIGURATION_SCHEMA = { + "settings": { + "type": "dict", + "schema": { + "include": { + "type": "path", + "coerce": "absolute_path", + "required": True, + "path_exists": True, + }, + "database": { + "type": "path", + "coerce": "absolute_path", + "required": True, + }, + "logging": { + "type": "path", + "coerce": "absolute_path", + "required": True, + "path_exists": True, + }, + }, + }, + "alpm": { + "type": "dict", + "schema": { + "database": { + "type": "path", + "coerce": "absolute_path", + "required": True, + }, + "mirror": { + "type": "string", + "required": True, + }, + "repositories": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + "required": True, + "empty": False, + }, + "root": { + "type": "path", + "coerce": "absolute_path", + "required": True, + "path_exists": True, + }, + "use_ahriman_cache": { + "type": "boolean", + "coerce": "boolean", + "required": True, + }, + }, + }, + "auth": { + "type": "dict", + "schema": { + "target": { + "type": "string", + "oneof": [ + {"allowed": ["disabled"]}, + {"allowed": ["configuration", "mapping"], "dependencies": ["salt"]}, + {"allowed": ["oauth"], "dependencies": [ + "client_id", "client_secret", "oauth_provider", "oauth_scopes", "salt" + ]}, + ], + }, + "allow_read_only": { + "type": "boolean", + "coerce": "boolean", + "required": True, + }, + "client_id": { + "type": "string", + }, + "client_secret": { + "type": "string", + }, + "max_age": { + "type": "integer", + "coerce": "integer", + }, + "oauth_provider": { + "type": "string", + }, + "oauth_scopes": { + "type": "string", + }, + "salt": { + "type": "string", + }, + }, + }, + "build": { + "type": "dict", + "schema": { + "archbuild_flags": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + }, + "build_command": { + "type": "string", + "required": True, + }, + "ignore_packages": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + }, + "makepkg_flags": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + }, + "makechrootpkg_flags": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + }, + "triggers": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + }, + "vcs_allowed_age": { + "type": "integer", + "coerce": "integer", + }, + }, + }, + "repository": { + "type": "dict", + "schema": { + "name": { + "type": "string", + "required": True, + }, + "root": { + "type": "string", + "required": True, + }, + }, + }, + "sign": { + "type": "dict", + "allow_unknown": True, + "keysrules": { + "type": "string", + "anyof_regex": ["^target$", "^key$", "^key_.*"], + }, + "schema": { + "target": { + "type": "list", + "coerce": "list", + "oneof": [ + {"allowed": []}, + {"allowed": ["package", "repository"], "dependencies": ["key"]}, + ], + }, + "key": { + "type": "string", + }, + }, + }, + "web": { + "type": "dict", + "schema": { + "address": { + "type": "string", + }, + "debug": { + "type": "boolean", + "coerce": "boolean", + }, + "debug_check_host": { + "type": "boolean", + "coerce": "boolean", + }, + "debug_allowed_hosts": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + }, + "host": { + "type": "string", + }, + "index_url": { + "type": "string", + }, + "password": { + "type": "string", + }, + "port": { + "type": "integer", + "coerce": "integer", + "min": 0, + "max": 65535, + }, + "static_path": { + "type": "path", + "coerce": "absolute_path", + "required": True, + "path_exists": True, + }, + "templates": { + "type": "path", + "coerce": "absolute_path", + "required": True, + "path_exists": True, + }, + "unix_socket": { + "type": "path", + "coerce": "absolute_path", + }, + "unix_socket_unsafe": { + "type": "boolean", + "coerce": "boolean", + }, + "username": { + "type": "string", + }, + }, + }, + "remote-pull": { + "type": "dict", + "schema": { + "target": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + }, + }, + }, + "remote-push": { + "type": "dict", + "schema": { + "target": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + }, + }, + }, + "report": { + "type": "dict", + "schema": { + "target": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + }, + }, + }, + "upload": { + "type": "dict", + "schema": { + "target": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + }, + }, + }, +} + + +GITREMOTE_REMOTE_PULL_SCHEMA = { + "type": "dict", + "schema": { + "pull_url": { + "type": "string", + "required": True, + }, + "pull_branch": { + "type": "string", + }, + }, +} + + +GITREMOTE_REMOTE_PUSH_SCHEMA = { + "type": "dict", + "schema": { + "commit_author": { + "type": "string", + }, + "push_url": { + "type": "string", + "required": True, + }, + "push_branch": { + "type": "string", + }, + }, +} + + +REPORT_CONSOLE_SCHEMA = { + "type": "dict", + "schema": { + "type": { + "type": "string", + "allowed": ["console"], + }, + "use_utf": { + "type": "boolean", + "coerce": "boolean", + }, + }, +} + + +REPORT_EMAIL_SCHEMA = { + "type": "dict", + "schema": { + "type": { + "type": "string", + "allowed": ["email"], + }, + "full_template_path": { + "type": "path", + "coerce": "absolute_path", + "path_exists": True, + }, + "homepage": { + "type": "string", + }, + "host": { + "type": "string", + "required": True, + }, + "link_path": { + "type": "string", + "required": True, + }, + "no_empty_report": { + "type": "boolean", + "coerce": "boolean", + }, + "password": { + "type": "string", + }, + "port": { + "type": "integer", + "coerce": "integer", + "required": True, + }, + "receivers": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + "required": True, + "empty": False, + }, + "sender": { + "type": "string", + "required": True, + }, + "ssl": { + "type": "string", + "allowed": ["ssl", "starttls", "disabled"], + }, + "template_path": { + "type": "path", + "coerce": "absolute_path", + "required": True, + "path_exists": True, + }, + "user": { + "type": "string", + }, + }, +} + + +REPORT_HTML_SCHEMA = { + "type": "dict", + "schema": { + "type": { + "type": "string", + "allowed": ["html"], + }, + "homepage": { + "type": "string", + }, + "link_path": { + "type": "string", + "required": True, + }, + "path": { + "type": "path", + "coerce": "absolute_path", + "required": True, + }, + "template_path": { + "type": "path", + "coerce": "absolute_path", + "required": True, + "path_exists": True, + }, + }, +} + + +REPORT_TELEGRAM_SCHEMA = { + "type": "dict", + "schema": { + "type": { + "type": "string", + "allowed": ["telegram"], + }, + "api_key": { + "type": "string", + "required": True, + }, + "chat_id": { + "type": "string", + "required": True, + }, + "homepage": { + "type": "string", + }, + "link_path": { + "type": "string", + "required": True, + }, + "template_path": { + "type": "path", + "coerce": "absolute_path", + "required": True, + "path_exists": True, + }, + "template_type": { + "type": "string", + "allowed": ["MarkdownV2", "HTML", "Markdown"], + }, + "timeout": { + "type": "integer", + "coerce": "integer", + }, + }, +} + + +UPLOAD_GITHUB_SCHEMA = { + "type": "dict", + "schema": { + "type": { + "type": "string", + "allowed": ["github"], + }, + "owner": { + "type": "string", + "required": True, + }, + "password": { + "type": "string", + "required": True, + }, + "repository": { + "type": "string", + "required": True, + }, + "timeout": { + "type": "integer", + "coerce": "integer", + }, + "username": { + "type": "string", + }, + }, +} + +UPLOAD_RSYNC_SCHEMA = { + "type": "dict", + "schema": { + "type": { + "type": "string", + "allowed": ["rsync"], + }, + "command": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + "required": True, + "empty": False, + }, + "remote": { + "type": "string", + "required": True, + }, + }, +} + + +UPLOAD_S3_SCHEMA = { + "type": "dict", + "schema": { + "type": { + "type": "string", + "allowed": ["s3"], + }, + "access_key": { + "type": "string", + "required": True, + }, + "bucket": { + "type": "string", + "required": True, + }, + "chunk_size": { + "type": "integer", + "coerce": "integer", + }, + "region": { + "type": "string", + "required": True, + }, + "secret_key": { + "type": "string", + "required": True, + }, + }, +} diff --git a/src/ahriman/core/configuration/validator.py b/src/ahriman/core/configuration/validator.py new file mode 100644 index 00000000..ca828b80 --- /dev/null +++ b/src/ahriman/core/configuration/validator.py @@ -0,0 +1,116 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from cerberus import TypeDefinition, Validator as RootValidator # type: ignore +from pathlib import Path +from typing import Any, List + +from ahriman.core.configuration import Configuration + + +class Validator(RootValidator): # type: ignore + """ + class which defines custom validation methods for the service configuration + + Attributes: + instance(Configuration): configuration instance + """ + + types_mapping = RootValidator.types_mapping.copy() + types_mapping["path"] = TypeDefinition("path", (Path,), ()) + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """ + default constructor + + Args: + instance(Configuration): configuration instance used for extraction + *args(Any): positional arguments to be passed to base validator + **kwargs(): keyword arguments to be passed to base validator + """ + RootValidator.__init__(self, *args, **kwargs) + self.instance: Configuration = kwargs["instance"] + + def _normalize_coerce_absolute_path(self, value: str) -> Path: + """ + extract path from string value + + Args: + value(str): converting value + + Returns: + Path: value converted to path instance according to configuration rules + """ + converted: Path = self.instance.converters["path"](value) + return converted + + def _normalize_coerce_boolean(self, value: str) -> bool: + """ + extract boolean from string value + + Args: + value(str): converting value + + Returns: + bool: value converted to boolean according to configuration rules + """ + # pylint: disable=protected-access + converted: bool = self.instance._convert_to_boolean(value) # type: ignore + return converted + + def _normalize_coerce_integer(self, value: str) -> int: + """ + extract integer from string value + + Args: + value(str): converting value + + Returns: + int: value converted to int according to configuration rules + """ + return int(value) + + def _normalize_coerce_list(self, value: str) -> List[str]: + """ + extract string list from string value + + Args: + value(str): converting value + + Returns: + List[str]: value converted to string list instance according to configuration rules + """ + converted: List[str] = self.instance.converters["list"](value) + return converted + + def _validate_path_exists(self, constraint: bool, field: str, value: Path) -> None: + """ + check if paths exists + + Args: + constraint(bool): True in case if path must exist and False otherwise + field(str): field name to be checked + value(Path): value to be checked + + Examples: + The rule's arguments are validated against this schema: + {"type": "boolean"} + """ + if constraint and not value.exists(): + self._error(field, f"Path {value} must exist") diff --git a/src/ahriman/core/formatters/__init__.py b/src/ahriman/core/formatters/__init__.py index f4f1f364..82b646e5 100644 --- a/src/ahriman/core/formatters/__init__.py +++ b/src/ahriman/core/formatters/__init__.py @@ -27,6 +27,7 @@ from ahriman.core.formatters.package_printer import PackagePrinter from ahriman.core.formatters.patch_printer import PatchPrinter from ahriman.core.formatters.status_printer import StatusPrinter from ahriman.core.formatters.tree_printer import TreePrinter +from ahriman.core.formatters.validation_printer import ValidationPrinter from ahriman.core.formatters.update_printer import UpdatePrinter from ahriman.core.formatters.user_printer import UserPrinter from ahriman.core.formatters.version_printer import VersionPrinter diff --git a/src/ahriman/core/formatters/printer.py b/src/ahriman/core/formatters/printer.py index ab090436..ab517f65 100644 --- a/src/ahriman/core/formatters/printer.py +++ b/src/ahriman/core/formatters/printer.py @@ -41,7 +41,8 @@ class Printer: for prop in self.properties(): if not verbose and not prop.is_required: continue - log_fn(f"\t{prop.name}{separator}{prop.value}") + indent = "\t" * prop.indent + log_fn(f"{indent}{prop.name}{separator}{prop.value}") def properties(self) -> List[Property]: """ diff --git a/src/ahriman/core/formatters/validation_printer.py b/src/ahriman/core/formatters/validation_printer.py new file mode 100644 index 00000000..21d0e2ee --- /dev/null +++ b/src/ahriman/core/formatters/validation_printer.py @@ -0,0 +1,77 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from typing import Any, Dict, Generator, List, Union + +from ahriman.core.formatters import StringPrinter +from ahriman.models.property import Property + + +class ValidationPrinter(StringPrinter): + """ + print content of the validation errors + + Attributes: + node(str): root level name + errors(List[Union[str, Dict[str, Any]]]): validation errors + """ + + def __init__(self, node: str, errors: List[Union[str, Dict[str, Any]]]) -> None: + """ + default constructor + + Args: + node(str): root level name + errors(List[Union[str, Dict[str, Any]]]): validation errors + """ + StringPrinter.__init__(self, node) + self.node = node + self.errors = errors + + @staticmethod + def get_error_messages(node: str, errors: List[Union[str, Dict[str, Any]]], + current_level: int = 1) -> Generator[Property, None, None]: + """ + extract default error message from cerberus class + + Args: + node(str): current node level name + errors(List[Union[str, Dict[str, Any]]]): current node validation errors + current_level(int, optional): current level number (Default value = 1) + + Yields: + Property: error messages from error tree + """ + for error in errors: + if not isinstance(error, str): # child nodes errors + for child_node, child_errors in error.items(): + # increase indentation instead of nodes concatenations + # sometimes it is not only nodes, but rules themselves + yield from ValidationPrinter.get_error_messages(child_node, child_errors, current_level + 1) + else: # current node errors + yield Property(node, error, is_required=True, indent=current_level) + + def properties(self) -> List[Property]: + """ + convert content into printable data + + Returns: + List[Property]: list of content properties + """ + return list(self.get_error_messages(self.node, self.errors)) diff --git a/src/ahriman/models/property.py b/src/ahriman/models/property.py index 269b03a5..b916f421 100644 --- a/src/ahriman/models/property.py +++ b/src/ahriman/models/property.py @@ -30,8 +30,10 @@ class Property: name(str): name of the property value(Any): property value is_required(bool): if set to True then this property is required + indent(int): property indentation level """ name: str value: Any is_required: bool = field(default=False, kw_only=True) + indent: int = 1 diff --git a/tests/ahriman/application/handlers/test_handler_dump.py b/tests/ahriman/application/handlers/test_handler_dump.py index d046ebfd..dcec01e6 100644 --- a/tests/ahriman/application/handlers/test_handler_dump.py +++ b/tests/ahriman/application/handlers/test_handler_dump.py @@ -4,15 +4,12 @@ from pytest_mock import MockerFixture from ahriman.application.handlers import Dump from ahriman.core.configuration import Configuration -from ahriman.core.repository import Repository -def test_run(args: argparse.Namespace, configuration: Configuration, repository: Repository, - mocker: MockerFixture) -> None: +def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None: """ must run command """ - mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) print_mock = mocker.patch("ahriman.core.formatters.Printer.print") application_mock = mocker.patch("ahriman.core.configuration.Configuration.dump", return_value=configuration.dump()) diff --git a/tests/ahriman/application/handlers/test_handler_validate.py b/tests/ahriman/application/handlers/test_handler_validate.py new file mode 100644 index 00000000..bc92097f --- /dev/null +++ b/tests/ahriman/application/handlers/test_handler_validate.py @@ -0,0 +1,104 @@ +import argparse +import json + +from pytest_mock import MockerFixture + +from ahriman.application.handlers import Validate +from ahriman.core.configuration import Configuration +from ahriman.core.configuration.schema import CONFIGURATION_SCHEMA, GITREMOTE_REMOTE_PULL_SCHEMA +from ahriman.core.configuration.validator import Validator + + +def _default_args(args: argparse.Namespace) -> argparse.Namespace: + """ + default arguments for these test cases + + Args: + args(argparse.Namespace): command line arguments fixture + + Returns: + argparse.Namespace: generated arguments for these test cases + """ + args.exit_code = False + return args + + +def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None: + """ + must run command + """ + args = _default_args(args) + mocker.patch.object(Validator, "errors", {"node": ["error"]}) + print_mock = mocker.patch("ahriman.core.formatters.Printer.print") + application_mock = mocker.patch("ahriman.core.configuration.validator.Validator.validate", return_value=False) + + Validate.run(args, "x86_64", configuration, report=False, unsafe=False) + + application_mock.assert_called_once_with(configuration.dump()) + print_mock.assert_called_once_with(verbose=True) + + +def test_run_skip(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None: + """ + must skip print if no errors found + """ + args = _default_args(args) + mocker.patch("ahriman.core.configuration.validator.Validator.validate", return_value=True) + print_mock = mocker.patch("ahriman.core.formatters.Printer.print") + + Validate.run(args, "x86_64", configuration, report=False, unsafe=False) + print_mock.assert_not_called() + + +def test_schema(configuration: Configuration) -> None: + """ + must generate full schema correctly + """ + schema = Validate.schema("x86_64", configuration) + + # defaults + assert schema.pop("console") + assert schema.pop("email") + assert schema.pop("github") + assert schema.pop("gitremote") + assert schema.pop("html") + assert schema.pop("rsync") + assert schema.pop("s3") + assert schema.pop("telegram") + + assert schema == CONFIGURATION_SCHEMA + + +def test_schema_erase_required() -> None: + """ + must remove required field from dictionaries recursively + """ + # the easiest way is to just dump to string and check + assert "required" not in json.dumps(Validate.schema_erase_required(CONFIGURATION_SCHEMA)) + + +def test_schema_insert(configuration: Configuration) -> None: + """ + must insert child schema to root + """ + result = Validate.schema_insert("x86_64", configuration, CONFIGURATION_SCHEMA, "remote-pull", + lambda _: GITREMOTE_REMOTE_PULL_SCHEMA) + assert result["gitremote"] == GITREMOTE_REMOTE_PULL_SCHEMA + + +def test_schema_insert_skip(configuration: Configuration) -> None: + """ + must do nothing in case if there is no such section or option + """ + configuration.remove_section("remote-pull") + + result = Validate.schema_insert("x86_64", configuration, CONFIGURATION_SCHEMA, "remote-pull", + lambda _: GITREMOTE_REMOTE_PULL_SCHEMA) + assert result == CONFIGURATION_SCHEMA + + +def test_disallow_auto_architecture_run() -> None: + """ + must not allow multi architecture run + """ + assert not Validate.ALLOW_AUTO_ARCHITECTURE_RUN diff --git a/tests/ahriman/application/test_ahriman.py b/tests/ahriman/application/test_ahriman.py index 70a89133..1fcde7eb 100644 --- a/tests/ahriman/application/test_ahriman.py +++ b/tests/ahriman/application/test_ahriman.py @@ -413,6 +413,18 @@ def test_subparsers_repo_config(parser: argparse.ArgumentParser) -> None: assert args.unsafe +def test_subparsers_repo_config_validate(parser: argparse.ArgumentParser) -> None: + """ + repo-config-validate command must imply lock, report, quiet and unsafe + """ + args = parser.parse_args(["-a", "x86_64", "repo-config-validate"]) + assert args.architecture == ["x86_64"] + assert args.lock is None + assert not args.report + assert args.quiet + assert args.unsafe + + def test_subparsers_repo_rebuild_architecture(parser: argparse.ArgumentParser) -> None: """ repo-rebuild command must correctly parse architecture list diff --git a/tests/ahriman/core/configuration/conftest.py b/tests/ahriman/core/configuration/conftest.py new file mode 100644 index 00000000..4cbc47b6 --- /dev/null +++ b/tests/ahriman/core/configuration/conftest.py @@ -0,0 +1,19 @@ +import pytest + +from ahriman.core.configuration import Configuration +from ahriman.core.configuration.schema import CONFIGURATION_SCHEMA +from ahriman.core.configuration.validator import Validator + + +@pytest.fixture +def validator(configuration: Configuration) -> Validator: + """ + fixture for validator + + Args: + configuration(Configuration): configuration fixture + + Returns: + Validator: validator test instance + """ + return Validator(instance=configuration, schema=CONFIGURATION_SCHEMA) diff --git a/tests/ahriman/core/test_configuration.py b/tests/ahriman/core/configuration/test_configuration.py similarity index 100% rename from tests/ahriman/core/test_configuration.py rename to tests/ahriman/core/configuration/test_configuration.py diff --git a/tests/ahriman/core/configuration/test_schema.py b/tests/ahriman/core/configuration/test_schema.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/ahriman/core/configuration/test_validator.py b/tests/ahriman/core/configuration/test_validator.py new file mode 100644 index 00000000..3ff52daf --- /dev/null +++ b/tests/ahriman/core/configuration/test_validator.py @@ -0,0 +1,70 @@ +from pathlib import Path +from pytest_mock import MockerFixture +from unittest.mock import MagicMock + +from ahriman.core.configuration.validator import Validator + + +def test_types_mapping() -> None: + """ + must set custom types + """ + assert "path" in Validator.types_mapping + assert Path in Validator.types_mapping["path"].included_types + + +def test_normalize_coerce_absolute_path(validator: Validator) -> None: + """ + must convert string value to path by using configuration converters + """ + convert_mock = MagicMock() + validator.instance.converters["path"] = convert_mock + + validator._normalize_coerce_absolute_path("value") + convert_mock.assert_called_once_with("value") + + +def test_normalize_coerce_boolean(validator: Validator, mocker: MockerFixture) -> None: + """ + must convert string value to boolean by using configuration converters + """ + convert_mock = mocker.patch("ahriman.core.configuration.Configuration._convert_to_boolean") + validator._normalize_coerce_boolean("1") + convert_mock.assert_called_once_with("1") + + +def test_normalize_coerce_integer(validator: Validator) -> None: + """ + must convert string value to integer by using configuration converters + """ + assert validator._normalize_coerce_integer("1") == 1 + assert validator._normalize_coerce_integer("42") == 42 + + +def test_normalize_coerce_list(validator: Validator) -> None: + """ + must convert string value to list by using configuration converters + """ + convert_mock = MagicMock() + validator.instance.converters["list"] = convert_mock + + validator._normalize_coerce_list("value") + convert_mock.assert_called_once_with("value") + + +def test_validate_path_exists(validator: Validator, mocker: MockerFixture) -> None: + """ + must validate that paths exists + """ + error_mock = mocker.patch("ahriman.core.configuration.validator.Validator._error") + + mocker.patch("pathlib.Path.exists", return_value=False) + validator._validate_path_exists(False, "field", Path("1")) + + mocker.patch("pathlib.Path.exists", return_value=False) + validator._validate_path_exists(True, "field", Path("2")) + + mocker.patch("pathlib.Path.exists", return_value=True) + validator._validate_path_exists(True, "field", Path("3")) + + error_mock.assert_called_once_with("field", "Path 2 must exist") diff --git a/tests/ahriman/core/formatters/conftest.py b/tests/ahriman/core/formatters/conftest.py index f3d4b37f..d37490d5 100644 --- a/tests/ahriman/core/formatters/conftest.py +++ b/tests/ahriman/core/formatters/conftest.py @@ -1,7 +1,7 @@ import pytest from ahriman.core.formatters import AurPrinter, ConfigurationPrinter, PackagePrinter, PatchPrinter, StatusPrinter, \ - StringPrinter, TreePrinter, UpdatePrinter, UserPrinter, VersionPrinter + StringPrinter, TreePrinter, UpdatePrinter, UserPrinter, ValidationPrinter, VersionPrinter from ahriman.models.aur_package import AURPackage from ahriman.models.build_status import BuildStatus from ahriman.models.package import Package @@ -126,6 +126,29 @@ def user_printer(user: User) -> UserPrinter: return UserPrinter(user) +@pytest.fixture +def validation_printer() -> ValidationPrinter: + """ + fixture for validation printer + + Returns: + ValidationPrinter: validation printer test instance + """ + return ValidationPrinter("root", [ + "root error", + { + "child": [ + "child error", + { + "grandchild": [ + "grandchild error", + ], + }, + ], + }, + ]) + + @pytest.fixture def version_printer(package_ahriman: Package) -> VersionPrinter: """ diff --git a/tests/ahriman/core/formatters/test_printer.py b/tests/ahriman/core/formatters/test_printer.py index 530dccac..ee1bed0b 100644 --- a/tests/ahriman/core/formatters/test_printer.py +++ b/tests/ahriman/core/formatters/test_printer.py @@ -1,7 +1,9 @@ -from unittest.mock import MagicMock +from pytest_mock import MockerFixture +from unittest.mock import MagicMock, call as MockCall from ahriman.core.formatters import PackagePrinter from ahriman.core.formatters import Printer +from ahriman.models.property import Property def test_print(package_ahriman_printer: PackagePrinter) -> None: @@ -31,6 +33,24 @@ def test_print_verbose(package_ahriman_printer: PackagePrinter) -> None: log_mock.assert_called() +def test_print_indent(mocker: MockerFixture) -> None: + """ + must correctly use indentation + """ + log_mock = MagicMock() + + mocker.patch("ahriman.core.formatters.Printer.properties", return_value=[Property("key", "value", indent=0)]) + Printer().print(verbose=True, log_fn=log_mock) + + mocker.patch("ahriman.core.formatters.Printer.properties", return_value=[Property("key", "value", indent=1)]) + Printer().print(verbose=True, log_fn=log_mock) + + mocker.patch("ahriman.core.formatters.Printer.properties", return_value=[Property("key", "value", indent=2)]) + Printer().print(verbose=True, log_fn=log_mock) + + log_mock.assert_has_calls([MockCall("key: value"), MockCall("\tkey: value"), MockCall("\t\tkey: value")]) + + def test_properties() -> None: """ must return empty properties list diff --git a/tests/ahriman/core/formatters/test_validation_printer.py b/tests/ahriman/core/formatters/test_validation_printer.py new file mode 100644 index 00000000..8c37c6c3 --- /dev/null +++ b/tests/ahriman/core/formatters/test_validation_printer.py @@ -0,0 +1,28 @@ +from ahriman.core.formatters import ValidationPrinter +from ahriman.models.property import Property + + +def test_properties(validation_printer: ValidationPrinter) -> None: + """ + must return non-empty properties list + """ + assert validation_printer.properties() + + +def test_title(validation_printer: ValidationPrinter) -> None: + """ + must return non-empty title + """ + assert validation_printer.title() is not None + + +def test_get_error_messages(validation_printer: ValidationPrinter) -> None: + """ + must get error messages from plain list + """ + result = ValidationPrinter.get_error_messages(validation_printer.node, validation_printer.errors) + assert list(result) == [ + Property("root", "root error", is_required=True, indent=1), + Property("child", "child error", is_required=True, indent=2), + Property("grandchild", "grandchild error", is_required=True, indent=3), + ]