diff --git a/docs/ahriman.1 b/docs/ahriman.1 index 80acdaee..b58ba2d2 100644 --- a/docs/ahriman.1 +++ b/docs/ahriman.1 @@ -3,7 +3,7 @@ ahriman .SH SYNOPSIS .B ahriman -[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--report | --no-report] [-q] [--unsafe] [-V] {aur-search,search,help,help-commands-unsafe,help-updates,help-version,version,package-add,add,package-update,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,patch-set-add,repo-backup,repo-check,check,repo-daemon,daemon,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-sign,sign,repo-status-update,repo-sync,sync,repo-tree,repo-triggers,repo-update,update,service-clean,clean,repo-clean,service-config,config,repo-config,service-config-validate,config-validate,repo-config-validate,service-key-import,key-import,service-setup,init,repo-init,repo-setup,setup,service-shell,shell,user-add,user-list,user-remove,web} ... +[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--report | --no-report] [-q] [--unsafe] [-V] {aur-search,search,help,help-commands-unsafe,help-updates,help-version,version,package-add,add,package-update,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,patch-set-add,repo-backup,repo-check,check,repo-create-keyring,repo-create-mirrorlist,repo-daemon,daemon,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-sign,sign,repo-status-update,repo-sync,sync,repo-tree,repo-triggers,repo-update,update,service-clean,clean,repo-clean,service-config,config,repo-config,service-config-validate,config-validate,repo-config-validate,service-key-import,key-import,service-setup,init,repo-init,repo-setup,setup,service-shell,shell,user-add,user-list,user-remove,web} ... .SH DESCRIPTION ArcH linux ReposItory MANager @@ -91,6 +91,12 @@ backup repository data \fBahriman\fR \fI\,repo\-check\/\fR check for updates .TP +\fBahriman\fR \fI\,repo\-create\-keyring\/\fR +create keyring package +.TP +\fBahriman\fR \fI\,repo\-create\-mirrorlist\/\fR +create mirrorlist package +.TP \fBahriman\fR \fI\,repo\-daemon\/\fR run application as daemon .TP @@ -400,6 +406,16 @@ fetch actual version of VCS packages \fB\-y\fR, \fB\-\-refresh\fR download fresh package databases from the mirror before actions, \-yy to force refresh even if up to date +.SH COMMAND \fI\,'ahriman repo\-create\-keyring'\/\fR +usage: ahriman repo\-create\-keyring [\-h] + +create package which contains list of trusted keys as set by configuration. Note, that this action will only create package, the package itself has to be built manually + +.SH COMMAND \fI\,'ahriman repo\-create\-mirrorlist'\/\fR +usage: ahriman repo\-create\-mirrorlist [\-h] + +create package which contains list of available mirrors as set by configuration. Note, that this action will only create package, the package itself has to be built manually + .SH COMMAND \fI\,'ahriman repo\-daemon'\/\fR usage: ahriman repo\-daemon [\-h] [\-i INTERVAL] [\-\-aur | \-\-no\-aur] [\-\-dependencies | \-\-no\-dependencies] [\-\-local | \-\-no\-local] [\-\-manual | \-\-no\-manual] [\-\-vcs | \-\-no\-vcs] [\-y] diff --git a/docs/ahriman.core.rst b/docs/ahriman.core.rst index 580b2315..f11f1231 100644 --- a/docs/ahriman.core.rst +++ b/docs/ahriman.core.rst @@ -19,6 +19,7 @@ Subpackages ahriman.core.repository ahriman.core.sign ahriman.core.status + ahriman.core.support ahriman.core.triggers ahriman.core.upload diff --git a/docs/ahriman.core.support.pkgbuild.rst b/docs/ahriman.core.support.pkgbuild.rst new file mode 100644 index 00000000..ca8d5d99 --- /dev/null +++ b/docs/ahriman.core.support.pkgbuild.rst @@ -0,0 +1,37 @@ +ahriman.core.support.pkgbuild package +===================================== + +Submodules +---------- + +ahriman.core.support.pkgbuild.keyring\_generator module +------------------------------------------------------- + +.. automodule:: ahriman.core.support.pkgbuild.keyring_generator + :members: + :no-undoc-members: + :show-inheritance: + +ahriman.core.support.pkgbuild.mirrorlist\_generator module +---------------------------------------------------------- + +.. automodule:: ahriman.core.support.pkgbuild.mirrorlist_generator + :members: + :no-undoc-members: + :show-inheritance: + +ahriman.core.support.pkgbuild.pkgbuild\_generator module +-------------------------------------------------------- + +.. automodule:: ahriman.core.support.pkgbuild.pkgbuild_generator + :members: + :no-undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: ahriman.core.support.pkgbuild + :members: + :no-undoc-members: + :show-inheritance: diff --git a/docs/ahriman.core.support.rst b/docs/ahriman.core.support.rst new file mode 100644 index 00000000..ebe39788 --- /dev/null +++ b/docs/ahriman.core.support.rst @@ -0,0 +1,45 @@ +ahriman.core.support package +============================ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + ahriman.core.support.pkgbuild + +Submodules +---------- + +ahriman.core.support.keyring\_trigger module +-------------------------------------------- + +.. automodule:: ahriman.core.support.keyring_trigger + :members: + :no-undoc-members: + :show-inheritance: + +ahriman.core.support.mirrorlist\_trigger module +----------------------------------------------- + +.. automodule:: ahriman.core.support.mirrorlist_trigger + :members: + :no-undoc-members: + :show-inheritance: + +ahriman.core.support.package\_creator module +-------------------------------------------- + +.. automodule:: ahriman.core.support.package_creator + :members: + :no-undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: ahriman.core.support + :members: + :no-undoc-members: + :show-inheritance: diff --git a/docs/completions/bash/_ahriman b/docs/completions/bash/_ahriman index 5b3170bc..efdc8c78 100644 --- a/docs/completions/bash/_ahriman +++ b/docs/completions/bash/_ahriman @@ -1,6 +1,6 @@ # AUTOMATICALLY GENERATED by `shtab` -_shtab_ahriman_subparsers=('aur-search' 'search' 'help' 'help-commands-unsafe' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'user-add' 'user-list' 'user-remove' 'web') +_shtab_ahriman_subparsers=('aur-search' 'search' 'help' 'help-commands-unsafe' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'user-add' 'user-list' 'user-remove' 'web') _shtab_ahriman_option_strings=('-h' '--help' '-a' '--architecture' '-c' '--configuration' '--force' '-l' '--lock' '--report' '--no-report' '-q' '--quiet' '--unsafe' '-V' '--version') _shtab_ahriman_aur_search_option_strings=('-h' '--help' '-e' '--exit-code' '--info' '--no-info' '--sort-by') @@ -27,6 +27,8 @@ _shtab_ahriman_patch_set_add_option_strings=('-h' '--help' '-t' '--track') _shtab_ahriman_repo_backup_option_strings=('-h' '--help') _shtab_ahriman_repo_check_option_strings=('-h' '--help' '-e' '--exit-code' '--vcs' '--no-vcs' '-y' '--refresh') _shtab_ahriman_check_option_strings=('-h' '--help' '-e' '--exit-code' '--vcs' '--no-vcs' '-y' '--refresh') +_shtab_ahriman_repo_create_keyring_option_strings=('-h' '--help') +_shtab_ahriman_repo_create_mirrorlist_option_strings=('-h' '--help') _shtab_ahriman_repo_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--dependencies' '--no-dependencies' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh') _shtab_ahriman_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--dependencies' '--no-dependencies' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh') _shtab_ahriman_repo_rebuild_option_strings=('-h' '--help' '--depends-on' '--dry-run' '--from-database' '-e' '--exit-code') @@ -70,7 +72,7 @@ _shtab_ahriman_web_option_strings=('-h' '--help') -_shtab_ahriman_pos_0_choices=('aur-search' 'search' 'help' 'help-commands-unsafe' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'user-add' 'user-list' 'user-remove' 'web') +_shtab_ahriman_pos_0_choices=('aur-search' 'search' 'help' 'help-commands-unsafe' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-create-keyring' 'repo-create-mirrorlist' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'user-add' 'user-list' 'user-remove' 'web') _shtab_ahriman_aur_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'submitter' 'url' 'url_path' 'version') _shtab_ahriman_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'submitter' 'url' 'url_path' 'version') _shtab_ahriman_package_add__s_choices=('auto' 'archive' 'aur' 'directory' 'local' 'remote' 'repository') @@ -230,6 +232,10 @@ _shtab_ahriman_check___vcs_nargs=0 _shtab_ahriman_check___no_vcs_nargs=0 _shtab_ahriman_check__y_nargs=0 _shtab_ahriman_check___refresh_nargs=0 +_shtab_ahriman_repo_create_keyring__h_nargs=0 +_shtab_ahriman_repo_create_keyring___help_nargs=0 +_shtab_ahriman_repo_create_mirrorlist__h_nargs=0 +_shtab_ahriman_repo_create_mirrorlist___help_nargs=0 _shtab_ahriman_repo_daemon__h_nargs=0 _shtab_ahriman_repo_daemon___help_nargs=0 _shtab_ahriman_repo_daemon___aur_nargs=0 diff --git a/docs/completions/zsh/_ahriman b/docs/completions/zsh/_ahriman index 5b56362d..f9e4e209 100644 --- a/docs/completions/zsh/_ahriman +++ b/docs/completions/zsh/_ahriman @@ -36,6 +36,8 @@ _shtab_ahriman_commands() { "repo-clean:remove local caches" "repo-config:dump configuration for the specified architecture" "repo-config-validate:validate configuration and print found errors" + "repo-create-keyring:create package which contains list of trusted keys as set by configuration. Note, that this action will only create package, the package itself has to be built manually" + "repo-create-mirrorlist:create package which contains list of available mirrors as set by configuration. Note, that this action will only create package, the package itself has to be built manually" "repo-daemon:start process which periodically will run update process" "repo-init:create initial service configuration, requires root" "repo-rebuild:force rebuild whole repository" @@ -303,6 +305,14 @@ _shtab_ahriman_repo_config_validate_options=( {-e,--exit-code}"[return non-zero exit status if configuration is invalid]" ) +_shtab_ahriman_repo_create_keyring_options=( + "(- : *)"{-h,--help}"[show this help message and exit]" +) + +_shtab_ahriman_repo_create_mirrorlist_options=( + "(- : *)"{-h,--help}"[show this help message and exit]" +) + _shtab_ahriman_repo_daemon_options=( "(- : *)"{-h,--help}"[show this help message and exit]" {-i,--interval}"[interval between runs in seconds]:interval:" @@ -595,6 +605,8 @@ _shtab_ahriman() { repo-clean) _arguments -C $_shtab_ahriman_repo_clean_options ;; repo-config) _arguments -C $_shtab_ahriman_repo_config_options ;; repo-config-validate) _arguments -C $_shtab_ahriman_repo_config_validate_options ;; + repo-create-keyring) _arguments -C $_shtab_ahriman_repo_create_keyring_options ;; + repo-create-mirrorlist) _arguments -C $_shtab_ahriman_repo_create_mirrorlist_options ;; repo-daemon) _arguments -C $_shtab_ahriman_repo_daemon_options ;; repo-init) _arguments -C $_shtab_ahriman_repo_init_options ;; repo-rebuild) _arguments -C $_shtab_ahriman_repo_rebuild_options ;; diff --git a/docs/configuration.rst b/docs/configuration.rst index 4568ce60..c57e80b5 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -30,8 +30,8 @@ Base configuration settings. * ``logging`` - path to logging configuration, string, required. Check ``logging.ini`` for reference. * ``suppress_http_log_errors`` - suppress http log errors, boolean, optional, default ``no``. If set to ``yes``, any http log errors (e.g. if web server is not available, but http logging is enabled) will be suppressed. -``alpm`` group --------------- +``alpm:*`` groups +----------------- libalpm and AUR related configuration. Group name can refer to architecture, e.g. ``alpm:x86_64`` can be used for x86_64 architecture specific settings. @@ -69,6 +69,7 @@ Build related configuration. Group name can refer to architecture, e.g. ``build: * ``makepkg_flags`` - additional flags passed to ``makepkg`` command, space separated list of strings, optional. * ``makechrootpkg_flags`` - additional flags passed to ``makechrootpkg`` command, space separated list of strings, optional. * ``triggers`` - list of ``ahriman.core.triggers.Trigger`` class implementation (e.g. ``ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger``) which will be loaded and run at the end of processing, space separated list of strings, optional. You can also specify triggers by their paths, e.g. ``/usr/lib/python3.10/site-packages/ahriman/core/report/report.py.ReportTrigger``. Triggers are run in the order of mention. +* ``triggers_known`` - optional list of ``ahriman.core.triggers.Trigger`` class implementations which are not run automatically and used only for trigger discovery and configuration validation. * ``vcs_allowed_age`` - maximal age in seconds of the VCS packages before their version will be updated with its remote source, int, optional, default ``604800``. ``repository`` group @@ -107,6 +108,41 @@ Web server settings. If any of ``host``/``port`` is not set, web integration wil * ``unix_socket_unsafe`` - set unsafe (o+w) permissions to unix socket, boolean, optional, default ``yes``. This option is enabled by default, because it is supposed that unix socket is created in safe environment (only web service is supposed to be used in unsafe), but it can be disabled by configuration. * ``username`` - username to authorize in web service in order to update service status, string, required in case if authorization enabled. +``keyring`` group +-------------------- + +Keyring package generator plugin. + +* ``target`` - list of generator settings sections, space separated list of strings, required. It must point to valid section name. + +Keyring generator plugin +^^^^^^^^^^^^^^^^^^^^^^^^ + +* ``description`` - keyring package description, string, optional, default is ``repo PGP keyring``, where ``repo`` is the repository name. +* ``homepage`` - url to homepage location if any, string, optional. +* ``license`` - list of licenses which are applied to this package, space separated list of strings, optional, default is ``Unlicense``. +* ``package`` - keyring package name, string, optional, default is ``repo-keyring``, where ``repo`` is the repository name. +* ``packagers`` - list of packagers keys, space separated list of strings, optional, if not set, the ``key_*`` options from ``sign`` group will be used. +* ``revoked`` - list of revoked packagers keys, space separated list of strings, optional. +* ``trusted`` - list of master keys, space separated list of strings, optional, if not set, the ``key`` option from ``sign`` group will be used. + +``mirrorlist`` group +-------------------- + +Mirrorlist package generator plugin. + +* ``target`` - list of generator settings sections, space separated list of strings, required. It must point to valid section name. + +Mirrorlist generator plugin +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* ``description`` - mirrorlist package description, string, optional, default is ``repo mirror list for use by pacman``, where ``repo`` is the repository name. +* ``homepage`` - url to homepage location if any, string, optional. +* ``license`` - list of licenses which are applied to this package, space separated list of strings, optional, default is ``Unlicense``. +* ``package`` - mirrorlist package name, string, optional, default is ``repo-mirrorlist``, where ``repo`` is the repository name. +* ``path`` - absolute path to generated mirrorlist file, string, optional, default is ``/etc/pacman.d/repo-mirrorlist``, where ``repo`` is the repository name. +* ``servers`` - list of repository mirrors, space separated list of strings, required. + ``remote-pull`` group --------------------- diff --git a/docs/faq.rst b/docs/faq.rst index 717625a7..5bf14b21 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -753,6 +753,80 @@ If you did everything fine you should receive the message with the next update. (replace ``${CHAT_ID}`` and ``${API_KEY}`` with the values from configuration). +Maintenance packages +-------------------- + +Generate keyring package +^^^^^^^^^^^^^^^^^^^^^^^^ + +The application provides special plugin which generates keyring package. This plugin heavily depends on ``sign`` group settings, however it is possible to override them. The minimal package can be generated in the following way: + +#. + Edit configuration: + + .. code-block:: ini + + [keyring] + target = keyring_generator + + By default it will use ``sign.key`` as trusted key and all other keys as packagers ones. For all available options refer to :doc:`configuration . + +#. + Create package source files: + + .. code-block:: shell + + sudo -u ahriman ahriman repo-create-keyring + + This command will generate PKGBUILD, revoked and trusted listings and keyring itself and will register the package in database. + +#. + Build new package as usual: + + .. code-block:: shell + + sudo -u ahriman ahriman package-add aur-clone-keyring --source local --now + + where ``aur-clone`` is your repository name. + +This plugin might have some issues, in case of any of them, kindly create `new issue `_. + +Generate mirrorlist package +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The application provides special plugin which generates mirrorlist package also. It is possible to distribute this package as usual later. The package can be generated in the following way: + +#. + Edit configuration: + + .. code-block:: ini + + [mirrorlist] + target = mirrorlist_generator + + [mirrorlist_generator] + servers = https://repo.example.com/$arch + + The ``mirrorlist_generator.servers`` must contain list of available mirrors, the ``$arch`` and ``$repo`` variables are supported. For more options kindly refer to :doc:`configuration . + +#. + Create package source files: + + .. code-block:: shell + + sudo -u ahriman ahriman repo-create-mirrorlist + + This command will generate PKGBUILD and mirrorlist file and will register the package in database. + +#. + Build new package as usual: + + .. code-block:: shell + + sudo -u ahriman ahriman package-add aur-clone-mirrorlist --source local --now + + where ``aur-clone`` is your repository name. + Web service ----------- diff --git a/docs/triggers.rst b/docs/triggers.rst index 8b452106..fb54e453 100644 --- a/docs/triggers.rst +++ b/docs/triggers.rst @@ -31,6 +31,16 @@ This trigger will be called right after build process (``on_result``). It will p Trigger which can be used for reporting. It implements ``on_result`` method and thus being called on each build update and generates report (e.g. html, telegram etc) according to the current settings. +``ahriman.core.support.KeyringTrigger`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Generator for keyring package. This trigger will extract keys from local keychain and pack them into keyring specific format. This trigger will generate sources including PKGBUILD, which can be used later for package building. + +``ahriman.core.support.MirrorlistTrigger`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Simple generator for mirrorlist package, based on the urls which were set by configuration. This trigger will generate sources including PKGBUILD, which can be used later for package building. + ``ahriman.core.upload.UploadTrigger`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/package/share/ahriman/settings/ahriman.ini b/package/share/ahriman/settings/ahriman.ini index e246a2f9..0588bdc6 100644 --- a/package/share/ahriman/settings/ahriman.ini +++ b/package/share/ahriman/settings/ahriman.ini @@ -25,6 +25,7 @@ ignore_packages = makechrootpkg_flags = makepkg_flags = --nocolor --ignorearch triggers = ahriman.core.gitremote.RemotePullTrigger ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger ahriman.core.gitremote.RemotePushTrigger +triggers_known = ahriman.core.support.KeyringTrigger ahriman.core.support.MirrorlistTrigger vcs_allowed_age = 604800 [repository] @@ -34,6 +35,12 @@ root = /var/lib/ahriman [sign] target = +[keyring] +target = + +[mirrorlist] +target = + [remote-pull] target = diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 3ace5315..4c8aae35 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -100,6 +100,8 @@ def _parser() -> argparse.ArgumentParser: _set_patch_set_add_parser(subparsers) _set_repo_backup_parser(subparsers) _set_repo_check_parser(subparsers) + _set_repo_create_keyring_parser(subparsers) + _set_repo_create_mirrorlist_parser(subparsers) _set_repo_daemon_parser(subparsers) _set_repo_rebuild_parser(subparsers) _set_repo_remove_unknown_parser(subparsers) @@ -478,6 +480,44 @@ def _set_repo_check_parser(root: SubParserAction) -> argparse.ArgumentParser: return parser +def _set_repo_create_keyring_parser(root: SubParserAction) -> argparse.ArgumentParser: + """ + add parser for create-keyring subcommand + + Args: + root(SubParserAction): subparsers for the commands + + Returns: + argparse.ArgumentParser: created argument parser + """ + parser = root.add_parser("repo-create-keyring", help="create keyring package", + description="create package which contains list of trusted keys as set by " + "configuration. Note, that this action will only create package, the package " + "itself has to be built manually", + formatter_class=_formatter) + parser.set_defaults(handler=handlers.Triggers, trigger=["ahriman.core.support.KeyringTrigger"]) + return parser + + +def _set_repo_create_mirrorlist_parser(root: SubParserAction) -> argparse.ArgumentParser: + """ + add parser for create-mirrorlist subcommand + + Args: + root(SubParserAction): subparsers for the commands + + Returns: + argparse.ArgumentParser: created argument parser + """ + parser = root.add_parser("repo-create-mirrorlist", help="create mirrorlist package", + description="create package which contains list of available mirrors as set by " + "configuration. Note, that this action will only create package, the package " + "itself has to be built manually", + formatter_class=_formatter) + parser.set_defaults(handler=handlers.Triggers, trigger=["ahriman.core.support.MirrorlistTrigger"]) + return parser + + def _set_repo_daemon_parser(root: SubParserAction) -> argparse.ArgumentParser: """ add parser for daemon subcommand diff --git a/src/ahriman/application/application/application_packages.py b/src/ahriman/application/application/application_packages.py index 2bb30216..0c78f6e6 100644 --- a/src/ahriman/application/application/application_packages.py +++ b/src/ahriman/application/application/application_packages.py @@ -26,6 +26,7 @@ from typing import Any from ahriman.application.application.application_properties import ApplicationProperties from ahriman.core.build_tools.sources import Sources +from ahriman.core.exceptions import UnknownPackageError from ahriman.core.util import package_like from ahriman.models.package import Package from ahriman.models.package_source import PackageSource @@ -43,8 +44,14 @@ class ApplicationPackages(ApplicationProperties): Args: source(str): path to package archive + + Raises: + UnknownPackageError: if specified path doesn't exist """ local_path = Path(source) + if not local_path.is_file(): + raise UnknownPackageError(source) + dst = self.repository.paths.packages / local_path.name shutil.copy(local_path, dst) @@ -68,6 +75,9 @@ class ApplicationPackages(ApplicationProperties): source(str): path to local directory """ local_dir = Path(source) + if not local_dir.is_dir(): + raise UnknownPackageError(source) + for full_path in filter(package_like, local_dir.iterdir()): self._add_archive(str(full_path)) @@ -77,12 +87,19 @@ class ApplicationPackages(ApplicationProperties): Args: source(str): path to directory with local source files + + Raises: + UnknownPackageError: if specified package is unknown or doesn't exist """ - source_dir = Path(source) - package = Package.from_build(source_dir, self.architecture) - cache_dir = self.repository.paths.cache_for(package.base) - shutil.copytree(source_dir, cache_dir) # copy package to store in caches - Sources.init(cache_dir) # we need to run init command in directory where we do have permissions + if (source_dir := Path(source)).is_dir(): + package = Package.from_build(source_dir, self.architecture) + cache_dir = self.repository.paths.cache_for(package.base) + shutil.copytree(source_dir, cache_dir) # copy package to store in caches + Sources.init(cache_dir) # we need to run init command in directory where we do have permissions + elif (source_dir := self.repository.paths.cache_for(source)).is_dir(): + package = Package.from_build(source_dir, self.architecture) + else: + raise UnknownPackageError(source) self.database.build_queue_insert(package) @@ -95,8 +112,11 @@ class ApplicationPackages(ApplicationProperties): """ dst = self.repository.paths.packages / Path(source).name # URL is path, is not it? # timeout=None to suppress pylint warns. Also suppress bandit warnings - response = requests.get(source, stream=True, timeout=None) # nosec - response.raise_for_status() + try: + response = requests.get(source, stream=True, timeout=None) # nosec + response.raise_for_status() + except Exception: + raise UnknownPackageError(source) with dst.open("wb") as local_file: for chunk in response.iter_content(chunk_size=1024): diff --git a/src/ahriman/application/handlers/validate.py b/src/ahriman/application/handlers/validate.py index f67812ee..857db3a5 100644 --- a/src/ahriman/application/handlers/validate.py +++ b/src/ahriman/application/handlers/validate.py @@ -78,7 +78,9 @@ class Validate(Handler): # create trigger loader instance loader = TriggerLoader() - for trigger in loader.selected_triggers(configuration): + triggers = loader.selected_triggers(configuration) + loader.known_triggers(configuration) + + for trigger in triggers: try: trigger_class = loader.load_trigger_class(trigger) except ExtensionError: diff --git a/src/ahriman/core/build_tools/sources.py b/src/ahriman/core/build_tools/sources.py index 93d90c06..3971ab22 100644 --- a/src/ahriman/core/build_tools/sources.py +++ b/src/ahriman/core/build_tools/sources.py @@ -125,6 +125,12 @@ class Sources(LazyLogging): Sources._check_output("git", "init", "--initial-branch", instance.DEFAULT_BRANCH, cwd=sources_dir, logger=instance.logger) + # extract local files... + files = ["PKGBUILD", ".SRCINFO"] + [str(path) for path in Package.local_files(sources_dir)] + instance.add(sources_dir, *files) + # ...and commit them + instance.commit(sources_dir, author="ahriman ") + @staticmethod def load(sources_dir: Path, package: Package, patches: list[PkgbuildPatch], paths: RepositoryPaths) -> None: """ diff --git a/src/ahriman/core/configuration/configuration.py b/src/ahriman/core/configuration/configuration.py index 4547a205..c2cd20c5 100644 --- a/src/ahriman/core/configuration/configuration.py +++ b/src/ahriman/core/configuration/configuration.py @@ -99,6 +99,16 @@ class Configuration(configparser.RawConfigParser): """ return self.getpath("settings", "logging") + @property + def repository_name(self) -> str: + """ + repository name as defined by configuration + + Returns: + str: repository name from configuration + """ + return self.get("repository", "name") + @property def repository_paths(self) -> RepositoryPaths: """ diff --git a/src/ahriman/core/configuration/schema.py b/src/ahriman/core/configuration/schema.py index e490d459..30da4764 100644 --- a/src/ahriman/core/configuration/schema.py +++ b/src/ahriman/core/configuration/schema.py @@ -163,6 +163,11 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = { "coerce": "list", "schema": {"type": "string"}, }, + "triggers_known": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + }, "vcs_allowed_age": { "type": "integer", "coerce": "integer", diff --git a/src/ahriman/core/configuration/validator.py b/src/ahriman/core/configuration/validator.py index c7e98915..d8edf399 100644 --- a/src/ahriman/core/configuration/validator.py +++ b/src/ahriman/core/configuration/validator.py @@ -144,6 +144,24 @@ class Validator(RootValidator): if constraint and url.scheme not in constraint: self._error(field, f"Url {value} scheme must be one of {constraint}") + def _validate_path_is_absolute(self, constraint: bool, field: str, value: Path) -> None: + """ + check if path is absolute or not + + Args: + constraint(bool): True in case if path must be absolute and False if it must be relative + field(str): field name to be checked + value(Path): value to be checked + + Examples: + The rule's arguments are validated against this schema: + {"type": "boolean"} + """ + if constraint and not value.is_absolute(): + self._error(field, f"Path {value} must be absolute") + if not constraint and value.is_absolute(): + self._error(field, f"Path {value} must be relative") + def _validate_path_exists(self, constraint: bool, field: str, value: Path) -> None: """ check if paths exists @@ -159,3 +177,5 @@ class Validator(RootValidator): """ if constraint and not value.exists(): self._error(field, f"Path {value} must exist") + if not constraint and value.exists(): + self._error(field, f"Path {value} must not exist") diff --git a/src/ahriman/core/exceptions.py b/src/ahriman/core/exceptions.py index 808e2bd2..e750efc9 100644 --- a/src/ahriman/core/exceptions.py +++ b/src/ahriman/core/exceptions.py @@ -194,6 +194,18 @@ class PasswordError(ValueError): ValueError.__init__(self, f"Password error: {details}") +class PkgbuildGeneratorError(RuntimeError): + """ + exception class for support type triggers + """ + + def __init__(self) -> None: + """ + default constructor + """ + RuntimeError.__init__(self, "Could not generate package") + + class ReportError(RuntimeError): """ report generation exception diff --git a/src/ahriman/core/report/html.py b/src/ahriman/core/report/html.py index 8c5edf13..ffc1ff08 100644 --- a/src/ahriman/core/report/html.py +++ b/src/ahriman/core/report/html.py @@ -57,4 +57,4 @@ class HTML(Report, JinjaTemplate): result(Result): build result """ html = self.make_html(Result(success=packages), self.template_path) - self.report_path.write_text(html) + self.report_path.write_text(html, encoding="utf8") diff --git a/src/ahriman/core/report/jinja_template.py b/src/ahriman/core/report/jinja_template.py index eb232706..a525ecf6 100644 --- a/src/ahriman/core/report/jinja_template.py +++ b/src/ahriman/core/report/jinja_template.py @@ -75,7 +75,7 @@ class JinjaTemplate: # base template vars self.homepage = configuration.get(section, "homepage", fallback=None) - self.name = configuration.get("repository", "name") + self.name = configuration.repository_name self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration) diff --git a/src/ahriman/core/repository/repository_properties.py b/src/ahriman/core/repository/repository_properties.py index 0c961d65..2a236f53 100644 --- a/src/ahriman/core/repository/repository_properties.py +++ b/src/ahriman/core/repository/repository_properties.py @@ -67,7 +67,7 @@ class RepositoryProperties(LazyLogging): self.configuration = configuration self.database = database - self.name = configuration.get("repository", "name") + self.name = configuration.repository_name self.vcs_allowed_age = configuration.getint("build", "vcs_allowed_age", fallback=0) self.paths: RepositoryPaths = configuration.repository_paths # additional workaround for pycharm typing @@ -79,7 +79,7 @@ class RepositoryProperties(LazyLogging): self.ignore_list = configuration.getlist("build", "ignore_packages", fallback=[]) self.pacman = Pacman(architecture, configuration, refresh_database=refresh_pacman_database) - self.sign = GPG(architecture, configuration) + self.sign = GPG(configuration) self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args) self.reporter = Client.load(configuration, report=report) self.triggers = TriggerLoader.load(architecture, configuration) diff --git a/src/ahriman/core/sign/gpg.py b/src/ahriman/core/sign/gpg.py index 976fac7e..be167c2c 100644 --- a/src/ahriman/core/sign/gpg.py +++ b/src/ahriman/core/sign/gpg.py @@ -19,6 +19,7 @@ # import requests +from collections.abc import Generator from pathlib import Path from ahriman.core.configuration import Configuration @@ -34,7 +35,6 @@ class GPG(LazyLogging): Attributes: DEFAULT_TIMEOUT(int): (class attribute) HTTP request timeout in seconds - architecture(str): repository architecture configuration(Configuration): configuration instance default_key(str | None): default PGP key ID to use targets(set[SignSettings]): list of targets to sign (repository, package etc) @@ -43,15 +43,13 @@ class GPG(LazyLogging): _check_output = check_output DEFAULT_TIMEOUT = 30 - def __init__(self, architecture: str, configuration: Configuration) -> None: + def __init__(self, configuration: Configuration) -> None: """ default constructor Args: - architecture(str): repository architecture configuration(Configuration): configuration instance """ - self.architecture = architecture self.configuration = configuration self.targets, self.default_key = self.sign_options(configuration) @@ -128,6 +126,34 @@ class GPG(LazyLogging): raise return response.text + def key_export(self, key: str) -> str: + """ + export public key from stored keychain + + Args: + key(str): key ID to export + + Returns: + str: PGP key in .asc format + """ + return GPG._check_output("gpg", "--armor", "--no-emit-version", "--export", key, logger=self.logger) + + def key_fingerprint(self, key: str) -> str: + """ + get full key fingerprint from short key id + + Args: + key(str): key ID to lookup + + Returns: + str: full PGP key fingerprint + """ + metadata = GPG._check_output("gpg", "--with-colons", "--fingerprint", key, logger=self.logger) + # fingerprint line will be like + # fpr:::::::::43A663569A07EE1E4ECC55CC7E3A4240CE3C45C2: + fingerprint = next(filter(lambda line: line[:3] == "fpr", metadata.splitlines())) + return fingerprint.split(":")[-2] + def key_import(self, server: str, key: str) -> None: """ import key to current user and sign it locally @@ -139,6 +165,21 @@ class GPG(LazyLogging): key_body = self.key_download(server, key) GPG._check_output("gpg", "--import", input_data=key_body, logger=self.logger) + def keys(self) -> list[str]: + """ + extract list of keys described in configuration + + Returns: + list[str]: list of unique keys which are set in configuration + """ + def generator() -> Generator[str, None, None]: + if self.default_key is not None: + yield self.default_key + for _, value in filter(lambda pair: pair[0].startswith("key_"), self.configuration["sign"].items()): + yield value + + return sorted(set(generator())) + def process(self, path: Path, key: str) -> list[Path]: """ gpg command wrapper diff --git a/src/ahriman/core/support/__init__.py b/src/ahriman/core/support/__init__.py new file mode 100644 index 00000000..f607effe --- /dev/null +++ b/src/ahriman/core/support/__init__.py @@ -0,0 +1,21 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from ahriman.core.support.keyring_trigger import KeyringTrigger +from ahriman.core.support.mirrorlist_trigger import MirrorlistTrigger diff --git a/src/ahriman/core/support/keyring_trigger.py b/src/ahriman/core/support/keyring_trigger.py new file mode 100644 index 00000000..6a83e889 --- /dev/null +++ b/src/ahriman/core/support/keyring_trigger.py @@ -0,0 +1,114 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from ahriman.core import context +from ahriman.core.configuration import Configuration +from ahriman.core.sign.gpg import GPG +from ahriman.core.support.package_creator import PackageCreator +from ahriman.core.support.pkgbuild.keyring_generator import KeyringGenerator +from ahriman.core.triggers import Trigger +from ahriman.models.context_key import ContextKey + + +class KeyringTrigger(Trigger): + """ + keyring generator trigger + + Attributes: + targets(list[str]): git remote target list + """ + + CONFIGURATION_SCHEMA = { + "keyring": { + "type": "dict", + "schema": { + "target": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + }, + }, + }, + "keyring_generator": { + "type": "dict", + "schema": { + "description": { + "type": "string", + }, + "homepage": { + "type": "string", + }, + "license": { + "type": "list", + "coerce": "list", + }, + "package": { + "type": "string", + }, + "packagers": { + "type": "list", + "coerce": "list", + }, + "revoked": { + "type": "list", + "coerce": "list", + }, + "trusted": { + "type": "list", + "coerce": "list", + }, + }, + }, + } + + def __init__(self, architecture: str, configuration: Configuration) -> None: + """ + default constructor + + Args: + architecture(str): repository architecture + configuration(Configuration): configuration instance + """ + Trigger.__init__(self, architecture, configuration) + self.targets = self.configuration_sections(configuration) + + @classmethod + def configuration_sections(cls, configuration: Configuration) -> list[str]: + """ + extract configuration sections from configuration + + Args: + configuration(Configuration): configuration instance + + Returns: + list[str]: read configuration sections belong to this trigger + """ + return configuration.getlist("keyring", "target", fallback=[]) + + def on_start(self) -> None: + """ + trigger action which will be called at the start of the application + """ + ctx = context.get() + sign = ctx.get(ContextKey("sign", GPG)) + + for target in self.targets: + generator = KeyringGenerator(sign, self.configuration, target) + runner = PackageCreator(self.configuration, generator) + runner.run() diff --git a/src/ahriman/core/support/mirrorlist_trigger.py b/src/ahriman/core/support/mirrorlist_trigger.py new file mode 100644 index 00000000..a1b89373 --- /dev/null +++ b/src/ahriman/core/support/mirrorlist_trigger.py @@ -0,0 +1,105 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from ahriman.core.configuration import Configuration +from ahriman.core.support.package_creator import PackageCreator +from ahriman.core.support.pkgbuild.mirrorlist_generator import MirrorlistGenerator +from ahriman.core.triggers import Trigger + + +class MirrorlistTrigger(Trigger): + """ + mirrorlist generator trigger + + Attributes: + targets(list[str]): git remote target list + """ + + CONFIGURATION_SCHEMA = { + "mirrorlist": { + "type": "dict", + "schema": { + "target": { + "type": "list", + "coerce": "list", + "schema": {"type": "string"}, + }, + }, + }, + "mirrorlist_generator": { + "type": "dict", + "schema": { + "description": { + "type": "string", + }, + "homepage": { + "type": "string", + }, + "license": { + "type": "list", + "coerce": "list", + }, + "package": { + "type": "string", + }, + "path": { + "type": "string", + "path_is_absolute": True, + }, + "servers": { + "type": "list", + "coerce": "list", + "required": True, + }, + }, + }, + } + + def __init__(self, architecture: str, configuration: Configuration) -> None: + """ + default constructor + + Args: + architecture(str): repository architecture + configuration(Configuration): configuration instance + """ + Trigger.__init__(self, architecture, configuration) + self.targets = self.configuration_sections(configuration) + + @classmethod + def configuration_sections(cls, configuration: Configuration) -> list[str]: + """ + extract configuration sections from configuration + + Args: + configuration(Configuration): configuration instance + + Returns: + list[str]: read configuration sections belong to this trigger + """ + return configuration.getlist("mirrorlist", "target", fallback=[]) + + def on_start(self) -> None: + """ + trigger action which will be called at the start of the application + """ + for target in self.targets: + generator = MirrorlistGenerator(self.configuration, target) + runner = PackageCreator(self.configuration, generator) + runner.run() diff --git a/src/ahriman/core/support/package_creator.py b/src/ahriman/core/support/package_creator.py new file mode 100644 index 00000000..6ab120df --- /dev/null +++ b/src/ahriman/core/support/package_creator.py @@ -0,0 +1,71 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import shutil + +from ahriman.core import context +from ahriman.core.build_tools.sources import Sources +from ahriman.core.configuration import Configuration +from ahriman.core.database import SQLite +from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator +from ahriman.models.build_status import BuildStatus +from ahriman.models.context_key import ContextKey +from ahriman.models.package import Package + + +class PackageCreator: + """ + helper which creates packages based on pkgbuild generator + + Attributes: + configuration(Configuration): configuration instance + generator(PkgbuildGenerator): PKGBUILD generator instance + """ + + def __init__(self, configuration: Configuration, generator: PkgbuildGenerator) -> None: + """ + default constructor + + Args: + configuration(Configuration): configuration instance + generator(PkgbuildGenerator): PKGBUILD generator instance + """ + self.configuration = configuration + self.generator = generator + + def run(self) -> None: + """ + create new local package + """ + local_path = self.configuration.repository_paths.cache_for(self.generator.pkgname) + + # clear old tree if any + shutil.rmtree(local_path, ignore_errors=True) + + # create local tree + local_path.mkdir(mode=0o755, parents=True, exist_ok=True) + self.generator.write_pkgbuild(local_path) + Sources.init(local_path) + + # register package + ctx = context.get() + database: SQLite = ctx.get(ContextKey("database", SQLite)) + _, architecture = self.configuration.check_loaded() + package = Package.from_build(local_path, architecture) + database.package_update(package, BuildStatus()) diff --git a/src/ahriman/core/support/pkgbuild/__init__.py b/src/ahriman/core/support/pkgbuild/__init__.py new file mode 100644 index 00000000..8fc622e9 --- /dev/null +++ b/src/ahriman/core/support/pkgbuild/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# diff --git a/src/ahriman/core/support/pkgbuild/keyring_generator.py b/src/ahriman/core/support/pkgbuild/keyring_generator.py new file mode 100644 index 00000000..0ddb8c3f --- /dev/null +++ b/src/ahriman/core/support/pkgbuild/keyring_generator.py @@ -0,0 +1,194 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from collections.abc import Callable +from pathlib import Path + +from ahriman.core.configuration import Configuration +from ahriman.core.exceptions import PkgbuildGeneratorError +from ahriman.core.sign.gpg import GPG +from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator + + +class KeyringGenerator(PkgbuildGenerator): + """ + generator for keyring PKGBUILD + + Attributes: + sign(GPG): GPG wrapper instance + name(str): repository name + packagers(list[str]): list of packagers PGP keys + pkgbuild_license(list[str]): keyring package license + pkgbuild_pkgdesc(str): keyring package description + pkgbuild_pkgname(str): keyring package name + pkgbuild_url(str): keyring package home page + revoked(list[str]): list of revoked PGP keys + trusted(list[str]): lif of trusted PGP keys + """ + + def __init__(self, sign: GPG, configuration: Configuration, section: str) -> None: + """ + default constructor + + Args: + sign(GPG): GPG wrapper instance + configuration(Configuration): configuration instance + section(str): settings section name + """ + self.sign = sign + self.name = configuration.repository_name + + # configuration fields + self.packagers = configuration.getlist(section, "packagers", fallback=sign.keys()) + self.revoked = configuration.getlist(section, "revoked", fallback=[]) + self.trusted = configuration.getlist( + section, "trusted", fallback=[sign.default_key] if sign.default_key is not None else []) + # pkgbuild description fields + self.pkgbuild_pkgname = configuration.get(section, "package", fallback=f"{self.name}-keyring") + self.pkgbuild_pkgdesc = configuration.get(section, "description", fallback=f"{self.name} PGP keyring") + self.pkgbuild_license = configuration.getlist(section, "license", fallback=["Unlicense"]) + self.pkgbuild_url = configuration.get(section, "homepage", fallback="") + + @property + def license(self) -> list[str]: + """ + package licenses list + + Returns: + list[str]: package licenses as PKGBUILD property + """ + return self.pkgbuild_license + + @property + def pkgdesc(self) -> str: + """ + package description + + Returns: + str: package description as PKGBUILD property + """ + return self.pkgbuild_pkgdesc + + @property + def pkgname(self) -> str: + """ + package name + + Returns: + str: package name as PKGBUILD property + """ + return self.pkgbuild_pkgname + + @property + def url(self) -> str: + """ + package upstream url + + Returns: + str: package upstream url as PKGBUILD property + """ + return self.pkgbuild_url + + def _generate_gpg(self, source_path: Path) -> None: + """ + generate GPG keychain + + Args: + source_path(Path): destination of the file content + """ + with source_path.open("w") as source_file: + for key in sorted(set(self.trusted + self.packagers + self.revoked)): + public_key = self.sign.key_export(key) + source_file.write(public_key) + source_file.write("\n") + + def _generate_revoked(self, source_path: Path) -> None: + """ + generate revoked PGP keys + + Args: + source_path(Path): destination of the file content + """ + with source_path.open("w") as source_file: + for key in sorted(set(self.revoked)): + fingerprint = self.sign.key_fingerprint(key) + source_file.write(fingerprint) + source_file.write("\n") + + def _generate_trusted(self, source_path: Path) -> None: + """ + generate trusted PGP keys + + Args: + source_path(Path): destination of the file content + """ + if not self.trusted: + raise PkgbuildGeneratorError + with source_path.open("w") as source_file: + for key in sorted(set(self.trusted)): + fingerprint = self.sign.key_fingerprint(key) + source_file.write(fingerprint) + source_file.write(":4:\n") + + def install(self) -> str | None: + """ + content of the install functions + + Returns: + str | None: content of the install functions if any + """ + # copy-paste from archlinux-keyring + return f"""post_upgrade() {{ + if usr/bin/pacman-key -l >/dev/null 2>&1; then + usr/bin/pacman-key --populate {self.name} + usr/bin/pacman-key --updatedb + fi +}} + +post_install() {{ + if [ -x usr/bin/pacman-key ]; then + post_upgrade + fi +}}""" + + def package(self) -> str: + """ + package function generator + + Returns: + str: package() function for PKGBUILD + """ + return f"""{{ + install -Dm644 "{Path("$srcdir") / f"{self.name}.gpg"}" "{Path("$pkgdir") / "usr" / "share" / "pacman" / "keyrings" / f"{self.name}.gpg"}" + install -Dm644 "{Path("$srcdir") / f"{self.name}-revoked"}" "{Path("$pkgdir") / "usr" / "share" / "pacman" / "keyrings" / f"{self.name}-revoked"}" + install -Dm644 "{Path("$srcdir") / f"{self.name}-trusted"}" "{Path("$pkgdir") / "usr" / "share" / "pacman" / "keyrings" / f"{self.name}-trusted"}" +}}""" + + def sources(self) -> dict[str, Callable[[Path], None]]: + """ + return list of sources for the package + + Returns: + dict[str, Callable[[Path], None]]: map of source identifier (e.g. filename) to its generator function + """ + return { + f"{self.name}.gpg": self._generate_gpg, + f"{self.name}-revoked": self._generate_revoked, + f"{self.name}-trusted": self._generate_trusted, + } diff --git a/src/ahriman/core/support/pkgbuild/mirrorlist_generator.py b/src/ahriman/core/support/pkgbuild/mirrorlist_generator.py new file mode 100644 index 00000000..36f3df08 --- /dev/null +++ b/src/ahriman/core/support/pkgbuild/mirrorlist_generator.py @@ -0,0 +1,143 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from collections.abc import Callable +from pathlib import Path + +from ahriman.core.configuration import Configuration +from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator +from ahriman.models.pkgbuild_patch import PkgbuildPatch + + +class MirrorlistGenerator(PkgbuildGenerator): + """ + generator for mirrorlist PKGBUILD + + Attributes: + path(Path): path to mirrorlist relative to / + pkgbuild_license(list[str]): mirrorlist package license + pkgbuild_pkgdesc(str): mirrorlist package description + pkgbuild_pkgname(str): mirrorlist package name + pkgbuild_url(str): mirrorlist package home page + servers(list[str]): list of mirror servers + """ + + def __init__(self, configuration: Configuration, section: str) -> None: + """ + default constructor + + Args: + configuration(Configuration): configuration instance + section(str): settings section name + """ + name = configuration.repository_name + + # configuration fields + self.servers = configuration.getlist(section, "servers") + self.path = configuration.getpath(section, "path", fallback=Path("/etc") / "pacman.d" / f"{name}-mirrorlist") + self.path = self.path.relative_to("/") # in pkgbuild we are always operating with relative to / path + # pkgbuild description fields + self.pkgbuild_pkgname = configuration.get(section, "package", fallback=f"{name}-mirrorlist") + self.pkgbuild_pkgdesc = configuration.get( + section, "description", fallback=f"{name} mirror list for use by pacman") + self.pkgbuild_license = configuration.getlist(section, "license", fallback=["Unlicense"]) + self.pkgbuild_url = configuration.get(section, "homepage", fallback="") + + @property + def license(self) -> list[str]: + """ + package licenses list + + Returns: + list[str]: package licenses as PKGBUILD property + """ + return self.pkgbuild_license + + @property + def pkgdesc(self) -> str: + """ + package description + + Returns: + str: package description as PKGBUILD property + """ + return self.pkgbuild_pkgdesc + + @property + def pkgname(self) -> str: + """ + package name + + Returns: + str: package name as PKGBUILD property + """ + return self.pkgbuild_pkgname + + @property + def url(self) -> str: + """ + package upstream url + + Returns: + str: package upstream url as PKGBUILD property + """ + return self.pkgbuild_url + + def _generate_mirrorlist(self, source_path: Path) -> None: + """ + generate mirrorlist file + + Args: + source_path(Path): destination of the mirrorlist content + """ + content = "".join([f"Server = {server}\n" for server in self.servers]) + source_path.write_text(content, encoding="utf8") + + def package(self) -> str: + """ + package function generator + + Returns: + str: package() function for PKGBUILD + """ + return f"""{{ + install -Dm644 "{Path("$srcdir") / "mirrorlist"}" "{Path("$pkgdir") / self.path}" +}}""" + + def patches(self) -> list[PkgbuildPatch]: + """ + list of additional PKGBUILD properties + + Returns: + list[PkgbuildPatch]: list of patches which generate PKGBUILD content + """ + return [ + PkgbuildPatch("backup", [str(self.path)]), + ] + + def sources(self) -> dict[str, Callable[[Path], None]]: + """ + return list of sources for the package + + Returns: + dict[str, Callable[[Path], None]]: map of source identifier (e.g. filename) to its generator function + """ + return { + "mirrorlist": self._generate_mirrorlist, + } diff --git a/src/ahriman/core/support/pkgbuild/pkgbuild_generator.py b/src/ahriman/core/support/pkgbuild/pkgbuild_generator.py new file mode 100644 index 00000000..42787e90 --- /dev/null +++ b/src/ahriman/core/support/pkgbuild/pkgbuild_generator.py @@ -0,0 +1,201 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import hashlib +import itertools + +from collections.abc import Callable, Generator +from pathlib import Path + +from ahriman.core.util import utcnow +from ahriman.models.pkgbuild_patch import PkgbuildPatch + + +class PkgbuildGenerator: + """ + main class for generating PKGBUILDs + + Attributes: + PKGBUILD_STATIC_PROPERTIES(list[PkgbuildPatch]): (class attribute) list of default pkgbuild static properties + """ + + PKGBUILD_STATIC_PROPERTIES = [ + PkgbuildPatch("pkgrel", "1"), + PkgbuildPatch("arch", ["any"]), + ] + + @property + def license(self) -> list[str]: + """ + package licenses list + + Returns: + list[str]: package licenses as PKGBUILD property + """ + return [] + + @property + def pkgdesc(self) -> str: + """ + package description + + Returns: + str: package description as PKGBUILD property + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + @property + def pkgname(self) -> str: + """ + package name + + Returns: + str: package name as PKGBUILD property + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + @property + def pkgver(self) -> str: + """ + package version + + Returns: + str: package version as PKGBUILD property + """ + return utcnow().strftime("%Y%m%d") + + @property + def url(self) -> str: + """ + package upstream url + + Returns: + str: package upstream url as PKGBUILD property + """ + return "" + + def install(self) -> str | None: + """ + content of the install functions + + Returns: + str | None: content of the install functions if any + """ + + def package(self) -> str: + """ + package function generator + + Returns: + str: package() function for PKGBUILD + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def patches(self) -> list[PkgbuildPatch]: + """ + list of additional PKGBUILD properties + + Returns: + list[PkgbuildPatch]: list of patches which generate PKGBUILD content + """ + return [] + + def sources(self) -> dict[str, Callable[[Path], None]]: + """ + return list of sources for the package + + Returns: + dict[str, Callable[[Path], None]]: map of source identifier (e.g. filename) to its generator function + """ + return {} + + def write_install(self, source_dir: Path) -> list[PkgbuildPatch]: + """ + generate content of install file + + Args: + source_dir(Path): path to directory in which sources must be generated + + Returns: + list[PkgbuildPatch]: patch for the pkgbuild if install file exists and empty list otherwise + """ + content: str | None = self.install() + if content is None: + return [] + + source_path = source_dir / f"{self.pkgname}.install" + source_path.write_text(content) + return [PkgbuildPatch("install", source_path.name)] + + def write_pkgbuild(self, source_dir: Path) -> None: + """ + generate PKGBUILD content to the specified path + + Args: + source_dir(Path): path to directory in which sources must be generated + """ + patches = self.PKGBUILD_STATIC_PROPERTIES # default static properties... + patches.extend([ + PkgbuildPatch("license", self.license), + PkgbuildPatch("pkgdesc", self.pkgdesc), + PkgbuildPatch("pkgname", self.pkgname), + PkgbuildPatch("pkgver", self.pkgver), + PkgbuildPatch("url", self.url), + ]) # ...main properties as defined by derived class... + patches.extend(self.patches()) # ...optional properties as defined by derived class... + patches.extend(self.write_install(source_dir)) # ...install function... + patches.append(PkgbuildPatch("package()", self.package())) # ...package function... + + patches.extend(self.write_sources(source_dir)) # ...and finally source files + + for patch in patches: + patch.write(source_dir / "PKGBUILD") + + def write_sources(self, source_dir: Path) -> list[PkgbuildPatch]: + """ + write sources and returns valid PKGBUILD properties for them + + Args: + source_dir(Path): path to directory in which sources must be generated + + Returns: + list[PkgbuildPatch]: list of patches to be applied to the PKGBUILD + """ + def sources_generator() -> Generator[tuple[str, str], None, None]: + for source, generator in sorted(self.sources().items()): + source_path = source_dir / source + generator(source_path) + with source_path.open("rb") as source_file: + source_hash = hashlib.sha512(source_file.read()) + yield source, source_hash.hexdigest() + + sources_iter, hashes_iter = itertools.tee(sources_generator()) + return [ + PkgbuildPatch("source", [source for source, _ in sources_iter]), + PkgbuildPatch("sha512sums", [sha512 for _, sha512 in hashes_iter]), + ] diff --git a/src/ahriman/core/triggers/trigger_loader.py b/src/ahriman/core/triggers/trigger_loader.py index ace455cf..0b0a7e88 100644 --- a/src/ahriman/core/triggers/trigger_loader.py +++ b/src/ahriman/core/triggers/trigger_loader.py @@ -84,6 +84,20 @@ class TriggerLoader(LazyLogging): return instance + @staticmethod + def known_triggers(configuration: Configuration) -> list[str]: + """ + read configuration and return list of known triggers. Unlike ``selected_triggers`` this option is used mainly + for configuration and validation and mentioned triggers are not being executed automatically + + Args: + configuration(Configuration): configuration instance + + Returns: + list[str]: list of registered, but not enabled, triggers + """ + return configuration.getlist("build", "triggers_known", fallback=[]) + @staticmethod def selected_triggers(configuration: Configuration) -> list[str]: """ diff --git a/src/ahriman/core/util.py b/src/ahriman/core/util.py index 239a5755..da1de322 100644 --- a/src/ahriman/core/util.py +++ b/src/ahriman/core/util.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +# pylint: disable=too-many-lines import datetime import io import itertools @@ -48,6 +49,8 @@ __all__ = [ "pretty_datetime", "pretty_size", "safe_filename", + "srcinfo_property", + "srcinfo_property_list", "trim_package", "utcnow", "walk", @@ -117,8 +120,6 @@ def check_output(*args: str, exception: Exception | None = None, cwd: Path | Non result: list[str] = [] for line in iter(get_io(process, "stdout").readline, ""): line = line.strip() - if not line: # skip empty lines - continue result.append(line) log(line) @@ -133,7 +134,7 @@ def check_output(*args: str, exception: Exception | None = None, cwd: Path | Non raise exception raise subprocess.CalledProcessError(status_code, process.args) - return "\n".join(result) + return "\n".join(result).rstrip("\n") # remove newline at the end of any def check_user(paths: RepositoryPaths, *, unsafe: bool) -> None: @@ -328,6 +329,47 @@ def safe_filename(source: str) -> str: return re.sub(r"[^A-Za-z\d\-._~:\[\]@]", "-", source) +def srcinfo_property(key: str, srcinfo: dict[str, Any], package_srcinfo: dict[str, Any], *, + default: Any = None) -> Any: + """ + extract property from SRCINFO. This method extracts property from package if this property is presented in + ``package``. Otherwise, it looks for the same property in root srcinfo. If none found, the default value will be + returned + + Args: + key(str): key to extract from srcinfo + srcinfo(dict[str, Any]): root structure of SRCINFO + package_srcinfo(dict[str, Any]): package specific SRCINFO + default(Any, optional): the default value for the specified key (Default value = None) + + Returns: + Any: extracted value from SRCINFO + """ + return package_srcinfo.get(key) or srcinfo.get(key) or default + + +def srcinfo_property_list(key: str, srcinfo: dict[str, Any], package_srcinfo: dict[str, Any], *, + architecture: str | None = None) -> list[Any]: + """ + extract list property from SRCINFO. Unlike ``srcinfo_property`` it supposes that default return value is always + empty list. If ``architecture`` is supplied, then it will try to lookup for architecture specific values and will + append it at the end of result + + Args: + key(str): key to extract from srcinfo + srcinfo(dict[str, Any]): root structure of SRCINFO + package_srcinfo(dict[str, Any]): package specific SRCINFO + architecture(str | None, optional): package architecture if set (Default value = None) + + Returns: + list[Any]: list of extracted properties from SRCINFO + """ + values: list[Any] = srcinfo_property(key, srcinfo, package_srcinfo, default=[]) + if architecture is not None: + values.extend(srcinfo_property(f"{key}_{architecture}", srcinfo, package_srcinfo, default=[])) + return values + + def trim_package(package_name: str) -> str: """ remove version bound and description from package name. Pacman allows to specify version bound (=, <=, >= etc) for diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 64315992..0dc15e15 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -22,18 +22,19 @@ from __future__ import annotations import copy -from collections.abc import Iterable +from collections.abc import Generator, Iterable from dataclasses import asdict, dataclass from pathlib import Path from pyalpm import vercmp # type: ignore[import] from srcinfo.parse import parse_srcinfo # type: ignore[import] from typing import Any, Self +from urllib.parse import urlparse from ahriman.core.alpm.pacman import Pacman from ahriman.core.alpm.remote import AUR, Official, OfficialSyncdb from ahriman.core.exceptions import PackageInfoError from ahriman.core.log import LazyLogging -from ahriman.core.util import check_output, full_version, utcnow +from ahriman.core.util import check_output, full_version, srcinfo_property_list, utcnow from ahriman.models.package_description import PackageDescription from ahriman.models.package_source import PackageSource from ahriman.models.remote_source import RemoteSource @@ -235,23 +236,25 @@ class Package(LazyLogging): if errors: raise PackageInfoError(errors) - def get_property(key: str, properties: dict[str, Any], default: Any) -> Any: - return properties.get(key) or srcinfo.get(key) or default - - def get_list(key: str, properties: dict[str, Any]) -> Any: - return get_property(key, properties, []) + get_property(f"{key}_{architecture}", properties, []) - packages = { package: PackageDescription( - depends=get_list("depends", properties), - make_depends=get_list("makedepends", properties), - opt_depends=get_list("optdepends", properties), + depends=srcinfo_property_list("depends", srcinfo, properties, architecture=architecture), + make_depends=srcinfo_property_list("makedepends", srcinfo, properties, architecture=architecture), + opt_depends=srcinfo_property_list("optdepends", srcinfo, properties, architecture=architecture), ) for package, properties in srcinfo["packages"].items() } version = full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"]) - return cls(base=srcinfo["pkgbase"], version=version, remote=None, packages=packages) + remote = RemoteSource( + git_url=path.absolute().as_uri(), + web_url="", + path=".", + branch="master", + source=PackageSource.Local, + ) + + return cls(base=srcinfo["pkgbase"], version=version, remote=remote, packages=packages) @classmethod def from_json(cls, dump: dict[str, Any]) -> Self: @@ -293,6 +296,41 @@ class Package(LazyLogging): remote=remote, packages={package.name: PackageDescription.from_aur(package)}) + @staticmethod + def local_files(path: Path) -> Generator[Path, None, None]: + """ + extract list of local files + + Args: + path(Path): path to package sources directory + + Returns: + Generator[Path, None, None]: list of paths of files which belong to the package and distributed together + with this tarball. All paths are relative to the ``path`` + """ + srcinfo_source = Package._check_output("makepkg", "--printsrcinfo", cwd=path) + srcinfo, errors = parse_srcinfo(srcinfo_source) + if errors: + raise PackageInfoError(errors) + + # we could use arch property, but for consistency it is better to call special method + architectures = Package.supported_architectures(path) + + for architecture in architectures: + for source in srcinfo_property_list("source", srcinfo, {}, architecture=architecture): + if "::" in source: + _, source = source.split("::", 1) # in case if filename is specified, remove it + + if urlparse(source).scheme: + # basically file schema should use absolute path which is impossible if we are distributing + # files together with PKGBUILD. In this case we are going to skip it also + continue + + yield Path(source) + + if (install := srcinfo.get("install", None)) is not None: + yield Path(install) + @staticmethod def supported_architectures(path: Path) -> set[str]: """ diff --git a/tests/ahriman/application/application/test_application_packages.py b/tests/ahriman/application/application/test_application_packages.py index 3ae3f6bb..dd03caa1 100644 --- a/tests/ahriman/application/application/test_application_packages.py +++ b/tests/ahriman/application/application/test_application_packages.py @@ -5,25 +5,36 @@ from pytest_mock import MockerFixture from unittest.mock import MagicMock from ahriman.application.application.application_packages import ApplicationPackages +from ahriman.core.exceptions import UnknownPackageError from ahriman.models.package import Package from ahriman.models.package_description import PackageDescription from ahriman.models.package_source import PackageSource from ahriman.models.result import Result -def test_add_archive( - application_packages: ApplicationPackages, - package_ahriman: Package, - mocker: MockerFixture) -> None: +def test_add_archive(application_packages: ApplicationPackages, package_ahriman: Package, + mocker: MockerFixture) -> None: """ must add package from archive """ + is_file_mock = mocker.patch("pathlib.Path.is_file", return_value=True) copy_mock = mocker.patch("shutil.copy") + application_packages._add_archive(package_ahriman.base) + is_file_mock.assert_called_once_with() copy_mock.assert_called_once_with( Path(package_ahriman.base), application_packages.repository.paths.packages / package_ahriman.base) +def test_add_archive_missing(application_packages: ApplicationPackages, mocker: MockerFixture) -> None: + """ + must raise UnknownPackageError on unknown path + """ + mocker.patch("pathlib.Path.is_file", return_value=False) + with pytest.raises(UnknownPackageError): + application_packages._add_archive("package") + + def test_add_aur(application_packages: ApplicationPackages, package_ahriman: Package, mocker: MockerFixture) -> None: """ must add package from AUR @@ -37,21 +48,29 @@ def test_add_aur(application_packages: ApplicationPackages, package_ahriman: Pac update_remote_mock.assert_called_once_with(package_ahriman) -def test_add_directory( - application_packages: ApplicationPackages, - package_ahriman: Package, - mocker: MockerFixture) -> None: +def test_add_directory(application_packages: ApplicationPackages, package_ahriman: Package, + mocker: MockerFixture) -> None: """ must add packages from directory """ - iterdir_mock = mocker.patch("pathlib.Path.iterdir", - return_value=[package.filepath for package in package_ahriman.packages.values()]) - copy_mock = mocker.patch("shutil.copy") + is_dir_mock = mocker.patch("pathlib.Path.is_dir", return_value=True) filename = package_ahriman.packages[package_ahriman.base].filepath + iterdir_mock = mocker.patch("pathlib.Path.iterdir", return_value=[filename]) + add_mock = mocker.patch("ahriman.application.application.application_packages.ApplicationPackages._add_archive") application_packages._add_directory(package_ahriman.base) + is_dir_mock.assert_called_once_with() iterdir_mock.assert_called_once_with() - copy_mock.assert_called_once_with(filename, application_packages.repository.paths.packages / filename.name) + add_mock.assert_called_once_with(str(filename)) + + +def test_add_directory_missing(application_packages: ApplicationPackages, mocker: MockerFixture) -> None: + """ + must raise UnknownPackageError on unknown directory path + """ + mocker.patch("pathlib.Path.is_dir", return_value=False) + with pytest.raises(UnknownPackageError): + application_packages._add_directory("package") def test_add_local(application_packages: ApplicationPackages, package_ahriman: Package, mocker: MockerFixture) -> None: @@ -59,17 +78,46 @@ def test_add_local(application_packages: ApplicationPackages, package_ahriman: P must add package from local sources """ mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman) + is_dir_mock = mocker.patch("pathlib.Path.is_dir", return_value=True) init_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.init") copytree_mock = mocker.patch("shutil.copytree") build_queue_mock = mocker.patch("ahriman.core.database.SQLite.build_queue_insert") application_packages._add_local(package_ahriman.base) + is_dir_mock.assert_called_once_with() copytree_mock.assert_called_once_with( Path(package_ahriman.base), application_packages.repository.paths.cache_for(package_ahriman.base)) init_mock.assert_called_once_with(application_packages.repository.paths.cache_for(package_ahriman.base)) build_queue_mock.assert_called_once_with(package_ahriman) +def test_add_local_cache(application_packages: ApplicationPackages, package_ahriman: Package, + mocker: MockerFixture) -> None: + """ + must add package from local source if there is cache + """ + mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman) + mocker.patch("pathlib.Path.is_dir", autospec=True, + side_effect=lambda p: True if p.is_relative_to(application_packages.repository.paths.cache) else False) + init_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.init") + copytree_mock = mocker.patch("shutil.copytree") + build_queue_mock = mocker.patch("ahriman.core.database.SQLite.build_queue_insert") + + application_packages._add_local(package_ahriman.base) + copytree_mock.assert_not_called() + init_mock.assert_not_called() + build_queue_mock.assert_called_once_with(package_ahriman) + + +def test_add_local_missing(application_packages: ApplicationPackages, mocker: MockerFixture) -> None: + """ + must raise UnknownPackageError if package wasn't found + """ + mocker.patch("pathlib.Path.is_dir", return_value=False) + with pytest.raises(UnknownPackageError): + application_packages._add_local("package") + + def test_add_remote(application_packages: ApplicationPackages, package_description_ahriman: PackageDescription, mocker: MockerFixture) -> None: """ @@ -87,6 +135,15 @@ def test_add_remote(application_packages: ApplicationPackages, package_descripti response_mock.raise_for_status.assert_called_once_with() +def test_add_remote_missing(application_packages: ApplicationPackages, mocker: MockerFixture) -> None: + """ + must add package from remote source + """ + mocker.patch("requests.get", side_effect=Exception()) + with pytest.raises(UnknownPackageError): + application_packages._add_remote("url") + + def test_add_repository(application_packages: ApplicationPackages, package_ahriman: Package, mocker: MockerFixture) -> None: """ @@ -112,10 +169,8 @@ def test_add_add_archive(application_packages: ApplicationPackages, package_ahri add_mock.assert_called_once_with(package_ahriman.base) -def test_add_add_aur( - application_packages: ApplicationPackages, - package_ahriman: Package, - mocker: MockerFixture) -> None: +def test_add_add_aur(application_packages: ApplicationPackages, package_ahriman: Package, + mocker: MockerFixture) -> None: """ must add package from AUR via add function """ diff --git a/tests/ahriman/application/handlers/test_handler_validate.py b/tests/ahriman/application/handlers/test_handler_validate.py index a488bd90..3bfb7d0d 100644 --- a/tests/ahriman/application/handlers/test_handler_validate.py +++ b/tests/ahriman/application/handlers/test_handler_validate.py @@ -62,6 +62,10 @@ def test_schema(configuration: Configuration) -> None: assert schema.pop("email") assert schema.pop("github") assert schema.pop("html") + assert schema.pop("keyring") + assert schema.pop("keyring_generator") + assert schema.pop("mirrorlist") + assert schema.pop("mirrorlist_generator") assert schema.pop("report") assert schema.pop("rsync") assert schema.pop("s3") @@ -76,6 +80,7 @@ def test_schema_invalid_trigger(configuration: Configuration) -> None: must skip trigger if it caused exception on load """ configuration.set_option("build", "triggers", "some.invalid.trigger.path.Trigger") + configuration.remove_option("build", "triggers_known") assert Validate.schema("x86_64", configuration) == CONFIGURATION_SCHEMA diff --git a/tests/ahriman/application/test_ahriman.py b/tests/ahriman/application/test_ahriman.py index 1fe5f9a3..f5af76fd 100644 --- a/tests/ahriman/application/test_ahriman.py +++ b/tests/ahriman/application/test_ahriman.py @@ -373,6 +373,42 @@ def test_subparsers_repo_check_option_refresh(parser: argparse.ArgumentParser) - assert args.refresh == 2 +def test_subparsers_repo_create_keyring(parser: argparse.ArgumentParser) -> None: + """ + repo-create-keyring command must imply trigger + """ + args = parser.parse_args(["repo-create-keyring"]) + assert args.trigger == ["ahriman.core.support.KeyringTrigger"] + + +def test_subparsers_repo_create_keyring_architecture(parser: argparse.ArgumentParser) -> None: + """ + repo-create-keyring command must correctly parse architecture list + """ + args = parser.parse_args(["repo-create-keyring"]) + assert args.architecture is None + args = parser.parse_args(["-a", "x86_64", "repo-create-keyring"]) + assert args.architecture == ["x86_64"] + + +def test_subparsers_repo_create_mirrorlist(parser: argparse.ArgumentParser) -> None: + """ + repo-create-mirrorlist command must imply trigger + """ + args = parser.parse_args(["repo-create-mirrorlist"]) + assert args.trigger == ["ahriman.core.support.MirrorlistTrigger"] + + +def test_subparsers_repo_create_mirrorlist_architecture(parser: argparse.ArgumentParser) -> None: + """ + repo-create-mirrorlist command must correctly parse architecture list + """ + args = parser.parse_args(["repo-create-mirrorlist"]) + assert args.architecture is None + args = parser.parse_args(["-a", "x86_64", "repo-create-mirrorlist"]) + assert args.architecture == ["x86_64"] + + def test_subparsers_repo_daemon(parser: argparse.ArgumentParser) -> None: """ repo-daemon command must imply dry run, exit code and package diff --git a/tests/ahriman/core/build_tools/test_sources.py b/tests/ahriman/core/build_tools/test_sources.py index 3d950e69..67adfbe6 100644 --- a/tests/ahriman/core/build_tools/test_sources.py +++ b/tests/ahriman/core/build_tools/test_sources.py @@ -135,12 +135,17 @@ def test_init(mocker: MockerFixture) -> None: """ must create empty repository at the specified path """ + mocker.patch("ahriman.models.package.Package.local_files", return_value=[Path("local")]) + add_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.add") check_output_mock = mocker.patch("ahriman.core.build_tools.sources.Sources._check_output") + commit_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.commit") local = Path("local") Sources.init(local) check_output_mock.assert_called_once_with("git", "init", "--initial-branch", Sources.DEFAULT_BRANCH, cwd=local, logger=pytest.helpers.anyvar(int)) + add_mock.assert_called_once_with(local, "PKGBUILD", ".SRCINFO", "local") + commit_mock.assert_called_once_with(local, author="ahriman ") def test_load(package_ahriman: Package, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/core/configuration/test_configuration.py b/tests/ahriman/core/configuration/test_configuration.py index f56ad7a0..9aa1d249 100644 --- a/tests/ahriman/core/configuration/test_configuration.py +++ b/tests/ahriman/core/configuration/test_configuration.py @@ -10,6 +10,13 @@ from ahriman.core.exceptions import InitializeError from ahriman.models.repository_paths import RepositoryPaths +def test_repository_name(configuration: Configuration) -> None: + """ + must return valid repository name + """ + assert configuration.repository_name == "aur-clone" + + def test_repository_paths(configuration: Configuration, repository_paths: RepositoryPaths) -> None: """ must return repository paths diff --git a/tests/ahriman/core/configuration/test_validator.py b/tests/ahriman/core/configuration/test_validator.py index db4bb34e..54535fa4 100644 --- a/tests/ahriman/core/configuration/test_validator.py +++ b/tests/ahriman/core/configuration/test_validator.py @@ -73,6 +73,30 @@ def test_validate_is_ip_address(validator: Validator, mocker: MockerFixture) -> ]) +def test_validate_path_is_absolute(validator: Validator, mocker: MockerFixture) -> None: + """ + must validate that path is absolute + """ + error_mock = mocker.patch("ahriman.core.configuration.validator.Validator._error") + + mocker.patch("pathlib.Path.is_absolute", return_value=False) + validator._validate_path_is_absolute(False, "field", Path("1")) + + mocker.patch("pathlib.Path.is_absolute", return_value=True) + validator._validate_path_is_absolute(False, "field", Path("2")) + + mocker.patch("pathlib.Path.is_absolute", return_value=False) + validator._validate_path_is_absolute(True, "field", Path("3")) + + mocker.patch("pathlib.Path.is_absolute", return_value=True) + validator._validate_path_is_absolute(True, "field", Path("4")) + + error_mock.assert_has_calls([ + MockCall("field", "Path 2 must be relative"), + MockCall("field", "Path 3 must be absolute"), + ]) + + def test_validate_is_url(validator: Validator, mocker: MockerFixture) -> None: """ must validate url correctly @@ -105,12 +129,16 @@ def test_validate_path_exists(validator: Validator, mocker: MockerFixture) -> No mocker.patch("pathlib.Path.exists", return_value=False) validator._validate_path_exists(False, "field", Path("1")) - mocker.patch("pathlib.Path.exists", return_value=False) - validator._validate_path_exists(True, "field", Path("2")) - mocker.patch("pathlib.Path.exists", return_value=True) + validator._validate_path_exists(False, "field", Path("2")) + + mocker.patch("pathlib.Path.exists", return_value=False) validator._validate_path_exists(True, "field", Path("3")) + mocker.patch("pathlib.Path.exists", return_value=True) + validator._validate_path_exists(True, "field", Path("4")) + error_mock.assert_has_calls([ - MockCall("field", "Path 2 must exist"), + MockCall("field", "Path 2 must not exist"), + MockCall("field", "Path 3 must exist"), ]) diff --git a/tests/ahriman/core/conftest.py b/tests/ahriman/core/conftest.py index 515b8c16..53cfcd1d 100644 --- a/tests/ahriman/core/conftest.py +++ b/tests/ahriman/core/conftest.py @@ -4,6 +4,7 @@ import pytest from ahriman.core.alpm.repo import Repo from ahriman.core.build_tools.task import Task from ahriman.core.configuration import Configuration +from ahriman.core.sign.gpg import GPG from ahriman.core.tree import Leaf from ahriman.models.package import Package from ahriman.models.repository_paths import RepositoryPaths @@ -63,6 +64,20 @@ def repo(configuration: Configuration, repository_paths: RepositoryPaths) -> Rep return Repo(configuration.get("repository", "name"), repository_paths, []) +@pytest.fixture +def gpg(configuration: Configuration) -> GPG: + """ + fixture for empty GPG + + Args: + configuration(Configuration): configuration fixture + + Returns: + GPG: GPG test instance + """ + return GPG(configuration) + + @pytest.fixture def task_ahriman(package_ahriman: Package, configuration: Configuration, repository_paths: RepositoryPaths) -> Task: """ diff --git a/tests/ahriman/core/report/test_html.py b/tests/ahriman/core/report/test_html.py index 1c4fc996..fa790b6c 100644 --- a/tests/ahriman/core/report/test_html.py +++ b/tests/ahriman/core/report/test_html.py @@ -16,4 +16,4 @@ def test_generate(configuration: Configuration, package_ahriman: Package, mocker report = HTML("x86_64", configuration, "html") report.generate([package_ahriman], Result()) - write_mock.assert_called_once_with(pytest.helpers.anyvar(int)) + write_mock.assert_called_once_with(pytest.helpers.anyvar(int), encoding="utf8") diff --git a/tests/ahriman/core/sign/conftest.py b/tests/ahriman/core/sign/conftest.py index 6351b062..e5bc6e7e 100644 --- a/tests/ahriman/core/sign/conftest.py +++ b/tests/ahriman/core/sign/conftest.py @@ -1,23 +1,8 @@ import pytest -from ahriman.core.configuration import Configuration from ahriman.core.sign.gpg import GPG -@pytest.fixture -def gpg(configuration: Configuration) -> GPG: - """ - fixture for empty GPG - - Args: - configuration(Configuration): configuration fixture - - Returns: - GPG: GPG test instance - """ - return GPG("x86_64", configuration) - - @pytest.fixture def gpg_with_key(gpg: GPG) -> GPG: """ diff --git a/tests/ahriman/core/sign/test_gpg.py b/tests/ahriman/core/sign/test_gpg.py index 9a36755a..e5427ffd 100644 --- a/tests/ahriman/core/sign/test_gpg.py +++ b/tests/ahriman/core/sign/test_gpg.py @@ -97,6 +97,33 @@ def test_key_download_failure(gpg: GPG, mocker: MockerFixture) -> None: gpg.key_download("keyserver.ubuntu.com", "0xE989490C") +def test_key_export(gpg: GPG, mocker: MockerFixture) -> None: + """ + must export gpg key correctly + """ + check_output_mock = mocker.patch("ahriman.core.sign.gpg.GPG._check_output", return_value="key") + assert gpg.key_export("k") == "key" + check_output_mock.assert_called_once_with("gpg", "--armor", "--no-emit-version", "--export", "k", + logger=pytest.helpers.anyvar(int)) + + +def test_key_fingerprint(gpg: GPG, mocker: MockerFixture) -> None: + """ + must extract fingerprint + """ + check_output_mock = mocker.patch( + "ahriman.core.sign.gpg.GPG._check_output", + return_value="""tru::1:1576103830:0:3:1:5 +fpr:::::::::C6EBB9222C3C8078631A0DE4BD2AC8C5E989490C: +sub:-:4096:1:7E3A4240CE3C45C2:1615121387::::::e::::::23: +fpr:::::::::43A663569A07EE1E4ECC55CC7E3A4240CE3C45C2:""") + + key = "0xCE3C45C2" + assert gpg.key_fingerprint(key) == "C6EBB9222C3C8078631A0DE4BD2AC8C5E989490C" + check_output_mock.assert_called_once_with("gpg", "--with-colons", "--fingerprint", key, + logger=pytest.helpers.anyvar(int)) + + def test_key_import(gpg: GPG, mocker: MockerFixture) -> None: """ must import PGP key from the server @@ -108,6 +135,21 @@ def test_key_import(gpg: GPG, mocker: MockerFixture) -> None: check_output_mock.assert_called_once_with("gpg", "--import", input_data="key", logger=pytest.helpers.anyvar(int)) +def test_keys(gpg: GPG) -> None: + """ + must extract keys + """ + assert gpg.keys() == [] + + gpg.default_key = "key" + assert gpg.keys() == [gpg.default_key] + + gpg.configuration.set_option("sign", "key_a", "key1") + gpg.configuration.set_option("sign", "key_b", "key1") + gpg.configuration.set_option("sign", "key_c", "key2") + assert gpg.keys() == ["key", "key1", "key2"] + + def test_process(gpg_with_key: GPG, mocker: MockerFixture) -> None: """ must call process method correctly diff --git a/tests/ahriman/core/support/conftest.py b/tests/ahriman/core/support/conftest.py new file mode 100644 index 00000000..6e85fca9 --- /dev/null +++ b/tests/ahriman/core/support/conftest.py @@ -0,0 +1,34 @@ +import pytest + +from ahriman.core.configuration import Configuration +from ahriman.core.support.package_creator import PackageCreator +from ahriman.core.support.pkgbuild.mirrorlist_generator import MirrorlistGenerator + + +@pytest.fixture +def mirrorlist_generator(configuration: Configuration) -> MirrorlistGenerator: + """ + fixture for mirrorlist pkgbuild generator + + Args: + configuration(Configuration): configuration fixture + + Returns: + MirrorlistGenerator: mirrorlist pkgbuild generator test instance + """ + return MirrorlistGenerator(configuration, "mirrorlist") + + +@pytest.fixture +def package_creator(configuration: Configuration, mirrorlist_generator: MirrorlistGenerator) -> PackageCreator: + """ + package creator fixture + + Args: + configuration(Configuration): configuration fixture + mirrorlist_generator(MirrorlistGenerator): + + Returns: + PackageCreator: package creator test instance + """ + return PackageCreator(configuration, mirrorlist_generator) diff --git a/tests/ahriman/core/support/pkgbuild/conftest.py b/tests/ahriman/core/support/pkgbuild/conftest.py new file mode 100644 index 00000000..af061f1d --- /dev/null +++ b/tests/ahriman/core/support/pkgbuild/conftest.py @@ -0,0 +1,32 @@ +import pytest + +from ahriman.core.configuration import Configuration +from ahriman.core.sign.gpg import GPG +from ahriman.core.support.pkgbuild.keyring_generator import KeyringGenerator +from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator + + +@pytest.fixture +def keyring_generator(gpg: GPG, configuration: Configuration) -> KeyringGenerator: + """ + fixture for keyring pkgbuild generator + + Args: + gpg(GPG): empty GPG fixture + configuration(Configuration): configuration fixture + + Returns: + KeyringGenerator: keyring generator test instance + """ + return KeyringGenerator(gpg, configuration, "keyring") + + +@pytest.fixture +def pkgbuild_generator() -> PkgbuildGenerator: + """ + fixture for dummy pkgbuild generator + + Returns: + PkgbuildGenerator: pkgbuild generator test instance + """ + return PkgbuildGenerator() diff --git a/tests/ahriman/core/support/pkgbuild/test_keyring_generator.py b/tests/ahriman/core/support/pkgbuild/test_keyring_generator.py new file mode 100644 index 00000000..5ae8b70d --- /dev/null +++ b/tests/ahriman/core/support/pkgbuild/test_keyring_generator.py @@ -0,0 +1,185 @@ +import pytest + +from pathlib import Path +from pytest_mock import MockerFixture +from unittest.mock import MagicMock, call as MockCall + +from ahriman.core.configuration import Configuration +from ahriman.core.exceptions import PkgbuildGeneratorError +from ahriman.core.sign.gpg import GPG +from ahriman.core.support.pkgbuild.keyring_generator import KeyringGenerator + + +def test_init_packagers(gpg: GPG, configuration: Configuration, mocker: MockerFixture) -> None: + """ + must extract packagers keys + """ + mocker.patch("ahriman.core.sign.gpg.GPG.keys", return_value=["key"]) + + assert KeyringGenerator(gpg, configuration, "keyring").packagers == ["key"] + + configuration.set_option("keyring", "packagers", "key1") + assert KeyringGenerator(gpg, configuration, "keyring").packagers == ["key1"] + + +def test_init_revoked(gpg: GPG, configuration: Configuration) -> None: + """ + must extract revoked keys + """ + assert KeyringGenerator(gpg, configuration, "keyring").revoked == [] + + configuration.set_option("keyring", "revoked", "key1") + assert KeyringGenerator(gpg, configuration, "keyring").revoked == ["key1"] + + +def test_init_trusted(gpg: GPG, configuration: Configuration) -> None: + """ + must extract trusted keys + """ + assert KeyringGenerator(gpg, configuration, "keyring").trusted == [] + + gpg.default_key = "key" + assert KeyringGenerator(gpg, configuration, "keyring").trusted == ["key"] + + configuration.set_option("keyring", "trusted", "key1") + assert KeyringGenerator(gpg, configuration, "keyring").trusted == ["key1"] + + +def test_license(gpg: GPG, configuration: Configuration) -> None: + """ + must generate correct licenses list + """ + assert KeyringGenerator(gpg, configuration, "keyring").license == ["Unlicense"] + + configuration.set_option("keyring", "license", "GPL MPL") + assert KeyringGenerator(gpg, configuration, "keyring").license == ["GPL", "MPL"] + + +def test_pkgdesc(gpg: GPG, configuration: Configuration) -> None: + """ + must generate correct pkgdesc property + """ + assert KeyringGenerator(gpg, configuration, "keyring").pkgdesc == "aur-clone PGP keyring" + + configuration.set_option("keyring", "description", "description") + assert KeyringGenerator(gpg, configuration, "keyring").pkgdesc == "description" + + +def test_pkgname(gpg: GPG, configuration: Configuration) -> None: + """ + must generate correct pkgname property + """ + assert KeyringGenerator(gpg, configuration, "keyring").pkgname == "aur-clone-keyring" + + configuration.set_option("keyring", "package", "keyring") + assert KeyringGenerator(gpg, configuration, "keyring").pkgname == "keyring" + + +def test_url(gpg: GPG, configuration: Configuration) -> None: + """ + must generate correct url property + """ + assert KeyringGenerator(gpg, configuration, "keyring").url == "" + + configuration.set_option("keyring", "homepage", "homepage") + assert KeyringGenerator(gpg, configuration, "keyring").url == "homepage" + + +def test_generate_gpg(keyring_generator: KeyringGenerator, mocker: MockerFixture) -> None: + """ + must correctly generate file with all PGP keys + """ + file_mock = MagicMock() + export_mock = mocker.patch("ahriman.core.sign.gpg.GPG.key_export", side_effect=lambda key: key) + open_mock = mocker.patch("pathlib.Path.open") + open_mock.return_value.__enter__.return_value = file_mock + keyring_generator.packagers = ["key"] + keyring_generator.revoked = ["revoked"] + keyring_generator.trusted = ["trusted", "key"] + + keyring_generator._generate_gpg(Path("local")) + open_mock.assert_called_once_with("w") + export_mock.assert_has_calls([MockCall("key"), MockCall("revoked"), MockCall("trusted")]) + file_mock.write.assert_has_calls([ + MockCall("key"), MockCall("\n"), + MockCall("revoked"), MockCall("\n"), + MockCall("trusted"), MockCall("\n"), + ]) + + +def test_generate_revoked(keyring_generator: KeyringGenerator, mocker: MockerFixture) -> None: + """ + must correctly generate file with revoked keys + """ + file_mock = MagicMock() + fingerprint_mock = mocker.patch("ahriman.core.sign.gpg.GPG.key_fingerprint", side_effect=lambda key: key) + open_mock = mocker.patch("pathlib.Path.open") + open_mock.return_value.__enter__.return_value = file_mock + keyring_generator.revoked = ["revoked"] + + keyring_generator._generate_revoked(Path("local")) + open_mock.assert_called_once_with("w") + fingerprint_mock.assert_called_once_with("revoked") + file_mock.write.assert_has_calls([MockCall("revoked"), MockCall("\n")]) + + +def test_generate_trusted(keyring_generator: KeyringGenerator, mocker: MockerFixture) -> None: + """ + must correctly generate file with trusted keys + """ + file_mock = MagicMock() + fingerprint_mock = mocker.patch("ahriman.core.sign.gpg.GPG.key_fingerprint", side_effect=lambda key: key) + open_mock = mocker.patch("pathlib.Path.open") + open_mock.return_value.__enter__.return_value = file_mock + keyring_generator.trusted = ["trusted", "trusted"] + + keyring_generator._generate_trusted(Path("local")) + open_mock.assert_called_once_with("w") + fingerprint_mock.assert_called_once_with("trusted") + file_mock.write.assert_has_calls([MockCall("trusted"), MockCall(":4:\n")]) + + +def test_generate_trusted_empty(keyring_generator: KeyringGenerator) -> None: + """ + must raise PkgbuildGeneratorError if no trusted keys set + """ + with pytest.raises(PkgbuildGeneratorError): + keyring_generator._generate_trusted(Path("local")) + + +def test_install(keyring_generator: KeyringGenerator) -> None: + """ + must return install functions + """ + assert keyring_generator.install() == """post_upgrade() { + if usr/bin/pacman-key -l >/dev/null 2>&1; then + usr/bin/pacman-key --populate aur-clone + usr/bin/pacman-key --updatedb + fi +} + +post_install() { + if [ -x usr/bin/pacman-key ]; then + post_upgrade + fi +}""" + + +def test_package(keyring_generator: KeyringGenerator) -> None: + """ + must generate package function correctly + """ + assert keyring_generator.package() == """{ + install -Dm644 "$srcdir/aur-clone.gpg" "$pkgdir/usr/share/pacman/keyrings/aur-clone.gpg" + install -Dm644 "$srcdir/aur-clone-revoked" "$pkgdir/usr/share/pacman/keyrings/aur-clone-revoked" + install -Dm644 "$srcdir/aur-clone-trusted" "$pkgdir/usr/share/pacman/keyrings/aur-clone-trusted" +}""" + + +def test_sources(keyring_generator: KeyringGenerator) -> None: + """ + must return valid sources files list + """ + assert keyring_generator.sources().get("aur-clone.gpg") + assert keyring_generator.sources().get("aur-clone-revoked") + assert keyring_generator.sources().get("aur-clone-trusted") diff --git a/tests/ahriman/core/support/pkgbuild/test_mirrorlist_generator.py b/tests/ahriman/core/support/pkgbuild/test_mirrorlist_generator.py new file mode 100644 index 00000000..6e5c3553 --- /dev/null +++ b/tests/ahriman/core/support/pkgbuild/test_mirrorlist_generator.py @@ -0,0 +1,90 @@ +from pathlib import Path +from pytest_mock import MockerFixture + +from ahriman.core.configuration import Configuration +from ahriman.core.support.pkgbuild.mirrorlist_generator import MirrorlistGenerator + + +def test_init_path(configuration: Configuration) -> None: + """ + must set relative path to mirrorlist + """ + assert MirrorlistGenerator(configuration, "mirrorlist").path == Path("etc") / "pacman.d" / "aur-clone-mirrorlist" + + configuration.set_option("mirrorlist", "path", "/etc") + assert MirrorlistGenerator(configuration, "mirrorlist").path == Path("etc") + + +def test_license(configuration: Configuration) -> None: + """ + must generate correct licenses list + """ + assert MirrorlistGenerator(configuration, "mirrorlist").license == ["Unlicense"] + + configuration.set_option("mirrorlist", "license", "GPL MPL") + assert MirrorlistGenerator(configuration, "mirrorlist").license == ["GPL", "MPL"] + + +def test_pkgdesc(configuration: Configuration) -> None: + """ + must generate correct pkgdesc property + """ + assert MirrorlistGenerator(configuration, "mirrorlist").pkgdesc == "aur-clone mirror list for use by pacman" + + configuration.set_option("mirrorlist", "description", "description") + assert MirrorlistGenerator(configuration, "mirrorlist").pkgdesc == "description" + + +def test_pkgname(configuration: Configuration) -> None: + """ + must generate correct pkgname property + """ + assert MirrorlistGenerator(configuration, "mirrorlist").pkgname == "aur-clone-mirrorlist" + + configuration.set_option("mirrorlist", "package", "mirrorlist") + assert MirrorlistGenerator(configuration, "mirrorlist").pkgname == "mirrorlist" + + +def test_url(configuration: Configuration) -> None: + """ + must generate correct url property + """ + assert MirrorlistGenerator(configuration, "mirrorlist").url == "" + + configuration.set_option("mirrorlist", "homepage", "homepage") + assert MirrorlistGenerator(configuration, "mirrorlist").url == "homepage" + + +def test_generate_mirrorlist(mirrorlist_generator: MirrorlistGenerator, mocker: MockerFixture) -> None: + """ + must correctly generate mirrorlist file + """ + write_mock = mocker.patch("pathlib.Path.write_text") + mirrorlist_generator._generate_mirrorlist(Path("local")) + write_mock.assert_called_once_with("Server = http://localhost\n", encoding="utf8") + + +def test_package(mirrorlist_generator: MirrorlistGenerator) -> None: + """ + must generate package function correctly + """ + assert mirrorlist_generator.package() == """{ + install -Dm644 "$srcdir/mirrorlist" "$pkgdir/etc/pacman.d/aur-clone-mirrorlist" +}""" + + +def test_patches(mirrorlist_generator: MirrorlistGenerator) -> None: + """ + must generate additional patch list + """ + patches = {patch.key: patch for patch in mirrorlist_generator.patches()} + + assert "backup" in patches + assert patches["backup"].value == [str(mirrorlist_generator.path)] + + +def test_sources(mirrorlist_generator: MirrorlistGenerator) -> None: + """ + must return valid sources files list + """ + assert mirrorlist_generator.sources().get("mirrorlist") diff --git a/tests/ahriman/core/support/pkgbuild/test_pkgbuild_generator.py b/tests/ahriman/core/support/pkgbuild/test_pkgbuild_generator.py new file mode 100644 index 00000000..d4857c71 --- /dev/null +++ b/tests/ahriman/core/support/pkgbuild/test_pkgbuild_generator.py @@ -0,0 +1,141 @@ +import datetime +import pytest + +from pathlib import Path +from pytest_mock import MockerFixture +from unittest.mock import MagicMock, call as MockCall + +from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator +from ahriman.core.util import utcnow +from ahriman.models.pkgbuild_patch import PkgbuildPatch + + +def test_license(pkgbuild_generator: PkgbuildGenerator) -> None: + """ + must return empty license list + """ + assert pkgbuild_generator.license == [] + + +def test_pkgdesc(pkgbuild_generator: PkgbuildGenerator) -> None: + """ + must raise NotImplementedError on missing pkgdesc property + """ + with pytest.raises(NotImplementedError): + assert pkgbuild_generator.pkgdesc + + +def test_pkgname(pkgbuild_generator: PkgbuildGenerator) -> None: + """ + must raise NotImplementedError on missing pkgname property + """ + with pytest.raises(NotImplementedError): + assert pkgbuild_generator.pkgname + + +def test_pkgver(pkgbuild_generator: PkgbuildGenerator, mocker: MockerFixture) -> None: + """ + must implement default version as current date + """ + mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.utcnow", return_value=datetime.datetime(2002, 3, 11)) + assert pkgbuild_generator.pkgver == utcnow().strftime("20020311") + + +def test_url(pkgbuild_generator: PkgbuildGenerator) -> None: + """ + must return empty url + """ + assert pkgbuild_generator.url == "" + + +def test_install(pkgbuild_generator: PkgbuildGenerator) -> None: + """ + must return empty install function + """ + assert pkgbuild_generator.install() is None + + +def test_package(pkgbuild_generator: PkgbuildGenerator) -> None: + """ + must raise NotImplementedError on missing package function + """ + with pytest.raises(NotImplementedError): + pkgbuild_generator.package() + + +def test_patches(pkgbuild_generator: PkgbuildGenerator) -> None: + """ + must return empty patches list + """ + assert pkgbuild_generator.patches() == [] + + +def test_sources(pkgbuild_generator: PkgbuildGenerator) -> None: + """ + must return empty sources list + """ + assert pkgbuild_generator.sources() == {} + + +def test_write_install(pkgbuild_generator: PkgbuildGenerator, mocker: MockerFixture) -> None: + """ + must write install file + """ + mocker.patch.object(PkgbuildGenerator, "pkgname", "package") + mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.PkgbuildGenerator.install", return_value="content") + write_mock = mocker.patch("pathlib.Path.write_text") + + assert pkgbuild_generator.write_install(Path("local")) == [PkgbuildPatch("install", "package.install")] + write_mock.assert_called_once_with("content") + + +def test_write_install_empty(pkgbuild_generator: PkgbuildGenerator) -> None: + """ + must return empty patch list for missing install function + """ + assert pkgbuild_generator.write_install(Path("local")) == [] + + +def test_write_pkgbuild(pkgbuild_generator: PkgbuildGenerator, mocker: MockerFixture) -> None: + """ + must write PKGBUILD content to file + """ + path = Path("local") + for prop in ("pkgdesc", "pkgname"): + mocker.patch.object(PkgbuildGenerator, prop, "") + mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.PkgbuildGenerator.package", return_value="{}") + patches_mock = mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.PkgbuildGenerator.patches", + return_value=[PkgbuildPatch("property", "value")]) + install_mock = mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.PkgbuildGenerator.write_install", + return_value=[PkgbuildPatch("install", "pkgname.install")]) + sources_mock = mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.PkgbuildGenerator.write_sources", + return_value=[PkgbuildPatch("source", []), PkgbuildPatch("sha512sums", [])]) + write_mock = mocker.patch("ahriman.models.pkgbuild_patch.PkgbuildPatch.write") + + pkgbuild_generator.write_pkgbuild(path) + patches_mock.assert_called_once_with() + install_mock.assert_called_once_with(path) + sources_mock.assert_called_once_with(path) + write_mock.assert_has_calls([MockCall(path / "PKGBUILD")] * 12) + + +def test_write_sources(pkgbuild_generator: PkgbuildGenerator, mocker: MockerFixture) -> None: + """ + must write sources files + """ + path = Path("local") + generator_mock = MagicMock() + sources_mock = mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.PkgbuildGenerator.sources", + return_value={"source": generator_mock}) + open_mock = mocker.patch("pathlib.Path.open") + hash_mock = MagicMock() + hash_mock.hexdigest.return_value = "hash" + mocker.patch("hashlib.sha512", return_value=hash_mock) + + assert pkgbuild_generator.write_sources(path) == [ + PkgbuildPatch("source", ["source"]), + PkgbuildPatch("sha512sums", ["hash"]), + ] + generator_mock.assert_called_once_with(path / "source") + sources_mock.assert_called_once_with() + open_mock.assert_called_once_with("rb") diff --git a/tests/ahriman/core/support/test_keyring_trigger.py b/tests/ahriman/core/support/test_keyring_trigger.py new file mode 100644 index 00000000..8c82703f --- /dev/null +++ b/tests/ahriman/core/support/test_keyring_trigger.py @@ -0,0 +1,30 @@ +from pytest_mock import MockerFixture + +from ahriman.core.configuration import Configuration +from ahriman.core.sign.gpg import GPG +from ahriman.core.support import KeyringTrigger +from ahriman.models.context_key import ContextKey + + +def test_configuration_sections(configuration: Configuration) -> None: + """ + must correctly parse target list + """ + configuration.set_option("keyring", "target", "a b c") + assert KeyringTrigger.configuration_sections(configuration) == ["a", "b", "c"] + + configuration.remove_option("keyring", "target") + assert KeyringTrigger.configuration_sections(configuration) == [] + + +def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must run report for specified targets + """ + gpg_mock = mocker.patch("ahriman.core._Context.get") + run_mock = mocker.patch("ahriman.core.support.package_creator.PackageCreator.run") + + trigger = KeyringTrigger("x86_64", configuration) + trigger.on_start() + gpg_mock.assert_called_once_with(ContextKey("sign", GPG)) + run_mock.assert_called_once_with() diff --git a/tests/ahriman/core/support/test_mirrorlist_trigger.py b/tests/ahriman/core/support/test_mirrorlist_trigger.py new file mode 100644 index 00000000..e5e79013 --- /dev/null +++ b/tests/ahriman/core/support/test_mirrorlist_trigger.py @@ -0,0 +1,26 @@ +from pytest_mock import MockerFixture + +from ahriman.core.configuration import Configuration +from ahriman.core.support import MirrorlistTrigger + + +def test_configuration_sections(configuration: Configuration) -> None: + """ + must correctly parse target list + """ + configuration.set_option("mirrorlist", "target", "a b c") + assert MirrorlistTrigger.configuration_sections(configuration) == ["a", "b", "c"] + + configuration.remove_option("mirrorlist", "target") + assert MirrorlistTrigger.configuration_sections(configuration) == [] + + +def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must run report for specified targets + """ + run_mock = mocker.patch("ahriman.core.support.package_creator.PackageCreator.run") + + trigger = MirrorlistTrigger("x86_64", configuration) + trigger.on_start() + run_mock.assert_called_once_with() diff --git a/tests/ahriman/core/support/test_package_creator.py b/tests/ahriman/core/support/test_package_creator.py new file mode 100644 index 00000000..616df485 --- /dev/null +++ b/tests/ahriman/core/support/test_package_creator.py @@ -0,0 +1,40 @@ +import pytest + +from pytest_mock import MockerFixture + +from ahriman.core.database import SQLite +from ahriman.core.support.package_creator import PackageCreator +from ahriman.models.context_key import ContextKey +from ahriman.models.package import Package +from ahriman.models.package_description import PackageDescription + + +def test_run(package_creator: PackageCreator, database: SQLite, mocker: MockerFixture) -> None: + """ + must correctly process package creation + """ + package = Package( + base=package_creator.generator.pkgname, + version=package_creator.generator.pkgver, + remote=None, + packages={package_creator.generator.pkgname: PackageDescription()}, + ) + local_path = package_creator.configuration.repository_paths.cache_for(package_creator.generator.pkgname) + + rmtree_mock = mocker.patch("shutil.rmtree") + database_mock = mocker.patch("ahriman.core._Context.get", return_value=database) + init_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.init") + insert_mock = mocker.patch("ahriman.core.database.SQLite.package_update") + mkdir_mock = mocker.patch("pathlib.Path.mkdir") + package_mock = mocker.patch("ahriman.models.package.Package.from_build", return_value=package) + write_mock = mocker.patch("ahriman.core.support.pkgbuild.pkgbuild_generator.PkgbuildGenerator.write_pkgbuild") + + package_creator.run() + rmtree_mock.assert_called_once_with(local_path, ignore_errors=True) + mkdir_mock.assert_called_once_with(mode=0o755, parents=True, exist_ok=True) + write_mock.assert_called_once_with(local_path) + init_mock.assert_called_once_with(local_path) + + package_mock.assert_called_once_with(local_path, "x86_64") + database_mock.assert_called_once_with(ContextKey("database", SQLite)) + insert_mock.assert_called_once_with(package, pytest.helpers.anyvar(int)) diff --git a/tests/ahriman/core/test_util.py b/tests/ahriman/core/test_util.py index 31c4bb53..a53a3d0c 100644 --- a/tests/ahriman/core/test_util.py +++ b/tests/ahriman/core/test_util.py @@ -12,7 +12,8 @@ from unittest.mock import MagicMock from ahriman.core.exceptions import BuildError, OptionError, UnsafeRunError from ahriman.core.util import check_output, check_user, enum_values, exception_response_text, filter_json, \ - full_version, package_like, partition, pretty_datetime, pretty_size, safe_filename, trim_package, utcnow, walk + full_version, package_like, partition, pretty_datetime, pretty_size, safe_filename, srcinfo_property, \ + srcinfo_property_list, trim_package, utcnow, walk from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.repository_paths import RepositoryPaths @@ -81,12 +82,12 @@ def test_check_output_with_user(passwd: Any, mocker: MockerFixture) -> None: """ must run command as specified user and set its homedir """ - assert check_output("python", "-c", "import os; print(os.getenv('HOME'))") != passwd.pw_dir + assert check_output("python", "-c", """import os; print(os.getenv("HOME"))""") != passwd.pw_dir getpwuid_mock = mocker.patch("ahriman.core.util.getpwuid", return_value=passwd) user = os.getuid() - assert check_output("python", "-c", "import os; print(os.getenv('HOME'))", user=user) == passwd.pw_dir + assert check_output("python", "-c", """import os; print(os.getenv("HOME"))""", user=user) == passwd.pw_dir getpwuid_mock.assert_called_once_with(user) @@ -331,6 +332,31 @@ def test_safe_filename() -> None: assert safe_filename("tolua++-1.0.93-4-x86_64.pkg.tar.zst") == "tolua---1.0.93-4-x86_64.pkg.tar.zst" +def test_srcinfo_property() -> None: + """ + must correctly extract properties + """ + assert srcinfo_property("key", {"key": "root"}, {"key": "overrides"}, default="default") == "overrides" + assert srcinfo_property("key", {"key": "root"}, {}, default="default") == "root" + assert srcinfo_property("key", {}, {"key": "overrides"}, default="default") == "overrides" + assert srcinfo_property("key", {}, {}, default="default") == "default" + assert srcinfo_property("key", {}, {}) is None + + +def test_srcinfo_property_list() -> None: + """ + must correctly extract property list + """ + assert srcinfo_property_list("key", {"key": ["root"]}, {"key": ["overrides"]}) == ["overrides"] + assert srcinfo_property_list("key", {"key": ["root"]}, {"key_x86_64": ["overrides"]}, architecture="x86_64") == [ + "root", "overrides" + ] + assert srcinfo_property_list("key", {"key": ["root"], "key_x86_64": ["overrides"]}, {}, architecture="x86_64") == [ + "root", "overrides" + ] + assert srcinfo_property_list("key", {"key_x86_64": ["overrides"]}, {}, architecture="x86_64") == ["overrides"] + + def test_trim_package() -> None: """ must trim package version diff --git a/tests/ahriman/core/triggers/test_trigger_loader.py b/tests/ahriman/core/triggers/test_trigger_loader.py index a43d1596..187a8451 100644 --- a/tests/ahriman/core/triggers/test_trigger_loader.py +++ b/tests/ahriman/core/triggers/test_trigger_loader.py @@ -11,6 +11,17 @@ from ahriman.models.package import Package from ahriman.models.result import Result +def test_known_triggers(configuration: Configuration) -> None: + """ + must return used triggers + """ + configuration.set_option("build", "triggers_known", "a b c") + assert TriggerLoader.known_triggers(configuration) == ["a", "b", "c"] + + configuration.remove_option("build", "triggers_known") + assert TriggerLoader.known_triggers(configuration) == [] + + def test_selected_triggers(configuration: Configuration) -> None: """ must return used triggers diff --git a/tests/ahriman/models/test_package.py b/tests/ahriman/models/test_package.py index 123e061d..932b5125 100644 --- a/tests/ahriman/models/test_package.py +++ b/tests/ahriman/models/test_package.py @@ -2,6 +2,7 @@ import pytest from pathlib import Path from pytest_mock import MockerFixture +from srcinfo.parse import parse_srcinfo from unittest.mock import MagicMock from ahriman.core.alpm.pacman import Pacman @@ -165,7 +166,7 @@ def test_from_build(package_ahriman: Package, mocker: MockerFixture, resource_pa package = Package.from_build(Path("path"), "x86_64") assert package_ahriman.packages.keys() == package.packages.keys() package_ahriman.packages = package.packages # we are not going to test PackageDescription here - package_ahriman.remote = None + package_ahriman.remote = package.remote assert package_ahriman == package @@ -269,6 +270,70 @@ def test_from_official(package_ahriman: Package, aur_package_ahriman: AURPackage assert package_ahriman.packages.keys() == package.packages.keys() +def test_local_files(mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must extract local file sources + """ + srcinfo = (resource_path_root / "models" / "package_yay_srcinfo").read_text() + parsed_srcinfo, _ = parse_srcinfo(srcinfo) + parsed_srcinfo["source"] = ["local-file.tar.gz"] + mocker.patch("ahriman.models.package.parse_srcinfo", return_value=(parsed_srcinfo, [])) + mocker.patch("ahriman.models.package.Package._check_output", return_value=srcinfo) + mocker.patch("ahriman.models.package.Package.supported_architectures", return_value=["any"]) + + assert list(Package.local_files(Path("path"))) == [Path("local-file.tar.gz")] + + +def test_local_files_empty(mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must extract empty local files list when there is no local files + """ + srcinfo = (resource_path_root / "models" / "package_yay_srcinfo").read_text() + mocker.patch("ahriman.models.package.Package._check_output", return_value=srcinfo) + mocker.patch("ahriman.models.package.Package.supported_architectures", return_value=["any"]) + + assert list(Package.local_files(Path("path"))) == [] + + +def test_local_files_error(mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must raise exception on package parsing for local sources + """ + mocker.patch("ahriman.models.package.Package._check_output", return_value="") + mocker.patch("ahriman.models.package.parse_srcinfo", return_value=({"packages": {}}, ["an error"])) + + with pytest.raises(PackageInfoError): + list(Package.local_files(Path("path"))) + + +def test_local_files_schema(mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must skip local file source when file schema is used + """ + srcinfo = (resource_path_root / "models" / "package_yay_srcinfo").read_text() + parsed_srcinfo, _ = parse_srcinfo(srcinfo) + parsed_srcinfo["source"] = ["file:///local-file.tar.gz"] + mocker.patch("ahriman.models.package.parse_srcinfo", return_value=(parsed_srcinfo, [])) + mocker.patch("ahriman.models.package.Package._check_output", return_value="") + mocker.patch("ahriman.models.package.Package.supported_architectures", return_value=["any"]) + + assert list(Package.local_files(Path("path"))) == [] + + +def test_local_files_with_install(mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must extract local file sources with install file + """ + srcinfo = (resource_path_root / "models" / "package_yay_srcinfo").read_text() + parsed_srcinfo, _ = parse_srcinfo(srcinfo) + parsed_srcinfo["install"] = "install" + mocker.patch("ahriman.models.package.parse_srcinfo", return_value=(parsed_srcinfo, [])) + mocker.patch("ahriman.models.package.Package._check_output", return_value="") + mocker.patch("ahriman.models.package.Package.supported_architectures", return_value=["any"]) + + assert list(Package.local_files(Path("path"))) == [Path("install")] + + def test_supported_architectures(mocker: MockerFixture, resource_path_root: Path) -> None: """ must generate list of available architectures diff --git a/tests/testresources/core/ahriman.ini b/tests/testresources/core/ahriman.ini index 8dfb33fe..ad35c8aa 100644 --- a/tests/testresources/core/ahriman.ini +++ b/tests/testresources/core/ahriman.ini @@ -25,6 +25,7 @@ ignore_packages = makechrootpkg_flags = makepkg_flags = --skippgpcheck triggers = ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger +triggers_known = ahriman.core.support.KeyringTrigger ahriman.core.support.MirrorlistTrigger [repository] name = aur-clone @@ -33,6 +34,13 @@ root = ../../../ [sign] target = +[keyring] +target = keyring + +[mirrorlist] +target = mirrorlist +servers = http://localhost + [remote-push] target = gitremote