From 22847aac686144f8b3a84492d9adc0624fde4c2b Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Sat, 4 Jun 2022 05:39:47 +0300 Subject: [PATCH] small docs update --- .pylintrc | 73 +------------------- README.md | 14 ++-- docs/architecture.rst | 10 +-- docs/configuration.rst | 11 +-- docs/index.rst | 6 +- docs/setup.rst | 33 ++++++--- docs/triggers.rst | 14 ++-- src/ahriman/application/handlers/status.py | 2 +- src/ahriman/application/handlers/versions.py | 2 +- src/ahriman/core/auth/auth.py | 6 +- src/ahriman/core/configuration.py | 2 +- src/ahriman/core/formatters/printer.py | 2 +- src/ahriman/core/status/client.py | 4 +- src/ahriman/models/package.py | 6 +- 14 files changed, 64 insertions(+), 121 deletions(-) diff --git a/.pylintrc b/.pylintrc index 9a893e97..70e65d54 100644 --- a/.pylintrc +++ b/.pylintrc @@ -60,17 +60,7 @@ confidence= # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use "--disable=all --enable=classes # --disable=W". -disable=print-statement, - parameter-unpacking, - unpacking-in-except, - old-raise-syntax, - backtick, - long-suffix, - old-ne-operator, - old-octal-literal, - import-star-module-level, - non-ascii-bytes-literal, - raw-checker-failed, +disable=raw-checker-failed, bad-inline-option, locally-disabled, file-ignored, @@ -78,67 +68,6 @@ disable=print-statement, useless-suppression, deprecated-pragma, use-symbolic-message-instead, - apply-builtin, - basestring-builtin, - buffer-builtin, - cmp-builtin, - coerce-builtin, - execfile-builtin, - file-builtin, - long-builtin, - raw_input-builtin, - reduce-builtin, - standarderror-builtin, - unicode-builtin, - xrange-builtin, - coerce-method, - delslice-method, - getslice-method, - setslice-method, - no-absolute-import, - old-division, - dict-iter-method, - dict-view-method, - next-method-called, - metaclass-assignment, - indexing-exception, - raising-string, - reload-builtin, - oct-method, - hex-method, - nonzero-method, - cmp-method, - input-builtin, - round-builtin, - intern-builtin, - unichr-builtin, - map-builtin-not-iterating, - zip-builtin-not-iterating, - range-builtin-not-iterating, - filter-builtin-not-iterating, - using-cmp-argument, - eq-without-hash, - div-method, - idiv-method, - rdiv-method, - exception-message-attribute, - invalid-str-codec, - sys-max-int, - bad-python3-import, - deprecated-string-function, - deprecated-str-translate-call, - deprecated-itertools-function, - deprecated-types-field, - next-method-defined, - dict-items-not-iterating, - dict-keys-not-iterating, - dict-values-not-iterating, - deprecated-operator-function, - deprecated-urllib-function, - xreadlines-attribute, - deprecated-sys-function, - exception-escape, - comprehension-escape, missing-module-docstring, line-too-long, no-name-in-module, diff --git a/README.md b/README.md index 3f9a3072..0d2f09bc 100644 --- a/README.md +++ b/README.md @@ -10,24 +10,26 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github ## Features -* Install-configure-forget manager for own repository. +* Install-configure-forget manager for the very own repository. * Multi-architecture support. +* Dependency manager. * VCS packages support. * Official repository support. +* Ability to patch AUR packages and even create package from local PKGBUILDs. * Sign support with gpg (repository, package, per package settings). * Triggers for repository updates, e.g. synchronization to remote services (rsync, s3 and github) and report generation (email, html, telegram). -* Dependency manager. -* Ability to patch AUR packages and even create package from local PKGBUILDs. * Repository status interface with optional authorization and control options: ![web interface](web.png) ## Installation and run -For installation details please refer to the [documentation](docs/setup.rst). For command help, `--help` subcommand must be used. Subcommands have own help message as well. The package also provides a [man page](docs/ahriman.1). +For installation details kindly refer to the [documentation](https://ahriman.readthedocs.io/en/latest/setup.html). For application commands it is possible to get information by using `--help`/`help` command or by using man page ([web version](https://ahriman.readthedocs.io/en/latest/command-line.html)). ## Configuration -Every available option is described in the [documentation](docs/configuration.rst). +Every available option is described in the [documentation](https://ahriman.readthedocs.io/en/latest/configuration.html). -## [FAQ](docs/faq.rst) +The application provides reasonable defaults which allow to use it out-of-box, though additional steps (like configuring build toolchain and sudoers) is recommended and can be easily achieved by following install instructions. + +## [FAQ](https://ahriman.readthedocs.io/en/latest/faq.html) diff --git a/docs/architecture.rst b/docs/architecture.rst index 07b7b62c..0659de8e 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -21,14 +21,14 @@ Full dependency diagram: This package contains application (aka executable) related classes and everything for that. It also contains package called ``ahriman.application.handlers`` in which all available subcommands are described as separated classes derived from base ``ahriman.application.handlers.Handler`` class. -``ahriman.application.application.Application`` (god class) is used for any interaction from parsers with repository, web etc. It is divided into multiple traits by functions (package related and repository related) in the same package. +``ahriman.application.application.Application`` (god class) is used for any interaction from parsers with repository. It is divided into multiple traits by functions (package related and repository related) in the same package. ``ahriman.application.ahriman`` contains only command line parses and executes specified ``Handler`` on success, ``ahriman.application.lock.Lock`` is additional class which provides file-based lock and also performs some common checks. ``ahriman.core`` package ^^^^^^^^^^^^^^^^^^^^^^^^ -This package contains everything which is required for any time of application run and separated to several packages: +This package contains everything which is required for any time of application run and separated into several packages: * ``ahriman.core.alpm`` package controls pacman related functions. It provides wrappers for ``pyalpm`` library and safe calls for repository tools (``repo-add`` and ``repo-remove``). Also this package contains ``ahriman.core.alpm.remote`` package which provides wrapper for remote sources (e.g. AUR RPC and official repositories RPC). * ``ahriman.core.auth`` package provides classes for authorization methods used by web mostly. Base class is ``ahriman.core.auth.Auth`` which must be called by ``load`` method. @@ -39,7 +39,7 @@ This package contains everything which is required for any time of application r * ``ahriman.core.repository`` contains several traits and base repository (``ahriman.core.repository.Repository`` class) implementation. * ``ahriman.core.sign`` package provides sign feature (only gpg calls are available). * ``ahriman.core.status`` contains helpers and watcher class which are required for web application. Reporter must be initialized by using ``ahriman.core.status.client.Client.load`` method. -* ``ahriman.core.triggers`` package contains base trigger classes. Classes from this package must be imported in order to implement user extensions. In fact, ``ahriman.core.report`` and ``ahriman.core.upload`` uses this package. +* ``ahriman.core.triggers`` package contains base trigger classes. Classes from this package must be imported in order to implement user extensions. In fact, ``ahriman.core.report`` and ``ahriman.core.upload`` use this package. * ``ahriman.core.upload`` package provides sync feature, must be called by ``ahriman.core.upload.Upload.load`` method. This package also provides some generic functions and classes which may be used by other packages: @@ -69,7 +69,7 @@ Application run * Parse command line arguments, find command and related handler which is set by parser. * Call ``Handler.execute`` method. -* Define list of architectures to run. In case if there is more than one architecture specified run several subprocesses or process in current process otherwise. Class attribute ``ALLOW_MULTI_ARCHITECTURE_RUN`` controls whether application can be run in multiple processes or not - this feature is required for some handlers (e.g. ``Web``) which should be able to spawn child process in daemon mode (it is impossible to do for daemonic processes). +* Define list of architectures to run. In case if there is more than one architecture specified run several subprocesses or process in current process otherwise. Class attribute ``ALLOW_MULTI_ARCHITECTURE_RUN`` controls whether application can be run in multiple processes or not - this feature is required for some handlers (e.g. ``Web``) which should be able to spawn child process in daemon mode (it is impossible to do from daemonic processes). * In each child process call lock functions. * After success checks pass control to ``Handler.run`` method defined by specific handler class. * Return result (success or failure) of each subprocess and exit from application. @@ -115,7 +115,7 @@ Idea is to copy package to the directory from which it will be handled at the ne * If supplied argument is file then application moves the file to the directory with built packages. Same rule applies for directory, but in this case it copies every package-like file from the specified directory. * If supplied argument is directory and there is ``PKGBUILD`` file there it will be treated as local package. In this case it will queue this package to build and copy source files (``PKGBUILD`` and ``.SRCINFO``) to caches. -* If supplied argument iis not file then application tries to lookup for the specified name in AUR and clones it into the directory with manual updates. This scenario can also handle package dependencies which are missing in repositories. +* If supplied argument is not file then application tries to lookup for the specified name in AUR and clones it into the directory with manual updates. This scenario can also handle package dependencies which are missing in repositories. This logic can be overwritten by specifying the ``source`` parameter, which is partially useful if you would like to add package from AUR, but there is local directory cloned from AUR. diff --git a/docs/configuration.rst b/docs/configuration.rst index c7356f1b..6aa5cf26 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1,7 +1,7 @@ Configuration ============= -Some groups can be specified for each architecture separately. E.g. if there are ``build`` and ``build:x86_64`` groups it will use the option from ``build:x86_64`` for the ``x86_64`` architecture and ``build`` for any other (architecture specific group has higher priority). In case if both groups are presented, architecture specific options will be merged into global ones overriding them. +Some groups can be specified for each architecture separately. E.g. if there are ``build`` and ``build:x86_64`` groups it will use an option from ``build:x86_64`` for the ``x86_64`` architecture and ``build`` for any other (architecture specific group has higher priority). In case if both groups are presented, architecture specific options will be merged into global ones overriding them. There are two variable types which have been added to default ones, they are paths and lists. List values will be read in the same way as shell does: @@ -36,12 +36,12 @@ libalpm and AUR related configuration. Base authorization settings. ``OAuth`` provider requires ``aioauth-client`` library to be installed. * ``target`` - specifies authorization provider, string, optional, default ``disabled``. Allowed values are ``disabled``, ``configuration``, ``oauth``. +* ``allow_read_only`` - allow requesting status APIs without authorization, boolean, required. * ``client_id`` - OAuth2 application client ID, string, required in case if ``oauth`` is used. * ``client_secret`` - OAuth2 application client secret key, string, required in case if ``oauth`` is used. * ``max_age`` - parameter which controls both cookie expiration and token expiration inside the service, integer, optional, default is 7 days. * ``oauth_provider`` - OAuth2 provider class name as is in ``aioauth-client`` (e.g. ``GoogleClient``, ``GithubClient`` etc), string, required in case if ``oauth`` is used. * ``oauth_scopes`` - scopes list for OAuth2 provider, which will allow retrieving user email (which is used for checking user permissions), e.g. ``https://www.googleapis.com/auth/userinfo.email`` for ``GoogleClient`` or ``user:email`` for ``GithubClient``, space separated list of strings, required in case if ``oauth`` is used. -* ``allow_read_only`` - allow requesting status APIs without authorization, boolean, required. * ``salt`` - password hash salt, string, required in case if authorization enabled (automatically generated by ``create-user`` subcommand). Authorized users are stored inside internal database, if any of external provides are used the password field for non-service users must be empty. @@ -56,7 +56,7 @@ Build related configuration. Group name can refer to architecture, e.g. ``build: * ``ignore_packages`` - list packages to ignore during a regular update (manual update will still work), space separated list of strings, optional. * ``makepkg_flags`` - additional flags passed to ``makepkg`` command, space separated list of strings, optional. * ``makechrootpkg_flags`` - additional flags passed to ``makechrootpkg`` command, space separated list of strings, optional. -* ``triggers`` - list of trigger classes (e.g. ``ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger``) which will be loaded and run at the end of processing, space separated list of strings, optional. You can also specify triggers by their paths, e.g. ``/usr/lib/python3.10/site-packages/ahriman/core/report/report.py.ReportTrigger``. Triggers are run in the order of mention. +* ``triggers`` - list of ``ahriman.core.triggers.Trigger`` class implementation (e.g. ``ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger``) which will be loaded and run at the end of processing, space separated list of strings, optional. You can also specify triggers by their paths, e.g. ``/usr/lib/python3.10/site-packages/ahriman/core/report/report.py.ReportTrigger``. Triggers are run in the order of mention. ``repository`` group -------------------- @@ -80,7 +80,7 @@ Settings for signing packages or repository. Group name can refer to architectur Report generation settings. -* ``target`` - list of reports to be generated, space separated list of strings, required. It must point to valid section (or to section with architecture), e.g. ``somerandomname`` must point to existing section, ``email`` must point to one of ``email`` of ``email:x86_64`` (the one with architecture has higher priority). +* ``target`` - list of reports to be generated, space separated list of strings, required. It must point to valid section (or to section with architecture), e.g. ``somerandomname`` must point to existing section, ``email`` must point to either ``email`` or ``email:x86_64`` (the one with architecture has higher priority). Type will be read from several ways: @@ -93,6 +93,7 @@ Type will be read from several ways: Section name must be either ``console`` (plus optional architecture name, e.g. ``console:x86_64``) or random name with ``type`` set. +* ``type`` - type of the report, string, optional, must be set to ``console`` if exists. * ``use_utf`` - use utf8 symbols in output if set and ascii otherwise, boolean, optional, default ``yes``. ``email`` type @@ -120,9 +121,9 @@ Section name must be either ``email`` (plus optional architecture name, e.g. ``e Section name must be either ``html`` (plus optional architecture name, e.g. ``html:x86_64``) or random name with ``type`` set. * ``type`` - type of the report, string, optional, must be set to ``html`` if exists. -* ``path`` - path to html report file, string, required. * ``homepage`` - link to homepage, string, optional. * ``link_path`` - prefix for HTML links, string, required. +* ``path`` - path to html report file, string, required. * ``template_path`` - path to Jinja2 template, string, required. ``telegram`` type diff --git a/docs/index.rst b/docs/index.rst index f8adc7c4..85757e13 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -6,14 +6,14 @@ Wrapper for managing custom repository inspired by `repo-scripts ` for more details. #. - TL;DR + TL;DR: .. code-block:: shell @@ -25,21 +25,24 @@ Initial setup Configure build tools (it is required for correct dependency management system): #. - Create build command, e.g. ``ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build`` (you can choose any name for command, basically it should be ``{name}-{arch}-build``). - #. - Create configuration file, e.g. ``cp /usr/share/devtools/pacman-{extra,ahriman}.conf`` (same as previous ``pacman-{name}.conf``). - #. - Change configuration file, add your own repository, add multilib repository etc; - #. - Set ``build_command`` option to point to your command. - #. - Configure ``/etc/sudoers.d/ahriman`` to allow running command without a password. + Create build command (you can choose any name for command, basically it should be ``{name}-{arch}-build``): .. code-block:: shell ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build + + #. + Create configuration file (same as previous ``pacman-{name}.conf``): + + .. code-block:: shell + cp /usr/share/devtools/pacman-{extra,ahriman}.conf + #. + Change configuration file, add your own repository, add multilib repository etc: + + .. code-block:: shell + echo '[multilib]' | tee -a /usr/share/devtools/pacman-ahriman.conf echo 'Include = /etc/pacman.d/mirrorlist' | tee -a /usr/share/devtools/pacman-ahriman.conf @@ -47,9 +50,19 @@ Initial setup echo 'SigLevel = Optional TrustAll' | tee -a /usr/share/devtools/pacman-ahriman.conf echo 'Server = file:///var/lib/ahriman/repository/$arch' | tee -a /usr/share/devtools/pacman-ahriman.conf + #. + Set ``build_command`` option to point to your command: + + .. code-block:: shell + echo '[build]' | tee -a /etc/ahriman.ini.d/build.ini echo 'build_command = ahriman-x86_64-build' | tee -a /etc/ahriman.ini.d/build.ini + #. + Configure ``/etc/sudoers.d/ahriman`` to allow running command without a password: + + .. code-block:: shell + echo 'Cmnd_Alias CARCHBUILD_CMD = /usr/local/bin/ahriman-x86_64-build *' | tee -a /etc/sudoers.d/ahriman echo 'ahriman ALL=(ALL) NOPASSWD: CARCHBUILD_CMD' | tee -a /etc/sudoers.d/ahriman chmod 400 /etc/sudoers.d/ahriman diff --git a/docs/triggers.rst b/docs/triggers.rst index c38cee3f..13fb5b52 100644 --- a/docs/triggers.rst +++ b/docs/triggers.rst @@ -1,16 +1,16 @@ Triggers ======== -The package provides ability to write custom extensions which will be run on (the most) actions, e.g. after updates. By default ahriman provides two types of extensions - reporting and files uploading. Each extension must derive from the ``Trigger`` class and implement ``run`` method +The package provides ability to write custom extensions which will be run on (the most) actions, e.g. after updates. By default ahriman provides two types of extensions - reporting and files uploading. Each extension must derive from the ``ahriman.core.triggers.Trigger`` class and implement ``run`` method Trigger example --------------- -Lets consider example of reporting trigger (e.g. `slack `_, which provides easy HTTP API for triggers for integrations). +Lets consider example of reporting trigger (e.g. `slack `_, which provides easy HTTP API for integration triggers). -In order to post message to slack we will need a specific trigger url (something like ``https://hooks.slack.com/services/company_id/trigger_id``), channel (e.g. ``#archrepo``) and username (``repo-bot``). It can be retrieved from the same application instance. +In order to post message to slack we will need a specific trigger url (something like ``https://hooks.slack.com/services/company_id/trigger_id``), channel (e.g. ``#archrepo``) and username (``repo-bot``). -As it has been mentioned, our trigger must derive from specific class +As it has been mentioned, our trigger must derive from specific class: .. code-block:: python @@ -24,9 +24,7 @@ As it has been mentioned, our trigger must derive from specific class self.channel = configuration.get("slack", "channel") self.username = configuration.get("slack", "username") -By now we have class with all required variables. - -Lets implement run method. Slack API requires positing data with specific payload by HTTP, thus +By now we have class with all required variables. Lets implement run method. Slack API requires positing data with specific payload by HTTP, thus: .. code-block:: python @@ -39,7 +37,7 @@ Lets implement run method. Slack API requires positing data with specific payloa response = requests.post(slack_url, data={"payload": json.dumps(payload)}) response.raise_for_status() -Obviously you can implement the specified method in class, but for guide purpose it has been done as separated method. Now we can merge this method into the class +Obviously you can implement the specified method in class, but for guide purpose it has been done as separated method. Now we can merge this method into the class: .. code-block:: python diff --git a/src/ahriman/application/handlers/status.py b/src/ahriman/application/handlers/status.py index 6339fd85..5c447be5 100644 --- a/src/ahriman/application/handlers/status.py +++ b/src/ahriman/application/handlers/status.py @@ -56,7 +56,7 @@ class Status(Handler): StatusPrinter(service_status.status).print(args.info) if args.package: packages: Iterable[Tuple[Package, BuildStatus]] = sum( - [client.get(base) for base in args.package], + (client.get(base) for base in args.package), start=[]) else: packages = client.get(None) diff --git a/src/ahriman/application/handlers/versions.py b/src/ahriman/application/handlers/versions.py index e849198e..484baa7c 100644 --- a/src/ahriman/application/handlers/versions.py +++ b/src/ahriman/application/handlers/versions.py @@ -77,7 +77,7 @@ class Versions(Handler): keys.extend(portion) portion = { key - for key in sum([dependencies_by_key(key) for key in portion], start=[]) + for key in sum((dependencies_by_key(key) for key in portion), start=[]) if key not in keys and key in resources } diff --git a/src/ahriman/core/auth/auth.py b/src/ahriman/core/auth/auth.py index c9c90797..94c0f073 100644 --- a/src/ahriman/core/auth/auth.py +++ b/src/ahriman/core/auth/auth.py @@ -89,7 +89,7 @@ class Auth: return OAuth(configuration, database) return cls(configuration) - async def check_credentials(self, username: Optional[str], password: Optional[str]) -> bool: # pylint: disable=no-self-use + async def check_credentials(self, username: Optional[str], password: Optional[str]) -> bool: """ validate user password @@ -103,7 +103,7 @@ class Auth: del username, password return True - async def known_username(self, username: Optional[str]) -> bool: # pylint: disable=no-self-use + async def known_username(self, username: Optional[str]) -> bool: """ check if user is known @@ -116,7 +116,7 @@ class Auth: del username return True - async def verify_access(self, username: str, required: UserAccess, context: Optional[str]) -> bool: # pylint: disable=no-self-use + async def verify_access(self, username: str, required: UserAccess, context: Optional[str]) -> bool: """ validate if user has access to requested resource diff --git a/src/ahriman/core/configuration.py b/src/ahriman/core/configuration.py index b0f37aaa..020c7cf6 100644 --- a/src/ahriman/core/configuration.py +++ b/src/ahriman/core/configuration.py @@ -220,7 +220,7 @@ class Configuration(configparser.RawConfigParser): } # pylint and mypy are too stupid to find these methods - # pylint: disable=missing-function-docstring,multiple-statements,unused-argument,no-self-use + # pylint: disable=missing-function-docstring,multiple-statements,unused-argument def getlist(self, *args: Any, **kwargs: Any) -> List[str]: ... def getpath(self, *args: Any, **kwargs: Any) -> Path: ... diff --git a/src/ahriman/core/formatters/printer.py b/src/ahriman/core/formatters/printer.py index ab950528..3bbd8403 100644 --- a/src/ahriman/core/formatters/printer.py +++ b/src/ahriman/core/formatters/printer.py @@ -44,7 +44,7 @@ class Printer: continue log_fn(f"\t{prop.name}{separator}{prop.value}") - def properties(self) -> List[Property]: # pylint: disable=no-self-use + def properties(self) -> List[Property]: """ convert content into printable data diff --git a/src/ahriman/core/status/client.py b/src/ahriman/core/status/client.py index a673718a..df2e1b6a 100644 --- a/src/ahriman/core/status/client.py +++ b/src/ahriman/core/status/client.py @@ -60,7 +60,7 @@ class Client: status(BuildStatusEnum): current package build status """ - def get(self, base: Optional[str]) -> List[Tuple[Package, BuildStatus]]: # pylint: disable=no-self-use + def get(self, base: Optional[str]) -> List[Tuple[Package, BuildStatus]]: """ get package status @@ -73,7 +73,7 @@ class Client: del base return [] - def get_internal(self) -> InternalStatus: # pylint: disable=no-self-use + def get_internal(self) -> InternalStatus: """ get internal service status diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 4c80499e..9559bfd4 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -85,7 +85,7 @@ class Package: Returns: List[str]: sum of dependencies per each package """ - return sorted(set(sum([package.depends for package in self.packages.values()], start=[]))) + return sorted(set(sum((package.depends for package in self.packages.values()), start=[]))) @property def groups(self) -> List[str]: @@ -95,7 +95,7 @@ class Package: Returns: List[str]: sum of groups per each package """ - return sorted(set(sum([package.groups for package in self.packages.values()], start=[]))) + return sorted(set(sum((package.groups for package in self.packages.values()), start=[]))) @property def is_single_package(self) -> bool: @@ -130,7 +130,7 @@ class Package: Returns: List[str]: sum of licenses per each package """ - return sorted(set(sum([package.licenses for package in self.packages.values()], start=[]))) + return sorted(set(sum((package.licenses for package in self.packages.values()), start=[]))) @classmethod def from_archive(cls: Type[Package], path: Path, pacman: Pacman, remote: Optional[RemoteSource]) -> Package: