diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d683a12f..8c912622 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,6 +36,7 @@ Again, the most checks can be performed by `tox` command, though some additional Notes: Very important note about this function + Probably multi-line Args: argument(str): an argument. This argument has @@ -70,6 +71,7 @@ Again, the most checks can be performed by `tox` command, though some additional Attributes: CLAZZ_ATTRIBUTE(int): (class attribute) a brand-new class attribute instance_attribute(str): an instance attribute + with the long description Examples: Very informative class usage example, e.g.:: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 4e4446d3..a705824c 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -8,9 +8,6 @@ cat < "/etc/ahriman.ini.d/00-docker.ini" [repository] root = $AHRIMAN_REPOSITORY_ROOT -[settings] -database = $AHRIMAN_REPOSITORY_ROOT/ahriman.db - [web] host = $AHRIMAN_HOST diff --git a/docs/ahriman.core.report.rst b/docs/ahriman.core.report.rst index 4738814d..c0f4d314 100644 --- a/docs/ahriman.core.report.rst +++ b/docs/ahriman.core.report.rst @@ -60,6 +60,14 @@ ahriman.core.report.report\_trigger module :no-undoc-members: :show-inheritance: +ahriman.core.report.rss module +------------------------------ + +.. automodule:: ahriman.core.report.rss + :members: + :no-undoc-members: + :show-inheritance: + ahriman.core.report.telegram module ----------------------------------- diff --git a/docs/configuration.rst b/docs/configuration.rst index 81cbc843..4a80125d 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -17,14 +17,33 @@ There are two variable types which have been added to default ones, they are pat Path values, except for casting to ``pathlib.Path`` type, will be also expanded to absolute paths relative to the configuration path. E.g. if path is set to ``ahriman.ini.d/logging.ini`` and root configuration path is ``/etc/ahriman.ini``, the value will be expanded to ``/etc/ahriman.ini.d/logging.ini``. In order to disable path expand, use the full path, e.g. ``/etc/ahriman.ini.d/logging.ini``. -Configuration allows string interpolation from environment variables, e.g.: +Configuration allows string interpolation from the same configuration file, e.g.: + +.. code-block:: ini + + [section] + key = ${anoher_key} + another_key = value + +will read value for the ``section.key`` option from ``section.another_key``. In case if the cross-section reference is required, the ``${section:another_key}`` notation must be used. It also allows string interpolation from environment variables, e.g.: .. code-block:: ini [section] key = $SECRET -will try to read value from ``SECRET`` environment variable. In case if the required environment variable wasn't found, it will keep original value (i.e. ``$SECRET`` in the example). Dollar sign can be set as ``$$``. +will try to read value from ``SECRET`` environment variable. In case if the required environment variable wasn't found, it will keep original value (i.e. ``$SECRET`` in the example). Dollar sign can be set as ``$$``. All those interpolations will be applied in succession and - expected to be - recursively, e.g. the following configuration: + +.. code-block:: ini + + [section1] + key = ${section2:key} + + [section2] + key = ${home} + home = $HOME + +will eventually lead ``section1.key`` option to be set to the value of ``HOME`` environment variable (if available). There is also additional subcommand which will allow to validate configuration and print found errors. In order to do so, run ``service-config-validate`` subcommand, e.g.: diff --git a/docs/faq/distributed.rst b/docs/faq/distributed.rst index e4dadde2..06f70884 100644 --- a/docs/faq/distributed.rst +++ b/docs/faq/distributed.rst @@ -292,7 +292,7 @@ Worker nodes (applicable for all workers) config (``worker.ini``) as: Command to run worker nodes (considering there will be two workers, one is on ``8081`` port and other is on ``8082``): -.. code-block:: ini +.. code-block:: shell docker run --privileged -p 8081:8081 -e AHRIMAN_PORT=8081 -v worker.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web docker run --privileged -p 8082:8082 -e AHRIMAN_PORT=8082 -v worker.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web diff --git a/docs/faq/reporting.rst b/docs/faq/reporting.rst index 08745ffd..7a86dab8 100644 --- a/docs/faq/reporting.rst +++ b/docs/faq/reporting.rst @@ -47,7 +47,7 @@ How to generate index page for S3 target = html [html] - path = /var/lib/ahriman/repository/aur-clone/x86_64/index.html + path = ${repository:root}/repository/aur-clone/x86_64/index.html link_path = http://example.com/aur-clone/x86_64 After these steps ``index.html`` file will be automatically synced to S3. diff --git a/package/share/ahriman/settings/ahriman.ini b/package/share/ahriman/settings/ahriman.ini index bd815a8c..52d8acf3 100644 --- a/package/share/ahriman/settings/ahriman.ini +++ b/package/share/ahriman/settings/ahriman.ini @@ -6,7 +6,7 @@ logging = ahriman.ini.d/logging.ini ; Perform database migrations on the application start. Do not touch this option unless you know what are you doing. ;apply_migrations = yes ; Path to the application SQLite database. -database = /var/lib/ahriman/ahriman.db +database = ${repository:root}/ahriman.db [alpm] ; Path to pacman system database cache. @@ -119,9 +119,9 @@ host = 127.0.0.1 ; Disable status (e.g. package status, logs, etc) endpoints. Useful for build only modes. ;service_only = no ; Path to directory with static files. -static_path = /usr/share/ahriman/templates/static +static_path = ${templates}/static ; List of directories with templates. -templates = /usr/share/ahriman/templates +templates = ${prefix}/share/ahriman/templates ; Path to unix socket. If none set, unix socket will be disabled. ;unix_socket = ; Allow unix socket to be world readable. @@ -246,7 +246,7 @@ template = email-index.jinja2 ; Template name to be used for full packages list generation (same as HTML report). ;template_full = ; List of directories with templates. -templates = /usr/share/ahriman/templates +templates = ${prefix}/share/ahriman/templates ; SMTP user. ;user = @@ -265,7 +265,7 @@ templates = /usr/share/ahriman/templates ; Template name to be used. template = repo-index.jinja2 ; List of directories with templates. -templates = /usr/share/ahriman/templates +templates = ${prefix}/share/ahriman/templates ; Remote service callback trigger configuration sample. [remote-call] @@ -295,7 +295,7 @@ templates = /usr/share/ahriman/templates ; Template name to be used. template = rss.jinja2 ; List of directories with templates. -templates = /usr/share/ahriman/templates +templates = ${prefix}/share/ahriman/templates ; Telegram reporting trigger configuration sample. [telegram] @@ -316,7 +316,7 @@ template = telegram-index.jinja2 ; Telegram specific template mode, one of MarkdownV2, HTML or Markdown. ;template_type = HTML ; List of directories with templates. -templates = /usr/share/ahriman/templates +templates = ${prefix}/share/ahriman/templates ; HTTP request timeout in seconds. ;timeout = 30 diff --git a/recipes/index/service.ini b/recipes/index/service.ini index 8972e4c3..80e3f016 100644 --- a/recipes/index/service.ini +++ b/recipes/index/service.ini @@ -2,5 +2,5 @@ target = html [html] -path = /var/lib/ahriman/ahriman/repository/ahriman-demo/x86_64/index.html +path = ${repository:root}/repository/ahriman-demo/x86_64/index.html link_path = http://localhost:8080/repo/ahriman-demo/x86_64 diff --git a/src/ahriman/application/application/application.py b/src/ahriman/application/application/application.py index 11f1d758..70639588 100644 --- a/src/ahriman/application/application/application.py +++ b/src/ahriman/application/application/application.py @@ -120,8 +120,7 @@ class Application(ApplicationPackages, ApplicationRepository): process_dependencies(bool): if no set, dependencies will not be processed Returns: - list[Package]: updated packages list. Packager for dependencies will be copied from - original package + list[Package]: updated packages list. Packager for dependencies will be copied from the original package Examples: In the most cases, in order to avoid build failure, it is required to add missing packages, which can be diff --git a/src/ahriman/core/build_tools/package_archive.py b/src/ahriman/core/build_tools/package_archive.py index 006724d5..63ee3919 100644 --- a/src/ahriman/core/build_tools/package_archive.py +++ b/src/ahriman/core/build_tools/package_archive.py @@ -236,7 +236,7 @@ class PackageArchive: extract list of the installed packages and their content Returns: - dict[str, FilesystemPackage]; map of package name to list of directories and files contained + dict[str, FilesystemPackage]: map of package name to list of directories and files contained by this package """ result = {} diff --git a/src/ahriman/core/configuration/shell_interpolator.py b/src/ahriman/core/configuration/shell_interpolator.py index 2c7b8fb5..c7614195 100644 --- a/src/ahriman/core/configuration/shell_interpolator.py +++ b/src/ahriman/core/configuration/shell_interpolator.py @@ -19,21 +19,93 @@ # import configparser import os +import sys -from collections.abc import Mapping, MutableMapping +from collections.abc import Generator, Mapping, MutableMapping from string import Template +class ExtendedTemplate(Template): + """ + extension to the default :class:`Template` class, which also enabled braces regex to lookup in sections + + Attributes: + braceidpattern(str): regular expression to match a colon inside braces + """ + + braceidpattern = r"(?a:[_a-z0-9][_a-z0-9:]*)" + + class ShellInterpolator(configparser.Interpolation): """ custom string interpolator, because we cannot use defaults argument due to config validation """ + DATA_LINK_ESCAPE = "\x10" + + @staticmethod + def _extract_variables(parser: MutableMapping[str, Mapping[str, str]], value: str, + defaults: Mapping[str, str]) -> Generator[tuple[str, str], None, None]: + """ + extract keys and values (if available) from the configuration. In case if a key is not available, it will be + silently skipped from the result + + Args: + parser(MutableMapping[str, Mapping[str, str]]): option parser + value(str): source (not-converted) value + defaults(Mapping[str, str]): default values + + Yields: + tuple[str, str]: variable name used for substitution and its value + """ + def identifiers() -> Generator[tuple[str | None, str], None, None]: + # extract all found identifiers and parse them + for identifier in ExtendedTemplate(value).get_identifiers(): + match identifier.split(":"): + case [lookup_option]: # single option from the same section + yield None, lookup_option + case [lookup_section, lookup_option]: # reference to another section + yield lookup_section, lookup_option + + for section, option in identifiers(): + # key to be substituted + key = f"{section}:{option}" if section else option + + if section is not None: # foreign section case + if section not in parser: + continue # section was not found, silently skip it + values = parser[section] + else: # same section + values = defaults + + if (raw := values.get(option)) is not None: + yield key, raw + + @staticmethod + def environment() -> dict[str, str]: + """ + extract environment variables + + Returns: + dict[str, str]: environment variables and some custom variables + """ + return os.environ | { + "prefix": sys.prefix, + } + def before_get(self, parser: MutableMapping[str, Mapping[str, str]], section: str, option: str, value: str, defaults: Mapping[str, str]) -> str: """ interpolate option value + Notes: + This method is using :class:`string.Template` class in order to render both cross-references and + environment variables, because it seems that it is the most legit way to handle it. In addition, + we are using shell-like variables in some cases (see :attr:`alpm.mirror` option), thus we would like + to keep them alive. + + First this method resolves substitution from the configuration and then renders environment variables + Args: parser(MutableMapping[str, Mapping[str, str]]): option parser section(str): section name @@ -44,8 +116,15 @@ class ShellInterpolator(configparser.Interpolation): Returns: str: substituted value """ - # At the moment it seems that it is the most legit way to handle environment variables - # Template behaviour is literally the same as shell - # In addition, we are using shell-like variables in some cases (see :attr:`alpm.mirror` option), - # thus we would like to keep them alive - return Template(value).safe_substitute(os.environ) + # because any substitution effectively replace escaped $ ($$) in result, we have to escape it manually + escaped = value.replace("$$", self.DATA_LINK_ESCAPE) + + # resolve internal references + variables = dict(self._extract_variables(parser, value, defaults)) + internal = ExtendedTemplate(escaped).safe_substitute(variables) + + # resolve enriched environment variables by using default Template class + environment = Template(internal).safe_substitute(self.environment()) + + # replace escaped values back + return environment.replace(self.DATA_LINK_ESCAPE, "$") diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 6016099f..9f988aba 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -161,10 +161,10 @@ class Package(LazyLogging): bool: ``True`` in case if package base looks like VCS package and ``False`` otherwise """ return self.base.endswith("-bzr") \ - or self.base.endswith("-csv")\ - or self.base.endswith("-darcs")\ - or self.base.endswith("-git")\ - or self.base.endswith("-hg")\ + or self.base.endswith("-csv") \ + or self.base.endswith("-darcs") \ + or self.base.endswith("-git") \ + or self.base.endswith("-hg") \ or self.base.endswith("-svn") @property @@ -358,7 +358,7 @@ class Package(LazyLogging): Yields: Path: list of paths of files which belong to the package and distributed together with this tarball. - All paths are relative to the ``path`` + All paths are relative to the ``path`` Raises: PackageInfoError: if there are parsing errors @@ -504,8 +504,8 @@ class Package(LazyLogging): timestamp(float | int): timestamp to check build date against Returns: - bool: ``True`` in case if package was built after the specified date and ``False`` otherwise. In case if build date - is not set by any of packages, it returns False + bool: ``True`` in case if package was built after the specified date and ``False`` otherwise. + In case if build date is not set by any of packages, it returns False """ return any( package.build_date > timestamp @@ -550,8 +550,8 @@ class Package(LazyLogging): Returns: str | None: new generated package release version if any. In case if the release contains dot (e.g. 1.2), - the minor part will be incremented by 1. If the release does not contain major.minor notation, the minor - version equals to 1 will be appended + the minor part will be incremented by 1. If the release does not contain major.minor notation, the minor + version equals to 1 will be appended """ if local_version is None: return None # local version not found, keep upstream pkgrel diff --git a/src/ahriman/models/waiter.py b/src/ahriman/models/waiter.py index 395ace94..c9917324 100644 --- a/src/ahriman/models/waiter.py +++ b/src/ahriman/models/waiter.py @@ -97,7 +97,7 @@ class Waiter: Attributes: interval(float): interval in seconds between checks wait_timeout(float): timeout in seconds to wait for. Negative value will result in immediate exit. Zero value - means infinite timeout + means infinite timeout """ wait_timeout: float diff --git a/tests/ahriman/core/configuration/test_shell_interpolator.py b/tests/ahriman/core/configuration/test_shell_interpolator.py index 92029a3d..bcacb712 100644 --- a/tests/ahriman/core/configuration/test_shell_interpolator.py +++ b/tests/ahriman/core/configuration/test_shell_interpolator.py @@ -1,6 +1,65 @@ import os -from ahriman.core.configuration.shell_interpolator import ShellInterpolator +from ahriman.core.configuration import Configuration +from ahriman.core.configuration.shell_interpolator import ExtendedTemplate, ShellInterpolator + + +def _parser() -> dict[str, dict[str, str]]: + """ + parser mock + + Returns: + dict[str, dict[str, str]]: options to be used as configparser mock + """ + return { + "section1": { + "home": "$HOME", + "key1": "value1", + "key4": "${home}", + }, + "section2": { + "key2": "value2", + }, + "section3": { + "key3": "${section1:home}", + "key5": "${section1:key4}", + }, + } + + +def test_extended_template() -> None: + """ + must match colons in braces + """ + assert ExtendedTemplate("$key:value").get_identifiers() == ["key"] + assert ExtendedTemplate("${key:value}").get_identifiers() == ["key:value"] + + +def test_extract_variables() -> None: + """ + must extract variables list + """ + parser = _parser() + + assert dict(ShellInterpolator._extract_variables(parser, "${key1}", parser["section1"])) == { + "key1": "value1", + } + assert not dict(ShellInterpolator._extract_variables(parser, "${key2}", parser["section1"])) + + assert dict(ShellInterpolator._extract_variables(parser, "${section2:key2}", parser["section1"])) == { + "section2:key2": "value2", + } + assert not dict(ShellInterpolator._extract_variables(parser, "${section2:key1}", parser["section1"])) + + assert not dict(ShellInterpolator._extract_variables(parser, "${section4:key1}", parser["section1"])) + + +def test_environment() -> None: + """ + must extend environment variables + """ + assert "HOME" in ShellInterpolator.environment() + assert "prefix" in ShellInterpolator.environment() def test_before_get() -> None: @@ -8,8 +67,35 @@ def test_before_get() -> None: must correctly extract environment variables """ interpolator = ShellInterpolator() - assert interpolator.before_get({}, "", "", "value", {}) == "value" assert interpolator.before_get({}, "", "", "$value", {}) == "$value" assert interpolator.before_get({}, "", "", "$HOME", {}) == os.environ["HOME"] + + +def test_before_get_escaped() -> None: + """ + must correctly read escaped variables + """ + interpolator = ShellInterpolator() assert interpolator.before_get({}, "", "", "$$HOME", {}) == "$HOME" + + +def test_before_get_reference() -> None: + """ + must correctly extract environment variables after resolving cross-reference + """ + interpolator = ShellInterpolator() + assert interpolator.before_get(_parser(), "", "", "${section1:home}", {}) == os.environ["HOME"] + + +def test_before_get_reference_recursive(configuration: Configuration) -> None: + """ + must correctly extract environment variables after resolving cross-reference recursively + """ + interpolator = ShellInterpolator() + for section, items in _parser().items(): + for option, value in items.items(): + configuration.set_option(section, option, value) + + assert interpolator.before_get(configuration, "", "", "${section1:home}", {}) == os.environ["HOME"] + assert interpolator.before_get(configuration, "", "", "${section3:key5}", {}) == os.environ["HOME"]