feat: allow append list options

docker/entrypoint.sh

@@ -4,7 +4,7 @@ set -e
 [ -n "$AHRIMAN_DEBUG" ] && set -x
 
 # configuration tune
-cat <<EOF > "/etc/ahriman.ini.d/00-docker.ini"
+cat <<EOF > "/etc/ahriman.ini.d/01-docker.ini"
 [repository]
 root = $AHRIMAN_REPOSITORY_ROOT
 

docs/ahriman.core.configuration.rst

@@ -12,6 +12,14 @@ ahriman.core.configuration.configuration module
    :no-undoc-members:
    :show-inheritance:
 
+ahriman.core.configuration.configuration\_multi\_dict module
+------------------------------------------------------------
+
+.. automodule:: ahriman.core.configuration.configuration_multi_dict
+   :members:
+   :no-undoc-members:
+   :show-inheritance:
+
 ahriman.core.configuration.schema module
 ----------------------------------------
 

docs/configuration.rst

@@ -13,7 +13,27 @@ There are two variable types which have been added to default ones, they are pat
 * By default, it splits value by spaces excluding empty elements. 
 * In case if quotation mark (``"`` or ``'``) will be found, any spaces inside will be ignored.
 * In order to use quotation mark inside value it is required to put it to another quotation mark, e.g. ``wor"'"d "with quote"`` will be parsed as ``["wor'd", "with quote"]`` and vice versa.
-* Unclosed quotation mark is not allowed and will rise an exception.
+* Unclosed quotation mark is not allowed and will raise an exception.
+
+It is also possible to split list option between multiple declarations. To do so, append key name with ``[]`` (like PHP, sorry!), e.g.:
+
+.. code-block:: ini
+
+   [section]
+   list[] = value1
+   list[] = value2
+
+will lead to ``${section:list}`` value to be set to ``value1 value2``. The values will be set in order of appearance, meaning that values which appear in different include files will be set in alphabetical order of file names. In order to reset list values, set option to empty string, e.g.:
+
+.. code-block:: ini
+
+   [section]
+   list[] = value1
+   list[] =
+   list[] = value2
+   list[] = value3
+
+will set option ``${section:list}`` to ``value2 value3``. Alternatively, setting the original option (e.g. ``list`` in the example above) will also reset value, though the subsequent options with leading ``[]`` will append the previous value.
 
 Path values, except for casting to ``pathlib.Path`` type, will be also expanded to absolute paths relative to the configuration path. E.g. if path is set to ``ahriman.ini.d/logging.ini`` and root configuration path is ``/etc/ahriman.ini``, the value will be expanded to ``/etc/ahriman.ini.d/logging.ini``. In order to disable path expand, use the full path, e.g. ``/etc/ahriman.ini.d/logging.ini``.
 
@@ -22,7 +42,7 @@ Configuration allows string interpolation from the same configuration file, e.g.
 .. code-block:: ini
 
    [section]
-   key = ${anoher_key}
+   key = ${another_key}
    another_key = value
 
 will read value for the ``key`` option from ``another_key`` in the same section. In case if the cross-section reference is required, the ``${section:another_key}`` notation must be used. It also allows string interpolation from environment variables, e.g.:

package/share/ahriman/settings/ahriman.ini

@@ -65,9 +65,19 @@ makepkg_flags = --nocolor --ignorearch
 ; List of paths to be used for implicit dependency scan. Regular expressions are supported.
 scan_paths = ^usr/lib(?!/cmake).*$
 ; List of enabled triggers in the order of calls.
-triggers = ahriman.core.gitremote.RemotePullTrigger ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger ahriman.core.gitremote.RemotePushTrigger
+triggers[] = ahriman.core.gitremote.RemotePullTrigger
+triggers[] = ahriman.core.report.ReportTrigger
+triggers[] = ahriman.core.upload.UploadTrigger
+triggers[] = ahriman.core.gitremote.RemotePushTrigger
 ; List of well-known triggers. Used only for configuration purposes.
-triggers_known = ahriman.core.distributed.WorkerLoaderTrigger ahriman.core.distributed.WorkerRegisterTrigger ahriman.core.distributed.WorkerTrigger ahriman.core.distributed.WorkerUnregisterTrigger ahriman.core.gitremote.RemotePullTrigger ahriman.core.gitremote.RemotePushTrigger ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger ahriman.core.support.KeyringTrigger ahriman.core.support.MirrorlistTrigger
+triggers_known[] = ahriman.core.distributed.WorkerLoaderTrigger
+triggers_known[] = ahriman.core.distributed.WorkerTrigger
+triggers_known[] = ahriman.core.gitremote.RemotePullTrigger
+triggers_known[] = ahriman.core.gitremote.RemotePushTrigger
+triggers_known[] = ahriman.core.report.ReportTrigger
+triggers_known[] = ahriman.core.support.KeyringTrigger
+triggers_known[] = ahriman.core.support.MirrorlistTrigger
+triggers_known[] = ahriman.core.upload.UploadTrigger
 ; Maximal age in seconds of the VCS packages before their version will be updated with its remote source.
 ;vcs_allowed_age = 604800
 ; List of worker nodes addresses used for build process, e.g.:
@@ -121,7 +131,7 @@ host = 127.0.0.1
 ; Path to directory with static files.
 static_path = ${templates}/static
 ; List of directories with templates.
-templates = ${prefix}/share/ahriman/templates
+templates[] = ${prefix}/share/ahriman/templates
 ; Path to unix socket. If none set, unix socket will be disabled.
 ;unix_socket =
 ; Allow unix socket to be world readable.
@@ -246,7 +256,7 @@ template = email-index.jinja2
 ; Template name to be used for full packages list generation (same as HTML report).
 ;template_full =
 ; List of directories with templates.
-templates = ${prefix}/share/ahriman/templates
+templates[] = ${prefix}/share/ahriman/templates
 ; SMTP user.
 ;user =
 
@@ -265,7 +275,7 @@ templates = ${prefix}/share/ahriman/templates
 ; Template name to be used.
 template = repo-index.jinja2
 ; List of directories with templates.
-templates = ${prefix}/share/ahriman/templates
+templates[] = ${prefix}/share/ahriman/templates
 
 ; Remote service callback trigger configuration sample.
 [remote-call]
@@ -295,7 +305,7 @@ templates = ${prefix}/share/ahriman/templates
 ; Template name to be used.
 template = rss.jinja2
 ; List of directories with templates.
-templates = ${prefix}/share/ahriman/templates
+templates[] = ${prefix}/share/ahriman/templates
 
 ; Telegram reporting trigger configuration sample.
 [telegram]
@@ -316,7 +326,7 @@ template = telegram-index.jinja2
 ; Telegram specific template mode, one of MarkdownV2, HTML or Markdown.
 ;template_type = HTML
 ; List of directories with templates.
-templates = ${prefix}/share/ahriman/templates
+templates[] = ${prefix}/share/ahriman/templates
 ; HTTP request timeout in seconds.
 ;timeout = 30
 

src/ahriman/application/handlers/setup.py

@@ -161,8 +161,8 @@ class Setup(Handler):
             repository_server(str): url of the repository
         """
         # allow_no_value=True is required because pacman uses boolean configuration in which just keys present
-        # (e.g. NoProgressBar) which will lead to exception
+        # (e.g. NoProgressBar) which will lead to exception. allow_multi_key=False is set just for fun
-        configuration = Configuration(allow_no_value=True)
+        configuration = Configuration(allow_no_value=True, allow_multi_key=False)
         # preserve case
         # stupid mypy thinks that it is impossible
         configuration.optionxform = lambda optionstr: optionstr  # type: ignore[method-assign]

src/ahriman/core/configuration/configuration.py

@@ -25,6 +25,7 @@ from collections.abc import Callable
 from pathlib import Path
 from typing import Any, Self
 
+from ahriman.core.configuration.configuration_multi_dict import ConfigurationMultiDict
 from ahriman.core.configuration.shell_interpolator import ShellInterpolator
 from ahriman.core.exceptions import InitializeError
 from ahriman.models.repository_id import RepositoryId
@@ -69,21 +70,27 @@ class Configuration(configparser.RawConfigParser):
     SYSTEM_CONFIGURATION_PATH = Path(sys.prefix) / "share" / "ahriman" / "settings" / "ahriman.ini"
     converters: dict[str, Callable[[str], Any]]  # typing guard
 
-    def __init__(self, allow_no_value: bool = False) -> None:
+    def __init__(self, allow_no_value: bool = False, allow_multi_key: bool = True) -> None:
         """
         Args:
             allow_no_value(bool, optional): copies :class:`configparser.RawConfigParser` behaviour. In case if it is set
                 to ``True``, the keys without values will be allowed (Default value = False)
+            allow_multi_key(bool, optional): if set to ``False``, then the default dictionary class will be used to
+                store keys internally. Otherwise, the special implementation will be used, which supports arrays
+                (Default value = True)
         """
         configparser.RawConfigParser.__init__(
             self,
+            dict_type=ConfigurationMultiDict if allow_multi_key else dict,  # type: ignore[arg-type]
             allow_no_value=allow_no_value,
+            strict=False,
+            empty_lines_in_values=not allow_multi_key,
             interpolation=ShellInterpolator(),
             converters={
                 "list": shlex.split,
                 "path": self._convert_path,
                 "pathlist": lambda value: [self._convert_path(element) for element in shlex.split(value)],
-            }
+            },
         )
 
         self.repository_id: RepositoryId | None = None

src/ahriman/core/configuration/configuration_multi_dict.py

@@ -0,0 +1,91 @@
+#
+# Copyright (c) 2021-2024 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from typing import Any
+
+from ahriman.core.exceptions import OptionError
+
+
+class ConfigurationMultiDict(dict[str, Any]):
+    """
+    wrapper around :class:`dict` to handle multiple configuration keys as lists if they end with ``[]``.
+
+    Examples:
+        This class is designed to be used only with :class:`configparser.RawConfigParser` class, but idea is that
+        if the key ends with ``[]`` it will be treated as array and the result will be appended to the current value.
+        In addition, if the value is empty, then it will clear previous values, e.g.:
+
+            >>> data = ConfigurationMultiDict()
+            >>>
+            >>> data["single"] = "value"  # append normal key
+            >>> print(data)  # {"single": "value"}
+            >>>
+            >>> data["array[]"] = ["value1"]  # append array value
+            >>> data["array[]"] = ["value2"]
+            >>> print(data)  # {"single": "value", "array": ["value1 value2"]}
+            >>>
+            >>> data["array[]"] = [""]  # clear previous values
+            >>> data["array[]"] = ["value3"]
+            >>> print(data)  # {"single": "value", "array": ["value3"]}
+    """
+
+    def _set_array_value(self, key: str, value: Any) -> None:
+        """
+        set array value. If the key already exists in the dictionary, its value will be prepended to new value
+
+        Args:
+            key(str): key to insert
+            value(Any): value of the related key
+
+        Raises:
+            OptionError: if the key already exists in the dictionary, but not a single value list or a string
+        """
+        match self.get(key):
+            case [current_value] | str(current_value):  # type: ignore[misc]
+                value = f"{current_value} {value}"
+            case None:
+                pass
+            case other:
+                raise OptionError(other)
+        super().__setitem__(key, [value])
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        """
+        set ``key`` to ``value``. If the value equals to ``[""]`` (array with empty string), then the key
+        will be removed (as expected from :class:`configparser.RawConfigParser`). If the key ends with
+        ``[]``, the value will be treated as an array and vice versa.
+
+        Args:
+            key(str): key to insert
+            value(Any): value of the related key
+
+        Raises:
+            OptionError: if ``key`` contains ``[]``, but not at the end of the string (e.g. ``prefix[]suffix``)
+        """
+        real_key, is_key_array, remaining = key.partition("[]")
+        if remaining:
+            raise OptionError(key)
+
+        match value:
+            case [""]:  # empty value key
+                self.pop(real_key, None)
+            case [array_value] if is_key_array:  # update array value
+                self._set_array_value(real_key, array_value)
+            case _:  # normal key
+                super().__setitem__(real_key, value)

tests/ahriman/core/configuration/test_configuration.py

@@ -1,4 +1,6 @@
 import configparser
+from io import StringIO
+
 import pytest
 
 from pathlib import Path
@@ -180,6 +182,32 @@ def test_getlist_unmatched_quote(configuration: Configuration) -> None:
         configuration.getlist("build", "test_list")
 
 
+def test_getlist_append() -> None:
+    """
+    must correctly append list values
+    """
+    configuration = Configuration()
+    configuration._read(
+        StringIO("""
+        [section]
+        list1[] = value1
+        list1[] = value2
+
+        list2[] = value3
+        list2[] =
+        list2[] = value4
+        list2[] = value5
+
+        list3[] = value6
+        list3 = value7
+        list3[] = value8
+        """), "io")
+
+    assert configuration.getlist("section", "list1") == ["value1", "value2"]
+    assert configuration.getlist("section", "list2") == ["value4", "value5"]
+    assert configuration.getlist("section", "list3") == ["value7", "value8"]
+
+
 def test_getpath_absolute_to_absolute(configuration: Configuration) -> None:
     """
     must not change path for absolute path in settings

tests/ahriman/core/configuration/test_configuration_multi_dict.py

@@ -0,0 +1,54 @@
+import pytest
+
+from ahriman.core.configuration.configuration_multi_dict import ConfigurationMultiDict
+from ahriman.core.exceptions import OptionError
+
+
+def test_setitem_non_list() -> None:
+    """
+    must insert not list correctly
+    """
+    instance = ConfigurationMultiDict()
+    instance["key"] = "value"
+    assert instance["key"] == "value"
+
+
+def test_setitem_remove() -> None:
+    """
+    must remove key
+    """
+    instance = ConfigurationMultiDict()
+    instance["key"] = "value"
+    instance["key"] = [""]
+
+    assert "key" not in instance
+
+
+def test_setitem_array() -> None:
+    """
+    must set array correctly
+    """
+    instance = ConfigurationMultiDict()
+    instance["key[]"] = ["value1"]
+    instance["key[]"] = ["value2"]
+
+    assert instance["key"] == ["value1 value2"]
+
+
+def test_setitem_array_exception() -> None:
+    """
+    must raise exception if the current value is not a single value array
+    """
+    instance = ConfigurationMultiDict()
+    instance["key[]"] = ["value1", "value2"]
+    with pytest.raises(OptionError):
+        instance["key[]"] = ["value3"]
+
+
+def test_setitem_exception() -> None:
+    """
+    must raise exception on invalid key
+    """
+    instance = ConfigurationMultiDict()
+    with pytest.raises(OptionError):
+        instance["prefix[]suffix"] = "value"

tests/testresources/core/ahriman.ini

@@ -60,7 +60,7 @@ target = console
 
 [email]
 host = 127.0.0.1
-link_path =
+link_path = http://example.com
 no_empty_report = no
 port = 587
 receivers = mail@example.com
@@ -72,9 +72,8 @@ templates = ../web/templates
 use_utf = yes
 
 [html]
-path =
+link_path = http://example.com
-homepage =
+path = local/path
-link_path =
 template = repo-index.jinja2
 templates = ../web/templates
 
@@ -82,17 +81,15 @@ templates = ../web/templates
 manual = yes
 
 [rss]
-path =
+link_path = http://example.com
-homepage =
+path = local/path
-link_path =
 template = rss.jinja2
 templates = ../web/templates
 
 [telegram]
-api_key = apikey
+api_key = api_key
 chat_id = @ahrimantestchat
-homepage =
+link_path = http://example.com
-link_path =
 template = telegram-index.jinja2
 templates = ../web/templates
 
@@ -101,20 +98,20 @@ target =
 
 [rsync]
 command = rsync --archive --verbose --compress --partial --delete
-remote =
+remote = remote@example.com
 
 [disabled]
 
 [customs3]
 type = s3
-access_key =
+access_key = access_key
 bucket = bucket
 region = eu-central-1
-secret_key =
+secret_key = secret_key
 
 [github:x86_64]
 owner = arcan1s
-password =
+password = pa55w0rd
 repository = ahriman
 username = arcan1s