expand bash

docs/ahriman.core.configuration.rst

@@ -28,6 +28,14 @@ ahriman.core.configuration.shell\_interpolator module
    :no-undoc-members:
    :show-inheritance:
 
+ahriman.core.configuration.shell\_template module
+-------------------------------------------------
+
+.. automodule:: ahriman.core.configuration.shell_template
+   :members:
+   :no-undoc-members:
+   :show-inheritance:
+
 ahriman.core.configuration.validator module
 -------------------------------------------
 

src/ahriman/core/alpm/pkgbuild_parser.py

@@ -270,9 +270,9 @@ class PkgbuildParser(shlex.shlex):
             PkgbuildPatch: extracted a PKGBUILD node
         """
         # simple assignment rule
-        if (match := self._STRING_ASSIGNMENT.match(token)) is not None:
-            key = match.group("key")
-            value = match.group("value")
+        if m := self._STRING_ASSIGNMENT.match(token):
+            key = m.group("key")
+            value = m.group("value")
             yield PkgbuildPatch(key, value)
             return
 
@@ -282,8 +282,8 @@ class PkgbuildParser(shlex.shlex):
 
         match self.get_token():
             # array processing. Arrays will be sent as "key=", "(", values, ")"
-            case PkgbuildToken.ArrayStarts if (match := self._ARRAY_ASSIGNMENT.match(token)) is not None:
-                key = match.group("key")
+            case PkgbuildToken.ArrayStarts if m := self._ARRAY_ASSIGNMENT.match(token):
+                key = m.group("key")
                 value = self._parse_array()
                 yield PkgbuildPatch(key, value)
 

src/ahriman/core/configuration/shell_interpolator.py

@@ -24,16 +24,7 @@ import sys
 from collections.abc import Generator, Mapping, MutableMapping
 from string import Template
 
-
-class ExtendedTemplate(Template):
-    """
-    extension to the default :class:`Template` class, which also enabled braces regex to lookup in sections
-
-    Attributes:
-        braceidpattern(str): regular expression to match a colon inside braces
-    """
-
-    braceidpattern = r"(?a:[_a-z0-9][_a-z0-9:]*)"
+from ahriman.core.configuration.shell_template import ShellTemplate
 
 
 class ShellInterpolator(configparser.Interpolation):
@@ -60,7 +51,7 @@ class ShellInterpolator(configparser.Interpolation):
         """
         def identifiers() -> Generator[tuple[str | None, str], None, None]:
             # extract all found identifiers and parse them
-            for identifier in ExtendedTemplate(value).get_identifiers():
+            for identifier in ShellTemplate(value).get_identifiers():
                 match identifier.split(":"):
                     case [lookup_option]:  # single option from the same section
                         yield None, lookup_option
@@ -121,7 +112,7 @@ class ShellInterpolator(configparser.Interpolation):
 
         # resolve internal references
         variables = dict(self._extract_variables(parser, value, defaults))
-        internal = ExtendedTemplate(escaped).safe_substitute(variables)
+        internal = ShellTemplate(escaped).safe_substitute(variables)
 
         # resolve enriched environment variables by using default Template class
         environment = Template(internal).safe_substitute(self.environment())

src/ahriman/core/configuration/shell_template.py

@@ -0,0 +1,158 @@
+#
+# Copyright (c) 2021-2024 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import fnmatch
+import re
+
+from collections.abc import Generator, Mapping
+from string import Template
+
+
+class ShellTemplate(Template):
+    """
+    extension to the default :class:`Template` class, which also adds additional tokens to braced regex and enables
+    bash expansion
+
+    Attributes:
+        braceidpattern(str): regular expression to match every character except for closing bracket
+    """
+
+    braceidpattern = r"(?a:[_a-z0-9][^}]*)"
+
+    _REMOVE_BACK = re.compile(r"^(?P<key>\w+)%(?P<pattern>.+)$")
+    _REMOVE_FRONT = re.compile(r"^(?P<key>\w+)#(?P<pattern>.+)$")
+    _REPLACE = re.compile(r"^(?P<key>\w+)/(?P<pattern>.+)/(?P<replacement>.+)$")
+
+    @staticmethod
+    def _remove_back(source: str, pattern: str, *, greedy: bool) -> str:
+        """
+        resolve "${var%(%)pattern}" constructions
+
+        Args:
+            source(str): source string to match the pattern inside
+            pattern(str): shell expression to match
+            greedy(bool): match as much as possible or not
+
+        Returns:
+            str: result after removal ``pattern`` from the end of the string
+        """
+        regex = fnmatch.translate(pattern)
+        compiled = re.compile(regex)
+
+        result = source
+        start_pos = 0
+
+        while m := compiled.search(source, start_pos):
+            result = source[:m.start()]
+            start_pos += m.start() + 1
+            if greedy:
+                break
+
+        return result
+
+    @staticmethod
+    def _remove_front(source: str, pattern: str, *, greedy: bool) -> str:
+        """
+        resolve "${var#(#)pattern}" constructions
+
+        Args:
+            source(str): source string to match the pattern inside
+            pattern(str): shell expression to match
+            greedy(bool): match as much as possible or not
+
+        Returns:
+            str: result after removal ``pattern`` from the start of the string
+        """
+        regex = fnmatch.translate(pattern)[:-2]  # remove \Z at the end of the regex
+        if not greedy:
+            regex = regex.replace("*", "*?")
+        compiled = re.compile(regex)
+
+        m = compiled.match(source)
+        if m is None:
+            return source
+
+        return source[m.end():]
+
+    @staticmethod
+    def _replace(source: str, pattern: str, replacement: str, *, greedy: bool) -> str:
+        """
+        resolve "${var/(/)pattern/replacement}" constructions
+
+        Args:
+            source(str): source string to match the pattern inside
+            pattern(str): shell expression to match
+            replacement(str): new substring
+            greedy(bool): replace as much as possible or not
+
+        Returns:
+            str: result after replacing ``pattern`` by ``replacement``
+        """
+        match pattern:
+            case from_back if from_back.startswith("%"):
+                removed = ShellTemplate._remove_back(source, from_back[1:], greedy=False)
+                return removed if removed == source else removed + replacement
+
+            case from_front if from_front.startswith("#"):
+                removed = ShellTemplate._remove_front(source, from_front[1:], greedy=False)
+                return removed if removed == source else replacement + removed
+
+            case regular:
+                regex = fnmatch.translate(regular)[:-2]  # remove \Z at the end of the regex
+                compiled = re.compile(regex)
+                return compiled.sub(replacement, source, count=not greedy)
+
+    def shell_substitute(self, mapping: Mapping[str, str], /, **kwargs: str) -> str:
+        """
+        this method behaves the same as :func:`safe_substitute`, however also expands bash string operations
+
+        Args:
+            mapping(Mapping[str, str]): key-value dictionary of variables
+            **kwargs(str): key-value dictionary of variables passed as kwargs
+
+        Returns:
+            str: string with replaced values
+        """
+        substitutions = (
+            (self._REMOVE_BACK, self._remove_back, "%"),
+            (self._REMOVE_FRONT, self._remove_front, "#"),
+            (self._REPLACE, self._replace, "/"),
+        )
+
+        def generator(variables: dict[str, str]) -> Generator[tuple[str, str], None, None]:
+            for identifier in self.get_identifiers():
+                for regex, function, greediness in substitutions:
+                    if m := regex.match(identifier):
+                        source = variables.get(m.group("key"))
+                        if source is None:
+                            continue
+
+                        # replace pattern with non-greedy
+                        pattern = m.group("pattern").removeprefix(greediness)
+                        greedy = m.group("pattern").startswith(greediness)
+                        # gather all additional args
+                        args = {key: value for key, value in m.groupdict().items() if key not in ("key", "pattern")}
+
+                        yield identifier, function(source, pattern, **args, greedy=greedy)
+                        break
+
+        kwargs.update(mapping)
+        substituted = dict(generator(kwargs))
+
+        return self.safe_substitute(kwargs | substituted)

src/ahriman/models/pkgbuild_patch.py

@@ -21,9 +21,9 @@ import shlex
 
 from dataclasses import dataclass, fields
 from pathlib import Path
-from string import Template
 from typing import Any, Generator, Self
 
+from ahriman.core.configuration.shell_template import ShellTemplate
 from ahriman.core.utils import dataclass_view, filter_json
 
 
@@ -180,8 +180,8 @@ class PkgbuildPatch:
             This function doesn't support recursive substitution
         """
         if isinstance(self.value, str):
-            return Template(self.value).safe_substitute(variables)
-        return [Template(value).safe_substitute(variables) for value in self.value]
+            return ShellTemplate(self.value).shell_substitute(variables)
+        return [ShellTemplate(value).shell_substitute(variables) for value in self.value]
 
     def view(self) -> dict[str, Any]:
         """

tests/ahriman/core/configuration/test_shell_interpolator.py

@@ -1,7 +1,7 @@
 import os
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.configuration.shell_interpolator import ExtendedTemplate, ShellInterpolator
+from ahriman.core.configuration.shell_interpolator import ShellInterpolator
 
 
 def _parser() -> dict[str, dict[str, str]]:
@@ -27,14 +27,6 @@ def _parser() -> dict[str, dict[str, str]]:
     }
 
 
-def test_extended_template() -> None:
-    """
-    must match colons in braces
-    """
-    assert ExtendedTemplate("$key:value").get_identifiers() == ["key"]
-    assert ExtendedTemplate("${key:value}").get_identifiers() == ["key:value"]
-
-
 def test_extract_variables() -> None:
     """
     must extract variables list

tests/ahriman/core/configuration/test_shell_template.py

@@ -0,0 +1,81 @@
+from ahriman.core.configuration.shell_template import ShellTemplate
+
+
+def test_shell_template_braceidpattern() -> None:
+    """
+    must match colons in braces
+    """
+    assert ShellTemplate("$k:value").get_identifiers() == ["k"]
+    assert ShellTemplate("${k:value}").get_identifiers() == ["k:value"]
+
+
+def test_remove_back() -> None:
+    """
+    must remove substring from the back
+    """
+    assert ShellTemplate("${k%removeme}").shell_substitute({"k": "please removeme"}) == "please "
+    assert ShellTemplate("${k%removeme*}").shell_substitute({"k": "please removeme removeme"}) == "please removeme "
+    assert ShellTemplate("${k%removem?}").shell_substitute({"k": "please removeme removeme"}) == "please removeme "
+
+    assert ShellTemplate("${k%%removeme}").shell_substitute({"k": "please removeme removeme"}) == "please removeme "
+    assert ShellTemplate("${k%%removeme*}").shell_substitute({"k": "please removeme removeme"}) == "please "
+    assert ShellTemplate("${k%%removem?}").shell_substitute({"k": "please removeme removeme"}) == "please removeme "
+
+    assert ShellTemplate("${k%removeme}").shell_substitute({}) == "${k%removeme}"
+    assert ShellTemplate("${k%%removeme}").shell_substitute({}) == "${k%%removeme}"
+
+    assert ShellTemplate("${k%r3m0v3m3}").shell_substitute({"k": "please removeme"}) == "please removeme"
+    assert ShellTemplate("${k%%r3m0v3m3}").shell_substitute({"k": "please removeme"}) == "please removeme"
+
+
+def test_remove_front() -> None:
+    """
+    must remove substring from the front
+    """
+    assert ShellTemplate("${k#removeme}").shell_substitute({"k": "removeme please"}) == " please"
+    assert ShellTemplate("${k#*removeme}").shell_substitute({"k": "removeme removeme please"}) == " removeme please"
+    assert ShellTemplate("${k#removem?}").shell_substitute({"k": "removeme removeme please"}) == " removeme please"
+
+    assert ShellTemplate("${k##removeme}").shell_substitute({"k": "removeme removeme please"}) == " removeme please"
+    assert ShellTemplate("${k##*removeme}").shell_substitute({"k": "removeme removeme please"}) == " please"
+    assert ShellTemplate("${k##removem?}").shell_substitute({"k": "removeme removeme please"}) == " removeme please"
+
+    assert ShellTemplate("${k#removeme}").shell_substitute({}) == "${k#removeme}"
+    assert ShellTemplate("${k##removeme}").shell_substitute({}) == "${k##removeme}"
+
+    assert ShellTemplate("${k#r3m0v3m3}").shell_substitute({"k": "removeme please"}) == "removeme please"
+    assert ShellTemplate("${k##r3m0v3m3}").shell_substitute({"k": "removeme please"}) == "removeme please"
+
+
+def test_replace() -> None:
+    """
+    must perform regular replacement
+    """
+    assert ShellTemplate("${k/in/out}").shell_substitute({"k": "in replace in"}) == "out replace in"
+    assert ShellTemplate("${k/in*/out}").shell_substitute({"k": "in replace in"}) == "out"
+    assert ShellTemplate("${k/*in/out}").shell_substitute({"k": "in replace in replace"}) == "out replace"
+    assert ShellTemplate("${k/i?/out}").shell_substitute({"k": "in replace in"}) == "out replace in"
+
+    assert ShellTemplate("${k//in/out}").shell_substitute({"k": "in replace in"}) == "out replace out"
+    assert ShellTemplate("${k//in*/out}").shell_substitute({"k": "in replace in"}) == "out"
+    assert ShellTemplate("${k//*in/out}").shell_substitute({"k": "in replace in replace"}) == "out replace"
+    assert ShellTemplate("${k//i?/out}").shell_substitute({"k": "in replace in replace"}) == "out replace out replace"
+
+    assert ShellTemplate("${k/in/out}").shell_substitute({}) == "${k/in/out}"
+    assert ShellTemplate("${k//in/out}").shell_substitute({}) == "${k//in/out}"
+
+
+def test_replace_back() -> None:
+    """
+    must replace substring from the back
+    """
+    assert ShellTemplate("${k/%in/out}").shell_substitute({"k": "in replace in"}) == "in replace out"
+    assert ShellTemplate("${k/%in/out}").shell_substitute({"k": "in replace in "}) == "in replace in "
+
+
+def test_replace_front() -> None:
+    """
+    must replace substring from the front
+    """
+    assert ShellTemplate("${k/#in/out}").shell_substitute({"k": "in replace in"}) == "out replace in"
+    assert ShellTemplate("${k/#in/out}").shell_substitute({"k": " in replace in"}) == " in replace in"