From e553d96d33eea0a43f015144b77ad40ee9d13298 Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Mon, 16 Sep 2024 17:53:24 +0300 Subject: [PATCH] expand bash --- docs/ahriman.core.configuration.rst | 8 + src/ahriman/core/alpm/pkgbuild_parser.py | 10 +- .../core/configuration/shell_interpolator.py | 15 +- .../core/configuration/shell_template.py | 158 ++++++++++++++++++ src/ahriman/models/pkgbuild_patch.py | 6 +- .../configuration/test_shell_interpolator.py | 10 +- .../core/configuration/test_shell_template.py | 81 +++++++++ 7 files changed, 259 insertions(+), 29 deletions(-) create mode 100644 src/ahriman/core/configuration/shell_template.py create mode 100644 tests/ahriman/core/configuration/test_shell_template.py diff --git a/docs/ahriman.core.configuration.rst b/docs/ahriman.core.configuration.rst index efd53d89..4bb208d9 100644 --- a/docs/ahriman.core.configuration.rst +++ b/docs/ahriman.core.configuration.rst @@ -28,6 +28,14 @@ ahriman.core.configuration.shell\_interpolator module :no-undoc-members: :show-inheritance: +ahriman.core.configuration.shell\_template module +------------------------------------------------- + +.. automodule:: ahriman.core.configuration.shell_template + :members: + :no-undoc-members: + :show-inheritance: + ahriman.core.configuration.validator module ------------------------------------------- diff --git a/src/ahriman/core/alpm/pkgbuild_parser.py b/src/ahriman/core/alpm/pkgbuild_parser.py index a5e5c43b..94e2be88 100644 --- a/src/ahriman/core/alpm/pkgbuild_parser.py +++ b/src/ahriman/core/alpm/pkgbuild_parser.py @@ -270,9 +270,9 @@ class PkgbuildParser(shlex.shlex): PkgbuildPatch: extracted a PKGBUILD node """ # simple assignment rule - if (match := self._STRING_ASSIGNMENT.match(token)) is not None: - key = match.group("key") - value = match.group("value") + if m := self._STRING_ASSIGNMENT.match(token): + key = m.group("key") + value = m.group("value") yield PkgbuildPatch(key, value) return @@ -282,8 +282,8 @@ class PkgbuildParser(shlex.shlex): match self.get_token(): # array processing. Arrays will be sent as "key=", "(", values, ")" - case PkgbuildToken.ArrayStarts if (match := self._ARRAY_ASSIGNMENT.match(token)) is not None: - key = match.group("key") + case PkgbuildToken.ArrayStarts if m := self._ARRAY_ASSIGNMENT.match(token): + key = m.group("key") value = self._parse_array() yield PkgbuildPatch(key, value) diff --git a/src/ahriman/core/configuration/shell_interpolator.py b/src/ahriman/core/configuration/shell_interpolator.py index c7614195..d2f35576 100644 --- a/src/ahriman/core/configuration/shell_interpolator.py +++ b/src/ahriman/core/configuration/shell_interpolator.py @@ -24,16 +24,7 @@ import sys from collections.abc import Generator, Mapping, MutableMapping from string import Template - -class ExtendedTemplate(Template): - """ - extension to the default :class:`Template` class, which also enabled braces regex to lookup in sections - - Attributes: - braceidpattern(str): regular expression to match a colon inside braces - """ - - braceidpattern = r"(?a:[_a-z0-9][_a-z0-9:]*)" +from ahriman.core.configuration.shell_template import ShellTemplate class ShellInterpolator(configparser.Interpolation): @@ -60,7 +51,7 @@ class ShellInterpolator(configparser.Interpolation): """ def identifiers() -> Generator[tuple[str | None, str], None, None]: # extract all found identifiers and parse them - for identifier in ExtendedTemplate(value).get_identifiers(): + for identifier in ShellTemplate(value).get_identifiers(): match identifier.split(":"): case [lookup_option]: # single option from the same section yield None, lookup_option @@ -121,7 +112,7 @@ class ShellInterpolator(configparser.Interpolation): # resolve internal references variables = dict(self._extract_variables(parser, value, defaults)) - internal = ExtendedTemplate(escaped).safe_substitute(variables) + internal = ShellTemplate(escaped).safe_substitute(variables) # resolve enriched environment variables by using default Template class environment = Template(internal).safe_substitute(self.environment()) diff --git a/src/ahriman/core/configuration/shell_template.py b/src/ahriman/core/configuration/shell_template.py new file mode 100644 index 00000000..ff135e7e --- /dev/null +++ b/src/ahriman/core/configuration/shell_template.py @@ -0,0 +1,158 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import fnmatch +import re + +from collections.abc import Generator, Mapping +from string import Template + + +class ShellTemplate(Template): + """ + extension to the default :class:`Template` class, which also adds additional tokens to braced regex and enables + bash expansion + + Attributes: + braceidpattern(str): regular expression to match every character except for closing bracket + """ + + braceidpattern = r"(?a:[_a-z0-9][^}]*)" + + _REMOVE_BACK = re.compile(r"^(?P\w+)%(?P.+)$") + _REMOVE_FRONT = re.compile(r"^(?P\w+)#(?P.+)$") + _REPLACE = re.compile(r"^(?P\w+)/(?P.+)/(?P.+)$") + + @staticmethod + def _remove_back(source: str, pattern: str, *, greedy: bool) -> str: + """ + resolve "${var%(%)pattern}" constructions + + Args: + source(str): source string to match the pattern inside + pattern(str): shell expression to match + greedy(bool): match as much as possible or not + + Returns: + str: result after removal ``pattern`` from the end of the string + """ + regex = fnmatch.translate(pattern) + compiled = re.compile(regex) + + result = source + start_pos = 0 + + while m := compiled.search(source, start_pos): + result = source[:m.start()] + start_pos += m.start() + 1 + if greedy: + break + + return result + + @staticmethod + def _remove_front(source: str, pattern: str, *, greedy: bool) -> str: + """ + resolve "${var#(#)pattern}" constructions + + Args: + source(str): source string to match the pattern inside + pattern(str): shell expression to match + greedy(bool): match as much as possible or not + + Returns: + str: result after removal ``pattern`` from the start of the string + """ + regex = fnmatch.translate(pattern)[:-2] # remove \Z at the end of the regex + if not greedy: + regex = regex.replace("*", "*?") + compiled = re.compile(regex) + + m = compiled.match(source) + if m is None: + return source + + return source[m.end():] + + @staticmethod + def _replace(source: str, pattern: str, replacement: str, *, greedy: bool) -> str: + """ + resolve "${var/(/)pattern/replacement}" constructions + + Args: + source(str): source string to match the pattern inside + pattern(str): shell expression to match + replacement(str): new substring + greedy(bool): replace as much as possible or not + + Returns: + str: result after replacing ``pattern`` by ``replacement`` + """ + match pattern: + case from_back if from_back.startswith("%"): + removed = ShellTemplate._remove_back(source, from_back[1:], greedy=False) + return removed if removed == source else removed + replacement + + case from_front if from_front.startswith("#"): + removed = ShellTemplate._remove_front(source, from_front[1:], greedy=False) + return removed if removed == source else replacement + removed + + case regular: + regex = fnmatch.translate(regular)[:-2] # remove \Z at the end of the regex + compiled = re.compile(regex) + return compiled.sub(replacement, source, count=not greedy) + + def shell_substitute(self, mapping: Mapping[str, str], /, **kwargs: str) -> str: + """ + this method behaves the same as :func:`safe_substitute`, however also expands bash string operations + + Args: + mapping(Mapping[str, str]): key-value dictionary of variables + **kwargs(str): key-value dictionary of variables passed as kwargs + + Returns: + str: string with replaced values + """ + substitutions = ( + (self._REMOVE_BACK, self._remove_back, "%"), + (self._REMOVE_FRONT, self._remove_front, "#"), + (self._REPLACE, self._replace, "/"), + ) + + def generator(variables: dict[str, str]) -> Generator[tuple[str, str], None, None]: + for identifier in self.get_identifiers(): + for regex, function, greediness in substitutions: + if m := regex.match(identifier): + source = variables.get(m.group("key")) + if source is None: + continue + + # replace pattern with non-greedy + pattern = m.group("pattern").removeprefix(greediness) + greedy = m.group("pattern").startswith(greediness) + # gather all additional args + args = {key: value for key, value in m.groupdict().items() if key not in ("key", "pattern")} + + yield identifier, function(source, pattern, **args, greedy=greedy) + break + + kwargs.update(mapping) + substituted = dict(generator(kwargs)) + + return self.safe_substitute(kwargs | substituted) diff --git a/src/ahriman/models/pkgbuild_patch.py b/src/ahriman/models/pkgbuild_patch.py index b4efe3a2..d060afab 100644 --- a/src/ahriman/models/pkgbuild_patch.py +++ b/src/ahriman/models/pkgbuild_patch.py @@ -21,9 +21,9 @@ import shlex from dataclasses import dataclass, fields from pathlib import Path -from string import Template from typing import Any, Generator, Self +from ahriman.core.configuration.shell_template import ShellTemplate from ahriman.core.utils import dataclass_view, filter_json @@ -180,8 +180,8 @@ class PkgbuildPatch: This function doesn't support recursive substitution """ if isinstance(self.value, str): - return Template(self.value).safe_substitute(variables) - return [Template(value).safe_substitute(variables) for value in self.value] + return ShellTemplate(self.value).shell_substitute(variables) + return [ShellTemplate(value).shell_substitute(variables) for value in self.value] def view(self) -> dict[str, Any]: """ diff --git a/tests/ahriman/core/configuration/test_shell_interpolator.py b/tests/ahriman/core/configuration/test_shell_interpolator.py index bcacb712..24b36b3f 100644 --- a/tests/ahriman/core/configuration/test_shell_interpolator.py +++ b/tests/ahriman/core/configuration/test_shell_interpolator.py @@ -1,7 +1,7 @@ import os from ahriman.core.configuration import Configuration -from ahriman.core.configuration.shell_interpolator import ExtendedTemplate, ShellInterpolator +from ahriman.core.configuration.shell_interpolator import ShellInterpolator def _parser() -> dict[str, dict[str, str]]: @@ -27,14 +27,6 @@ def _parser() -> dict[str, dict[str, str]]: } -def test_extended_template() -> None: - """ - must match colons in braces - """ - assert ExtendedTemplate("$key:value").get_identifiers() == ["key"] - assert ExtendedTemplate("${key:value}").get_identifiers() == ["key:value"] - - def test_extract_variables() -> None: """ must extract variables list diff --git a/tests/ahriman/core/configuration/test_shell_template.py b/tests/ahriman/core/configuration/test_shell_template.py new file mode 100644 index 00000000..fd569eef --- /dev/null +++ b/tests/ahriman/core/configuration/test_shell_template.py @@ -0,0 +1,81 @@ +from ahriman.core.configuration.shell_template import ShellTemplate + + +def test_shell_template_braceidpattern() -> None: + """ + must match colons in braces + """ + assert ShellTemplate("$k:value").get_identifiers() == ["k"] + assert ShellTemplate("${k:value}").get_identifiers() == ["k:value"] + + +def test_remove_back() -> None: + """ + must remove substring from the back + """ + assert ShellTemplate("${k%removeme}").shell_substitute({"k": "please removeme"}) == "please " + assert ShellTemplate("${k%removeme*}").shell_substitute({"k": "please removeme removeme"}) == "please removeme " + assert ShellTemplate("${k%removem?}").shell_substitute({"k": "please removeme removeme"}) == "please removeme " + + assert ShellTemplate("${k%%removeme}").shell_substitute({"k": "please removeme removeme"}) == "please removeme " + assert ShellTemplate("${k%%removeme*}").shell_substitute({"k": "please removeme removeme"}) == "please " + assert ShellTemplate("${k%%removem?}").shell_substitute({"k": "please removeme removeme"}) == "please removeme " + + assert ShellTemplate("${k%removeme}").shell_substitute({}) == "${k%removeme}" + assert ShellTemplate("${k%%removeme}").shell_substitute({}) == "${k%%removeme}" + + assert ShellTemplate("${k%r3m0v3m3}").shell_substitute({"k": "please removeme"}) == "please removeme" + assert ShellTemplate("${k%%r3m0v3m3}").shell_substitute({"k": "please removeme"}) == "please removeme" + + +def test_remove_front() -> None: + """ + must remove substring from the front + """ + assert ShellTemplate("${k#removeme}").shell_substitute({"k": "removeme please"}) == " please" + assert ShellTemplate("${k#*removeme}").shell_substitute({"k": "removeme removeme please"}) == " removeme please" + assert ShellTemplate("${k#removem?}").shell_substitute({"k": "removeme removeme please"}) == " removeme please" + + assert ShellTemplate("${k##removeme}").shell_substitute({"k": "removeme removeme please"}) == " removeme please" + assert ShellTemplate("${k##*removeme}").shell_substitute({"k": "removeme removeme please"}) == " please" + assert ShellTemplate("${k##removem?}").shell_substitute({"k": "removeme removeme please"}) == " removeme please" + + assert ShellTemplate("${k#removeme}").shell_substitute({}) == "${k#removeme}" + assert ShellTemplate("${k##removeme}").shell_substitute({}) == "${k##removeme}" + + assert ShellTemplate("${k#r3m0v3m3}").shell_substitute({"k": "removeme please"}) == "removeme please" + assert ShellTemplate("${k##r3m0v3m3}").shell_substitute({"k": "removeme please"}) == "removeme please" + + +def test_replace() -> None: + """ + must perform regular replacement + """ + assert ShellTemplate("${k/in/out}").shell_substitute({"k": "in replace in"}) == "out replace in" + assert ShellTemplate("${k/in*/out}").shell_substitute({"k": "in replace in"}) == "out" + assert ShellTemplate("${k/*in/out}").shell_substitute({"k": "in replace in replace"}) == "out replace" + assert ShellTemplate("${k/i?/out}").shell_substitute({"k": "in replace in"}) == "out replace in" + + assert ShellTemplate("${k//in/out}").shell_substitute({"k": "in replace in"}) == "out replace out" + assert ShellTemplate("${k//in*/out}").shell_substitute({"k": "in replace in"}) == "out" + assert ShellTemplate("${k//*in/out}").shell_substitute({"k": "in replace in replace"}) == "out replace" + assert ShellTemplate("${k//i?/out}").shell_substitute({"k": "in replace in replace"}) == "out replace out replace" + + assert ShellTemplate("${k/in/out}").shell_substitute({}) == "${k/in/out}" + assert ShellTemplate("${k//in/out}").shell_substitute({}) == "${k//in/out}" + + +def test_replace_back() -> None: + """ + must replace substring from the back + """ + assert ShellTemplate("${k/%in/out}").shell_substitute({"k": "in replace in"}) == "in replace out" + assert ShellTemplate("${k/%in/out}").shell_substitute({"k": "in replace in "}) == "in replace in " + + +def test_replace_front() -> None: + """ + must replace substring from the front + """ + assert ShellTemplate("${k/#in/out}").shell_substitute({"k": "in replace in"}) == "out replace in" + assert ShellTemplate("${k/#in/out}").shell_substitute({"k": " in replace in"}) == " in replace in"