Compare commits

...
2 Commits
Author SHA1 Message Date
arcanis c282b0f569 fix: handle array expansion without regex 2026-10-29 18:43:00 +02:00
arcanis 96558886bd fix: handle array append correctly 2026-10-29 14:27:14 +02:00
3 changed files with 158 additions and 34 deletions

No files matched your search

@@ -17,7 +17,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import re
import shlex
import tree_sitter_bash
@@ -26,6 +25,7 @@ from enum import StrEnum
from tree_sitter import Language, Node, Parser
from typing import IO, cast
from ahriman.core.configuration.shell_interpolator import ShellInterpolator
from ahriman.core.exceptions import PkgbuildParserError
from ahriman.models.pkgbuild_patch import PkgbuildPatch
@@ -113,8 +113,6 @@ class PkgbuildParser(Iterable[PkgbuildPatch]):
... print(patch.key, patch.value)
"""
_BRACE_EXPANSION = re.compile(r"(.*)\{([^{}]+)\}(.*)")
def __init__(self, stream: IO[bytes]) -> None:
"""
Args:
@@ -122,26 +120,6 @@ class PkgbuildParser(Iterable[PkgbuildPatch]):
"""
self._content = stream.read()
@staticmethod
def _expand_array(value: str, *, expand: bool = True) -> list[str]:
"""
expand comma-separated brace expressions in a value
Args:
value(str): value that may contain a brace expression
expand(bool, optional): whether brace expansion is allowed (Default value = True)
Returns:
list[str]: expanded values if applicable
"""
match = PkgbuildParser._BRACE_EXPANSION.fullmatch(value)
# if there is no comma inside it should be treated as normal value
if match is None or not expand or "," not in match.group(2):
return [value]
prefix, values, suffix = match.groups()
return [f"{prefix}{item}{suffix}" for item in values.split(",")]
@staticmethod
def _text(node: Node) -> str:
"""
@@ -155,6 +133,19 @@ class PkgbuildParser(Iterable[PkgbuildPatch]):
"""
return cast(bytes, node.text).decode("utf8", errors="backslashreplace")
@staticmethod
def _unquote(value: str) -> str:
"""
remove shell quotes and escapes from a single value
Args:
value(str): shell value to unquote
Returns:
str: unquoted value
"""
return "".join(shlex.split(value, comments=False, posix=True))
def _assignment(self, node: Node) -> PkgbuildPatch:
"""
convert an assignment node
@@ -178,27 +169,75 @@ class PkgbuildParser(Iterable[PkgbuildPatch]):
case PkgbuildNodeType.Array:
array_values = []
for element in filter(lambda child: child.type != PkgbuildNodeType.Comment, value.named_children):
item = self._value(element)
values = self._expand_array(item, expand=element.type == PkgbuildNodeType.Concatenation)
array_values.extend(values)
array_values.extend(self._expand_array(element))
return PkgbuildPatch(key, array_values)
case _:
return PkgbuildPatch(key, self._value(value))
def _read(self, node: Node) -> Iterator[PkgbuildPatch]:
def _expand_array(self, node: Node) -> list[str]:
"""
expand simple comma-separated brace groups in an array element
Args:
node(Node): array element to expand
Returns:
list[str]: expanded element values
"""
if node.type != PkgbuildNodeType.Concatenation:
return [self._value(node)]
values = [""]
children = node.named_children
while children:
match children[:3]:
case [
Node(type=PkgbuildNodeType.Word, text=b"{"),
Node(type=PkgbuildNodeType.Word) as content,
Node(type=PkgbuildNodeType.Word, text=b"}"),
] if "," in (escaped := self._text(content).replace("\\,", ShellInterpolator.DATA_LINK_ESCAPE)):
parts = [
self._unquote(part).replace(ShellInterpolator.DATA_LINK_ESCAPE, ",")
for part in escaped.split(",")
]
children = children[3:]
case [head, *_]:
parts = [self._value(head)]
children = children[1:]
values = [prefix + part for prefix in values for part in parts]
return values
def _read(self, node: Node, values: dict[str | None, PkgbuildPatch]) -> Iterator[PkgbuildPatch]:
"""
find assignments and functions in a syntax tree node
Args:
node(Node): syntax tree node to inspect
values(dict[str | None, PkgbuildPatch]): cached assignments
Yields:
PkgbuildPatch: parsed entries
"""
match node.type:
case PkgbuildNodeType.VariableAssignment if not any(child.type == PkgbuildNodeType.Append for child in node.children):
yield self._assignment(node)
case PkgbuildNodeType.VariableAssignment if any(child.type == PkgbuildNodeType.Append for child in node.children):
patch = self._assignment(node)
match values.get(patch.key), patch.value:
case PkgbuildPatch(value=str() as previous), str() as appended:
patch = PkgbuildPatch(patch.key, previous + appended)
case PkgbuildPatch(value=str() as previous), list() as appended:
patch = PkgbuildPatch(patch.key, [previous, *appended])
case PkgbuildPatch(value=list() as previous), list() as appended:
patch = PkgbuildPatch(patch.key, previous + appended)
values[patch.key] = patch
yield patch
case PkgbuildNodeType.VariableAssignment:
patch = self._assignment(node)
values[patch.key] = patch
yield patch
case PkgbuildNodeType.FunctionDefinition:
name = cast(Node, node.child_by_field_name(PkgbuildNodeField.Name))
@@ -215,7 +254,7 @@ class PkgbuildParser(Iterable[PkgbuildPatch]):
PkgbuildNodeType.String,
PkgbuildNodeType.Word,
):
yield from self._read(child)
yield from self._read(child, values)
def _value(self, node: Node) -> str:
"""
@@ -239,7 +278,7 @@ class PkgbuildParser(Iterable[PkgbuildPatch]):
case _:
return value
return "".join(shlex.split(value, comments=False, posix=True))
return self._unquote(value)
def parse(self) -> Iterator[PkgbuildPatch]:
"""
@@ -265,7 +304,7 @@ class PkgbuildParser(Iterable[PkgbuildPatch]):
f"syntax error at line {error.start_point.row + 1}, column {error.start_point.column + 1}"
)
yield from self._read(tree.root_node)
yield from self._read(tree.root_node, {})
def __iter__(self) -> Iterator[PkgbuildPatch]:
"""
@@ -18,9 +18,48 @@ def test_expand_array() -> None:
parser = PkgbuildParser(BytesIO(b"array=(prefix{first,second}suffix)"))
assert list(parser.parse()) == [PkgbuildPatch("array", ["prefixfirstsuffix", "prefixsecondsuffix"])]
parser = PkgbuildParser(BytesIO(br"array=(prefix{first\,second,third}suffix)"))
assert list(parser.parse()) == [PkgbuildPatch("array", ["prefixfirst,secondsuffix", "prefixthirdsuffix"])]
parser = PkgbuildParser(BytesIO(b"""array=("prefix{first,second}suffix")"""))
assert list(parser.parse()) == [PkgbuildPatch("array", ["prefix{first,second}suffix"])]
parser = PkgbuildParser(BytesIO(b"""array=(x"{a,b}" x'{a,b}' "{x,y}"{a,b})"""))
assert list(parser) == [PkgbuildPatch("array", ["x{a,b}", "x{a,b}", "{x,y}a", "{x,y}b"])]
parser = PkgbuildParser(BytesIO(b"""array=(${v:-a,b}suffix ${v:-a,b}{,.sig} $(echo '{a,b}')suffix)"""))
assert list(parser) == [PkgbuildPatch("array", [
"${v:-a,b}suffix", "${v:-a,b}", "${v:-a,b}.sig", "$(echo '{a,b}')suffix",
])]
def test_parse_append_array() -> None:
"""
must retain original array elements when appending and iterating again
"""
parser = PkgbuildParser(BytesIO(
b"source=(base.patch)\nsource+=(extra.patch)\nsource+=(last.patch)\nsource=(reset.patch)\nsource+=(extra.patch)"
))
expected = [
PkgbuildPatch("source", ["base.patch"]),
PkgbuildPatch("source", ["base.patch", "extra.patch"]),
PkgbuildPatch("source", ["base.patch", "extra.patch", "last.patch"]),
PkgbuildPatch("source", ["reset.patch"]),
PkgbuildPatch("source", ["reset.patch", "extra.patch"]),
]
assert list(parser) == expected
def test_parse_append_array_to_string() -> None:
"""
must preserve a string as the first element when appending an array
"""
parser = PkgbuildParser(BytesIO(b"source=base.patch\nsource+=(extra.patch)"))
assert list(parser) == [
PkgbuildPatch("source", "base.patch"),
PkgbuildPatch("source", ["base.patch", "extra.patch"]),
]
def test_parse_array() -> None:
"""
@@ -451,10 +451,56 @@ def test_parse_pacman_static(resource_path_root: Path) -> None:
"source": [
"git+https://gitlab.archlinux.org/pacman/pacman.git#tag=v${_git_tag}?signed",
"pacman-revertme-makepkg-remove-libdepends-and-libprovides.patch::https://gitlab.archlinux.org/pacman/pacman/-/commit/354a300cd26bb1c7e6551473596be5ecced921de.patch",
"https://github.com/nghttp2/nghttp2/releases/download/v$_nghttp2_ver/nghttp2-$_nghttp2_ver.tar.xz",
"https://github.com/c-ares/c-ares/releases/download/v${_cares_ver}/c-ares-${_cares_ver}.tar.gz",
"https://github.com/c-ares/c-ares/releases/download/v${_cares_ver}/c-ares-${_cares_ver}.tar.gz.asc",
"https://curl.haxx.se/download/curl-${_curlver}.tar.gz",
"https://curl.haxx.se/download/curl-${_curlver}.tar.gz.asc",
"https://github.com/openssl/openssl/releases/download/openssl-${_sslver}/openssl-${_sslver}.tar.gz",
"https://github.com/openssl/openssl/releases/download/openssl-${_sslver}/openssl-${_sslver}.tar.gz.asc",
"ca-dir.patch",
"openssl-3.0.7-no-atomic.patch",
"https://zlib.net/zlib-${_zlibver}.tar.gz",
"https://zlib.net/zlib-${_zlibver}.tar.gz.asc",
"git+https://github.com/tukaani-project/xz#tag=v${_xzver}",
"https://sourceware.org/pub/bzip2/bzip2-${_bzipver}.tar.gz",
"https://sourceware.org/pub/bzip2/bzip2-${_bzipver}.tar.gz.sig",
"https://github.com/facebook/zstd/releases/download/v${_zstdver}/zstd-${_zstdver}.tar.zst",
"https://github.com/facebook/zstd/releases/download/v${_zstdver}/zstd-${_zstdver}.tar.zst.sig",
"https://gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-${_gpgerrorver}.tar.bz2",
"https://gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-${_gpgerrorver}.tar.bz2.sig",
"https://gnupg.org/ftp/gcrypt/libassuan/libassuan-${_libassuanver}.tar.bz2",
"https://gnupg.org/ftp/gcrypt/libassuan/libassuan-${_libassuanver}.tar.bz2.sig",
"https://www.gnupg.org/ftp/gcrypt/gpgme/gpgme-${_gpgmever}.tar.bz2",
"https://www.gnupg.org/ftp/gcrypt/gpgme/gpgme-${_gpgmever}.tar.bz2.sig",
"https://github.com/libarchive/libarchive/releases/download/v${_libarchive_ver}/libarchive-${_libarchive_ver}.tar.xz",
"https://github.com/libarchive/libarchive/releases/download/v${_libarchive_ver}/libarchive-${_libarchive_ver}.tar.xz.asc",
],
"validpgpkeys": [
"6645B0A8C7005E78DB1D7864F99FFE0FEAE999BD",
"B8151B117037781095514CA7BBDFFC92306B1121",
"27EDEAF22F3ABCEB50DB9A125CC908FDB71E12C2",
"DA7D64E4C82C6294CB73A20E22E3D13B5411B7CA",
"27EDEAF22F3ABCEB50DB9A125CC908FDB71E12C2",
"8657ABB260F056B1E5190839D9C4D26D0E604491",
"7953AC1FBC3DC8B3B292393ED5E9E43F7DF9EE8C",
"A21FAB74B0088AA361152586B8EF1A6BA9DA2D5C",
"EFC0A467D613CB83C7ED6D30D894E2CE8B3D79F5",
"BA5473A2B0587B07FB27CF2D216094DFD0CB81EF",
"8657ABB260F056B1E5190839D9C4D26D0E604491",
"7953AC1FBC3DC8B3B292393ED5E9E43F7DF9EE8C",
"A21FAB74B0088AA361152586B8EF1A6BA9DA2D5C",
"EFC0A467D613CB83C7ED6D30D894E2CE8B3D79F5",
"5ED46A6721D365587791E2AA783FCD8E58BCAFBA",
"3690C240CE51B4670D30AD1C38EE757D69184620",
"EC3CFE88F6CA0788774F5C1D1AA44BE649DE760A",
"4EF4AC63455FC9F4545D9B7DEF8FE99528B52FFD",
"D8692123C4065DEA5E0F3AB5249B39D24F25E3B6",
"031EC2536E580D8EA286A9F22071B08A33BD3F06",
"6DAA6E64A76D2840571B4902528897B826403ADA",
"AC8E115BF73E2D8D47FA9908E98E9B2D19C6C8BD",
"A5A45B12AD92D964B89EEE2DEC560C81CEC2276E",
"DB2C7CF1B4C265FAEF56E3FC5848A18B8F14184B",
],
"sha512sums": [
"44e00c2bc259fe6a85de71f7fd8a43fcfd1b8fb7d920d2267bd5b347e02f1dab736b3d96e31faf7b535480398e2348f7c0b9914e51ca7e12bab2d5b8003926b4",
@@ -487,8 +533,8 @@ def test_parse_pacman_static(resource_path_root: Path) -> None:
"LDFLAGS": "$LDFLAGS -static",
"CC": "musl-gcc -fno-stack-protector",
"CXX": "musl-gcc -fno-stack-protector",
"CFLAGS": "${CFLAGS/-fstack-protector-strong/}",
"CXXFLAGS": "${CXXFLAGS/-fstack-protector-strong/}",
"CFLAGS": "${CFLAGS/-fstack-protector-strong/} -D_LARGEFILE64_SOURCE",
"CXXFLAGS": "${CXXFLAGS/-fstack-protector-strong/} -D_LARGEFILE64_SOURCE",
"PKGEXT": ".pkg.tar.xz",
}