feat: migrate pkgbuild parser to tree-sitter (#168)

2026-10-20 11:47:57 +00:00 · 2026-10-19 23:10:58 +03:00
1 parent 9b65e01d2f
commit 9e5468313e
@@ -10,15 +10,15 @@ echo -e '[arcanisrepo]\nServer = https://repo.arcanis.me/$arch\nSigLevel = Never
 # refresh the image
 pacman -Syyu --noconfirm
 # main dependencies
-pacman -S --noconfirm devtools git npm pyalpm python-bcrypt python-filelock python-inflection python-pyelftools python-requests python-systemd sudo
+pacman -S --noconfirm --asdeps devtools git pyalpm python-bcrypt python-filelock python-inflection python-pyelftools python-requests python-tree-sitter python-tree-sitter-bash
 # make dependencies
-pacman -S --noconfirm --asdeps base-devel python-build python-hatchling python-installer python-tox python-wheel
+pacman -S --noconfirm --asdeps base-devel npm python-build python-hatchling python-installer python-tox python-wheel
 # optional dependencies
 if [[ -z $MINIMAL_INSTALL ]]; then
    # web server
-    pacman -S --noconfirm python-aioauth-client python-aiohttp python-aiohttp-apispec-git python-aiohttp-cors python-aiohttp-jinja2 python-aiohttp-security python-aiohttp-session python-aiohttp-sse-git python-cryptography python-jinja
+    pacman -S --noconfirm --asdeps python-aioauth-client python-aiohttp python-aiohttp-apispec-git python-aiohttp-cors python-aiohttp-jinja2 python-aiohttp-security python-aiohttp-session python-aiohttp-sse-git python-cryptography python-jinja
    # additional features
-    pacman -S --noconfirm gnupg ipython python-boto3 python-cerberus python-matplotlib rsync
+    pacman -S --noconfirm --asdeps gnupg ipython python-boto3 python-cerberus python-matplotlib python-systemd rsync
 fi
 # FIXME since 1.0.4 devtools requires dbus to be run, which doesn't work now in container
 cp "docker/systemd-nspawn.sh" "/usr/local/bin/systemd-nspawn"
@@ -12,6 +12,8 @@ dependencies = [
    "inflection",
    "pyelftools",
    "requests",
    "tree-sitter",
    "tree-sitter-bash",
 ]
 description = "ArcH linux ReposItory MANager, core package"
 dynamic = [
@@ -17,355 +17,242 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import itertools
 import re
 import shlex
 import tree_sitter_bash
 from collections.abc import Iterator
 from enum import StrEnum
-from typing import IO
+from tree_sitter import Language, Node, Parser
 from typing import IO, cast
 from ahriman.core.exceptions import PkgbuildParserError
 from ahriman.models.pkgbuild_patch import PkgbuildPatch
-class PkgbuildToken(StrEnum):
+class PkgbuildNodeType(StrEnum):
    """
-    well-known tokens dictionary
+    tree-sitter node types used by parser
    Attributes:
-        ArrayEnds(PkgbuildToken): array ends token
+        Append(PkgbuildNodeType): append node
-        ArrayStarts(PkgbuildToken): array starts token
+        Array(PkgbuildNodeType): array node
-        Comma(PkgbuildToken): comma token
+        Comment(PkgbuildNodeType): comment node
-        Comment(PkgbuildToken): comment token
+        Concatenation(PkgbuildNodeType): concatenation node
-        FunctionDeclaration(PkgbuildToken): function declaration token
+        Error(PkgbuildNodeType): syntax error node
-        FunctionEnds(PkgbuildToken): function ends token
+        FunctionDefinition(PkgbuildNodeType): function definition node
-        FunctionStarts(PkgbuildToken): function starts token
+        Number(PkgbuildNodeType): numeric word node
-        NewLine(PkgbuildToken): new line token
+        RawString(PkgbuildNodeType): single-quoted string node
        String(PkgbuildNodeType): double-quoted string node
        StringContent(PkgbuildNodeType): string content node
        VariableAssignment(PkgbuildNodeType): assignment node
        Word(PkgbuildNodeType): word node
    """
-    ArrayStarts = "("
+    Append = "+="
-    ArrayEnds = ")"
+    Array = "array"
-
+    Comment = "comment"
-    Comma = ","
+    Concatenation = "concatenation"
-
+    Error = "ERROR"
-    Comment = "#"
+    FunctionDefinition = "function_definition"
-
+    Number = "number"
-    FunctionDeclaration = "()"
+    RawString = "raw_string"
-    FunctionStarts = "{"
+    String = "string"
-    FunctionEnds = "}"
+    StringContent = "string_content"
-
+    VariableAssignment = "variable_assignment"
-    NewLine = "\n"
+    Word = "word"
-class PkgbuildParser(shlex.shlex):
+class PkgbuildNodeField(StrEnum):
    """
-    simple pkgbuild reader implementation in pure python, because others suck.
+    tree-sitter node fields used by parser
-    What is it:
+    Attributes:
        Body(PkgbuildNodeField): function body field
        Name(PkgbuildNodeField): variable or function name field
        Value(PkgbuildNodeField): assignment value field
    """
-    #. Simple PKGBUILD parser written in python.
+    Body = "body"
-    #. No shell execution, so it is free from random shell attacks.
+    Name = "name"
-    #. Able to parse simple constructions (assignments, comments, functions, arrays).
+    Value = "value"
 class PkgbuildParser:
    """
    PKGBUILD parser implementation in python
    What it is:
    #. A PKGBUILD parser based on tree-sitter's bash grammar.
    #. A parser for assignments, arrays, and functions.
    #. A parser which does not execute shell code.
    What it is not:
-    #. Fully functional shell parser.
+    #. A shell executor.
-    #. Shell executor.
+    #. A shell interpreter. Variable and command substitutions are returned as written.
    #. No parameter expansion.
-    For more details what does it support, please, consult with the test cases.
+    Simple comma-separated brace expressions in array elements are expanded. Function bodies are returned from the
    original source, with their formatting preserved. Assignments inside shell statements and control-flow nodes are
    extracted without checking whether those statements would run.
    Examples:
-        This class is heavily based on :mod:`shlex` parser, but instead of strings operates with the
+        Parse package assignments::
        :class:`ahriman.models.pkgbuild_patch.PkgbuildPatch` objects. The main way to use it is to call :func:`parse()`
        function and collect parsed objects, e.g.::
-            >>> parser = PkgbuildParser(StringIO("input string"))
+            >>> source = b"pkgname=example\\npkgver=1\\n"
-            >>> for patch in parser.parse():
+            >>> for patch in PkgbuildParser(BytesIO(source)).parse():
-            >>>     print(f"{patch.key} = {patch.value}")
+            ...     print(patch.key, patch.value)
-        It doesn't store the state of the fields (but operates with the :mod:`shlex` parser state), so no shell
+        Function bodies are returned as function patches::
-        post-processing is performed (e.g. variable substitution).
+
            >>> source = b"pkgver() { printf 1; }"
            >>> for patch in list(PkgbuildParser(BytesIO(source)).parse())
            ...     print(patch.key, patch.value)
    """
-    _ARRAY_ASSIGNMENT = re.compile(r"^(?P<key>\w+)=$")
+    _BRACE_EXPANSION = re.compile(r"(.*)\{([^{}]+)\}(.*)")
    # in addition to usual assignment, functions can have dash
    _FUNCTION_DECLARATION = re.compile(r"^(?P<key>[\w-]+)$")
    _STRING_ASSIGNMENT = re.compile(r"^(?P<key>\w+)=(?P<value>.+)$")
-    def __init__(self, stream: IO[str]) -> None:
+    def __init__(self, stream: IO[bytes]) -> None:
-        """
+        """"""
-        Args:
+        self._content = stream.read()
            stream(IO[str]): input stream containing PKGBUILD content
        """
        shlex.shlex.__init__(self, stream, posix=True, punctuation_chars=True)
        self._io = stream  # direct access without type casting
        # ignore substitution and extend bash symbols
        self.wordchars += "${}#:+-@!"
        # in case of default behavior, it will ignore, for example, segment part of url outside of quotes
        self.commenters = ""
    @staticmethod
-    def _expand_array(array: list[str]) -> list[str]:
+    def _expand_array(value: str, *, expand: bool = True) -> list[str]:
        """
-        bash array expansion simulator. It takes raw array and tries to expand constructions like
+        expand comma-separated brace expressions in a value
        ``(first prefix-{mid1,mid2}-suffix last)`` into ``(first, prefix-mid1-suffix prefix-mid2-suffix last)``
        Args:
-            array(list[str]): input array
+            value(str): value that may contain a brace expression
            expand(bool, optional): whether brace expansion is allowed (Default value = True)
        Returns:
-            list[str]: either source array or expanded array if possible
+            list[str]: expanded values if applicable
        Raises:
            PkgbuildParserError: if there are errors in parser
        """
-        # we are using comma as marker for expansion (if any)
+        match = PkgbuildParser._BRACE_EXPANSION.fullmatch(value)
-        if PkgbuildToken.Comma not in array:
+        # if there is no comma inside it should be treated as normal value
-            return array
+        if match is None or not expand or "," not in match.group(2):
-        # again sanity check, for expansion there are at least 3 elements (first, last and comma)
+            return [value]
        if len(array) < 3:
            return array
-        result = []
+        prefix, values, suffix = match.groups()
-        buffer, prefix = [], None
+        return [f"{prefix}{item}{suffix}" for item in values.split(",")]
-        for index, (first, second) in enumerate(itertools.pairwise(array)):
+    @staticmethod
-            match (first, second):
+    def _text(node: Node) -> str:
                # in this case we check if expansion should be started
                # this condition matches "prefix{first", ","
                case (_, PkgbuildToken.Comma) if PkgbuildToken.FunctionStarts in first:
                    prefix, part = first.rsplit(PkgbuildToken.FunctionStarts, maxsplit=1)
                    buffer.append(f"{prefix}{part}")
                # the last element case, it matches either ",", "last}" or ",", "last}suffix"
                # in case if there is suffix, it must be appended to all list elements
                case (PkgbuildToken.Comma, _) if prefix is not None and PkgbuildToken.FunctionEnds in second:
                    part, suffix = second.rsplit(PkgbuildToken.FunctionEnds, maxsplit=1)
                    buffer.append(f"{prefix}{part}")
                    result.extend([f"{part}{suffix}" for part in buffer])
                    # reset state
                    buffer, prefix = [], None
                # we have already got prefix string, so we are in progress of expansion
                # we always operate the last element, so this matches ",", "next"
                case (PkgbuildToken.Comma, _) if prefix is not None:
                    buffer.append(f"{prefix}{second}")
                # exactly first element of the list
                case (_, _) if prefix is None and index == 0:
                    result.append(first)
                # any next normal element
                case (_, _) if prefix is None:
                    result.append(second)
        # small sanity check
        if prefix is not None:
            raise PkgbuildParserError("error in array expansion", array)
        return result
    def _is_escaped(self) -> bool:
        """
-        check if the last element was quoted. ``shlex.shlex`` parser doesn't provide information about was the token
+        read the original source text represented by a syntax tree node
        quoted or not, thus there is no difference between "'#'" (sharp in quotes) and "#" (sharp without quotes). This
        method simply rolls back to the last non-space character and check if it is a quotation mark
        Returns:
            bool: ``True`` if the previous element of the stream is a quote or escaped and ``False`` otherwise
        """
        current_position = self._io.tell()
        last_char = penultimate_char = None
        index = current_position - 1
        while index > 0:
            index, last_char = self._read_last(index)
            if last_char.isspace():
                index -= 1
                continue
            if index > 1:
                _, penultimate_char = self._read_last(index - 1)
            break
        self._io.seek(current_position)  # reset position of the stream
        is_quoted = last_char is not None and last_char in self.quotes
        is_escaped = penultimate_char is not None and penultimate_char in self.escape
        return is_quoted or is_escaped
    def _parse_array(self) -> list[str]:
        """
        parse array from the PKGBUILD. This method will extract tokens from parser until it matches closing array,
        modifying source parser state
        Returns:
            list[str]: extracted arrays elements
        Raises:
            PkgbuildParserError: if array is not closed
        """
        def extract() -> Iterator[str]:
            while token := self.get_token():
                match token:
                    case _ if self._is_escaped():
                        pass
                    case PkgbuildToken.ArrayEnds:
                        break
                    case comment if comment.startswith(PkgbuildToken.Comment):
                        self._read_comment()
                        continue
                yield token
            if token != PkgbuildToken.ArrayEnds:
                raise PkgbuildParserError("no closing array bracket found")
        return self._expand_array(list(extract()))
    def _parse_function(self) -> str:
        """
        parse function from the PKGBUILD. This method will extract tokens from parser until it matches closing function,
        modifying source parser state. Instead of trying to combine tokens together, it uses positions of the file
        and reads content again in this range
        Returns:
            str: function body
        Raises:
            PkgbuildParserError: if function body wasn't found or parser input stream doesn't support position reading
        """
        # find start and end positions
        start_position = end_position = -1
        counter = 0  # simple processing of the inner "{" and "}"
        for token in self:
            match token:
                case _ if self._is_escaped():
                    continue
                case PkgbuildToken.FunctionStarts:
                    if counter == 0:
                        start_position = self._io.tell() - 1
                    counter += 1
                case PkgbuildToken.FunctionEnds:
                    end_position = self._io.tell()
                    if self.state != self.eof:  # type: ignore[attr-defined]
                        end_position -= 1  # if we are not at the end of the file, position is _after_ the token
                    counter -= 1
                    if counter == 0:
                        break
                case comment if comment.startswith(PkgbuildToken.Comment):
                    self._read_comment()
        if not 0 < start_position < end_position:
            raise PkgbuildParserError("function body wasn't found")
        # read the specified interval from source stream
        self._io.seek(start_position - 1)  # start from the previous symbol
        # we cannot use :func:`read()` here, because it reads characters, not bytes
        content = ""
        while self._io.tell() != end_position and (next_char := self._io.read(1)):
            content += next_char
        # special case of the end of file
        if self.state == self.eof:  # type: ignore[attr-defined]
            content += self._io.read(1)
        return content
    def _parse_token(self, token: str) -> Iterator[PkgbuildPatch]:
        """
        parse single token to the PKGBUILD field
        Args:
-            token(str): current token
+            node(Node): syntax tree node to read
        Returns:
            str: source text for the node
        """
        return cast(bytes, node.text).decode("utf8", errors="backslashreplace")
    def _assignment(self, node: Node) -> PkgbuildPatch:
        """
        convert an assignment node
        Args:
            node(Node): assignment node to convert
        Returns:
            PkgbuildPatch: parsed assignment
        """
        name = cast(Node, node.child_by_field_name(PkgbuildNodeField.Name))
        value = cast(Node, node.child_by_field_name(PkgbuildNodeField.Value))
        key = self._text(name)
        match value.type:
            case PkgbuildNodeType.Array:
                array_values = []
                for element in filter(lambda child: child.type != PkgbuildNodeType.Comment, value.named_children):
                    item = self._value(element)
                    values = self._expand_array(item, expand=element.type == PkgbuildNodeType.Concatenation)
                    array_values.extend(values)
                return PkgbuildPatch(key, array_values)
            case _:
                return PkgbuildPatch(key, self._value(value))
    def _read(self, node: Node) -> Iterator[PkgbuildPatch]:
        """
        find assignments and functions in a syntax tree node
        Args:
            node(Node): syntax tree node to inspect
        Yields:
-            PkgbuildPatch: extracted a PKGBUILD node
+            PkgbuildPatch: parsed entries
        """
-        # simple assignment rule
+        match node.type:
-        if m := self._STRING_ASSIGNMENT.match(token):
+            case PkgbuildNodeType.VariableAssignment if not any(child.type == PkgbuildNodeType.Append for child in node.children):
-            key = m.group("key")
+                yield self._assignment(node)
            value = m.group("value")
            yield PkgbuildPatch(key, value)
            return
-        if token.startswith(PkgbuildToken.Comment):
+            case PkgbuildNodeType.FunctionDefinition:
-            self._read_comment()
+                name = cast(Node, node.child_by_field_name(PkgbuildNodeField.Name))
-            return
+                body = cast(Node, node.child_by_field_name(PkgbuildNodeField.Body))
                yield PkgbuildPatch(f"{self._text(name)}()", self._text(body))
-        match self.get_token():
+            case _:
-            # array processing. Arrays will be sent as "key=", "(", values, ")"
+                # descend into statements, but skip command arguments and values
-            case PkgbuildToken.ArrayStarts if m := self._ARRAY_ASSIGNMENT.match(token):
+                for child in node.named_children:
-                key = m.group("key")
+                    if child.type not in (
-                value = self._parse_array()
+                            PkgbuildNodeType.Array,
-                yield PkgbuildPatch(key, value)
+                            PkgbuildNodeType.Comment,
                            PkgbuildNodeType.RawString,
                            PkgbuildNodeType.String,
                            PkgbuildNodeType.Word,
                    ):
                        yield from self._read(child)
-            # functions processing. Function will be sent as "name", "()", "{", body, "}"
+    def _value(self, node: Node) -> str:
            case PkgbuildToken.FunctionDeclaration if self._FUNCTION_DECLARATION.match(token):
                key = f"{token}{PkgbuildToken.FunctionDeclaration}"
                value = self._parse_function()
                yield PkgbuildPatch(key, value)  # this is not mistake, assign to token without ()
            # special function case, where "(" and ")" are separated tokens, e.g. "pkgver ( )"
            case PkgbuildToken.ArrayStarts if self._FUNCTION_DECLARATION.match(token):
                next_token = self.get_token()
                if next_token == PkgbuildToken.ArrayEnds:  # replace closing bracket with "()"
                    next_token = PkgbuildToken.FunctionDeclaration
                self.push_token(next_token)  # type: ignore[arg-type]
                yield from self._parse_token(token)
            # some random token received without continuation, lets guess it is empty assignment (i.e. key=)
            case other if other is not None:
                yield from self._parse_token(other)
    def _read_comment(self) -> None:
        """
-        read comment from the current position. This method doesn't check comment itself, just read the stream
+        extract a string value from a syntax tree node
        until the comment line ends
        """
        _, last_symbol = self._read_last()
        if last_symbol != PkgbuildToken.NewLine:
            self.instream.readline()
    def _read_last(self, initial_index: int | None = None) -> tuple[int, str]:
        """
        wrapper around read to read the last symbol from the input stream. This method is designed to process UTF-8
        symbols correctly. This method does not reset current stream position
        Args:
-            initial_index(int | None, optional): initial index to start reading from. If none set, the previous position
+            node(Node): syntax tree node to extract
                will be used (Default value = None)
        Returns:
-            tuple[int, str]: last symbol and its position in the stream
+            str: extracted string value
        Raises:
            PkgbuildParserError: in case if stream reached starting position, but no valid symbols were found
        """
-        if initial_index is None:
+        if node.type in (PkgbuildNodeType.Concatenation, PkgbuildNodeType.String):
-            initial_index = self._io.tell() - 1
+            return "".join(self._value(child) for child in node.named_children)
        if initial_index < 0:
            raise PkgbuildParserError("stream is on starting position")
        self._io.seek(initial_index)
-        while (position := self._io.tell()) > 0:
+        value = self._text(node)
-            try:
+        match node.type:
-                return position, self._io.read(1)
+            case PkgbuildNodeType.StringContent:
-            except UnicodeDecodeError:
+                value = f'"{value}"'
-                self._io.seek(position - 1)
+            case PkgbuildNodeType.Number | PkgbuildNodeType.RawString | PkgbuildNodeType.Word:
                pass
            case _:
                return value
-        raise PkgbuildParserError("reached starting position, no valid symbols found")
+        return "".join(shlex.split(value, comments=False, posix=True))
    def parse(self) -> Iterator[PkgbuildPatch]:
        """
-        parse source stream and yield parsed entries
+        parse source and yield extracted entries
        Yields:
-            PkgbuildPatch: extracted a PKGBUILD node
+            PkgbuildPatch: parsed entries
        Raises:
            PkgbuildParserError: source contains invalid shell syntax
        """
-        for token in self:
+        def walk(node: Node) -> Iterator[Node]:
-            yield from self._parse_token(token)
+            yield node
            for child in node.children:
                yield from walk(child)
        parser = Parser(Language(tree_sitter_bash.language()))
        tree = parser.parse(self._content)
        if tree.root_node.has_error:
            error = next((node for node in walk(tree.root_node) if node.is_error or node.is_missing), tree.root_node)
            raise PkgbuildParserError(
                f"syntax error at line {error.start_point.row + 1}, column {error.start_point.column + 1}"
            )
        yield from self._read(tree.root_node)
@@ -238,16 +238,12 @@ class PkgbuildParserError(ValueError):
    exception raises in case of PKGBUILD parser errors
    """
-    def __init__(self, reason: str, source: Any = None) -> None:
+    def __init__(self, reason: str) -> None:
        """
        Args:
            reason(str): parser error reason
            source(Any, optional): source line if available (Default value = None)
        """
-        message = f"Could not parse PKGBUILD: {reason}"
+        ValueError.__init__(self, f"Could not parse PKGBUILD: {reason}")
        if source is not None:
            message += f", source: `{source}`"
        ValueError.__init__(self, message)
 class PasswordError(ValueError):
@@ -19,12 +19,12 @@
 #
 from collections.abc import Iterator, Mapping
 from dataclasses import dataclass
-from io import StringIO
+from io import BytesIO
 from pathlib import Path
-from typing import Any, ClassVar, IO, Self
+from typing import Any, IO, Self
 from urllib.parse import urlparse
-from ahriman.core.alpm.pkgbuild_parser import PkgbuildParser, PkgbuildToken
+from ahriman.core.alpm.pkgbuild_parser import PkgbuildParser
 from ahriman.core.utils import srcinfo_property_list
 from ahriman.models.pkgbuild_patch import PkgbuildPatch
@@ -35,14 +35,11 @@ class Pkgbuild(Mapping[str, Any]):
    model and proxy for PKGBUILD properties
    Attributes:
        DEFAULT_ENCODINGS(str): (class attribute) default encoding to be applied on the file content
        fields(dict[str, PkgbuildPatch]): PKGBUILD fields
    """
    fields: dict[str, PkgbuildPatch]
    DEFAULT_ENCODINGS: ClassVar[str] = "utf8"
    @property
    def variables(self) -> dict[str, str]:
        """
@@ -59,37 +56,26 @@ class Pkgbuild(Mapping[str, Any]):
        }
    @classmethod
-    def from_file(cls, path: Path, encoding: str | None = None) -> Self:
+    def from_file(cls, path: Path) -> Self:
        """
        parse PKGBUILD from the file
        Args:
            path(Path): path to the PKGBUILD file
            encoding(str | None, optional): the encoding of the file (Default value = None)
        Returns:
            Self: constructed instance of self
        Raises:
            EncodeError: if encoding is unknown
        """
        # read raw bytes from file
        with path.open("rb") as input_file:
-            content = input_file.read()
+            return cls.from_io(input_file)
        # decode bytes content based on either
        encoding = encoding or cls.DEFAULT_ENCODINGS
        io = StringIO(content.decode(encoding, errors="backslashreplace"))
        return cls.from_io(io)
    @classmethod
-    def from_io(cls, stream: IO[str]) -> Self:
+    def from_io(cls, stream: IO[bytes]) -> Self:
        """
        parse PKGBUILD from input stream
        Args:
-            stream(IO[str]): input stream containing PKGBUILD content
+            stream(IO[bytes]): input stream containing PKGBUILD content
        Returns:
            Self: constructed instance of self
@@ -162,10 +148,10 @@ class Pkgbuild(Mapping[str, Any]):
        """
        packages = [self["pkgname"]] if isinstance(self["pkgname"], str) else self["pkgname"]
-        def io(package_name: str) -> IO[str]:
+        def io(package_name: str) -> IO[bytes]:
            # try to read package specific function and fallback to default otherwise
            content = self.get(f"package_{package_name}") or self.get("package") or ""
-            return StringIO(content)
+            return BytesIO(content.encode("utf8"))
        return {package: self.from_io(io(package)) for package in packages}
@@ -185,8 +171,8 @@ class Pkgbuild(Mapping[str, Any]):
        """
        value = self.fields.get(item)
        # if the key wasn't found and user didn't ask for function explicitly, we can try to get by function name
-        if value is None and not item.endswith(PkgbuildToken.FunctionDeclaration):
+        if value is None and not item.endswith("()"):
-            value = self.fields.get(f"{item}{PkgbuildToken.FunctionDeclaration}")
+            value = self.fields.get(f"{item}()")
        # if we still didn't find anything, we can just raise the exception
        if value is None:
@@ -1,6 +1,6 @@
 import pytest
-from io import StringIO
+from io import BytesIO
 from pathlib import Path
 from ahriman.core.alpm.pkgbuild_parser import PkgbuildParser
@@ -12,60 +12,46 @@ def test_expand_array() -> None:
    """
    must correctly expand array
    """
-    assert PkgbuildParser._expand_array(["${pkgbase}{", ",", "-libs", ",", "-fortran}"]) == [
+    parser = PkgbuildParser(BytesIO(b"array=(prefix{only}suffix)"))
-        "${pkgbase}", "${pkgbase}-libs", "${pkgbase}-fortran"
+    assert list(parser.parse()) == [PkgbuildPatch("array", ["prefix{only}suffix"])]
    ]
    assert PkgbuildParser._expand_array(["first", "prefix{1", ",", "2", ",", "3}suffix", "last"]) == [
        "first", "prefix1suffix", "prefix2suffix", "prefix3suffix", "last"
    ]
    parser = PkgbuildParser(BytesIO(b"array=(prefix{first,second}suffix)"))
    assert list(parser.parse()) == [PkgbuildPatch("array", ["prefixfirstsuffix", "prefixsecondsuffix"])]
-def test_expand_array_no_comma() -> None:
+    parser = PkgbuildParser(BytesIO(b"""array=("prefix{first,second}suffix")"""))
-    """
+    assert list(parser.parse()) == [PkgbuildPatch("array", ["prefix{first,second}suffix"])]
    must skip array extraction if there is no comma
    """
    assert PkgbuildParser._expand_array(["${pkgbase}{", "-libs", "-fortran}"]) == ["${pkgbase}{", "-libs", "-fortran}"]
 def test_expand_array_short() -> None:
    """
    must skip array extraction if it is short
    """
    assert PkgbuildParser._expand_array(["${pkgbase}{", ","]) == ["${pkgbase}{", ","]
 def test_expand_array_exception() -> None:
    """
    must raise exception if there is unclosed element
    """
    with pytest.raises(PkgbuildParserError):
        assert PkgbuildParser._expand_array(["${pkgbase}{", ",", "-libs"])
 def test_is_escaped_exception(resource_path_root: Path) -> None:
    """
    must raise PkgbuildParserError if no valid utf symbols found
    """
    utf8 = resource_path_root / "models" / "utf8"
    with utf8.open(encoding="utf8") as content:
        content.seek(2)
        with pytest.raises(PkgbuildParserError):
            assert not PkgbuildParser(content)._is_escaped()
 def test_parse_array() -> None:
    """
    must parse array
    """
-    parser = PkgbuildParser(StringIO("var=(first second)"))
+    parser = PkgbuildParser(BytesIO(b"var=(first second)"))
    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", "second"])]
 def test_parse_array_escaped() -> None:
    """
    must correctly process quoted brackets
    """
    parser = PkgbuildParser(BytesIO(b"""var=(first "(" second)"""))
    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", "(", "second"])]
    parser = PkgbuildParser(BytesIO(b"""var=(first ")" second)"""))
    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
    parser = PkgbuildParser(BytesIO(b"""var=(first ')' second)"""))
    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
    parser = PkgbuildParser(BytesIO(b"""var=(first \\) second)"""))
    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
 def test_parse_array_comment() -> None:
    """
    must parse array with comments inside
    """
-    parser = PkgbuildParser(StringIO("""validpgpkeys=(
+    parser = PkgbuildParser(BytesIO(b"""validpgpkeys=(
  'F3691687D867B81B51CE07D9BBE43771487328A9'  # bpiotrowski@archlinux.org
  '86CFFCA918CF3AF47147588051E8B148A9999C34'  # evangelos@foutrelis.com
  '13975A70E63C361C73AE69EF6EEB81F8981C74C7'  # richard.guenther@gmail.com
@@ -79,28 +65,11 @@ def test_parse_array_comment() -> None:
    ])]
 def test_parse_array_escaped() -> None:
    """
    must correctly process quoted brackets
    """
    parser = PkgbuildParser(StringIO("""var=(first "(" second)"""))
    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", "(", "second"])]
    parser = PkgbuildParser(StringIO("""var=(first ")" second)"""))
    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
    parser = PkgbuildParser(StringIO("""var=(first ')' second)"""))
    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
    parser = PkgbuildParser(StringIO("""var=(first \\) second)"""))
    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
 def test_parse_array_exception() -> None:
    """
    must raise exception if there is no closing bracket
    """
-    parser = PkgbuildParser(StringIO("var=(first second"))
+    parser = PkgbuildParser(BytesIO(b"var=(first second"))
    with pytest.raises(PkgbuildParserError):
        assert list(parser.parse())
@@ -109,86 +78,75 @@ def test_parse_function() -> None:
    """
    must parse function
    """
-    parser = PkgbuildParser(StringIO("var() { echo hello world } "))
+    parser = PkgbuildParser(BytesIO(b"var() { echo hello world; } "))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")]
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world; }")]
 def test_parse_function_eof() -> None:
    """
    must parse function with "}" at the end of the file
    """
-    parser = PkgbuildParser(StringIO("var() { echo hello world }"))
+    parser = PkgbuildParser(BytesIO(b"var() { echo hello world; }"))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")]
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world; }")]
 def test_parse_function_spaces() -> None:
    """
    must parse function with spaces in declaration
    """
    parser = PkgbuildParser(StringIO("var ( ) { echo hello world } "))
    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")]
 def test_parse_function_inner_shell() -> None:
    """
    must parse function with inner shell
    """
    parser = PkgbuildParser(StringIO("var ( ) { { echo hello world } } "))
    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ { echo hello world } }")]
 def test_parse_function_escaped() -> None:
    """
    must parse function with bracket in quotes
    """
-    parser = PkgbuildParser(StringIO("""var ( ) { echo "hello world {" } """))
+    parser = PkgbuildParser(BytesIO(b"""var() { echo "hello world {"; } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo "hello world {" }""")]
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo "hello world {"; }""")]
-    parser = PkgbuildParser(StringIO("""var ( ) { echo hello world "{" } """))
+    parser = PkgbuildParser(BytesIO(b"""var() { echo hello world "{"; } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world "{" }""")]
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world "{"; }""")]
-    parser = PkgbuildParser(StringIO("""var ( ) { echo "hello world }" } """))
+    parser = PkgbuildParser(BytesIO(b"""var() { echo "hello world }"; } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo "hello world }" }""")]
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo "hello world }"; }""")]
-    parser = PkgbuildParser(StringIO("""var ( ) { echo hello world "}" } """))
+    parser = PkgbuildParser(BytesIO(b"""var() { echo hello world "}"; } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world "}" }""")]
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world "}"; }""")]
-    parser = PkgbuildParser(StringIO("""var ( ) { echo hello world '}' } """))
+    parser = PkgbuildParser(BytesIO(b"""var() { echo hello world '}'; } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world '}' }""")]
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world '}'; }""")]
-    parser = PkgbuildParser(StringIO("""var ( ) { echo hello world \\} } """))
+    parser = PkgbuildParser(BytesIO(b"""var() { echo hello world \\}; } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world \\} }""")]
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world \\}; }""")]
 def test_parse_function_exception() -> None:
    """
    must raise exception if no bracket found
    """
-    parser = PkgbuildParser(StringIO("var() echo hello world } "))
+    parser = PkgbuildParser(BytesIO(b"var() echo hello world } "))
    with pytest.raises(PkgbuildParserError):
        assert list(parser.parse())
-    parser = PkgbuildParser(StringIO("var() { echo hello world"))
+    parser = PkgbuildParser(BytesIO(b"var() { echo hello world"))
    with pytest.raises(PkgbuildParserError):
        assert list(parser.parse())
-def test_parse_token_assignment() -> None:
+def test_parse_function_inner_shell() -> None:
    """
-    must parse simple assignment
+    must parse function with inner shell
    """
-    parser = PkgbuildParser(StringIO())
+    parser = PkgbuildParser(BytesIO(b"var() { { echo hello world; }; } "))
-    assert next(parser._parse_token("var=value")) == PkgbuildPatch("var", "value")
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ { echo hello world; }; }")]
-    assert next(parser._parse_token("var=$value")) == PkgbuildPatch("var", "$value")
+
-    assert next(parser._parse_token("var=${value}")) == PkgbuildPatch("var", "${value}")
+
-    assert next(parser._parse_token("var=${value/-/_}")) == PkgbuildPatch("var", "${value/-/_}")
+def test_parse_function_spaces() -> None:
    """
    must parse function with spaces before body
    """
    parser = PkgbuildParser(BytesIO(b"var()   { echo hello world; } "))
    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world; }")]
 def test_parse_token_comment() -> None:
    """
    must correctly parse comment
    """
-    parser = PkgbuildParser(StringIO("""first=1 # comment
+    parser = PkgbuildParser(BytesIO(b"""first=1 # comment
    # comment line
    second=2
    #third=3
@@ -199,58 +157,12 @@ def test_parse_token_comment() -> None:
    ]
 def test_read_comment() -> None:
    """
    must read comment correctly
    """
    io = StringIO("# comment\nnew line")
    io.seek(2)
    PkgbuildParser(io)._read_comment()
    assert io.tell() == 10
 def test_read_comment_skip() -> None:
    """
    must skip reading new line if comment ends with new line
    """
    io = StringIO("#comment\nnew line")
    io.seek(7)
    PkgbuildParser(io)._read_comment()
    assert io.tell() == 9
 def test_read_last() -> None:
    """
    must read last symbol from current position
    """
    io = StringIO("mock")
    io.seek(2)
    assert PkgbuildParser(io)._read_last() == (1, "o")
 def test_read_last_starting() -> None:
    """
    must raise exception if it reads from starting position
    """
    with pytest.raises(PkgbuildParserError):
        assert PkgbuildParser(StringIO("mock"))._read_last()
 def test_read_last_from_position() -> None:
    """
    must read last symbol from the specified position
    """
    assert PkgbuildParser(StringIO("mock"))._read_last(2) == (2, "c")
 def test_parse(resource_path_root: Path) -> None:
    """
    must parse complex file
    """
    pkgbuild = resource_path_root / "models" / "pkgbuild"
-    with pkgbuild.open(encoding="utf8") as content:
+    with pkgbuild.open("rb") as content:
        parser = PkgbuildParser(content)
        assert list(parser.parse()) == [
            PkgbuildPatch("var", "value"),
@@ -277,53 +189,58 @@ def test_parse(resource_path_root: Path) -> None:
            PkgbuildPatch("array", ["first", ")", "second"]),
            PkgbuildPatch("array", ["first", "(", "second"]),
            PkgbuildPatch("array", ["first", ")", "second"]),
-            PkgbuildPatch("function()", """{ single line }"""),
+            PkgbuildPatch("func()", """{ single line; }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
    multi
    line
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
    c
    multi
    line
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
    # comment
    multi
    line
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
    body
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
    body
 }"""),
-            PkgbuildPatch("function_with-package-name()", """{ body }"""),
+            PkgbuildPatch("function_with-package-name()", """{ body; }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
    first
-    { inner shell }
+    { inner shell; }
    last
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
    body "{" argument
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
    body "}" argument
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
    body '{' argument
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
    body '}' argument
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
    # we don't care about unclosed quotation in comments
    body # no, I said we really don't care
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
  mv "$pkgdir"/usr/share/fonts/站酷小薇体 "$pkgdir"/usr/share/fonts/zcool-xiaowei-regular
  mv "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.站酷小薇体 "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.zcool-xiaowei-regular
 }"""),
            PkgbuildPatch("var", "value"),
            PkgbuildPatch("array", ["first", "second", "third"]),
            PkgbuildPatch("pkgver()", """{
  cd aften
  _ver="$(cat CMakeLists.txt | grep -e SO_MAJOR_VERSION -e SO_MINOR_VERSION -e SO_BUILD_VERSION | cut -d '"' -f2 | head -n3)"
  echo "$(echo ${_ver} | tr ' ' .).r$(git rev-list --count HEAD).$(git rev-parse --short HEAD)"
 }"""),
        ]
@@ -1,6 +1,6 @@
 import pytest
-from io import BytesIO, StringIO
+from io import BytesIO
 from pathlib import Path
 from pytest_mock import MockerFixture
@@ -33,10 +33,11 @@ def test_from_file(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
 def test_from_file_latin(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
    """
-    must correctly load from file with latin encoding
+    must correctly load bytes containing latin encoding
    """
    open_mock = mocker.patch("pathlib.Path.open")
    open_mock.return_value.__enter__.return_value = BytesIO("contént".encode("latin-1"))
    mocker.patch("ahriman.models.pkgbuild.Pkgbuild.from_io", return_value=pkgbuild_ahriman)
    load_mock = mocker.patch("ahriman.models.pkgbuild.Pkgbuild.from_io", return_value=pkgbuild_ahriman)
    assert Pkgbuild.from_file(Path("local"))
@@ -50,7 +51,7 @@ def test_from_io(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
    """
    load_mock = mocker.patch("ahriman.core.alpm.pkgbuild_parser.PkgbuildParser.parse",
                             return_value=pkgbuild_ahriman.fields.values())
-    assert Pkgbuild.from_io(StringIO("mock")) == pkgbuild_ahriman
+    assert Pkgbuild.from_io(BytesIO(b"mock")) == pkgbuild_ahriman
    load_mock.assert_called_once_with()
@@ -64,9 +65,9 @@ def test_from_io_pkgbase(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> N
        [value for key, value in pkgbuild_ahriman.fields.items()] + [PkgbuildPatch("pkgbase", "pkgbase")],
    ])
-    assert Pkgbuild.from_io(StringIO("mock"))["pkgbase"] == pkgbuild_ahriman["pkgname"]
+    assert Pkgbuild.from_io(BytesIO(b"mock"))["pkgbase"] == pkgbuild_ahriman["pkgname"]
-    assert "pkgbase" not in Pkgbuild.from_io(StringIO("mock"))
+    assert "pkgbase" not in Pkgbuild.from_io(BytesIO(b"mock"))
-    assert Pkgbuild.from_io(StringIO("mock"))["pkgbase"] == "pkgbase"
+    assert Pkgbuild.from_io(BytesIO(b"mock"))["pkgbase"] == "pkgbase"
 def test_from_io_empty(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
@@ -75,7 +76,7 @@ def test_from_io_empty(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> Non
    """
    mocker.patch("ahriman.core.alpm.pkgbuild_parser.PkgbuildParser.parse",
                 return_value=list(pkgbuild_ahriman.fields.values()) + [PkgbuildPatch("", "")])
-    assert Pkgbuild.from_io(StringIO("mock")) == pkgbuild_ahriman
+    assert Pkgbuild.from_io(BytesIO(b"mock")) == pkgbuild_ahriman
 def test_local_files(mocker: MockerFixture, resource_path_root: Path) -> None:
@@ -567,7 +568,6 @@ def test_parse_pacman_static(resource_path_root: Path) -> None:
    """
    pkgbuild = Pkgbuild.from_file(resource_path_root / "models" / "package_pacman-static_pkgbuild")
    values = {key: value.value for key, value in pkgbuild.fields.items() if not value.is_function}
    print(values)
    assert values == {
        "pkgbase": "pacman-static",
        "pkgname": "pacman-static",
@@ -8,7 +8,7 @@ pkgdesc="ArcH linux ReposItory MANager"
 arch=('any')
 url="https://ahriman.readthedocs.io/"
 license=('GPL-3.0-or-later')
-depends=('devtools>=1:1.0.0' 'git' 'pyalpm' 'python-bcrypt' 'python-filelock' 'python-inflection' 'python-pyelftools' 'python-requests')
+depends=('devtools>=1:1.0.0' 'git' 'pyalpm' 'python-bcrypt' 'python-filelock' 'python-inflection' 'python-pyelftools' 'python-requests' 'python-tree-sitter' 'python-tree-sitter-bash')
 makedepends=('npm' 'python-build' 'python-hatchling' 'python-installer' 'python-wheel')
 source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgbase-$pkgver.tar.gz"
        "$pkgbase.sysusers"
@@ -426,13 +426,13 @@ There are several supported synchronization providers, currently they are ``rsyn
 PKGBUILD parsing
 ^^^^^^^^^^^^^^^^
-The application provides a house-made shell parser ``ahriman.core.alpm.pkgbuild_parser.PkgbuildParser`` to process PKGBUILDs and extract package data from them. It relies on the ``shlex.shlex`` parser with some configuration tweaks and adds some token post-processing.
+The application uses the tree-sitter bash grammar in ``ahriman.core.alpm.pkgbuild_parser.PkgbuildParser`` to process
 PKGBUILDs without executing them. It extracts assignment and function nodes from the syntax tree and preserves function
 bodies from the original source.
-#. During the parser process, firstly, it extracts the next token from the source file (basically, the word) and tries to match it to the variable assignment. If so, then just processes accordingly.
+#. Assignments, arrays, and functions are read from the syntax tree. Quote and escape handling follows bash syntax.
-#. If it is not an assignment, the parser checks if the token was quoted.
+#. Function bodies are sliced from the source so their original formatting is preserved.
-#. If it wasn't quoted then the parser tries to match the array starts (two consecutive tokens like ``array=`` and ``(``) or it is function (``function``, ``()`` and ``{``).
+#. Bash syntax errors raise ``PkgbuildParserError`` with the source location.
 #. The arrays are processed until the next closing bracket ``)``. After extraction, the parser tries to expand an array according to bash rules (``prefix{first,second}suffix`` constructions).
 #. The functions are just read until the closing bracket ``}`` and then reread whole text from the input string without a tokenization.
 All extracted fields are packed as ``ahriman.models.pkgbuild_patch.PkgbuildPatch`` and then can be used as ``ahriman.models.pkgbuild.Pkgbuild`` instance.
@@ -138,7 +138,7 @@ prepare() {
    # apply patch level commits on top of annotated tag for pacman
    if [[ -n ${_git_patch_level_commit} ]]; then
-        if [[ v${_git_tag} != $(git describe --tags --abbrev=0 "${_git_patch_level_commit}") ]] then
+        if [[ v${_git_tag} != $(git describe --tags --abbrev=0 "${_git_patch_level_commit}") ]]; then
            error "patch level commit ${_git_patch_level_commit} is not a descendant of v${_git_tag}"
            exit 1
        fi
@@ -41,55 +41,55 @@ array=(first '(' second)
 array=(first ')' second)
 # functions
-function() { single line }
+func() { single line; }
-function() {
+func() {
    multi
    line
 }
-function()
+func()
 {
    c
    multi
    line
 }
-function() {
+func() {
    # comment
    multi
    line
 }
-function () {
+func() {
    body
 }
-function ( ){
+func() {
    body
 }
-function_with-package-name() { body }
+function_with-package-name() { body; }
-function() {
+func() {
    first
-    { inner shell }
+    { inner shell; }
    last
 }
-function() {
+func() {
    body "{" argument
 }
-function() {
+func() {
    body "}" argument
 }
-function() {
+func() {
    body '{' argument
 }
-function() {
+func() {
    body '}' argument
 }
 # special case with quotes in comments
-function() {
+func() {
    # we don't care about unclosed quotation in comments
    body # no, I said we really don't care
 }
 # some random unicode symbols
-function() {
+func() {
  mv "$pkgdir"/usr/share/fonts/站酷小薇体 "$pkgdir"/usr/share/fonts/zcool-xiaowei-regular
  mv "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.站酷小薇体 "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.zcool-xiaowei-regular
 }
@@ -107,3 +107,9 @@ array=(
    second #comment-without-whitespace
    third
 )
 pkgver() {
  cd aften
  _ver="$(cat CMakeLists.txt | grep -e SO_MAJOR_VERSION -e SO_MINOR_VERSION -e SO_BUILD_VERSION | cut -d '"' -f2 | head -n3)"
  echo "$(echo ${_ver} | tr ' ' .).r$(git rev-list --count HEAD).$(git rev-parse --short HEAD)"
 }