draft of migration to tree-sitter

ahriman-core/pyproject.toml

@@ -12,6 +12,8 @@ dependencies = [
     "inflection",
     "pyelftools",
     "requests",
+    "tree-sitter",
+    "tree-sitter-bash",
 ]
 description = "ArcH linux ReposItory MANager, core package"
 dynamic = [

ahriman-core/src/ahriman/core/alpm/pkgbuild_parser.py

@@ -17,355 +17,242 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-import itertools
 import re
 import shlex
+import tree_sitter_bash
 
 from collections.abc import Iterator
 from enum import StrEnum
-from typing import IO
+from tree_sitter import Language, Node, Parser
+from typing import IO, cast
 
 from ahriman.core.exceptions import PkgbuildParserError
 from ahriman.models.pkgbuild_patch import PkgbuildPatch
 
 
-class PkgbuildToken(StrEnum):
+class PkgbuildNodeType(StrEnum):
     """
-    well-known tokens dictionary
+    tree-sitter node types used by parser
 
     Attributes:
-        ArrayEnds(PkgbuildToken): array ends token
-        ArrayStarts(PkgbuildToken): array starts token
-        Comma(PkgbuildToken): comma token
-        Comment(PkgbuildToken): comment token
-        FunctionDeclaration(PkgbuildToken): function declaration token
-        FunctionEnds(PkgbuildToken): function ends token
-        FunctionStarts(PkgbuildToken): function starts token
-        NewLine(PkgbuildToken): new line token
+        Append(PkgbuildNodeType): append node
+        Array(PkgbuildNodeType): array node
+        Comment(PkgbuildNodeType): comment node
+        Concatenation(PkgbuildNodeType): concatenation node
+        Error(PkgbuildNodeType): syntax error node
+        FunctionDefinition(PkgbuildNodeType): function definition node
+        Number(PkgbuildNodeType): numeric word node
+        RawString(PkgbuildNodeType): single-quoted string node
+        String(PkgbuildNodeType): double-quoted string node
+        StringContent(PkgbuildNodeType): string content node
+        VariableAssignment(PkgbuildNodeType): assignment node
+        Word(PkgbuildNodeType): word node
     """
 
-    ArrayStarts = "("
-    ArrayEnds = ")"
-
-    Comma = ","
-
-    Comment = "#"
-
-    FunctionDeclaration = "()"
-    FunctionStarts = "{"
-    FunctionEnds = "}"
-
-    NewLine = "\n"
+    Append = "+="
+    Array = "array"
+    Comment = "comment"
+    Concatenation = "concatenation"
+    Error = "ERROR"
+    FunctionDefinition = "function_definition"
+    Number = "number"
+    RawString = "raw_string"
+    String = "string"
+    StringContent = "string_content"
+    VariableAssignment = "variable_assignment"
+    Word = "word"
 
 
-class PkgbuildParser(shlex.shlex):
+class PkgbuildNodeField(StrEnum):
     """
-    simple pkgbuild reader implementation in pure python, because others suck.
+    tree-sitter node fields used by parser
 
-    What is it:
+    Attributes:
+        Body(PkgbuildNodeField): function body field
+        Name(PkgbuildNodeField): variable or function name field
+        Value(PkgbuildNodeField): assignment value field
+    """
 
-    #. Simple PKGBUILD parser written in python.
-    #. No shell execution, so it is free from random shell attacks.
-    #. Able to parse simple constructions (assignments, comments, functions, arrays).
+    Body = "body"
+    Name = "name"
+    Value = "value"
+
+
+class PkgbuildParser:
+    """
+    PKGBUILD parser implementation in python
+
+    What it is:
+
+    #. A PKGBUILD parser based on tree-sitter's bash grammar.
+    #. A parser for assignments, arrays, and functions.
+    #. A parser which does not execute shell code.
 
     What it is not:
 
-    #. Fully functional shell parser.
-    #. Shell executor.
-    #. No parameter expansion.
+    #. A shell executor.
+    #. A shell interpreter. Variable and command substitutions are returned as written.
 
-    For more details what does it support, please, consult with the test cases.
+    Simple comma-separated brace expressions in array elements are expanded. Function bodies are returned from the
+    original source, with their formatting preserved. Assignments inside shell statements and control-flow nodes are
+    extracted without checking whether those statements would run.
 
     Examples:
-        This class is heavily based on :mod:`shlex` parser, but instead of strings operates with the
-        :class:`ahriman.models.pkgbuild_patch.PkgbuildPatch` objects. The main way to use it is to call :func:`parse()`
-        function and collect parsed objects, e.g.::
+        Parse package assignments::
 
-            >>> parser = PkgbuildParser(StringIO("input string"))
-            >>> for patch in parser.parse():
-            >>>     print(f"{patch.key} = {patch.value}")
+            >>> source = b"pkgname=example\\npkgver=1\\n"
+            >>> for patch in PkgbuildParser(BytesIO(source)).parse():
+            ...     print(patch.key, patch.value)
 
-        It doesn't store the state of the fields (but operates with the :mod:`shlex` parser state), so no shell
-        post-processing is performed (e.g. variable substitution).
+        Function bodies are returned as function patches::
+
+            >>> source = b"pkgver() { printf 1; }"
+            >>> for patch in list(PkgbuildParser(BytesIO(source)).parse())
+            ...     print(patch.key, patch.value)
     """
 
-    _ARRAY_ASSIGNMENT = re.compile(r"^(?P<key>\w+)=$")
-    # in addition to usual assignment, functions can have dash
-    _FUNCTION_DECLARATION = re.compile(r"^(?P<key>[\w-]+)$")
-    _STRING_ASSIGNMENT = re.compile(r"^(?P<key>\w+)=(?P<value>.+)$")
+    _BRACE_EXPANSION = re.compile(r"(.*)\{([^{}]+)\}(.*)")
 
-    def __init__(self, stream: IO[str]) -> None:
-        """
-        Args:
-            stream(IO[str]): input stream containing PKGBUILD content
-        """
-        shlex.shlex.__init__(self, stream, posix=True, punctuation_chars=True)
-        self._io = stream  # direct access without type casting
-
-        # ignore substitution and extend bash symbols
-        self.wordchars += "${}#:+-@!"
-        # in case of default behavior, it will ignore, for example, segment part of url outside of quotes
-        self.commenters = ""
+    def __init__(self, stream: IO[bytes]) -> None:
+        """"""
+        self._content = stream.read()
 
     @staticmethod
-    def _expand_array(array: list[str]) -> list[str]:
+    def _expand_braces(value: str, *, expand: bool = True) -> list[str]:
         """
-        bash array expansion simulator. It takes raw array and tries to expand constructions like
-        ``(first prefix-{mid1,mid2}-suffix last)`` into ``(first, prefix-mid1-suffix prefix-mid2-suffix last)``
+        expand comma-separated brace expressions in a value
 
         Args:
-            array(list[str]): input array
+            value(str): value that may contain a brace expression
+            expand(bool, optional): whether brace expansion is allowed (Default value = True)
 
         Returns:
-            list[str]: either source array or expanded array if possible
-
-        Raises:
-            PkgbuildParserError: if there are errors in parser
+            list[str]: expanded values if applicable
         """
-        # we are using comma as marker for expansion (if any)
-        if PkgbuildToken.Comma not in array:
-            return array
-        # again sanity check, for expansion there are at least 3 elements (first, last and comma)
-        if len(array) < 3:
-            return array
+        match = PkgbuildParser._BRACE_EXPANSION.fullmatch(value)
+        # if there is no comma inside it should be treated as normal value
+        if match is None or not expand or "," not in match.group(2):
+            return [value]
 
-        result = []
-        buffer, prefix = [], None
+        prefix, values, suffix = match.groups()
+        return [f"{prefix}{item}{suffix}" for item in values.split(",")]
 
-        for index, (first, second) in enumerate(itertools.pairwise(array)):
-            match (first, second):
-                # in this case we check if expansion should be started
-                # this condition matches "prefix{first", ","
-                case (_, PkgbuildToken.Comma) if PkgbuildToken.FunctionStarts in first:
-                    prefix, part = first.rsplit(PkgbuildToken.FunctionStarts, maxsplit=1)
-                    buffer.append(f"{prefix}{part}")
-
-                # the last element case, it matches either ",", "last}" or ",", "last}suffix"
-                # in case if there is suffix, it must be appended to all list elements
-                case (PkgbuildToken.Comma, _) if prefix is not None and PkgbuildToken.FunctionEnds in second:
-                    part, suffix = second.rsplit(PkgbuildToken.FunctionEnds, maxsplit=1)
-                    buffer.append(f"{prefix}{part}")
-                    result.extend([f"{part}{suffix}" for part in buffer])
-                    # reset state
-                    buffer, prefix = [], None
-
-                # we have already got prefix string, so we are in progress of expansion
-                # we always operate the last element, so this matches ",", "next"
-                case (PkgbuildToken.Comma, _) if prefix is not None:
-                    buffer.append(f"{prefix}{second}")
-
-                # exactly first element of the list
-                case (_, _) if prefix is None and index == 0:
-                    result.append(first)
-
-                # any next normal element
-                case (_, _) if prefix is None:
-                    result.append(second)
-
-        # small sanity check
-        if prefix is not None:
-            raise PkgbuildParserError("error in array expansion", array)
-
-        return result
-
-    def _is_escaped(self) -> bool:
+    @staticmethod
+    def _text(node: Node) -> str:
         """
-        check if the last element was quoted. ``shlex.shlex`` parser doesn't provide information about was the token
-        quoted or not, thus there is no difference between "'#'" (sharp in quotes) and "#" (sharp without quotes). This
-        method simply rolls back to the last non-space character and check if it is a quotation mark
-
-        Returns:
-            bool: ``True`` if the previous element of the stream is a quote or escaped and ``False`` otherwise
-        """
-        current_position = self._io.tell()
-
-        last_char = penultimate_char = None
-        index = current_position - 1
-        while index > 0:
-            index, last_char = self._read_last(index)
-            if last_char.isspace():
-                index -= 1
-                continue
-
-            if index > 1:
-                _, penultimate_char = self._read_last(index - 1)
-
-            break
-
-        self._io.seek(current_position)  # reset position of the stream
-        is_quoted = last_char is not None and last_char in self.quotes
-        is_escaped = penultimate_char is not None and penultimate_char in self.escape
-
-        return is_quoted or is_escaped
-
-    def _parse_array(self) -> list[str]:
-        """
-        parse array from the PKGBUILD. This method will extract tokens from parser until it matches closing array,
-        modifying source parser state
-
-        Returns:
-            list[str]: extracted arrays elements
-
-        Raises:
-            PkgbuildParserError: if array is not closed
-        """
-        def extract() -> Iterator[str]:
-            while token := self.get_token():
-                match token:
-                    case _ if self._is_escaped():
-                        pass
-                    case PkgbuildToken.ArrayEnds:
-                        break
-                    case comment if comment.startswith(PkgbuildToken.Comment):
-                        self._read_comment()
-                        continue
-
-                yield token
-
-            if token != PkgbuildToken.ArrayEnds:
-                raise PkgbuildParserError("no closing array bracket found")
-
-        return self._expand_array(list(extract()))
-
-    def _parse_function(self) -> str:
-        """
-        parse function from the PKGBUILD. This method will extract tokens from parser until it matches closing function,
-        modifying source parser state. Instead of trying to combine tokens together, it uses positions of the file
-        and reads content again in this range
-
-        Returns:
-            str: function body
-
-        Raises:
-            PkgbuildParserError: if function body wasn't found or parser input stream doesn't support position reading
-        """
-        # find start and end positions
-        start_position = end_position = -1
-        counter = 0  # simple processing of the inner "{" and "}"
-        for token in self:
-            match token:
-                case _ if self._is_escaped():
-                    continue
-                case PkgbuildToken.FunctionStarts:
-                    if counter == 0:
-                        start_position = self._io.tell() - 1
-                    counter += 1
-                case PkgbuildToken.FunctionEnds:
-                    end_position = self._io.tell()
-                    if self.state != self.eof:  # type: ignore[attr-defined]
-                        end_position -= 1  # if we are not at the end of the file, position is _after_ the token
-                    counter -= 1
-                    if counter == 0:
-                        break
-                case comment if comment.startswith(PkgbuildToken.Comment):
-                    self._read_comment()
-
-        if not 0 < start_position < end_position:
-            raise PkgbuildParserError("function body wasn't found")
-
-        # read the specified interval from source stream
-        self._io.seek(start_position - 1)  # start from the previous symbol
-        # we cannot use :func:`read()` here, because it reads characters, not bytes
-        content = ""
-        while self._io.tell() != end_position and (next_char := self._io.read(1)):
-            content += next_char
-
-        # special case of the end of file
-        if self.state == self.eof:  # type: ignore[attr-defined]
-            content += self._io.read(1)
-
-        return content
-
-    def _parse_token(self, token: str) -> Iterator[PkgbuildPatch]:
-        """
-        parse single token to the PKGBUILD field
+        read the original source text represented by a syntax tree node
 
         Args:
-            token(str): current token
+            node(Node): syntax tree node to read
+
+        Returns:
+            str: source text for the node
+        """
+        return cast(bytes, node.text).decode("utf8", errors="backslashreplace")
+
+    def _assignment(self, node: Node) -> PkgbuildPatch:
+        """
+        convert an assignment node
+
+        Args:
+            node(Node): assignment node to convert
+
+        Returns:
+            PkgbuildPatch: parsed assignment
+        """
+        name = cast(Node, node.child_by_field_name(PkgbuildNodeField.Name))
+        value = cast(Node, node.child_by_field_name(PkgbuildNodeField.Value))
+        key = self._text(name)
+
+        match value.type:
+            case PkgbuildNodeType.Array:
+                array_values = []
+                for element in filter(lambda child: child.type != PkgbuildNodeType.Comment, value.named_children):
+                    item = self._value(element)
+                    values = self._expand_braces(item, expand=element.type == PkgbuildNodeType.Concatenation)
+                    array_values.extend(values)
+                return PkgbuildPatch(key, array_values)
+
+            case _:
+                return PkgbuildPatch(key, self._value(value))
+
+    def _read(self, node: Node) -> Iterator[PkgbuildPatch]:
+        """
+        find assignments and functions in a syntax tree node
+
+        Args:
+            node(Node): syntax tree node to inspect
 
         Yields:
-            PkgbuildPatch: extracted a PKGBUILD node
+            PkgbuildPatch: parsed entries
         """
-        # simple assignment rule
-        if m := self._STRING_ASSIGNMENT.match(token):
-            key = m.group("key")
-            value = m.group("value")
-            yield PkgbuildPatch(key, value)
-            return
+        match node.type:
+            case PkgbuildNodeType.VariableAssignment if not any(child.type == PkgbuildNodeType.Append for child in node.children):
+                yield self._assignment(node)
 
-        if token.startswith(PkgbuildToken.Comment):
-            self._read_comment()
-            return
+            case PkgbuildNodeType.FunctionDefinition:
+                name = cast(Node, node.child_by_field_name(PkgbuildNodeField.Name))
+                body = cast(Node, node.child_by_field_name(PkgbuildNodeField.Body))
+                yield PkgbuildPatch(f"{self._text(name)}()", self._text(body))
 
-        match self.get_token():
-            # array processing. Arrays will be sent as "key=", "(", values, ")"
-            case PkgbuildToken.ArrayStarts if m := self._ARRAY_ASSIGNMENT.match(token):
-                key = m.group("key")
-                value = self._parse_array()
-                yield PkgbuildPatch(key, value)
+            case _:
+                # descend into statements, but skip command arguments and values
+                for child in node.named_children:
+                    if child.type not in (
+                            PkgbuildNodeType.Array,
+                            PkgbuildNodeType.Comment,
+                            PkgbuildNodeType.RawString,
+                            PkgbuildNodeType.String,
+                            PkgbuildNodeType.Word,
+                    ):
+                        yield from self._read(child)
 
-            # functions processing. Function will be sent as "name", "()", "{", body, "}"
-            case PkgbuildToken.FunctionDeclaration if self._FUNCTION_DECLARATION.match(token):
-                key = f"{token}{PkgbuildToken.FunctionDeclaration}"
-                value = self._parse_function()
-                yield PkgbuildPatch(key, value)  # this is not mistake, assign to token without ()
-
-            # special function case, where "(" and ")" are separated tokens, e.g. "pkgver ( )"
-            case PkgbuildToken.ArrayStarts if self._FUNCTION_DECLARATION.match(token):
-                next_token = self.get_token()
-                if next_token == PkgbuildToken.ArrayEnds:  # replace closing bracket with "()"
-                    next_token = PkgbuildToken.FunctionDeclaration
-                self.push_token(next_token)  # type: ignore[arg-type]
-                yield from self._parse_token(token)
-
-            # some random token received without continuation, lets guess it is empty assignment (i.e. key=)
-            case other if other is not None:
-                yield from self._parse_token(other)
-
-    def _read_comment(self) -> None:
+    def _value(self, node: Node) -> str:
         """
-        read comment from the current position. This method doesn't check comment itself, just read the stream
-        until the comment line ends
-        """
-        _, last_symbol = self._read_last()
-        if last_symbol != PkgbuildToken.NewLine:
-            self.instream.readline()
-
-    def _read_last(self, initial_index: int | None = None) -> tuple[int, str]:
-        """
-        wrapper around read to read the last symbol from the input stream. This method is designed to process UTF-8
-        symbols correctly. This method does not reset current stream position
+        extract a string value from a syntax tree node
 
         Args:
-            initial_index(int | None, optional): initial index to start reading from. If none set, the previous position
-                will be used (Default value = None)
+            node(Node): syntax tree node to extract
 
         Returns:
-            tuple[int, str]: last symbol and its position in the stream
-
-        Raises:
-            PkgbuildParserError: in case if stream reached starting position, but no valid symbols were found
+            str: extracted string value
         """
-        if initial_index is None:
-            initial_index = self._io.tell() - 1
-        if initial_index < 0:
-            raise PkgbuildParserError("stream is on starting position")
-        self._io.seek(initial_index)
+        if node.type in (PkgbuildNodeType.Concatenation, PkgbuildNodeType.String):
+            return "".join(self._value(child) for child in node.named_children)
 
-        while (position := self._io.tell()) > 0:
-            try:
-                return position, self._io.read(1)
-            except UnicodeDecodeError:
-                self._io.seek(position - 1)
+        value = self._text(node)
+        match node.type:
+            case PkgbuildNodeType.StringContent:
+                value = f'"{value}"'
+            case PkgbuildNodeType.Number | PkgbuildNodeType.RawString | PkgbuildNodeType.Word:
+                pass
+            case _:
+                return value
 
-        raise PkgbuildParserError("reached starting position, no valid symbols found")
+        return "".join(shlex.split(value, comments=False, posix=True))
 
     def parse(self) -> Iterator[PkgbuildPatch]:
         """
-        parse source stream and yield parsed entries
+        parse source and yield extracted entries
 
         Yields:
-            PkgbuildPatch: extracted a PKGBUILD node
+            PkgbuildPatch: parsed entries
+
+        Raises:
+            PkgbuildParserError: source contains invalid shell syntax
         """
-        for token in self:
-            yield from self._parse_token(token)
+        def walk(node: Node) -> Iterator[Node]:
+            yield node
+            for child in node.children:
+                yield from walk(child)
+
+        parser = Parser(Language(tree_sitter_bash.language()))
+        tree = parser.parse(self._content)
+
+        if tree.root_node.has_error:
+            error = next((node for node in walk(tree.root_node) if node.is_error or node.is_missing), tree.root_node)
+            raise PkgbuildParserError(
+                f"syntax error at line {error.start_point.row + 1}, column {error.start_point.column + 1}"
+            )
+
+        yield from self._read(tree.root_node)

ahriman-core/src/ahriman/core/exceptions.py

@@ -238,16 +238,12 @@ class PkgbuildParserError(ValueError):
     exception raises in case of PKGBUILD parser errors
     """
 
-    def __init__(self, reason: str, source: Any = None) -> None:
+    def __init__(self, reason: str) -> None:
         """
         Args:
             reason(str): parser error reason
-            source(Any, optional): source line if available (Default value = None)
         """
-        message = f"Could not parse PKGBUILD: {reason}"
-        if source is not None:
-            message += f", source: `{source}`"
-        ValueError.__init__(self, message)
+        ValueError.__init__(self, f"Could not parse PKGBUILD: {reason}")
 
 
 class PasswordError(ValueError):

ahriman-core/src/ahriman/models/pkgbuild.py

@@ -19,12 +19,12 @@
 #
 from collections.abc import Iterator, Mapping
 from dataclasses import dataclass
-from io import StringIO
+from io import BytesIO
 from pathlib import Path
-from typing import Any, ClassVar, IO, Self
+from typing import Any, IO, Self
 from urllib.parse import urlparse
 
-from ahriman.core.alpm.pkgbuild_parser import PkgbuildParser, PkgbuildToken
+from ahriman.core.alpm.pkgbuild_parser import PkgbuildParser
 from ahriman.core.utils import srcinfo_property_list
 from ahriman.models.pkgbuild_patch import PkgbuildPatch
 
@@ -35,14 +35,11 @@ class Pkgbuild(Mapping[str, Any]):
     model and proxy for PKGBUILD properties
 
     Attributes:
-        DEFAULT_ENCODINGS(str): (class attribute) default encoding to be applied on the file content
         fields(dict[str, PkgbuildPatch]): PKGBUILD fields
     """
 
     fields: dict[str, PkgbuildPatch]
 
-    DEFAULT_ENCODINGS: ClassVar[str] = "utf8"
-
     @property
     def variables(self) -> dict[str, str]:
         """
@@ -59,37 +56,26 @@ class Pkgbuild(Mapping[str, Any]):
         }
 
     @classmethod
-    def from_file(cls, path: Path, encoding: str | None = None) -> Self:
+    def from_file(cls, path: Path) -> Self:
         """
         parse PKGBUILD from the file
 
         Args:
             path(Path): path to the PKGBUILD file
-            encoding(str | None, optional): the encoding of the file (Default value = None)
 
         Returns:
             Self: constructed instance of self
-
-        Raises:
-            EncodeError: if encoding is unknown
         """
-        # read raw bytes from file
         with path.open("rb") as input_file:
-            content = input_file.read()
-
-        # decode bytes content based on either
-        encoding = encoding or cls.DEFAULT_ENCODINGS
-        io = StringIO(content.decode(encoding, errors="backslashreplace"))
-
-        return cls.from_io(io)
+            return cls.from_io(input_file)
 
     @classmethod
-    def from_io(cls, stream: IO[str]) -> Self:
+    def from_io(cls, stream: IO[bytes]) -> Self:
         """
         parse PKGBUILD from input stream
 
         Args:
-            stream(IO[str]): input stream containing PKGBUILD content
+            stream(IO[bytes]): input stream containing PKGBUILD content
 
         Returns:
             Self: constructed instance of self
@@ -162,10 +148,10 @@ class Pkgbuild(Mapping[str, Any]):
         """
         packages = [self["pkgname"]] if isinstance(self["pkgname"], str) else self["pkgname"]
 
-        def io(package_name: str) -> IO[str]:
+        def io(package_name: str) -> IO[bytes]:
             # try to read package specific function and fallback to default otherwise
             content = self.get(f"package_{package_name}") or self.get("package") or ""
-            return StringIO(content)
+            return BytesIO(content.encode("utf8"))
 
         return {package: self.from_io(io(package)) for package in packages}
 
@@ -185,8 +171,8 @@ class Pkgbuild(Mapping[str, Any]):
         """
         value = self.fields.get(item)
         # if the key wasn't found and user didn't ask for function explicitly, we can try to get by function name
-        if value is None and not item.endswith(PkgbuildToken.FunctionDeclaration):
-            value = self.fields.get(f"{item}{PkgbuildToken.FunctionDeclaration}")
+        if value is None and not item.endswith("()"):
+            value = self.fields.get(f"{item}()")
 
         # if we still didn't find anything, we can just raise the exception
         if value is None:

ahriman-core/tests/ahriman/core/alpm/test_pkgbuild_parser.py

@@ -1,71 +1,56 @@
 import pytest
 
-from io import StringIO
-from pathlib import Path
-
 from ahriman.core.alpm.pkgbuild_parser import PkgbuildParser
 from ahriman.core.exceptions import PkgbuildParserError
 from ahriman.models.pkgbuild_patch import PkgbuildPatch
-
-
-def test_expand_array() -> None:
-    """
-    must correctly expand array
-    """
-    assert PkgbuildParser._expand_array(["${pkgbase}{", ",", "-libs", ",", "-fortran}"]) == [
-        "${pkgbase}", "${pkgbase}-libs", "${pkgbase}-fortran"
-    ]
-    assert PkgbuildParser._expand_array(["first", "prefix{1", ",", "2", ",", "3}suffix", "last"]) == [
-        "first", "prefix1suffix", "prefix2suffix", "prefix3suffix", "last"
-    ]
-
-
-def test_expand_array_no_comma() -> None:
-    """
-    must skip array extraction if there is no comma
-    """
-    assert PkgbuildParser._expand_array(["${pkgbase}{", "-libs", "-fortran}"]) == ["${pkgbase}{", "-libs", "-fortran}"]
-
-
-def test_expand_array_short() -> None:
-    """
-    must skip array extraction if it is short
-    """
-    assert PkgbuildParser._expand_array(["${pkgbase}{", ","]) == ["${pkgbase}{", ","]
-
-
-def test_expand_array_exception() -> None:
-    """
-    must raise exception if there is unclosed element
-    """
-    with pytest.raises(PkgbuildParserError):
-        assert PkgbuildParser._expand_array(["${pkgbase}{", ",", "-libs"])
-
-
-def test_is_escaped_exception(resource_path_root: Path) -> None:
-    """
-    must raise PkgbuildParserError if no valid utf symbols found
-    """
-    utf8 = resource_path_root / "models" / "utf8"
-    with utf8.open(encoding="utf8") as content:
-        content.seek(2)
-        with pytest.raises(PkgbuildParserError):
-            assert not PkgbuildParser(content)._is_escaped()
+from io import BytesIO
+from pathlib import Path
 
 
 def test_parse_array() -> None:
     """
     must parse array
     """
-    parser = PkgbuildParser(StringIO("var=(first second)"))
+    parser = PkgbuildParser(BytesIO(b"var=(first second)"))
     assert list(parser.parse()) == [PkgbuildPatch("var", ["first", "second"])]
 
 
+def test_parse_array_brace_expansion() -> None:
+    """
+    must correctly expand bash braces in array values
+    """
+    parser = PkgbuildParser(BytesIO(b"array=(prefix{only}suffix)"))
+    assert list(parser.parse()) == [PkgbuildPatch("array", ["prefix{only}suffix"])]
+
+    parser = PkgbuildParser(BytesIO(b"array=(prefix{first,second}suffix)"))
+    assert list(parser.parse()) == [PkgbuildPatch("array", ["prefixfirstsuffix", "prefixsecondsuffix"])]
+
+    parser = PkgbuildParser(BytesIO(b"""array=("prefix{first,second}suffix")"""))
+    assert list(parser.parse()) == [PkgbuildPatch("array", ["prefix{first,second}suffix"])]
+
+
+def test_parse_array_brackets() -> None:
+    """
+    must correctly process quoted brackets
+    """
+    parser = PkgbuildParser(BytesIO(b"""var=(first "(" second)"""))
+    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", "(", "second"])]
+
+    parser = PkgbuildParser(BytesIO(b"""var=(first ")" second)"""))
+    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
+
+    parser = PkgbuildParser(BytesIO(b"""var=(first ')' second)"""))
+    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
+
+    parser = PkgbuildParser(BytesIO(b"""var=(first \\) second)"""))
+    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
+
+
 def test_parse_array_comment() -> None:
     """
     must parse array with comments inside
     """
-    parser = PkgbuildParser(StringIO("""validpgpkeys=(
+    parser = PkgbuildParser(BytesIO(b"""validpgpkeys=(
   'F3691687D867B81B51CE07D9BBE43771487328A9'  # bpiotrowski@archlinux.org
   '86CFFCA918CF3AF47147588051E8B148A9999C34'  # evangelos@foutrelis.com
   '13975A70E63C361C73AE69EF6EEB81F8981C74C7'  # richard.guenther@gmail.com
@@ -79,28 +64,23 @@ def test_parse_array_comment() -> None:
     ])]
 
 
-def test_parse_array_escaped() -> None:
-    """
-    must correctly process quoted brackets
-    """
-    parser = PkgbuildParser(StringIO("""var=(first "(" second)"""))
-    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", "(", "second"])]
-
-    parser = PkgbuildParser(StringIO("""var=(first ")" second)"""))
-    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
-
-    parser = PkgbuildParser(StringIO("""var=(first ')' second)"""))
-    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
-
-    parser = PkgbuildParser(StringIO("""var=(first \\) second)"""))
-    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
-
-
 def test_parse_array_exception() -> None:
     """
     must raise exception if there is no closing bracket
     """
-    parser = PkgbuildParser(StringIO("var=(first second"))
+    parser = PkgbuildParser(BytesIO(b"var=(first second"))
+    with pytest.raises(PkgbuildParserError, match="line 1"):
+        assert list(parser.parse())
+
+
+def test_parse_conditional_assignment() -> None:
+    """
+    must parse an assignment inside a Bash conditional
+    """
+    source = b"if [[ -n value ]]; then\n  pkgver=1\nfi\n"
+    assert list(PkgbuildParser(BytesIO(source)).parse()) == [PkgbuildPatch("pkgver", "1")]
+
+    parser = PkgbuildParser(BytesIO(b"var() { echo hello world"))
     with pytest.raises(PkgbuildParserError):
         assert list(parser.parse())
 
@@ -109,86 +89,92 @@ def test_parse_function() -> None:
     """
     must parse function
     """
-    parser = PkgbuildParser(StringIO("var() { echo hello world } "))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")]
+    parser = PkgbuildParser(BytesIO(b"var() { echo hello world; } "))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world; }")]
 
 
 def test_parse_function_eof() -> None:
     """
     must parse function with "}" at the end of the file
     """
-    parser = PkgbuildParser(StringIO("var() { echo hello world }"))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")]
-
-
-def test_parse_function_spaces() -> None:
-    """
-    must parse function with spaces in declaration
-    """
-    parser = PkgbuildParser(StringIO("var ( ) { echo hello world } "))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")]
-
-
-def test_parse_function_inner_shell() -> None:
-    """
-    must parse function with inner shell
-    """
-    parser = PkgbuildParser(StringIO("var ( ) { { echo hello world } } "))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ { echo hello world } }")]
+    parser = PkgbuildParser(BytesIO(b"var() { echo hello world; }"))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world; }")]
 
 
 def test_parse_function_escaped() -> None:
     """
     must parse function with bracket in quotes
     """
-    parser = PkgbuildParser(StringIO("""var ( ) { echo "hello world {" } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo "hello world {" }""")]
+    parser = PkgbuildParser(BytesIO(b"""var() { echo "hello world {"; } """))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo "hello world {"; }""")]
 
-    parser = PkgbuildParser(StringIO("""var ( ) { echo hello world "{" } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world "{" }""")]
+    parser = PkgbuildParser(BytesIO(b"""var() { echo hello world "{"; } """))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world "{"; }""")]
 
-    parser = PkgbuildParser(StringIO("""var ( ) { echo "hello world }" } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo "hello world }" }""")]
+    parser = PkgbuildParser(BytesIO(b"""var() { echo "hello world }"; } """))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo "hello world }"; }""")]
 
-    parser = PkgbuildParser(StringIO("""var ( ) { echo hello world "}" } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world "}" }""")]
+    parser = PkgbuildParser(BytesIO(b"""var() { echo hello world "}"; } """))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world "}"; }""")]
 
-    parser = PkgbuildParser(StringIO("""var ( ) { echo hello world '}' } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world '}' }""")]
+    parser = PkgbuildParser(BytesIO(b"""var() { echo hello world '}'; } """))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world '}'; }""")]
 
-    parser = PkgbuildParser(StringIO("""var ( ) { echo hello world \\} } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world \\} }""")]
+    parser = PkgbuildParser(BytesIO(b"""var() { echo hello world \\}; } """))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world \\}; }""")]
 
 
 def test_parse_function_exception() -> None:
     """
     must raise exception if no bracket found
     """
-    parser = PkgbuildParser(StringIO("var() echo hello world } "))
-    with pytest.raises(PkgbuildParserError):
-        assert list(parser.parse())
-
-    parser = PkgbuildParser(StringIO("var() { echo hello world"))
+    parser = PkgbuildParser(BytesIO(b"var() echo hello world } "))
     with pytest.raises(PkgbuildParserError):
         assert list(parser.parse())
 
 
-def test_parse_token_assignment() -> None:
+def test_parse_function_inner_shell() -> None:
     """
-    must parse simple assignment
+    must parse function with inner shell
     """
-    parser = PkgbuildParser(StringIO())
-    assert next(parser._parse_token("var=value")) == PkgbuildPatch("var", "value")
-    assert next(parser._parse_token("var=$value")) == PkgbuildPatch("var", "$value")
-    assert next(parser._parse_token("var=${value}")) == PkgbuildPatch("var", "${value}")
-    assert next(parser._parse_token("var=${value/-/_}")) == PkgbuildPatch("var", "${value/-/_}")
+    parser = PkgbuildParser(BytesIO(b"var() { { echo hello world; }; } "))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ { echo hello world; }; }")]
+
+
+def test_parse_function_spaces() -> None:
+    """
+    must parse function with spaces before body
+    """
+    parser = PkgbuildParser(BytesIO(b"var()   { echo hello world; } "))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world; }")]
+
+
+def test_parse_nested_quotes() -> None:
+    """
+    must parse command substitutions that contain their own quote characters
+    """
+    source = b"""pkgbase="$(grep X f | cut -d '"' -f2)"
+pkgver() {
+  cd aften
+  _ver="$(cat CMakeLists.txt | grep -e SO_MAJOR_VERSION -e SO_MINOR_VERSION -e SO_BUILD_VERSION | cut -d '"' -f2 | head -n3)"
+  echo "$(echo ${_ver} | tr ' ' .).r$(git rev-list --count HEAD).$(git rev-parse --short HEAD)"
+}
+"""
+    assert list(PkgbuildParser(BytesIO(source)).parse()) == [
+        PkgbuildPatch("pkgbase", """$(grep X f | cut -d '"' -f2)"""),
+        PkgbuildPatch("pkgver()", """{
+  cd aften
+  _ver="$(cat CMakeLists.txt | grep -e SO_MAJOR_VERSION -e SO_MINOR_VERSION -e SO_BUILD_VERSION | cut -d '"' -f2 | head -n3)"
+  echo "$(echo ${_ver} | tr ' ' .).r$(git rev-list --count HEAD).$(git rev-parse --short HEAD)"
+}"""),
+    ]
 
 
 def test_parse_token_comment() -> None:
     """
     must correctly parse comment
     """
-    parser = PkgbuildParser(StringIO("""first=1 # comment
+    parser = PkgbuildParser(BytesIO(b"""first=1 # comment
     # comment line
     second=2
     #third=3
@@ -199,58 +185,12 @@ def test_parse_token_comment() -> None:
     ]
 
 
-def test_read_comment() -> None:
-    """
-    must read comment correctly
-    """
-    io = StringIO("# comment\nnew line")
-    io.seek(2)
-
-    PkgbuildParser(io)._read_comment()
-    assert io.tell() == 10
-
-
-def test_read_comment_skip() -> None:
-    """
-    must skip reading new line if comment ends with new line
-    """
-    io = StringIO("#comment\nnew line")
-    io.seek(7)
-
-    PkgbuildParser(io)._read_comment()
-    assert io.tell() == 9
-
-
-def test_read_last() -> None:
-    """
-    must read last symbol from current position
-    """
-    io = StringIO("mock")
-    io.seek(2)
-    assert PkgbuildParser(io)._read_last() == (1, "o")
-
-
-def test_read_last_starting() -> None:
-    """
-    must raise exception if it reads from starting position
-    """
-    with pytest.raises(PkgbuildParserError):
-        assert PkgbuildParser(StringIO("mock"))._read_last()
-
-
-def test_read_last_from_position() -> None:
-    """
-    must read last symbol from the specified position
-    """
-    assert PkgbuildParser(StringIO("mock"))._read_last(2) == (2, "c")
-
-
 def test_parse(resource_path_root: Path) -> None:
     """
     must parse complex file
     """
     pkgbuild = resource_path_root / "models" / "pkgbuild"
-    with pkgbuild.open(encoding="utf8") as content:
+    with pkgbuild.open("rb") as content:
         parser = PkgbuildParser(content)
         assert list(parser.parse()) == [
             PkgbuildPatch("var", "value"),
@@ -277,50 +217,50 @@ def test_parse(resource_path_root: Path) -> None:
             PkgbuildPatch("array", ["first", ")", "second"]),
             PkgbuildPatch("array", ["first", "(", "second"]),
             PkgbuildPatch("array", ["first", ")", "second"]),
-            PkgbuildPatch("function()", """{ single line }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{ single line; }"""),
+            PkgbuildPatch("func()", """{
     multi
     line
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     c
     multi
     line
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     # comment
     multi
     line
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     body
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     body
 }"""),
-            PkgbuildPatch("function_with-package-name()", """{ body }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("function_with-package-name()", """{ body; }"""),
+            PkgbuildPatch("func()", """{
     first
-    { inner shell }
+    { inner shell; }
     last
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     body "{" argument
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     body "}" argument
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     body '{' argument
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     body '}' argument
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     # we don't care about unclosed quotation in comments
     body # no, I said we really don't care
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
   mv "$pkgdir"/usr/share/fonts/站酷小薇体 "$pkgdir"/usr/share/fonts/zcool-xiaowei-regular
   mv "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.站酷小薇体 "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.zcool-xiaowei-regular
 }"""),

ahriman-core/tests/ahriman/models/test_pkgbuild.py

@@ -1,6 +1,6 @@
 import pytest
 
-from io import BytesIO, StringIO
+from io import BytesIO
 from pathlib import Path
 from pytest_mock import MockerFixture
 
@@ -22,26 +22,14 @@ def test_from_file(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
     """
     must correctly load from file
     """
+    stream = BytesIO(b"content")
     open_mock = mocker.patch("pathlib.Path.open")
-    open_mock.return_value.__enter__.return_value = BytesIO(b"content")
+    open_mock.return_value.__enter__.return_value = stream
     load_mock = mocker.patch("ahriman.models.pkgbuild.Pkgbuild.from_io", return_value=pkgbuild_ahriman)
 
     assert Pkgbuild.from_file(Path("local"))
     open_mock.assert_called_once_with("rb")
-    load_mock.assert_called_once_with(pytest.helpers.anyvar(int))
-
-
-def test_from_file_latin(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
-    """
-    must correctly load from file with latin encoding
-    """
-    open_mock = mocker.patch("pathlib.Path.open")
-    open_mock.return_value.__enter__.return_value = BytesIO("contént".encode("latin-1"))
-    load_mock = mocker.patch("ahriman.models.pkgbuild.Pkgbuild.from_io", return_value=pkgbuild_ahriman)
-
-    assert Pkgbuild.from_file(Path("local"))
-    open_mock.assert_called_once_with("rb")
-    load_mock.assert_called_once_with(pytest.helpers.anyvar(int))
+    load_mock.assert_called_once_with(stream)
 
 
 def test_from_io(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
@@ -50,7 +38,7 @@ def test_from_io(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
     """
     load_mock = mocker.patch("ahriman.core.alpm.pkgbuild_parser.PkgbuildParser.parse",
                              return_value=pkgbuild_ahriman.fields.values())
-    assert Pkgbuild.from_io(StringIO("mock")) == pkgbuild_ahriman
+    assert Pkgbuild.from_io(BytesIO(b"mock")) == pkgbuild_ahriman
     load_mock.assert_called_once_with()
 
 
@@ -64,9 +52,9 @@ def test_from_io_pkgbase(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> N
         [value for key, value in pkgbuild_ahriman.fields.items()] + [PkgbuildPatch("pkgbase", "pkgbase")],
     ])
 
-    assert Pkgbuild.from_io(StringIO("mock"))["pkgbase"] == pkgbuild_ahriman["pkgname"]
-    assert "pkgbase" not in Pkgbuild.from_io(StringIO("mock"))
-    assert Pkgbuild.from_io(StringIO("mock"))["pkgbase"] == "pkgbase"
+    assert Pkgbuild.from_io(BytesIO(b"mock"))["pkgbase"] == pkgbuild_ahriman["pkgname"]
+    assert "pkgbase" not in Pkgbuild.from_io(BytesIO(b"mock"))
+    assert Pkgbuild.from_io(BytesIO(b"mock"))["pkgbase"] == "pkgbase"
 
 
 def test_from_io_empty(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
@@ -75,7 +63,7 @@ def test_from_io_empty(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> Non
     """
     mocker.patch("ahriman.core.alpm.pkgbuild_parser.PkgbuildParser.parse",
                  return_value=list(pkgbuild_ahriman.fields.values()) + [PkgbuildPatch("", "")])
-    assert Pkgbuild.from_io(StringIO("mock")) == pkgbuild_ahriman
+    assert Pkgbuild.from_io(BytesIO(b"mock")) == pkgbuild_ahriman
 
 
 def test_local_files(mocker: MockerFixture, resource_path_root: Path) -> None:
@@ -567,7 +555,6 @@ def test_parse_pacman_static(resource_path_root: Path) -> None:
     """
     pkgbuild = Pkgbuild.from_file(resource_path_root / "models" / "package_pacman-static_pkgbuild")
     values = {key: value.value for key, value in pkgbuild.fields.items() if not value.is_function}
-    print(values)
     assert values == {
         "pkgbase": "pacman-static",
         "pkgname": "pacman-static",

archlinux/PKGBUILD

@@ -8,7 +8,7 @@ pkgdesc="ArcH linux ReposItory MANager"
 arch=('any')
 url="https://ahriman.readthedocs.io/"
 license=('GPL-3.0-or-later')
-depends=('devtools>=1:1.0.0' 'git' 'pyalpm' 'python-bcrypt' 'python-filelock' 'python-inflection' 'python-pyelftools' 'python-requests')
+depends=('devtools>=1:1.0.0' 'git' 'pyalpm' 'python-bcrypt' 'python-filelock' 'python-inflection' 'python-pyelftools' 'python-requests' 'python-tree-sitter' 'python-tree-sitter-bash')
 makedepends=('npm' 'python-build' 'python-hatchling' 'python-installer' 'python-wheel')
 source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgbase-$pkgver.tar.gz"
         "$pkgbase.sysusers"

docs/architecture.rst

@@ -426,13 +426,13 @@ There are several supported synchronization providers, currently they are ``rsyn
 PKGBUILD parsing
 ^^^^^^^^^^^^^^^^
 
-The application provides a house-made shell parser ``ahriman.core.alpm.pkgbuild_parser.PkgbuildParser`` to process PKGBUILDs and extract package data from them. It relies on the ``shlex.shlex`` parser with some configuration tweaks and adds some token post-processing.
+The application uses the Tree-sitter Bash grammar in ``ahriman.core.alpm.pkgbuild_parser.PkgbuildParser`` to process
+PKGBUILDs without executing them. It extracts assignment and function nodes from the syntax tree and preserves function
+bodies from the original source.
 
-#. During the parser process, firstly, it extracts the next token from the source file (basically, the word) and tries to match it to the variable assignment. If so, then just processes accordingly.
-#. If it is not an assignment, the parser checks if the token was quoted.
-#. If it wasn't quoted then the parser tries to match the array starts (two consecutive tokens like ``array=`` and ``(``) or it is function (``function``, ``()`` and ``{``).
-#. The arrays are processed until the next closing bracket ``)``. After extraction, the parser tries to expand an array according to bash rules (``prefix{first,second}suffix`` constructions).
-#. The functions are just read until the closing bracket ``}`` and then reread whole text from the input string without a tokenization.
+#. Assignments, arrays, and functions are read from the syntax tree. Quote and escape handling follows Bash syntax.
+#. Function bodies are sliced from the source so their original formatting is preserved.
+#. Bash syntax errors raise ``PkgbuildParserError`` with the source location.
 
 All extracted fields are packed as ``ahriman.models.pkgbuild_patch.PkgbuildPatch`` and then can be used as ``ahriman.models.pkgbuild.Pkgbuild`` instance.
 

tests/testresources/models/package_pacman-static_pkgbuild

@@ -138,7 +138,7 @@ prepare() {
 
     # apply patch level commits on top of annotated tag for pacman
     if [[ -n ${_git_patch_level_commit} ]]; then
-        if [[ v${_git_tag} != $(git describe --tags --abbrev=0 "${_git_patch_level_commit}") ]] then
+        if [[ v${_git_tag} != $(git describe --tags --abbrev=0 "${_git_patch_level_commit}") ]]; then
             error "patch level commit ${_git_patch_level_commit} is not a descendant of v${_git_tag}"
             exit 1
         fi

tests/testresources/models/pkgbuild

@@ -41,55 +41,55 @@ array=(first '(' second)
 array=(first ')' second)
 
 # functions
-function() { single line }
-function() {
+func() { single line; }
+func() {
     multi
     line
 }
-function()
+func()
 {
     c
     multi
     line
 }
-function() {
+func() {
     # comment
     multi
     line
 }
-function () {
+func() {
     body
 }
-function ( ){
+func() {
     body
 }
-function_with-package-name() { body }
-function() {
+function_with-package-name() { body; }
+func() {
     first
-    { inner shell }
+    { inner shell; }
     last
 }
-function() {
+func() {
     body "{" argument
 }
-function() {
+func() {
     body "}" argument
 }
-function() {
+func() {
     body '{' argument
 }
-function() {
+func() {
     body '}' argument
 }
 
 # special case with quotes in comments
-function() {
+func() {
     # we don't care about unclosed quotation in comments
     body # no, I said we really don't care
 }
 
 # some random unicode symbols
-function() {
+func() {
   mv "$pkgdir"/usr/share/fonts/站酷小薇体 "$pkgdir"/usr/share/fonts/zcool-xiaowei-regular
   mv "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.站酷小薇体 "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.zcool-xiaowei-regular
 }