draft of migration to tree-sitter

It has been reported that AUR repositories might contain .gitignore,
which prevents PKGBUILD/.SRCINFO/whatever to be added without --force
flag. This commit simply adds force flag for all those operations, which
should be fine, because we don't commit wildcards

Closes #167

ahriman-core/pyproject.toml

@@ -12,6 +12,8 @@ dependencies = [
     "inflection",
     "pyelftools",
     "requests",
+    "tree-sitter",
+    "tree-sitter-bash",
 ]
 description = "ArcH linux ReposItory MANager, core package"
 dynamic = [

ahriman-core/src/ahriman/application/application/application_repository.py

@@ -158,6 +158,8 @@ class ApplicationRepository(ApplicationProperties):
         # process already built packages if any
         if built_packages := self.repository.packages_built():  # speedup a bit
             build_result = self.repository.process_update(built_packages, packagers)
+            for package in build_result.success:
+                self.database.build_queue_clear(package.base)
             self.on_result(build_result)
             result.merge(build_result)
 
@@ -172,6 +174,8 @@ class ApplicationRepository(ApplicationProperties):
         for num, partition in enumerate(partitions):
             self.logger.info("processing chunk #%i %s", num, [package.base for package in partition])
             build_result = builder.update(partition, packagers, bump_pkgrel=bump_pkgrel)
+            for package in partition:
+                self.database.build_queue_clear(package.base)
             self.on_result(build_result)
             result.merge(build_result)
 

ahriman-core/src/ahriman/core/alpm/pkgbuild_parser.py

@@ -17,355 +17,242 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-import itertools
 import re
 import shlex
+import tree_sitter_bash
 
 from collections.abc import Iterator
 from enum import StrEnum
-from typing import IO
+from tree_sitter import Language, Node, Parser
+from typing import IO, cast
 
 from ahriman.core.exceptions import PkgbuildParserError
 from ahriman.models.pkgbuild_patch import PkgbuildPatch
 
 
-class PkgbuildToken(StrEnum):
+class PkgbuildNodeType(StrEnum):
     """
-    well-known tokens dictionary
+    tree-sitter node types used by parser
 
     Attributes:
-        ArrayEnds(PkgbuildToken): array ends token
-        ArrayStarts(PkgbuildToken): array starts token
-        Comma(PkgbuildToken): comma token
-        Comment(PkgbuildToken): comment token
-        FunctionDeclaration(PkgbuildToken): function declaration token
-        FunctionEnds(PkgbuildToken): function ends token
-        FunctionStarts(PkgbuildToken): function starts token
-        NewLine(PkgbuildToken): new line token
+        Append(PkgbuildNodeType): append node
+        Array(PkgbuildNodeType): array node
+        Comment(PkgbuildNodeType): comment node
+        Concatenation(PkgbuildNodeType): concatenation node
+        Error(PkgbuildNodeType): syntax error node
+        FunctionDefinition(PkgbuildNodeType): function definition node
+        Number(PkgbuildNodeType): numeric word node
+        RawString(PkgbuildNodeType): single-quoted string node
+        String(PkgbuildNodeType): double-quoted string node
+        StringContent(PkgbuildNodeType): string content node
+        VariableAssignment(PkgbuildNodeType): assignment node
+        Word(PkgbuildNodeType): word node
     """
 
-    ArrayStarts = "("
-    ArrayEnds = ")"
-
-    Comma = ","
-
-    Comment = "#"
-
-    FunctionDeclaration = "()"
-    FunctionStarts = "{"
-    FunctionEnds = "}"
-
-    NewLine = "\n"
+    Append = "+="
+    Array = "array"
+    Comment = "comment"
+    Concatenation = "concatenation"
+    Error = "ERROR"
+    FunctionDefinition = "function_definition"
+    Number = "number"
+    RawString = "raw_string"
+    String = "string"
+    StringContent = "string_content"
+    VariableAssignment = "variable_assignment"
+    Word = "word"
 
 
-class PkgbuildParser(shlex.shlex):
+class PkgbuildNodeField(StrEnum):
     """
-    simple pkgbuild reader implementation in pure python, because others suck.
+    tree-sitter node fields used by parser
 
-    What is it:
+    Attributes:
+        Body(PkgbuildNodeField): function body field
+        Name(PkgbuildNodeField): variable or function name field
+        Value(PkgbuildNodeField): assignment value field
+    """
 
-    #. Simple PKGBUILD parser written in python.
-    #. No shell execution, so it is free from random shell attacks.
-    #. Able to parse simple constructions (assignments, comments, functions, arrays).
+    Body = "body"
+    Name = "name"
+    Value = "value"
+
+
+class PkgbuildParser:
+    """
+    PKGBUILD parser implementation in python
+
+    What it is:
+
+    #. A PKGBUILD parser based on tree-sitter's bash grammar.
+    #. A parser for assignments, arrays, and functions.
+    #. A parser which does not execute shell code.
 
     What it is not:
 
-    #. Fully functional shell parser.
-    #. Shell executor.
-    #. No parameter expansion.
+    #. A shell executor.
+    #. A shell interpreter. Variable and command substitutions are returned as written.
 
-    For more details what does it support, please, consult with the test cases.
+    Simple comma-separated brace expressions in array elements are expanded. Function bodies are returned from the
+    original source, with their formatting preserved. Assignments inside shell statements and control-flow nodes are
+    extracted without checking whether those statements would run.
 
     Examples:
-        This class is heavily based on :mod:`shlex` parser, but instead of strings operates with the
-        :class:`ahriman.models.pkgbuild_patch.PkgbuildPatch` objects. The main way to use it is to call :func:`parse()`
-        function and collect parsed objects, e.g.::
+        Parse package assignments::
 
-            >>> parser = PkgbuildParser(StringIO("input string"))
-            >>> for patch in parser.parse():
-            >>>     print(f"{patch.key} = {patch.value}")
+            >>> source = b"pkgname=example\\npkgver=1\\n"
+            >>> for patch in PkgbuildParser(BytesIO(source)).parse():
+            ...     print(patch.key, patch.value)
 
-        It doesn't store the state of the fields (but operates with the :mod:`shlex` parser state), so no shell
-        post-processing is performed (e.g. variable substitution).
+        Function bodies are returned as function patches::
+
+            >>> source = b"pkgver() { printf 1; }"
+            >>> for patch in list(PkgbuildParser(BytesIO(source)).parse())
+            ...     print(patch.key, patch.value)
     """
 
-    _ARRAY_ASSIGNMENT = re.compile(r"^(?P<key>\w+)=$")
-    # in addition to usual assignment, functions can have dash
-    _FUNCTION_DECLARATION = re.compile(r"^(?P<key>[\w-]+)$")
-    _STRING_ASSIGNMENT = re.compile(r"^(?P<key>\w+)=(?P<value>.+)$")
+    _BRACE_EXPANSION = re.compile(r"(.*)\{([^{}]+)\}(.*)")
 
-    def __init__(self, stream: IO[str]) -> None:
-        """
-        Args:
-            stream(IO[str]): input stream containing PKGBUILD content
-        """
-        shlex.shlex.__init__(self, stream, posix=True, punctuation_chars=True)
-        self._io = stream  # direct access without type casting
-
-        # ignore substitution and extend bash symbols
-        self.wordchars += "${}#:+-@!"
-        # in case of default behavior, it will ignore, for example, segment part of url outside of quotes
-        self.commenters = ""
+    def __init__(self, stream: IO[bytes]) -> None:
+        """"""
+        self._content = stream.read()
 
     @staticmethod
-    def _expand_array(array: list[str]) -> list[str]:
+    def _expand_braces(value: str, *, expand: bool = True) -> list[str]:
         """
-        bash array expansion simulator. It takes raw array and tries to expand constructions like
-        ``(first prefix-{mid1,mid2}-suffix last)`` into ``(first, prefix-mid1-suffix prefix-mid2-suffix last)``
+        expand comma-separated brace expressions in a value
 
         Args:
-            array(list[str]): input array
+            value(str): value that may contain a brace expression
+            expand(bool, optional): whether brace expansion is allowed (Default value = True)
 
         Returns:
-            list[str]: either source array or expanded array if possible
-
-        Raises:
-            PkgbuildParserError: if there are errors in parser
+            list[str]: expanded values if applicable
         """
-        # we are using comma as marker for expansion (if any)
-        if PkgbuildToken.Comma not in array:
-            return array
-        # again sanity check, for expansion there are at least 3 elements (first, last and comma)
-        if len(array) < 3:
-            return array
+        match = PkgbuildParser._BRACE_EXPANSION.fullmatch(value)
+        # if there is no comma inside it should be treated as normal value
+        if match is None or not expand or "," not in match.group(2):
+            return [value]
 
-        result = []
-        buffer, prefix = [], None
+        prefix, values, suffix = match.groups()
+        return [f"{prefix}{item}{suffix}" for item in values.split(",")]
 
-        for index, (first, second) in enumerate(itertools.pairwise(array)):
-            match (first, second):
-                # in this case we check if expansion should be started
-                # this condition matches "prefix{first", ","
-                case (_, PkgbuildToken.Comma) if PkgbuildToken.FunctionStarts in first:
-                    prefix, part = first.rsplit(PkgbuildToken.FunctionStarts, maxsplit=1)
-                    buffer.append(f"{prefix}{part}")
-
-                # the last element case, it matches either ",", "last}" or ",", "last}suffix"
-                # in case if there is suffix, it must be appended to all list elements
-                case (PkgbuildToken.Comma, _) if prefix is not None and PkgbuildToken.FunctionEnds in second:
-                    part, suffix = second.rsplit(PkgbuildToken.FunctionEnds, maxsplit=1)
-                    buffer.append(f"{prefix}{part}")
-                    result.extend([f"{part}{suffix}" for part in buffer])
-                    # reset state
-                    buffer, prefix = [], None
-
-                # we have already got prefix string, so we are in progress of expansion
-                # we always operate the last element, so this matches ",", "next"
-                case (PkgbuildToken.Comma, _) if prefix is not None:
-                    buffer.append(f"{prefix}{second}")
-
-                # exactly first element of the list
-                case (_, _) if prefix is None and index == 0:
-                    result.append(first)
-
-                # any next normal element
-                case (_, _) if prefix is None:
-                    result.append(second)
-
-        # small sanity check
-        if prefix is not None:
-            raise PkgbuildParserError("error in array expansion", array)
-
-        return result
-
-    def _is_escaped(self) -> bool:
+    @staticmethod
+    def _text(node: Node) -> str:
         """
-        check if the last element was quoted. ``shlex.shlex`` parser doesn't provide information about was the token
-        quoted or not, thus there is no difference between "'#'" (sharp in quotes) and "#" (sharp without quotes). This
-        method simply rolls back to the last non-space character and check if it is a quotation mark
-
-        Returns:
-            bool: ``True`` if the previous element of the stream is a quote or escaped and ``False`` otherwise
-        """
-        current_position = self._io.tell()
-
-        last_char = penultimate_char = None
-        index = current_position - 1
-        while index > 0:
-            index, last_char = self._read_last(index)
-            if last_char.isspace():
-                index -= 1
-                continue
-
-            if index > 1:
-                _, penultimate_char = self._read_last(index - 1)
-
-            break
-
-        self._io.seek(current_position)  # reset position of the stream
-        is_quoted = last_char is not None and last_char in self.quotes
-        is_escaped = penultimate_char is not None and penultimate_char in self.escape
-
-        return is_quoted or is_escaped
-
-    def _parse_array(self) -> list[str]:
-        """
-        parse array from the PKGBUILD. This method will extract tokens from parser until it matches closing array,
-        modifying source parser state
-
-        Returns:
-            list[str]: extracted arrays elements
-
-        Raises:
-            PkgbuildParserError: if array is not closed
-        """
-        def extract() -> Iterator[str]:
-            while token := self.get_token():
-                match token:
-                    case _ if self._is_escaped():
-                        pass
-                    case PkgbuildToken.ArrayEnds:
-                        break
-                    case comment if comment.startswith(PkgbuildToken.Comment):
-                        self._read_comment()
-                        continue
-
-                yield token
-
-            if token != PkgbuildToken.ArrayEnds:
-                raise PkgbuildParserError("no closing array bracket found")
-
-        return self._expand_array(list(extract()))
-
-    def _parse_function(self) -> str:
-        """
-        parse function from the PKGBUILD. This method will extract tokens from parser until it matches closing function,
-        modifying source parser state. Instead of trying to combine tokens together, it uses positions of the file
-        and reads content again in this range
-
-        Returns:
-            str: function body
-
-        Raises:
-            PkgbuildParserError: if function body wasn't found or parser input stream doesn't support position reading
-        """
-        # find start and end positions
-        start_position = end_position = -1
-        counter = 0  # simple processing of the inner "{" and "}"
-        for token in self:
-            match token:
-                case _ if self._is_escaped():
-                    continue
-                case PkgbuildToken.FunctionStarts:
-                    if counter == 0:
-                        start_position = self._io.tell() - 1
-                    counter += 1
-                case PkgbuildToken.FunctionEnds:
-                    end_position = self._io.tell()
-                    if self.state != self.eof:  # type: ignore[attr-defined]
-                        end_position -= 1  # if we are not at the end of the file, position is _after_ the token
-                    counter -= 1
-                    if counter == 0:
-                        break
-                case comment if comment.startswith(PkgbuildToken.Comment):
-                    self._read_comment()
-
-        if not 0 < start_position < end_position:
-            raise PkgbuildParserError("function body wasn't found")
-
-        # read the specified interval from source stream
-        self._io.seek(start_position - 1)  # start from the previous symbol
-        # we cannot use :func:`read()` here, because it reads characters, not bytes
-        content = ""
-        while self._io.tell() != end_position and (next_char := self._io.read(1)):
-            content += next_char
-
-        # special case of the end of file
-        if self.state == self.eof:  # type: ignore[attr-defined]
-            content += self._io.read(1)
-
-        return content
-
-    def _parse_token(self, token: str) -> Iterator[PkgbuildPatch]:
-        """
-        parse single token to the PKGBUILD field
+        read the original source text represented by a syntax tree node
 
         Args:
-            token(str): current token
+            node(Node): syntax tree node to read
+
+        Returns:
+            str: source text for the node
+        """
+        return cast(bytes, node.text).decode("utf8", errors="backslashreplace")
+
+    def _assignment(self, node: Node) -> PkgbuildPatch:
+        """
+        convert an assignment node
+
+        Args:
+            node(Node): assignment node to convert
+
+        Returns:
+            PkgbuildPatch: parsed assignment
+        """
+        name = cast(Node, node.child_by_field_name(PkgbuildNodeField.Name))
+        value = cast(Node, node.child_by_field_name(PkgbuildNodeField.Value))
+        key = self._text(name)
+
+        match value.type:
+            case PkgbuildNodeType.Array:
+                array_values = []
+                for element in filter(lambda child: child.type != PkgbuildNodeType.Comment, value.named_children):
+                    item = self._value(element)
+                    values = self._expand_braces(item, expand=element.type == PkgbuildNodeType.Concatenation)
+                    array_values.extend(values)
+                return PkgbuildPatch(key, array_values)
+
+            case _:
+                return PkgbuildPatch(key, self._value(value))
+
+    def _read(self, node: Node) -> Iterator[PkgbuildPatch]:
+        """
+        find assignments and functions in a syntax tree node
+
+        Args:
+            node(Node): syntax tree node to inspect
 
         Yields:
-            PkgbuildPatch: extracted a PKGBUILD node
+            PkgbuildPatch: parsed entries
         """
-        # simple assignment rule
-        if m := self._STRING_ASSIGNMENT.match(token):
-            key = m.group("key")
-            value = m.group("value")
-            yield PkgbuildPatch(key, value)
-            return
+        match node.type:
+            case PkgbuildNodeType.VariableAssignment if not any(child.type == PkgbuildNodeType.Append for child in node.children):
+                yield self._assignment(node)
 
-        if token.startswith(PkgbuildToken.Comment):
-            self._read_comment()
-            return
+            case PkgbuildNodeType.FunctionDefinition:
+                name = cast(Node, node.child_by_field_name(PkgbuildNodeField.Name))
+                body = cast(Node, node.child_by_field_name(PkgbuildNodeField.Body))
+                yield PkgbuildPatch(f"{self._text(name)}()", self._text(body))
 
-        match self.get_token():
-            # array processing. Arrays will be sent as "key=", "(", values, ")"
-            case PkgbuildToken.ArrayStarts if m := self._ARRAY_ASSIGNMENT.match(token):
-                key = m.group("key")
-                value = self._parse_array()
-                yield PkgbuildPatch(key, value)
+            case _:
+                # descend into statements, but skip command arguments and values
+                for child in node.named_children:
+                    if child.type not in (
+                            PkgbuildNodeType.Array,
+                            PkgbuildNodeType.Comment,
+                            PkgbuildNodeType.RawString,
+                            PkgbuildNodeType.String,
+                            PkgbuildNodeType.Word,
+                    ):
+                        yield from self._read(child)
 
-            # functions processing. Function will be sent as "name", "()", "{", body, "}"
-            case PkgbuildToken.FunctionDeclaration if self._FUNCTION_DECLARATION.match(token):
-                key = f"{token}{PkgbuildToken.FunctionDeclaration}"
-                value = self._parse_function()
-                yield PkgbuildPatch(key, value)  # this is not mistake, assign to token without ()
-
-            # special function case, where "(" and ")" are separated tokens, e.g. "pkgver ( )"
-            case PkgbuildToken.ArrayStarts if self._FUNCTION_DECLARATION.match(token):
-                next_token = self.get_token()
-                if next_token == PkgbuildToken.ArrayEnds:  # replace closing bracket with "()"
-                    next_token = PkgbuildToken.FunctionDeclaration
-                self.push_token(next_token)  # type: ignore[arg-type]
-                yield from self._parse_token(token)
-
-            # some random token received without continuation, lets guess it is empty assignment (i.e. key=)
-            case other if other is not None:
-                yield from self._parse_token(other)
-
-    def _read_comment(self) -> None:
+    def _value(self, node: Node) -> str:
         """
-        read comment from the current position. This method doesn't check comment itself, just read the stream
-        until the comment line ends
-        """
-        _, last_symbol = self._read_last()
-        if last_symbol != PkgbuildToken.NewLine:
-            self.instream.readline()
-
-    def _read_last(self, initial_index: int | None = None) -> tuple[int, str]:
-        """
-        wrapper around read to read the last symbol from the input stream. This method is designed to process UTF-8
-        symbols correctly. This method does not reset current stream position
+        extract a string value from a syntax tree node
 
         Args:
-            initial_index(int | None, optional): initial index to start reading from. If none set, the previous position
-                will be used (Default value = None)
+            node(Node): syntax tree node to extract
 
         Returns:
-            tuple[int, str]: last symbol and its position in the stream
-
-        Raises:
-            PkgbuildParserError: in case if stream reached starting position, but no valid symbols were found
+            str: extracted string value
         """
-        if initial_index is None:
-            initial_index = self._io.tell() - 1
-        if initial_index < 0:
-            raise PkgbuildParserError("stream is on starting position")
-        self._io.seek(initial_index)
+        if node.type in (PkgbuildNodeType.Concatenation, PkgbuildNodeType.String):
+            return "".join(self._value(child) for child in node.named_children)
 
-        while (position := self._io.tell()) > 0:
-            try:
-                return position, self._io.read(1)
-            except UnicodeDecodeError:
-                self._io.seek(position - 1)
+        value = self._text(node)
+        match node.type:
+            case PkgbuildNodeType.StringContent:
+                value = f'"{value}"'
+            case PkgbuildNodeType.Number | PkgbuildNodeType.RawString | PkgbuildNodeType.Word:
+                pass
+            case _:
+                return value
 
-        raise PkgbuildParserError("reached starting position, no valid symbols found")
+        return "".join(shlex.split(value, comments=False, posix=True))
 
     def parse(self) -> Iterator[PkgbuildPatch]:
         """
-        parse source stream and yield parsed entries
+        parse source and yield extracted entries
 
         Yields:
-            PkgbuildPatch: extracted a PKGBUILD node
+            PkgbuildPatch: parsed entries
+
+        Raises:
+            PkgbuildParserError: source contains invalid shell syntax
         """
-        for token in self:
-            yield from self._parse_token(token)
+        def walk(node: Node) -> Iterator[Node]:
+            yield node
+            for child in node.children:
+                yield from walk(child)
+
+        parser = Parser(Language(tree_sitter_bash.language()))
+        tree = parser.parse(self._content)
+
+        if tree.root_node.has_error:
+            error = next((node for node in walk(tree.root_node) if node.is_error or node.is_missing), tree.root_node)
+            raise PkgbuildParserError(
+                f"syntax error at line {error.start_point.row + 1}, column {error.start_point.column + 1}"
+            )
+
+        yield from self._read(tree.root_node)

ahriman-core/src/ahriman/core/build_tools/sources.py

@@ -165,7 +165,7 @@ class Sources(LazyLogging):
 
         # extract local files...
         files = ["PKGBUILD", ".SRCINFO"] + [str(path) for path in Pkgbuild.local_files(sources_dir)]
-        instance.add(sources_dir, *files)
+        instance.add(sources_dir, *files, force=True)
         # ...and commit them
         instance.commit(sources_dir)
 
@@ -232,7 +232,7 @@ class Sources(LazyLogging):
         git_url, branch = remote.git_source()
         check_output(*instance.git(), "push", "--quiet", git_url, branch, cwd=sources_dir, logger=instance.logger)
 
-    def add(self, sources_dir: Path, *pattern: str, intent_to_add: bool = False) -> None:
+    def add(self, sources_dir: Path, *pattern: str, intent_to_add: bool = False, force: bool = False) -> None:
         """
         track found files via git
 
@@ -241,6 +241,7 @@ class Sources(LazyLogging):
             *pattern(str): glob patterns
             intent_to_add(bool, optional): record only the fact that it will be added later, acts as
                 --intent-to-add git flag (Default value = False)
+            force(bool, optional): allow adding ignored files (Default value = False)
         """
         # glob directory to find files which match the specified patterns
         found_files: list[Path] = []
@@ -251,6 +252,8 @@ class Sources(LazyLogging):
         self.logger.info("found matching files %s", found_files)
         # add them to index
         args = ["--intent-to-add"] if intent_to_add else []
+        if force:
+            args.append("--force")
         check_output(*self.git(), "add", *args, *[str(fn.relative_to(sources_dir)) for fn in found_files],
                      cwd=sources_dir, logger=self.logger)
 

ahriman-core/src/ahriman/core/exceptions.py

@@ -238,16 +238,12 @@ class PkgbuildParserError(ValueError):
     exception raises in case of PKGBUILD parser errors
     """
 
-    def __init__(self, reason: str, source: Any = None) -> None:
+    def __init__(self, reason: str) -> None:
         """
         Args:
             reason(str): parser error reason
-            source(Any, optional): source line if available (Default value = None)
         """
-        message = f"Could not parse PKGBUILD: {reason}"
-        if source is not None:
-            message += f", source: `{source}`"
-        ValueError.__init__(self, message)
+        ValueError.__init__(self, f"Could not parse PKGBUILD: {reason}")
 
 
 class PasswordError(ValueError):

ahriman-core/src/ahriman/core/housekeeping/archive_rotation_trigger.py

@@ -87,7 +87,12 @@ class ArchiveRotationTrigger(Trigger):
         if self.keep_built_packages == 0:
             return
 
-        to_remove = repository.package_archives(package.base)
+        current = next(iter(repository.packages([package.base])), None)
+        to_remove = [
+            single
+            for single in repository.package_archives(package.base)
+            if current is None or single.version != current.version
+        ]
 
         for single in to_remove[:-self.keep_built_packages]:
             self.logger.info("removing version %s of package %s", single.version, single.base)

ahriman-core/src/ahriman/core/repository/update_handler.py

@@ -36,6 +36,17 @@ class UpdateHandler(PackageInfo, Cleaner):
     trait to get package update list
     """
 
+    def _updates_ignored(self) -> set[str]:
+        """
+        collect package bases excluded from automatic updates
+
+        Returns:
+            set[str]: held and configured ignored package bases
+        """
+        return set(self._ignore_list) | {
+            package.base for package, status in self.reporter.package_get(None) if status.is_held
+        }
+
     def updates_aur(self, filter_packages: Iterable[str], *, vcs: bool) -> list[Package]:
         """
         check AUR for updates
@@ -58,9 +69,7 @@ class UpdateHandler(PackageInfo, Cleaner):
                     continue
             raise UnknownPackageError(package.base)
 
-        ignore_list = self._ignore_list + [
-            package.base for package, status in self.reporter.package_get(None) if status.is_held
-        ]
+        ignore_list = self._updates_ignored()
 
         result: list[Package] = []
         for local in self.packages(filter_packages):
@@ -106,8 +115,13 @@ class UpdateHandler(PackageInfo, Cleaner):
 
             return files
 
+        ignore_list = self._updates_ignored()
+
         result: list[Package] = []
         for local in self.packages(filter_packages):
+            if local.base in ignore_list:
+                self.logger.info("package %s is held, skip update check", local.base)
+                continue
             dependencies = self.reporter.package_dependencies_get(local.base)
             if not dependencies.paths:
                 continue  # skip check if no package dependencies found
@@ -141,9 +155,13 @@ class UpdateHandler(PackageInfo, Cleaner):
         """
         packages = {local.base: local for local in self.packages()}
         local_versions = {package_base: package.version for package_base, package in packages.items()}
+        ignore_list = self._updates_ignored()
 
         result: list[Package] = []
         for cache_dir in self.paths.cache.iterdir():
+            if cache_dir.name in ignore_list:
+                self.logger.info("package %s is held, skip update check", cache_dir.name)
+                continue
             with self.in_package_context(cache_dir.name, local_versions.get(cache_dir.name)):
                 try:
                     source = RemoteSource(
@@ -162,6 +180,9 @@ class UpdateHandler(PackageInfo, Cleaner):
                         continue  # we don't add packages automatically
                     if local.remote.is_remote:
                         continue  # avoid checking AUR packages
+                    if local.base in ignore_list:
+                        self.logger.info("package %s is held, skip update check", local.base)
+                        continue
 
                     if PackageVersion(local).is_outdated(remote, self.configuration, calculate_version=vcs):
                         self.reporter.set_pending(local.base)
@@ -191,7 +212,6 @@ class UpdateHandler(PackageInfo, Cleaner):
                     self.reporter.set_pending(local.base)
                 self.event(local.base, EventType.PackageOutdated, "Manual update is requested")
 
-            self.clear_queue()
         except Exception:
             self.logger.exception("could not load packages from database")
 

ahriman-core/src/ahriman/models/pkgbuild.py

@@ -19,12 +19,12 @@
 #
 from collections.abc import Iterator, Mapping
 from dataclasses import dataclass
-from io import StringIO
+from io import BytesIO
 from pathlib import Path
-from typing import Any, ClassVar, IO, Self
+from typing import Any, IO, Self
 from urllib.parse import urlparse
 
-from ahriman.core.alpm.pkgbuild_parser import PkgbuildParser, PkgbuildToken
+from ahriman.core.alpm.pkgbuild_parser import PkgbuildParser
 from ahriman.core.utils import srcinfo_property_list
 from ahriman.models.pkgbuild_patch import PkgbuildPatch
 
@@ -35,14 +35,11 @@ class Pkgbuild(Mapping[str, Any]):
     model and proxy for PKGBUILD properties
 
     Attributes:
-        DEFAULT_ENCODINGS(str): (class attribute) default encoding to be applied on the file content
         fields(dict[str, PkgbuildPatch]): PKGBUILD fields
     """
 
     fields: dict[str, PkgbuildPatch]
 
-    DEFAULT_ENCODINGS: ClassVar[str] = "utf8"
-
     @property
     def variables(self) -> dict[str, str]:
         """
@@ -59,37 +56,26 @@ class Pkgbuild(Mapping[str, Any]):
         }
 
     @classmethod
-    def from_file(cls, path: Path, encoding: str | None = None) -> Self:
+    def from_file(cls, path: Path) -> Self:
         """
         parse PKGBUILD from the file
 
         Args:
             path(Path): path to the PKGBUILD file
-            encoding(str | None, optional): the encoding of the file (Default value = None)
 
         Returns:
             Self: constructed instance of self
-
-        Raises:
-            EncodeError: if encoding is unknown
         """
-        # read raw bytes from file
         with path.open("rb") as input_file:
-            content = input_file.read()
-
-        # decode bytes content based on either
-        encoding = encoding or cls.DEFAULT_ENCODINGS
-        io = StringIO(content.decode(encoding, errors="backslashreplace"))
-
-        return cls.from_io(io)
+            return cls.from_io(input_file)
 
     @classmethod
-    def from_io(cls, stream: IO[str]) -> Self:
+    def from_io(cls, stream: IO[bytes]) -> Self:
         """
         parse PKGBUILD from input stream
 
         Args:
-            stream(IO[str]): input stream containing PKGBUILD content
+            stream(IO[bytes]): input stream containing PKGBUILD content
 
         Returns:
             Self: constructed instance of self
@@ -162,10 +148,10 @@ class Pkgbuild(Mapping[str, Any]):
         """
         packages = [self["pkgname"]] if isinstance(self["pkgname"], str) else self["pkgname"]
 
-        def io(package_name: str) -> IO[str]:
+        def io(package_name: str) -> IO[bytes]:
             # try to read package specific function and fallback to default otherwise
             content = self.get(f"package_{package_name}") or self.get("package") or ""
-            return StringIO(content)
+            return BytesIO(content.encode("utf8"))
 
         return {package: self.from_io(io(package)) for package in packages}
 
@@ -185,8 +171,8 @@ class Pkgbuild(Mapping[str, Any]):
         """
         value = self.fields.get(item)
         # if the key wasn't found and user didn't ask for function explicitly, we can try to get by function name
-        if value is None and not item.endswith(PkgbuildToken.FunctionDeclaration):
-            value = self.fields.get(f"{item}{PkgbuildToken.FunctionDeclaration}")
+        if value is None and not item.endswith("()"):
+            value = self.fields.get(f"{item}()")
 
         # if we still didn't find anything, we can just raise the exception
         if value is None:

ahriman-core/tests/ahriman/application/application/test_application_repository.py

@@ -188,6 +188,7 @@ def test_update(application_repository: ApplicationRepository, package_ahriman:
     """
     must process package updates
     """
+    application_repository.database.build_queue_insert(package_ahriman)
     paths = [package.filepath for package in package_ahriman.packages.values()]
     tree = Tree([Leaf(package_ahriman)])
     prebuilt_result = Result()
@@ -207,12 +208,15 @@ def test_update(application_repository: ApplicationRepository, package_ahriman:
     update_mock.assert_called_once_with(paths, Packagers("username"))
     on_result_mock.assert_has_calls([MockCall(prebuilt_result), MockCall(result)])
 
+    assert application_repository.database.build_queue_get() == []
+
 
 def test_update_prebuilt_filter(application_repository: ApplicationRepository, package_ahriman: Package, result: Result,
                                 mocker: MockerFixture) -> None:
     """
     must filter out packages which were successfully prebuilt
     """
+    application_repository.database.build_queue_insert(package_ahriman)
     paths = [package.filepath for package in package_ahriman.packages.values()]
 
     resolve_mock = mocker.patch("ahriman.application.application.workers.local_updater.LocalUpdater.partition",
@@ -224,6 +228,8 @@ def test_update_prebuilt_filter(application_repository: ApplicationRepository, p
     application_repository.update([package_ahriman], Packagers("username"), bump_pkgrel=True)
     resolve_mock.assert_called_once_with([])
 
+    assert application_repository.database.build_queue_get() == []
+
 
 def test_update_empty(application_repository: ApplicationRepository, package_ahriman: Package, result: Result,
                       mocker: MockerFixture) -> None:

ahriman-core/tests/ahriman/core/alpm/test_pkgbuild_parser.py

@@ -1,71 +1,56 @@
 import pytest
 
-from io import StringIO
-from pathlib import Path
-
 from ahriman.core.alpm.pkgbuild_parser import PkgbuildParser
 from ahriman.core.exceptions import PkgbuildParserError
 from ahriman.models.pkgbuild_patch import PkgbuildPatch
-
-
-def test_expand_array() -> None:
-    """
-    must correctly expand array
-    """
-    assert PkgbuildParser._expand_array(["${pkgbase}{", ",", "-libs", ",", "-fortran}"]) == [
-        "${pkgbase}", "${pkgbase}-libs", "${pkgbase}-fortran"
-    ]
-    assert PkgbuildParser._expand_array(["first", "prefix{1", ",", "2", ",", "3}suffix", "last"]) == [
-        "first", "prefix1suffix", "prefix2suffix", "prefix3suffix", "last"
-    ]
-
-
-def test_expand_array_no_comma() -> None:
-    """
-    must skip array extraction if there is no comma
-    """
-    assert PkgbuildParser._expand_array(["${pkgbase}{", "-libs", "-fortran}"]) == ["${pkgbase}{", "-libs", "-fortran}"]
-
-
-def test_expand_array_short() -> None:
-    """
-    must skip array extraction if it is short
-    """
-    assert PkgbuildParser._expand_array(["${pkgbase}{", ","]) == ["${pkgbase}{", ","]
-
-
-def test_expand_array_exception() -> None:
-    """
-    must raise exception if there is unclosed element
-    """
-    with pytest.raises(PkgbuildParserError):
-        assert PkgbuildParser._expand_array(["${pkgbase}{", ",", "-libs"])
-
-
-def test_is_escaped_exception(resource_path_root: Path) -> None:
-    """
-    must raise PkgbuildParserError if no valid utf symbols found
-    """
-    utf8 = resource_path_root / "models" / "utf8"
-    with utf8.open(encoding="utf8") as content:
-        content.seek(2)
-        with pytest.raises(PkgbuildParserError):
-            assert not PkgbuildParser(content)._is_escaped()
+from io import BytesIO
+from pathlib import Path
 
 
 def test_parse_array() -> None:
     """
     must parse array
     """
-    parser = PkgbuildParser(StringIO("var=(first second)"))
+    parser = PkgbuildParser(BytesIO(b"var=(first second)"))
     assert list(parser.parse()) == [PkgbuildPatch("var", ["first", "second"])]
 
 
+def test_parse_array_brace_expansion() -> None:
+    """
+    must correctly expand bash braces in array values
+    """
+    parser = PkgbuildParser(BytesIO(b"array=(prefix{only}suffix)"))
+    assert list(parser.parse()) == [PkgbuildPatch("array", ["prefix{only}suffix"])]
+
+    parser = PkgbuildParser(BytesIO(b"array=(prefix{first,second}suffix)"))
+    assert list(parser.parse()) == [PkgbuildPatch("array", ["prefixfirstsuffix", "prefixsecondsuffix"])]
+
+    parser = PkgbuildParser(BytesIO(b"""array=("prefix{first,second}suffix")"""))
+    assert list(parser.parse()) == [PkgbuildPatch("array", ["prefix{first,second}suffix"])]
+
+
+def test_parse_array_brackets() -> None:
+    """
+    must correctly process quoted brackets
+    """
+    parser = PkgbuildParser(BytesIO(b"""var=(first "(" second)"""))
+    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", "(", "second"])]
+
+    parser = PkgbuildParser(BytesIO(b"""var=(first ")" second)"""))
+    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
+
+    parser = PkgbuildParser(BytesIO(b"""var=(first ')' second)"""))
+    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
+
+    parser = PkgbuildParser(BytesIO(b"""var=(first \\) second)"""))
+    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
+
+
 def test_parse_array_comment() -> None:
     """
     must parse array with comments inside
     """
-    parser = PkgbuildParser(StringIO("""validpgpkeys=(
+    parser = PkgbuildParser(BytesIO(b"""validpgpkeys=(
   'F3691687D867B81B51CE07D9BBE43771487328A9'  # bpiotrowski@archlinux.org
   '86CFFCA918CF3AF47147588051E8B148A9999C34'  # evangelos@foutrelis.com
   '13975A70E63C361C73AE69EF6EEB81F8981C74C7'  # richard.guenther@gmail.com
@@ -79,28 +64,23 @@ def test_parse_array_comment() -> None:
     ])]
 
 
-def test_parse_array_escaped() -> None:
-    """
-    must correctly process quoted brackets
-    """
-    parser = PkgbuildParser(StringIO("""var=(first "(" second)"""))
-    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", "(", "second"])]
-
-    parser = PkgbuildParser(StringIO("""var=(first ")" second)"""))
-    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
-
-    parser = PkgbuildParser(StringIO("""var=(first ')' second)"""))
-    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
-
-    parser = PkgbuildParser(StringIO("""var=(first \\) second)"""))
-    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", ")", "second"])]
-
-
 def test_parse_array_exception() -> None:
     """
     must raise exception if there is no closing bracket
     """
-    parser = PkgbuildParser(StringIO("var=(first second"))
+    parser = PkgbuildParser(BytesIO(b"var=(first second"))
+    with pytest.raises(PkgbuildParserError, match="line 1"):
+        assert list(parser.parse())
+
+
+def test_parse_conditional_assignment() -> None:
+    """
+    must parse an assignment inside a Bash conditional
+    """
+    source = b"if [[ -n value ]]; then\n  pkgver=1\nfi\n"
+    assert list(PkgbuildParser(BytesIO(source)).parse()) == [PkgbuildPatch("pkgver", "1")]
+
+    parser = PkgbuildParser(BytesIO(b"var() { echo hello world"))
     with pytest.raises(PkgbuildParserError):
         assert list(parser.parse())
 
@@ -109,86 +89,92 @@ def test_parse_function() -> None:
     """
     must parse function
     """
-    parser = PkgbuildParser(StringIO("var() { echo hello world } "))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")]
+    parser = PkgbuildParser(BytesIO(b"var() { echo hello world; } "))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world; }")]
 
 
 def test_parse_function_eof() -> None:
     """
     must parse function with "}" at the end of the file
     """
-    parser = PkgbuildParser(StringIO("var() { echo hello world }"))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")]
-
-
-def test_parse_function_spaces() -> None:
-    """
-    must parse function with spaces in declaration
-    """
-    parser = PkgbuildParser(StringIO("var ( ) { echo hello world } "))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")]
-
-
-def test_parse_function_inner_shell() -> None:
-    """
-    must parse function with inner shell
-    """
-    parser = PkgbuildParser(StringIO("var ( ) { { echo hello world } } "))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ { echo hello world } }")]
+    parser = PkgbuildParser(BytesIO(b"var() { echo hello world; }"))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world; }")]
 
 
 def test_parse_function_escaped() -> None:
     """
     must parse function with bracket in quotes
     """
-    parser = PkgbuildParser(StringIO("""var ( ) { echo "hello world {" } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo "hello world {" }""")]
+    parser = PkgbuildParser(BytesIO(b"""var() { echo "hello world {"; } """))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo "hello world {"; }""")]
 
-    parser = PkgbuildParser(StringIO("""var ( ) { echo hello world "{" } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world "{" }""")]
+    parser = PkgbuildParser(BytesIO(b"""var() { echo hello world "{"; } """))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world "{"; }""")]
 
-    parser = PkgbuildParser(StringIO("""var ( ) { echo "hello world }" } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo "hello world }" }""")]
+    parser = PkgbuildParser(BytesIO(b"""var() { echo "hello world }"; } """))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo "hello world }"; }""")]
 
-    parser = PkgbuildParser(StringIO("""var ( ) { echo hello world "}" } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world "}" }""")]
+    parser = PkgbuildParser(BytesIO(b"""var() { echo hello world "}"; } """))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world "}"; }""")]
 
-    parser = PkgbuildParser(StringIO("""var ( ) { echo hello world '}' } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world '}' }""")]
+    parser = PkgbuildParser(BytesIO(b"""var() { echo hello world '}'; } """))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world '}'; }""")]
 
-    parser = PkgbuildParser(StringIO("""var ( ) { echo hello world \\} } """))
-    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world \\} }""")]
+    parser = PkgbuildParser(BytesIO(b"""var() { echo hello world \\}; } """))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", """{ echo hello world \\}; }""")]
 
 
 def test_parse_function_exception() -> None:
     """
     must raise exception if no bracket found
     """
-    parser = PkgbuildParser(StringIO("var() echo hello world } "))
-    with pytest.raises(PkgbuildParserError):
-        assert list(parser.parse())
-
-    parser = PkgbuildParser(StringIO("var() { echo hello world"))
+    parser = PkgbuildParser(BytesIO(b"var() echo hello world } "))
     with pytest.raises(PkgbuildParserError):
         assert list(parser.parse())
 
 
-def test_parse_token_assignment() -> None:
+def test_parse_function_inner_shell() -> None:
     """
-    must parse simple assignment
+    must parse function with inner shell
     """
-    parser = PkgbuildParser(StringIO())
-    assert next(parser._parse_token("var=value")) == PkgbuildPatch("var", "value")
-    assert next(parser._parse_token("var=$value")) == PkgbuildPatch("var", "$value")
-    assert next(parser._parse_token("var=${value}")) == PkgbuildPatch("var", "${value}")
-    assert next(parser._parse_token("var=${value/-/_}")) == PkgbuildPatch("var", "${value/-/_}")
+    parser = PkgbuildParser(BytesIO(b"var() { { echo hello world; }; } "))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ { echo hello world; }; }")]
+
+
+def test_parse_function_spaces() -> None:
+    """
+    must parse function with spaces before body
+    """
+    parser = PkgbuildParser(BytesIO(b"var()   { echo hello world; } "))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world; }")]
+
+
+def test_parse_nested_quotes() -> None:
+    """
+    must parse command substitutions that contain their own quote characters
+    """
+    source = b"""pkgbase="$(grep X f | cut -d '"' -f2)"
+pkgver() {
+  cd aften
+  _ver="$(cat CMakeLists.txt | grep -e SO_MAJOR_VERSION -e SO_MINOR_VERSION -e SO_BUILD_VERSION | cut -d '"' -f2 | head -n3)"
+  echo "$(echo ${_ver} | tr ' ' .).r$(git rev-list --count HEAD).$(git rev-parse --short HEAD)"
+}
+"""
+    assert list(PkgbuildParser(BytesIO(source)).parse()) == [
+        PkgbuildPatch("pkgbase", """$(grep X f | cut -d '"' -f2)"""),
+        PkgbuildPatch("pkgver()", """{
+  cd aften
+  _ver="$(cat CMakeLists.txt | grep -e SO_MAJOR_VERSION -e SO_MINOR_VERSION -e SO_BUILD_VERSION | cut -d '"' -f2 | head -n3)"
+  echo "$(echo ${_ver} | tr ' ' .).r$(git rev-list --count HEAD).$(git rev-parse --short HEAD)"
+}"""),
+    ]
 
 
 def test_parse_token_comment() -> None:
     """
     must correctly parse comment
     """
-    parser = PkgbuildParser(StringIO("""first=1 # comment
+    parser = PkgbuildParser(BytesIO(b"""first=1 # comment
     # comment line
     second=2
     #third=3
@@ -199,58 +185,12 @@ def test_parse_token_comment() -> None:
     ]
 
 
-def test_read_comment() -> None:
-    """
-    must read comment correctly
-    """
-    io = StringIO("# comment\nnew line")
-    io.seek(2)
-
-    PkgbuildParser(io)._read_comment()
-    assert io.tell() == 10
-
-
-def test_read_comment_skip() -> None:
-    """
-    must skip reading new line if comment ends with new line
-    """
-    io = StringIO("#comment\nnew line")
-    io.seek(7)
-
-    PkgbuildParser(io)._read_comment()
-    assert io.tell() == 9
-
-
-def test_read_last() -> None:
-    """
-    must read last symbol from current position
-    """
-    io = StringIO("mock")
-    io.seek(2)
-    assert PkgbuildParser(io)._read_last() == (1, "o")
-
-
-def test_read_last_starting() -> None:
-    """
-    must raise exception if it reads from starting position
-    """
-    with pytest.raises(PkgbuildParserError):
-        assert PkgbuildParser(StringIO("mock"))._read_last()
-
-
-def test_read_last_from_position() -> None:
-    """
-    must read last symbol from the specified position
-    """
-    assert PkgbuildParser(StringIO("mock"))._read_last(2) == (2, "c")
-
-
 def test_parse(resource_path_root: Path) -> None:
     """
     must parse complex file
     """
     pkgbuild = resource_path_root / "models" / "pkgbuild"
-    with pkgbuild.open(encoding="utf8") as content:
+    with pkgbuild.open("rb") as content:
         parser = PkgbuildParser(content)
         assert list(parser.parse()) == [
             PkgbuildPatch("var", "value"),
@@ -277,50 +217,50 @@ def test_parse(resource_path_root: Path) -> None:
             PkgbuildPatch("array", ["first", ")", "second"]),
             PkgbuildPatch("array", ["first", "(", "second"]),
             PkgbuildPatch("array", ["first", ")", "second"]),
-            PkgbuildPatch("function()", """{ single line }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{ single line; }"""),
+            PkgbuildPatch("func()", """{
     multi
     line
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     c
     multi
     line
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     # comment
     multi
     line
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     body
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     body
 }"""),
-            PkgbuildPatch("function_with-package-name()", """{ body }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("function_with-package-name()", """{ body; }"""),
+            PkgbuildPatch("func()", """{
     first
-    { inner shell }
+    { inner shell; }
     last
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     body "{" argument
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     body "}" argument
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     body '{' argument
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     body '}' argument
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
     # we don't care about unclosed quotation in comments
     body # no, I said we really don't care
 }"""),
-            PkgbuildPatch("function()", """{
+            PkgbuildPatch("func()", """{
   mv "$pkgdir"/usr/share/fonts/站酷小薇体 "$pkgdir"/usr/share/fonts/zcool-xiaowei-regular
   mv "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.站酷小薇体 "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.zcool-xiaowei-regular
 }"""),

ahriman-core/tests/ahriman/core/build_tools/test_sources.py

@@ -195,7 +195,7 @@ def test_init(sources: Sources, mocker: MockerFixture) -> None:
     sources.init(local)
     check_output_mock.assert_called_once_with(*sources.git(), "init", "--quiet", "--initial-branch",
                                               sources.DEFAULT_BRANCH, cwd=local, logger=pytest.helpers.anyvar(int))
-    add_mock.assert_called_once_with(local, "PKGBUILD", ".SRCINFO", "local")
+    add_mock.assert_called_once_with(local, "PKGBUILD", ".SRCINFO", "local", force=True)
     commit_mock.assert_called_once_with(local)
 
 
@@ -347,6 +347,17 @@ def test_add_skip(sources: Sources, mocker: MockerFixture) -> None:
     check_output_mock.assert_not_called()
 
 
+def test_add_force(sources: Sources, tmp_path: Path, mocker: MockerFixture) -> None:
+    """
+    must add files to git with --force flag
+    """
+    mocker.patch("pathlib.Path.glob", return_value=[Path("local/1"), Path("local/2")])
+    check_output_mock = mocker.patch("ahriman.core.build_tools.sources.check_output")
+
+    sources.add(Path("local"), "PKGBUILD", force=True)
+    assert "--force" in check_output_mock.call_args.args
+
+
 def test_commit(sources: Sources, mocker: MockerFixture) -> None:
     """
     must commit changes

ahriman-core/tests/ahriman/core/housekeeping/test_archive_rotation_trigger.py

@@ -22,10 +22,10 @@ def test_configuration_sections(configuration: Configuration) -> None:
 def test_archives_remove(archive_rotation_trigger: ArchiveRotationTrigger, package_ahriman: Package,
                          repository: Repository, mocker: MockerFixture) -> None:
     """
-    must remove older packages
+    must keep the current package and the configured amount of additional packages
     """
     packages = []
-    for i in range(5):
+    for i in range(1, 7):
         generated = replace(package_ahriman, version=str(i))
         generated.packages = {
             key: replace(value, filename=str(i))
@@ -33,14 +33,17 @@ def test_archives_remove(archive_rotation_trigger: ArchiveRotationTrigger, packa
         }
         packages.append(generated)
 
+    archive_rotation_trigger.keep_built_packages = 2
+    mocker.patch.object(repository, "packages", return_value=[replace(package_ahriman, version="3")])
     mocker.patch("ahriman.core.repository.package_info.PackageInfo.package_archives", return_value=packages)
-    mocker.patch("pathlib.Path.glob", return_value=[Path(str(i)) for i in range(5)])
+    mocker.patch("pathlib.Path.glob", side_effect=lambda pattern: [Path(pattern.removesuffix("*"))])
     unlink_mock = mocker.patch("pathlib.Path.unlink", autospec=True)
 
     archive_rotation_trigger.archives_remove(package_ahriman, repository)
     unlink_mock.assert_has_calls([
-        MockCall(Path("0")),
         MockCall(Path("1")),
+        MockCall(Path("2")),
+        MockCall(Path("4")),
     ])
 
 

ahriman-core/tests/ahriman/core/repository/test_update_handler.py

@@ -14,6 +14,20 @@ from ahriman.models.package_source import PackageSource
 from ahriman.models.remote_source import RemoteSource
 
 
+def test_updates_ignored(update_handler: UpdateHandler, package_ahriman: Package,
+                         package_python_schedule: Package, mocker: MockerFixture) -> None:
+    """
+    must combine configured ignored packages and held packages only
+    """
+    update_handler._ignore_list = ["ignored-package"]
+    mocker.patch("ahriman.core.status.local_client.LocalClient.package_get", return_value=[
+        (package_ahriman, BuildStatus(is_held=True)),
+        (package_python_schedule, BuildStatus(is_held=False)),
+    ])
+
+    assert update_handler._updates_ignored() == {"ignored-package", package_ahriman.base}
+
+
 def test_updates_aur(update_handler: UpdateHandler, package_ahriman: Package,
                      mocker: MockerFixture) -> None:
     """
@@ -194,6 +208,18 @@ def test_updates_dependencies(update_handler: UpdateHandler, package_ahriman: Pa
                                        pytest.helpers.anyvar(str, True))
 
 
+def test_updates_dependencies_ignore(update_handler: UpdateHandler, package_ahriman: Package,
+                                     mocker: MockerFixture) -> None:
+    """
+    must skip held packages when checking broken dependencies
+    """
+    mocker.patch("ahriman.core.status.local_client.LocalClient.package_get",
+                 return_value=[(package_ahriman, BuildStatus(is_held=True))])
+    mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman])
+
+    assert update_handler.updates_dependencies([]) == []
+
+
 def test_updates_dependencies_skip_unknown(update_handler: UpdateHandler, package_ahriman: Package,
                                            mocker: MockerFixture) -> None:
     """
@@ -249,6 +275,19 @@ def test_updates_local(update_handler: UpdateHandler, package_ahriman: Package,
         calculate_version=True)
 
 
+def test_updates_local_ignore(update_handler: UpdateHandler, package_ahriman: Package,
+                              mocker: MockerFixture) -> None:
+    """
+    must skip held packages before fetching local sources
+    """
+    mocker.patch("ahriman.core.status.local_client.LocalClient.package_get",
+                 return_value=[(package_ahriman, BuildStatus(is_held=True))])
+    mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman])
+    mocker.patch("pathlib.Path.iterdir", return_value=[Path(package_ahriman.base)])
+
+    assert update_handler.updates_local(vcs=True) == []
+
+
 def test_updates_local_ignore_vcs(update_handler: UpdateHandler, package_ahriman: Package,
                                   mocker: MockerFixture) -> None:
     """
@@ -306,17 +345,32 @@ def test_updates_local_with_failures(update_handler: UpdateHandler, package_ahri
     assert not update_handler.updates_local(vcs=True)
 
 
-def test_updates_manual_clear(update_handler: UpdateHandler, mocker: MockerFixture) -> None:
+def test_updates_local_held_metadata(update_handler: UpdateHandler, package_ahriman: Package,
+                                     mocker: MockerFixture) -> None:
     """
-    requesting manual updates must clear packages directory
+    must respect the held base even when the source directory has a different name
     """
-    mocker.patch("pathlib.Path.iterdir", return_value=[])
-    mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages")
+    package_ahriman.remote = RemoteSource(source=PackageSource.Local)
+    mocker.patch.object(update_handler.reporter, "package_get",
+                        return_value=[(package_ahriman, BuildStatus(is_held=True))])
+    mocker.patch.object(update_handler, "packages", return_value=[package_ahriman])
+    mocker.patch("pathlib.Path.iterdir", return_value=[Path("different-base")])
+    mocker.patch("ahriman.core.build_tools.sources.Sources.fetch")
+    mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman)
 
-    update_handler.updates_manual()
+    assert update_handler.updates_local(vcs=True) == []
 
-    from ahriman.core.repository.cleaner import Cleaner
-    Cleaner.clear_queue.assert_called_once_with()
+
+def test_updates_manual_preserves_queue(update_handler: UpdateHandler, package_ahriman: Package,
+                                        mocker: MockerFixture) -> None:
+    """
+    discovery must leave manual requests queued for execution
+    """
+    update_handler.database.build_queue_insert(package_ahriman)
+    mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.packages", return_value=[package_ahriman])
+
+    assert update_handler.updates_manual() == [package_ahriman]
+    assert update_handler.database.build_queue_get() == [package_ahriman]
 
 
 def test_updates_manual_status_known(update_handler: UpdateHandler, package_ahriman: Package,

ahriman-core/tests/ahriman/models/test_pkgbuild.py

@@ -1,6 +1,6 @@
 import pytest
 
-from io import BytesIO, StringIO
+from io import BytesIO
 from pathlib import Path
 from pytest_mock import MockerFixture
 
@@ -22,26 +22,14 @@ def test_from_file(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
     """
     must correctly load from file
     """
+    stream = BytesIO(b"content")
     open_mock = mocker.patch("pathlib.Path.open")
-    open_mock.return_value.__enter__.return_value = BytesIO(b"content")
+    open_mock.return_value.__enter__.return_value = stream
     load_mock = mocker.patch("ahriman.models.pkgbuild.Pkgbuild.from_io", return_value=pkgbuild_ahriman)
 
     assert Pkgbuild.from_file(Path("local"))
     open_mock.assert_called_once_with("rb")
-    load_mock.assert_called_once_with(pytest.helpers.anyvar(int))
-
-
-def test_from_file_latin(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
-    """
-    must correctly load from file with latin encoding
-    """
-    open_mock = mocker.patch("pathlib.Path.open")
-    open_mock.return_value.__enter__.return_value = BytesIO("contént".encode("latin-1"))
-    load_mock = mocker.patch("ahriman.models.pkgbuild.Pkgbuild.from_io", return_value=pkgbuild_ahriman)
-
-    assert Pkgbuild.from_file(Path("local"))
-    open_mock.assert_called_once_with("rb")
-    load_mock.assert_called_once_with(pytest.helpers.anyvar(int))
+    load_mock.assert_called_once_with(stream)
 
 
 def test_from_io(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
@@ -50,7 +38,7 @@ def test_from_io(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
     """
     load_mock = mocker.patch("ahriman.core.alpm.pkgbuild_parser.PkgbuildParser.parse",
                              return_value=pkgbuild_ahriman.fields.values())
-    assert Pkgbuild.from_io(StringIO("mock")) == pkgbuild_ahriman
+    assert Pkgbuild.from_io(BytesIO(b"mock")) == pkgbuild_ahriman
     load_mock.assert_called_once_with()
 
 
@@ -64,9 +52,9 @@ def test_from_io_pkgbase(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> N
         [value for key, value in pkgbuild_ahriman.fields.items()] + [PkgbuildPatch("pkgbase", "pkgbase")],
     ])
 
-    assert Pkgbuild.from_io(StringIO("mock"))["pkgbase"] == pkgbuild_ahriman["pkgname"]
-    assert "pkgbase" not in Pkgbuild.from_io(StringIO("mock"))
-    assert Pkgbuild.from_io(StringIO("mock"))["pkgbase"] == "pkgbase"
+    assert Pkgbuild.from_io(BytesIO(b"mock"))["pkgbase"] == pkgbuild_ahriman["pkgname"]
+    assert "pkgbase" not in Pkgbuild.from_io(BytesIO(b"mock"))
+    assert Pkgbuild.from_io(BytesIO(b"mock"))["pkgbase"] == "pkgbase"
 
 
 def test_from_io_empty(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
@@ -75,7 +63,7 @@ def test_from_io_empty(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> Non
     """
     mocker.patch("ahriman.core.alpm.pkgbuild_parser.PkgbuildParser.parse",
                  return_value=list(pkgbuild_ahriman.fields.values()) + [PkgbuildPatch("", "")])
-    assert Pkgbuild.from_io(StringIO("mock")) == pkgbuild_ahriman
+    assert Pkgbuild.from_io(BytesIO(b"mock")) == pkgbuild_ahriman
 
 
 def test_local_files(mocker: MockerFixture, resource_path_root: Path) -> None:
@@ -567,7 +555,6 @@ def test_parse_pacman_static(resource_path_root: Path) -> None:
     """
     pkgbuild = Pkgbuild.from_file(resource_path_root / "models" / "package_pacman-static_pkgbuild")
     values = {key: value.value for key, value in pkgbuild.fields.items() if not value.is_function}
-    print(values)
     assert values == {
         "pkgbase": "pacman-static",
         "pkgname": "pacman-static",

archlinux/PKGBUILD

@@ -8,7 +8,7 @@ pkgdesc="ArcH linux ReposItory MANager"
 arch=('any')
 url="https://ahriman.readthedocs.io/"
 license=('GPL-3.0-or-later')
-depends=('devtools>=1:1.0.0' 'git' 'pyalpm' 'python-bcrypt' 'python-filelock' 'python-inflection' 'python-pyelftools' 'python-requests')
+depends=('devtools>=1:1.0.0' 'git' 'pyalpm' 'python-bcrypt' 'python-filelock' 'python-inflection' 'python-pyelftools' 'python-requests' 'python-tree-sitter' 'python-tree-sitter-bash')
 makedepends=('npm' 'python-build' 'python-hatchling' 'python-installer' 'python-wheel')
 source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgbase-$pkgver.tar.gz"
         "$pkgbase.sysusers"

docs/architecture.rst

@@ -257,6 +257,10 @@ There are few ways for packages to be marked as out-of-date and hence requiring
 #. In addition to the above, if package is named as VCS (e.g. has suffix ``-git``) and the last update was more than specified threshold ago, the service will also try to fetch sources and check if the revision is newer than the built one.
 #. In addition, there is ability to check if the dependencies of the package have been updated (e.g. if linked library has been renamed or the modules directory - e.g. in case of python and ruby packages - has been changed). And if so, the package will be marked as out-of-date as well.
 
+Held packages and bases listed in ``build.ignore_packages`` are excluded from automatic AUR, official-repository, local-source, and broken-dependency update checks. These checks leave their package status unchanged. Manual updates and explicit rebuilds remain available.
+
+Reading manual updates, including during ``repo-update --dry-run``, leaves the build queue intact. Execution removes requests by package base after a build chunk returns or a remote worker accepts it; successful prebuilt imports also remove requests by package base. Interrupted chunks and work that was not selected remain queued. Concurrent requests for the same package base are not distinguished; request revisions and crash recovery are separate work.
+
 Update packages
 ^^^^^^^^^^^^^^^
 
@@ -422,13 +426,13 @@ There are several supported synchronization providers, currently they are ``rsyn
 PKGBUILD parsing
 ^^^^^^^^^^^^^^^^
 
-The application provides a house-made shell parser ``ahriman.core.alpm.pkgbuild_parser.PkgbuildParser`` to process PKGBUILDs and extract package data from them. It relies on the ``shlex.shlex`` parser with some configuration tweaks and adds some token post-processing.
+The application uses the Tree-sitter Bash grammar in ``ahriman.core.alpm.pkgbuild_parser.PkgbuildParser`` to process
+PKGBUILDs without executing them. It extracts assignment and function nodes from the syntax tree and preserves function
+bodies from the original source.
 
-#. During the parser process, firstly, it extracts the next token from the source file (basically, the word) and tries to match it to the variable assignment. If so, then just processes accordingly.
-#. If it is not an assignment, the parser checks if the token was quoted.
-#. If it wasn't quoted then the parser tries to match the array starts (two consecutive tokens like ``array=`` and ``(``) or it is function (``function``, ``()`` and ``{``).
-#. The arrays are processed until the next closing bracket ``)``. After extraction, the parser tries to expand an array according to bash rules (``prefix{first,second}suffix`` constructions).
-#. The functions are just read until the closing bracket ``}`` and then reread whole text from the input string without a tokenization.
+#. Assignments, arrays, and functions are read from the syntax tree. Quote and escape handling follows Bash syntax.
+#. Function bodies are sliced from the source so their original formatting is preserved.
+#. Bash syntax errors raise ``PkgbuildParserError`` with the source location.
 
 All extracted fields are packed as ``ahriman.models.pkgbuild_patch.PkgbuildPatch`` and then can be used as ``ahriman.models.pkgbuild.Pkgbuild`` instance.
 

docs/configuration.rst

@@ -207,7 +207,7 @@ Web server settings. This feature requires ``aiohttp`` libraries to be installed
 
 Describes settings for packages archives management extensions.
 
-* ``keep_built_packages`` - keep this amount of built packages with different versions, integer, required. ``0`` will effectively disable archives removal.
+* ``keep_built_packages`` - keep this amount of additional built package versions alongside the version currently published in the repository, integer, required. ``0`` will effectively disable archives removal.
 
 ``keyring`` group
 -----------------

docs/triggers.rst

@@ -44,7 +44,7 @@ This trigger will be called right after build process (``on_result``). It will p
 ``ahriman.core.housekeeping.ArchiveRotationTrigger``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-This trigger removes old package versions from the archive directory. It implements ``on_result`` and, after each build, compares available versions for updated packages and removes the older ones, keeping only the last N versions as configured by ``keep_built_packages`` option.
+This trigger removes old package versions from the archive directory. It implements ``on_result`` and, after each build, compares available versions for updated packages and removes older ones, keeping the version currently published in the repository plus the last N other versions as configured by ``keep_built_packages`` option.
 
 ``ahriman.core.housekeeping.LogsRotationTrigger``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

frontend/src/components/dialogs/PackageInfoDialog.tsx

@@ -126,6 +126,7 @@ export default function PackageInfoDialog({
         try {
             const newHeldStatus = !(status?.is_held ?? false);
             await client.service.servicePackageHold(localPackageBase, currentRepository, newHeldStatus);
+            showSuccess("Success", `Package ${newHeldStatus ? "held" : "unheld"}`);
             void queryClient.invalidateQueries({ queryKey: QueryKeys.package(localPackageBase, currentRepository) });
         } catch (exception) {
             showError("Action failed", `Could not update hold status: ${ApiError.errorDetail(exception)}`);

frontend/src/components/table/PackageTable.tsx

@@ -143,9 +143,9 @@ export default function PackageTable(): React.JSX.Element {
                     .then(() => table.setSelectionModel([])),
             }}
             isAuthorized={table.isAuthorized}
-            hasSelection={table.selectionModel.length > 0}
             onSearchChange={table.setSearchText}
             searchText={table.searchText}
+            selectedPackageCount={table.selectionModel.length}
             status={table.status}
         />
 

frontend/src/components/table/PackageTableToolbar.tsx

@@ -48,19 +48,19 @@ export interface ToolbarActions {
 
 interface PackageTableToolbarProps {
     actions: ToolbarActions;
-    hasSelection: boolean;
     isAuthorized: boolean;
     onSearchChange: (text: string) => void;
     searchText: string;
+    selectedPackageCount: number;
     status?: BuildStatus;
 }
 
 export default function PackageTableToolbar({
     actions,
-    hasSelection,
     isAuthorized,
     onSearchChange,
     searchText,
+    selectedPackageCount,
     status,
 }: PackageTableToolbarProps): React.JSX.Element {
     const [packagesAnchorEl, setPackagesAnchorEl] = useState<HTMLElement | null>(null);
@@ -103,7 +103,8 @@ export default function PackageTableToolbar({
                     <MenuItem onClick={() => {
                         setPackagesAnchorEl(null); actions.onUpdateClick();
                     }}>
-                        <PlayArrowIcon fontSize="small" sx={{ mr: 1 }} /> update
+                        <PlayArrowIcon fontSize="small" sx={{ mr: 1 }} />
+                        {selectedPackageCount === 0 ? "update all packages" : `update selected packages (${selectedPackageCount})`}
                     </MenuItem>
                     <MenuItem onClick={() => {
                         setPackagesAnchorEl(null); actions.onRefreshDatabaseClick();
@@ -116,7 +117,7 @@ export default function PackageTableToolbar({
                         <ReplayIcon fontSize="small" sx={{ mr: 1 }} /> rebuild
                     </MenuItem>
                     <Divider />
-                    <MenuItem disabled={!hasSelection} onClick={() => {
+                    <MenuItem disabled={selectedPackageCount === 0} onClick={() => {
                         setPackagesAnchorEl(null); actions.onRemoveClick();
                     }}>
                         <DeleteIcon fontSize="small" sx={{ mr: 1 }} /> remove

frontend/src/components/table/StatusCell.tsx

@@ -18,7 +18,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 import PauseCircleIcon from "@mui/icons-material/PauseCircle";
-import { Chip } from "@mui/material";
+import { Chip, Tooltip } from "@mui/material";
 import type { BuildStatus } from "models/BuildStatus";
 import type React from "react";
 import { StatusColors } from "theme/StatusColors";
@@ -29,14 +29,17 @@ interface StatusCellProps {
 }
 
 export default function StatusCell({ isHeld, status }: StatusCellProps): React.JSX.Element {
-    return <Chip
-        icon={isHeld ? <PauseCircleIcon /> : undefined}
-        label={status}
-        size="small"
-        sx={{
-            backgroundColor: StatusColors[status],
-            color: "common.white",
-            fontWeight: 500,
-        }}
-    />;
+    return <Tooltip title={isHeld ? "Held: automatic updates are skipped" : ""}>
+        <Chip
+            aria-label={isHeld ? `${status}; held: automatic updates are skipped` : status}
+            icon={isHeld ? <PauseCircleIcon /> : undefined}
+            label={status}
+            size="small"
+            sx={{
+                backgroundColor: StatusColors[status],
+                color: "common.white",
+                fontWeight: 500,
+            }}
+        />
+    </Tooltip>;
 }

frontend/src/theme/Theme.ts

@@ -18,6 +18,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 import { createTheme, type Theme } from "@mui/material/styles";
+import type {} from "@mui/x-data-grid/themeAugmentation";
 
 export function createAppTheme(mode: "light" | "dark"): Theme {
     return createTheme({
@@ -30,6 +31,14 @@ export function createAppTheme(mode: "light" | "dark"): Theme {
                     },
                 },
             },
+            MuiDataGrid: {
+                styleOverrides: {
+                    cell: {
+                        alignItems: "center",
+                        display: "flex",
+                    },
+                },
+            },
             MuiDialog: {
                 defaultProps: {
                     fullWidth: true,

tests/testresources/models/package_pacman-static_pkgbuild

@@ -138,7 +138,7 @@ prepare() {
 
     # apply patch level commits on top of annotated tag for pacman
     if [[ -n ${_git_patch_level_commit} ]]; then
-        if [[ v${_git_tag} != $(git describe --tags --abbrev=0 "${_git_patch_level_commit}") ]] then
+        if [[ v${_git_tag} != $(git describe --tags --abbrev=0 "${_git_patch_level_commit}") ]]; then
             error "patch level commit ${_git_patch_level_commit} is not a descendant of v${_git_tag}"
             exit 1
         fi

tests/testresources/models/pkgbuild

@@ -41,55 +41,55 @@ array=(first '(' second)
 array=(first ')' second)
 
 # functions
-function() { single line }
-function() {
+func() { single line; }
+func() {
     multi
     line
 }
-function()
+func()
 {
     c
     multi
     line
 }
-function() {
+func() {
     # comment
     multi
     line
 }
-function () {
+func() {
     body
 }
-function ( ){
+func() {
     body
 }
-function_with-package-name() { body }
-function() {
+function_with-package-name() { body; }
+func() {
     first
-    { inner shell }
+    { inner shell; }
     last
 }
-function() {
+func() {
     body "{" argument
 }
-function() {
+func() {
     body "}" argument
 }
-function() {
+func() {
     body '{' argument
 }
-function() {
+func() {
     body '}' argument
 }
 
 # special case with quotes in comments
-function() {
+func() {
     # we don't care about unclosed quotation in comments
     body # no, I said we really don't care
 }
 
 # some random unicode symbols
-function() {
+func() {
   mv "$pkgdir"/usr/share/fonts/站酷小薇体 "$pkgdir"/usr/share/fonts/zcool-xiaowei-regular
   mv "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.站酷小薇体 "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.zcool-xiaowei-regular
 }