in-house bytes pkgbuild parser

src/ahriman/core/alpm/bytes_pkgbuild_parser.py

@@ -0,0 +1,183 @@
+#
+# Copyright (c) 2021-2025 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+from collections import defaultdict
+from collections.abc import Iterator
+from dataclasses import dataclass
+from enum import ReprEnum
+from types import SimpleNamespace
+from typing import Generator, IO, Self
+
+from ahriman.models.pkgbuild_patch import PkgbuildPatch
+
+
+class PkgbuildToken(bytes, ReprEnum):
+
+    Comment = b"#"
+    Assignment = b"="
+    SingleQuote = b"'"
+    DoubleQuote = b"\""
+    Space = b" "
+    NewLine = b"\n"
+
+    ParenthesisOpen = b"("
+    ParenthesisClose = b")"
+
+    FunctionStarts = b"function"
+    FunctionDeclaration = b"()"
+    BraceOpen = b"{"
+    BraceClose = b"}"
+
+
+@dataclass
+class PkgbuildWord:
+
+    word: bytes
+    quote: bytes | None
+
+    @property
+    def closing(self) -> PkgbuildToken | None:
+        if self.quote:
+            return None
+        match self.word:
+            case PkgbuildToken.ParenthesisOpen:
+                return PkgbuildToken.ParenthesisClose
+            case PkgbuildToken.BraceOpen:
+                return PkgbuildToken.BraceClose
+        return None
+
+    @property
+    def original(self) -> bytes:
+        quote = self.quote or b""
+        return quote + self.word + quote
+
+    def __bool__(self) -> bool:
+        return bool(self.original)
+
+
+class BytesPkgbuildParser(Iterator[PkgbuildPatch]):
+
+    def __init__(self, stream: IO[bytes]) -> None:
+        self._io = stream
+
+    def _next(self, *, declaration: bool) -> bytes:
+        while not (token := self._next_token(declaration=declaration)):
+            continue
+        return token
+
+    def _next_token(self, *, declaration: bool) -> bytes:
+        buffer = b""
+        while word := self._next_word():
+            match word:
+                case PkgbuildWord(PkgbuildToken.Comment, None):
+                    self._io.readline()
+
+                case PkgbuildWord(PkgbuildToken.NewLine, None):
+                    if declaration:
+                        buffer = b""
+                    return buffer
+
+                case PkgbuildWord(PkgbuildToken.Assignment, None) if declaration:
+                    return buffer
+
+                case PkgbuildWord(PkgbuildToken.Space, None) if declaration:
+                    if buffer.endswith(PkgbuildToken.FunctionDeclaration):
+                        return buffer
+                    buffer = b""
+                    continue
+
+                case PkgbuildWord(PkgbuildToken.Space, None):
+                    return buffer
+
+                case PkgbuildWord(PkgbuildToken.ParenthesisOpen, None):
+                    buffer += PkgbuildToken.ParenthesisOpen
+                    buffer += b"".join(self._next_words_until(PkgbuildWord(PkgbuildToken.ParenthesisClose, None)))
+
+                case PkgbuildWord(PkgbuildToken.BraceOpen, None):
+                    buffer += PkgbuildToken.BraceOpen
+                    buffer += b"".join(self._next_words_until(PkgbuildWord(PkgbuildToken.BraceClose, None)))
+
+                case PkgbuildWord(token, _):
+                    buffer += token
+
+        raise StopIteration
+
+    def _next_word(self) -> PkgbuildWord:
+        # pass SimpleNamespace as an argument to implement side effects
+        def generator(quote: SimpleNamespace) -> Generator[bytes, None, None]:
+            while token := self._io.read(1):
+                match token:
+                    case (PkgbuildToken.SingleQuote | PkgbuildToken.DoubleQuote) if quote.open is None:
+                        quote.open = token
+                    case closing_quote if closing_quote == quote.open:
+                        return
+                    case part:
+                        yield part
+                        if quote.open is None:
+                            return
+
+            if quote.open is not None:
+                raise ValueError("No closing quotation")
+
+        open_quote = SimpleNamespace(open=None)
+        value = b"".join(generator(open_quote))
+
+        return PkgbuildWord(value, open_quote.open)
+
+    def _next_words_until(self, ending: PkgbuildWord) -> Generator[bytes, None, None]:
+        braces = defaultdict(int)
+        while element := self._next_word():
+            yield element.original
+            match element:
+                case PkgbuildWord(token, None) if braces[token] > 0:
+                    braces[token] -= 1
+                case with_closure if (closing := with_closure.closing) is not None:
+                    braces[closing] += 1
+                case _ if element == ending:
+                    return
+
+        if any(brace for brace in braces.values() if brace > 0):
+            raise ValueError("Unclosed parenthesis and/or braces found")
+        raise ValueError(f"No matching ending element {ending.word} found")
+
+    def parse(self) -> Generator[PkgbuildPatch, None, None]:
+        """
+        parse source stream and yield parsed entries
+
+        Yields:
+            PkgbuildPatch: extracted a PKGBUILD node
+        """
+        yield from self
+
+    def __iter__(self) -> Self:
+        """
+        base iterator method
+
+        Returns:
+            Self: iterator instance
+        """
+        return self
+
+    def __next__(self) -> PkgbuildPatch:
+        key = self._next(declaration=True)
+        value = self._next(declaration=False)
+
+        return PkgbuildPatch(key.decode(encoding="utf8"), value.decode(encoding="utf8"))

src/ahriman/core/exceptions.py

@@ -95,19 +95,6 @@ class DuplicateRunError(RuntimeError):
             self, "Another application instance is run. This error can be suppressed by using --force flag.")
 
 
-class EncodeError(ValueError):
-    """
-    exception used for bytes encoding errors
-    """
-
-    def __init__(self, encodings: list[str]) -> None:
-        """
-        Args:
-            encodings(list[str]): list of encodings tried
-        """
-        ValueError.__init__(self, f"Could not encode bytes by using {encodings}")
-
-
 class ExitCode(RuntimeError):
     """
     special exception which has to be thrown to return non-zero status without error message

src/ahriman/models/pkgbuild.py

@@ -24,7 +24,6 @@ from pathlib import Path
 from typing import Any, ClassVar, IO, Self
 
 from ahriman.core.alpm.pkgbuild_parser import PkgbuildParser, PkgbuildToken
-from ahriman.core.exceptions import EncodeError
 from ahriman.models.pkgbuild_patch import PkgbuildPatch
 
 
@@ -34,13 +33,13 @@ class Pkgbuild(Mapping[str, Any]):
     model and proxy for PKGBUILD properties
 
     Attributes:
-        DEFAULT_ENCODINGS(list[str]): (class attribute) list of encoding to be applied on the file content
+        DEFAULT_ENCODINGS(str): (class attribute) default encoding to be applied on the file content
         fields(dict[str, PkgbuildPatch]): PKGBUILD fields
     """
 
     fields: dict[str, PkgbuildPatch]
 
-    DEFAULT_ENCODINGS: ClassVar[list[str]] = ["utf8", "latin-1"]
+    DEFAULT_ENCODINGS: ClassVar[str] = "utf8"
 
     @property
     def variables(self) -> dict[str, str]:
@@ -58,13 +57,13 @@ class Pkgbuild(Mapping[str, Any]):
         }
 
     @classmethod
-    def from_file(cls, path: Path, encodings: list[str] | None = None) -> Self:
+    def from_file(cls, path: Path, encoding: str | None = None) -> Self:
         """
         parse PKGBUILD from the file
 
         Args:
             path(Path): path to the PKGBUILD file
-            encodings(list[str] | None, optional): the encoding of the file (Default value = None)
+            encoding(str | None, optional): the encoding of the file (Default value = None)
 
         Returns:
             Self: constructed instance of self
@@ -77,15 +76,10 @@ class Pkgbuild(Mapping[str, Any]):
             content = input_file.read()
 
         # decode bytes content based on either
-        encodings = encodings or cls.DEFAULT_ENCODINGS
-        for encoding in encodings:
-            try:
-                io = StringIO(content.decode(encoding))
-                return cls.from_io(io)
-            except ValueError:
-                pass
+        encoding = encoding or cls.DEFAULT_ENCODINGS
+        io = StringIO(content.decode(encoding, errors="backslashreplace"))
 
-        raise EncodeError(encodings)
+        return cls.from_io(io)
 
     @classmethod
     def from_io(cls, stream: IO[str]) -> Self:

tests/ahriman/models/test_pkgbuild.py

@@ -3,9 +3,7 @@ import pytest
 from io import BytesIO, StringIO
 from pathlib import Path
 from pytest_mock import MockerFixture
-from unittest.mock import MagicMock
 
-from ahriman.core.exceptions import EncodeError
 from ahriman.models.pkgbuild import Pkgbuild
 from ahriman.models.pkgbuild_patch import PkgbuildPatch
 
@@ -46,18 +44,6 @@ def test_from_file_latin(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> N
     load_mock.assert_called_once_with(pytest.helpers.anyvar(int))
 
 
-def test_from_file_unknown_encoding(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
-    """
-    must raise exception when encoding is unknown
-    """
-    open_mock = mocker.patch("pathlib.Path.open")
-    io_mock = open_mock.return_value.__enter__.return_value = MagicMock()
-    io_mock.read.return_value.decode.side_effect = EncodeError(pkgbuild_ahriman.DEFAULT_ENCODINGS)
-
-    with pytest.raises(EncodeError):
-        assert Pkgbuild.from_file(Path("local"))
-
-
 def test_from_io(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
     """
     must correctly load from io

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 envlist = check, tests
-isolated_build = True
+isolated_build = true
 labels =
 	release = version, docs, publish
 dependencies = -e .[journald,pacman,s3,shell,stats,validator,web]
@@ -27,7 +27,9 @@ description = Run common checks like linter, mypy, etc
 deps =
 	{[tox]dependencies}
 	-e .[check]
+pip_pre = true
 setenv =
+	CFLAGS="-Wno-unterminated-string-initialization"
 	MYPYPATH=src
 commands =
 	autopep8 --exit-code --max-line-length 120 -aa -i -j 0 -r "src/{[tox]project_name}" "tests/{[tox]project_name}"
@@ -65,7 +67,7 @@ description = Generate html documentation
 deps =
 	{[tox]dependencies}
 	-e .[docs]
-recreate = True
+recreate = true
 commands =
 	sphinx-build -b html -a -j auto -W docs {envtmpdir}{/}html
 
@@ -89,6 +91,7 @@ description = Run tests
 deps =
 	{[tox]dependencies}
 	-e .[tests]
+pip_pre = true
 commands =
 	pytest {posargs}