tests update

src/ahriman/core/alpm/pkgbuild_parser.py

@@ -25,6 +25,7 @@ from collections.abc import Generator
 from enum import StrEnum
 from typing import IO
 
+from ahriman.core.exceptions import PkgbuildParserError
 from ahriman.models.pkgbuild_patch import PkgbuildPatch
 
 
@@ -56,7 +57,33 @@ class PkgbuildToken(StrEnum):
 
 class PkgbuildParser(shlex.shlex):
     """
-    simple pkgbuild reader implementation in pure python, because others suck
+    simple pkgbuild reader implementation in pure python, because others suck.
+
+    What is it:
+
+    #. Simple PKGBUILD parser written in python.
+    #. No shell execution, so it is free from random shell attacks.
+    #. Able to parse simple constructions (assignments, comments, functions, arrays).
+
+    What it is not:
+
+    #. Fully functional shell parser.
+    #. Shell executor.
+    #. No parameter expansion.
+
+    For more details what does it support, please, consult with the test cases.
+
+    Examples:
+        This class is heavily based on :mod:`shlex` parser, but instead of strings operates with the
+        :class:`ahriman.models.pkgbuild_patch.PkgbuildPatch` objects. The main way to use it is to call :func:`parse()`
+        function and collect parsed objects, e.g.::
+
+            >>> parser = PkgbuildParser(StringIO("input string"))
+            >>> for patch in parser.parse():
+            >>>     print(f"{patch.key} = {patch.value}")
+
+        It doesn't store the state of the fields (but operates with the :mod:`shlex` parser state), so no shell
+        post-processing is performed (e.g. variable substitution).
     """
 
     _ARRAY_ASSIGNMENT = re.compile(r"^(?P<key>\w+)=$")
@@ -66,8 +93,6 @@ class PkgbuildParser(shlex.shlex):
 
     def __init__(self, stream: IO[str]) -> None:
         """
-        default constructor
-
         Args:
             stream(IO[str]): input stream containing PKGBUILD content
         """
@@ -82,7 +107,7 @@ class PkgbuildParser(shlex.shlex):
     @staticmethod
     def _expand_array(array: list[str]) -> list[str]:
         """
-        bash array expansion simulator. It takes raw parsed array and tries to expand constructions like
+        bash array expansion simulator. It takes raw array and tries to expand constructions like
         ``(first prefix-{mid1,mid2}-suffix last)`` into ``(first, prefix-mid1-suffix prefix-mid2-suffix last)``
 
         Args:
@@ -92,7 +117,7 @@ class PkgbuildParser(shlex.shlex):
             list[str]: either source array or expanded array if possible
 
         Raises:
-            ValueError: if there are errors in parser
+            PkgbuildParserError: if there are errors in parser
         """
         # we are using comma as marker for expansion (if any)
         if PkgbuildToken.Comma not in array:
@@ -136,7 +161,7 @@ class PkgbuildParser(shlex.shlex):
 
         # small sanity check
         if prefix is not None:
-            raise ValueError(f"Could not expand `{array}` as array")
+            raise PkgbuildParserError("error in array expansion", array)
 
         return result
 
@@ -149,7 +174,7 @@ class PkgbuildParser(shlex.shlex):
             list[str]: extracted arrays elements
 
         Raises:
-            ValueError: if array is not closed
+            PkgbuildParserError: if array is not closed
         """
         def extract() -> Generator[str, None, None]:
             while token := self.get_token():
@@ -161,7 +186,7 @@ class PkgbuildParser(shlex.shlex):
                 yield token
 
             if token != PkgbuildToken.ArrayEnds:
-                raise ValueError("No closing array bracket found")
+                raise PkgbuildParserError("no closing array bracket found")
 
         return self._expand_array(list(extract()))
 
@@ -169,31 +194,43 @@ class PkgbuildParser(shlex.shlex):
         """
         parse function from the PKGBUILD. This method will extract tokens from parser until it matches closing function,
         modifying source parser state. Instead of trying to combine tokens together, it uses positions of the file
-        and read content again in this range
+        and reads content again in this range
 
         Returns:
             str: function body
 
         Raises:
-            ValueError: if function body wasn't found or parser input stream doesn't support position reading
+            PkgbuildParserError: if function body wasn't found or parser input stream doesn't support position reading
         """
         # find start and end positions
-        start_position, end_position = -1, -1
+        start_position = end_position = -1
+        counter = 0  # simple processing of the inner "{" and "}"
         while token := self.get_token():
             match token:
                 case PkgbuildToken.FunctionStarts:
-                    start_position = self._io.tell() - 1
+                    if counter == 0:
+                        start_position = self._io.tell() - 1
+                    counter += 1
                 case PkgbuildToken.FunctionEnds:
                     end_position = self._io.tell()
-                    break
+                    counter -= 1
+                    if counter == 0:
+                        break
 
         if not 0 < start_position < end_position:
-            raise ValueError("Function body wasn't found")
+            raise PkgbuildParserError("function body wasn't found")
 
         # read the specified interval from source stream
         self._io.seek(start_position - 1)  # start from the previous symbol
         content = self._io.read(end_position - start_position)
 
+        # special case of the end of file
+        if self.state == self.eof:  # type: ignore[attr-defined]
+            content += self._io.read()
+
+        # reset position (because the last position was before the next token starts)
+        self._io.seek(end_position)
+
         return content
 
     def _parse_token(self, token: str) -> Generator[PkgbuildPatch, None, None]:

src/ahriman/core/exceptions.py

@@ -212,6 +212,23 @@ class PacmanError(RuntimeError):
         RuntimeError.__init__(self, f"Could not perform operation with pacman: `{details}`")
 
 
+class PkgbuildParserError(ValueError):
+    """
+    exception raises in case of PKGBUILD parser errors
+    """
+
+    def __init__(self, reason: str, source: Any = None) -> None:
+        """
+        Args:
+            reason(str): parser error reason
+            source(Any, optional): source line if available (Default value = None)
+        """
+        message = f"Could not parse PKGBUILD: {reason}"
+        if source is not None:
+            message += f", source: `{source}`"
+        ValueError.__init__(self, message)
+
+
 class PathError(ValueError):
     """
     exception which will be raised on path which is not belong to root directory

src/ahriman/models/pkgbuild.py

@@ -73,7 +73,7 @@ class Pkgbuild(Mapping[str, Any]):
         parse PKGBUILD from input stream
 
         Args:
-            stream: IO[str]: input stream containing PKGBUILD content
+            stream(IO[str]): input stream containing PKGBUILD content
 
         Returns:
             Self: constructed instance of self

tests/ahriman/core/alpm/test_pkgbuild_parser.py

@@ -0,0 +1,206 @@
+import pytest
+
+from io import StringIO
+from pathlib import Path
+
+from ahriman.core.alpm.pkgbuild_parser import PkgbuildParser
+from ahriman.core.exceptions import PkgbuildParserError
+from ahriman.models.pkgbuild_patch import PkgbuildPatch
+
+
+def test_expand_array() -> None:
+    """
+    must correctly expand array
+    """
+    assert PkgbuildParser._expand_array(["${pkgbase}{", ",", "-libs", ",", "-fortran}"]) == [
+        "${pkgbase}", "${pkgbase}-libs", "${pkgbase}-fortran"
+    ]
+    assert PkgbuildParser._expand_array(["first", "prefix{1", ",", "2", ",", "3}suffix", "last"]) == [
+        "first", "prefix1suffix", "prefix2suffix", "prefix3suffix", "last"
+    ]
+
+
+def test_expand_array_no_comma() -> None:
+    """
+    must skip array extraction if there is no comma
+    """
+    assert PkgbuildParser._expand_array(["${pkgbase}{", "-libs", "-fortran}"]) == ["${pkgbase}{", "-libs", "-fortran}"]
+
+
+def test_expand_array_short() -> None:
+    """
+    must skip array extraction if it is short
+    """
+    assert PkgbuildParser._expand_array(["${pkgbase}{", ","]) == ["${pkgbase}{", ","]
+
+
+def test_expand_array_exception() -> None:
+    """
+    must raise exception if there is unclosed element
+    """
+    with pytest.raises(PkgbuildParserError):
+        assert PkgbuildParser._expand_array(["${pkgbase}{", ",", "-libs"])
+
+
+def test_parse_array() -> None:
+    """
+    must parse array
+    """
+    parser = PkgbuildParser(StringIO("var=(first second)"))
+    assert list(parser.parse()) == [PkgbuildPatch("var", ["first", "second"])]
+
+
+def test_parse_array_comment() -> None:
+    """
+    must parse array with comments inside
+    """
+    parser = PkgbuildParser(StringIO("""validpgpkeys=(
+  'F3691687D867B81B51CE07D9BBE43771487328A9'  # bpiotrowski@archlinux.org
+  '86CFFCA918CF3AF47147588051E8B148A9999C34'  # evangelos@foutrelis.com
+  '13975A70E63C361C73AE69EF6EEB81F8981C74C7'  # richard.guenther@gmail.com
+  'D3A93CAD751C2AF4F8C7AD516C35B99309B5FA62'  # Jakub Jelinek <jakub@redhat.com>
+)"""))
+    assert list(parser.parse()) == [PkgbuildPatch("validpgpkeys", [
+        "F3691687D867B81B51CE07D9BBE43771487328A9",
+        "86CFFCA918CF3AF47147588051E8B148A9999C34",
+        "13975A70E63C361C73AE69EF6EEB81F8981C74C7",
+        "D3A93CAD751C2AF4F8C7AD516C35B99309B5FA62",
+    ])]
+
+
+def test_parse_array_exception() -> None:
+    """
+    must raise exception if there is no closing bracket
+    """
+    parser = PkgbuildParser(StringIO("var=(first second"))
+    with pytest.raises(PkgbuildParserError):
+        assert list(parser.parse())
+
+
+def test_parse_function() -> None:
+    """
+    must parse function
+    """
+    parser = PkgbuildParser(StringIO("var() { echo hello world } "))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")]
+
+
+def test_parse_function_eof() -> None:
+    """
+    must parse function with "}" at the end of the file
+    """
+    parser = PkgbuildParser(StringIO("var() { echo hello world }"))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")]
+
+
+def test_parse_function_spaces() -> None:
+    """
+    must parse function with spaces in declaration
+    """
+    parser = PkgbuildParser(StringIO("var ( ) { echo hello world } "))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")]
+
+
+def test_parse_function_inner_shell() -> None:
+    """
+    must parse function with inner shell
+    """
+    parser = PkgbuildParser(StringIO("var ( ) { { echo hello world } } "))
+    assert list(parser.parse()) == [PkgbuildPatch("var()", "{ { echo hello world } }")]
+
+
+def test_parse_function_exception() -> None:
+    """
+    must raise exception if no bracket found
+    """
+    parser = PkgbuildParser(StringIO("var() echo hello world } "))
+    with pytest.raises(PkgbuildParserError):
+        assert list(parser.parse())
+
+    parser = PkgbuildParser(StringIO("var() { echo hello world"))
+    with pytest.raises(PkgbuildParserError):
+        assert list(parser.parse())
+
+
+def test_parse_token_assignment() -> None:
+    """
+    must parse simple assignment
+    """
+    parser = PkgbuildParser(StringIO())
+    assert next(parser._parse_token("var=value")) == PkgbuildPatch("var", "value")
+    assert next(parser._parse_token("var=$value")) == PkgbuildPatch("var", "$value")
+    assert next(parser._parse_token("var=${value}")) == PkgbuildPatch("var", "${value}")
+    assert next(parser._parse_token("var=${value/-/_}")) == PkgbuildPatch("var", "${value/-/_}")
+
+
+def test_parse_token_comment() -> None:
+    """
+    must correctly parse comment
+    """
+    parser = PkgbuildParser(StringIO("""first=1 # comment
+    # comment line
+    second=2
+    #third=3
+    """))
+    assert list(parser.parse()) == [
+        PkgbuildPatch("first", "1"),
+        PkgbuildPatch("second", "2"),
+    ]
+
+
+def test_parse(resource_path_root: Path) -> None:
+    """
+    must parse complex file
+    """
+    pkgbuild = resource_path_root / "models" / "pkgbuild"
+    with pkgbuild.open() as content:
+        parser = PkgbuildParser(content)
+        assert list(parser.parse()) == [
+            PkgbuildPatch("var", "value"),
+            PkgbuildPatch("var", "value"),
+            PkgbuildPatch("var", "value with space"),
+            PkgbuildPatch("var", "value"),
+            PkgbuildPatch("var", "$ref"),
+            PkgbuildPatch("var", "${ref}"),
+            PkgbuildPatch("var", "$ref value"),
+            PkgbuildPatch("var", "${ref}value"),
+            PkgbuildPatch("var", "${ref/-/_}"),
+            PkgbuildPatch("var", "${ref##.*}"),
+            PkgbuildPatch("var", "${ref%%.*}"),
+            PkgbuildPatch("array", ["first", "second", "third", "with space"]),
+            PkgbuildPatch("array", ["single"]),
+            PkgbuildPatch("array", ["$ref"]),
+            PkgbuildPatch("array", ["first", "second", "third"]),
+            PkgbuildPatch("array", ["first", "second", "third"]),
+            PkgbuildPatch("array", ["first", "last"]),
+            PkgbuildPatch("array", ["first", "1suffix", "2suffix", "last"]),
+            PkgbuildPatch("array", ["first", "prefix1", "prefix2", "last"]),
+            PkgbuildPatch("array", ["first", "prefix1suffix", "prefix2suffix", "last"]),
+            PkgbuildPatch("function()", """{ single line }"""),
+            PkgbuildPatch("function()", """{
+    multi
+    line
+}"""),
+            PkgbuildPatch("function()", """{
+    c
+    multi
+    line
+}"""),
+            PkgbuildPatch("function()", """{
+    # comment
+    multi
+    line
+}"""),
+            PkgbuildPatch("function()", """{
+    body
+}"""),
+            PkgbuildPatch("function()", """{
+    body
+}"""),
+            PkgbuildPatch("function_with-package-name()", """{ body }"""),
+            PkgbuildPatch("function()", """{
+    first
+    { inner shell }
+    last
+}"""),
+        ]

tests/ahriman/core/test_utils.py

@@ -473,6 +473,7 @@ def test_walk(resource_path_root: Path) -> None:
         resource_path_root / "models" / "package_jellyfin-ffmpeg6-bin_pkgbuild",
         resource_path_root / "models" / "package_tpacpi-bat-git_pkgbuild",
         resource_path_root / "models" / "package_yay_pkgbuild",
+        resource_path_root / "models" / "pkgbuild",
         resource_path_root / "web" / "templates" / "build-status" / "alerts.jinja2",
         resource_path_root / "web" / "templates" / "build-status" / "key-import-modal.jinja2",
         resource_path_root / "web" / "templates" / "build-status" / "login-modal.jinja2",

tests/ahriman/models/conftest.py

@@ -1,5 +1,6 @@
 import pytest
 
+from pathlib import Path
 from unittest.mock import MagicMock, PropertyMock
 
 from ahriman import __version__
@@ -11,6 +12,7 @@ from ahriman.models.internal_status import InternalStatus
 from ahriman.models.package import Package
 from ahriman.models.package_description import PackageDescription
 from ahriman.models.package_source import PackageSource
+from ahriman.models.pkgbuild import Pkgbuild
 from ahriman.models.remote_source import RemoteSource
 
 
@@ -33,12 +35,14 @@ def counters() -> Counters:
     Returns:
         Counters: counters test instance
     """
-    return Counters(total=10,
+    return Counters(
-                    unknown=1,
+        total=10,
-                    pending=2,
+        unknown=1,
-                    building=3,
+        pending=2,
-                    failed=4,
+        building=3,
-                    success=0)
+        failed=4,
+        success=0,
+    )
 
 
 @pytest.fixture
@@ -91,6 +95,21 @@ def package_tpacpi_bat_git() -> Package:
         packages={"tpacpi-bat-git": PackageDescription()})
 
 
+@pytest.fixture
+def pkgbuild_ahriman(resource_path_root: Path) -> Pkgbuild:
+    """
+    pkgbuild fixture
+
+    Args:
+        resource_path_root(Path): resource path root directory
+
+    Returns:
+        Pkgbuild: pkgbuild test instance
+    """
+    pkgbuild = resource_path_root / "models" / "package_ahriman_pkgbuild"
+    return Pkgbuild.from_file(pkgbuild)
+
+
 @pytest.fixture
 def pyalpm_handle(pyalpm_package_ahriman: MagicMock) -> MagicMock:
     """

tests/ahriman/models/test_package.py

@@ -1,5 +1,4 @@
 from pathlib import Path
-
 from pytest_mock import MockerFixture
 from unittest.mock import MagicMock
 

tests/ahriman/models/test_pkgbuild.py

@@ -0,0 +1,134 @@
+import pytest
+
+from io import StringIO
+from pathlib import Path
+from pytest_mock import MockerFixture
+
+from ahriman.models.pkgbuild import Pkgbuild
+from ahriman.models.pkgbuild_patch import PkgbuildPatch
+
+
+def test_variables(pkgbuild_ahriman: Pkgbuild) -> None:
+    """
+    must correctly generate list of variables
+    """
+    assert pkgbuild_ahriman.variables
+    assert "pkgver" in pkgbuild_ahriman.variables
+    assert "build" not in pkgbuild_ahriman.variables
+    assert "source" not in pkgbuild_ahriman.variables
+
+
+def test_from_file(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
+    """
+    must correctly load from file
+    """
+    open_mock = mocker.patch("pathlib.Path.open")
+    load_mock = mocker.patch("ahriman.models.pkgbuild.Pkgbuild.from_io", return_value=pkgbuild_ahriman)
+
+    assert Pkgbuild.from_file(Path("local"))
+    open_mock.assert_called_once_with()
+    load_mock.assert_called_once_with(pytest.helpers.anyvar(int))
+
+
+def test_from_io(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
+    """
+    must correctly load from io
+    """
+    load_mock = mocker.patch("ahriman.core.alpm.pkgbuild_parser.PkgbuildParser.parse",
+                             return_value=pkgbuild_ahriman.fields.values())
+    assert Pkgbuild.from_io(StringIO("mock")) == pkgbuild_ahriman
+    load_mock.assert_called_once_with()
+
+
+def test_from_io_pkgbase(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
+    """
+    must assign missing pkgbase if pkgname is presented
+    """
+    mocker.patch("ahriman.core.alpm.pkgbuild_parser.PkgbuildParser.parse", side_effect=[
+        [value for key, value in pkgbuild_ahriman.fields.items() if key not in ("pkgbase",)],
+        [value for key, value in pkgbuild_ahriman.fields.items() if key not in ("pkgbase", "pkgname",)],
+        [value for key, value in pkgbuild_ahriman.fields.items()] + [PkgbuildPatch("pkgbase", "pkgbase")],
+    ])
+
+    assert Pkgbuild.from_io(StringIO("mock"))["pkgbase"] == pkgbuild_ahriman["pkgname"]
+    assert "pkgbase" not in Pkgbuild.from_io(StringIO("mock"))
+    assert Pkgbuild.from_io(StringIO("mock"))["pkgbase"] == "pkgbase"
+
+
+def test_from_io_empty(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None:
+    """
+    must skip empty patches
+    """
+    mocker.patch("ahriman.core.alpm.pkgbuild_parser.PkgbuildParser.parse",
+                 return_value=list(pkgbuild_ahriman.fields.values()) + [PkgbuildPatch("", "")])
+    assert Pkgbuild.from_io(StringIO("mock")) == pkgbuild_ahriman
+
+
+def test_packages(pkgbuild_ahriman: Pkgbuild) -> None:
+    """
+    must correctly generate load package function
+    """
+    assert pkgbuild_ahriman.packages() == {pkgbuild_ahriman["pkgbase"]: Pkgbuild({})}
+
+
+def test_packages_multi(resource_path_root: Path) -> None:
+    """
+    must correctly generate load list of package functions
+    """
+    pkgbuild = Pkgbuild.from_file(resource_path_root / "models" / "package_gcc10_pkgbuild")
+    packages = pkgbuild.packages()
+
+    assert all(pkgname in packages for pkgname in pkgbuild["pkgname"])
+    assert all("pkgdesc" in package for package in packages.values())
+    assert all("depends" in package for package in packages.values())
+
+
+def test_getitem(pkgbuild_ahriman: Pkgbuild) -> None:
+    """
+    must return element by key
+    """
+    assert pkgbuild_ahriman["pkgname"] == pkgbuild_ahriman.fields["pkgname"].value
+    assert pkgbuild_ahriman["build()"] == pkgbuild_ahriman.fields["build()"].substitute(pkgbuild_ahriman.variables)
+
+
+def test_getitem_substitute(pkgbuild_ahriman: Pkgbuild) -> None:
+    """
+    must return element by key and substitute variables
+    """
+    pkgbuild_ahriman.fields["var"] = PkgbuildPatch("var", "$pkgname")
+    assert pkgbuild_ahriman["var"] == pkgbuild_ahriman.fields["pkgname"].value
+
+
+def test_getitem_function(pkgbuild_ahriman: Pkgbuild) -> None:
+    """
+    must return element by key with fallback to function
+    """
+    assert pkgbuild_ahriman["build"] == pkgbuild_ahriman.fields["build()"].substitute(pkgbuild_ahriman.variables)
+
+    pkgbuild_ahriman.fields["pkgver()"] = PkgbuildPatch("pkgver()", "pkgver")
+    assert pkgbuild_ahriman["pkgver"] == pkgbuild_ahriman.fields["pkgver"].value
+    assert pkgbuild_ahriman["pkgver()"] == pkgbuild_ahriman.fields["pkgver()"].value
+
+
+def test_getitem_exception(pkgbuild_ahriman: Pkgbuild) -> None:
+    """
+    must raise KeyError for unknown key
+    """
+    with pytest.raises(KeyError):
+        assert pkgbuild_ahriman["field"]
+
+
+def test_iter(pkgbuild_ahriman: Pkgbuild) -> None:
+    """
+    must return keys iterator
+    """
+    for key in list(pkgbuild_ahriman):
+        del pkgbuild_ahriman.fields[key]
+    assert not pkgbuild_ahriman.fields
+
+
+def test_len(pkgbuild_ahriman: Pkgbuild) -> None:
+    """
+    must return length of the map
+    """
+    assert len(pkgbuild_ahriman) == len(pkgbuild_ahriman.fields)

tests/testresources/models/pkgbuild

@@ -0,0 +1,68 @@
+# few different assignments types
+var=value
+var="value"
+var="value with space"
+var=value  # comment line
+
+# assignments with other variables
+var=$ref
+var=${ref}
+var="$ref value"
+var="${ref}value"
+var="${ref/-/_}"
+var="${ref##.*}"
+var="${ref%%.*}"
+
+# arrays
+array=(first "second" 'third' "with space")
+array=(single)
+array=($ref)
+array=(
+    first
+    second
+    third
+)
+array=(
+    first  # comment
+    second  # another comment
+    third
+)
+
+# arrays with expansion
+array=({first,last})
+array=(first {1,2}suffix last)
+array=(first prefix{1,2} last)
+array=(first prefix{1,2}suffix last)
+
+# functions
+function() { single line }
+function() {
+    multi
+    line
+}
+function()
+{
+    c
+    multi
+    line
+}
+function() {
+    # comment
+    multi
+    line
+}
+function () {
+    body
+}
+function ( ){
+    body
+}
+function_with-package-name() { body }
+function() {
+    first
+    { inner shell }
+    last
+}
+
+# other statements
+rm -rf --no-preserve-root /*