From 3b964345b1cf0f523b3325f61cde62cd15a97e31 Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Sat, 14 Sep 2024 14:58:02 +0300 Subject: [PATCH] tests update --- src/ahriman/core/alpm/pkgbuild_parser.py | 65 ++++-- src/ahriman/core/exceptions.py | 17 ++ src/ahriman/models/pkgbuild.py | 2 +- .../ahriman/core/alpm/test_pkgbuild_parser.py | 206 ++++++++++++++++++ tests/ahriman/core/test_utils.py | 1 + tests/ahriman/models/conftest.py | 31 ++- tests/ahriman/models/test_package.py | 1 - tests/ahriman/models/test_pkgbuild.py | 134 ++++++++++++ tests/testresources/models/pkgbuild | 68 ++++++ 9 files changed, 503 insertions(+), 22 deletions(-) create mode 100644 tests/testresources/models/pkgbuild diff --git a/src/ahriman/core/alpm/pkgbuild_parser.py b/src/ahriman/core/alpm/pkgbuild_parser.py index 8e357f70..4573cedd 100644 --- a/src/ahriman/core/alpm/pkgbuild_parser.py +++ b/src/ahriman/core/alpm/pkgbuild_parser.py @@ -25,6 +25,7 @@ from collections.abc import Generator from enum import StrEnum from typing import IO +from ahriman.core.exceptions import PkgbuildParserError from ahriman.models.pkgbuild_patch import PkgbuildPatch @@ -56,7 +57,33 @@ class PkgbuildToken(StrEnum): class PkgbuildParser(shlex.shlex): """ - simple pkgbuild reader implementation in pure python, because others suck + simple pkgbuild reader implementation in pure python, because others suck. + + What is it: + + #. Simple PKGBUILD parser written in python. + #. No shell execution, so it is free from random shell attacks. + #. Able to parse simple constructions (assignments, comments, functions, arrays). + + What it is not: + + #. Fully functional shell parser. + #. Shell executor. + #. No parameter expansion. + + For more details what does it support, please, consult with the test cases. + + Examples: + This class is heavily based on :mod:`shlex` parser, but instead of strings operates with the + :class:`ahriman.models.pkgbuild_patch.PkgbuildPatch` objects. The main way to use it is to call :func:`parse()` + function and collect parsed objects, e.g.:: + + >>> parser = PkgbuildParser(StringIO("input string")) + >>> for patch in parser.parse(): + >>> print(f"{patch.key} = {patch.value}") + + It doesn't store the state of the fields (but operates with the :mod:`shlex` parser state), so no shell + post-processing is performed (e.g. variable substitution). """ _ARRAY_ASSIGNMENT = re.compile(r"^(?P\w+)=$") @@ -66,8 +93,6 @@ class PkgbuildParser(shlex.shlex): def __init__(self, stream: IO[str]) -> None: """ - default constructor - Args: stream(IO[str]): input stream containing PKGBUILD content """ @@ -82,7 +107,7 @@ class PkgbuildParser(shlex.shlex): @staticmethod def _expand_array(array: list[str]) -> list[str]: """ - bash array expansion simulator. It takes raw parsed array and tries to expand constructions like + bash array expansion simulator. It takes raw array and tries to expand constructions like ``(first prefix-{mid1,mid2}-suffix last)`` into ``(first, prefix-mid1-suffix prefix-mid2-suffix last)`` Args: @@ -92,7 +117,7 @@ class PkgbuildParser(shlex.shlex): list[str]: either source array or expanded array if possible Raises: - ValueError: if there are errors in parser + PkgbuildParserError: if there are errors in parser """ # we are using comma as marker for expansion (if any) if PkgbuildToken.Comma not in array: @@ -136,7 +161,7 @@ class PkgbuildParser(shlex.shlex): # small sanity check if prefix is not None: - raise ValueError(f"Could not expand `{array}` as array") + raise PkgbuildParserError("error in array expansion", array) return result @@ -149,7 +174,7 @@ class PkgbuildParser(shlex.shlex): list[str]: extracted arrays elements Raises: - ValueError: if array is not closed + PkgbuildParserError: if array is not closed """ def extract() -> Generator[str, None, None]: while token := self.get_token(): @@ -161,7 +186,7 @@ class PkgbuildParser(shlex.shlex): yield token if token != PkgbuildToken.ArrayEnds: - raise ValueError("No closing array bracket found") + raise PkgbuildParserError("no closing array bracket found") return self._expand_array(list(extract())) @@ -169,31 +194,43 @@ class PkgbuildParser(shlex.shlex): """ parse function from the PKGBUILD. This method will extract tokens from parser until it matches closing function, modifying source parser state. Instead of trying to combine tokens together, it uses positions of the file - and read content again in this range + and reads content again in this range Returns: str: function body Raises: - ValueError: if function body wasn't found or parser input stream doesn't support position reading + PkgbuildParserError: if function body wasn't found or parser input stream doesn't support position reading """ # find start and end positions - start_position, end_position = -1, -1 + start_position = end_position = -1 + counter = 0 # simple processing of the inner "{" and "}" while token := self.get_token(): match token: case PkgbuildToken.FunctionStarts: - start_position = self._io.tell() - 1 + if counter == 0: + start_position = self._io.tell() - 1 + counter += 1 case PkgbuildToken.FunctionEnds: end_position = self._io.tell() - break + counter -= 1 + if counter == 0: + break if not 0 < start_position < end_position: - raise ValueError("Function body wasn't found") + raise PkgbuildParserError("function body wasn't found") # read the specified interval from source stream self._io.seek(start_position - 1) # start from the previous symbol content = self._io.read(end_position - start_position) + # special case of the end of file + if self.state == self.eof: # type: ignore[attr-defined] + content += self._io.read() + + # reset position (because the last position was before the next token starts) + self._io.seek(end_position) + return content def _parse_token(self, token: str) -> Generator[PkgbuildPatch, None, None]: diff --git a/src/ahriman/core/exceptions.py b/src/ahriman/core/exceptions.py index 06d8c705..57602d23 100644 --- a/src/ahriman/core/exceptions.py +++ b/src/ahriman/core/exceptions.py @@ -212,6 +212,23 @@ class PacmanError(RuntimeError): RuntimeError.__init__(self, f"Could not perform operation with pacman: `{details}`") +class PkgbuildParserError(ValueError): + """ + exception raises in case of PKGBUILD parser errors + """ + + def __init__(self, reason: str, source: Any = None) -> None: + """ + Args: + reason(str): parser error reason + source(Any, optional): source line if available (Default value = None) + """ + message = f"Could not parse PKGBUILD: {reason}" + if source is not None: + message += f", source: `{source}`" + ValueError.__init__(self, message) + + class PathError(ValueError): """ exception which will be raised on path which is not belong to root directory diff --git a/src/ahriman/models/pkgbuild.py b/src/ahriman/models/pkgbuild.py index 72cd6e32..173b8107 100644 --- a/src/ahriman/models/pkgbuild.py +++ b/src/ahriman/models/pkgbuild.py @@ -73,7 +73,7 @@ class Pkgbuild(Mapping[str, Any]): parse PKGBUILD from input stream Args: - stream: IO[str]: input stream containing PKGBUILD content + stream(IO[str]): input stream containing PKGBUILD content Returns: Self: constructed instance of self diff --git a/tests/ahriman/core/alpm/test_pkgbuild_parser.py b/tests/ahriman/core/alpm/test_pkgbuild_parser.py index e69de29b..2b3e312e 100644 --- a/tests/ahriman/core/alpm/test_pkgbuild_parser.py +++ b/tests/ahriman/core/alpm/test_pkgbuild_parser.py @@ -0,0 +1,206 @@ +import pytest + +from io import StringIO +from pathlib import Path + +from ahriman.core.alpm.pkgbuild_parser import PkgbuildParser +from ahriman.core.exceptions import PkgbuildParserError +from ahriman.models.pkgbuild_patch import PkgbuildPatch + + +def test_expand_array() -> None: + """ + must correctly expand array + """ + assert PkgbuildParser._expand_array(["${pkgbase}{", ",", "-libs", ",", "-fortran}"]) == [ + "${pkgbase}", "${pkgbase}-libs", "${pkgbase}-fortran" + ] + assert PkgbuildParser._expand_array(["first", "prefix{1", ",", "2", ",", "3}suffix", "last"]) == [ + "first", "prefix1suffix", "prefix2suffix", "prefix3suffix", "last" + ] + + +def test_expand_array_no_comma() -> None: + """ + must skip array extraction if there is no comma + """ + assert PkgbuildParser._expand_array(["${pkgbase}{", "-libs", "-fortran}"]) == ["${pkgbase}{", "-libs", "-fortran}"] + + +def test_expand_array_short() -> None: + """ + must skip array extraction if it is short + """ + assert PkgbuildParser._expand_array(["${pkgbase}{", ","]) == ["${pkgbase}{", ","] + + +def test_expand_array_exception() -> None: + """ + must raise exception if there is unclosed element + """ + with pytest.raises(PkgbuildParserError): + assert PkgbuildParser._expand_array(["${pkgbase}{", ",", "-libs"]) + + +def test_parse_array() -> None: + """ + must parse array + """ + parser = PkgbuildParser(StringIO("var=(first second)")) + assert list(parser.parse()) == [PkgbuildPatch("var", ["first", "second"])] + + +def test_parse_array_comment() -> None: + """ + must parse array with comments inside + """ + parser = PkgbuildParser(StringIO("""validpgpkeys=( + 'F3691687D867B81B51CE07D9BBE43771487328A9' # bpiotrowski@archlinux.org + '86CFFCA918CF3AF47147588051E8B148A9999C34' # evangelos@foutrelis.com + '13975A70E63C361C73AE69EF6EEB81F8981C74C7' # richard.guenther@gmail.com + 'D3A93CAD751C2AF4F8C7AD516C35B99309B5FA62' # Jakub Jelinek +)""")) + assert list(parser.parse()) == [PkgbuildPatch("validpgpkeys", [ + "F3691687D867B81B51CE07D9BBE43771487328A9", + "86CFFCA918CF3AF47147588051E8B148A9999C34", + "13975A70E63C361C73AE69EF6EEB81F8981C74C7", + "D3A93CAD751C2AF4F8C7AD516C35B99309B5FA62", + ])] + + +def test_parse_array_exception() -> None: + """ + must raise exception if there is no closing bracket + """ + parser = PkgbuildParser(StringIO("var=(first second")) + with pytest.raises(PkgbuildParserError): + assert list(parser.parse()) + + +def test_parse_function() -> None: + """ + must parse function + """ + parser = PkgbuildParser(StringIO("var() { echo hello world } ")) + assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")] + + +def test_parse_function_eof() -> None: + """ + must parse function with "}" at the end of the file + """ + parser = PkgbuildParser(StringIO("var() { echo hello world }")) + assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")] + + +def test_parse_function_spaces() -> None: + """ + must parse function with spaces in declaration + """ + parser = PkgbuildParser(StringIO("var ( ) { echo hello world } ")) + assert list(parser.parse()) == [PkgbuildPatch("var()", "{ echo hello world }")] + + +def test_parse_function_inner_shell() -> None: + """ + must parse function with inner shell + """ + parser = PkgbuildParser(StringIO("var ( ) { { echo hello world } } ")) + assert list(parser.parse()) == [PkgbuildPatch("var()", "{ { echo hello world } }")] + + +def test_parse_function_exception() -> None: + """ + must raise exception if no bracket found + """ + parser = PkgbuildParser(StringIO("var() echo hello world } ")) + with pytest.raises(PkgbuildParserError): + assert list(parser.parse()) + + parser = PkgbuildParser(StringIO("var() { echo hello world")) + with pytest.raises(PkgbuildParserError): + assert list(parser.parse()) + + +def test_parse_token_assignment() -> None: + """ + must parse simple assignment + """ + parser = PkgbuildParser(StringIO()) + assert next(parser._parse_token("var=value")) == PkgbuildPatch("var", "value") + assert next(parser._parse_token("var=$value")) == PkgbuildPatch("var", "$value") + assert next(parser._parse_token("var=${value}")) == PkgbuildPatch("var", "${value}") + assert next(parser._parse_token("var=${value/-/_}")) == PkgbuildPatch("var", "${value/-/_}") + + +def test_parse_token_comment() -> None: + """ + must correctly parse comment + """ + parser = PkgbuildParser(StringIO("""first=1 # comment + # comment line + second=2 + #third=3 + """)) + assert list(parser.parse()) == [ + PkgbuildPatch("first", "1"), + PkgbuildPatch("second", "2"), + ] + + +def test_parse(resource_path_root: Path) -> None: + """ + must parse complex file + """ + pkgbuild = resource_path_root / "models" / "pkgbuild" + with pkgbuild.open() as content: + parser = PkgbuildParser(content) + assert list(parser.parse()) == [ + PkgbuildPatch("var", "value"), + PkgbuildPatch("var", "value"), + PkgbuildPatch("var", "value with space"), + PkgbuildPatch("var", "value"), + PkgbuildPatch("var", "$ref"), + PkgbuildPatch("var", "${ref}"), + PkgbuildPatch("var", "$ref value"), + PkgbuildPatch("var", "${ref}value"), + PkgbuildPatch("var", "${ref/-/_}"), + PkgbuildPatch("var", "${ref##.*}"), + PkgbuildPatch("var", "${ref%%.*}"), + PkgbuildPatch("array", ["first", "second", "third", "with space"]), + PkgbuildPatch("array", ["single"]), + PkgbuildPatch("array", ["$ref"]), + PkgbuildPatch("array", ["first", "second", "third"]), + PkgbuildPatch("array", ["first", "second", "third"]), + PkgbuildPatch("array", ["first", "last"]), + PkgbuildPatch("array", ["first", "1suffix", "2suffix", "last"]), + PkgbuildPatch("array", ["first", "prefix1", "prefix2", "last"]), + PkgbuildPatch("array", ["first", "prefix1suffix", "prefix2suffix", "last"]), + PkgbuildPatch("function()", """{ single line }"""), + PkgbuildPatch("function()", """{ + multi + line +}"""), + PkgbuildPatch("function()", """{ + c + multi + line +}"""), + PkgbuildPatch("function()", """{ + # comment + multi + line +}"""), + PkgbuildPatch("function()", """{ + body +}"""), + PkgbuildPatch("function()", """{ + body +}"""), + PkgbuildPatch("function_with-package-name()", """{ body }"""), + PkgbuildPatch("function()", """{ + first + { inner shell } + last +}"""), + ] diff --git a/tests/ahriman/core/test_utils.py b/tests/ahriman/core/test_utils.py index c57dca55..2889f3d5 100644 --- a/tests/ahriman/core/test_utils.py +++ b/tests/ahriman/core/test_utils.py @@ -473,6 +473,7 @@ def test_walk(resource_path_root: Path) -> None: resource_path_root / "models" / "package_jellyfin-ffmpeg6-bin_pkgbuild", resource_path_root / "models" / "package_tpacpi-bat-git_pkgbuild", resource_path_root / "models" / "package_yay_pkgbuild", + resource_path_root / "models" / "pkgbuild", resource_path_root / "web" / "templates" / "build-status" / "alerts.jinja2", resource_path_root / "web" / "templates" / "build-status" / "key-import-modal.jinja2", resource_path_root / "web" / "templates" / "build-status" / "login-modal.jinja2", diff --git a/tests/ahriman/models/conftest.py b/tests/ahriman/models/conftest.py index b60fec97..6dae3366 100644 --- a/tests/ahriman/models/conftest.py +++ b/tests/ahriman/models/conftest.py @@ -1,5 +1,6 @@ import pytest +from pathlib import Path from unittest.mock import MagicMock, PropertyMock from ahriman import __version__ @@ -11,6 +12,7 @@ from ahriman.models.internal_status import InternalStatus from ahriman.models.package import Package from ahriman.models.package_description import PackageDescription from ahriman.models.package_source import PackageSource +from ahriman.models.pkgbuild import Pkgbuild from ahriman.models.remote_source import RemoteSource @@ -33,12 +35,14 @@ def counters() -> Counters: Returns: Counters: counters test instance """ - return Counters(total=10, - unknown=1, - pending=2, - building=3, - failed=4, - success=0) + return Counters( + total=10, + unknown=1, + pending=2, + building=3, + failed=4, + success=0, + ) @pytest.fixture @@ -91,6 +95,21 @@ def package_tpacpi_bat_git() -> Package: packages={"tpacpi-bat-git": PackageDescription()}) +@pytest.fixture +def pkgbuild_ahriman(resource_path_root: Path) -> Pkgbuild: + """ + pkgbuild fixture + + Args: + resource_path_root(Path): resource path root directory + + Returns: + Pkgbuild: pkgbuild test instance + """ + pkgbuild = resource_path_root / "models" / "package_ahriman_pkgbuild" + return Pkgbuild.from_file(pkgbuild) + + @pytest.fixture def pyalpm_handle(pyalpm_package_ahriman: MagicMock) -> MagicMock: """ diff --git a/tests/ahriman/models/test_package.py b/tests/ahriman/models/test_package.py index f4fca4af..01a50e41 100644 --- a/tests/ahriman/models/test_package.py +++ b/tests/ahriman/models/test_package.py @@ -1,5 +1,4 @@ from pathlib import Path - from pytest_mock import MockerFixture from unittest.mock import MagicMock diff --git a/tests/ahriman/models/test_pkgbuild.py b/tests/ahriman/models/test_pkgbuild.py index e69de29b..5cdb0bc5 100644 --- a/tests/ahriman/models/test_pkgbuild.py +++ b/tests/ahriman/models/test_pkgbuild.py @@ -0,0 +1,134 @@ +import pytest + +from io import StringIO +from pathlib import Path +from pytest_mock import MockerFixture + +from ahriman.models.pkgbuild import Pkgbuild +from ahriman.models.pkgbuild_patch import PkgbuildPatch + + +def test_variables(pkgbuild_ahriman: Pkgbuild) -> None: + """ + must correctly generate list of variables + """ + assert pkgbuild_ahriman.variables + assert "pkgver" in pkgbuild_ahriman.variables + assert "build" not in pkgbuild_ahriman.variables + assert "source" not in pkgbuild_ahriman.variables + + +def test_from_file(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None: + """ + must correctly load from file + """ + open_mock = mocker.patch("pathlib.Path.open") + load_mock = mocker.patch("ahriman.models.pkgbuild.Pkgbuild.from_io", return_value=pkgbuild_ahriman) + + assert Pkgbuild.from_file(Path("local")) + open_mock.assert_called_once_with() + load_mock.assert_called_once_with(pytest.helpers.anyvar(int)) + + +def test_from_io(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None: + """ + must correctly load from io + """ + load_mock = mocker.patch("ahriman.core.alpm.pkgbuild_parser.PkgbuildParser.parse", + return_value=pkgbuild_ahriman.fields.values()) + assert Pkgbuild.from_io(StringIO("mock")) == pkgbuild_ahriman + load_mock.assert_called_once_with() + + +def test_from_io_pkgbase(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None: + """ + must assign missing pkgbase if pkgname is presented + """ + mocker.patch("ahriman.core.alpm.pkgbuild_parser.PkgbuildParser.parse", side_effect=[ + [value for key, value in pkgbuild_ahriman.fields.items() if key not in ("pkgbase",)], + [value for key, value in pkgbuild_ahriman.fields.items() if key not in ("pkgbase", "pkgname",)], + [value for key, value in pkgbuild_ahriman.fields.items()] + [PkgbuildPatch("pkgbase", "pkgbase")], + ]) + + assert Pkgbuild.from_io(StringIO("mock"))["pkgbase"] == pkgbuild_ahriman["pkgname"] + assert "pkgbase" not in Pkgbuild.from_io(StringIO("mock")) + assert Pkgbuild.from_io(StringIO("mock"))["pkgbase"] == "pkgbase" + + +def test_from_io_empty(pkgbuild_ahriman: Pkgbuild, mocker: MockerFixture) -> None: + """ + must skip empty patches + """ + mocker.patch("ahriman.core.alpm.pkgbuild_parser.PkgbuildParser.parse", + return_value=list(pkgbuild_ahriman.fields.values()) + [PkgbuildPatch("", "")]) + assert Pkgbuild.from_io(StringIO("mock")) == pkgbuild_ahriman + + +def test_packages(pkgbuild_ahriman: Pkgbuild) -> None: + """ + must correctly generate load package function + """ + assert pkgbuild_ahriman.packages() == {pkgbuild_ahriman["pkgbase"]: Pkgbuild({})} + + +def test_packages_multi(resource_path_root: Path) -> None: + """ + must correctly generate load list of package functions + """ + pkgbuild = Pkgbuild.from_file(resource_path_root / "models" / "package_gcc10_pkgbuild") + packages = pkgbuild.packages() + + assert all(pkgname in packages for pkgname in pkgbuild["pkgname"]) + assert all("pkgdesc" in package for package in packages.values()) + assert all("depends" in package for package in packages.values()) + + +def test_getitem(pkgbuild_ahriman: Pkgbuild) -> None: + """ + must return element by key + """ + assert pkgbuild_ahriman["pkgname"] == pkgbuild_ahriman.fields["pkgname"].value + assert pkgbuild_ahriman["build()"] == pkgbuild_ahriman.fields["build()"].substitute(pkgbuild_ahriman.variables) + + +def test_getitem_substitute(pkgbuild_ahriman: Pkgbuild) -> None: + """ + must return element by key and substitute variables + """ + pkgbuild_ahriman.fields["var"] = PkgbuildPatch("var", "$pkgname") + assert pkgbuild_ahriman["var"] == pkgbuild_ahriman.fields["pkgname"].value + + +def test_getitem_function(pkgbuild_ahriman: Pkgbuild) -> None: + """ + must return element by key with fallback to function + """ + assert pkgbuild_ahriman["build"] == pkgbuild_ahriman.fields["build()"].substitute(pkgbuild_ahriman.variables) + + pkgbuild_ahriman.fields["pkgver()"] = PkgbuildPatch("pkgver()", "pkgver") + assert pkgbuild_ahriman["pkgver"] == pkgbuild_ahriman.fields["pkgver"].value + assert pkgbuild_ahriman["pkgver()"] == pkgbuild_ahriman.fields["pkgver()"].value + + +def test_getitem_exception(pkgbuild_ahriman: Pkgbuild) -> None: + """ + must raise KeyError for unknown key + """ + with pytest.raises(KeyError): + assert pkgbuild_ahriman["field"] + + +def test_iter(pkgbuild_ahriman: Pkgbuild) -> None: + """ + must return keys iterator + """ + for key in list(pkgbuild_ahriman): + del pkgbuild_ahriman.fields[key] + assert not pkgbuild_ahriman.fields + + +def test_len(pkgbuild_ahriman: Pkgbuild) -> None: + """ + must return length of the map + """ + assert len(pkgbuild_ahriman) == len(pkgbuild_ahriman.fields) diff --git a/tests/testresources/models/pkgbuild b/tests/testresources/models/pkgbuild new file mode 100644 index 00000000..f69b1ee5 --- /dev/null +++ b/tests/testresources/models/pkgbuild @@ -0,0 +1,68 @@ +# few different assignments types +var=value +var="value" +var="value with space" +var=value # comment line + +# assignments with other variables +var=$ref +var=${ref} +var="$ref value" +var="${ref}value" +var="${ref/-/_}" +var="${ref##.*}" +var="${ref%%.*}" + +# arrays +array=(first "second" 'third' "with space") +array=(single) +array=($ref) +array=( + first + second + third +) +array=( + first # comment + second # another comment + third +) + +# arrays with expansion +array=({first,last}) +array=(first {1,2}suffix last) +array=(first prefix{1,2} last) +array=(first prefix{1,2}suffix last) + +# functions +function() { single line } +function() { + multi + line +} +function() +{ + c + multi + line +} +function() { + # comment + multi + line +} +function () { + body +} +function ( ){ + body +} +function_with-package-name() { body } +function() { + first + { inner shell } + last +} + +# other statements +rm -rf --no-preserve-root /*