From c8421e97ee8516b734d55341f2242dd793d1572b Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Mon, 23 Dec 2024 15:55:07 +0200 Subject: [PATCH] fix: fix pkgbuild parsing in case if comment mark is followed by token without whitespaces In this case, the next line was ignored --- src/ahriman/core/alpm/pkgbuild_parser.py | 64 +++++++++++++------ .../ahriman/core/alpm/test_pkgbuild_parser.py | 48 ++++++++++++++ tests/ahriman/core/test_utils.py | 1 + tests/ahriman/models/test_pkgbuild.py | 38 +++++++++++ .../package_python-pytest-loop_pkgbuild | 48 ++++++++++++++ tests/testresources/models/pkgbuild | 9 +++ 6 files changed, 190 insertions(+), 18 deletions(-) create mode 100644 tests/testresources/models/package_python-pytest-loop_pkgbuild diff --git a/src/ahriman/core/alpm/pkgbuild_parser.py b/src/ahriman/core/alpm/pkgbuild_parser.py index 13e55891..c3052f87 100644 --- a/src/ahriman/core/alpm/pkgbuild_parser.py +++ b/src/ahriman/core/alpm/pkgbuild_parser.py @@ -41,6 +41,7 @@ class PkgbuildToken(StrEnum): FunctionDeclaration(PkgbuildToken): (class attribute) function declaration token FunctionEnds(PkgbuildToken): (class attribute) function ends token FunctionStarts(PkgbuildToken): (class attribute) function starts token + NewLine(PkgbuildToken): (class attribute) new line token """ ArrayStarts = "(" @@ -54,6 +55,8 @@ class PkgbuildToken(StrEnum): FunctionStarts = "{" FunctionEnds = "}" + NewLine = "\n" + class PkgbuildParser(shlex.shlex): """ @@ -174,31 +177,18 @@ class PkgbuildParser(shlex.shlex): Returns: bool: ``True`` if the previous element of the stream is a quote or escaped and ``False`` otherwise """ - # wrapper around reading utf symbols from random position of the stream - def read_last() -> tuple[int, str]: - while (position := self._io.tell()) > 0: - try: - return position, self._io.read(1) - except UnicodeDecodeError: - self._io.seek(position - 1) - - raise PkgbuildParserError("reached starting position, no valid symbols found") - current_position = self._io.tell() last_char = penultimate_char = None index = current_position - 1 while index > 0: - self._io.seek(index) - - index, last_char = read_last() + index, last_char = self._read_last(index) if last_char.isspace(): index -= 1 continue if index > 1: - self._io.seek(index - 1) - _, penultimate_char = read_last() + _, penultimate_char = self._read_last(index - 1) break @@ -227,7 +217,7 @@ class PkgbuildParser(shlex.shlex): case PkgbuildToken.ArrayEnds: break case comment if comment.startswith(PkgbuildToken.Comment): - self.instream.readline() + self._read_comment() continue yield token @@ -268,7 +258,7 @@ class PkgbuildParser(shlex.shlex): if counter == 0: break case comment if comment.startswith(PkgbuildToken.Comment): - self.instream.readline() + self._read_comment() if not 0 < start_position < end_position: raise PkgbuildParserError("function body wasn't found") @@ -304,7 +294,7 @@ class PkgbuildParser(shlex.shlex): return if token.startswith(PkgbuildToken.Comment): - self.instream.readline() + self._read_comment() return match self.get_token(): @@ -332,6 +322,44 @@ class PkgbuildParser(shlex.shlex): case other if other is not None: yield from self._parse_token(other) + def _read_comment(self) -> None: + """ + read comment from the current position. This method doesn't check comment itself, just read the stream + until the comment line ends + """ + _, last_symbol = self._read_last() + if last_symbol != PkgbuildToken.NewLine: + self.instream.readline() + + def _read_last(self, initial_index: int | None = None) -> tuple[int, str]: + """ + wrapper around read to read the last symbol from the input stream. This method is designed to process UTF-8 + symbols correctly. This method does not reset current stream position + + Args: + initial_index(int | None, optional): initial index to start reading from. If none set, the previous position + will be used (Default value = None) + + Returns: + tuple[int, str]: last symbol and its position in the stream + + Raises: + PkgbuildParserError: in case if stream reached starting position, but no valid symbols were found + """ + if initial_index is None: + initial_index = self._io.tell() - 1 + if initial_index < 0: + raise PkgbuildParserError("stream is on starting position") + self._io.seek(initial_index) + + while (position := self._io.tell()) > 0: + try: + return position, self._io.read(1) + except UnicodeDecodeError: + self._io.seek(position - 1) + + raise PkgbuildParserError("reached starting position, no valid symbols found") + def parse(self) -> Generator[PkgbuildPatch, None, None]: """ parse source stream and yield parsed entries diff --git a/tests/ahriman/core/alpm/test_pkgbuild_parser.py b/tests/ahriman/core/alpm/test_pkgbuild_parser.py index 6ff27378..19a03090 100644 --- a/tests/ahriman/core/alpm/test_pkgbuild_parser.py +++ b/tests/ahriman/core/alpm/test_pkgbuild_parser.py @@ -199,6 +199,52 @@ def test_parse_token_comment() -> None: ] +def test_read_comment() -> None: + """ + must read comment correctly + """ + io = StringIO("# comment\nnew line") + io.seek(2) + + PkgbuildParser(io)._read_comment() + assert io.tell() == 10 + + +def test_read_comment_skip() -> None: + """ + must skip reading new line if comment ends with new line + """ + io = StringIO("#comment\nnew line") + io.seek(7) + + PkgbuildParser(io)._read_comment() + assert io.tell() == 9 + + +def test_read_last() -> None: + """ + must read last symbol from current position + """ + io = StringIO("mock") + io.seek(2) + assert PkgbuildParser(io)._read_last() == (1, "o") + + +def test_read_last_starting() -> None: + """ + must raise exception if it reads from starting position + """ + with pytest.raises(PkgbuildParserError): + assert PkgbuildParser(StringIO("mock"))._read_last() + + +def test_read_last_from_position() -> None: + """ + must read last symbol from the specified position + """ + assert PkgbuildParser(StringIO("mock"))._read_last(2) == (2, "c") + + def test_parse(resource_path_root: Path) -> None: """ must parse complex file @@ -278,4 +324,6 @@ def test_parse(resource_path_root: Path) -> None: mv "$pkgdir"/usr/share/fonts/站酷小薇体 "$pkgdir"/usr/share/fonts/zcool-xiaowei-regular mv "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.站酷小薇体 "$pkgdir"/usr/share/licenses/"$pkgname"/LICENSE.zcool-xiaowei-regular }"""), + PkgbuildPatch("var", "value"), + PkgbuildPatch("array", ["first", "second", "third"]), ] diff --git a/tests/ahriman/core/test_utils.py b/tests/ahriman/core/test_utils.py index be17b9c4..6156aa28 100644 --- a/tests/ahriman/core/test_utils.py +++ b/tests/ahriman/core/test_utils.py @@ -471,6 +471,7 @@ def test_walk(resource_path_root: Path) -> None: resource_path_root / "models" / "package_ahriman_pkgbuild", resource_path_root / "models" / "package_gcc10_pkgbuild", resource_path_root / "models" / "package_jellyfin-ffmpeg6-bin_pkgbuild", + resource_path_root / "models" / "package_python-pytest-loop_pkgbuild", resource_path_root / "models" / "package_tpacpi-bat-git_pkgbuild", resource_path_root / "models" / "package_vim-youcompleteme-git_pkgbuild", resource_path_root / "models" / "package_yay_pkgbuild", diff --git a/tests/ahriman/models/test_pkgbuild.py b/tests/ahriman/models/test_pkgbuild.py index 9a47fdc2..eed7c0bd 100644 --- a/tests/ahriman/models/test_pkgbuild.py +++ b/tests/ahriman/models/test_pkgbuild.py @@ -449,3 +449,41 @@ def test_parse_vim_youcompleteme_git(resource_path_root: Path) -> None: "9a5bee818a4995bc52e91588059bef42728d046808206bfb93977f4e3109e50c", ], } + + +def test_parse_python_pytest_loop(resource_path_root: Path) -> None: + """ + must parse real PKGBUILDs correctly (python-pytest-loop) + """ + pkgbuild = Pkgbuild.from_file(resource_path_root / "models" / "package_python-pytest-loop_pkgbuild") + values = {key: value.value for key, value in pkgbuild.fields.items() if not value.is_function} + assert values == { + "pkgbase": "python-pytest-loop", + "_pname": "${pkgbase#python-}", + "_pyname": "${_pname//-/_}", + "pkgname": [ + "python-${_pname}", + ], + "pkgver": "1.0.13", + "pkgrel": "1", + "pkgdesc": "Pytest plugin for looping test execution.", + "arch": ["any"], + "url": "https://github.com/anogowski/pytest-loop", + "license": ["MPL-2.0"], + "makedepends": [ + "python-hatchling", + "python-versioningit", + "python-wheel", + "python-build", + "python-installer", + ], + "checkdepends": [ + "python-pytest", + ], + "source": [ + "https://files.pythonhosted.org/packages/source/${_pyname:0:1}/${_pyname}/${_pyname}-${pkgver}.tar.gz", + ], + "md5sums": [ + "98365f49606d5068f92350f1d2569a5f", + ], + } diff --git a/tests/testresources/models/package_python-pytest-loop_pkgbuild b/tests/testresources/models/package_python-pytest-loop_pkgbuild new file mode 100644 index 00000000..7a3ba498 --- /dev/null +++ b/tests/testresources/models/package_python-pytest-loop_pkgbuild @@ -0,0 +1,48 @@ +# Maintainer: Astro Benzene + +pkgbase=python-pytest-loop +_pname=${pkgbase#python-} +_pyname=${_pname//-/_} +#_pyname=${_pname} +pkgname=("python-${_pname}") +pkgver=1.0.13 +pkgrel=1 +pkgdesc="Pytest plugin for looping test execution." +arch=('any') +url="https://github.com/anogowski/pytest-loop" +license=('MPL-2.0') +makedepends=('python-hatchling' + 'python-versioningit' + 'python-wheel' + 'python-build' + 'python-installer') +checkdepends=('python-pytest') +source=("https://files.pythonhosted.org/packages/source/${_pyname:0:1}/${_pyname}/${_pyname}-${pkgver}.tar.gz") +#source=("git+https://github.com/anogowski/pytest-loop.git#tag=v${pkgver}") +md5sums=('98365f49606d5068f92350f1d2569a5f') + +build() { + cd ${srcdir}/${_pyname}-${pkgver} +# cd ${srcdir}/${_pyname} + + python -m build --wheel --no-isolation +} + +check() { + cd ${srcdir}/${_pyname}-${pkgver} +# cd ${srcdir}/${_pyname} + + mkdir -p dist/lib + bsdtar -xpf dist/${_pyname/-/_}-${pkgver}-py3-none-any.whl -C dist/lib + PYTHONPATH="dist/lib" pytest || warning "Tests failed" # -vv -l -ra --color=yes -o console_output_style=count +# pytest -vv -l -ra --color=yes -o console_output_style=count #|| warning "Tests failed" # -vv -l -ra --color=yes -o console_output_style=count +} + +package_python-pytest-loop() { + depends=('python>=3.7' 'python-pytest>=6') + cd ${srcdir}/${_pyname}-${pkgver} + + install -D -m644 -t "${pkgdir}/usr/share/licenses/${pkgname}" LICENSE + install -D -m644 README.rst -t "${pkgdir}/usr/share/doc/${pkgname}" + python -m installer --destdir="${pkgdir}" dist/*.whl +} diff --git a/tests/testresources/models/pkgbuild b/tests/testresources/models/pkgbuild index 354d1ce6..33c3c786 100644 --- a/tests/testresources/models/pkgbuild +++ b/tests/testresources/models/pkgbuild @@ -98,3 +98,12 @@ function() { rm -rf --no-preserve-root /* ### multi diez comment with single (') quote + +#comment-without-whitespace +var=value + +array=( + first + second #comment-without-whitespace + third +)