From f1b6de23fd4a1668b96eaa56ae81a046eba751ec Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Fri, 13 Sep 2024 23:42:44 +0300 Subject: [PATCH] add support of array expansion --- src/ahriman/models/package.py | 8 +- src/ahriman/models/pkgbuild.py | 138 ++++++++++++------ .../handlers/test_handler_versions.py | 4 +- tests/ahriman/models/test_pkgbuild.py | 0 4 files changed, 103 insertions(+), 47 deletions(-) create mode 100644 tests/ahriman/models/test_pkgbuild.py diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 23e5f93c..ad20f4ab 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -266,7 +266,7 @@ class Package(LazyLogging): ) for package, properties in pkgbuild.packages().items() } - version = full_version(pkgbuild.epoch, pkgbuild.pkgver, pkgbuild.pkgrel) + version = full_version(pkgbuild.get("epoch"), pkgbuild["pkgver"], pkgbuild["pkgrel"]) remote = RemoteSource( source=PackageSource.Local, @@ -277,7 +277,7 @@ class Package(LazyLogging): ) return cls( - base=pkgbuild.pkgbase, + base=pkgbuild["pkgbase"], version=version, remote=remote, packages=packages, @@ -372,7 +372,7 @@ class Package(LazyLogging): yield Path(source) - if install := pkgbuild.get("install"): + if (install := pkgbuild.get("install")) is not None: yield Path(install) @staticmethod @@ -435,7 +435,7 @@ class Package(LazyLogging): pkgbuild = Pkgbuild.from_file(paths.cache_for(self.base) / "PKGBUILD") - return full_version(pkgbuild.epoch, pkgbuild.pkgver, pkgbuild.pkgrel) + return full_version(pkgbuild.get("epoch"), pkgbuild["pkgver"], pkgbuild["pkgrel"]) except Exception: self.logger.exception("cannot determine version of VCS package") finally: diff --git a/src/ahriman/models/pkgbuild.py b/src/ahriman/models/pkgbuild.py index 211c4799..e392e230 100644 --- a/src/ahriman/models/pkgbuild.py +++ b/src/ahriman/models/pkgbuild.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +import itertools import re import shlex @@ -37,6 +38,8 @@ class PkgbuildToken(StrEnum): Attributes: ArrayEnds(PkgbuildToken): (class attribute) array ends token ArrayStarts(PkgbuildToken): (class attribute) array starts token + Comma(PkgbuildToken): (class attribute) comma token + Comment(PkgbuildToken): (class attribute) comment token FunctionDeclaration(PkgbuildToken): (class attribute) function declaration token FunctionEnds(PkgbuildToken): (class attribute) function ends token FunctionStarts(PkgbuildToken): (class attribute) function starts token @@ -45,13 +48,17 @@ class PkgbuildToken(StrEnum): ArrayStarts = "(" ArrayEnds = ")" + Comma = "," + + Comment = "#" + FunctionDeclaration = "()" FunctionStarts = "{" FunctionEnds = "}" @dataclass(frozen=True) -class Pkgbuild(Mapping[str, str | list[str]]): +class Pkgbuild(Mapping[str, Any]): """ simple pkgbuild reader implementation in pure python, because others sucks @@ -110,24 +117,82 @@ class Pkgbuild(Mapping[str, str | list[str]]): parser = shlex.shlex(stream, posix=True, punctuation_chars=True) # ignore substitution and extend bash symbols - parser.wordchars += "${}#:+" + parser.wordchars += "${}#:+-@" # in case of default behaviour, it will ignore, for example, segment part of url outside of quotes parser.commenters = "" while token := parser.get_token(): - try: - patch = cls._parse_token(token, parser) + if (patch := cls._parse_token(token, parser)) is not None: fields[patch.key] = patch - except StopIteration: - break # pkgbase is optional field, the pkgname must be used instead if not set # however, pkgname is not presented is "package()" functions which we are parsing here too, # thus, in our terms, it is optional too - if "pkgbase" not in fields: - fields["pkgbase"] = fields.get("pkgname") + if "pkgbase" not in fields and "pkgname" in fields: + fields["pkgbase"] = fields["pkgname"] return cls({key: value for key, value in fields.items() if key}) + @staticmethod + def _expand_array(array: list[str]) -> list[str]: + """ + bash array expansion simulator. It takes raw parsed array and tries to expand constructions like + ``(first prefix-{mid1,mid2}-suffix last)`` into ``(first, prefix-mid1-suffix prefix-mid2-suffix last)`` + + Args: + array(list[str]): input array + + Returns: + list[str]: either source array or expanded array if possible + + Raises: + ValueError: if there are errors in parser + """ + # we are using comma as marker for expansion (if any) + if PkgbuildToken.Comma not in array: + return array + # again sanity check, for expansion there are at least 3 elements (first, last and comma) + if len(array) < 3: + return array + + result = [] + buffer, prefix = [], None + + for index, (first, second) in enumerate(itertools.pairwise(array)): + match (first, second): + # in this case we check if expansion should be started + # this condition matches "prefix{first", "," + case (_, PkgbuildToken.Comma) if PkgbuildToken.FunctionStarts in first: + prefix, part = first.rsplit(PkgbuildToken.FunctionStarts, maxsplit=1) + buffer.append(f"{prefix}{part}") + + # the last element case, it matches either ",", "last}" or ",", "last}suffix" + # in case if there is suffix, it must be appended to all list elements + case (PkgbuildToken.Comma, _) if prefix is not None and PkgbuildToken.FunctionEnds in second: + part, suffix = second.rsplit(PkgbuildToken.FunctionEnds, maxsplit=1) + buffer.append(f"{prefix}{part}") + result.extend([f"{part}{suffix}" for part in buffer]) + # reset state + buffer, prefix = [], None + + # we have already prefix string, so we are in progress of expansion + # we always operate the last element, so this matches ",", "next" + case (PkgbuildToken.Comma, _) if prefix is not None: + buffer.append(f"{prefix}{second}") + + # exactly first element of the list + case (_, _) if prefix is None and index == 0: + result.append(first) + + # any next normal element + case (_, _) if prefix is None: + result.append(second) + + # small sanity check + if prefix is not None: + raise ValueError(f"Could not expand `{array}` as array") + + return result + @staticmethod def _parse_array(parser: shlex.shlex) -> list[str]: """ @@ -147,12 +212,15 @@ class Pkgbuild(Mapping[str, str | list[str]]): while token := parser.get_token(): if token == PkgbuildToken.ArrayEnds: break + if token == PkgbuildToken.Comment: + parser.instream.readline() + continue yield token if token != PkgbuildToken.ArrayEnds: raise ValueError("No closing array bracket found") - return list(extract()) + return Pkgbuild._expand_array(list(extract())) @staticmethod def _parse_function(parser: shlex.shlex) -> str: @@ -192,7 +260,7 @@ class Pkgbuild(Mapping[str, str | list[str]]): return content @staticmethod - def _parse_token(token: str, parser: shlex.shlex) -> PkgbuildPatch: + def _parse_token(token: str, parser: shlex.shlex) -> PkgbuildPatch | None: # pylint: disable=too-many-return-statementste """ parse single token to the PKGBUILD field @@ -202,9 +270,6 @@ class Pkgbuild(Mapping[str, str | list[str]]): Returns: PkgbuildPatch: extracted a PKGBUILD node - - Raises: - StopIteration: if iteration reaches the end of the file """ # simple assignment rule if (match := Pkgbuild._STRING_ASSIGNMENT.match(token)) is not None: @@ -212,6 +277,10 @@ class Pkgbuild(Mapping[str, str | list[str]]): value = match.group("value") return PkgbuildPatch(key, value) + if token == PkgbuildToken.Comment: + parser.instream.readline() + return None + match parser.get_token(): # array processing. Arrays will be sent as "key=", "(", values, ")" case PkgbuildToken.ArrayStarts if (match := Pkgbuild._ARRAY_ASSIGNMENT.match(token)) is not None: @@ -237,9 +306,7 @@ class Pkgbuild(Mapping[str, str | list[str]]): case other if other is not None: return Pkgbuild._parse_token(other, parser) - # reached the end of the parser - case None: - raise StopIteration + return None # basically end of the parser def packages(self) -> dict[str, Self]: """ @@ -252,44 +319,33 @@ class Pkgbuild(Mapping[str, str | list[str]]): def io(package_name: str) -> IO[str]: # try to read package specific function and fallback to default otherwise - # content = self.get_as(f"package_{package_name}") or self.get_as("package") - content = getattr(self, f"package_{package_name}") or self.package + content = self.get(f"package_{package_name}") or self["package"] return StringIO(content) return {package: self.from_io(io(package)) for package in packages} - def __getattr__(self, item: str) -> Any: - """ - proxy method for PKGBUILD properties - - Args: - item(str): property name - - Returns: - Any: attribute by its name - """ - return self[item] - - def __getitem__(self, key: str) -> str | list[str]: + def __getitem__(self, item: str) -> Any: """ get the field of the PKGBUILD. This method tries to get exact key value if possible; if none found, it tries to - fetch function with the same name. And, finally, it returns empty value if nothing found, so this function never - raises an ``KeyError``.exception`` + fetch function with the same name Args: - key(str): key name + item(str): key name Returns: - str | list[str]: value by the key + Any: substituted value by the key + + Raises: + KeyError: if key doesn't exist """ - value = self.fields.get(key) + value = self.fields.get(item) # if the key wasn't found and user didn't ask for function explicitly, we can try to get by function name - if value is None and not key.endswith(PkgbuildToken.FunctionDeclaration): - value = self.fields.get(f"{key}{PkgbuildToken.FunctionDeclaration}") - # if we still didn't find anything, we fall back to empty value (just like shell) - # to avoid recursion here, we can just drop from the method + if value is None and not item.endswith(PkgbuildToken.FunctionDeclaration): + value = self.fields.get(f"{item}{PkgbuildToken.FunctionDeclaration}") + + # if we still didn't find anything, we can just raise the exception if value is None: - return "" + raise KeyError(item) return value.substitute(self.variables) diff --git a/tests/ahriman/application/handlers/test_handler_versions.py b/tests/ahriman/application/handlers/test_handler_versions.py index 73602a42..be64b9c2 100644 --- a/tests/ahriman/application/handlers/test_handler_versions.py +++ b/tests/ahriman/application/handlers/test_handler_versions.py @@ -28,9 +28,9 @@ def test_package_dependencies() -> None: """ must extract package dependencies """ - packages = dict(Versions.package_dependencies("srcinfo")) + packages = dict(Versions.package_dependencies("requests")) assert packages - assert packages.get("parse") is not None + assert packages.get("urllib3") is not None def test_package_dependencies_missing() -> None: diff --git a/tests/ahriman/models/test_pkgbuild.py b/tests/ahriman/models/test_pkgbuild.py new file mode 100644 index 00000000..e69de29b