diff --git a/docs/faq.rst b/docs/faq.rst index 2a2d0909..6caaad2e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -200,6 +200,14 @@ Alternatively you can create full-diff patches, which are calculated by using `` The last command will calculate diff from current tree to the ``HEAD`` and will store it locally. Patches will be applied on any package actions (e.g. it can be used for dependency management). +It is also possible to create simple patch during package addition, e.g.: + +.. code-block:: shell + + sudo -u ahriman ahriman package-add ahriman --variable PKGEXT=.pkg.tar.xz + +The ``--variable`` argument accepts variables in shell like format: quotation and lists are supported as usual, but functions are not. This feature is useful in particular in order to override specific makepkg variables during build. + How to build package from official repository ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 61f7298f..f29f1749 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -276,6 +276,7 @@ def _set_package_add_parser(root: SubParserAction) -> argparse.ArgumentParser: parser.add_argument("-s", "--source", help="explicitly specify the package source for this command", type=PackageSource, choices=enum_values(PackageSource), default=PackageSource.Auto) parser.add_argument("-u", "--username", help="build as user", default=extract_user()) + parser.add_argument("-v", "--variable", help="apply specified makepkg variables to the next build", action="append") parser.set_defaults(handler=handlers.Add) return parser diff --git a/src/ahriman/application/handlers/add.py b/src/ahriman/application/handlers/add.py index 2892f004..587fff9b 100644 --- a/src/ahriman/application/handlers/add.py +++ b/src/ahriman/application/handlers/add.py @@ -23,6 +23,7 @@ from ahriman.application.application import Application from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.models.packagers import Packagers +from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.repository_id import RepositoryId @@ -45,7 +46,12 @@ class Add(Handler): """ application = Application(repository_id, configuration, report=report, refresh_pacman_database=args.refresh) application.on_start() + application.add(args.package, args.source, args.username) + patches = [PkgbuildPatch.from_env(patch) for patch in args.variable] if args.variable is not None else [] + for package in args.package: # for each requested package insert patch + application.database.patches_insert(package, patches) + if not args.now: return diff --git a/src/ahriman/application/handlers/handler.py b/src/ahriman/application/handlers/handler.py index 928587b8..b01fb282 100644 --- a/src/ahriman/application/handlers/handler.py +++ b/src/ahriman/application/handlers/handler.py @@ -162,7 +162,7 @@ class Handler: if args.repository_id is not None: separator = "/" if "/" in args.repository_id else "-" # systemd and non-systemd identifiers # repository parts is optional for backward compatibility - architecture, *repository_parts = args.repository_id.split(separator) + architecture, *repository_parts = args.repository_id.split(separator) # maxsplit isn't used intentionally args.architecture = architecture if repository_parts: args.repository = "-".join(repository_parts) # replace slash with dash diff --git a/src/ahriman/application/handlers/patch.py b/src/ahriman/application/handlers/patch.py index 1cdba013..6e7240b0 100644 --- a/src/ahriman/application/handlers/patch.py +++ b/src/ahriman/application/handlers/patch.py @@ -115,7 +115,7 @@ class Patch(Handler): package_base(str): package base patch(PkgbuildPatch): patch descriptor """ - application.database.patches_insert(package_base, patch) + application.database.patches_insert(package_base, [patch]) @staticmethod def patch_set_list(application: Application, package_base: str | None, variables: list[str] | None, diff --git a/src/ahriman/core/database/operations/package_operations.py b/src/ahriman/core/database/operations/package_operations.py index dc684aa1..ac2c7ce1 100644 --- a/src/ahriman/core/database/operations/package_operations.py +++ b/src/ahriman/core/database/operations/package_operations.py @@ -33,8 +33,8 @@ class PackageOperations(Operations): package operations """ - def _package_remove_package_base(self, connection: Connection, package_base: str, - repository_id: RepositoryId) -> None: + @staticmethod + def _package_remove_package_base(connection: Connection, package_base: str, repository_id: RepositoryId) -> None: """ remove package base information @@ -50,8 +50,9 @@ class PackageOperations(Operations): """delete from package_bases where package_base = :package_base and repository = :repository""", {"package_base": package_base, "repository": repository_id.id}) - def _package_remove_packages(self, connection: Connection, package_base: str, - current_packages: Iterable[str], repository_id: RepositoryId) -> None: + @staticmethod + def _package_remove_packages(connection: Connection, package_base: str, current_packages: Iterable[str], + repository_id: RepositoryId) -> None: """ remove packages belong to the package base @@ -74,8 +75,8 @@ class PackageOperations(Operations): """delete from packages where package = :package and repository = :repository""", packages) - def _package_update_insert_base(self, connection: Connection, package: Package, - repository_id: RepositoryId) -> None: + @staticmethod + def _package_update_insert_base(connection: Connection, package: Package, repository_id: RepositoryId) -> None: """ insert base package into table @@ -107,8 +108,8 @@ class PackageOperations(Operations): } ) - def _package_update_insert_packages(self, connection: Connection, package: Package, - repository_id: RepositoryId) -> None: + @staticmethod + def _package_update_insert_packages(connection: Connection, package: Package, repository_id: RepositoryId) -> None: """ insert packages into table @@ -149,7 +150,8 @@ class PackageOperations(Operations): """, package_list) - def _package_update_insert_status(self, connection: Connection, package_base: str, status: BuildStatus, + @staticmethod + def _package_update_insert_status(connection: Connection, package_base: str, status: BuildStatus, repository_id: RepositoryId) -> None: """ insert base package status into table @@ -176,8 +178,8 @@ class PackageOperations(Operations): "repository": repository_id.id, }) - def _packages_get_select_package_bases(self, connection: Connection, - repository_id: RepositoryId) -> dict[str, Package]: + @staticmethod + def _packages_get_select_package_bases(connection: Connection, repository_id: RepositoryId) -> dict[str, Package]: """ select package bases from the table @@ -201,7 +203,8 @@ class PackageOperations(Operations): ) } - def _packages_get_select_packages(self, connection: Connection, packages: dict[str, Package], + @staticmethod + def _packages_get_select_packages(connection: Connection, packages: dict[str, Package], repository_id: RepositoryId) -> dict[str, Package]: """ select packages from the table @@ -223,8 +226,8 @@ class PackageOperations(Operations): packages[row["package_base"]].packages[row["package"]] = PackageDescription.from_json(row) return packages - def _packages_get_select_statuses(self, connection: Connection, - repository_id: RepositoryId) -> dict[str, BuildStatus]: + @staticmethod + def _packages_get_select_statuses(connection: Connection, repository_id: RepositoryId) -> dict[str, BuildStatus]: """ select package build statuses from the table diff --git a/src/ahriman/core/database/operations/patch_operations.py b/src/ahriman/core/database/operations/patch_operations.py index fd33fea8..48ce6317 100644 --- a/src/ahriman/core/database/operations/patch_operations.py +++ b/src/ahriman/core/database/operations/patch_operations.py @@ -18,7 +18,6 @@ # along with this program. If not, see . # from collections import defaultdict - from sqlite3 import Connection from ahriman.core.database.operations import Operations @@ -42,16 +41,16 @@ class PatchOperations(Operations): """ return self.patches_list(package_base, None).get(package_base, []) - def patches_insert(self, package_base: str, patch: PkgbuildPatch) -> None: + def patches_insert(self, package_base: str, patches: list[PkgbuildPatch]) -> None: """ insert or update patch in database Args: package_base(str): package base to insert - patch(PkgbuildPatch): patch content + patches(list[PkgbuildPatch]): patch content """ def run(connection: Connection) -> None: - connection.execute( + connection.executemany( """ insert into patches (package_base, variable, patch) @@ -60,7 +59,14 @@ class PatchOperations(Operations): on conflict (package_base, coalesce(variable, '')) do update set patch = :patch """, - {"package_base": package_base, "variable": patch.key, "patch": patch.value}) + [ + { + "package_base": package_base, + "variable": patch.key, + "patch": patch.value, + } for patch in patches + ] + ) return self.with_connection(run, commit=True) @@ -89,7 +95,7 @@ class PatchOperations(Operations): if variables is not None and patch.key not in variables: continue patches[package].append(patch) - return dict(patches) + return patches def patches_remove(self, package_base: str, variables: list[str] | None) -> None: """ @@ -102,12 +108,21 @@ class PatchOperations(Operations): def run_many(connection: Connection) -> None: patches = variables or [] # suppress mypy warning connection.executemany( - """delete from patches where package_base = :package_base and variable = :variable""", - [{"package_base": package_base, "variable": variable} for variable in patches]) + """ + delete from patches where package_base = :package_base and variable = :variable + """, + [ + { + "package_base": package_base, + "variable": variable, + } for variable in patches + ]) def run(connection: Connection) -> None: connection.execute( - """delete from patches where package_base = :package_base""", + """ + delete from patches where package_base = :package_base + """, {"package_base": package_base}) if variables is not None: diff --git a/src/ahriman/core/spawn.py b/src/ahriman/core/spawn.py index c126f490..48ea1d91 100644 --- a/src/ahriman/core/spawn.py +++ b/src/ahriman/core/spawn.py @@ -28,6 +28,7 @@ from multiprocessing import Process, Queue from threading import Lock, Thread from ahriman.core.log import LazyLogging +from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.process_status import ProcessStatus from ahriman.models.repository_id import RepositoryId @@ -96,7 +97,8 @@ class Spawn(Thread, LazyLogging): queue.put(ProcessStatus(process_id, result, consumed_time)) - def _spawn_process(self, repository_id: RepositoryId, command: str, *args: str, **kwargs: str | None) -> str: + def _spawn_process(self, repository_id: RepositoryId, command: str, *args: str, + **kwargs: str | list[str] | None) -> str: """ spawn external ahriman process with supplied arguments @@ -104,7 +106,7 @@ class Spawn(Thread, LazyLogging): repository_id(RepositoryId): repository unique identifier command(str): subcommand to run *args(str): positional command arguments - **kwargs(str): named command arguments + **kwargs(str | list[str] | None): named command arguments Returns: str: spawned process identifier @@ -118,9 +120,13 @@ class Spawn(Thread, LazyLogging): for argument, value in kwargs.items(): if value is None: continue # skip null values - arguments.append(f"--{argument}") - if value: - arguments.append(value) + flag = f"--{argument}" + if isinstance(value, list): + arguments.extend(list(sum(((flag, v) for v in value), ()))) + elif value: + arguments.extend([flag, value]) + else: + arguments.append(flag) # boolean argument process_id = str(uuid.uuid4()) self.logger.info("full command line arguments of %s are %s using repository %s", @@ -167,7 +173,7 @@ class Spawn(Thread, LazyLogging): return self._spawn_process(repository_id, "service-key-import", key, **kwargs) def packages_add(self, repository_id: RepositoryId, packages: Iterable[str], username: str | None, *, - now: bool) -> str: + patches: list[PkgbuildPatch], now: bool) -> str: """ add packages @@ -175,14 +181,18 @@ class Spawn(Thread, LazyLogging): repository_id(RepositoryId): repository unique identifier packages(Iterable[str]): packages list to add username(str | None): optional override of username for build process + patches(list[PkgbuildPatch]): list of patches to be passed now(bool): build packages now Returns: str: spawned process identifier """ - kwargs = {"username": username} + kwargs: dict[str, str | list[str] | None] = {"username": username} if now: kwargs["now"] = "" + if patches: + kwargs["variable"] = [patch.serialize() for patch in patches] + return self._spawn_process(repository_id, "package-add", *packages, **kwargs) def packages_rebuild(self, repository_id: RepositoryId, depends_on: str, username: str | None) -> str: diff --git a/src/ahriman/core/util.py b/src/ahriman/core/util.py index 36cfdef8..b90a1210 100644 --- a/src/ahriman/core/util.py +++ b/src/ahriman/core/util.py @@ -56,6 +56,7 @@ __all__ = [ "srcinfo_property", "srcinfo_property_list", "trim_package", + "unquote", "utcnow", "walk", ] @@ -465,6 +466,38 @@ def trim_package(package_name: str) -> str: return package_name +def unquote(source: str) -> str: + """ + like ``shlex.quote``, but opposite + + Args: + source(str): source string to remove quotes + + Returns: + str: string with quotes removed + + Raises: + ValueError: if no closing quotation + """ + def generator() -> Generator[str, None, None]: + token = None + for char in source: + if token is not None: + if char == token: + token = None # closed quote + else: + yield char # character inside quotes + elif char in ("'", "\""): + token = char # first quote found + else: + yield char # normal character + + if token is not None: + raise ValueError("No closing quotation") + + return "".join(generator()) + + def utcnow() -> datetime.datetime: """ get current time diff --git a/src/ahriman/models/pkgbuild_patch.py b/src/ahriman/models/pkgbuild_patch.py index a547874c..6736214c 100644 --- a/src/ahriman/models/pkgbuild_patch.py +++ b/src/ahriman/models/pkgbuild_patch.py @@ -19,8 +19,11 @@ # import shlex -from dataclasses import dataclass, field +from dataclasses import dataclass from pathlib import Path +from typing import Self + +from ahriman.core.util import unquote @dataclass(frozen=True) @@ -33,12 +36,12 @@ class PkgbuildPatch: considered as full PKGBUILD diffs value(str | list[str]): value of the stored PKGBUILD property. It must be either string or list of string values - unsafe(bool): if set, value will be not quoted, might break PKGBUILD """ key: str | None value: str | list[str] - unsafe: bool = field(default=False, kw_only=True) + + quote = shlex.quote def __post_init__(self) -> None: """ @@ -66,17 +69,26 @@ class PkgbuildPatch: """ return self.key is None - def quote(self, value: str) -> str: + @classmethod + def from_env(cls, variable: str) -> Self: """ - quote value according to the unsafe flag + construct patch from environment variable. Functions are not supported Args: - value(str): value to be quoted + variable(str): variable in bash form, i.e. KEY=VALUE Returns: - str: quoted string in case if unsafe is False and as is otherwise + Self: package properties """ - return value if self.unsafe else shlex.quote(value) + key, *value_parts = variable.split("=", maxsplit=1) + + raw_value = next(iter(value_parts), "") # extract raw value + if raw_value.startswith("(") and raw_value.endswith(")"): + value: str | list[str] = shlex.split(raw_value[1:-1]) # arrays for poor + else: + value = unquote(raw_value) + + return cls(key, value) def serialize(self) -> str: """ @@ -88,14 +100,14 @@ class PkgbuildPatch: str: serialized key-value pair, print-friendly """ if isinstance(self.value, list): # list like - value = " ".join(map(self.quote, self.value)) + value = " ".join(map(PkgbuildPatch.quote, self.value)) return f"""{self.key}=({value})""" if self.is_plain_diff: # no additional logic for plain diffs return self.value # we suppose that function values are only supported in string-like values if self.is_function: return f"{self.key} {self.value}" # no quoting enabled here - return f"""{self.key}={self.quote(self.value)}""" + return f"""{self.key}={PkgbuildPatch.quote(self.value)}""" def write(self, pkgbuild_path: Path) -> None: """ diff --git a/src/ahriman/models/repository_id.py b/src/ahriman/models/repository_id.py index ac410828..541761f2 100644 --- a/src/ahriman/models/repository_id.py +++ b/src/ahriman/models/repository_id.py @@ -52,6 +52,8 @@ class RepositoryId: Returns: str: unique id for this repository """ + if self.is_empty: + return "" return f"{self.architecture}-{self.name}" # basically the same as used for command line def query(self) -> list[tuple[str, str]]: diff --git a/src/ahriman/web/schemas/__init__.py b/src/ahriman/web/schemas/__init__.py index 3effcca8..6a26332b 100644 --- a/src/ahriman/web/schemas/__init__.py +++ b/src/ahriman/web/schemas/__init__.py @@ -33,6 +33,7 @@ from ahriman.web.schemas.package_properties_schema import PackagePropertiesSchem from ahriman.web.schemas.package_schema import PackageSchema from ahriman.web.schemas.package_status_schema import PackageStatusSimplifiedSchema, PackageStatusSchema from ahriman.web.schemas.pagination_schema import PaginationSchema +from ahriman.web.schemas.patch_schema import PackagePatchSchema, PatchSchema from ahriman.web.schemas.pgp_key_id_schema import PGPKeyIdSchema from ahriman.web.schemas.pgp_key_schema import PGPKeySchema from ahriman.web.schemas.process_id_schema import ProcessIdSchema diff --git a/src/ahriman/web/schemas/patch_schema.py b/src/ahriman/web/schemas/patch_schema.py new file mode 100644 index 00000000..09cc1b13 --- /dev/null +++ b/src/ahriman/web/schemas/patch_schema.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from marshmallow import Schema, fields + +from ahriman.web.schemas.package_names_schema import PackageNamesSchema + + +class PatchSchema(Schema): + """ + request patch schema + """ + + key = fields.String(required=True, metadata={ + "description": "environment variable name", + }) + value = fields.String(metadata={ + "description": "environment variable value", + }) + + +class PackagePatchSchema(PackageNamesSchema): + """ + request schema with packages and patches + """ + + patches = fields.Nested(PatchSchema(many=True), metadata={ + "description": "optional environment variables to be applied as patches" + }) diff --git a/src/ahriman/web/views/v1/service/add.py b/src/ahriman/web/views/v1/service/add.py index 776c7d72..9260c71e 100644 --- a/src/ahriman/web/views/v1/service/add.py +++ b/src/ahriman/web/views/v1/service/add.py @@ -21,8 +21,9 @@ import aiohttp_apispec # type: ignore[import-untyped] from aiohttp.web import HTTPBadRequest, Response, json_response +from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.user_access import UserAccess -from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema, RepositoryIdSchema +from ahriman.web.schemas import AuthSchema, ErrorSchema, PackagePatchSchema, ProcessIdSchema, RepositoryIdSchema from ahriman.web.views.base import BaseView @@ -53,7 +54,7 @@ class AddView(BaseView): ) @aiohttp_apispec.cookies_schema(AuthSchema) @aiohttp_apispec.querystring_schema(RepositoryIdSchema) - @aiohttp_apispec.json_schema(PackageNamesSchema) + @aiohttp_apispec.json_schema(PackagePatchSchema) async def post(self) -> Response: """ add new package @@ -65,13 +66,14 @@ class AddView(BaseView): HTTPBadRequest: if bad data is supplied """ try: - data = await self.extract_data(["packages"]) + data = await self.extract_data(["packages", "patches"]) packages = self.get_non_empty(lambda key: [package for package in data[key] if package], "packages") + patches = [PkgbuildPatch(patch["key"], patch.get("value", "")) for patch in data.get("patches", [])] except Exception as ex: raise HTTPBadRequest(reason=str(ex)) repository_id = self.repository_id() username = await self.username() - process_id = self.spawner.packages_add(repository_id, packages, username, now=True) + process_id = self.spawner.packages_add(repository_id, packages, username, patches=patches, now=True) return json_response({"process_id": process_id}) diff --git a/src/ahriman/web/views/v1/service/request.py b/src/ahriman/web/views/v1/service/request.py index aa7eebf3..2919f88e 100644 --- a/src/ahriman/web/views/v1/service/request.py +++ b/src/ahriman/web/views/v1/service/request.py @@ -21,8 +21,9 @@ import aiohttp_apispec # type: ignore[import-untyped] from aiohttp.web import HTTPBadRequest, Response, json_response +from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.user_access import UserAccess -from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema, RepositoryIdSchema +from ahriman.web.schemas import AuthSchema, ErrorSchema, PackagePatchSchema, ProcessIdSchema, RepositoryIdSchema from ahriman.web.views.base import BaseView @@ -53,7 +54,7 @@ class RequestView(BaseView): ) @aiohttp_apispec.cookies_schema(AuthSchema) @aiohttp_apispec.querystring_schema(RepositoryIdSchema) - @aiohttp_apispec.json_schema(PackageNamesSchema) + @aiohttp_apispec.json_schema(PackagePatchSchema) async def post(self) -> Response: """ request to add new package @@ -65,13 +66,14 @@ class RequestView(BaseView): HTTPBadRequest: if bad data is supplied """ try: - data = await self.extract_data(["packages"]) + data = await self.extract_data(["packages", "patches"]) packages = self.get_non_empty(lambda key: [package for package in data[key] if package], "packages") + patches = [PkgbuildPatch(patch["key"], patch.get("value", "")) for patch in data.get("patches", [])] except Exception as ex: raise HTTPBadRequest(reason=str(ex)) username = await self.username() repository_id = self.repository_id() - process_id = self.spawner.packages_add(repository_id, packages, username, now=False) + process_id = self.spawner.packages_add(repository_id, packages, username, patches=patches, now=False) return json_response({"process_id": process_id}) diff --git a/tests/ahriman/application/handlers/test_handler_add.py b/tests/ahriman/application/handlers/test_handler_add.py index 8cd5177e..c09e767f 100644 --- a/tests/ahriman/application/handlers/test_handler_add.py +++ b/tests/ahriman/application/handlers/test_handler_add.py @@ -9,6 +9,7 @@ from ahriman.core.repository import Repository from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.packagers import Packagers +from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.result import Result @@ -22,7 +23,7 @@ def _default_args(args: argparse.Namespace) -> argparse.Namespace: Returns: argparse.Namespace: generated arguments for these test cases """ - args.package = [] + args.package = ["ahriman"] args.exit_code = False args.increment = True args.now = False @@ -30,6 +31,7 @@ def _default_args(args: argparse.Namespace) -> argparse.Namespace: args.source = PackageSource.Auto args.dependencies = True args.username = "username" + args.variable = None return args @@ -51,6 +53,22 @@ def test_run(args: argparse.Namespace, configuration: Configuration, repository: on_start_mock.assert_called_once_with() +def test_run_with_patches(args: argparse.Namespace, configuration: Configuration, repository: Repository, + mocker: MockerFixture) -> None: + """ + must run command and insert temporary patches + """ + args = _default_args(args) + args.variable = ["KEY=VALUE"] + mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) + mocker.patch("ahriman.application.application.Application.add") + application_mock = mocker.patch("ahriman.core.database.SQLite.patches_insert") + + _, repository_id = configuration.check_loaded() + Add.run(args, repository_id, configuration, report=False) + application_mock.assert_called_once_with(args.package[0], [PkgbuildPatch("KEY", "VALUE")]) + + def test_run_with_updates(args: argparse.Namespace, configuration: Configuration, repository: Repository, package_ahriman: Package, mocker: MockerFixture) -> None: """ diff --git a/tests/ahriman/application/handlers/test_handler_patch.py b/tests/ahriman/application/handlers/test_handler_patch.py index 054aedb9..d8d3cad8 100644 --- a/tests/ahriman/application/handlers/test_handler_patch.py +++ b/tests/ahriman/application/handlers/test_handler_patch.py @@ -180,7 +180,7 @@ def test_patch_set_create(application: Application, package_ahriman: Package, mo """ create_mock = mocker.patch("ahriman.core.database.SQLite.patches_insert") Patch.patch_set_create(application, package_ahriman.base, PkgbuildPatch("version", package_ahriman.version)) - create_mock.assert_called_once_with(package_ahriman.base, PkgbuildPatch("version", package_ahriman.version)) + create_mock.assert_called_once_with(package_ahriman.base, [PkgbuildPatch("version", package_ahriman.version)]) def test_patch_set_remove(application: Application, package_ahriman: Package, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/application/test_ahriman.py b/tests/ahriman/application/test_ahriman.py index 84d2af82..91b0060f 100644 --- a/tests/ahriman/application/test_ahriman.py +++ b/tests/ahriman/application/test_ahriman.py @@ -254,6 +254,22 @@ def test_subparsers_package_add_option_refresh(parser: argparse.ArgumentParser) assert args.refresh == 2 +def test_subparsers_package_add_option_variable_empty(parser: argparse.ArgumentParser) -> None: + """ + package-add command must accept empty variable list as None + """ + args = parser.parse_args(["package-add", "ahriman"]) + assert args.variable is None + + +def test_subparsers_package_add_option_variable_multiple(parser: argparse.ArgumentParser) -> None: + """ + repo-rebuild command must accept multiple depends-on + """ + args = parser.parse_args(["package-add", "ahriman", "-v", "var1", "-v", "var2"]) + assert args.variable == ["var1", "var2"] + + def test_subparsers_package_remove_option_architecture(parser: argparse.ArgumentParser) -> None: """ package-remove command must correctly parse architecture list diff --git a/tests/ahriman/core/build_tools/test_task.py b/tests/ahriman/core/build_tools/test_task.py index e59ef5b5..1846cfab 100644 --- a/tests/ahriman/core/build_tools/test_task.py +++ b/tests/ahriman/core/build_tools/test_task.py @@ -20,6 +20,7 @@ def test_init(task_ahriman: Task, database: SQLite, mocker: MockerFixture) -> No """ mocker.patch("ahriman.models.package.Package.from_build", return_value=task_ahriman.package) load_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.load") + task_ahriman.init(Path("ahriman"), database, None) load_mock.assert_called_once_with(Path("ahriman"), task_ahriman.package, [], task_ahriman.paths) diff --git a/tests/ahriman/core/database/migrations/test_m011_repository_name.py b/tests/ahriman/core/database/migrations/test_m011_repository_name.py index 319ccc51..970d482e 100644 --- a/tests/ahriman/core/database/migrations/test_m011_repository_name.py +++ b/tests/ahriman/core/database/migrations/test_m011_repository_name.py @@ -8,7 +8,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.database.migrations.m011_repository_name import migrate_data, migrate_package_repository, steps -def test_migration_check_depends() -> None: +def test_migration_repository_name() -> None: """ migration must not be empty """ diff --git a/tests/ahriman/core/database/operations/test_patch_operations.py b/tests/ahriman/core/database/operations/test_patch_operations.py index c7f2669b..26f9c417 100644 --- a/tests/ahriman/core/database/operations/test_patch_operations.py +++ b/tests/ahriman/core/database/operations/test_patch_operations.py @@ -7,9 +7,9 @@ def test_patches_get_insert(database: SQLite, package_ahriman: Package, package_ """ must insert patch to database """ - database.patches_insert(package_ahriman.base, PkgbuildPatch(None, "patch_1")) - database.patches_insert(package_ahriman.base, PkgbuildPatch("key", "patch_3")) - database.patches_insert(package_python_schedule.base, PkgbuildPatch(None, "patch_2")) + database.patches_insert(package_ahriman.base, [PkgbuildPatch(None, "patch_1")]) + database.patches_insert(package_ahriman.base, [PkgbuildPatch("key", "patch_3")]) + database.patches_insert(package_python_schedule.base, [PkgbuildPatch(None, "patch_2")]) assert database.patches_get(package_ahriman.base) == [ PkgbuildPatch(None, "patch_1"), PkgbuildPatch("key", "patch_3") ] @@ -19,9 +19,9 @@ def test_patches_list(database: SQLite, package_ahriman: Package, package_python """ must list all patches """ - database.patches_insert(package_ahriman.base, PkgbuildPatch(None, "patch1")) - database.patches_insert(package_ahriman.base, PkgbuildPatch("key", "patch3")) - database.patches_insert(package_python_schedule.base, PkgbuildPatch(None, "patch2")) + database.patches_insert(package_ahriman.base, [PkgbuildPatch(None, "patch1")]) + database.patches_insert(package_ahriman.base, [PkgbuildPatch("key", "patch3")]) + database.patches_insert(package_python_schedule.base, [PkgbuildPatch(None, "patch2")]) assert database.patches_list(None, None) == { package_ahriman.base: [PkgbuildPatch(None, "patch1"), PkgbuildPatch("key", "patch3")], package_python_schedule.base: [PkgbuildPatch(None, "patch2")], @@ -32,8 +32,8 @@ def test_patches_list_filter(database: SQLite, package_ahriman: Package, package """ must list all patches filtered by package name (same as get) """ - database.patches_insert(package_ahriman.base, PkgbuildPatch(None, "patch1")) - database.patches_insert(package_python_schedule.base, PkgbuildPatch(None, "patch2")) + database.patches_insert(package_ahriman.base, [PkgbuildPatch(None, "patch1")]) + database.patches_insert(package_python_schedule.base, [PkgbuildPatch(None, "patch2")]) assert database.patches_list(package_ahriman.base, None) == {package_ahriman.base: [PkgbuildPatch(None, "patch1")]} assert database.patches_list(package_python_schedule.base, None) == { @@ -46,9 +46,9 @@ def test_patches_list_filter_by_variable(database: SQLite, package_ahriman: Pack """ must list all patches filtered by package name (same as get) """ - database.patches_insert(package_ahriman.base, PkgbuildPatch(None, "patch1")) - database.patches_insert(package_ahriman.base, PkgbuildPatch("key", "patch2")) - database.patches_insert(package_python_schedule.base, PkgbuildPatch(None, "patch3")) + database.patches_insert(package_ahriman.base, [PkgbuildPatch(None, "patch1")]) + database.patches_insert(package_ahriman.base, [PkgbuildPatch("key", "patch2")]) + database.patches_insert(package_python_schedule.base, [PkgbuildPatch(None, "patch3")]) assert database.patches_list(None, None) == { package_ahriman.base: [PkgbuildPatch(None, "patch1"), PkgbuildPatch("key", "patch2")], @@ -63,8 +63,8 @@ def test_patches_insert_remove(database: SQLite, package_ahriman: Package, packa """ must remove patch from database """ - database.patches_insert(package_ahriman.base, PkgbuildPatch(None, "patch1")) - database.patches_insert(package_python_schedule.base, PkgbuildPatch(None, "patch2")) + database.patches_insert(package_ahriman.base, [PkgbuildPatch(None, "patch1")]) + database.patches_insert(package_python_schedule.base, [PkgbuildPatch(None, "patch2")]) database.patches_remove(package_ahriman.base, None) assert database.patches_get(package_ahriman.base) == [] @@ -76,9 +76,9 @@ def test_patches_insert_remove_by_variable(database: SQLite, package_ahriman: Pa """ must remove patch from database by variable """ - database.patches_insert(package_ahriman.base, PkgbuildPatch(None, "patch1")) - database.patches_insert(package_ahriman.base, PkgbuildPatch("key", "patch3")) - database.patches_insert(package_python_schedule.base, PkgbuildPatch(None, "patch2")) + database.patches_insert(package_ahriman.base, [PkgbuildPatch(None, "patch1")]) + database.patches_insert(package_ahriman.base, [PkgbuildPatch("key", "patch3")]) + database.patches_insert(package_python_schedule.base, [PkgbuildPatch(None, "patch2")]) database.patches_remove(package_ahriman.base, ["key"]) assert database.patches_get(package_ahriman.base) == [PkgbuildPatch(None, "patch1")] @@ -89,8 +89,14 @@ def test_patches_insert_insert(database: SQLite, package_ahriman: Package) -> No """ must update patch in database """ - database.patches_insert(package_ahriman.base, PkgbuildPatch(None, "patch1")) + database.patches_insert(package_ahriman.base, [PkgbuildPatch(None, "patch1")]) assert database.patches_get(package_ahriman.base) == [PkgbuildPatch(None, "patch1")] - database.patches_insert(package_ahriman.base, PkgbuildPatch(None, "patch2")) + database.patches_insert(package_ahriman.base, [PkgbuildPatch(None, "patch2")]) assert database.patches_get(package_ahriman.base) == [PkgbuildPatch(None, "patch2")] + + database.patches_insert(package_ahriman.base, [PkgbuildPatch(None, "patch3"), PkgbuildPatch("key", "patch4")]) + assert database.patches_get(package_ahriman.base) == [ + PkgbuildPatch(None, "patch3"), + PkgbuildPatch("key", "patch4"), + ] diff --git a/tests/ahriman/core/test_spawn.py b/tests/ahriman/core/test_spawn.py index 00349c90..cb0e1355 100644 --- a/tests/ahriman/core/test_spawn.py +++ b/tests/ahriman/core/test_spawn.py @@ -4,6 +4,7 @@ from pytest_mock import MockerFixture from unittest.mock import MagicMock from ahriman.core.spawn import Spawn +from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.process_status import ProcessStatus from ahriman.models.repository_id import RepositoryId @@ -56,11 +57,12 @@ def test_spawn_process(spawner: Spawn, repository_id: RepositoryId, mocker: Mock """ start_mock = mocker.patch("multiprocessing.Process.start") - assert spawner._spawn_process(repository_id, "add", "ahriman", now="", maybe="?", none=None) + assert spawner._spawn_process(repository_id, "command", "argument", + empty="", string="v", list=["a", "b"], empty_list=[], none=None) start_mock.assert_called_once_with() spawner.args_parser.parse_args.assert_called_once_with( spawner.command_arguments + [ - "add", "ahriman", "--now", "--maybe", "?" + "command", "argument", "--empty", "--string", "v", "--list", "a", "--list", "b", ] ) @@ -99,7 +101,7 @@ def test_packages_add(spawner: Spawn, repository_id: RepositoryId, mocker: Mocke must call package addition """ spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") - assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, now=False) + assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=[], now=False) spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None) @@ -108,7 +110,7 @@ def test_packages_add_with_build(spawner: Spawn, repository_id: RepositoryId, mo must call package addition with update """ spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") - assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, now=True) + assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=[], now=True) spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None, now="") @@ -117,10 +119,21 @@ def test_packages_add_with_username(spawner: Spawn, repository_id: RepositoryId, must call package addition with username """ spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") - assert spawner.packages_add(repository_id, ["ahriman", "linux"], "username", now=False) + assert spawner.packages_add(repository_id, ["ahriman", "linux"], "username", patches=[], now=False) spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username="username") +def test_packages_add_with_patches(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None: + """ + must call package addition with patches + """ + patches = [PkgbuildPatch("key", "value"), PkgbuildPatch("key", "value")] + spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") + assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=patches, now=False) + spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None, + variable=[patch.serialize() for patch in patches]) + + def test_packages_rebuild(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None: """ must call package rebuild diff --git a/tests/ahriman/core/test_util.py b/tests/ahriman/core/test_util.py index 25709742..5cb1179c 100644 --- a/tests/ahriman/core/test_util.py +++ b/tests/ahriman/core/test_util.py @@ -2,6 +2,7 @@ import datetime import logging import os import pytest +import shlex from pathlib import Path from pytest_mock import MockerFixture @@ -11,7 +12,7 @@ from unittest.mock import call as MockCall from ahriman.core.exceptions import BuildError, CalledProcessError, OptionError, UnsafeRunError from ahriman.core.util import check_output, check_user, dataclass_view, enum_values, extract_user, filter_json, \ full_version, minmax, package_like, parse_version, partition, pretty_datetime, pretty_size, safe_filename, \ - srcinfo_property, srcinfo_property_list, trim_package, utcnow, walk + srcinfo_property, srcinfo_property_list, trim_package, unquote, utcnow, walk from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.repository_paths import RepositoryPaths @@ -443,6 +444,26 @@ def test_trim_package() -> None: assert trim_package("package: a description") == "package" +def test_unquote() -> None: + """ + must remove quotation marks + """ + for source in ( + "abc", + "ab'c", + "ab\"c", + ): + assert unquote(shlex.quote(source)) == source + + +def test_unquote_error() -> None: + """ + must raise value error on invalid quotation + """ + with pytest.raises(ValueError): + unquote("ab'c") + + def test_utcnow() -> None: """ must generate correct timestamp diff --git a/tests/ahriman/models/test_pkgbuild_patch.py b/tests/ahriman/models/test_pkgbuild_patch.py index 0d40134d..b151a82e 100644 --- a/tests/ahriman/models/test_pkgbuild_patch.py +++ b/tests/ahriman/models/test_pkgbuild_patch.py @@ -34,9 +34,18 @@ def test_quote() -> None: """ must quote strings if unsafe flag is not set """ - assert PkgbuildPatch("key", "value").quote("value") == """value""" - assert PkgbuildPatch("key", "va'lue").quote("va'lue") == """'va'"'"'lue'""" - assert PkgbuildPatch("key", "va'lue", unsafe=True).quote("va'lue") == """va'lue""" + assert PkgbuildPatch.quote("value") == """value""" + assert PkgbuildPatch.quote("va'lue") == """'va'"'"'lue'""" + + +def test_from_env() -> None: + """ + must construct patch from environment variable + """ + assert PkgbuildPatch.from_env("KEY=VALUE") == PkgbuildPatch("KEY", "VALUE") + assert PkgbuildPatch.from_env("KEY=VA=LUE") == PkgbuildPatch("KEY", "VA=LUE") + assert PkgbuildPatch.from_env("KEY=") == PkgbuildPatch("KEY", "") + assert PkgbuildPatch.from_env("KEY") == PkgbuildPatch("KEY", "") def test_serialize() -> None: @@ -46,7 +55,19 @@ def test_serialize() -> None: assert PkgbuildPatch("key", "value").serialize() == "key=value" assert PkgbuildPatch("key", "42").serialize() == "key=42" assert PkgbuildPatch("key", "4'2").serialize() == """key='4'"'"'2'""" - assert PkgbuildPatch("key", "4'2", unsafe=True).serialize() == "key=4'2" + + +def test_from_env_serialize() -> None: + """ + must serialize and parse back + """ + for patch in ( + PkgbuildPatch("key", "value"), + PkgbuildPatch("key", "4'2"), + PkgbuildPatch("arch", ["i686", "x86_64"]), + PkgbuildPatch("key", ["val'ue", "val\"ue2"]), + ): + assert PkgbuildPatch.from_env(patch.serialize()) == patch def test_serialize_plain_diff() -> None: @@ -60,7 +81,7 @@ def test_serialize_function() -> None: """ must correctly serialize function values """ - assert PkgbuildPatch("key()", "{ value }", unsafe=True).serialize() == "key() { value }" + assert PkgbuildPatch("key()", "{ value }").serialize() == "key() { value }" def test_serialize_list() -> None: @@ -69,7 +90,6 @@ def test_serialize_list() -> None: """ assert PkgbuildPatch("arch", ["i686", "x86_64"]).serialize() == """arch=(i686 x86_64)""" assert PkgbuildPatch("key", ["val'ue", "val\"ue2"]).serialize() == """key=('val'"'"'ue' 'val"ue2')""" - assert PkgbuildPatch("key", ["val'ue", "val\"ue2"], unsafe=True).serialize() == """key=(val'ue val"ue2)""" def test_write(mocker: MockerFixture) -> None: diff --git a/tests/ahriman/models/test_repository_id.py b/tests/ahriman/models/test_repository_id.py index 6ce39956..99795958 100644 --- a/tests/ahriman/models/test_repository_id.py +++ b/tests/ahriman/models/test_repository_id.py @@ -17,7 +17,7 @@ def test_id() -> None: """ must correctly generate id """ - assert RepositoryId("", "").id == "-" + assert RepositoryId("", "").id == "" assert RepositoryId("arch", "repo").id == "arch-repo" diff --git a/tests/ahriman/web/schemas/test_patch_schema.py b/tests/ahriman/web/schemas/test_patch_schema.py new file mode 100644 index 00000000..1982fb6b --- /dev/null +++ b/tests/ahriman/web/schemas/test_patch_schema.py @@ -0,0 +1 @@ +# schema testing goes in view class tests diff --git a/tests/ahriman/web/views/v1/service/test_view_v1_service_add.py b/tests/ahriman/web/views/v1/service/test_view_v1_service_add.py index ef2e371e..e1ffec43 100644 --- a/tests/ahriman/web/views/v1/service/test_view_v1_service_add.py +++ b/tests/ahriman/web/views/v1/service/test_view_v1_service_add.py @@ -4,6 +4,7 @@ from aiohttp.test_utils import TestClient from pytest_mock import MockerFixture from unittest.mock import AsyncMock +from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.repository_id import RepositoryId from ahriman.models.user_access import UserAccess from ahriman.web.views.v1.service.add import AddView @@ -40,13 +41,42 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc assert not request_schema.validate(payload) response = await client.post("/api/v1/service/add", json=payload) assert response.ok - add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", now=True) + add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", patches=[], now=True) json = await response.json() assert json["process_id"] == "abc" assert not response_schema.validate(json) +async def test_post_patches(client: TestClient, repository_id: RepositoryId, mocker: MockerFixture) -> None: + """ + must call post request with patches correctly + """ + add_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_add", return_value="abc") + user_mock = AsyncMock() + user_mock.return_value = "username" + mocker.patch("ahriman.web.views.base.BaseView.username", side_effect=user_mock) + request_schema = pytest.helpers.schema_request(AddView.post) + + payload = { + "packages": ["ahriman"], + "patches": [ + { + "key": "k", + "value": "v", + }, + { + "key": "k2", + }, + ] + } + assert not request_schema.validate(payload) + response = await client.post("/api/v1/service/add", json=payload) + assert response.ok + add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", + patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")], now=True) + + async def test_post_empty(client: TestClient, mocker: MockerFixture) -> None: """ must call raise 400 on empty request diff --git a/tests/ahriman/web/views/v1/service/test_view_v1_service_request.py b/tests/ahriman/web/views/v1/service/test_view_v1_service_request.py index bd6dfb73..d2f8b7ce 100644 --- a/tests/ahriman/web/views/v1/service/test_view_v1_service_request.py +++ b/tests/ahriman/web/views/v1/service/test_view_v1_service_request.py @@ -4,6 +4,7 @@ from aiohttp.test_utils import TestClient from pytest_mock import MockerFixture from unittest.mock import AsyncMock +from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.repository_id import RepositoryId from ahriman.models.user_access import UserAccess from ahriman.web.views.v1.service.request import RequestView @@ -40,13 +41,42 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc assert not request_schema.validate(payload) response = await client.post("/api/v1/service/request", json=payload) assert response.ok - add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", now=False) + add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", patches=[], now=False) json = await response.json() assert json["process_id"] == "abc" assert not response_schema.validate(json) +async def test_post_patches(client: TestClient, repository_id: RepositoryId, mocker: MockerFixture) -> None: + """ + must call post request with patches correctly + """ + add_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_add", return_value="abc") + user_mock = AsyncMock() + user_mock.return_value = "username" + mocker.patch("ahriman.web.views.base.BaseView.username", side_effect=user_mock) + request_schema = pytest.helpers.schema_request(RequestView.post) + + payload = { + "packages": ["ahriman"], + "patches": [ + { + "key": "k", + "value": "v", + }, + { + "key": "k2", + }, + ] + } + assert not request_schema.validate(payload) + response = await client.post("/api/v1/service/request", json=payload) + assert response.ok + add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", + patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")], now=False) + + async def test_post_exception(client: TestClient, mocker: MockerFixture) -> None: """ must raise exception on missing packages payload