diff --git a/docs/ahriman.application.application.rst b/docs/ahriman.application.application.rst index 63d399c3..c3672dde 100644 --- a/docs/ahriman.application.application.rst +++ b/docs/ahriman.application.application.rst @@ -1,6 +1,14 @@ ahriman.application.application package ======================================= +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + ahriman.application.application.workers + Submodules ---------- diff --git a/docs/ahriman.application.application.workers.rst b/docs/ahriman.application.application.workers.rst new file mode 100644 index 00000000..eb61f08f --- /dev/null +++ b/docs/ahriman.application.application.workers.rst @@ -0,0 +1,37 @@ +ahriman.application.application.workers package +=============================================== + +Submodules +---------- + +ahriman.application.application.workers.local\_updater module +------------------------------------------------------------- + +.. automodule:: ahriman.application.application.workers.local_updater + :members: + :no-undoc-members: + :show-inheritance: + +ahriman.application.application.workers.remote\_updater module +-------------------------------------------------------------- + +.. automodule:: ahriman.application.application.workers.remote_updater + :members: + :no-undoc-members: + :show-inheritance: + +ahriman.application.application.workers.updater module +------------------------------------------------------ + +.. automodule:: ahriman.application.application.workers.updater + :members: + :no-undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: ahriman.application.application.workers + :members: + :no-undoc-members: + :show-inheritance: diff --git a/docs/ahriman.models.rst b/docs/ahriman.models.rst index 13fc69a2..461c044e 100644 --- a/docs/ahriman.models.rst +++ b/docs/ahriman.models.rst @@ -252,6 +252,14 @@ ahriman.models.waiter module :no-undoc-members: :show-inheritance: +ahriman.models.worker module +---------------------------- + +.. automodule:: ahriman.models.worker + :members: + :no-undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/ahriman.web.schemas.rst b/docs/ahriman.web.schemas.rst index 83691510..fa3bc8b0 100644 --- a/docs/ahriman.web.schemas.rst +++ b/docs/ahriman.web.schemas.rst @@ -20,6 +20,14 @@ ahriman.web.schemas.auth\_schema module :no-undoc-members: :show-inheritance: +ahriman.web.schemas.build\_options\_schema module +------------------------------------------------- + +.. automodule:: ahriman.web.schemas.build_options_schema + :members: + :no-undoc-members: + :show-inheritance: + ahriman.web.schemas.changes\_schema module ------------------------------------------ diff --git a/docs/configuration.rst b/docs/configuration.rst index 79064e07..a3f41665 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -86,6 +86,7 @@ Build related configuration. Group name can refer to architecture, e.g. ``build: * ``triggers`` - list of ``ahriman.core.triggers.Trigger`` class implementation (e.g. ``ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger``) which will be loaded and run at the end of processing, space separated list of strings, optional. You can also specify triggers by their paths, e.g. ``/usr/lib/python3.10/site-packages/ahriman/core/report/report.py.ReportTrigger``. Triggers are run in the order of mention. * ``triggers_known`` - optional list of ``ahriman.core.triggers.Trigger`` class implementations which are not run automatically and used only for trigger discovery and configuration validation. * ``vcs_allowed_age`` - maximal age in seconds of the VCS packages before their version will be updated with its remote source, integer, optional, default ``604800``. +* ``workers`` - list of worker nodes addresses used for build process, space separated list of strings, optional. Each worker address must be valid and reachable url, e.g. ``https://10.0.0.1:8080``. If none set, the build process will be run on the current node. ``repository`` group -------------------- diff --git a/docs/faq.rst b/docs/faq.rst index d90c11af..c4bf6595 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -1022,6 +1022,116 @@ This action must be done in two steps: #. Remove package on worker. #. Remove package on master node. +Delegate builds to remote workers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This setup heavily uses upload feature described above and, in addition, also delegates build process automatically to build machines. Same as above, there must be at least two instances available (``master`` and ``worker``), however, all ``worker`` nodes must be run in the web service mode. + +Master node configuration +""""""""""""""""""""""""" + +In addition to the configuration above, the worker list must be defined in configuration file (``build.workers`` option), i.e.: + +.. code-block:: ini + + [build] + workers = https://worker1.example.com https://worker2.example.com + + [web] + enable_archive_upload = yes + wait_timeout = 0 + +In the example above, ``https://worker1.example.com`` and ``https://worker2.example.com`` are remote ``worker`` node addresses available for ``master`` node. + +In case if authentication is required (which is recommended way to setup it), it can be set by using ``status`` section as usual. + +Worker nodes configuration +"""""""""""""""""""""""""" + +It is required to point to the master node repository, otherwise internal dependencies will not be handled correctly. In order to do so, the ``--server`` argument (or ``AHRIMAN_REPOSITORY_SERVER`` environment variable for docker images) can be used. + +Also, in case if authentication is enabled, the same user with the same password must be created for all workers. + +It is also recommended to set ``web.wait_timeout`` to infinte in case of multiple conflicting runs. + +Other settings are the same as mentioned above. + +Triple node minimal docker example +"""""""""""""""""""""""""""""""""" + +In this example, all instances are run on the same machine with address ``172.17.0.1`` with ports available outside of container. Master node config (``master.ini``) as: + +.. code-block:: ini + + [auth] + target = mapping + + [status] + username = builder-user + password = very-secure-password + + [build] + workers = http://172.17.0.1:8081 http://172.17.0.1:8082 + + [web] + enable_archive_upload = yes + wait_timeout = 0 + +Command to run master node: + +.. code-block:: shell + + docker run --privileged -p 8080:8080 -e AHRIMAN_PORT=8080 -v master.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web + +Worker nodes (applicable for all workers) config (``worker.ini``) as: + +.. code-block:: ini + + [auth] + target = mapping + + [status] + address = http://172.17.0.1:8080 + username = builder-user + password = very-secure-password + + [upload] + target = remote-service + + [remote-service] + + [report] + target = remote-call + + [remote-call] + manual = yes + wait_timeout = 0 + + [build] + triggers = ahriman.core.upload.UploadTrigger ahriman.core.report.ReportTrigger + +Command to run worker nodes (considering there will be two workers, one is on ``8081`` port and other is on ``8082``): + +.. code-block:: ini + + docker run --privileged -p 8081:8081 -e AHRIMAN_PORT=8081 -v worker.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web + docker run --privileged -p 8082:8082 -e AHRIMAN_PORT=8082 -v worker.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web + +Unlike the previous setup, it doesn't require to mount repository root for ``worker`` nodes, because ``worker`` nodes don't use it anyway. + +Addition of new package, package removal, repository update +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +In all scenarios, update process must be run only on ``master`` node. Unlike the setup described above, automatic update must be enabled only for ``master`` node also. + +Known limitations +""""""""""""""""" + +* Workers don't support local packages. However, it is possible to build custom packages by providing sources by using ``ahriman.core.gitremote.RemotePullTrigger`` trigger. +* No dynamic nodes discovery. In case if one of worker nodes is unavailable, the build process will fail. +* No pkgrel bump on conflicts. Well, it works, however, it isn't guaranteed. +* The identical user must be created for all workers. However, the ``master`` node user can be different from this one. + Maintenance packages -------------------- diff --git a/package/share/man/man1/ahriman.1 b/package/share/man/man1/ahriman.1 index 68e411d0..b8f350ab 100644 --- a/package/share/man/man1/ahriman.1 +++ b/package/share/man/man1/ahriman.1 @@ -1,4 +1,4 @@ -.TH AHRIMAN "1" "2023\-11\-29" "ahriman" "Generated Python Manual" +.TH AHRIMAN "1" "2023\-12\-08" "ahriman" "Generated Python Manual" .SH NAME ahriman .SH SYNOPSIS diff --git a/src/ahriman/application/application/application_repository.py b/src/ahriman/application/application/application_repository.py index 0b4ac8c6..4fad83de 100644 --- a/src/ahriman/application/application/application_repository.py +++ b/src/ahriman/application/application/application_repository.py @@ -18,11 +18,10 @@ # along with this program. If not, see . # from collections.abc import Iterable -from pathlib import Path from ahriman.application.application.application_properties import ApplicationProperties +from ahriman.application.application.workers import Updater from ahriman.core.build_tools.sources import Sources -from ahriman.core.tree import Tree from ahriman.models.package import Package from ahriman.models.packagers import Packagers from ahriman.models.result import Result @@ -154,26 +153,25 @@ class ApplicationRepository(ApplicationProperties): Returns: Result: update result """ - def process_update(paths: Iterable[Path], result: Result) -> None: - if not paths: - return # don't need to process if no update supplied - update_result = self.repository.process_update(paths, packagers) - self.on_result(result.merge(update_result)) + result = Result() - # process built packages - build_result = Result() - packages = self.repository.packages_built() - process_update(packages, build_result) + # process already built packages if any + built_packages = self.repository.packages_built() + if built_packages: # speedup a bit + build_result = self.repository.process_update(built_packages, packagers) + result.merge(build_result) + self.on_result(result.merge(build_result)) - # process manual packages - tree = Tree.resolve(updates) - for num, level in enumerate(tree): - self.logger.info("processing level #%i %s", num, [package.base for package in level]) - build_result = self.repository.process_build(level, packagers, bump_pkgrel=bump_pkgrel) - packages = self.repository.packages_built() - process_update(packages, build_result) + builder = Updater.load(self.repository_id, self.configuration, self.repository) - return build_result + # ok so for now we split all packages into chunks and process each chunk accordingly + partitions = builder.partition(updates) + for num, partition in enumerate(partitions): + self.logger.info("processing chunk #%i %s", num, [package.base for package in partition]) + build_result = builder.update(partition, packagers, bump_pkgrel=bump_pkgrel) + self.on_result(result.merge(build_result)) + + return result def updates(self, filter_packages: Iterable[str], *, aur: bool, local: bool, manual: bool, vcs: bool) -> list[Package]: diff --git a/src/ahriman/application/application/workers/__init__.py b/src/ahriman/application/application/workers/__init__.py new file mode 100644 index 00000000..7dd71d6b --- /dev/null +++ b/src/ahriman/application/application/workers/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from ahriman.application.application.workers.updater import Updater diff --git a/src/ahriman/application/application/workers/local_updater.py b/src/ahriman/application/application/workers/local_updater.py new file mode 100644 index 00000000..35b8e7df --- /dev/null +++ b/src/ahriman/application/application/workers/local_updater.py @@ -0,0 +1,77 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from collections.abc import Iterable + +from ahriman.application.application.workers.updater import Updater +from ahriman.core.repository import Repository +from ahriman.core.tree import Tree +from ahriman.models.package import Package +from ahriman.models.packagers import Packagers +from ahriman.models.result import Result + + +class LocalUpdater(Updater): + """ + local build process implementation + + Attributes: + repository(Repository): repository instance + """ + + def __init__(self, repository: Repository) -> None: + """ + default constructor + + Args: + repository(Repository): repository instance + """ + self.repository = repository + + def partition(self, packages: Iterable[Package]) -> list[list[Package]]: + """ + split packages into partitions to be processed by this worker + + Args: + packages(Iterable[Package]): list of packages to partition + + Returns: + list[list[Package]]: packages partitioned by this worker type + """ + return Tree.resolve(packages) + + def update(self, updates: Iterable[Package], packagers: Packagers | None = None, *, + bump_pkgrel: bool = False) -> Result: + """ + run package updates + + Args: + updates(Iterable[Package]): list of packages to update + packagers(Packagers | None, optional): optional override of username for build process + (Default value = None) + bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False) + + Returns: + Result: update result + """ + build_result = self.repository.process_build(updates, packagers, bump_pkgrel=bump_pkgrel) + packages = self.repository.packages_built() + update_result = self.repository.process_update(packages, packagers) + + return build_result.merge(update_result) diff --git a/src/ahriman/application/application/workers/remote_updater.py b/src/ahriman/application/application/workers/remote_updater.py new file mode 100644 index 00000000..0064adbb --- /dev/null +++ b/src/ahriman/application/application/workers/remote_updater.py @@ -0,0 +1,140 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from collections import deque +from collections.abc import Iterable + +from ahriman.application.application.workers.updater import Updater +from ahriman.core.configuration import Configuration +from ahriman.core.http import SyncAhrimanClient +from ahriman.core.tree import Tree +from ahriman.models.package import Package +from ahriman.models.packagers import Packagers +from ahriman.models.repository_id import RepositoryId +from ahriman.models.result import Result +from ahriman.models.worker import Worker + + +class RemoteUpdater(Updater): + """ + remote update worker + + Attributes: + configuration(Configuration): configuration instance + repository_id(RepositoryId): repository unique identifier + workers(list[Worker]): worker identifiers + """ + + def __init__(self, workers: list[Worker], repository_id: RepositoryId, configuration: Configuration) -> None: + """ + default constructor + + Args: + workers(list[Worker]): worker identifiers + repository_id(RepositoryId): repository unique identifier + configuration(Configuration): configuration instance + """ + self.workers = workers + self.repository_id = repository_id + self.configuration = configuration + + self._clients: deque[tuple[Worker, SyncAhrimanClient]] = deque() + + @property + def clients(self) -> dict[Worker, SyncAhrimanClient]: + """ + extract loaded clients. Note that this method yields only workers which have been already loaded + + Returns: + dict[Worker, SyncAhrimanClient]: map of the worker to the related web client + """ + return dict(self._clients) + + @staticmethod + def _update_url(worker: Worker) -> str: + """ + get url for updates + + Args: + worker(Worker): worker identifier + + Returns: + str: full url for web service to run update process + """ + return f"{worker.address}/api/v1/service/add" + + def next_worker(self) -> tuple[Worker, SyncAhrimanClient]: + """ + generate next not-used web client. In case if all clients have been already used, it yields next not used client + + Returns: + tuple[Worker, SyncAhrimanClient]: worker and constructed client instance for the web + """ + # check if there is not used yet worker + worker = next((worker for worker in self.workers if worker not in self.clients), None) + if worker is not None: + client = SyncAhrimanClient(self.configuration, "status") + client.address = worker.address + else: + worker, client = self._clients.popleft() + + # register worker in the queue + self._clients.append((worker, client)) + + return worker, client + + def partition(self, packages: Iterable[Package]) -> list[list[Package]]: + """ + split packages into partitions to be processed by this worker + + Args: + packages(Iterable[Package]): list of packages to partition + + Returns: + list[list[Package]]: packages partitioned by this worker type + """ + return Tree.partition(packages, count=len(self.workers)) + + def update(self, updates: Iterable[Package], packagers: Packagers | None = None, *, + bump_pkgrel: bool = False) -> Result: + """ + run package updates + + Args: + updates(Iterable[Package]): list of packages to update + packagers(Packagers | None, optional): optional override of username for build process + (Default value = None) + bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False) + + Returns: + Result: update result + """ + payload = { + "increment": bump_pkgrel, + "packager": packagers.default if packagers is not None else None, + "packages": [package.base for package in updates], + "patches": [], # might be used later + "refresh": True, + } + + worker, client = self.next_worker() + client.make_request("POST", self._update_url(worker), params=self.repository_id.query(), json=payload) + # we don't block here for process + + return Result() diff --git a/src/ahriman/application/application/workers/updater.py b/src/ahriman/application/application/workers/updater.py new file mode 100644 index 00000000..a1852a4c --- /dev/null +++ b/src/ahriman/application/application/workers/updater.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from __future__ import annotations + +from collections.abc import Iterable + +from ahriman.core.configuration import Configuration +from ahriman.core.log import LazyLogging +from ahriman.core.repository import Repository +from ahriman.models.package import Package +from ahriman.models.packagers import Packagers +from ahriman.models.repository_id import RepositoryId +from ahriman.models.result import Result +from ahriman.models.worker import Worker + + +class Updater(LazyLogging): + """ + updater handler interface + + Attributes: + split_method(Callable[[Iterable[Package]], list[list[Package]]]): method to split packages into chunks + """ + + @staticmethod + def load(repository_id: RepositoryId, configuration: Configuration, + repository: Repository, workers: list[Worker] | None = None) -> Updater: + """ + construct updaters from parameters + + Args: + repository_id(RepositoryId): repository unique identifier + configuration(Configuration): configuration instance + repository(Repository): repository instance + workers(list[Worker] | None, optional): worker identifiers if any (Default value = None) + + Returns: + Updater: constructed updater worker + """ + if workers is None: + # no workers set explicitly, try to guess from configuration + workers = [Worker(address) for address in configuration.getlist("build", "workers", fallback=[])] + + if workers: + # there is something we could use as remote workers + from ahriman.application.application.workers.remote_updater import RemoteUpdater + return RemoteUpdater(workers, repository_id, configuration) + + # and finally no workers available, just use local service + from ahriman.application.application.workers.local_updater import LocalUpdater + return LocalUpdater(repository) + + def partition(self, packages: Iterable[Package]) -> list[list[Package]]: + """ + split packages into partitions to be processed by this worker + + Args: + packages(Iterable[Package]): list of packages to partition + + Returns: + list[list[Package]]: packages partitioned by this worker type + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def update(self, updates: Iterable[Package], packagers: Packagers | None = None, *, + bump_pkgrel: bool = False) -> Result: + """ + run package updates + + Args: + updates(Iterable[Package]): list of packages to update + packagers(Packagers | None, optional): optional override of username for build process + (Default value = None) + bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False) + + Returns: + Result: update result + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError diff --git a/src/ahriman/core/configuration/schema.py b/src/ahriman/core/configuration/schema.py index a8a59fd3..b6ed7fd6 100644 --- a/src/ahriman/core/configuration/schema.py +++ b/src/ahriman/core/configuration/schema.py @@ -213,6 +213,15 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = { "coerce": "integer", "min": 0, }, + "workers": { + "type": "list", + "coerce": "list", + "schema": { + "type": "string", + "empty": False, + "is_url": [], + }, + }, }, }, "repository": { diff --git a/src/ahriman/core/spawn.py b/src/ahriman/core/spawn.py index 19e00a58..00b277a7 100644 --- a/src/ahriman/core/spawn.py +++ b/src/ahriman/core/spawn.py @@ -174,7 +174,7 @@ class Spawn(Thread, LazyLogging): return self._spawn_process(repository_id, "service-key-import", key, **kwargs) def packages_add(self, repository_id: RepositoryId, packages: Iterable[str], username: str | None, *, - patches: list[PkgbuildPatch], now: bool) -> str: + patches: list[PkgbuildPatch], now: bool, increment: bool, refresh: bool) -> str: """ add packages @@ -184,19 +184,26 @@ class Spawn(Thread, LazyLogging): username(str | None): optional override of username for build process patches(list[PkgbuildPatch]): list of patches to be passed now(bool): build packages now + increment(bool): increment pkgrel on conflict + refresh(bool): refresh pacman database before process Returns: str: spawned process identifier """ - kwargs: dict[str, str | list[str] | None] = {"username": username} + kwargs: dict[str, str | list[str] | None] = { + "username": username, + "variable": [patch.serialize() for patch in patches], + self.boolean_action_argument("increment", increment): "", + } if now: kwargs["now"] = "" - if patches: - kwargs["variable"] = [patch.serialize() for patch in patches] + if refresh: + kwargs["refresh"] = "" return self._spawn_process(repository_id, "package-add", *packages, **kwargs) - def packages_rebuild(self, repository_id: RepositoryId, depends_on: str, username: str | None) -> str: + def packages_rebuild(self, repository_id: RepositoryId, depends_on: str, username: str | None, *, + increment: bool) -> str: """ rebuild packages which depend on the specified package @@ -204,11 +211,16 @@ class Spawn(Thread, LazyLogging): repository_id(RepositoryId): repository unique identifier depends_on(str): packages dependency username(str | None): optional override of username for build process + increment(bool): increment pkgrel on conflict Returns: str: spawned process identifier """ - kwargs = {"depends-on": depends_on, "username": username} + kwargs = { + "depends-on": depends_on, + "username": username, + self.boolean_action_argument("increment", increment): "", + } return self._spawn_process(repository_id, "repo-rebuild", **kwargs) def packages_remove(self, repository_id: RepositoryId, packages: Iterable[str]) -> str: @@ -225,7 +237,7 @@ class Spawn(Thread, LazyLogging): return self._spawn_process(repository_id, "package-remove", *packages) def packages_update(self, repository_id: RepositoryId, username: str | None, *, - aur: bool, local: bool, manual: bool) -> str: + aur: bool, local: bool, manual: bool, increment: bool, refresh: bool) -> str: """ run full repository update @@ -235,6 +247,8 @@ class Spawn(Thread, LazyLogging): aur(bool): check for aur updates local(bool): check for local packages updates manual(bool): check for manual packages + increment(bool): increment pkgrel on conflict + refresh(bool): refresh pacman database before process Returns: str: spawned process identifier @@ -244,7 +258,11 @@ class Spawn(Thread, LazyLogging): self.boolean_action_argument("aur", aur): "", self.boolean_action_argument("local", local): "", self.boolean_action_argument("manual", manual): "", + self.boolean_action_argument("increment", increment): "", } + if refresh: + kwargs["refresh"] = "" + return self._spawn_process(repository_id, "repo-update", **kwargs) def run(self) -> None: diff --git a/src/ahriman/core/tree.py b/src/ahriman/core/tree.py index f7b04b23..3b9a762b 100644 --- a/src/ahriman/core/tree.py +++ b/src/ahriman/core/tree.py @@ -148,6 +148,8 @@ class Tree: sorted(part, key=lambda leaf: leaf.package.base) for part in partitions if part ] + if not partitions: # nothing to balance + return partitions while True: min_part, max_part = minmax(partitions, key=len) diff --git a/src/ahriman/models/worker.py b/src/ahriman/models/worker.py new file mode 100644 index 00000000..72215ff1 --- /dev/null +++ b/src/ahriman/models/worker.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from dataclasses import dataclass, field +from urllib.parse import urlparse + + +@dataclass(frozen=True) +class Worker: + """ + worker descriptor + + Attributes: + address(str): worker address to be reachable outside + identifier(str): worker unique identifier. If none set it will be automatically generated from the address + """ + + address: str + identifier: str = field(default="", kw_only=True) + + def __post_init__(self) -> None: + """ + update identifier based on settings + """ + object.__setattr__(self, "identifier", self.identifier or urlparse(self.address).netloc) diff --git a/src/ahriman/web/schemas/__init__.py b/src/ahriman/web/schemas/__init__.py index 91cf090a..2fe4d73f 100644 --- a/src/ahriman/web/schemas/__init__.py +++ b/src/ahriman/web/schemas/__init__.py @@ -19,6 +19,7 @@ # from ahriman.web.schemas.aur_package_schema import AURPackageSchema from ahriman.web.schemas.auth_schema import AuthSchema +from ahriman.web.schemas.build_options_schema import BuildOptionsSchema from ahriman.web.schemas.changes_schema import ChangesSchema from ahriman.web.schemas.counters_schema import CountersSchema from ahriman.web.schemas.error_schema import ErrorSchema diff --git a/src/ahriman/web/schemas/build_options_schema.py b/src/ahriman/web/schemas/build_options_schema.py new file mode 100644 index 00000000..a5bb2de5 --- /dev/null +++ b/src/ahriman/web/schemas/build_options_schema.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from marshmallow import Schema, fields + + +class BuildOptionsSchema(Schema): + """ + request build options schema + """ + + increment = fields.Boolean(dump_default=True, metadata={ + "description": "Increment pkgrel on conflicts", + }) + packager = fields.String(metadata={ + "description": "Packager identity if applicable", + }) + refresh = fields.Boolean(dump_default=True, metadata={ + "description": "Refresh pacman database" + }) diff --git a/src/ahriman/web/schemas/package_names_schema.py b/src/ahriman/web/schemas/package_names_schema.py index 5bba154e..fb7438ec 100644 --- a/src/ahriman/web/schemas/package_names_schema.py +++ b/src/ahriman/web/schemas/package_names_schema.py @@ -17,10 +17,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from marshmallow import Schema, fields +from marshmallow import fields + +from ahriman.web.schemas.build_options_schema import BuildOptionsSchema -class PackageNamesSchema(Schema): +class PackageNamesSchema(BuildOptionsSchema): """ request package names schema """ diff --git a/src/ahriman/web/schemas/update_flags_schema.py b/src/ahriman/web/schemas/update_flags_schema.py index 7ff09a59..35820d63 100644 --- a/src/ahriman/web/schemas/update_flags_schema.py +++ b/src/ahriman/web/schemas/update_flags_schema.py @@ -17,20 +17,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from marshmallow import Schema, fields +from marshmallow import fields + +from ahriman.web.schemas.build_options_schema import BuildOptionsSchema -class UpdateFlagsSchema(Schema): +class UpdateFlagsSchema(BuildOptionsSchema): """ update flags request schema """ - aur = fields.Bool(dump_default=True, metadata={ + aur = fields.Boolean(dump_default=True, metadata={ "description": "Check AUR for updates", }) - local = fields.Bool(dump_default=True, metadata={ + local = fields.Boolean(dump_default=True, metadata={ "description": "Check local packages for updates", }) - manual = fields.Bool(dump_default=True, metadata={ + manual = fields.Boolean(dump_default=True, metadata={ "description": "Check manually built packages", }) diff --git a/src/ahriman/web/views/base.py b/src/ahriman/web/views/base.py index ddf15599..356cec78 100644 --- a/src/ahriman/web/views/base.py +++ b/src/ahriman/web/views/base.py @@ -222,6 +222,13 @@ class BaseView(View, CorsViewMixin): Returns: str | None: authorized username if any and None otherwise (e.g. if authorization is disabled) """ + try: # try to read from payload + data: dict[str, str] = await self.request.json() # technically it is not, but we only need str here + if (packager := data.get("packager")) is not None: + return packager + except Exception: + self.request.app.logger.exception("could not extract json data for packager") + policy = self.request.app.get("identity") if policy is not None: identity: str = await policy.identify(self.request) diff --git a/src/ahriman/web/views/v1/service/add.py b/src/ahriman/web/views/v1/service/add.py index 7957cfa3..cfb98980 100644 --- a/src/ahriman/web/views/v1/service/add.py +++ b/src/ahriman/web/views/v1/service/add.py @@ -74,6 +74,14 @@ class AddView(BaseView): repository_id = self.repository_id() username = await self.username() - process_id = self.spawner.packages_add(repository_id, packages, username, patches=patches, now=True) + process_id = self.spawner.packages_add( + repository_id, + packages, + username, + patches=patches, + now=True, + increment=data.get("increment", True), + refresh=data.get("refresh", False), + ) return json_response({"process_id": process_id}) diff --git a/src/ahriman/web/views/v1/service/rebuild.py b/src/ahriman/web/views/v1/service/rebuild.py index 5952101b..990ec255 100644 --- a/src/ahriman/web/views/v1/service/rebuild.py +++ b/src/ahriman/web/views/v1/service/rebuild.py @@ -73,6 +73,11 @@ class RebuildView(BaseView): repository_id = self.repository_id() username = await self.username() - process_id = self.spawner.packages_rebuild(repository_id, depends_on, username) + process_id = self.spawner.packages_rebuild( + repository_id, + depends_on, + username, + increment=data.get("increment", True), + ) return json_response({"process_id": process_id}) diff --git a/src/ahriman/web/views/v1/service/request.py b/src/ahriman/web/views/v1/service/request.py index ce99420a..19a850c8 100644 --- a/src/ahriman/web/views/v1/service/request.py +++ b/src/ahriman/web/views/v1/service/request.py @@ -74,6 +74,14 @@ class RequestView(BaseView): username = await self.username() repository_id = self.repository_id() - process_id = self.spawner.packages_add(repository_id, packages, username, patches=patches, now=False) + process_id = self.spawner.packages_add( + repository_id, + packages, + username, + patches=patches, + now=False, + increment=False, # no-increment doesn't work here + refresh=False, # refresh doesn't work here + ) return json_response({"process_id": process_id}) diff --git a/src/ahriman/web/views/v1/service/update.py b/src/ahriman/web/views/v1/service/update.py index f5587436..c46ed702 100644 --- a/src/ahriman/web/views/v1/service/update.py +++ b/src/ahriman/web/views/v1/service/update.py @@ -77,6 +77,8 @@ class UpdateView(BaseView): aur=data.get("aur", True), local=data.get("local", True), manual=data.get("manual", True), + increment=data.get("increment", True), + refresh=data.get("refresh", False), ) return json_response({"process_id": process_id}) diff --git a/tests/ahriman/application/application/test_application_repository.py b/tests/ahriman/application/application/test_application_repository.py index 67c22dba..21138e18 100644 --- a/tests/ahriman/application/application/test_application_repository.py +++ b/tests/ahriman/application/application/test_application_repository.py @@ -197,33 +197,34 @@ def test_update(application_repository: ApplicationRepository, package_ahriman: paths = [package.filepath for package in package_ahriman.packages.values()] tree = Tree([Leaf(package_ahriman)]) - mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels()) + resolve_mock = mocker.patch("ahriman.application.application.workers.local_updater.LocalUpdater.partition", + return_value=tree.levels()) mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=paths) - build_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_build", return_value=result) - update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update", return_value=result) + build_mock = mocker.patch("ahriman.application.application.workers.local_updater.LocalUpdater.update", + return_value=result) + update_mock = mocker.patch("ahriman.core.repository.Repository.process_update", return_value=result) on_result_mock = mocker.patch( "ahriman.application.application.application_repository.ApplicationRepository.on_result") application_repository.update([package_ahriman], Packagers("username"), bump_pkgrel=True) + resolve_mock.assert_called_once_with([package_ahriman]) build_mock.assert_called_once_with([package_ahriman], Packagers("username"), bump_pkgrel=True) - update_mock.assert_has_calls([ - MockCall(paths, Packagers("username")), - MockCall(paths, Packagers("username")), - ]) + update_mock.assert_called_once_with(paths, Packagers("username")) on_result_mock.assert_has_calls([MockCall(result), MockCall(result)]) -def test_update_empty(application_repository: ApplicationRepository, package_ahriman: Package, +def test_update_empty(application_repository: ApplicationRepository, package_ahriman: Package, result: Result, mocker: MockerFixture) -> None: """ must skip updating repository if no packages supplied """ tree = Tree([Leaf(package_ahriman)]) - mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels()) - mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=[]) - mocker.patch("ahriman.core.repository.executor.Executor.process_build") - update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update") + mocker.patch("ahriman.application.application.workers.Updater.partition", return_value=tree.levels()) + mocker.patch("ahriman.core.repository.Repository.packages_built", return_value=[]) + mocker.patch("ahriman.application.application.workers.local_updater.LocalUpdater.update", return_value=result) + mocker.patch("ahriman.application.application.application_repository.ApplicationRepository.on_result") + update_mock = mocker.patch("ahriman.core.repository.Repository.process_update") application_repository.update([package_ahriman]) update_mock.assert_not_called() diff --git a/tests/ahriman/application/application/workers/conftest.py b/tests/ahriman/application/application/workers/conftest.py new file mode 100644 index 00000000..9e080702 --- /dev/null +++ b/tests/ahriman/application/application/workers/conftest.py @@ -0,0 +1,48 @@ +import pytest + +from ahriman.application.application.workers import Updater +from ahriman.application.application.workers.local_updater import LocalUpdater +from ahriman.application.application.workers.remote_updater import RemoteUpdater +from ahriman.core.configuration import Configuration +from ahriman.core.repository import Repository +from ahriman.models.worker import Worker + + +@pytest.fixture +def local_updater(repository: Repository) -> LocalUpdater: + """ + local updater fixture + + Args: + repository(Repository): repository fixture + + Returns: + LocalUpdater: local updater test instance + """ + return LocalUpdater(repository) + + +@pytest.fixture +def remote_updater(configuration: Configuration) -> RemoteUpdater: + """ + local updater fixture + + Args: + configuration(Configuration): configuration fixture + + Returns: + RemoteUpdater: remote updater test instance + """ + _, repository_id = configuration.check_loaded() + return RemoteUpdater([Worker("remote1"), Worker("remote2")], repository_id, configuration) + + +@pytest.fixture +def updater() -> Updater: + """ + empty updater fixture + + Returns: + Updater: empty updater test instance + """ + return Updater() diff --git a/tests/ahriman/application/application/workers/test_local_updater.py b/tests/ahriman/application/application/workers/test_local_updater.py new file mode 100644 index 00000000..72599cfa --- /dev/null +++ b/tests/ahriman/application/application/workers/test_local_updater.py @@ -0,0 +1,30 @@ +from pytest_mock import MockerFixture + +from ahriman.application.application.workers.local_updater import LocalUpdater +from ahriman.models.package import Package +from ahriman.models.packagers import Packagers +from ahriman.models.result import Result + + +def test_partition(local_updater: LocalUpdater, mocker: MockerFixture) -> None: + """ + must partition as tree resolution + """ + resolve_mock = mocker.patch("ahriman.core.tree.Tree.resolve") + local_updater.partition([]) + resolve_mock.assert_called_once_with([]) + + +def test_update(local_updater: LocalUpdater, package_ahriman: Package, result: Result, + mocker: MockerFixture) -> None: + """ + must process package updates + """ + paths = [package.filepath for package in package_ahriman.packages.values()] + mocker.patch("ahriman.core.repository.Repository.packages_built", return_value=paths) + build_mock = mocker.patch("ahriman.core.repository.Repository.process_build", return_value=result) + update_mock = mocker.patch("ahriman.core.repository.Repository.process_update", return_value=result) + + assert local_updater.update([package_ahriman], Packagers("username"), bump_pkgrel=True) == result + build_mock.assert_called_once_with([package_ahriman], Packagers("username"), bump_pkgrel=True) + update_mock.assert_called_once_with(paths, Packagers("username")) diff --git a/tests/ahriman/application/application/workers/test_remote_updater.py b/tests/ahriman/application/application/workers/test_remote_updater.py new file mode 100644 index 00000000..9b9580a4 --- /dev/null +++ b/tests/ahriman/application/application/workers/test_remote_updater.py @@ -0,0 +1,84 @@ +from pytest_mock import MockerFixture + +from ahriman.application.application.workers.remote_updater import RemoteUpdater +from ahriman.core.http import SyncAhrimanClient +from ahriman.models.package import Package +from ahriman.models.packagers import Packagers +from ahriman.models.result import Result + + +def test_clients(remote_updater: RemoteUpdater) -> None: + """ + must return map of clients + """ + worker = remote_updater.workers[0] + client = SyncAhrimanClient() + remote_updater._clients.append((worker, client)) + + assert remote_updater.clients == {worker: client} + + +def test_update_url(remote_updater: RemoteUpdater) -> None: + """ + must generate update url correctly + """ + worker = remote_updater.workers[0] + assert remote_updater._update_url(worker).startswith(worker.address) + assert remote_updater._update_url(worker).endswith("/api/v1/service/add") + + +def test_next_worker(remote_updater: RemoteUpdater) -> None: + """ + must return next not used worker + """ + assert remote_updater.next_worker()[0] == remote_updater.workers[0] + assert len(remote_updater.clients) == 1 + assert remote_updater.workers[0] in remote_updater.clients + + assert remote_updater.next_worker()[0] == remote_updater.workers[1] + assert remote_updater.workers[1] in remote_updater.clients + assert len(remote_updater.clients) == 2 + + +def test_next_worker_cycle(remote_updater: RemoteUpdater) -> None: + """ + must return first used worker if no free workers left + """ + worker1, client1 = remote_updater.next_worker() + worker2, client2 = remote_updater.next_worker() + + assert remote_updater.next_worker() == (worker1, client1) + assert remote_updater.next_worker() == (worker2, client2) + assert remote_updater.next_worker() == (worker1, client1) + + +def test_partition(remote_updater: RemoteUpdater, mocker: MockerFixture) -> None: + """ + must partition as tree partition + """ + resolve_mock = mocker.patch("ahriman.core.tree.Tree.partition") + remote_updater.partition([]) + resolve_mock.assert_called_once_with([], count=2) + + +def test_update(remote_updater: RemoteUpdater, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must process remote package updates + """ + worker, client = remote_updater.next_worker() + worker_mock = mocker.patch("ahriman.application.application.workers.remote_updater.RemoteUpdater.next_worker", + return_value=(worker, client)) + request_mock = mocker.patch("ahriman.core.http.SyncAhrimanClient.make_request") + + assert remote_updater.update([package_ahriman], Packagers("username"), bump_pkgrel=True) == Result() + worker_mock.assert_called_once_with() + request_mock.assert_called_once_with("POST", remote_updater._update_url(worker), + params=remote_updater.repository_id.query(), + json={ + "increment": True, + "packager": "username", + "packages": [package_ahriman.base], + "patches": [], + "refresh": True, + } + ) diff --git a/tests/ahriman/application/application/workers/test_updater.py b/tests/ahriman/application/application/workers/test_updater.py new file mode 100644 index 00000000..85a85ce9 --- /dev/null +++ b/tests/ahriman/application/application/workers/test_updater.py @@ -0,0 +1,50 @@ +import pytest + +from ahriman.application.application.workers import Updater +from ahriman.application.application.workers.local_updater import LocalUpdater +from ahriman.application.application.workers.remote_updater import RemoteUpdater +from ahriman.core.configuration import Configuration +from ahriman.core.repository import Repository +from ahriman.models.worker import Worker + + +def test_load(configuration: Configuration, repository: Repository) -> None: + """ + must load local updater if empty worker list is set + """ + _, repository_id = configuration.check_loaded() + assert isinstance(Updater.load(repository_id, configuration, repository), LocalUpdater) + assert isinstance(Updater.load(repository_id, configuration, repository, []), LocalUpdater) + + +def test_load_from_option(configuration: Configuration, repository: Repository) -> None: + """ + must load remote updater if nonempty worker list is set + """ + _, repository_id = configuration.check_loaded() + assert isinstance(Updater.load(repository_id, configuration, repository, [Worker("remote")]), RemoteUpdater) + + +def test_load_from_configuration(configuration: Configuration, repository: Repository) -> None: + """ + must load remote updater from settings + """ + configuration.set_option("build", "workers", "remote") + _, repository_id = configuration.check_loaded() + assert isinstance(Updater.load(repository_id, configuration, repository), RemoteUpdater) + + +def test_partition(updater: Updater) -> None: + """ + must raise not implemented error for missing partition method + """ + with pytest.raises(NotImplementedError): + updater.partition([]) + + +def test_update(updater: Updater) -> None: + """ + must raise not implemented error for missing update method + """ + with pytest.raises(NotImplementedError): + updater.update([]) diff --git a/tests/ahriman/core/test_spawn.py b/tests/ahriman/core/test_spawn.py index cb0e1355..40377faf 100644 --- a/tests/ahriman/core/test_spawn.py +++ b/tests/ahriman/core/test_spawn.py @@ -101,8 +101,10 @@ def test_packages_add(spawner: Spawn, repository_id: RepositoryId, mocker: Mocke must call package addition """ spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") - assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=[], now=False) - spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None) + assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, + patches=[], now=False, increment=False, refresh=False) + spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", + **{"username": None, "variable": [], "no-increment": ""}) def test_packages_add_with_build(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None: @@ -110,8 +112,10 @@ def test_packages_add_with_build(spawner: Spawn, repository_id: RepositoryId, mo must call package addition with update """ spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") - assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=[], now=True) - spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None, now="") + assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, + patches=[], now=True, increment=False, refresh=False) + spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", + **{"username": None, "variable": [], "no-increment": "", "now": ""}) def test_packages_add_with_username(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None: @@ -119,8 +123,10 @@ def test_packages_add_with_username(spawner: Spawn, repository_id: RepositoryId, must call package addition with username """ spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") - assert spawner.packages_add(repository_id, ["ahriman", "linux"], "username", patches=[], now=False) - spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username="username") + assert spawner.packages_add(repository_id, ["ahriman", "linux"], "username", + patches=[], now=False, increment=False, refresh=False) + spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", + **{"username": "username", "variable": [], "no-increment": ""}) def test_packages_add_with_patches(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None: @@ -129,9 +135,36 @@ def test_packages_add_with_patches(spawner: Spawn, repository_id: RepositoryId, """ patches = [PkgbuildPatch("key", "value"), PkgbuildPatch("key", "value")] spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") - assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=patches, now=False) - spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None, - variable=[patch.serialize() for patch in patches]) + assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, + patches=patches, now=False, increment=False, refresh=False) + spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", + **{ + "username": None, + "variable": [patch.serialize() for patch in patches], + "no-increment": "" + }) + + +def test_packages_add_with_increment(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None: + """ + must call package addition with increment + """ + spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") + assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, + patches=[], now=False, increment=True, refresh=False) + spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", + **{"username": None, "variable": [], "increment": ""}) + + +def test_packages_add_with_refresh(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None: + """ + must call package addition with refresh + """ + spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") + assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, + patches=[], now=False, increment=False, refresh=True) + spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", + **{"username": None, "variable": [], "no-increment": "", "refresh": ""}) def test_packages_rebuild(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None: @@ -139,9 +172,19 @@ def test_packages_rebuild(spawner: Spawn, repository_id: RepositoryId, mocker: M must call package rebuild """ spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") - assert spawner.packages_rebuild(repository_id, "python", "packager") + assert spawner.packages_rebuild(repository_id, "python", "packager", increment=False) spawn_mock.assert_called_once_with(repository_id, "repo-rebuild", - **{"depends-on": "python", "username": "packager"}) + **{"depends-on": "python", "username": "packager", "no-increment": ""}) + + +def test_packages_rebuild_with_increment(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None: + """ + must call package rebuild with increment + """ + spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") + assert spawner.packages_rebuild(repository_id, "python", "packager", increment=True) + spawn_mock.assert_called_once_with(repository_id, "repo-rebuild", + **{"depends-on": "python", "username": "packager", "increment": ""}) def test_packages_remove(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None: @@ -159,27 +202,61 @@ def test_packages_update(spawner: Spawn, repository_id: RepositoryId, mocker: Mo """ spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") - assert spawner.packages_update(repository_id, "packager", aur=True, local=True, manual=True) - args = {"username": "packager", "aur": "", "local": "", "manual": ""} + assert spawner.packages_update(repository_id, "packager", + aur=True, local=True, manual=True, increment=False, refresh=False) + args = {"username": "packager", "aur": "", "local": "", "manual": "", "no-increment": ""} spawn_mock.assert_called_once_with(repository_id, "repo-update", **args) spawn_mock.reset_mock() - assert spawner.packages_update(repository_id, "packager", aur=False, local=True, manual=True) - args = {"username": "packager", "no-aur": "", "local": "", "manual": ""} + assert spawner.packages_update(repository_id, "packager", + aur=False, local=True, manual=True, increment=False, refresh=False) + args = {"username": "packager", "no-aur": "", "local": "", "manual": "", "no-increment": ""} spawn_mock.assert_called_once_with(repository_id, "repo-update", **args) spawn_mock.reset_mock() - assert spawner.packages_update(repository_id, "packager", aur=True, local=False, manual=True) - args = {"username": "packager", "aur": "", "no-local": "", "manual": ""} + assert spawner.packages_update(repository_id, "packager", + aur=True, local=False, manual=True, increment=False, refresh=False) + args = {"username": "packager", "aur": "", "no-local": "", "manual": "", "no-increment": ""} spawn_mock.assert_called_once_with(repository_id, "repo-update", **args) spawn_mock.reset_mock() - assert spawner.packages_update(repository_id, "packager", aur=True, local=True, manual=False) - args = {"username": "packager", "aur": "", "local": "", "no-manual": ""} + assert spawner.packages_update(repository_id, "packager", + aur=True, local=True, manual=False, increment=False, refresh=False) + args = {"username": "packager", "aur": "", "local": "", "no-manual": "", "no-increment": ""} spawn_mock.assert_called_once_with(repository_id, "repo-update", **args) spawn_mock.reset_mock() +def test_packages_update_with_increment(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None: + """ + must call repo update with increment + """ + spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") + assert spawner.packages_update(repository_id, None, + aur=True, local=True, manual=True, increment=True, refresh=False) + spawn_mock.assert_called_once_with(repository_id, "repo-update", + **{"username": None, "aur": "", "local": "", "manual": "", "increment": ""}) + + +def test_packages_update_with_refresh(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None: + """ + must call repo update with refresh + """ + spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process") + assert spawner.packages_update(repository_id, None, + aur=True, local=True, manual=True, increment=False, refresh=True) + spawn_mock.assert_called_once_with(repository_id, "repo-update", + **{ + "username": None, + "aur": "", + "local": "", + "manual": "", + "no-increment": "", + "refresh": "", + } + ) + + def test_run(spawner: Spawn, mocker: MockerFixture) -> None: """ must implement run method diff --git a/tests/ahriman/core/test_tree.py b/tests/ahriman/core/test_tree.py index 906e21d9..42775f88 100644 --- a/tests/ahriman/core/test_tree.py +++ b/tests/ahriman/core/test_tree.py @@ -88,6 +88,13 @@ def test_tree_balance() -> None: assert third == [leaf5] +def test_tree_balance_empty() -> None: + """ + must do not fail on empty tree balancing + """ + assert Tree.balance([]) == [] + + def test_tree_partition(package_ahriman: Package, package_python_schedule: Package) -> None: """ must partition dependencies tree diff --git a/tests/ahriman/models/test_worker.py b/tests/ahriman/models/test_worker.py new file mode 100644 index 00000000..b6e5e72e --- /dev/null +++ b/tests/ahriman/models/test_worker.py @@ -0,0 +1,10 @@ +from ahriman.models.worker import Worker + + +def test_post_init() -> None: + """ + must read identifier from location if not set + """ + assert Worker("http://localhost:8080").identifier == "localhost:8080" + assert Worker("remote").identifier == "" # not a valid url + assert Worker("remote", identifier="id").identifier == "id" diff --git a/tests/ahriman/web/schemas/test_build_options_schema.py b/tests/ahriman/web/schemas/test_build_options_schema.py new file mode 100644 index 00000000..1982fb6b --- /dev/null +++ b/tests/ahriman/web/schemas/test_build_options_schema.py @@ -0,0 +1 @@ +# schema testing goes in view class tests diff --git a/tests/ahriman/web/views/test_view_base.py b/tests/ahriman/web/views/test_view_base.py index 9c4ef4fe..64331a09 100644 --- a/tests/ahriman/web/views/test_view_base.py +++ b/tests/ahriman/web/views/test_view_base.py @@ -193,6 +193,9 @@ async def test_username(base: BaseView, mocker: MockerFixture) -> None: policy = AsyncMock() policy.identify.return_value = "identity" mocker.patch("aiohttp.web.Application.get", return_value=policy) + json = AsyncMock() + json.return_value = {} + base._request = pytest.helpers.request(base.request.app, "", "", json=json) assert await base.username() == "identity" policy.identify.assert_called_once_with(base.request) @@ -202,4 +205,26 @@ async def test_username_no_auth(base: BaseView) -> None: """ must return None in case if auth is disabled """ + json = AsyncMock() + json.return_value = {} + base._request = pytest.helpers.request(base.request.app, "", "", json=json) + + assert await base.username() is None + + +async def test_username_request(base: BaseView) -> None: + """ + must read packager from request + """ + json = AsyncMock() + json.return_value = {"packager": "identity"} + base._request = pytest.helpers.request(base.request.app, "", "", json=json) + + assert await base.username() == "identity" + + +async def test_username_request_exception(base: BaseView) -> None: + """ + must not fail in case if cannot read request + """ assert await base.username() is None diff --git a/tests/ahriman/web/views/v1/service/test_view_v1_service_add.py b/tests/ahriman/web/views/v1/service/test_view_v1_service_add.py index e1ffec43..f80b8bab 100644 --- a/tests/ahriman/web/views/v1/service/test_view_v1_service_add.py +++ b/tests/ahriman/web/views/v1/service/test_view_v1_service_add.py @@ -41,7 +41,8 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc assert not request_schema.validate(payload) response = await client.post("/api/v1/service/add", json=payload) assert response.ok - add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", patches=[], now=True) + add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", + patches=[], now=True, increment=True, refresh=False) json = await response.json() assert json["process_id"] == "abc" @@ -74,7 +75,8 @@ async def test_post_patches(client: TestClient, repository_id: RepositoryId, moc response = await client.post("/api/v1/service/add", json=payload) assert response.ok add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", - patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")], now=True) + patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")], + now=True, increment=True, refresh=False) async def test_post_empty(client: TestClient, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/web/views/v1/service/test_view_v1_service_rebuild.py b/tests/ahriman/web/views/v1/service/test_view_v1_service_rebuild.py index 1fe740fb..fbe74b13 100644 --- a/tests/ahriman/web/views/v1/service/test_view_v1_service_rebuild.py +++ b/tests/ahriman/web/views/v1/service/test_view_v1_service_rebuild.py @@ -40,7 +40,7 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc assert not request_schema.validate(payload) response = await client.post("/api/v1/service/rebuild", json=payload) assert response.ok - rebuild_mock.assert_called_once_with(repository_id, "python", "username") + rebuild_mock.assert_called_once_with(repository_id, "python", "username", increment=True) json = await response.json() assert json["process_id"] == "abc" diff --git a/tests/ahriman/web/views/v1/service/test_view_v1_service_request.py b/tests/ahriman/web/views/v1/service/test_view_v1_service_request.py index d2f8b7ce..5cc9b789 100644 --- a/tests/ahriman/web/views/v1/service/test_view_v1_service_request.py +++ b/tests/ahriman/web/views/v1/service/test_view_v1_service_request.py @@ -41,7 +41,8 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc assert not request_schema.validate(payload) response = await client.post("/api/v1/service/request", json=payload) assert response.ok - add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", patches=[], now=False) + add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", + patches=[], now=False, increment=False, refresh=False) json = await response.json() assert json["process_id"] == "abc" @@ -74,7 +75,8 @@ async def test_post_patches(client: TestClient, repository_id: RepositoryId, moc response = await client.post("/api/v1/service/request", json=payload) assert response.ok add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", - patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")], now=False) + patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")], + now=False, increment=False, refresh=False) async def test_post_exception(client: TestClient, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/web/views/v1/service/test_view_v1_service_update.py b/tests/ahriman/web/views/v1/service/test_view_v1_service_update.py index 2990599a..56121190 100644 --- a/tests/ahriman/web/views/v1/service/test_view_v1_service_update.py +++ b/tests/ahriman/web/views/v1/service/test_view_v1_service_update.py @@ -40,6 +40,8 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc "aur": True, "local": True, "manual": True, + "increment": True, + "refresh": False, } for payload in (