diff --git a/docs/ahriman.application.application.rst b/docs/ahriman.application.application.rst
index 63d399c3..c3672dde 100644
--- a/docs/ahriman.application.application.rst
+++ b/docs/ahriman.application.application.rst
@@ -1,6 +1,14 @@
ahriman.application.application package
=======================================
+Subpackages
+-----------
+
+.. toctree::
+ :maxdepth: 4
+
+ ahriman.application.application.workers
+
Submodules
----------
diff --git a/docs/ahriman.application.application.workers.rst b/docs/ahriman.application.application.workers.rst
new file mode 100644
index 00000000..eb61f08f
--- /dev/null
+++ b/docs/ahriman.application.application.workers.rst
@@ -0,0 +1,37 @@
+ahriman.application.application.workers package
+===============================================
+
+Submodules
+----------
+
+ahriman.application.application.workers.local\_updater module
+-------------------------------------------------------------
+
+.. automodule:: ahriman.application.application.workers.local_updater
+ :members:
+ :no-undoc-members:
+ :show-inheritance:
+
+ahriman.application.application.workers.remote\_updater module
+--------------------------------------------------------------
+
+.. automodule:: ahriman.application.application.workers.remote_updater
+ :members:
+ :no-undoc-members:
+ :show-inheritance:
+
+ahriman.application.application.workers.updater module
+------------------------------------------------------
+
+.. automodule:: ahriman.application.application.workers.updater
+ :members:
+ :no-undoc-members:
+ :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ahriman.application.application.workers
+ :members:
+ :no-undoc-members:
+ :show-inheritance:
diff --git a/docs/ahriman.models.rst b/docs/ahriman.models.rst
index 13fc69a2..461c044e 100644
--- a/docs/ahriman.models.rst
+++ b/docs/ahriman.models.rst
@@ -252,6 +252,14 @@ ahriman.models.waiter module
:no-undoc-members:
:show-inheritance:
+ahriman.models.worker module
+----------------------------
+
+.. automodule:: ahriman.models.worker
+ :members:
+ :no-undoc-members:
+ :show-inheritance:
+
Module contents
---------------
diff --git a/docs/ahriman.web.schemas.rst b/docs/ahriman.web.schemas.rst
index 83691510..fa3bc8b0 100644
--- a/docs/ahriman.web.schemas.rst
+++ b/docs/ahriman.web.schemas.rst
@@ -20,6 +20,14 @@ ahriman.web.schemas.auth\_schema module
:no-undoc-members:
:show-inheritance:
+ahriman.web.schemas.build\_options\_schema module
+-------------------------------------------------
+
+.. automodule:: ahriman.web.schemas.build_options_schema
+ :members:
+ :no-undoc-members:
+ :show-inheritance:
+
ahriman.web.schemas.changes\_schema module
------------------------------------------
diff --git a/docs/configuration.rst b/docs/configuration.rst
index 79064e07..a3f41665 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -86,6 +86,7 @@ Build related configuration. Group name can refer to architecture, e.g. ``build:
* ``triggers`` - list of ``ahriman.core.triggers.Trigger`` class implementation (e.g. ``ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger``) which will be loaded and run at the end of processing, space separated list of strings, optional. You can also specify triggers by their paths, e.g. ``/usr/lib/python3.10/site-packages/ahriman/core/report/report.py.ReportTrigger``. Triggers are run in the order of mention.
* ``triggers_known`` - optional list of ``ahriman.core.triggers.Trigger`` class implementations which are not run automatically and used only for trigger discovery and configuration validation.
* ``vcs_allowed_age`` - maximal age in seconds of the VCS packages before their version will be updated with its remote source, integer, optional, default ``604800``.
+* ``workers`` - list of worker nodes addresses used for build process, space separated list of strings, optional. Each worker address must be valid and reachable url, e.g. ``https://10.0.0.1:8080``. If none set, the build process will be run on the current node.
``repository`` group
--------------------
diff --git a/docs/faq.rst b/docs/faq.rst
index d90c11af..c4bf6595 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -1022,6 +1022,116 @@ This action must be done in two steps:
#. Remove package on worker.
#. Remove package on master node.
+Delegate builds to remote workers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This setup heavily uses upload feature described above and, in addition, also delegates build process automatically to build machines. Same as above, there must be at least two instances available (``master`` and ``worker``), however, all ``worker`` nodes must be run in the web service mode.
+
+Master node configuration
+"""""""""""""""""""""""""
+
+In addition to the configuration above, the worker list must be defined in configuration file (``build.workers`` option), i.e.:
+
+.. code-block:: ini
+
+ [build]
+ workers = https://worker1.example.com https://worker2.example.com
+
+ [web]
+ enable_archive_upload = yes
+ wait_timeout = 0
+
+In the example above, ``https://worker1.example.com`` and ``https://worker2.example.com`` are remote ``worker`` node addresses available for ``master`` node.
+
+In case if authentication is required (which is recommended way to setup it), it can be set by using ``status`` section as usual.
+
+Worker nodes configuration
+""""""""""""""""""""""""""
+
+It is required to point to the master node repository, otherwise internal dependencies will not be handled correctly. In order to do so, the ``--server`` argument (or ``AHRIMAN_REPOSITORY_SERVER`` environment variable for docker images) can be used.
+
+Also, in case if authentication is enabled, the same user with the same password must be created for all workers.
+
+It is also recommended to set ``web.wait_timeout`` to infinte in case of multiple conflicting runs.
+
+Other settings are the same as mentioned above.
+
+Triple node minimal docker example
+""""""""""""""""""""""""""""""""""
+
+In this example, all instances are run on the same machine with address ``172.17.0.1`` with ports available outside of container. Master node config (``master.ini``) as:
+
+.. code-block:: ini
+
+ [auth]
+ target = mapping
+
+ [status]
+ username = builder-user
+ password = very-secure-password
+
+ [build]
+ workers = http://172.17.0.1:8081 http://172.17.0.1:8082
+
+ [web]
+ enable_archive_upload = yes
+ wait_timeout = 0
+
+Command to run master node:
+
+.. code-block:: shell
+
+ docker run --privileged -p 8080:8080 -e AHRIMAN_PORT=8080 -v master.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web
+
+Worker nodes (applicable for all workers) config (``worker.ini``) as:
+
+.. code-block:: ini
+
+ [auth]
+ target = mapping
+
+ [status]
+ address = http://172.17.0.1:8080
+ username = builder-user
+ password = very-secure-password
+
+ [upload]
+ target = remote-service
+
+ [remote-service]
+
+ [report]
+ target = remote-call
+
+ [remote-call]
+ manual = yes
+ wait_timeout = 0
+
+ [build]
+ triggers = ahriman.core.upload.UploadTrigger ahriman.core.report.ReportTrigger
+
+Command to run worker nodes (considering there will be two workers, one is on ``8081`` port and other is on ``8082``):
+
+.. code-block:: ini
+
+ docker run --privileged -p 8081:8081 -e AHRIMAN_PORT=8081 -v worker.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web
+ docker run --privileged -p 8082:8082 -e AHRIMAN_PORT=8082 -v worker.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web
+
+Unlike the previous setup, it doesn't require to mount repository root for ``worker`` nodes, because ``worker`` nodes don't use it anyway.
+
+Addition of new package, package removal, repository update
+"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+In all scenarios, update process must be run only on ``master`` node. Unlike the setup described above, automatic update must be enabled only for ``master`` node also.
+
+Known limitations
+"""""""""""""""""
+
+* Workers don't support local packages. However, it is possible to build custom packages by providing sources by using ``ahriman.core.gitremote.RemotePullTrigger`` trigger.
+* No dynamic nodes discovery. In case if one of worker nodes is unavailable, the build process will fail.
+* No pkgrel bump on conflicts. Well, it works, however, it isn't guaranteed.
+* The identical user must be created for all workers. However, the ``master`` node user can be different from this one.
+
Maintenance packages
--------------------
diff --git a/package/share/man/man1/ahriman.1 b/package/share/man/man1/ahriman.1
index 68e411d0..b8f350ab 100644
--- a/package/share/man/man1/ahriman.1
+++ b/package/share/man/man1/ahriman.1
@@ -1,4 +1,4 @@
-.TH AHRIMAN "1" "2023\-11\-29" "ahriman" "Generated Python Manual"
+.TH AHRIMAN "1" "2023\-12\-08" "ahriman" "Generated Python Manual"
.SH NAME
ahriman
.SH SYNOPSIS
diff --git a/src/ahriman/application/application/application_repository.py b/src/ahriman/application/application/application_repository.py
index 0b4ac8c6..4fad83de 100644
--- a/src/ahriman/application/application/application_repository.py
+++ b/src/ahriman/application/application/application_repository.py
@@ -18,11 +18,10 @@
# along with this program. If not, see .
#
from collections.abc import Iterable
-from pathlib import Path
from ahriman.application.application.application_properties import ApplicationProperties
+from ahriman.application.application.workers import Updater
from ahriman.core.build_tools.sources import Sources
-from ahriman.core.tree import Tree
from ahriman.models.package import Package
from ahriman.models.packagers import Packagers
from ahriman.models.result import Result
@@ -154,26 +153,25 @@ class ApplicationRepository(ApplicationProperties):
Returns:
Result: update result
"""
- def process_update(paths: Iterable[Path], result: Result) -> None:
- if not paths:
- return # don't need to process if no update supplied
- update_result = self.repository.process_update(paths, packagers)
- self.on_result(result.merge(update_result))
+ result = Result()
- # process built packages
- build_result = Result()
- packages = self.repository.packages_built()
- process_update(packages, build_result)
+ # process already built packages if any
+ built_packages = self.repository.packages_built()
+ if built_packages: # speedup a bit
+ build_result = self.repository.process_update(built_packages, packagers)
+ result.merge(build_result)
+ self.on_result(result.merge(build_result))
- # process manual packages
- tree = Tree.resolve(updates)
- for num, level in enumerate(tree):
- self.logger.info("processing level #%i %s", num, [package.base for package in level])
- build_result = self.repository.process_build(level, packagers, bump_pkgrel=bump_pkgrel)
- packages = self.repository.packages_built()
- process_update(packages, build_result)
+ builder = Updater.load(self.repository_id, self.configuration, self.repository)
- return build_result
+ # ok so for now we split all packages into chunks and process each chunk accordingly
+ partitions = builder.partition(updates)
+ for num, partition in enumerate(partitions):
+ self.logger.info("processing chunk #%i %s", num, [package.base for package in partition])
+ build_result = builder.update(partition, packagers, bump_pkgrel=bump_pkgrel)
+ self.on_result(result.merge(build_result))
+
+ return result
def updates(self, filter_packages: Iterable[str], *,
aur: bool, local: bool, manual: bool, vcs: bool) -> list[Package]:
diff --git a/src/ahriman/application/application/workers/__init__.py b/src/ahriman/application/application/workers/__init__.py
new file mode 100644
index 00000000..7dd71d6b
--- /dev/null
+++ b/src/ahriman/application/application/workers/__init__.py
@@ -0,0 +1,20 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+from ahriman.application.application.workers.updater import Updater
diff --git a/src/ahriman/application/application/workers/local_updater.py b/src/ahriman/application/application/workers/local_updater.py
new file mode 100644
index 00000000..35b8e7df
--- /dev/null
+++ b/src/ahriman/application/application/workers/local_updater.py
@@ -0,0 +1,77 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+from collections.abc import Iterable
+
+from ahriman.application.application.workers.updater import Updater
+from ahriman.core.repository import Repository
+from ahriman.core.tree import Tree
+from ahriman.models.package import Package
+from ahriman.models.packagers import Packagers
+from ahriman.models.result import Result
+
+
+class LocalUpdater(Updater):
+ """
+ local build process implementation
+
+ Attributes:
+ repository(Repository): repository instance
+ """
+
+ def __init__(self, repository: Repository) -> None:
+ """
+ default constructor
+
+ Args:
+ repository(Repository): repository instance
+ """
+ self.repository = repository
+
+ def partition(self, packages: Iterable[Package]) -> list[list[Package]]:
+ """
+ split packages into partitions to be processed by this worker
+
+ Args:
+ packages(Iterable[Package]): list of packages to partition
+
+ Returns:
+ list[list[Package]]: packages partitioned by this worker type
+ """
+ return Tree.resolve(packages)
+
+ def update(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
+ bump_pkgrel: bool = False) -> Result:
+ """
+ run package updates
+
+ Args:
+ updates(Iterable[Package]): list of packages to update
+ packagers(Packagers | None, optional): optional override of username for build process
+ (Default value = None)
+ bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False)
+
+ Returns:
+ Result: update result
+ """
+ build_result = self.repository.process_build(updates, packagers, bump_pkgrel=bump_pkgrel)
+ packages = self.repository.packages_built()
+ update_result = self.repository.process_update(packages, packagers)
+
+ return build_result.merge(update_result)
diff --git a/src/ahriman/application/application/workers/remote_updater.py b/src/ahriman/application/application/workers/remote_updater.py
new file mode 100644
index 00000000..0064adbb
--- /dev/null
+++ b/src/ahriman/application/application/workers/remote_updater.py
@@ -0,0 +1,140 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+from collections import deque
+from collections.abc import Iterable
+
+from ahriman.application.application.workers.updater import Updater
+from ahriman.core.configuration import Configuration
+from ahriman.core.http import SyncAhrimanClient
+from ahriman.core.tree import Tree
+from ahriman.models.package import Package
+from ahriman.models.packagers import Packagers
+from ahriman.models.repository_id import RepositoryId
+from ahriman.models.result import Result
+from ahriman.models.worker import Worker
+
+
+class RemoteUpdater(Updater):
+ """
+ remote update worker
+
+ Attributes:
+ configuration(Configuration): configuration instance
+ repository_id(RepositoryId): repository unique identifier
+ workers(list[Worker]): worker identifiers
+ """
+
+ def __init__(self, workers: list[Worker], repository_id: RepositoryId, configuration: Configuration) -> None:
+ """
+ default constructor
+
+ Args:
+ workers(list[Worker]): worker identifiers
+ repository_id(RepositoryId): repository unique identifier
+ configuration(Configuration): configuration instance
+ """
+ self.workers = workers
+ self.repository_id = repository_id
+ self.configuration = configuration
+
+ self._clients: deque[tuple[Worker, SyncAhrimanClient]] = deque()
+
+ @property
+ def clients(self) -> dict[Worker, SyncAhrimanClient]:
+ """
+ extract loaded clients. Note that this method yields only workers which have been already loaded
+
+ Returns:
+ dict[Worker, SyncAhrimanClient]: map of the worker to the related web client
+ """
+ return dict(self._clients)
+
+ @staticmethod
+ def _update_url(worker: Worker) -> str:
+ """
+ get url for updates
+
+ Args:
+ worker(Worker): worker identifier
+
+ Returns:
+ str: full url for web service to run update process
+ """
+ return f"{worker.address}/api/v1/service/add"
+
+ def next_worker(self) -> tuple[Worker, SyncAhrimanClient]:
+ """
+ generate next not-used web client. In case if all clients have been already used, it yields next not used client
+
+ Returns:
+ tuple[Worker, SyncAhrimanClient]: worker and constructed client instance for the web
+ """
+ # check if there is not used yet worker
+ worker = next((worker for worker in self.workers if worker not in self.clients), None)
+ if worker is not None:
+ client = SyncAhrimanClient(self.configuration, "status")
+ client.address = worker.address
+ else:
+ worker, client = self._clients.popleft()
+
+ # register worker in the queue
+ self._clients.append((worker, client))
+
+ return worker, client
+
+ def partition(self, packages: Iterable[Package]) -> list[list[Package]]:
+ """
+ split packages into partitions to be processed by this worker
+
+ Args:
+ packages(Iterable[Package]): list of packages to partition
+
+ Returns:
+ list[list[Package]]: packages partitioned by this worker type
+ """
+ return Tree.partition(packages, count=len(self.workers))
+
+ def update(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
+ bump_pkgrel: bool = False) -> Result:
+ """
+ run package updates
+
+ Args:
+ updates(Iterable[Package]): list of packages to update
+ packagers(Packagers | None, optional): optional override of username for build process
+ (Default value = None)
+ bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False)
+
+ Returns:
+ Result: update result
+ """
+ payload = {
+ "increment": bump_pkgrel,
+ "packager": packagers.default if packagers is not None else None,
+ "packages": [package.base for package in updates],
+ "patches": [], # might be used later
+ "refresh": True,
+ }
+
+ worker, client = self.next_worker()
+ client.make_request("POST", self._update_url(worker), params=self.repository_id.query(), json=payload)
+ # we don't block here for process
+
+ return Result()
diff --git a/src/ahriman/application/application/workers/updater.py b/src/ahriman/application/application/workers/updater.py
new file mode 100644
index 00000000..a1852a4c
--- /dev/null
+++ b/src/ahriman/application/application/workers/updater.py
@@ -0,0 +1,102 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.log import LazyLogging
+from ahriman.core.repository import Repository
+from ahriman.models.package import Package
+from ahriman.models.packagers import Packagers
+from ahriman.models.repository_id import RepositoryId
+from ahriman.models.result import Result
+from ahriman.models.worker import Worker
+
+
+class Updater(LazyLogging):
+ """
+ updater handler interface
+
+ Attributes:
+ split_method(Callable[[Iterable[Package]], list[list[Package]]]): method to split packages into chunks
+ """
+
+ @staticmethod
+ def load(repository_id: RepositoryId, configuration: Configuration,
+ repository: Repository, workers: list[Worker] | None = None) -> Updater:
+ """
+ construct updaters from parameters
+
+ Args:
+ repository_id(RepositoryId): repository unique identifier
+ configuration(Configuration): configuration instance
+ repository(Repository): repository instance
+ workers(list[Worker] | None, optional): worker identifiers if any (Default value = None)
+
+ Returns:
+ Updater: constructed updater worker
+ """
+ if workers is None:
+ # no workers set explicitly, try to guess from configuration
+ workers = [Worker(address) for address in configuration.getlist("build", "workers", fallback=[])]
+
+ if workers:
+ # there is something we could use as remote workers
+ from ahriman.application.application.workers.remote_updater import RemoteUpdater
+ return RemoteUpdater(workers, repository_id, configuration)
+
+ # and finally no workers available, just use local service
+ from ahriman.application.application.workers.local_updater import LocalUpdater
+ return LocalUpdater(repository)
+
+ def partition(self, packages: Iterable[Package]) -> list[list[Package]]:
+ """
+ split packages into partitions to be processed by this worker
+
+ Args:
+ packages(Iterable[Package]): list of packages to partition
+
+ Returns:
+ list[list[Package]]: packages partitioned by this worker type
+
+ Raises:
+ NotImplementedError: not implemented method
+ """
+ raise NotImplementedError
+
+ def update(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
+ bump_pkgrel: bool = False) -> Result:
+ """
+ run package updates
+
+ Args:
+ updates(Iterable[Package]): list of packages to update
+ packagers(Packagers | None, optional): optional override of username for build process
+ (Default value = None)
+ bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False)
+
+ Returns:
+ Result: update result
+
+ Raises:
+ NotImplementedError: not implemented method
+ """
+ raise NotImplementedError
diff --git a/src/ahriman/core/configuration/schema.py b/src/ahriman/core/configuration/schema.py
index a8a59fd3..b6ed7fd6 100644
--- a/src/ahriman/core/configuration/schema.py
+++ b/src/ahriman/core/configuration/schema.py
@@ -213,6 +213,15 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"coerce": "integer",
"min": 0,
},
+ "workers": {
+ "type": "list",
+ "coerce": "list",
+ "schema": {
+ "type": "string",
+ "empty": False,
+ "is_url": [],
+ },
+ },
},
},
"repository": {
diff --git a/src/ahriman/core/spawn.py b/src/ahriman/core/spawn.py
index 19e00a58..00b277a7 100644
--- a/src/ahriman/core/spawn.py
+++ b/src/ahriman/core/spawn.py
@@ -174,7 +174,7 @@ class Spawn(Thread, LazyLogging):
return self._spawn_process(repository_id, "service-key-import", key, **kwargs)
def packages_add(self, repository_id: RepositoryId, packages: Iterable[str], username: str | None, *,
- patches: list[PkgbuildPatch], now: bool) -> str:
+ patches: list[PkgbuildPatch], now: bool, increment: bool, refresh: bool) -> str:
"""
add packages
@@ -184,19 +184,26 @@ class Spawn(Thread, LazyLogging):
username(str | None): optional override of username for build process
patches(list[PkgbuildPatch]): list of patches to be passed
now(bool): build packages now
+ increment(bool): increment pkgrel on conflict
+ refresh(bool): refresh pacman database before process
Returns:
str: spawned process identifier
"""
- kwargs: dict[str, str | list[str] | None] = {"username": username}
+ kwargs: dict[str, str | list[str] | None] = {
+ "username": username,
+ "variable": [patch.serialize() for patch in patches],
+ self.boolean_action_argument("increment", increment): "",
+ }
if now:
kwargs["now"] = ""
- if patches:
- kwargs["variable"] = [patch.serialize() for patch in patches]
+ if refresh:
+ kwargs["refresh"] = ""
return self._spawn_process(repository_id, "package-add", *packages, **kwargs)
- def packages_rebuild(self, repository_id: RepositoryId, depends_on: str, username: str | None) -> str:
+ def packages_rebuild(self, repository_id: RepositoryId, depends_on: str, username: str | None, *,
+ increment: bool) -> str:
"""
rebuild packages which depend on the specified package
@@ -204,11 +211,16 @@ class Spawn(Thread, LazyLogging):
repository_id(RepositoryId): repository unique identifier
depends_on(str): packages dependency
username(str | None): optional override of username for build process
+ increment(bool): increment pkgrel on conflict
Returns:
str: spawned process identifier
"""
- kwargs = {"depends-on": depends_on, "username": username}
+ kwargs = {
+ "depends-on": depends_on,
+ "username": username,
+ self.boolean_action_argument("increment", increment): "",
+ }
return self._spawn_process(repository_id, "repo-rebuild", **kwargs)
def packages_remove(self, repository_id: RepositoryId, packages: Iterable[str]) -> str:
@@ -225,7 +237,7 @@ class Spawn(Thread, LazyLogging):
return self._spawn_process(repository_id, "package-remove", *packages)
def packages_update(self, repository_id: RepositoryId, username: str | None, *,
- aur: bool, local: bool, manual: bool) -> str:
+ aur: bool, local: bool, manual: bool, increment: bool, refresh: bool) -> str:
"""
run full repository update
@@ -235,6 +247,8 @@ class Spawn(Thread, LazyLogging):
aur(bool): check for aur updates
local(bool): check for local packages updates
manual(bool): check for manual packages
+ increment(bool): increment pkgrel on conflict
+ refresh(bool): refresh pacman database before process
Returns:
str: spawned process identifier
@@ -244,7 +258,11 @@ class Spawn(Thread, LazyLogging):
self.boolean_action_argument("aur", aur): "",
self.boolean_action_argument("local", local): "",
self.boolean_action_argument("manual", manual): "",
+ self.boolean_action_argument("increment", increment): "",
}
+ if refresh:
+ kwargs["refresh"] = ""
+
return self._spawn_process(repository_id, "repo-update", **kwargs)
def run(self) -> None:
diff --git a/src/ahriman/core/tree.py b/src/ahriman/core/tree.py
index f7b04b23..3b9a762b 100644
--- a/src/ahriman/core/tree.py
+++ b/src/ahriman/core/tree.py
@@ -148,6 +148,8 @@ class Tree:
sorted(part, key=lambda leaf: leaf.package.base)
for part in partitions if part
]
+ if not partitions: # nothing to balance
+ return partitions
while True:
min_part, max_part = minmax(partitions, key=len)
diff --git a/src/ahriman/models/worker.py b/src/ahriman/models/worker.py
new file mode 100644
index 00000000..72215ff1
--- /dev/null
+++ b/src/ahriman/models/worker.py
@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+from dataclasses import dataclass, field
+from urllib.parse import urlparse
+
+
+@dataclass(frozen=True)
+class Worker:
+ """
+ worker descriptor
+
+ Attributes:
+ address(str): worker address to be reachable outside
+ identifier(str): worker unique identifier. If none set it will be automatically generated from the address
+ """
+
+ address: str
+ identifier: str = field(default="", kw_only=True)
+
+ def __post_init__(self) -> None:
+ """
+ update identifier based on settings
+ """
+ object.__setattr__(self, "identifier", self.identifier or urlparse(self.address).netloc)
diff --git a/src/ahriman/web/schemas/__init__.py b/src/ahriman/web/schemas/__init__.py
index 91cf090a..2fe4d73f 100644
--- a/src/ahriman/web/schemas/__init__.py
+++ b/src/ahriman/web/schemas/__init__.py
@@ -19,6 +19,7 @@
#
from ahriman.web.schemas.aur_package_schema import AURPackageSchema
from ahriman.web.schemas.auth_schema import AuthSchema
+from ahriman.web.schemas.build_options_schema import BuildOptionsSchema
from ahriman.web.schemas.changes_schema import ChangesSchema
from ahriman.web.schemas.counters_schema import CountersSchema
from ahriman.web.schemas.error_schema import ErrorSchema
diff --git a/src/ahriman/web/schemas/build_options_schema.py b/src/ahriman/web/schemas/build_options_schema.py
new file mode 100644
index 00000000..a5bb2de5
--- /dev/null
+++ b/src/ahriman/web/schemas/build_options_schema.py
@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+from marshmallow import Schema, fields
+
+
+class BuildOptionsSchema(Schema):
+ """
+ request build options schema
+ """
+
+ increment = fields.Boolean(dump_default=True, metadata={
+ "description": "Increment pkgrel on conflicts",
+ })
+ packager = fields.String(metadata={
+ "description": "Packager identity if applicable",
+ })
+ refresh = fields.Boolean(dump_default=True, metadata={
+ "description": "Refresh pacman database"
+ })
diff --git a/src/ahriman/web/schemas/package_names_schema.py b/src/ahriman/web/schemas/package_names_schema.py
index 5bba154e..fb7438ec 100644
--- a/src/ahriman/web/schemas/package_names_schema.py
+++ b/src/ahriman/web/schemas/package_names_schema.py
@@ -17,10 +17,12 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#
-from marshmallow import Schema, fields
+from marshmallow import fields
+
+from ahriman.web.schemas.build_options_schema import BuildOptionsSchema
-class PackageNamesSchema(Schema):
+class PackageNamesSchema(BuildOptionsSchema):
"""
request package names schema
"""
diff --git a/src/ahriman/web/schemas/update_flags_schema.py b/src/ahriman/web/schemas/update_flags_schema.py
index 7ff09a59..35820d63 100644
--- a/src/ahriman/web/schemas/update_flags_schema.py
+++ b/src/ahriman/web/schemas/update_flags_schema.py
@@ -17,20 +17,22 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#
-from marshmallow import Schema, fields
+from marshmallow import fields
+
+from ahriman.web.schemas.build_options_schema import BuildOptionsSchema
-class UpdateFlagsSchema(Schema):
+class UpdateFlagsSchema(BuildOptionsSchema):
"""
update flags request schema
"""
- aur = fields.Bool(dump_default=True, metadata={
+ aur = fields.Boolean(dump_default=True, metadata={
"description": "Check AUR for updates",
})
- local = fields.Bool(dump_default=True, metadata={
+ local = fields.Boolean(dump_default=True, metadata={
"description": "Check local packages for updates",
})
- manual = fields.Bool(dump_default=True, metadata={
+ manual = fields.Boolean(dump_default=True, metadata={
"description": "Check manually built packages",
})
diff --git a/src/ahriman/web/views/base.py b/src/ahriman/web/views/base.py
index ddf15599..356cec78 100644
--- a/src/ahriman/web/views/base.py
+++ b/src/ahriman/web/views/base.py
@@ -222,6 +222,13 @@ class BaseView(View, CorsViewMixin):
Returns:
str | None: authorized username if any and None otherwise (e.g. if authorization is disabled)
"""
+ try: # try to read from payload
+ data: dict[str, str] = await self.request.json() # technically it is not, but we only need str here
+ if (packager := data.get("packager")) is not None:
+ return packager
+ except Exception:
+ self.request.app.logger.exception("could not extract json data for packager")
+
policy = self.request.app.get("identity")
if policy is not None:
identity: str = await policy.identify(self.request)
diff --git a/src/ahriman/web/views/v1/service/add.py b/src/ahriman/web/views/v1/service/add.py
index 7957cfa3..cfb98980 100644
--- a/src/ahriman/web/views/v1/service/add.py
+++ b/src/ahriman/web/views/v1/service/add.py
@@ -74,6 +74,14 @@ class AddView(BaseView):
repository_id = self.repository_id()
username = await self.username()
- process_id = self.spawner.packages_add(repository_id, packages, username, patches=patches, now=True)
+ process_id = self.spawner.packages_add(
+ repository_id,
+ packages,
+ username,
+ patches=patches,
+ now=True,
+ increment=data.get("increment", True),
+ refresh=data.get("refresh", False),
+ )
return json_response({"process_id": process_id})
diff --git a/src/ahriman/web/views/v1/service/rebuild.py b/src/ahriman/web/views/v1/service/rebuild.py
index 5952101b..990ec255 100644
--- a/src/ahriman/web/views/v1/service/rebuild.py
+++ b/src/ahriman/web/views/v1/service/rebuild.py
@@ -73,6 +73,11 @@ class RebuildView(BaseView):
repository_id = self.repository_id()
username = await self.username()
- process_id = self.spawner.packages_rebuild(repository_id, depends_on, username)
+ process_id = self.spawner.packages_rebuild(
+ repository_id,
+ depends_on,
+ username,
+ increment=data.get("increment", True),
+ )
return json_response({"process_id": process_id})
diff --git a/src/ahriman/web/views/v1/service/request.py b/src/ahriman/web/views/v1/service/request.py
index ce99420a..19a850c8 100644
--- a/src/ahriman/web/views/v1/service/request.py
+++ b/src/ahriman/web/views/v1/service/request.py
@@ -74,6 +74,14 @@ class RequestView(BaseView):
username = await self.username()
repository_id = self.repository_id()
- process_id = self.spawner.packages_add(repository_id, packages, username, patches=patches, now=False)
+ process_id = self.spawner.packages_add(
+ repository_id,
+ packages,
+ username,
+ patches=patches,
+ now=False,
+ increment=False, # no-increment doesn't work here
+ refresh=False, # refresh doesn't work here
+ )
return json_response({"process_id": process_id})
diff --git a/src/ahriman/web/views/v1/service/update.py b/src/ahriman/web/views/v1/service/update.py
index f5587436..c46ed702 100644
--- a/src/ahriman/web/views/v1/service/update.py
+++ b/src/ahriman/web/views/v1/service/update.py
@@ -77,6 +77,8 @@ class UpdateView(BaseView):
aur=data.get("aur", True),
local=data.get("local", True),
manual=data.get("manual", True),
+ increment=data.get("increment", True),
+ refresh=data.get("refresh", False),
)
return json_response({"process_id": process_id})
diff --git a/tests/ahriman/application/application/test_application_repository.py b/tests/ahriman/application/application/test_application_repository.py
index 67c22dba..21138e18 100644
--- a/tests/ahriman/application/application/test_application_repository.py
+++ b/tests/ahriman/application/application/test_application_repository.py
@@ -197,33 +197,34 @@ def test_update(application_repository: ApplicationRepository, package_ahriman:
paths = [package.filepath for package in package_ahriman.packages.values()]
tree = Tree([Leaf(package_ahriman)])
- mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels())
+ resolve_mock = mocker.patch("ahriman.application.application.workers.local_updater.LocalUpdater.partition",
+ return_value=tree.levels())
mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=paths)
- build_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_build", return_value=result)
- update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update", return_value=result)
+ build_mock = mocker.patch("ahriman.application.application.workers.local_updater.LocalUpdater.update",
+ return_value=result)
+ update_mock = mocker.patch("ahriman.core.repository.Repository.process_update", return_value=result)
on_result_mock = mocker.patch(
"ahriman.application.application.application_repository.ApplicationRepository.on_result")
application_repository.update([package_ahriman], Packagers("username"), bump_pkgrel=True)
+ resolve_mock.assert_called_once_with([package_ahriman])
build_mock.assert_called_once_with([package_ahriman], Packagers("username"), bump_pkgrel=True)
- update_mock.assert_has_calls([
- MockCall(paths, Packagers("username")),
- MockCall(paths, Packagers("username")),
- ])
+ update_mock.assert_called_once_with(paths, Packagers("username"))
on_result_mock.assert_has_calls([MockCall(result), MockCall(result)])
-def test_update_empty(application_repository: ApplicationRepository, package_ahriman: Package,
+def test_update_empty(application_repository: ApplicationRepository, package_ahriman: Package, result: Result,
mocker: MockerFixture) -> None:
"""
must skip updating repository if no packages supplied
"""
tree = Tree([Leaf(package_ahriman)])
- mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels())
- mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=[])
- mocker.patch("ahriman.core.repository.executor.Executor.process_build")
- update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update")
+ mocker.patch("ahriman.application.application.workers.Updater.partition", return_value=tree.levels())
+ mocker.patch("ahriman.core.repository.Repository.packages_built", return_value=[])
+ mocker.patch("ahriman.application.application.workers.local_updater.LocalUpdater.update", return_value=result)
+ mocker.patch("ahriman.application.application.application_repository.ApplicationRepository.on_result")
+ update_mock = mocker.patch("ahriman.core.repository.Repository.process_update")
application_repository.update([package_ahriman])
update_mock.assert_not_called()
diff --git a/tests/ahriman/application/application/workers/conftest.py b/tests/ahriman/application/application/workers/conftest.py
new file mode 100644
index 00000000..9e080702
--- /dev/null
+++ b/tests/ahriman/application/application/workers/conftest.py
@@ -0,0 +1,48 @@
+import pytest
+
+from ahriman.application.application.workers import Updater
+from ahriman.application.application.workers.local_updater import LocalUpdater
+from ahriman.application.application.workers.remote_updater import RemoteUpdater
+from ahriman.core.configuration import Configuration
+from ahriman.core.repository import Repository
+from ahriman.models.worker import Worker
+
+
+@pytest.fixture
+def local_updater(repository: Repository) -> LocalUpdater:
+ """
+ local updater fixture
+
+ Args:
+ repository(Repository): repository fixture
+
+ Returns:
+ LocalUpdater: local updater test instance
+ """
+ return LocalUpdater(repository)
+
+
+@pytest.fixture
+def remote_updater(configuration: Configuration) -> RemoteUpdater:
+ """
+ local updater fixture
+
+ Args:
+ configuration(Configuration): configuration fixture
+
+ Returns:
+ RemoteUpdater: remote updater test instance
+ """
+ _, repository_id = configuration.check_loaded()
+ return RemoteUpdater([Worker("remote1"), Worker("remote2")], repository_id, configuration)
+
+
+@pytest.fixture
+def updater() -> Updater:
+ """
+ empty updater fixture
+
+ Returns:
+ Updater: empty updater test instance
+ """
+ return Updater()
diff --git a/tests/ahriman/application/application/workers/test_local_updater.py b/tests/ahriman/application/application/workers/test_local_updater.py
new file mode 100644
index 00000000..72599cfa
--- /dev/null
+++ b/tests/ahriman/application/application/workers/test_local_updater.py
@@ -0,0 +1,30 @@
+from pytest_mock import MockerFixture
+
+from ahriman.application.application.workers.local_updater import LocalUpdater
+from ahriman.models.package import Package
+from ahriman.models.packagers import Packagers
+from ahriman.models.result import Result
+
+
+def test_partition(local_updater: LocalUpdater, mocker: MockerFixture) -> None:
+ """
+ must partition as tree resolution
+ """
+ resolve_mock = mocker.patch("ahriman.core.tree.Tree.resolve")
+ local_updater.partition([])
+ resolve_mock.assert_called_once_with([])
+
+
+def test_update(local_updater: LocalUpdater, package_ahriman: Package, result: Result,
+ mocker: MockerFixture) -> None:
+ """
+ must process package updates
+ """
+ paths = [package.filepath for package in package_ahriman.packages.values()]
+ mocker.patch("ahriman.core.repository.Repository.packages_built", return_value=paths)
+ build_mock = mocker.patch("ahriman.core.repository.Repository.process_build", return_value=result)
+ update_mock = mocker.patch("ahriman.core.repository.Repository.process_update", return_value=result)
+
+ assert local_updater.update([package_ahriman], Packagers("username"), bump_pkgrel=True) == result
+ build_mock.assert_called_once_with([package_ahriman], Packagers("username"), bump_pkgrel=True)
+ update_mock.assert_called_once_with(paths, Packagers("username"))
diff --git a/tests/ahriman/application/application/workers/test_remote_updater.py b/tests/ahriman/application/application/workers/test_remote_updater.py
new file mode 100644
index 00000000..9b9580a4
--- /dev/null
+++ b/tests/ahriman/application/application/workers/test_remote_updater.py
@@ -0,0 +1,84 @@
+from pytest_mock import MockerFixture
+
+from ahriman.application.application.workers.remote_updater import RemoteUpdater
+from ahriman.core.http import SyncAhrimanClient
+from ahriman.models.package import Package
+from ahriman.models.packagers import Packagers
+from ahriman.models.result import Result
+
+
+def test_clients(remote_updater: RemoteUpdater) -> None:
+ """
+ must return map of clients
+ """
+ worker = remote_updater.workers[0]
+ client = SyncAhrimanClient()
+ remote_updater._clients.append((worker, client))
+
+ assert remote_updater.clients == {worker: client}
+
+
+def test_update_url(remote_updater: RemoteUpdater) -> None:
+ """
+ must generate update url correctly
+ """
+ worker = remote_updater.workers[0]
+ assert remote_updater._update_url(worker).startswith(worker.address)
+ assert remote_updater._update_url(worker).endswith("/api/v1/service/add")
+
+
+def test_next_worker(remote_updater: RemoteUpdater) -> None:
+ """
+ must return next not used worker
+ """
+ assert remote_updater.next_worker()[0] == remote_updater.workers[0]
+ assert len(remote_updater.clients) == 1
+ assert remote_updater.workers[0] in remote_updater.clients
+
+ assert remote_updater.next_worker()[0] == remote_updater.workers[1]
+ assert remote_updater.workers[1] in remote_updater.clients
+ assert len(remote_updater.clients) == 2
+
+
+def test_next_worker_cycle(remote_updater: RemoteUpdater) -> None:
+ """
+ must return first used worker if no free workers left
+ """
+ worker1, client1 = remote_updater.next_worker()
+ worker2, client2 = remote_updater.next_worker()
+
+ assert remote_updater.next_worker() == (worker1, client1)
+ assert remote_updater.next_worker() == (worker2, client2)
+ assert remote_updater.next_worker() == (worker1, client1)
+
+
+def test_partition(remote_updater: RemoteUpdater, mocker: MockerFixture) -> None:
+ """
+ must partition as tree partition
+ """
+ resolve_mock = mocker.patch("ahriman.core.tree.Tree.partition")
+ remote_updater.partition([])
+ resolve_mock.assert_called_once_with([], count=2)
+
+
+def test_update(remote_updater: RemoteUpdater, package_ahriman: Package, mocker: MockerFixture) -> None:
+ """
+ must process remote package updates
+ """
+ worker, client = remote_updater.next_worker()
+ worker_mock = mocker.patch("ahriman.application.application.workers.remote_updater.RemoteUpdater.next_worker",
+ return_value=(worker, client))
+ request_mock = mocker.patch("ahriman.core.http.SyncAhrimanClient.make_request")
+
+ assert remote_updater.update([package_ahriman], Packagers("username"), bump_pkgrel=True) == Result()
+ worker_mock.assert_called_once_with()
+ request_mock.assert_called_once_with("POST", remote_updater._update_url(worker),
+ params=remote_updater.repository_id.query(),
+ json={
+ "increment": True,
+ "packager": "username",
+ "packages": [package_ahriman.base],
+ "patches": [],
+ "refresh": True,
+ }
+ )
diff --git a/tests/ahriman/application/application/workers/test_updater.py b/tests/ahriman/application/application/workers/test_updater.py
new file mode 100644
index 00000000..85a85ce9
--- /dev/null
+++ b/tests/ahriman/application/application/workers/test_updater.py
@@ -0,0 +1,50 @@
+import pytest
+
+from ahriman.application.application.workers import Updater
+from ahriman.application.application.workers.local_updater import LocalUpdater
+from ahriman.application.application.workers.remote_updater import RemoteUpdater
+from ahriman.core.configuration import Configuration
+from ahriman.core.repository import Repository
+from ahriman.models.worker import Worker
+
+
+def test_load(configuration: Configuration, repository: Repository) -> None:
+ """
+ must load local updater if empty worker list is set
+ """
+ _, repository_id = configuration.check_loaded()
+ assert isinstance(Updater.load(repository_id, configuration, repository), LocalUpdater)
+ assert isinstance(Updater.load(repository_id, configuration, repository, []), LocalUpdater)
+
+
+def test_load_from_option(configuration: Configuration, repository: Repository) -> None:
+ """
+ must load remote updater if nonempty worker list is set
+ """
+ _, repository_id = configuration.check_loaded()
+ assert isinstance(Updater.load(repository_id, configuration, repository, [Worker("remote")]), RemoteUpdater)
+
+
+def test_load_from_configuration(configuration: Configuration, repository: Repository) -> None:
+ """
+ must load remote updater from settings
+ """
+ configuration.set_option("build", "workers", "remote")
+ _, repository_id = configuration.check_loaded()
+ assert isinstance(Updater.load(repository_id, configuration, repository), RemoteUpdater)
+
+
+def test_partition(updater: Updater) -> None:
+ """
+ must raise not implemented error for missing partition method
+ """
+ with pytest.raises(NotImplementedError):
+ updater.partition([])
+
+
+def test_update(updater: Updater) -> None:
+ """
+ must raise not implemented error for missing update method
+ """
+ with pytest.raises(NotImplementedError):
+ updater.update([])
diff --git a/tests/ahriman/core/test_spawn.py b/tests/ahriman/core/test_spawn.py
index cb0e1355..40377faf 100644
--- a/tests/ahriman/core/test_spawn.py
+++ b/tests/ahriman/core/test_spawn.py
@@ -101,8 +101,10 @@ def test_packages_add(spawner: Spawn, repository_id: RepositoryId, mocker: Mocke
must call package addition
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
- assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=[], now=False)
- spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None)
+ assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
+ patches=[], now=False, increment=False, refresh=False)
+ spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
+ **{"username": None, "variable": [], "no-increment": ""})
def test_packages_add_with_build(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@@ -110,8 +112,10 @@ def test_packages_add_with_build(spawner: Spawn, repository_id: RepositoryId, mo
must call package addition with update
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
- assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=[], now=True)
- spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None, now="")
+ assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
+ patches=[], now=True, increment=False, refresh=False)
+ spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
+ **{"username": None, "variable": [], "no-increment": "", "now": ""})
def test_packages_add_with_username(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@@ -119,8 +123,10 @@ def test_packages_add_with_username(spawner: Spawn, repository_id: RepositoryId,
must call package addition with username
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
- assert spawner.packages_add(repository_id, ["ahriman", "linux"], "username", patches=[], now=False)
- spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username="username")
+ assert spawner.packages_add(repository_id, ["ahriman", "linux"], "username",
+ patches=[], now=False, increment=False, refresh=False)
+ spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
+ **{"username": "username", "variable": [], "no-increment": ""})
def test_packages_add_with_patches(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@@ -129,9 +135,36 @@ def test_packages_add_with_patches(spawner: Spawn, repository_id: RepositoryId,
"""
patches = [PkgbuildPatch("key", "value"), PkgbuildPatch("key", "value")]
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
- assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=patches, now=False)
- spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None,
- variable=[patch.serialize() for patch in patches])
+ assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
+ patches=patches, now=False, increment=False, refresh=False)
+ spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
+ **{
+ "username": None,
+ "variable": [patch.serialize() for patch in patches],
+ "no-increment": ""
+ })
+
+
+def test_packages_add_with_increment(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
+ """
+ must call package addition with increment
+ """
+ spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
+ assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
+ patches=[], now=False, increment=True, refresh=False)
+ spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
+ **{"username": None, "variable": [], "increment": ""})
+
+
+def test_packages_add_with_refresh(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
+ """
+ must call package addition with refresh
+ """
+ spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
+ assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
+ patches=[], now=False, increment=False, refresh=True)
+ spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
+ **{"username": None, "variable": [], "no-increment": "", "refresh": ""})
def test_packages_rebuild(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@@ -139,9 +172,19 @@ def test_packages_rebuild(spawner: Spawn, repository_id: RepositoryId, mocker: M
must call package rebuild
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
- assert spawner.packages_rebuild(repository_id, "python", "packager")
+ assert spawner.packages_rebuild(repository_id, "python", "packager", increment=False)
spawn_mock.assert_called_once_with(repository_id, "repo-rebuild",
- **{"depends-on": "python", "username": "packager"})
+ **{"depends-on": "python", "username": "packager", "no-increment": ""})
+
+
+def test_packages_rebuild_with_increment(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
+ """
+ must call package rebuild with increment
+ """
+ spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
+ assert spawner.packages_rebuild(repository_id, "python", "packager", increment=True)
+ spawn_mock.assert_called_once_with(repository_id, "repo-rebuild",
+ **{"depends-on": "python", "username": "packager", "increment": ""})
def test_packages_remove(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@@ -159,27 +202,61 @@ def test_packages_update(spawner: Spawn, repository_id: RepositoryId, mocker: Mo
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
- assert spawner.packages_update(repository_id, "packager", aur=True, local=True, manual=True)
- args = {"username": "packager", "aur": "", "local": "", "manual": ""}
+ assert spawner.packages_update(repository_id, "packager",
+ aur=True, local=True, manual=True, increment=False, refresh=False)
+ args = {"username": "packager", "aur": "", "local": "", "manual": "", "no-increment": ""}
spawn_mock.assert_called_once_with(repository_id, "repo-update", **args)
spawn_mock.reset_mock()
- assert spawner.packages_update(repository_id, "packager", aur=False, local=True, manual=True)
- args = {"username": "packager", "no-aur": "", "local": "", "manual": ""}
+ assert spawner.packages_update(repository_id, "packager",
+ aur=False, local=True, manual=True, increment=False, refresh=False)
+ args = {"username": "packager", "no-aur": "", "local": "", "manual": "", "no-increment": ""}
spawn_mock.assert_called_once_with(repository_id, "repo-update", **args)
spawn_mock.reset_mock()
- assert spawner.packages_update(repository_id, "packager", aur=True, local=False, manual=True)
- args = {"username": "packager", "aur": "", "no-local": "", "manual": ""}
+ assert spawner.packages_update(repository_id, "packager",
+ aur=True, local=False, manual=True, increment=False, refresh=False)
+ args = {"username": "packager", "aur": "", "no-local": "", "manual": "", "no-increment": ""}
spawn_mock.assert_called_once_with(repository_id, "repo-update", **args)
spawn_mock.reset_mock()
- assert spawner.packages_update(repository_id, "packager", aur=True, local=True, manual=False)
- args = {"username": "packager", "aur": "", "local": "", "no-manual": ""}
+ assert spawner.packages_update(repository_id, "packager",
+ aur=True, local=True, manual=False, increment=False, refresh=False)
+ args = {"username": "packager", "aur": "", "local": "", "no-manual": "", "no-increment": ""}
spawn_mock.assert_called_once_with(repository_id, "repo-update", **args)
spawn_mock.reset_mock()
+def test_packages_update_with_increment(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
+ """
+ must call repo update with increment
+ """
+ spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
+ assert spawner.packages_update(repository_id, None,
+ aur=True, local=True, manual=True, increment=True, refresh=False)
+ spawn_mock.assert_called_once_with(repository_id, "repo-update",
+ **{"username": None, "aur": "", "local": "", "manual": "", "increment": ""})
+
+
+def test_packages_update_with_refresh(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
+ """
+ must call repo update with refresh
+ """
+ spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
+ assert spawner.packages_update(repository_id, None,
+ aur=True, local=True, manual=True, increment=False, refresh=True)
+ spawn_mock.assert_called_once_with(repository_id, "repo-update",
+ **{
+ "username": None,
+ "aur": "",
+ "local": "",
+ "manual": "",
+ "no-increment": "",
+ "refresh": "",
+ }
+ )
+
+
def test_run(spawner: Spawn, mocker: MockerFixture) -> None:
"""
must implement run method
diff --git a/tests/ahriman/core/test_tree.py b/tests/ahriman/core/test_tree.py
index 906e21d9..42775f88 100644
--- a/tests/ahriman/core/test_tree.py
+++ b/tests/ahriman/core/test_tree.py
@@ -88,6 +88,13 @@ def test_tree_balance() -> None:
assert third == [leaf5]
+def test_tree_balance_empty() -> None:
+ """
+ must do not fail on empty tree balancing
+ """
+ assert Tree.balance([]) == []
+
+
def test_tree_partition(package_ahriman: Package, package_python_schedule: Package) -> None:
"""
must partition dependencies tree
diff --git a/tests/ahriman/models/test_worker.py b/tests/ahriman/models/test_worker.py
new file mode 100644
index 00000000..b6e5e72e
--- /dev/null
+++ b/tests/ahriman/models/test_worker.py
@@ -0,0 +1,10 @@
+from ahriman.models.worker import Worker
+
+
+def test_post_init() -> None:
+ """
+ must read identifier from location if not set
+ """
+ assert Worker("http://localhost:8080").identifier == "localhost:8080"
+ assert Worker("remote").identifier == "" # not a valid url
+ assert Worker("remote", identifier="id").identifier == "id"
diff --git a/tests/ahriman/web/schemas/test_build_options_schema.py b/tests/ahriman/web/schemas/test_build_options_schema.py
new file mode 100644
index 00000000..1982fb6b
--- /dev/null
+++ b/tests/ahriman/web/schemas/test_build_options_schema.py
@@ -0,0 +1 @@
+# schema testing goes in view class tests
diff --git a/tests/ahriman/web/views/test_view_base.py b/tests/ahriman/web/views/test_view_base.py
index 9c4ef4fe..64331a09 100644
--- a/tests/ahriman/web/views/test_view_base.py
+++ b/tests/ahriman/web/views/test_view_base.py
@@ -193,6 +193,9 @@ async def test_username(base: BaseView, mocker: MockerFixture) -> None:
policy = AsyncMock()
policy.identify.return_value = "identity"
mocker.patch("aiohttp.web.Application.get", return_value=policy)
+ json = AsyncMock()
+ json.return_value = {}
+ base._request = pytest.helpers.request(base.request.app, "", "", json=json)
assert await base.username() == "identity"
policy.identify.assert_called_once_with(base.request)
@@ -202,4 +205,26 @@ async def test_username_no_auth(base: BaseView) -> None:
"""
must return None in case if auth is disabled
"""
+ json = AsyncMock()
+ json.return_value = {}
+ base._request = pytest.helpers.request(base.request.app, "", "", json=json)
+
+ assert await base.username() is None
+
+
+async def test_username_request(base: BaseView) -> None:
+ """
+ must read packager from request
+ """
+ json = AsyncMock()
+ json.return_value = {"packager": "identity"}
+ base._request = pytest.helpers.request(base.request.app, "", "", json=json)
+
+ assert await base.username() == "identity"
+
+
+async def test_username_request_exception(base: BaseView) -> None:
+ """
+ must not fail in case if cannot read request
+ """
assert await base.username() is None
diff --git a/tests/ahriman/web/views/v1/service/test_view_v1_service_add.py b/tests/ahriman/web/views/v1/service/test_view_v1_service_add.py
index e1ffec43..f80b8bab 100644
--- a/tests/ahriman/web/views/v1/service/test_view_v1_service_add.py
+++ b/tests/ahriman/web/views/v1/service/test_view_v1_service_add.py
@@ -41,7 +41,8 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc
assert not request_schema.validate(payload)
response = await client.post("/api/v1/service/add", json=payload)
assert response.ok
- add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", patches=[], now=True)
+ add_mock.assert_called_once_with(repository_id, ["ahriman"], "username",
+ patches=[], now=True, increment=True, refresh=False)
json = await response.json()
assert json["process_id"] == "abc"
@@ -74,7 +75,8 @@ async def test_post_patches(client: TestClient, repository_id: RepositoryId, moc
response = await client.post("/api/v1/service/add", json=payload)
assert response.ok
add_mock.assert_called_once_with(repository_id, ["ahriman"], "username",
- patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")], now=True)
+ patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")],
+ now=True, increment=True, refresh=False)
async def test_post_empty(client: TestClient, mocker: MockerFixture) -> None:
diff --git a/tests/ahriman/web/views/v1/service/test_view_v1_service_rebuild.py b/tests/ahriman/web/views/v1/service/test_view_v1_service_rebuild.py
index 1fe740fb..fbe74b13 100644
--- a/tests/ahriman/web/views/v1/service/test_view_v1_service_rebuild.py
+++ b/tests/ahriman/web/views/v1/service/test_view_v1_service_rebuild.py
@@ -40,7 +40,7 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc
assert not request_schema.validate(payload)
response = await client.post("/api/v1/service/rebuild", json=payload)
assert response.ok
- rebuild_mock.assert_called_once_with(repository_id, "python", "username")
+ rebuild_mock.assert_called_once_with(repository_id, "python", "username", increment=True)
json = await response.json()
assert json["process_id"] == "abc"
diff --git a/tests/ahriman/web/views/v1/service/test_view_v1_service_request.py b/tests/ahriman/web/views/v1/service/test_view_v1_service_request.py
index d2f8b7ce..5cc9b789 100644
--- a/tests/ahriman/web/views/v1/service/test_view_v1_service_request.py
+++ b/tests/ahriman/web/views/v1/service/test_view_v1_service_request.py
@@ -41,7 +41,8 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc
assert not request_schema.validate(payload)
response = await client.post("/api/v1/service/request", json=payload)
assert response.ok
- add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", patches=[], now=False)
+ add_mock.assert_called_once_with(repository_id, ["ahriman"], "username",
+ patches=[], now=False, increment=False, refresh=False)
json = await response.json()
assert json["process_id"] == "abc"
@@ -74,7 +75,8 @@ async def test_post_patches(client: TestClient, repository_id: RepositoryId, moc
response = await client.post("/api/v1/service/request", json=payload)
assert response.ok
add_mock.assert_called_once_with(repository_id, ["ahriman"], "username",
- patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")], now=False)
+ patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")],
+ now=False, increment=False, refresh=False)
async def test_post_exception(client: TestClient, mocker: MockerFixture) -> None:
diff --git a/tests/ahriman/web/views/v1/service/test_view_v1_service_update.py b/tests/ahriman/web/views/v1/service/test_view_v1_service_update.py
index 2990599a..56121190 100644
--- a/tests/ahriman/web/views/v1/service/test_view_v1_service_update.py
+++ b/tests/ahriman/web/views/v1/service/test_view_v1_service_update.py
@@ -40,6 +40,8 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc
"aur": True,
"local": True,
"manual": True,
+ "increment": True,
+ "refresh": False,
}
for payload in (