add ability to run build process to remote instances

docs/ahriman.application.application.rst

@@ -1,6 +1,14 @@
 ahriman.application.application package
 =======================================
 
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   ahriman.application.application.workers
+
 Submodules
 ----------
 

docs/ahriman.application.application.workers.rst

@@ -0,0 +1,37 @@
+ahriman.application.application.workers package
+===============================================
+
+Submodules
+----------
+
+ahriman.application.application.workers.local\_updater module
+-------------------------------------------------------------
+
+.. automodule:: ahriman.application.application.workers.local_updater
+   :members:
+   :no-undoc-members:
+   :show-inheritance:
+
+ahriman.application.application.workers.remote\_updater module
+--------------------------------------------------------------
+
+.. automodule:: ahriman.application.application.workers.remote_updater
+   :members:
+   :no-undoc-members:
+   :show-inheritance:
+
+ahriman.application.application.workers.updater module
+------------------------------------------------------
+
+.. automodule:: ahriman.application.application.workers.updater
+   :members:
+   :no-undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: ahriman.application.application.workers
+   :members:
+   :no-undoc-members:
+   :show-inheritance:

docs/ahriman.models.rst

@@ -252,6 +252,14 @@ ahriman.models.waiter module
    :no-undoc-members:
    :show-inheritance:
 
+ahriman.models.worker module
+----------------------------
+
+.. automodule:: ahriman.models.worker
+   :members:
+   :no-undoc-members:
+   :show-inheritance:
+
 Module contents
 ---------------
 

docs/ahriman.web.schemas.rst

@@ -20,6 +20,14 @@ ahriman.web.schemas.auth\_schema module
    :no-undoc-members:
    :show-inheritance:
 
+ahriman.web.schemas.build\_options\_schema module
+-------------------------------------------------
+
+.. automodule:: ahriman.web.schemas.build_options_schema
+   :members:
+   :no-undoc-members:
+   :show-inheritance:
+
 ahriman.web.schemas.changes\_schema module
 ------------------------------------------
 

docs/configuration.rst

@@ -86,6 +86,7 @@ Build related configuration. Group name can refer to architecture, e.g. ``build:
 * ``triggers`` - list of ``ahriman.core.triggers.Trigger`` class implementation (e.g. ``ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger``) which will be loaded and run at the end of processing, space separated list of strings, optional. You can also specify triggers by their paths, e.g. ``/usr/lib/python3.10/site-packages/ahriman/core/report/report.py.ReportTrigger``. Triggers are run in the order of mention.
 * ``triggers_known`` - optional list of ``ahriman.core.triggers.Trigger`` class implementations which are not run automatically and used only for trigger discovery and configuration validation.
 * ``vcs_allowed_age`` - maximal age in seconds of the VCS packages before their version will be updated with its remote source, integer, optional, default ``604800``.
+* ``workers`` - list of worker nodes addresses used for build process, space separated list of strings, optional. Each worker address must be valid and reachable url, e.g. ``https://10.0.0.1:8080``. If none set, the build process will be run on the current node.
 
 ``repository`` group
 --------------------

docs/faq.rst

@@ -1022,6 +1022,116 @@ This action must be done in two steps:
 #. Remove package on worker.
 #. Remove package on master node.
 
+Delegate builds to remote workers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This setup heavily uses upload feature described above and, in addition, also delegates build process automatically to build machines. Same as above, there must be at least two instances available (``master`` and ``worker``), however, all ``worker`` nodes must be run in the web service mode.
+
+Master node configuration
+"""""""""""""""""""""""""
+
+In addition to the configuration above, the worker list must be defined in configuration file (``build.workers`` option), i.e.:
+
+.. code-block:: ini
+
+   [build]
+   workers = https://worker1.example.com https://worker2.example.com
+
+   [web]
+   enable_archive_upload = yes
+   wait_timeout = 0
+
+In the example above, ``https://worker1.example.com`` and ``https://worker2.example.com`` are remote ``worker`` node addresses available for ``master`` node.
+
+In case if authentication is required (which is recommended way to setup it), it can be set by using ``status`` section as usual.
+
+Worker nodes configuration
+""""""""""""""""""""""""""
+
+It is required to point to the master node repository, otherwise internal dependencies will not be handled correctly. In order to do so, the ``--server`` argument (or ``AHRIMAN_REPOSITORY_SERVER`` environment variable for docker images) can be used.
+
+Also, in case if authentication is enabled, the same user with the same password must be created for all workers.
+
+It is also recommended to set ``web.wait_timeout`` to infinte in case of multiple conflicting runs.
+
+Other settings are the same as mentioned above.
+
+Triple node minimal docker example
+""""""""""""""""""""""""""""""""""
+
+In this example, all instances are run on the same machine with address ``172.17.0.1`` with ports available outside of container. Master node config (``master.ini``) as:
+
+.. code-block:: ini
+
+   [auth]
+   target = mapping
+
+   [status]
+   username = builder-user
+   password = very-secure-password
+
+   [build]
+   workers = http://172.17.0.1:8081 http://172.17.0.1:8082
+
+   [web]
+   enable_archive_upload = yes
+   wait_timeout = 0
+
+Command to run master node:
+
+.. code-block:: shell
+
+   docker run --privileged -p 8080:8080 -e AHRIMAN_PORT=8080 -v master.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web
+
+Worker nodes (applicable for all workers) config (``worker.ini``) as:
+
+.. code-block:: ini
+
+   [auth]
+   target = mapping
+
+   [status]
+   address = http://172.17.0.1:8080
+   username = builder-user
+   password = very-secure-password
+
+   [upload]
+   target = remote-service
+
+   [remote-service]
+
+   [report]
+   target = remote-call
+
+   [remote-call]
+   manual = yes
+   wait_timeout = 0
+
+   [build]
+   triggers = ahriman.core.upload.UploadTrigger ahriman.core.report.ReportTrigger
+
+Command to run worker nodes (considering there will be two workers, one is on ``8081`` port and other is on ``8082``):
+
+.. code-block:: ini
+
+   docker run --privileged -p 8081:8081 -e AHRIMAN_PORT=8081 -v worker.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web
+   docker run --privileged -p 8082:8082 -e AHRIMAN_PORT=8082 -v worker.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web
+
+Unlike the previous setup, it doesn't require to mount repository root for ``worker`` nodes, because ``worker`` nodes don't use it anyway.
+
+Addition of new package, package removal, repository update
+"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+
+In all scenarios, update process must be run only on ``master`` node. Unlike the setup described above, automatic update must be enabled only for ``master`` node also.
+
+Known limitations
+"""""""""""""""""
+
+* Workers don't support local packages. However, it is possible to build custom packages by providing sources by using ``ahriman.core.gitremote.RemotePullTrigger`` trigger.
+* No dynamic nodes discovery. In case if one of worker nodes is unavailable, the build process will fail.
+* No pkgrel bump on conflicts. Well, it works, however, it isn't guaranteed.
+* The identical user must be created for all workers. However, the ``master`` node user can be different from this one.
+
 Maintenance packages
 --------------------
 

package/share/man/man1/ahriman.1

@@ -1,4 +1,4 @@
-.TH AHRIMAN "1" "2023\-11\-29" "ahriman" "Generated Python Manual"
+.TH AHRIMAN "1" "2023\-12\-08" "ahriman" "Generated Python Manual"
 .SH NAME
 ahriman
 .SH SYNOPSIS

src/ahriman/application/application/application_repository.py

@@ -18,11 +18,10 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from collections.abc import Iterable
-from pathlib import Path
 
 from ahriman.application.application.application_properties import ApplicationProperties
+from ahriman.application.application.workers import Updater
 from ahriman.core.build_tools.sources import Sources
-from ahriman.core.tree import Tree
 from ahriman.models.package import Package
 from ahriman.models.packagers import Packagers
 from ahriman.models.result import Result
@@ -154,26 +153,25 @@ class ApplicationRepository(ApplicationProperties):
         Returns:
             Result: update result
         """
-        def process_update(paths: Iterable[Path], result: Result) -> None:
+        result = Result()
-            if not paths:
-                return  # don't need to process if no update supplied
-            update_result = self.repository.process_update(paths, packagers)
-            self.on_result(result.merge(update_result))
 
-        # process built packages
+        # process already built packages if any
-        build_result = Result()
+        built_packages = self.repository.packages_built()
-        packages = self.repository.packages_built()
+        if built_packages:  # speedup a bit
-        process_update(packages, build_result)
+            build_result = self.repository.process_update(built_packages, packagers)
+            result.merge(build_result)
+            self.on_result(result.merge(build_result))
 
-        # process manual packages
+        builder = Updater.load(self.repository_id, self.configuration, self.repository)
-        tree = Tree.resolve(updates)
-        for num, level in enumerate(tree):
-            self.logger.info("processing level #%i %s", num, [package.base for package in level])
-            build_result = self.repository.process_build(level, packagers, bump_pkgrel=bump_pkgrel)
-            packages = self.repository.packages_built()
-            process_update(packages, build_result)
 
-        return build_result
+        # ok so for now we split all packages into chunks and process each chunk accordingly
+        partitions = builder.partition(updates)
+        for num, partition in enumerate(partitions):
+            self.logger.info("processing chunk #%i %s", num, [package.base for package in partition])
+            build_result = builder.update(partition, packagers, bump_pkgrel=bump_pkgrel)
+            self.on_result(result.merge(build_result))
+
+        return result
 
     def updates(self, filter_packages: Iterable[str], *,
                 aur: bool, local: bool, manual: bool, vcs: bool) -> list[Package]:

src/ahriman/application/application/workers/__init__.py

@@ -0,0 +1,20 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from ahriman.application.application.workers.updater import Updater

src/ahriman/application/application/workers/local_updater.py

@@ -0,0 +1,77 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from collections.abc import Iterable
+
+from ahriman.application.application.workers.updater import Updater
+from ahriman.core.repository import Repository
+from ahriman.core.tree import Tree
+from ahriman.models.package import Package
+from ahriman.models.packagers import Packagers
+from ahriman.models.result import Result
+
+
+class LocalUpdater(Updater):
+    """
+    local build process implementation
+
+    Attributes:
+        repository(Repository): repository instance
+    """
+
+    def __init__(self, repository: Repository) -> None:
+        """
+        default constructor
+
+        Args:
+            repository(Repository): repository instance
+        """
+        self.repository = repository
+
+    def partition(self, packages: Iterable[Package]) -> list[list[Package]]:
+        """
+        split packages into partitions to be processed by this worker
+
+        Args:
+            packages(Iterable[Package]): list of packages to partition
+
+        Returns:
+            list[list[Package]]: packages partitioned by this worker type
+        """
+        return Tree.resolve(packages)
+
+    def update(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
+               bump_pkgrel: bool = False) -> Result:
+        """
+        run package updates
+
+        Args:
+            updates(Iterable[Package]): list of packages to update
+            packagers(Packagers | None, optional): optional override of username for build process
+                (Default value = None)
+            bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False)
+
+        Returns:
+            Result: update result
+        """
+        build_result = self.repository.process_build(updates, packagers, bump_pkgrel=bump_pkgrel)
+        packages = self.repository.packages_built()
+        update_result = self.repository.process_update(packages, packagers)
+
+        return build_result.merge(update_result)

src/ahriman/application/application/workers/remote_updater.py

@@ -0,0 +1,140 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from collections import deque
+from collections.abc import Iterable
+
+from ahriman.application.application.workers.updater import Updater
+from ahriman.core.configuration import Configuration
+from ahriman.core.http import SyncAhrimanClient
+from ahriman.core.tree import Tree
+from ahriman.models.package import Package
+from ahriman.models.packagers import Packagers
+from ahriman.models.repository_id import RepositoryId
+from ahriman.models.result import Result
+from ahriman.models.worker import Worker
+
+
+class RemoteUpdater(Updater):
+    """
+    remote update worker
+
+    Attributes:
+        configuration(Configuration): configuration instance
+        repository_id(RepositoryId): repository unique identifier
+        workers(list[Worker]): worker identifiers
+    """
+
+    def __init__(self, workers: list[Worker], repository_id: RepositoryId, configuration: Configuration) -> None:
+        """
+        default constructor
+
+        Args:
+            workers(list[Worker]): worker identifiers
+            repository_id(RepositoryId): repository unique identifier
+            configuration(Configuration): configuration instance
+        """
+        self.workers = workers
+        self.repository_id = repository_id
+        self.configuration = configuration
+
+        self._clients: deque[tuple[Worker, SyncAhrimanClient]] = deque()
+
+    @property
+    def clients(self) -> dict[Worker, SyncAhrimanClient]:
+        """
+        extract loaded clients. Note that this method yields only workers which have been already loaded
+
+        Returns:
+            dict[Worker, SyncAhrimanClient]: map of the worker to the related web client
+        """
+        return dict(self._clients)
+
+    @staticmethod
+    def _update_url(worker: Worker) -> str:
+        """
+        get url for updates
+
+        Args:
+            worker(Worker): worker identifier
+
+        Returns:
+            str: full url for web service to run update process
+        """
+        return f"{worker.address}/api/v1/service/add"
+
+    def next_worker(self) -> tuple[Worker, SyncAhrimanClient]:
+        """
+        generate next not-used web client. In case if all clients have been already used, it yields next not used client
+
+        Returns:
+            tuple[Worker, SyncAhrimanClient]: worker and constructed client instance for the web
+        """
+        # check if there is not used yet worker
+        worker = next((worker for worker in self.workers if worker not in self.clients), None)
+        if worker is not None:
+            client = SyncAhrimanClient(self.configuration, "status")
+            client.address = worker.address
+        else:
+            worker, client = self._clients.popleft()
+
+        # register worker in the queue
+        self._clients.append((worker, client))
+
+        return worker, client
+
+    def partition(self, packages: Iterable[Package]) -> list[list[Package]]:
+        """
+        split packages into partitions to be processed by this worker
+
+        Args:
+            packages(Iterable[Package]): list of packages to partition
+
+        Returns:
+            list[list[Package]]: packages partitioned by this worker type
+        """
+        return Tree.partition(packages, count=len(self.workers))
+
+    def update(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
+               bump_pkgrel: bool = False) -> Result:
+        """
+        run package updates
+
+        Args:
+            updates(Iterable[Package]): list of packages to update
+            packagers(Packagers | None, optional): optional override of username for build process
+                (Default value = None)
+            bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False)
+
+        Returns:
+            Result: update result
+        """
+        payload = {
+            "increment": bump_pkgrel,
+            "packager": packagers.default if packagers is not None else None,
+            "packages": [package.base for package in updates],
+            "patches": [],  # might be used later
+            "refresh": True,
+        }
+
+        worker, client = self.next_worker()
+        client.make_request("POST", self._update_url(worker), params=self.repository_id.query(), json=payload)
+        # we don't block here for process
+
+        return Result()

src/ahriman/application/application/workers/updater.py

@@ -0,0 +1,102 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.log import LazyLogging
+from ahriman.core.repository import Repository
+from ahriman.models.package import Package
+from ahriman.models.packagers import Packagers
+from ahriman.models.repository_id import RepositoryId
+from ahriman.models.result import Result
+from ahriman.models.worker import Worker
+
+
+class Updater(LazyLogging):
+    """
+    updater handler interface
+
+    Attributes:
+        split_method(Callable[[Iterable[Package]], list[list[Package]]]): method to split packages into chunks
+    """
+
+    @staticmethod
+    def load(repository_id: RepositoryId, configuration: Configuration,
+             repository: Repository, workers: list[Worker] | None = None) -> Updater:
+        """
+        construct updaters from parameters
+
+        Args:
+            repository_id(RepositoryId): repository unique identifier
+            configuration(Configuration): configuration instance
+            repository(Repository): repository instance
+            workers(list[Worker] | None, optional): worker identifiers if any (Default value = None)
+
+        Returns:
+            Updater: constructed updater worker
+        """
+        if workers is None:
+            # no workers set explicitly, try to guess from configuration
+            workers = [Worker(address) for address in configuration.getlist("build", "workers", fallback=[])]
+
+        if workers:
+            # there is something we could use as remote workers
+            from ahriman.application.application.workers.remote_updater import RemoteUpdater
+            return RemoteUpdater(workers, repository_id, configuration)
+
+        # and finally no workers available, just use local service
+        from ahriman.application.application.workers.local_updater import LocalUpdater
+        return LocalUpdater(repository)
+
+    def partition(self, packages: Iterable[Package]) -> list[list[Package]]:
+        """
+        split packages into partitions to be processed by this worker
+
+        Args:
+            packages(Iterable[Package]): list of packages to partition
+
+        Returns:
+            list[list[Package]]: packages partitioned by this worker type
+
+        Raises:
+            NotImplementedError: not implemented method
+        """
+        raise NotImplementedError
+
+    def update(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
+               bump_pkgrel: bool = False) -> Result:
+        """
+        run package updates
+
+        Args:
+            updates(Iterable[Package]): list of packages to update
+            packagers(Packagers | None, optional): optional override of username for build process
+                (Default value = None)
+            bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False)
+
+        Returns:
+            Result: update result
+
+        Raises:
+            NotImplementedError: not implemented method
+        """
+        raise NotImplementedError

src/ahriman/core/configuration/schema.py

@@ -213,6 +213,15 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
                 "coerce": "integer",
                 "min": 0,
             },
+            "workers": {
+                "type": "list",
+                "coerce": "list",
+                "schema": {
+                    "type": "string",
+                    "empty": False,
+                    "is_url": [],
+                },
+            },
         },
     },
     "repository": {

src/ahriman/core/spawn.py

@@ -174,7 +174,7 @@ class Spawn(Thread, LazyLogging):
         return self._spawn_process(repository_id, "service-key-import", key, **kwargs)
 
     def packages_add(self, repository_id: RepositoryId, packages: Iterable[str], username: str | None, *,
-                     patches: list[PkgbuildPatch], now: bool) -> str:
+                     patches: list[PkgbuildPatch], now: bool, increment: bool, refresh: bool) -> str:
         """
         add packages
 
@@ -184,19 +184,26 @@ class Spawn(Thread, LazyLogging):
             username(str | None): optional override of username for build process
             patches(list[PkgbuildPatch]): list of patches to be passed
             now(bool): build packages now
+            increment(bool): increment pkgrel on conflict
+            refresh(bool): refresh pacman database before process
 
         Returns:
             str: spawned process identifier
         """
-        kwargs: dict[str, str | list[str] | None] = {"username": username}
+        kwargs: dict[str, str | list[str] | None] = {
+            "username": username,
+            "variable": [patch.serialize() for patch in patches],
+            self.boolean_action_argument("increment", increment): "",
+        }
         if now:
             kwargs["now"] = ""
-        if patches:
+        if refresh:
-            kwargs["variable"] = [patch.serialize() for patch in patches]
+            kwargs["refresh"] = ""
 
         return self._spawn_process(repository_id, "package-add", *packages, **kwargs)
 
-    def packages_rebuild(self, repository_id: RepositoryId, depends_on: str, username: str | None) -> str:
+    def packages_rebuild(self, repository_id: RepositoryId, depends_on: str, username: str | None, *,
+                         increment: bool) -> str:
         """
         rebuild packages which depend on the specified package
 
@@ -204,11 +211,16 @@ class Spawn(Thread, LazyLogging):
             repository_id(RepositoryId): repository unique identifier
             depends_on(str): packages dependency
             username(str | None): optional override of username for build process
+            increment(bool): increment pkgrel on conflict
 
         Returns:
             str: spawned process identifier
         """
-        kwargs = {"depends-on": depends_on, "username": username}
+        kwargs = {
+            "depends-on": depends_on,
+            "username": username,
+            self.boolean_action_argument("increment", increment): "",
+        }
         return self._spawn_process(repository_id, "repo-rebuild", **kwargs)
 
     def packages_remove(self, repository_id: RepositoryId, packages: Iterable[str]) -> str:
@@ -225,7 +237,7 @@ class Spawn(Thread, LazyLogging):
         return self._spawn_process(repository_id, "package-remove", *packages)
 
     def packages_update(self, repository_id: RepositoryId, username: str | None, *,
-                        aur: bool, local: bool, manual: bool) -> str:
+                        aur: bool, local: bool, manual: bool, increment: bool, refresh: bool) -> str:
         """
         run full repository update
 
@@ -235,6 +247,8 @@ class Spawn(Thread, LazyLogging):
             aur(bool): check for aur updates
             local(bool): check for local packages updates
             manual(bool): check for manual packages
+            increment(bool): increment pkgrel on conflict
+            refresh(bool): refresh pacman database before process
 
         Returns:
             str: spawned process identifier
@@ -244,7 +258,11 @@ class Spawn(Thread, LazyLogging):
             self.boolean_action_argument("aur", aur): "",
             self.boolean_action_argument("local", local): "",
             self.boolean_action_argument("manual", manual): "",
+            self.boolean_action_argument("increment", increment): "",
         }
+        if refresh:
+            kwargs["refresh"] = ""
+
         return self._spawn_process(repository_id, "repo-update", **kwargs)
 
     def run(self) -> None:

src/ahriman/core/tree.py

@@ -148,6 +148,8 @@ class Tree:
             sorted(part, key=lambda leaf: leaf.package.base)
             for part in partitions if part
         ]
+        if not partitions:  # nothing to balance
+            return partitions
 
         while True:
             min_part, max_part = minmax(partitions, key=len)

src/ahriman/models/worker.py

@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from dataclasses import dataclass, field
+from urllib.parse import urlparse
+
+
+@dataclass(frozen=True)
+class Worker:
+    """
+    worker descriptor
+
+    Attributes:
+        address(str): worker address to be reachable outside
+        identifier(str): worker unique identifier. If none set it will be automatically generated from the address
+    """
+
+    address: str
+    identifier: str = field(default="", kw_only=True)
+
+    def __post_init__(self) -> None:
+        """
+        update identifier based on settings
+        """
+        object.__setattr__(self, "identifier", self.identifier or urlparse(self.address).netloc)

src/ahriman/web/schemas/__init__.py

@@ -19,6 +19,7 @@
 #
 from ahriman.web.schemas.aur_package_schema import AURPackageSchema
 from ahriman.web.schemas.auth_schema import AuthSchema
+from ahriman.web.schemas.build_options_schema import BuildOptionsSchema
 from ahriman.web.schemas.changes_schema import ChangesSchema
 from ahriman.web.schemas.counters_schema import CountersSchema
 from ahriman.web.schemas.error_schema import ErrorSchema

src/ahriman/web/schemas/build_options_schema.py

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class BuildOptionsSchema(Schema):
+    """
+    request build options schema
+    """
+
+    increment = fields.Boolean(dump_default=True, metadata={
+        "description": "Increment pkgrel on conflicts",
+    })
+    packager = fields.String(metadata={
+        "description": "Packager identity if applicable",
+    })
+    refresh = fields.Boolean(dump_default=True, metadata={
+        "description": "Refresh pacman database"
+    })

src/ahriman/web/schemas/package_names_schema.py

@@ -17,10 +17,12 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from marshmallow import Schema, fields
+from marshmallow import fields
+
+from ahriman.web.schemas.build_options_schema import BuildOptionsSchema
 
 
-class PackageNamesSchema(Schema):
+class PackageNamesSchema(BuildOptionsSchema):
     """
     request package names schema
     """

src/ahriman/web/schemas/update_flags_schema.py

@@ -17,20 +17,22 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from marshmallow import Schema, fields
+from marshmallow import fields
+
+from ahriman.web.schemas.build_options_schema import BuildOptionsSchema
 
 
-class UpdateFlagsSchema(Schema):
+class UpdateFlagsSchema(BuildOptionsSchema):
     """
     update flags request schema
     """
 
-    aur = fields.Bool(dump_default=True, metadata={
+    aur = fields.Boolean(dump_default=True, metadata={
         "description": "Check AUR for updates",
     })
-    local = fields.Bool(dump_default=True, metadata={
+    local = fields.Boolean(dump_default=True, metadata={
         "description": "Check local packages for updates",
     })
-    manual = fields.Bool(dump_default=True, metadata={
+    manual = fields.Boolean(dump_default=True, metadata={
         "description": "Check manually built packages",
     })

src/ahriman/web/views/base.py

@@ -222,6 +222,13 @@ class BaseView(View, CorsViewMixin):
         Returns:
             str | None: authorized username if any and None otherwise (e.g. if authorization is disabled)
         """
+        try:  # try to read from payload
+            data: dict[str, str] = await self.request.json()  # technically it is not, but we only need str here
+            if (packager := data.get("packager")) is not None:
+                return packager
+        except Exception:
+            self.request.app.logger.exception("could not extract json data for packager")
+
         policy = self.request.app.get("identity")
         if policy is not None:
             identity: str = await policy.identify(self.request)

src/ahriman/web/views/v1/service/add.py

@@ -74,6 +74,14 @@ class AddView(BaseView):
 
         repository_id = self.repository_id()
         username = await self.username()
-        process_id = self.spawner.packages_add(repository_id, packages, username, patches=patches, now=True)
+        process_id = self.spawner.packages_add(
+            repository_id,
+            packages,
+            username,
+            patches=patches,
+            now=True,
+            increment=data.get("increment", True),
+            refresh=data.get("refresh", False),
+        )
 
         return json_response({"process_id": process_id})

src/ahriman/web/views/v1/service/rebuild.py

@@ -73,6 +73,11 @@ class RebuildView(BaseView):
 
         repository_id = self.repository_id()
         username = await self.username()
-        process_id = self.spawner.packages_rebuild(repository_id, depends_on, username)
+        process_id = self.spawner.packages_rebuild(
+            repository_id,
+            depends_on,
+            username,
+            increment=data.get("increment", True),
+        )
 
         return json_response({"process_id": process_id})

src/ahriman/web/views/v1/service/request.py

@@ -74,6 +74,14 @@ class RequestView(BaseView):
 
         username = await self.username()
         repository_id = self.repository_id()
-        process_id = self.spawner.packages_add(repository_id, packages, username, patches=patches, now=False)
+        process_id = self.spawner.packages_add(
+            repository_id,
+            packages,
+            username,
+            patches=patches,
+            now=False,
+            increment=False,  # no-increment doesn't work here
+            refresh=False,  # refresh doesn't work here
+        )
 
         return json_response({"process_id": process_id})

src/ahriman/web/views/v1/service/update.py

@@ -77,6 +77,8 @@ class UpdateView(BaseView):
             aur=data.get("aur", True),
             local=data.get("local", True),
             manual=data.get("manual", True),
+            increment=data.get("increment", True),
+            refresh=data.get("refresh", False),
         )
 
         return json_response({"process_id": process_id})

tests/ahriman/application/application/test_application_repository.py

@@ -197,33 +197,34 @@ def test_update(application_repository: ApplicationRepository, package_ahriman:
     paths = [package.filepath for package in package_ahriman.packages.values()]
     tree = Tree([Leaf(package_ahriman)])
 
-    mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels())
+    resolve_mock = mocker.patch("ahriman.application.application.workers.local_updater.LocalUpdater.partition",
+                                return_value=tree.levels())
     mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=paths)
-    build_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_build", return_value=result)
+    build_mock = mocker.patch("ahriman.application.application.workers.local_updater.LocalUpdater.update",
-    update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update", return_value=result)
+                              return_value=result)
+    update_mock = mocker.patch("ahriman.core.repository.Repository.process_update", return_value=result)
     on_result_mock = mocker.patch(
         "ahriman.application.application.application_repository.ApplicationRepository.on_result")
 
     application_repository.update([package_ahriman], Packagers("username"), bump_pkgrel=True)
+    resolve_mock.assert_called_once_with([package_ahriman])
     build_mock.assert_called_once_with([package_ahriman], Packagers("username"), bump_pkgrel=True)
-    update_mock.assert_has_calls([
+    update_mock.assert_called_once_with(paths, Packagers("username"))
-        MockCall(paths, Packagers("username")),
-        MockCall(paths, Packagers("username")),
-    ])
     on_result_mock.assert_has_calls([MockCall(result), MockCall(result)])
 
 
-def test_update_empty(application_repository: ApplicationRepository, package_ahriman: Package,
+def test_update_empty(application_repository: ApplicationRepository, package_ahriman: Package, result: Result,
                       mocker: MockerFixture) -> None:
     """
     must skip updating repository if no packages supplied
     """
     tree = Tree([Leaf(package_ahriman)])
 
-    mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels())
+    mocker.patch("ahriman.application.application.workers.Updater.partition", return_value=tree.levels())
-    mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=[])
+    mocker.patch("ahriman.core.repository.Repository.packages_built", return_value=[])
-    mocker.patch("ahriman.core.repository.executor.Executor.process_build")
+    mocker.patch("ahriman.application.application.workers.local_updater.LocalUpdater.update", return_value=result)
-    update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update")
+    mocker.patch("ahriman.application.application.application_repository.ApplicationRepository.on_result")
+    update_mock = mocker.patch("ahriman.core.repository.Repository.process_update")
 
     application_repository.update([package_ahriman])
     update_mock.assert_not_called()

tests/ahriman/application/application/workers/conftest.py

@@ -0,0 +1,48 @@
+import pytest
+
+from ahriman.application.application.workers import Updater
+from ahriman.application.application.workers.local_updater import LocalUpdater
+from ahriman.application.application.workers.remote_updater import RemoteUpdater
+from ahriman.core.configuration import Configuration
+from ahriman.core.repository import Repository
+from ahriman.models.worker import Worker
+
+
+@pytest.fixture
+def local_updater(repository: Repository) -> LocalUpdater:
+    """
+    local updater fixture
+
+    Args:
+        repository(Repository): repository fixture
+
+    Returns:
+        LocalUpdater: local updater test instance
+    """
+    return LocalUpdater(repository)
+
+
+@pytest.fixture
+def remote_updater(configuration: Configuration) -> RemoteUpdater:
+    """
+    local updater fixture
+
+    Args:
+        configuration(Configuration): configuration fixture
+
+    Returns:
+        RemoteUpdater: remote updater test instance
+    """
+    _, repository_id = configuration.check_loaded()
+    return RemoteUpdater([Worker("remote1"), Worker("remote2")], repository_id, configuration)
+
+
+@pytest.fixture
+def updater() -> Updater:
+    """
+    empty updater fixture
+
+    Returns:
+        Updater: empty updater test instance
+    """
+    return Updater()

tests/ahriman/application/application/workers/test_local_updater.py

@@ -0,0 +1,30 @@
+from pytest_mock import MockerFixture
+
+from ahriman.application.application.workers.local_updater import LocalUpdater
+from ahriman.models.package import Package
+from ahriman.models.packagers import Packagers
+from ahriman.models.result import Result
+
+
+def test_partition(local_updater: LocalUpdater, mocker: MockerFixture) -> None:
+    """
+    must partition as tree resolution
+    """
+    resolve_mock = mocker.patch("ahriman.core.tree.Tree.resolve")
+    local_updater.partition([])
+    resolve_mock.assert_called_once_with([])
+
+
+def test_update(local_updater: LocalUpdater, package_ahriman: Package, result: Result,
+                mocker: MockerFixture) -> None:
+    """
+    must process package updates
+    """
+    paths = [package.filepath for package in package_ahriman.packages.values()]
+    mocker.patch("ahriman.core.repository.Repository.packages_built", return_value=paths)
+    build_mock = mocker.patch("ahriman.core.repository.Repository.process_build", return_value=result)
+    update_mock = mocker.patch("ahriman.core.repository.Repository.process_update", return_value=result)
+
+    assert local_updater.update([package_ahriman], Packagers("username"), bump_pkgrel=True) == result
+    build_mock.assert_called_once_with([package_ahriman], Packagers("username"), bump_pkgrel=True)
+    update_mock.assert_called_once_with(paths, Packagers("username"))

tests/ahriman/application/application/workers/test_remote_updater.py

@@ -0,0 +1,84 @@
+from pytest_mock import MockerFixture
+
+from ahriman.application.application.workers.remote_updater import RemoteUpdater
+from ahriman.core.http import SyncAhrimanClient
+from ahriman.models.package import Package
+from ahriman.models.packagers import Packagers
+from ahriman.models.result import Result
+
+
+def test_clients(remote_updater: RemoteUpdater) -> None:
+    """
+    must return map of clients
+    """
+    worker = remote_updater.workers[0]
+    client = SyncAhrimanClient()
+    remote_updater._clients.append((worker, client))
+
+    assert remote_updater.clients == {worker: client}
+
+
+def test_update_url(remote_updater: RemoteUpdater) -> None:
+    """
+    must generate update url correctly
+    """
+    worker = remote_updater.workers[0]
+    assert remote_updater._update_url(worker).startswith(worker.address)
+    assert remote_updater._update_url(worker).endswith("/api/v1/service/add")
+
+
+def test_next_worker(remote_updater: RemoteUpdater) -> None:
+    """
+    must return next not used worker
+    """
+    assert remote_updater.next_worker()[0] == remote_updater.workers[0]
+    assert len(remote_updater.clients) == 1
+    assert remote_updater.workers[0] in remote_updater.clients
+
+    assert remote_updater.next_worker()[0] == remote_updater.workers[1]
+    assert remote_updater.workers[1] in remote_updater.clients
+    assert len(remote_updater.clients) == 2
+
+
+def test_next_worker_cycle(remote_updater: RemoteUpdater) -> None:
+    """
+    must return first used worker if no free workers left
+    """
+    worker1, client1 = remote_updater.next_worker()
+    worker2, client2 = remote_updater.next_worker()
+
+    assert remote_updater.next_worker() == (worker1, client1)
+    assert remote_updater.next_worker() == (worker2, client2)
+    assert remote_updater.next_worker() == (worker1, client1)
+
+
+def test_partition(remote_updater: RemoteUpdater, mocker: MockerFixture) -> None:
+    """
+    must partition as tree partition
+    """
+    resolve_mock = mocker.patch("ahriman.core.tree.Tree.partition")
+    remote_updater.partition([])
+    resolve_mock.assert_called_once_with([], count=2)
+
+
+def test_update(remote_updater: RemoteUpdater, package_ahriman: Package, mocker: MockerFixture) -> None:
+    """
+    must process remote package updates
+    """
+    worker, client = remote_updater.next_worker()
+    worker_mock = mocker.patch("ahriman.application.application.workers.remote_updater.RemoteUpdater.next_worker",
+                               return_value=(worker, client))
+    request_mock = mocker.patch("ahriman.core.http.SyncAhrimanClient.make_request")
+
+    assert remote_updater.update([package_ahriman], Packagers("username"), bump_pkgrel=True) == Result()
+    worker_mock.assert_called_once_with()
+    request_mock.assert_called_once_with("POST", remote_updater._update_url(worker),
+                                         params=remote_updater.repository_id.query(),
+                                         json={
+                                             "increment": True,
+                                             "packager": "username",
+                                             "packages": [package_ahriman.base],
+                                             "patches": [],
+                                             "refresh": True,
+    }
+    )

tests/ahriman/application/application/workers/test_updater.py

@@ -0,0 +1,50 @@
+import pytest
+
+from ahriman.application.application.workers import Updater
+from ahriman.application.application.workers.local_updater import LocalUpdater
+from ahriman.application.application.workers.remote_updater import RemoteUpdater
+from ahriman.core.configuration import Configuration
+from ahriman.core.repository import Repository
+from ahriman.models.worker import Worker
+
+
+def test_load(configuration: Configuration, repository: Repository) -> None:
+    """
+    must load local updater if empty worker list is set
+    """
+    _, repository_id = configuration.check_loaded()
+    assert isinstance(Updater.load(repository_id, configuration, repository), LocalUpdater)
+    assert isinstance(Updater.load(repository_id, configuration, repository, []), LocalUpdater)
+
+
+def test_load_from_option(configuration: Configuration, repository: Repository) -> None:
+    """
+    must load remote updater if nonempty worker list is set
+    """
+    _, repository_id = configuration.check_loaded()
+    assert isinstance(Updater.load(repository_id, configuration, repository, [Worker("remote")]), RemoteUpdater)
+
+
+def test_load_from_configuration(configuration: Configuration, repository: Repository) -> None:
+    """
+    must load remote updater from settings
+    """
+    configuration.set_option("build", "workers", "remote")
+    _, repository_id = configuration.check_loaded()
+    assert isinstance(Updater.load(repository_id, configuration, repository), RemoteUpdater)
+
+
+def test_partition(updater: Updater) -> None:
+    """
+    must raise not implemented error for missing partition method
+    """
+    with pytest.raises(NotImplementedError):
+        updater.partition([])
+
+
+def test_update(updater: Updater) -> None:
+    """
+    must raise not implemented error for missing update method
+    """
+    with pytest.raises(NotImplementedError):
+        updater.update([])

tests/ahriman/core/test_spawn.py

@@ -101,8 +101,10 @@ def test_packages_add(spawner: Spawn, repository_id: RepositoryId, mocker: Mocke
     must call package addition
     """
     spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
-    assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=[], now=False)
+    assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
-    spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None)
+                                patches=[], now=False, increment=False, refresh=False)
+    spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
+                                       **{"username": None, "variable": [], "no-increment": ""})
 
 
 def test_packages_add_with_build(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@@ -110,8 +112,10 @@ def test_packages_add_with_build(spawner: Spawn, repository_id: RepositoryId, mo
     must call package addition with update
     """
     spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
-    assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=[], now=True)
+    assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
-    spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None, now="")
+                                patches=[], now=True, increment=False, refresh=False)
+    spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
+                                       **{"username": None, "variable": [], "no-increment": "", "now": ""})
 
 
 def test_packages_add_with_username(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@@ -119,8 +123,10 @@ def test_packages_add_with_username(spawner: Spawn, repository_id: RepositoryId,
     must call package addition with username
     """
     spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
-    assert spawner.packages_add(repository_id, ["ahriman", "linux"], "username", patches=[], now=False)
+    assert spawner.packages_add(repository_id, ["ahriman", "linux"], "username",
-    spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username="username")
+                                patches=[], now=False, increment=False, refresh=False)
+    spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
+                                       **{"username": "username", "variable": [], "no-increment": ""})
 
 
 def test_packages_add_with_patches(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@@ -129,9 +135,36 @@ def test_packages_add_with_patches(spawner: Spawn, repository_id: RepositoryId,
     """
     patches = [PkgbuildPatch("key", "value"), PkgbuildPatch("key", "value")]
     spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
-    assert spawner.packages_add(repository_id, ["ahriman", "linux"], None, patches=patches, now=False)
+    assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
-    spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux", username=None,
+                                patches=patches, now=False, increment=False, refresh=False)
-                                       variable=[patch.serialize() for patch in patches])
+    spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
+                                       **{
+                                           "username": None,
+                                           "variable": [patch.serialize() for patch in patches],
+                                           "no-increment": ""
+                                       })
+
+
+def test_packages_add_with_increment(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
+    """
+    must call package addition with increment
+    """
+    spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
+    assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
+                                patches=[], now=False, increment=True, refresh=False)
+    spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
+                                       **{"username": None, "variable": [], "increment": ""})
+
+
+def test_packages_add_with_refresh(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
+    """
+    must call package addition with refresh
+    """
+    spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
+    assert spawner.packages_add(repository_id, ["ahriman", "linux"], None,
+                                patches=[], now=False, increment=False, refresh=True)
+    spawn_mock.assert_called_once_with(repository_id, "package-add", "ahriman", "linux",
+                                       **{"username": None, "variable": [], "no-increment": "", "refresh": ""})
 
 
 def test_packages_rebuild(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@@ -139,9 +172,19 @@ def test_packages_rebuild(spawner: Spawn, repository_id: RepositoryId, mocker: M
     must call package rebuild
     """
     spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
-    assert spawner.packages_rebuild(repository_id, "python", "packager")
+    assert spawner.packages_rebuild(repository_id, "python", "packager", increment=False)
     spawn_mock.assert_called_once_with(repository_id, "repo-rebuild",
-                                       **{"depends-on": "python", "username": "packager"})
+                                       **{"depends-on": "python", "username": "packager", "no-increment": ""})
+
+
+def test_packages_rebuild_with_increment(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
+    """
+    must call package rebuild with increment
+    """
+    spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
+    assert spawner.packages_rebuild(repository_id, "python", "packager", increment=True)
+    spawn_mock.assert_called_once_with(repository_id, "repo-rebuild",
+                                       **{"depends-on": "python", "username": "packager", "increment": ""})
 
 
 def test_packages_remove(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
@@ -159,27 +202,61 @@ def test_packages_update(spawner: Spawn, repository_id: RepositoryId, mocker: Mo
     """
     spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
 
-    assert spawner.packages_update(repository_id, "packager", aur=True, local=True, manual=True)
+    assert spawner.packages_update(repository_id, "packager",
-    args = {"username": "packager", "aur": "", "local": "", "manual": ""}
+                                   aur=True, local=True, manual=True, increment=False, refresh=False)
+    args = {"username": "packager", "aur": "", "local": "", "manual": "", "no-increment": ""}
     spawn_mock.assert_called_once_with(repository_id, "repo-update", **args)
     spawn_mock.reset_mock()
 
-    assert spawner.packages_update(repository_id, "packager", aur=False, local=True, manual=True)
+    assert spawner.packages_update(repository_id, "packager",
-    args = {"username": "packager", "no-aur": "", "local": "", "manual": ""}
+                                   aur=False, local=True, manual=True, increment=False, refresh=False)
+    args = {"username": "packager", "no-aur": "", "local": "", "manual": "", "no-increment": ""}
     spawn_mock.assert_called_once_with(repository_id, "repo-update", **args)
     spawn_mock.reset_mock()
 
-    assert spawner.packages_update(repository_id, "packager", aur=True, local=False, manual=True)
+    assert spawner.packages_update(repository_id, "packager",
-    args = {"username": "packager", "aur": "", "no-local": "", "manual": ""}
+                                   aur=True, local=False, manual=True, increment=False, refresh=False)
+    args = {"username": "packager", "aur": "", "no-local": "", "manual": "", "no-increment": ""}
     spawn_mock.assert_called_once_with(repository_id, "repo-update", **args)
     spawn_mock.reset_mock()
 
-    assert spawner.packages_update(repository_id, "packager", aur=True, local=True, manual=False)
+    assert spawner.packages_update(repository_id, "packager",
-    args = {"username": "packager", "aur": "", "local": "", "no-manual": ""}
+                                   aur=True, local=True, manual=False, increment=False, refresh=False)
+    args = {"username": "packager", "aur": "", "local": "", "no-manual": "", "no-increment": ""}
     spawn_mock.assert_called_once_with(repository_id, "repo-update", **args)
     spawn_mock.reset_mock()
 
 
+def test_packages_update_with_increment(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
+    """
+    must call repo update with increment
+    """
+    spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
+    assert spawner.packages_update(repository_id, None,
+                                   aur=True, local=True, manual=True, increment=True, refresh=False)
+    spawn_mock.assert_called_once_with(repository_id, "repo-update",
+                                       **{"username": None, "aur": "", "local": "", "manual": "", "increment": ""})
+
+
+def test_packages_update_with_refresh(spawner: Spawn, repository_id: RepositoryId, mocker: MockerFixture) -> None:
+    """
+    must call repo update with refresh
+    """
+    spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
+    assert spawner.packages_update(repository_id, None,
+                                   aur=True, local=True, manual=True, increment=False, refresh=True)
+    spawn_mock.assert_called_once_with(repository_id, "repo-update",
+                                       **{
+                                           "username": None,
+                                           "aur": "",
+                                           "local": "",
+                                           "manual": "",
+                                           "no-increment": "",
+                                           "refresh": "",
+                                       }
+                                       )
+
+
 def test_run(spawner: Spawn, mocker: MockerFixture) -> None:
     """
     must implement run method

tests/ahriman/core/test_tree.py

@@ -88,6 +88,13 @@ def test_tree_balance() -> None:
     assert third == [leaf5]
 
 
+def test_tree_balance_empty() -> None:
+    """
+    must do not fail on empty tree balancing
+    """
+    assert Tree.balance([]) == []
+
+
 def test_tree_partition(package_ahriman: Package, package_python_schedule: Package) -> None:
     """
     must partition dependencies tree

tests/ahriman/models/test_worker.py

@@ -0,0 +1,10 @@
+from ahriman.models.worker import Worker
+
+
+def test_post_init() -> None:
+    """
+    must read identifier from location if not set
+    """
+    assert Worker("http://localhost:8080").identifier == "localhost:8080"
+    assert Worker("remote").identifier == ""  # not a valid url
+    assert Worker("remote", identifier="id").identifier == "id"

tests/ahriman/web/schemas/test_build_options_schema.py

@@ -0,0 +1 @@
+# schema testing goes in view class tests

tests/ahriman/web/views/test_view_base.py

@@ -193,6 +193,9 @@ async def test_username(base: BaseView, mocker: MockerFixture) -> None:
     policy = AsyncMock()
     policy.identify.return_value = "identity"
     mocker.patch("aiohttp.web.Application.get", return_value=policy)
+    json = AsyncMock()
+    json.return_value = {}
+    base._request = pytest.helpers.request(base.request.app, "", "", json=json)
 
     assert await base.username() == "identity"
     policy.identify.assert_called_once_with(base.request)
@@ -202,4 +205,26 @@ async def test_username_no_auth(base: BaseView) -> None:
     """
     must return None in case if auth is disabled
     """
+    json = AsyncMock()
+    json.return_value = {}
+    base._request = pytest.helpers.request(base.request.app, "", "", json=json)
+
+    assert await base.username() is None
+
+
+async def test_username_request(base: BaseView) -> None:
+    """
+    must read packager from request
+    """
+    json = AsyncMock()
+    json.return_value = {"packager": "identity"}
+    base._request = pytest.helpers.request(base.request.app, "", "", json=json)
+
+    assert await base.username() == "identity"
+
+
+async def test_username_request_exception(base: BaseView) -> None:
+    """
+    must not fail in case if cannot read request
+    """
     assert await base.username() is None

tests/ahriman/web/views/v1/service/test_view_v1_service_add.py

@@ -41,7 +41,8 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc
     assert not request_schema.validate(payload)
     response = await client.post("/api/v1/service/add", json=payload)
     assert response.ok
-    add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", patches=[], now=True)
+    add_mock.assert_called_once_with(repository_id, ["ahriman"], "username",
+                                     patches=[], now=True, increment=True, refresh=False)
 
     json = await response.json()
     assert json["process_id"] == "abc"
@@ -74,7 +75,8 @@ async def test_post_patches(client: TestClient, repository_id: RepositoryId, moc
     response = await client.post("/api/v1/service/add", json=payload)
     assert response.ok
     add_mock.assert_called_once_with(repository_id, ["ahriman"], "username",
-                                     patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")], now=True)
+                                     patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")],
+                                     now=True, increment=True, refresh=False)
 
 
 async def test_post_empty(client: TestClient, mocker: MockerFixture) -> None:

tests/ahriman/web/views/v1/service/test_view_v1_service_rebuild.py

@@ -40,7 +40,7 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc
     assert not request_schema.validate(payload)
     response = await client.post("/api/v1/service/rebuild", json=payload)
     assert response.ok
-    rebuild_mock.assert_called_once_with(repository_id, "python", "username")
+    rebuild_mock.assert_called_once_with(repository_id, "python", "username", increment=True)
 
     json = await response.json()
     assert json["process_id"] == "abc"

tests/ahriman/web/views/v1/service/test_view_v1_service_request.py

@@ -41,7 +41,8 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc
     assert not request_schema.validate(payload)
     response = await client.post("/api/v1/service/request", json=payload)
     assert response.ok
-    add_mock.assert_called_once_with(repository_id, ["ahriman"], "username", patches=[], now=False)
+    add_mock.assert_called_once_with(repository_id, ["ahriman"], "username",
+                                     patches=[], now=False, increment=False, refresh=False)
 
     json = await response.json()
     assert json["process_id"] == "abc"
@@ -74,7 +75,8 @@ async def test_post_patches(client: TestClient, repository_id: RepositoryId, moc
     response = await client.post("/api/v1/service/request", json=payload)
     assert response.ok
     add_mock.assert_called_once_with(repository_id, ["ahriman"], "username",
-                                     patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")], now=False)
+                                     patches=[PkgbuildPatch("k", "v"), PkgbuildPatch("k2", "")],
+                                     now=False, increment=False, refresh=False)
 
 
 async def test_post_exception(client: TestClient, mocker: MockerFixture) -> None:

tests/ahriman/web/views/v1/service/test_view_v1_service_update.py

@@ -40,6 +40,8 @@ async def test_post(client: TestClient, repository_id: RepositoryId, mocker: Moc
         "aur": True,
         "local": True,
         "manual": True,
+        "increment": True,
+        "refresh": False,
     }
 
     for payload in (