feat: add ability to run build process to remote instances (#118)

src/ahriman/application/application/application_repository.py

@@ -18,11 +18,10 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from collections.abc import Iterable
-from pathlib import Path
 
 from ahriman.application.application.application_properties import ApplicationProperties
+from ahriman.application.application.workers import Updater
 from ahriman.core.build_tools.sources import Sources
-from ahriman.core.tree import Tree
 from ahriman.models.package import Package
 from ahriman.models.packagers import Packagers
 from ahriman.models.result import Result
@@ -154,26 +153,25 @@ class ApplicationRepository(ApplicationProperties):
         Returns:
             Result: update result
         """
-        def process_update(paths: Iterable[Path], result: Result) -> None:
-            if not paths:
-                return  # don't need to process if no update supplied
-            update_result = self.repository.process_update(paths, packagers)
-            self.on_result(result.merge(update_result))
+        result = Result()
 
-        # process built packages
-        build_result = Result()
-        packages = self.repository.packages_built()
-        process_update(packages, build_result)
+        # process already built packages if any
+        built_packages = self.repository.packages_built()
+        if built_packages:  # speedup a bit
+            build_result = self.repository.process_update(built_packages, packagers)
+            result.merge(build_result)
+            self.on_result(result.merge(build_result))
 
-        # process manual packages
-        tree = Tree.resolve(updates)
-        for num, level in enumerate(tree):
-            self.logger.info("processing level #%i %s", num, [package.base for package in level])
-            build_result = self.repository.process_build(level, packagers, bump_pkgrel=bump_pkgrel)
-            packages = self.repository.packages_built()
-            process_update(packages, build_result)
+        builder = Updater.load(self.repository_id, self.configuration, self.repository)
 
-        return build_result
+        # ok so for now we split all packages into chunks and process each chunk accordingly
+        partitions = builder.partition(updates)
+        for num, partition in enumerate(partitions):
+            self.logger.info("processing chunk #%i %s", num, [package.base for package in partition])
+            build_result = builder.update(partition, packagers, bump_pkgrel=bump_pkgrel)
+            self.on_result(result.merge(build_result))
+
+        return result
 
     def updates(self, filter_packages: Iterable[str], *,
                 aur: bool, local: bool, manual: bool, vcs: bool) -> list[Package]:

src/ahriman/application/application/workers/__init__.py

@@ -0,0 +1,20 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from ahriman.application.application.workers.updater import Updater

src/ahriman/application/application/workers/local_updater.py

@@ -0,0 +1,77 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from collections.abc import Iterable
+
+from ahriman.application.application.workers.updater import Updater
+from ahriman.core.repository import Repository
+from ahriman.core.tree import Tree
+from ahriman.models.package import Package
+from ahriman.models.packagers import Packagers
+from ahriman.models.result import Result
+
+
+class LocalUpdater(Updater):
+    """
+    local build process implementation
+
+    Attributes:
+        repository(Repository): repository instance
+    """
+
+    def __init__(self, repository: Repository) -> None:
+        """
+        default constructor
+
+        Args:
+            repository(Repository): repository instance
+        """
+        self.repository = repository
+
+    def partition(self, packages: Iterable[Package]) -> list[list[Package]]:
+        """
+        split packages into partitions to be processed by this worker
+
+        Args:
+            packages(Iterable[Package]): list of packages to partition
+
+        Returns:
+            list[list[Package]]: packages partitioned by this worker type
+        """
+        return Tree.resolve(packages)
+
+    def update(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
+               bump_pkgrel: bool = False) -> Result:
+        """
+        run package updates
+
+        Args:
+            updates(Iterable[Package]): list of packages to update
+            packagers(Packagers | None, optional): optional override of username for build process
+                (Default value = None)
+            bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False)
+
+        Returns:
+            Result: update result
+        """
+        build_result = self.repository.process_build(updates, packagers, bump_pkgrel=bump_pkgrel)
+        packages = self.repository.packages_built()
+        update_result = self.repository.process_update(packages, packagers)
+
+        return build_result.merge(update_result)

src/ahriman/application/application/workers/remote_updater.py

@@ -0,0 +1,140 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from collections import deque
+from collections.abc import Iterable
+
+from ahriman.application.application.workers.updater import Updater
+from ahriman.core.configuration import Configuration
+from ahriman.core.http import SyncAhrimanClient
+from ahriman.core.tree import Tree
+from ahriman.models.package import Package
+from ahriman.models.packagers import Packagers
+from ahriman.models.repository_id import RepositoryId
+from ahriman.models.result import Result
+from ahriman.models.worker import Worker
+
+
+class RemoteUpdater(Updater):
+    """
+    remote update worker
+
+    Attributes:
+        configuration(Configuration): configuration instance
+        repository_id(RepositoryId): repository unique identifier
+        workers(list[Worker]): worker identifiers
+    """
+
+    def __init__(self, workers: list[Worker], repository_id: RepositoryId, configuration: Configuration) -> None:
+        """
+        default constructor
+
+        Args:
+            workers(list[Worker]): worker identifiers
+            repository_id(RepositoryId): repository unique identifier
+            configuration(Configuration): configuration instance
+        """
+        self.workers = workers
+        self.repository_id = repository_id
+        self.configuration = configuration
+
+        self._clients: deque[tuple[Worker, SyncAhrimanClient]] = deque()
+
+    @property
+    def clients(self) -> dict[Worker, SyncAhrimanClient]:
+        """
+        extract loaded clients. Note that this method yields only workers which have been already loaded
+
+        Returns:
+            dict[Worker, SyncAhrimanClient]: map of the worker to the related web client
+        """
+        return dict(self._clients)
+
+    @staticmethod
+    def _update_url(worker: Worker) -> str:
+        """
+        get url for updates
+
+        Args:
+            worker(Worker): worker identifier
+
+        Returns:
+            str: full url for web service to run update process
+        """
+        return f"{worker.address}/api/v1/service/add"
+
+    def next_worker(self) -> tuple[Worker, SyncAhrimanClient]:
+        """
+        generate next not-used web client. In case if all clients have been already used, it yields next not used client
+
+        Returns:
+            tuple[Worker, SyncAhrimanClient]: worker and constructed client instance for the web
+        """
+        # check if there is not used yet worker
+        worker = next((worker for worker in self.workers if worker not in self.clients), None)
+        if worker is not None:
+            client = SyncAhrimanClient(self.configuration, "status")
+            client.address = worker.address
+        else:
+            worker, client = self._clients.popleft()
+
+        # register worker in the queue
+        self._clients.append((worker, client))
+
+        return worker, client
+
+    def partition(self, packages: Iterable[Package]) -> list[list[Package]]:
+        """
+        split packages into partitions to be processed by this worker
+
+        Args:
+            packages(Iterable[Package]): list of packages to partition
+
+        Returns:
+            list[list[Package]]: packages partitioned by this worker type
+        """
+        return Tree.partition(packages, count=len(self.workers))
+
+    def update(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
+               bump_pkgrel: bool = False) -> Result:
+        """
+        run package updates
+
+        Args:
+            updates(Iterable[Package]): list of packages to update
+            packagers(Packagers | None, optional): optional override of username for build process
+                (Default value = None)
+            bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False)
+
+        Returns:
+            Result: update result
+        """
+        payload = {
+            "increment": bump_pkgrel,
+            "packager": packagers.default if packagers is not None else None,
+            "packages": [package.base for package in updates],
+            "patches": [],  # might be used later
+            "refresh": True,
+        }
+
+        worker, client = self.next_worker()
+        client.make_request("POST", self._update_url(worker), params=self.repository_id.query(), json=payload)
+        # we don't block here for process
+
+        return Result()

src/ahriman/application/application/workers/updater.py

@@ -0,0 +1,102 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.log import LazyLogging
+from ahriman.core.repository import Repository
+from ahriman.models.package import Package
+from ahriman.models.packagers import Packagers
+from ahriman.models.repository_id import RepositoryId
+from ahriman.models.result import Result
+from ahriman.models.worker import Worker
+
+
+class Updater(LazyLogging):
+    """
+    updater handler interface
+
+    Attributes:
+        split_method(Callable[[Iterable[Package]], list[list[Package]]]): method to split packages into chunks
+    """
+
+    @staticmethod
+    def load(repository_id: RepositoryId, configuration: Configuration,
+             repository: Repository, workers: list[Worker] | None = None) -> Updater:
+        """
+        construct updaters from parameters
+
+        Args:
+            repository_id(RepositoryId): repository unique identifier
+            configuration(Configuration): configuration instance
+            repository(Repository): repository instance
+            workers(list[Worker] | None, optional): worker identifiers if any (Default value = None)
+
+        Returns:
+            Updater: constructed updater worker
+        """
+        if workers is None:
+            # no workers set explicitly, try to guess from configuration
+            workers = [Worker(address) for address in configuration.getlist("build", "workers", fallback=[])]
+
+        if workers:
+            # there is something we could use as remote workers
+            from ahriman.application.application.workers.remote_updater import RemoteUpdater
+            return RemoteUpdater(workers, repository_id, configuration)
+
+        # and finally no workers available, just use local service
+        from ahriman.application.application.workers.local_updater import LocalUpdater
+        return LocalUpdater(repository)
+
+    def partition(self, packages: Iterable[Package]) -> list[list[Package]]:
+        """
+        split packages into partitions to be processed by this worker
+
+        Args:
+            packages(Iterable[Package]): list of packages to partition
+
+        Returns:
+            list[list[Package]]: packages partitioned by this worker type
+
+        Raises:
+            NotImplementedError: not implemented method
+        """
+        raise NotImplementedError
+
+    def update(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
+               bump_pkgrel: bool = False) -> Result:
+        """
+        run package updates
+
+        Args:
+            updates(Iterable[Package]): list of packages to update
+            packagers(Packagers | None, optional): optional override of username for build process
+                (Default value = None)
+            bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False)
+
+        Returns:
+            Result: update result
+
+        Raises:
+            NotImplementedError: not implemented method
+        """
+        raise NotImplementedError

src/ahriman/core/configuration/schema.py

@@ -213,6 +213,15 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
                 "coerce": "integer",
                 "min": 0,
             },
+            "workers": {
+                "type": "list",
+                "coerce": "list",
+                "schema": {
+                    "type": "string",
+                    "empty": False,
+                    "is_url": [],
+                },
+            },
         },
     },
     "repository": {

src/ahriman/core/spawn.py

@@ -174,7 +174,7 @@ class Spawn(Thread, LazyLogging):
         return self._spawn_process(repository_id, "service-key-import", key, **kwargs)
 
     def packages_add(self, repository_id: RepositoryId, packages: Iterable[str], username: str | None, *,
-                     patches: list[PkgbuildPatch], now: bool) -> str:
+                     patches: list[PkgbuildPatch], now: bool, increment: bool, refresh: bool) -> str:
         """
         add packages
 
@@ -184,19 +184,26 @@ class Spawn(Thread, LazyLogging):
             username(str | None): optional override of username for build process
             patches(list[PkgbuildPatch]): list of patches to be passed
             now(bool): build packages now
+            increment(bool): increment pkgrel on conflict
+            refresh(bool): refresh pacman database before process
 
         Returns:
             str: spawned process identifier
         """
-        kwargs: dict[str, str | list[str] | None] = {"username": username}
+        kwargs: dict[str, str | list[str] | None] = {
+            "username": username,
+            "variable": [patch.serialize() for patch in patches],
+            self.boolean_action_argument("increment", increment): "",
+        }
         if now:
             kwargs["now"] = ""
-        if patches:
-            kwargs["variable"] = [patch.serialize() for patch in patches]
+        if refresh:
+            kwargs["refresh"] = ""
 
         return self._spawn_process(repository_id, "package-add", *packages, **kwargs)
 
-    def packages_rebuild(self, repository_id: RepositoryId, depends_on: str, username: str | None) -> str:
+    def packages_rebuild(self, repository_id: RepositoryId, depends_on: str, username: str | None, *,
+                         increment: bool) -> str:
         """
         rebuild packages which depend on the specified package
 
@@ -204,11 +211,16 @@ class Spawn(Thread, LazyLogging):
             repository_id(RepositoryId): repository unique identifier
             depends_on(str): packages dependency
             username(str | None): optional override of username for build process
+            increment(bool): increment pkgrel on conflict
 
         Returns:
             str: spawned process identifier
         """
-        kwargs = {"depends-on": depends_on, "username": username}
+        kwargs = {
+            "depends-on": depends_on,
+            "username": username,
+            self.boolean_action_argument("increment", increment): "",
+        }
         return self._spawn_process(repository_id, "repo-rebuild", **kwargs)
 
     def packages_remove(self, repository_id: RepositoryId, packages: Iterable[str]) -> str:
@@ -225,7 +237,7 @@ class Spawn(Thread, LazyLogging):
         return self._spawn_process(repository_id, "package-remove", *packages)
 
     def packages_update(self, repository_id: RepositoryId, username: str | None, *,
-                        aur: bool, local: bool, manual: bool) -> str:
+                        aur: bool, local: bool, manual: bool, increment: bool, refresh: bool) -> str:
         """
         run full repository update
 
@@ -235,6 +247,8 @@ class Spawn(Thread, LazyLogging):
             aur(bool): check for aur updates
             local(bool): check for local packages updates
             manual(bool): check for manual packages
+            increment(bool): increment pkgrel on conflict
+            refresh(bool): refresh pacman database before process
 
         Returns:
             str: spawned process identifier
@@ -244,7 +258,11 @@ class Spawn(Thread, LazyLogging):
             self.boolean_action_argument("aur", aur): "",
             self.boolean_action_argument("local", local): "",
             self.boolean_action_argument("manual", manual): "",
+            self.boolean_action_argument("increment", increment): "",
         }
+        if refresh:
+            kwargs["refresh"] = ""
+
         return self._spawn_process(repository_id, "repo-update", **kwargs)
 
     def run(self) -> None:

src/ahriman/core/tree.py

@@ -148,6 +148,8 @@ class Tree:
             sorted(part, key=lambda leaf: leaf.package.base)
             for part in partitions if part
         ]
+        if not partitions:  # nothing to balance
+            return partitions
 
         while True:
             min_part, max_part = minmax(partitions, key=len)

src/ahriman/models/worker.py

@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from dataclasses import dataclass, field
+from urllib.parse import urlparse
+
+
+@dataclass(frozen=True)
+class Worker:
+    """
+    worker descriptor
+
+    Attributes:
+        address(str): worker address to be reachable outside
+        identifier(str): worker unique identifier. If none set it will be automatically generated from the address
+    """
+
+    address: str
+    identifier: str = field(default="", kw_only=True)
+
+    def __post_init__(self) -> None:
+        """
+        update identifier based on settings
+        """
+        object.__setattr__(self, "identifier", self.identifier or urlparse(self.address).netloc)

src/ahriman/web/schemas/__init__.py

@@ -19,6 +19,7 @@
 #
 from ahriman.web.schemas.aur_package_schema import AURPackageSchema
 from ahriman.web.schemas.auth_schema import AuthSchema
+from ahriman.web.schemas.build_options_schema import BuildOptionsSchema
 from ahriman.web.schemas.changes_schema import ChangesSchema
 from ahriman.web.schemas.counters_schema import CountersSchema
 from ahriman.web.schemas.error_schema import ErrorSchema

src/ahriman/web/schemas/build_options_schema.py

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from marshmallow import Schema, fields
+
+
+class BuildOptionsSchema(Schema):
+    """
+    request build options schema
+    """
+
+    increment = fields.Boolean(dump_default=True, metadata={
+        "description": "Increment pkgrel on conflicts",
+    })
+    packager = fields.String(metadata={
+        "description": "Packager identity if applicable",
+    })
+    refresh = fields.Boolean(dump_default=True, metadata={
+        "description": "Refresh pacman database"
+    })

src/ahriman/web/schemas/package_names_schema.py

@@ -17,10 +17,12 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from marshmallow import Schema, fields
+from marshmallow import fields
+
+from ahriman.web.schemas.build_options_schema import BuildOptionsSchema
 
 
-class PackageNamesSchema(Schema):
+class PackageNamesSchema(BuildOptionsSchema):
     """
     request package names schema
     """

src/ahriman/web/schemas/update_flags_schema.py

@@ -17,20 +17,22 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from marshmallow import Schema, fields
+from marshmallow import fields
+
+from ahriman.web.schemas.build_options_schema import BuildOptionsSchema
 
 
-class UpdateFlagsSchema(Schema):
+class UpdateFlagsSchema(BuildOptionsSchema):
     """
     update flags request schema
     """
 
-    aur = fields.Bool(dump_default=True, metadata={
+    aur = fields.Boolean(dump_default=True, metadata={
         "description": "Check AUR for updates",
     })
-    local = fields.Bool(dump_default=True, metadata={
+    local = fields.Boolean(dump_default=True, metadata={
         "description": "Check local packages for updates",
     })
-    manual = fields.Bool(dump_default=True, metadata={
+    manual = fields.Boolean(dump_default=True, metadata={
         "description": "Check manually built packages",
     })

src/ahriman/web/views/base.py

@@ -222,6 +222,13 @@ class BaseView(View, CorsViewMixin):
         Returns:
             str | None: authorized username if any and None otherwise (e.g. if authorization is disabled)
         """
+        try:  # try to read from payload
+            data: dict[str, str] = await self.request.json()  # technically it is not, but we only need str here
+            if (packager := data.get("packager")) is not None:
+                return packager
+        except Exception:
+            self.request.app.logger.exception("could not extract json data for packager")
+
         policy = self.request.app.get("identity")
         if policy is not None:
             identity: str = await policy.identify(self.request)

src/ahriman/web/views/v1/service/add.py

@@ -74,6 +74,14 @@ class AddView(BaseView):
 
         repository_id = self.repository_id()
         username = await self.username()
-        process_id = self.spawner.packages_add(repository_id, packages, username, patches=patches, now=True)
+        process_id = self.spawner.packages_add(
+            repository_id,
+            packages,
+            username,
+            patches=patches,
+            now=True,
+            increment=data.get("increment", True),
+            refresh=data.get("refresh", False),
+        )
 
         return json_response({"process_id": process_id})

src/ahriman/web/views/v1/service/rebuild.py

@@ -73,6 +73,11 @@ class RebuildView(BaseView):
 
         repository_id = self.repository_id()
         username = await self.username()
-        process_id = self.spawner.packages_rebuild(repository_id, depends_on, username)
+        process_id = self.spawner.packages_rebuild(
+            repository_id,
+            depends_on,
+            username,
+            increment=data.get("increment", True),
+        )
 
         return json_response({"process_id": process_id})

src/ahriman/web/views/v1/service/request.py

@@ -74,6 +74,14 @@ class RequestView(BaseView):
 
         username = await self.username()
         repository_id = self.repository_id()
-        process_id = self.spawner.packages_add(repository_id, packages, username, patches=patches, now=False)
+        process_id = self.spawner.packages_add(
+            repository_id,
+            packages,
+            username,
+            patches=patches,
+            now=False,
+            increment=False,  # no-increment doesn't work here
+            refresh=False,  # refresh doesn't work here
+        )
 
         return json_response({"process_id": process_id})

src/ahriman/web/views/v1/service/update.py

@@ -77,6 +77,8 @@ class UpdateView(BaseView):
             aur=data.get("aur", True),
             local=data.get("local", True),
             manual=data.get("manual", True),
+            increment=data.get("increment", True),
+            refresh=data.get("refresh", False),
         )
 
         return json_response({"process_id": process_id})