Remote call trigger support (#105)

* add support of remote task tracking
* add remote call trigger implementation
* docs update
* add cross-service upload
* add notes about user
* add more ability to control upload
* multipart upload with signatures as well as safe file save
* configuration reference update
* rename watcher methods
* erase logs based on current package version

Old implementation has used process id instead, but it leads to log
removal in case of remote process trigger

* add --server flag for setup command
* restore behavior of the httploghandler

src/ahriman/web/views/api/swagger.py

@@ -43,7 +43,7 @@ class SwaggerView(BaseView):
             Response: 200 with json api specification
         """
         spec = self.request.app["swagger_dict"]
-        is_body_parameter: Callable[[dict[str, str]], bool] = lambda p: p["in"] == "body"
+        is_body_parameter: Callable[[dict[str, str]], bool] = lambda p: p["in"] == "body" or p["in"] == "formData"
 
         # special workaround because it writes request body to parameters section
         paths = spec["paths"]
@@ -56,11 +56,14 @@ class SwaggerView(BaseView):
                 if not body:
                     continue  # there were no ``body`` parameters found
 
+                schema = next(iter(body))
+                content_type = "multipart/form-data" if schema["in"] == "formData" else "application/json"
+
                 # there should be only one body parameters
                 method["requestBody"] = {
                     "content": {
-                        "application/json": {
-                            "schema": next(iter(body))["schema"]
+                        content_type: {
+                            "schema": schema["schema"]
                         }
                     }
                 }

src/ahriman/web/views/service/add.py

@@ -19,10 +19,10 @@
 #
 import aiohttp_apispec  # type: ignore[import]
 
-from aiohttp.web import HTTPBadRequest, HTTPNoContent
+from aiohttp.web import HTTPBadRequest, Response, json_response
 
 from ahriman.models.user_access import UserAccess
-from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
+from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
 from ahriman.web.views.base import BaseView
 
 
@@ -41,7 +41,7 @@ class AddView(BaseView):
         summary="Add new package",
         description="Add new package(s) from AUR",
         responses={
-            204: {"description": "Success response"},
+            200: {"description": "Success response", "schema": ProcessIdSchema},
             400: {"description": "Bad data is supplied", "schema": ErrorSchema},
             401: {"description": "Authorization required", "schema": ErrorSchema},
             403: {"description": "Access is forbidden", "schema": ErrorSchema},
@@ -51,13 +51,15 @@ class AddView(BaseView):
     )
     @aiohttp_apispec.cookies_schema(AuthSchema)
     @aiohttp_apispec.json_schema(PackageNamesSchema)
-    async def post(self) -> None:
+    async def post(self) -> Response:
         """
         add new package
 
+        Returns:
+            Response: 200 with spawned process id
+
         Raises:
             HTTPBadRequest: if bad data is supplied
-            HTTPNoContent: in case of success response
         """
         try:
             data = await self.extract_data(["packages"])
@@ -66,6 +68,6 @@ class AddView(BaseView):
             raise HTTPBadRequest(reason=str(e))
 
         username = await self.username()
-        self.spawner.packages_add(packages, username, now=True)
+        process_id = self.spawner.packages_add(packages, username, now=True)
 
-        raise HTTPNoContent()
+        return json_response({"process_id": process_id})

src/ahriman/web/views/service/pgp.py

@@ -19,10 +19,10 @@
 #
 import aiohttp_apispec  # type: ignore[import]
 
-from aiohttp.web import HTTPBadRequest, HTTPNoContent, HTTPNotFound, Response, json_response
+from aiohttp.web import HTTPBadRequest, HTTPNotFound, Response, json_response
 
 from ahriman.models.user_access import UserAccess
-from ahriman.web.schemas import AuthSchema, ErrorSchema, PGPKeyIdSchema, PGPKeySchema
+from ahriman.web.schemas import AuthSchema, ErrorSchema, PGPKeyIdSchema, PGPKeySchema, ProcessIdSchema
 from ahriman.web.views.base import BaseView
 
 
@@ -83,7 +83,7 @@ class PGPView(BaseView):
         summary="Fetch PGP key",
         description="Fetch PGP key from the key server",
         responses={
-            204: {"description": "Success response"},
+            200: {"description": "Success response", "schema": ProcessIdSchema},
             400: {"description": "Bad data is supplied", "schema": ErrorSchema},
             401: {"description": "Authorization required", "schema": ErrorSchema},
             403: {"description": "Access is forbidden", "schema": ErrorSchema},
@@ -93,13 +93,15 @@ class PGPView(BaseView):
     )
     @aiohttp_apispec.cookies_schema(AuthSchema)
     @aiohttp_apispec.json_schema(PGPKeyIdSchema)
-    async def post(self) -> None:
+    async def post(self) -> Response:
         """
         store key to the local service environment
 
+        Returns:
+            Response: 200 with spawned process id
+
         Raises:
             HTTPBadRequest: if bad data is supplied
-            HTTPNoContent: in case of success response
         """
         data = await self.extract_data()
 
@@ -108,6 +110,6 @@ class PGPView(BaseView):
         except Exception as e:
             raise HTTPBadRequest(reason=str(e))
 
-        self.spawner.key_import(key, data.get("server"))
+        process_id = self.spawner.key_import(key, data.get("server"))
 
-        raise HTTPNoContent()
+        return json_response({"process_id": process_id})

src/ahriman/web/views/service/process.py

@@ -0,0 +1,74 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import aiohttp_apispec  # type: ignore[import]
+
+from aiohttp.web import HTTPNotFound, Response, json_response
+
+from ahriman.models.user_access import UserAccess
+from ahriman.web.schemas import AuthSchema, ErrorSchema, ProcessIdSchema, ProcessSchema
+from ahriman.web.views.base import BaseView
+
+
+class ProcessView(BaseView):
+    """
+    Process information web view
+
+    Attributes:
+        GET_PERMISSION(UserAccess): (class attribute) get permissions of self
+    """
+
+    GET_PERMISSION = UserAccess.Reporter
+
+    @aiohttp_apispec.docs(
+        tags=["Actions"],
+        summary="Get process",
+        description="Get process information",
+        responses={
+            200: {"description": "Success response", "schema": ProcessSchema},
+            401: {"description": "Authorization required", "schema": ErrorSchema},
+            403: {"description": "Access is forbidden", "schema": ErrorSchema},
+            404: {"description": "Not found", "schema": ErrorSchema},
+            500: {"description": "Internal server error", "schema": ErrorSchema},
+        },
+        security=[{"token": [GET_PERMISSION]}],
+    )
+    @aiohttp_apispec.cookies_schema(AuthSchema)
+    @aiohttp_apispec.match_info_schema(ProcessIdSchema)
+    async def get(self) -> Response:
+        """
+        get spawned process status
+
+        Returns:
+            Response: 200 with process information
+
+        Raises:
+            HTTPNotFound: if no process found
+        """
+        process_id = self.request.match_info["process_id"]
+
+        is_alive = self.spawner.has_process(process_id)
+        if not is_alive:
+            raise HTTPNotFound(reason=f"No process {process_id} found")
+
+        response = {
+            "is_alive": is_alive,
+        }
+
+        return json_response(response)

src/ahriman/web/views/service/rebuild.py

@@ -19,10 +19,10 @@
 #
 import aiohttp_apispec  # type: ignore[import]
 
-from aiohttp.web import HTTPBadRequest, HTTPNoContent
+from aiohttp.web import HTTPBadRequest, Response, json_response
 
 from ahriman.models.user_access import UserAccess
-from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
+from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
 from ahriman.web.views.base import BaseView
 
 
@@ -41,7 +41,7 @@ class RebuildView(BaseView):
         summary="Rebuild packages",
         description="Rebuild packages which depend on specified one",
         responses={
-            204: {"description": "Success response"},
+            200: {"description": "Success response", "schema": ProcessIdSchema},
             400: {"description": "Bad data is supplied", "schema": ErrorSchema},
             401: {"description": "Authorization required", "schema": ErrorSchema},
             403: {"description": "Access is forbidden", "schema": ErrorSchema},
@@ -51,13 +51,15 @@ class RebuildView(BaseView):
     )
     @aiohttp_apispec.cookies_schema(AuthSchema)
     @aiohttp_apispec.json_schema(PackageNamesSchema)
-    async def post(self) -> None:
+    async def post(self) -> Response:
         """
         rebuild packages based on their dependency
 
+        Returns:
+            Response: 200 with spawned process id
+
         Raises:
             HTTPBadRequest: if bad data is supplied
-            HTTPNoContent: in case of success response
         """
         try:
             data = await self.extract_data(["packages"])
@@ -67,6 +69,6 @@ class RebuildView(BaseView):
             raise HTTPBadRequest(reason=str(e))
 
         username = await self.username()
-        self.spawner.packages_rebuild(depends_on, username)
+        process_id = self.spawner.packages_rebuild(depends_on, username)
 
-        raise HTTPNoContent()
+        return json_response({"process_id": process_id})

src/ahriman/web/views/service/remove.py

@@ -19,10 +19,10 @@
 #
 import aiohttp_apispec  # type: ignore[import]
 
-from aiohttp.web import HTTPBadRequest, HTTPNoContent
+from aiohttp.web import HTTPBadRequest, Response, json_response
 
 from ahriman.models.user_access import UserAccess
-from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
+from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
 from ahriman.web.views.base import BaseView
 
 
@@ -41,7 +41,7 @@ class RemoveView(BaseView):
         summary="Remove packages",
         description="Remove specified packages from the repository",
         responses={
-            204: {"description": "Success response"},
+            200: {"description": "Success response", "schema": ProcessIdSchema},
             400: {"description": "Bad data is supplied", "schema": ErrorSchema},
             401: {"description": "Authorization required", "schema": ErrorSchema},
             403: {"description": "Access is forbidden", "schema": ErrorSchema},
@@ -51,13 +51,15 @@ class RemoveView(BaseView):
     )
     @aiohttp_apispec.cookies_schema(AuthSchema)
     @aiohttp_apispec.json_schema(PackageNamesSchema)
-    async def post(self) -> None:
+    async def post(self) -> Response:
         """
         remove existing packages
 
+        Returns:
+            Response: 200 with spawned process id
+
         Raises:
             HTTPBadRequest: if bad data is supplied
-            HTTPNoContent: in case of success response
         """
         try:
             data = await self.extract_data(["packages"])
@@ -65,6 +67,6 @@ class RemoveView(BaseView):
         except Exception as e:
             raise HTTPBadRequest(reason=str(e))
 
-        self.spawner.packages_remove(packages)
+        process_id = self.spawner.packages_remove(packages)
 
-        raise HTTPNoContent()
+        return json_response({"process_id": process_id})

src/ahriman/web/views/service/request.py

@@ -19,10 +19,10 @@
 #
 import aiohttp_apispec  # type: ignore[import]
 
-from aiohttp.web import HTTPBadRequest, HTTPNoContent
+from aiohttp.web import HTTPBadRequest, Response, json_response
 
 from ahriman.models.user_access import UserAccess
-from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
+from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
 from ahriman.web.views.base import BaseView
 
 
@@ -41,7 +41,7 @@ class RequestView(BaseView):
         summary="Request new package",
         description="Request new package(s) to be added from AUR",
         responses={
-            204: {"description": "Success response"},
+            200: {"description": "Success response", "schema": ProcessIdSchema},
             400: {"description": "Bad data is supplied", "schema": ErrorSchema},
             401: {"description": "Authorization required", "schema": ErrorSchema},
             403: {"description": "Access is forbidden", "schema": ErrorSchema},
@@ -51,13 +51,15 @@ class RequestView(BaseView):
     )
     @aiohttp_apispec.cookies_schema(AuthSchema)
     @aiohttp_apispec.json_schema(PackageNamesSchema)
-    async def post(self) -> None:
+    async def post(self) -> Response:
         """
         request to add new package
 
+        Returns:
+            Response: 200 with spawned process id
+
         Raises:
             HTTPBadRequest: if bad data is supplied
-            HTTPNoContent: in case of success response
         """
         try:
             data = await self.extract_data(["packages"])
@@ -66,6 +68,6 @@ class RequestView(BaseView):
             raise HTTPBadRequest(reason=str(e))
 
         username = await self.username()
-        self.spawner.packages_add(packages, username, now=False)
+        process_id = self.spawner.packages_add(packages, username, now=False)
 
-        raise HTTPNoContent()
+        return json_response({"process_id": process_id})

src/ahriman/web/views/service/update.py

@@ -19,10 +19,10 @@
 #
 import aiohttp_apispec  # type: ignore[import]
 
-from aiohttp.web import HTTPNoContent
+from aiohttp.web import HTTPBadRequest, Response, json_response
 
 from ahriman.models.user_access import UserAccess
-from ahriman.web.schemas import AuthSchema, ErrorSchema
+from ahriman.web.schemas import AuthSchema, ErrorSchema, ProcessIdSchema, UpdateFlagsSchema
 from ahriman.web.views.base import BaseView
 
 
@@ -41,7 +41,8 @@ class UpdateView(BaseView):
         summary="Update packages",
         description="Run repository update process",
         responses={
-            204: {"description": "Success response"},
+            200: {"description": "Success response", "schema": ProcessIdSchema},
+            400: {"description": "Bad data is supplied", "schema": ErrorSchema},
             401: {"description": "Authorization required", "schema": ErrorSchema},
             403: {"description": "Access is forbidden", "schema": ErrorSchema},
             500: {"description": "Internal server error", "schema": ErrorSchema},
@@ -49,14 +50,28 @@ class UpdateView(BaseView):
         security=[{"token": [POST_PERMISSION]}],
     )
     @aiohttp_apispec.cookies_schema(AuthSchema)
-    async def post(self) -> None:
+    @aiohttp_apispec.json_schema(UpdateFlagsSchema)
+    async def post(self) -> Response:
         """
         run repository update. No parameters supported here
 
-        Raises:
-            HTTPNoContent: in case of success response
-        """
-        username = await self.username()
-        self.spawner.packages_update(username)
+        Returns:
+            Response: 200 with spawned process id
 
-        raise HTTPNoContent()
+        Raises:
+            HTTPBadRequest: if bad data is supplied
+        """
+        try:
+            data = await self.extract_data()
+        except Exception as e:
+            raise HTTPBadRequest(reason=str(e))
+
+        username = await self.username()
+        process_id = self.spawner.packages_update(
+            username,
+            aur=data.get("aur", True),
+            local=data.get("local", True),
+            manual=data.get("manual", True),
+        )
+
+        return json_response({"process_id": process_id})

src/ahriman/web/views/service/upload.py

@@ -0,0 +1,144 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import aiohttp_apispec  # type: ignore[import]
+import shutil
+
+from aiohttp import BodyPartReader
+from aiohttp.web import HTTPBadRequest, HTTPCreated, HTTPNotFound
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+
+from ahriman.models.user_access import UserAccess
+from ahriman.web.schemas import AuthSchema, ErrorSchema, FileSchema
+from ahriman.web.views.base import BaseView
+
+
+class UploadView(BaseView):
+    """
+    upload file to repository
+
+    Attributes:
+        POST_PERMISSION(UserAccess): (class attribute) post permissions of self
+    """
+
+    POST_PERMISSION = UserAccess.Full
+
+    @staticmethod
+    async def save_file(part: BodyPartReader, target: Path, *, max_body_size: int | None = None) -> tuple[str, Path]:
+        """
+        save file to local cache
+
+        Args:
+            part(BodyPartReader): multipart part to be saved
+            target(Path): path to directory to which file should be saved
+            max_body_size(int | None, optional): max body size in bytes (Default value = None)
+
+        Returns:
+            tuple[str, Path]: map of received filename to its local path
+
+        Raises:
+            HTTPBadRequest: if bad data is supplied
+        """
+        archive_name = part.filename
+        if archive_name is None:
+            raise HTTPBadRequest(reason="Filename must be set")
+        # some magic inside. We would like to make sure that passed filename is filename
+        # without slashes, dots, etc
+        if Path(archive_name).resolve().name != archive_name:
+            raise HTTPBadRequest(reason="Filename must be valid archive name")
+
+        current_size = 0
+
+        # in order to handle errors automatically we create temporary file for long operation (transfer)
+        # and then copy it to valid location
+        with NamedTemporaryFile() as cache:
+            while True:
+                chunk = await part.read_chunk()
+                if not chunk:
+                    break
+
+                current_size += len(chunk)
+                if max_body_size is not None and current_size > max_body_size:
+                    raise HTTPBadRequest(reason="Body part is too large")
+
+                cache.write(chunk)
+
+            cache.seek(0)  # reset file position
+
+            # and now copy temporary file to target location as hidden file
+            # we put it as hidden in order to make sure that it will not be handled during some random process
+            temporary_output = target / f".{archive_name}"
+            with temporary_output.open("wb") as archive:
+                shutil.copyfileobj(cache, archive)
+
+            return archive_name, temporary_output
+
+    @aiohttp_apispec.docs(
+        tags=["Actions"],
+        summary="Upload package",
+        description="Upload package to local filesystem",
+        responses={
+            201: {"description": "Success response"},
+            400: {"description": "Bad data is supplied", "schema": ErrorSchema},
+            401: {"description": "Authorization required", "schema": ErrorSchema},
+            403: {"description": "Access is forbidden", "schema": ErrorSchema},
+            404: {"description": "Not found", "schema": ErrorSchema},
+            500: {"description": "Internal server error", "schema": ErrorSchema},
+        },
+        security=[{"token": [POST_PERMISSION]}],
+    )
+    @aiohttp_apispec.cookies_schema(AuthSchema)
+    @aiohttp_apispec.form_schema(FileSchema)
+    async def post(self) -> None:
+        """
+        upload file from another instance to the server
+
+        Raises:
+            HTTPBadRequest: if bad data is supplied
+            HTTPCreated: on success response
+        """
+        if not self.configuration.getboolean("web", "enable_archive_upload", fallback=False):
+            raise HTTPNotFound()
+
+        try:
+            reader = await self.request.multipart()
+        except Exception as e:
+            raise HTTPBadRequest(reason=str(e))
+
+        max_body_size = self.configuration.getint("web", "max_body_size", fallback=None)
+        target = self.configuration.repository_paths.packages
+
+        files = []
+        while (part := await reader.next()) is not None:
+            if not isinstance(part, BodyPartReader):
+                raise HTTPBadRequest(reason="Invalid multipart message received")
+
+            if part.name not in ("package", "signature"):
+                raise HTTPBadRequest(reason="Multipart field isn't package or signature")
+
+            files.append(await self.save_file(part, target, max_body_size=max_body_size))
+
+        # and now we can rename files, which is relatively fast operation
+        # it is probably good way to call lock here, however
+        for filename, current_location in files:
+            target_location = current_location.parent / filename
+            current_location.rename(target_location)
+
+        raise HTTPCreated()

src/ahriman/web/views/status/logs.py

@@ -63,7 +63,7 @@ class LogsView(BaseView):
             HTTPNoContent: on success response
         """
         package_base = self.request.match_info["package"]
-        self.service.remove_logs(package_base, None)
+        self.service.logs_remove(package_base, None)
 
         raise HTTPNoContent()
 
@@ -95,10 +95,10 @@ class LogsView(BaseView):
         package_base = self.request.match_info["package"]
 
         try:
-            _, status = self.service.get(package_base)
+            _, status = self.service.package_get(package_base)
         except UnknownPackageError:
             raise HTTPNotFound()
-        logs = self.service.get_logs(package_base)
+        logs = self.service.logs_get(package_base)
 
         response = {
             "package_base": package_base,
@@ -137,10 +137,10 @@ class LogsView(BaseView):
         try:
             created = data["created"]
             record = data["message"]
-            process_id = data["process_id"]
+            version = data["version"]
         except Exception as e:
             raise HTTPBadRequest(reason=str(e))
 
-        self.service.update_logs(LogRecordId(package_base, process_id), created, record)
+        self.service.logs_update(LogRecordId(package_base, version), created, record)
 
         raise HTTPNoContent()

src/ahriman/web/views/status/package.py

@@ -64,7 +64,7 @@ class PackageView(BaseView):
             HTTPNoContent: on success response
         """
         package_base = self.request.match_info["package"]
-        self.service.remove(package_base)
+        self.service.package_remove(package_base)
 
         raise HTTPNoContent()
 
@@ -96,7 +96,7 @@ class PackageView(BaseView):
         package_base = self.request.match_info["package"]
 
         try:
-            package, status = self.service.get(package_base)
+            package, status = self.service.package_get(package_base)
         except UnknownPackageError:
             raise HTTPNotFound()
 
@@ -142,7 +142,7 @@ class PackageView(BaseView):
             raise HTTPBadRequest(reason=str(e))
 
         try:
-            self.service.update(package_base, status, package)
+            self.service.package_update(package_base, status, package)
         except UnknownPackageError:
             raise HTTPBadRequest(reason=f"Package {package_base} is unknown, but no package body set")
 

src/ahriman/web/views/status/status.py

@@ -102,6 +102,6 @@ class StatusView(BaseView):
         except Exception as e:
             raise HTTPBadRequest(reason=str(e))
 
-        self.service.update_self(status)
+        self.service.status_update(status)
 
         raise HTTPNoContent()