Add ability to trigger updates from the web (#31)

* add external process spawner and update test cases

* pass no_report to handlers

* provide service api endpoints

* do not spawn process for single architecture run

* pass no report to handlers

* make _call method of handlers public and also simplify process spawn

* move update under add

* implement actions from web page

* clear logging & improve l&f

src/ahriman/web/middlewares/auth_handler.py

@@ -73,9 +73,7 @@ def auth_handler(validator: Auth) -> MiddlewareType:
     """
     @middleware
     async def handle(request: Request, handler: HandlerType) -> StreamResponse:
-        if request.path.startswith("/status-api"):
-            permission = UserAccess.Status
-        elif request.method in ("GET", "HEAD", "OPTIONS"):
+        if request.method in ("GET", "HEAD", "OPTIONS"):
             permission = UserAccess.Read
         else:
             permission = UserAccess.Write

src/ahriman/web/routes.py

@@ -19,13 +19,16 @@
 #
 from aiohttp.web import Application
 
-from ahriman.web.views.ahriman import AhrimanView
 from ahriman.web.views.index import IndexView
-from ahriman.web.views.login import LoginView
-from ahriman.web.views.logout import LogoutView
-from ahriman.web.views.package import PackageView
-from ahriman.web.views.packages import PackagesView
-from ahriman.web.views.status import StatusView
+from ahriman.web.views.service.add import AddView
+from ahriman.web.views.service.remove import RemoveView
+from ahriman.web.views.service.search import SearchView
+from ahriman.web.views.status.ahriman import AhrimanView
+from ahriman.web.views.status.package import PackageView
+from ahriman.web.views.status.packages import PackagesView
+from ahriman.web.views.status.status import StatusView
+from ahriman.web.views.user.login import LoginView
+from ahriman.web.views.user.logout import LogoutView
 
 
 def setup_routes(application: Application) -> None:
@@ -37,8 +40,13 @@ def setup_routes(application: Application) -> None:
         GET /                                  get build status page
         GET /index.html                        same as above
 
-        POST /login                            login to service
-        POST /logout                           logout from service
+        POST /service-api/v1/add               add new packages to repository
+
+        POST /service-api/v1/remove            remove existing package from repository
+
+        POST /service-api/v1/update            update packages in repository, actually it is just alias for add
+
+        GET /service-api/v1/search             search for substring in AUR
 
         GET /status-api/v1/ahriman             get current service status
         POST /status-api/v1/ahriman            update service status
@@ -52,13 +60,21 @@ def setup_routes(application: Application) -> None:
 
         GET /status-api/v1/status              get web service status itself
 
+        POST /user-api/v1/login                login to service
+        POST /user-api/v1/logout               logout from service
+
     :param application: web application instance
     """
     application.router.add_get("/", IndexView, allow_head=True)
     application.router.add_get("/index.html", IndexView, allow_head=True)
 
-    application.router.add_post("/login", LoginView)
-    application.router.add_post("/logout", LogoutView)
+    application.router.add_post("/service-api/v1/add", AddView)
+
+    application.router.add_post("/service-api/v1/remove", RemoveView)
+
+    application.router.add_get("/service-api/v1/search", SearchView, allow_head=False)
+
+    application.router.add_post("/service-api/v1/update", AddView)
 
     application.router.add_get("/status-api/v1/ahriman", AhrimanView, allow_head=True)
     application.router.add_post("/status-api/v1/ahriman", AhrimanView)
@@ -71,3 +87,6 @@ def setup_routes(application: Application) -> None:
     application.router.add_post("/status-api/v1/packages/{package}", PackageView)
 
     application.router.add_get("/status-api/v1/status", StatusView, allow_head=True)
+
+    application.router.add_post("/user-api/v1/login", LoginView)
+    application.router.add_post("/user-api/v1/logout", LogoutView)

src/ahriman/web/views/base.py

@@ -18,9 +18,10 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from aiohttp.web import View
-from typing import Any, Dict
+from typing import Any, Dict, List, Optional
 
 from ahriman.core.auth.auth import Auth
+from ahriman.core.spawn import Spawn
 from ahriman.core.status.watcher import Watcher
 
 
@@ -37,6 +38,14 @@ class BaseView(View):
         watcher: Watcher = self.request.app["watcher"]
         return watcher
 
+    @property
+    def spawner(self) -> Spawn:
+        """
+        :return: external process spawner instance
+        """
+        spawner: Spawn = self.request.app["spawn"]
+        return spawner
+
     @property
     def validator(self) -> Auth:
         """
@@ -45,13 +54,33 @@ class BaseView(View):
         validator: Auth = self.request.app["validator"]
         return validator
 
-    async def extract_data(self) -> Dict[str, Any]:
+    async def extract_data(self, list_keys: Optional[List[str]] = None) -> Dict[str, Any]:
         """
         extract json data from either json or form data
+        :param list_keys: optional list of keys which must be forced to list from form data
         :return: raw json object or form data converted to json
         """
         try:
             json: Dict[str, Any] = await self.request.json()
             return json
         except ValueError:
-            return dict(await self.request.post())
+            return await self.data_as_json(list_keys or [])
+
+    async def data_as_json(self, list_keys: List[str]) -> Dict[str, Any]:
+        """
+        extract form data and convert it to json object
+        :param list_keys: list of keys which must be forced to list from form data
+        :return: form data converted to json. In case if a key is found multiple times it will be returned as list
+        """
+        raw = await self.request.post()
+        json: Dict[str, Any] = {}
+        for key, value in raw.items():
+            if key in json and isinstance(json[key], list):
+                json[key].append(value)
+            elif key in json:
+                json[key] = [json[key], value]
+            elif key in list_keys:
+                json[key] = [value]
+            else:
+                json[key] = value
+        return json

src/ahriman/web/views/service/__init__.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#

src/ahriman/web/views/service/add.py

@@ -0,0 +1,52 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from aiohttp.web import HTTPFound, Response, json_response
+
+from ahriman.web.views.base import BaseView
+
+
+class AddView(BaseView):
+    """
+    add package web view
+    """
+
+    async def post(self) -> Response:
+        """
+        add new package
+
+        JSON body must be supplied, the following model is used:
+        {
+            "packages": "ahriman",   # either list of packages or package name as in AUR
+            "build_now": true       # optional flag which runs build
+        }
+
+        :return: redirect to main page on success
+        """
+        data = await self.extract_data(["packages"])
+
+        try:
+            now = data.get("build_now", True)
+            packages = data["packages"]
+        except Exception as e:
+            return json_response(text=str(e), status=400)
+
+        self.spawner.packages_add(packages, now)
+
+        return HTTPFound("/")

src/ahriman/web/views/service/remove.py

@@ -0,0 +1,50 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from aiohttp.web import HTTPFound, Response, json_response
+
+from ahriman.web.views.base import BaseView
+
+
+class RemoveView(BaseView):
+    """
+    remove package web view
+    """
+
+    async def post(self) -> Response:
+        """
+        remove existing packages
+
+        JSON body must be supplied, the following model is used:
+        {
+            "packages": "ahriman",   # either list of packages or package name
+        }
+
+        :return: redirect to main page on success
+        """
+        data = await self.extract_data(["packages"])
+
+        try:
+            packages = data["packages"]
+        except Exception as e:
+            return json_response(text=str(e), status=400)
+
+        self.spawner.packages_remove(packages)
+
+        return HTTPFound("/")

src/ahriman/web/views/service/search.py

@@ -0,0 +1,48 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import aur  # type: ignore
+
+from aiohttp.web import Response, json_response
+from typing import Iterator
+
+from ahriman.web.views.base import BaseView
+
+
+class SearchView(BaseView):
+    """
+    AUR search web view
+    """
+
+    async def get(self) -> Response:
+        """
+        search packages in AUR
+
+        search string (non empty) must be supplied as `for` parameter
+
+        :return: 200 with found package bases sorted by name
+        """
+        search: Iterator[str] = filter(lambda s: len(s) > 3, self.request.query.getall("for", default=[]))
+        search_string = " ".join(search)
+
+        if not search_string:
+            return json_response(text="Search string must not be empty", status=400)
+        packages = aur.search(search_string)
+
+        return json_response(sorted(package.package_base for package in packages))

src/ahriman/web/views/status/__init__.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#

src/ahriman/web/views/status/ahriman.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from aiohttp.web import HTTPBadRequest, HTTPNoContent, Response, json_response
+from aiohttp.web import HTTPNoContent, Response, json_response
 
 from ahriman.models.build_status import BuildStatusEnum
 from ahriman.web.views.base import BaseView
@@ -51,7 +51,7 @@ class AhrimanView(BaseView):
         try:
             status = BuildStatusEnum(data["status"])
         except Exception as e:
-            raise HTTPBadRequest(text=str(e))
+            return json_response(text=str(e), status=400)
 
         self.service.update_self(status)
 

src/ahriman/web/views/status/package.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from aiohttp.web import HTTPBadRequest, HTTPNoContent, HTTPNotFound, Response, json_response
+from aiohttp.web import HTTPNoContent, HTTPNotFound, Response, json_response
 
 from ahriman.core.exceptions import UnknownPackage
 from ahriman.models.build_status import BuildStatusEnum
@@ -80,11 +80,11 @@ class PackageView(BaseView):
             package = Package.from_json(data["package"]) if "package" in data else None
             status = BuildStatusEnum(data["status"])
         except Exception as e:
-            raise HTTPBadRequest(text=str(e))
+            return json_response(text=str(e), status=400)
 
         try:
             self.service.update(base, status, package)
         except UnknownPackage:
-            raise HTTPBadRequest(text=f"Package {base} is unknown, but no package body set")
+            return json_response(text=f"Package {base} is unknown, but no package body set", status=400)
 
         return HTTPNoContent()

src/ahriman/web/views/user/__init__.py

@@ -0,0 +1,19 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#

src/ahriman/web/web.py

@@ -26,6 +26,7 @@ from aiohttp import web
 from ahriman.core.auth.auth import Auth
 from ahriman.core.configuration import Configuration
 from ahriman.core.exceptions import InitializeException
+from ahriman.core.spawn import Spawn
 from ahriman.core.status.watcher import Watcher
 from ahriman.web.middlewares.exception_handler import exception_handler
 from ahriman.web.routes import setup_routes
@@ -67,11 +68,12 @@ def run_server(application: web.Application) -> None:
                 access_log=logging.getLogger("http"))
 
 
-def setup_service(architecture: str, configuration: Configuration) -> web.Application:
+def setup_service(architecture: str, configuration: Configuration, spawner: Spawn) -> web.Application:
     """
     create web application
     :param architecture: repository architecture
     :param configuration: configuration instance
+    :param spawner: spawner thread
     :return: web application instance
     """
     application = web.Application(logger=logging.getLogger("http"))
@@ -93,6 +95,9 @@ def setup_service(architecture: str, configuration: Configuration) -> web.Applic
     application.logger.info("setup watcher")
     application["watcher"] = Watcher(architecture, configuration)
 
+    application.logger.info("setup process spawner")
+    application["spawn"] = spawner
+
     application.logger.info("setup authorization")
     validator = application["validator"] = Auth.load(configuration)
     if validator.enabled: