diff --git a/docs/faq.rst b/docs/faq.rst
index 6ec4c8c1..764da421 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -773,29 +773,7 @@ In this example the following settings are assumed:
Master node configuration
"""""""""""""""""""""""""
-The only requirements for the master node is that API must be available for worker nodes to call (e.g. port must be exposed to internet, or local network in case of VPN, etc).
-
-#.
- Create ssh key for ``ahriman`` user, e.g.:
-
- .. code-block:: shell
-
- sudo -u ahriman ssh-keygen
-
-#.
- Copy private key as it will be used later for workers configuration.
-
-#.
- Allow login via ssh with the generated key, e.g.:
-
- .. code-block:: shell
-
- sudo -u ahriman ln -s id_rsa.pub /var/lib/ahriman/.ssh/authorized_keys
-
-#.
- Make sure that ssh server is enabled and run. Optionally check possibility to login as ``ahriman`` user (note, however, that system user is not allowed to login into shell).
-
-In addition, the following settings are recommended:
+The only requirements for the master node is that API must be available for worker nodes to call (e.g. port must be exposed to internet, or local network in case of VPN, etc). In addition, the following settings are recommended:
*
As it has been mentioned above, it is recommended to enable authentication (see `How to enable basic authorization`_) and create system user which will be used later. Later this user (if any) will be referenced as ``worker-user``.
@@ -817,25 +795,18 @@ Worker nodes configuration
* Worker #1: ``A``.
* Worker #2: ``B`` and ``C``.
-#.
- Copy private key generated before to ``/var/lib/ahriman/.ssh/id_rsa``. Make sure that file owner is ``ahriman`` and file permissions are ``600``.
-
#.
Each worker must be configured to upload files to master node:
.. code-block:: ini
[upload]
- target = rsync
+ target = remote-service
- [rsync]
- remote = master.example.com:/var/lib/ahriman/packages/x86_64
- command = rsync --archive --compress --partial
-
- ``/var/lib/ahriman/packages/x86_64`` must refer to built packages directory inside repository tree (change to ``/var/lib/ahriman/ahriman/packages/x86_64`` in case if master node is run inside container). Unlike default settings ``rsync.command`` option must not include ``--delete`` flag.
+ [remote-service]
#.
- Worker must be configured to report package logs and build status to master node:
+ Worker must be configured to access web on master node:
.. code-block:: ini
@@ -867,6 +838,55 @@ Worker nodes configuration
[build]
triggers = ahriman.core.gitremote.RemotePullTrigger ahriman.core.upload.UploadTrigger ahriman.core.report.ReportTrigger ahriman.core.gitremote.RemotePushTrigger
+Double node docker example
+""""""""""""""""""""""""""
+
+Master node config (``master.ini``) as:
+
+.. code-block:: ini
+
+ [auth]
+ target = mapping
+
+Command to run master node:
+
+.. code-block:: shell
+
+ docker run --privileged -p 8080:8080 -e AHRIMAN_PORT=8080 -v master.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web
+
+Worker node config (``worker.ini``) as:
+
+.. code-block:: ini
+
+ [web]
+ address = http://172.17.0.1:8080
+ username = test
+ password = test
+
+ [upload]
+ target = remote-service
+
+ [remote-service]
+
+ [report]
+ target = remote-call
+
+ [remote-call]
+ manual = yes
+
+ [build]
+ triggers = ahriman.core.gitremote.RemotePullTrigger ahriman.core.upload.UploadTrigger ahriman.core.report.ReportTrigger ahriman.core.gitremote.RemotePushTrigger
+
+The address above (``http://172.17.0.1:8080``) is something available for worker container.
+
+Command to run worker node:
+
+.. code-block:: shell
+
+ docker run --privileged -v worker.ini:/etc/ahriman.ini.d/overrides.ini -it arcan1s/ahriman:latest package-add arhiman --now
+
+The command above will successfully build ``ahriman`` package, upload it on master node and, finally, will update master node repository.
+
Addition of new package and repository update
"""""""""""""""""""""""""""""""""""""""""""""
diff --git a/src/ahriman/core/report/remote_call.py b/src/ahriman/core/report/remote_call.py
index 222b365b..160b1aaf 100644
--- a/src/ahriman/core/report/remote_call.py
+++ b/src/ahriman/core/report/remote_call.py
@@ -77,7 +77,7 @@ class RemoteCall(Report):
Returns:
bool: True in case if remote process is alive and False otherwise
"""
- response = self.client.make_request("GET", f"{self.client.address}/api/v1/service/process/{process_id}")
+ response = self.client.make_request("GET", f"/api/v1/service/process/{process_id}")
if response is None:
return False
@@ -93,15 +93,11 @@ class RemoteCall(Report):
Returns:
str | None: remote process id on success and ``None`` otherwise
"""
- response = self.client.make_request(
- "POST",
- f"{self.client.address}/api/v1/service/update",
- json={
- "aur": self.update_aur,
- "local": self.update_local,
- "manual": self.update_manual,
- }
- )
+ response = self.client.make_request("POST", "/api/v1/service/update", json={
+ "aur": self.update_aur,
+ "local": self.update_local,
+ "manual": self.update_manual,
+ })
if response is None:
return None # request terminated with error
diff --git a/src/ahriman/core/status/web_client.py b/src/ahriman/core/status/web_client.py
index a69fa047..882fa860 100644
--- a/src/ahriman/core/status/web_client.py
+++ b/src/ahriman/core/status/web_client.py
@@ -22,7 +22,7 @@ import logging
import requests
from collections.abc import Generator
-from typing import Any, Literal
+from typing import Any, IO, Literal
from urllib.parse import quote_plus as urlencode
from ahriman import __version__
@@ -46,6 +46,9 @@ class WebClient(Client, LazyLogging):
user(User | None): web service user descriptor
"""
+ _login_url = "/api/v1/login"
+ _status_url = "/api/v1/status"
+
def __init__(self, configuration: Configuration) -> None:
"""
default constructor
@@ -61,25 +64,33 @@ class WebClient(Client, LazyLogging):
self.__session = self._create_session(use_unix_socket=use_unix_socket)
- @property
- def _login_url(self) -> str:
+ @staticmethod
+ def _logs_url(package_base: str) -> str:
"""
- get url for the login api
+ get url for the logs api
+
+ Args:
+ package_base(str): package base
Returns:
- str: full url for web service to log in
+ str: full url for web service for logs
"""
- return f"{self.address}/api/v1/login"
+ return f"/api/v1/packages/{package_base}/logs"
- @property
- def _status_url(self) -> str:
+ @staticmethod
+ def _package_url(package_base: str = "") -> str:
"""
- get url for the status api
+ url generator
+
+ Args:
+ package_base(str, optional): package base to generate url (Default value = "")
Returns:
- str: full url for web service for status
+ str: full url of web service for specific package base
"""
- return f"{self.address}/api/v1/status"
+ # in case if unix socket is used we need to normalize url
+ suffix = f"/{package_base}" if package_base else ""
+ return f"/api/v1/packages{suffix}"
@staticmethod
def parse_address(configuration: Configuration) -> tuple[str, bool]:
@@ -167,34 +178,9 @@ class WebClient(Client, LazyLogging):
}
self.make_request("POST", self._login_url, json=payload, session=session)
- def _logs_url(self, package_base: str) -> str:
- """
- get url for the logs api
-
- Args:
- package_base(str): package base
-
- Returns:
- str: full url for web service for logs
- """
- return f"{self.address}/api/v1/packages/{package_base}/logs"
-
- def _package_url(self, package_base: str = "") -> str:
- """
- url generator
-
- Args:
- package_base(str, optional): package base to generate url (Default value = "")
-
- Returns:
- str: full url of web service for specific package base
- """
- # in case if unix socket is used we need to normalize url
- suffix = f"/{package_base}" if package_base else ""
- return f"{self.address}/api/v1/packages{suffix}"
-
def make_request(self, method: Literal["DELETE", "GET", "POST"], url: str,
params: list[tuple[str, str]] | None = None, json: dict[str, Any] | None = None,
+ files: dict[str, tuple[str, IO[bytes], str, dict[str, str]]] | None = None,
session: requests.Session | None = None) -> requests.Response | None:
"""
perform request with specified parameters
@@ -204,13 +190,15 @@ class WebClient(Client, LazyLogging):
url(str): remote url to call
params(list[tuple[str, str]] | None, optional): request query parameters (Default value = None)
json(dict[str, Any] | None, optional): request json parameters (Default value = None)
+ files(dict[str, tuple[str, IO, str, dict[str, str]]] | None, optional): multipart upload
+ (Default value = None)
session(requests.Session | None, optional): session object if any (Default value = None)
Returns:
requests.Response | None: response object or None in case of errors
"""
with self.__get_session(session) as _session:
- response = _session.request(method, url, params=params, json=json)
+ response = _session.request(method, f"{self.address}{url}", params=params, json=json, files=files)
response.raise_for_status()
return response
diff --git a/src/ahriman/core/upload/remote_service.py b/src/ahriman/core/upload/remote_service.py
new file mode 100644
index 00000000..4fb66c2d
--- /dev/null
+++ b/src/ahriman/core/upload/remote_service.py
@@ -0,0 +1,80 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+from pathlib import Path
+from typing import IO
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.status.web_client import WebClient
+from ahriman.core.upload.upload import Upload
+from ahriman.models.package import Package
+
+
+class RemoteService(Upload):
+ """
+ upload files to another server instance
+
+ Attributes:
+ client(WebClient): web client instance
+ """
+
+ def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
+ """
+ default constructor
+
+ Args:
+ architecture(str): repository architecture
+ configuration(Configuration): configuration instance
+ section(str): settings section name
+ """
+ Upload.__init__(self, architecture, configuration)
+ del section
+
+ self.client = WebClient(configuration)
+
+ def package_upload(self, path: Path, package: Package) -> None:
+ """
+ upload single package to remote
+
+ Args:
+ path(Path): local path to sync
+ package(Package): package to upload
+ """
+ for key, descriptor in package.packages.items():
+ if descriptor.filename is None:
+ self.logger.warning("package %s of %s doesn't have filename set", key, package.base)
+ continue
+
+ with (path / descriptor.filename).open("rb") as archive:
+ # filename, file, content-type, headers
+ part: tuple[str, IO[bytes], str, dict[str, str]] = (
+ descriptor.filename, archive, "application/octet-stream", {}
+ )
+ self.client.make_request("POST", "/api/v1/service/upload", files={"archive": part})
+
+ def sync(self, path: Path, built_packages: list[Package]) -> None:
+ """
+ sync data to remote server
+
+ Args:
+ path(Path): local path to sync
+ built_packages(list[Package]): list of packages which has just been built
+ """
+ for package in built_packages:
+ self.package_upload(path, package)
diff --git a/src/ahriman/core/upload/upload.py b/src/ahriman/core/upload/upload.py
index 3b8f9fb1..b2ac61bb 100644
--- a/src/ahriman/core/upload/upload.py
+++ b/src/ahriman/core/upload/upload.py
@@ -90,6 +90,9 @@ class Upload(LazyLogging):
if provider == UploadSettings.Github:
from ahriman.core.upload.github import Github
return Github(architecture, configuration, section)
+ if provider == UploadSettings.RemoteService:
+ from ahriman.core.upload.remote_service import RemoteService
+ return RemoteService(architecture, configuration, section)
return Upload(architecture, configuration) # should never happen
def run(self, path: Path, built_packages: list[Package]) -> None:
diff --git a/src/ahriman/core/upload/upload_trigger.py b/src/ahriman/core/upload/upload_trigger.py
index a3af9b6e..e12c145e 100644
--- a/src/ahriman/core/upload/upload_trigger.py
+++ b/src/ahriman/core/upload/upload_trigger.py
@@ -92,6 +92,11 @@ class UploadTrigger(Trigger):
},
},
},
+ "remote-service": {
+ "type": "dict",
+ "schema": {
+ },
+ },
"s3": {
"type": "dict",
"schema": {
diff --git a/src/ahriman/models/report_settings.py b/src/ahriman/models/report_settings.py
index 5b37473b..7b086636 100644
--- a/src/ahriman/models/report_settings.py
+++ b/src/ahriman/models/report_settings.py
@@ -32,7 +32,7 @@ class ReportSettings(str, Enum):
Email(ReportSettings): (class attribute) email report generation
Console(ReportSettings): (class attribute) print result to console
Telegram(ReportSettings): (class attribute) markdown report to telegram channel
- RemoteCall(ReportSettings): (class attribute) remote server call
+ RemoteCall(ReportSettings): (class attribute) remote ahriman server call
"""
Disabled = "disabled" # for testing purpose
@@ -61,6 +61,6 @@ class ReportSettings(str, Enum):
return ReportSettings.Console
if value.lower() in ("telegram",):
return ReportSettings.Telegram
- if value.lower() in ("remote-call",):
+ if value.lower() in ("ahriman", "remote-call",):
return ReportSettings.RemoteCall
return ReportSettings.Disabled
diff --git a/src/ahriman/models/upload_settings.py b/src/ahriman/models/upload_settings.py
index d6f959e6..02f7a738 100644
--- a/src/ahriman/models/upload_settings.py
+++ b/src/ahriman/models/upload_settings.py
@@ -31,12 +31,14 @@ class UploadSettings(str, Enum):
Rsync(UploadSettings): (class attribute) sync via rsync
S3(UploadSettings): (class attribute) sync to Amazon S3
Github(UploadSettings): (class attribute) sync to github releases page
+ RemoteService(UploadSettings): (class attribute) sync to another ahriman instance
"""
Disabled = "disabled" # for testing purpose
Rsync = "rsync"
S3 = "s3"
Github = "github"
+ RemoteService = "remote-service"
@staticmethod
def from_option(value: str) -> UploadSettings:
@@ -55,4 +57,6 @@ class UploadSettings(str, Enum):
return UploadSettings.S3
if value.lower() in ("github",):
return UploadSettings.Github
+ if value.lower() in ("ahriman", "remote-service",):
+ return UploadSettings.RemoteService
return UploadSettings.Disabled
diff --git a/src/ahriman/web/routes.py b/src/ahriman/web/routes.py
index ecc2a428..7a34d330 100644
--- a/src/ahriman/web/routes.py
+++ b/src/ahriman/web/routes.py
@@ -31,6 +31,7 @@ from ahriman.web.views.service.remove import RemoveView
from ahriman.web.views.service.request import RequestView
from ahriman.web.views.service.search import SearchView
from ahriman.web.views.service.update import UpdateView
+from ahriman.web.views.service.upload import UploadView
from ahriman.web.views.status.logs import LogsView
from ahriman.web.views.status.package import PackageView
from ahriman.web.views.status.packages import PackagesView
@@ -66,6 +67,7 @@ def setup_routes(application: Application, static_path: Path) -> None:
application.router.add_view("/api/v1/service/request", RequestView)
application.router.add_view("/api/v1/service/search", SearchView)
application.router.add_view("/api/v1/service/update", UpdateView)
+ application.router.add_view("/api/v1/service/upload", UploadView)
application.router.add_view("/api/v1/packages", PackagesView)
application.router.add_view("/api/v1/packages/{package}", PackageView)
diff --git a/src/ahriman/web/schemas/__init__.py b/src/ahriman/web/schemas/__init__.py
index 08e810c4..5253204b 100644
--- a/src/ahriman/web/schemas/__init__.py
+++ b/src/ahriman/web/schemas/__init__.py
@@ -21,6 +21,7 @@ from ahriman.web.schemas.aur_package_schema import AURPackageSchema
from ahriman.web.schemas.auth_schema import AuthSchema
from ahriman.web.schemas.counters_schema import CountersSchema
from ahriman.web.schemas.error_schema import ErrorSchema
+from ahriman.web.schemas.file_schema import FileSchema
from ahriman.web.schemas.internal_status_schema import InternalStatusSchema
from ahriman.web.schemas.log_schema import LogSchema
from ahriman.web.schemas.login_schema import LoginSchema
diff --git a/src/ahriman/web/schemas/file_schema.py b/src/ahriman/web/schemas/file_schema.py
new file mode 100644
index 00000000..95f3f863
--- /dev/null
+++ b/src/ahriman/web/schemas/file_schema.py
@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+from marshmallow import Schema, fields
+
+
+class FileSchema(Schema):
+ """
+ request file upload schema
+ """
+
+ archive = fields.Field(required=True, metadata={
+ "description": "Package archive to be uploaded",
+ })
diff --git a/src/ahriman/web/views/api/swagger.py b/src/ahriman/web/views/api/swagger.py
index a27ae60a..e3517a07 100644
--- a/src/ahriman/web/views/api/swagger.py
+++ b/src/ahriman/web/views/api/swagger.py
@@ -43,7 +43,7 @@ class SwaggerView(BaseView):
Response: 200 with json api specification
"""
spec = self.request.app["swagger_dict"]
- is_body_parameter: Callable[[dict[str, str]], bool] = lambda p: p["in"] == "body"
+ is_body_parameter: Callable[[dict[str, str]], bool] = lambda p: p["in"] == "body" or p["in"] == "formData"
# special workaround because it writes request body to parameters section
paths = spec["paths"]
@@ -56,11 +56,14 @@ class SwaggerView(BaseView):
if not body:
continue # there were no ``body`` parameters found
+ schema = next(iter(body))
+ content_type = "multipart/form-data" if schema["in"] == "formData" else "application/json"
+
# there should be only one body parameters
method["requestBody"] = {
"content": {
- "application/json": {
- "schema": next(iter(body))["schema"]
+ content_type: {
+ "schema": schema["schema"]
}
}
}
diff --git a/src/ahriman/web/views/service/upload.py b/src/ahriman/web/views/service/upload.py
new file mode 100644
index 00000000..232862a6
--- /dev/null
+++ b/src/ahriman/web/views/service/upload.py
@@ -0,0 +1,92 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+import aiohttp_apispec # type: ignore[import]
+
+from aiohttp import BodyPartReader
+from aiohttp.web import HTTPBadRequest, HTTPCreated
+from pathlib import Path
+
+from ahriman.models.user_access import UserAccess
+from ahriman.web.schemas import AuthSchema, ErrorSchema, FileSchema
+from ahriman.web.views.base import BaseView
+
+
+class UploadView(BaseView):
+ """
+ upload file to repository
+
+ Attributes:
+ POST_PERMISSION(UserAccess): (class attribute) post permissions of self
+ """
+
+ POST_PERMISSION = UserAccess.Full
+
+ @aiohttp_apispec.docs(
+ tags=["Actions"],
+ summary="Upload package",
+ description="Upload package to local filesystem",
+ responses={
+ 201: {"description": "Success response"},
+ 400: {"description": "Bad data is supplied", "schema": ErrorSchema},
+ 401: {"description": "Authorization required", "schema": ErrorSchema},
+ 403: {"description": "Access is forbidden", "schema": ErrorSchema},
+ 500: {"description": "Internal server error", "schema": ErrorSchema},
+ },
+ security=[{"token": [POST_PERMISSION]}],
+ )
+ @aiohttp_apispec.cookies_schema(AuthSchema)
+ @aiohttp_apispec.form_schema(FileSchema)
+ async def post(self) -> None:
+ """
+ upload file from another instance to the server
+
+ Raises:
+ HTTPBadRequest: if bad data is supplied
+ HTTPCreated: on success response
+ """
+ try:
+ reader = await self.request.multipart()
+ except Exception as e:
+ raise HTTPBadRequest(reason=str(e))
+
+ part = await reader.next()
+ if not isinstance(part, BodyPartReader):
+ raise HTTPBadRequest(reason="Invalid multipart message received")
+
+ if part.name != "archive":
+ raise HTTPBadRequest(reason="Multipart field isn't archive")
+
+ archive_name = part.filename
+ if archive_name is None:
+ raise HTTPBadRequest(reason="Filename must be set") # pragma: no cover
+ # some magic inside. We would like to make sure that passed filename is filename
+ # without slashes, dots, etc
+ if Path(archive_name).resolve().name != archive_name:
+ raise HTTPBadRequest(reason="Filename must be valid archive name")
+
+ output = self.configuration.repository_paths.packages / archive_name
+ with output.open("wb") as archive:
+ while True:
+ chunk = await part.read_chunk()
+ if not chunk:
+ break
+ archive.write(chunk)
+
+ raise HTTPCreated()
diff --git a/tests/ahriman/application/handlers/test_handler_validate.py b/tests/ahriman/application/handlers/test_handler_validate.py
index 62391380..05a95de6 100644
--- a/tests/ahriman/application/handlers/test_handler_validate.py
+++ b/tests/ahriman/application/handlers/test_handler_validate.py
@@ -70,6 +70,7 @@ def test_schema(configuration: Configuration) -> None:
assert schema.pop("remote-call")
assert schema.pop("remote-pull")
assert schema.pop("remote-push")
+ assert schema.pop("remote-service")
assert schema.pop("report")
assert schema.pop("rsync")
assert schema.pop("s3")
diff --git a/tests/ahriman/core/report/test_remote_call.py b/tests/ahriman/core/report/test_remote_call.py
index 666df97c..6f71c9b9 100644
--- a/tests/ahriman/core/report/test_remote_call.py
+++ b/tests/ahriman/core/report/test_remote_call.py
@@ -30,7 +30,7 @@ def test_is_process_alive(remote_call: RemoteCall, mocker: MockerFixture) -> Non
request_mock = mocker.patch("ahriman.core.status.web_client.WebClient.make_request", return_value=response_obj)
assert remote_call.is_process_alive("id")
- request_mock.assert_called_once_with("GET", f"{remote_call.client.address}/api/v1/service/process/id")
+ request_mock.assert_called_once_with("GET", "/api/v1/service/process/id")
def test_is_process_alive_unknown(remote_call: RemoteCall, mocker: MockerFixture) -> None:
@@ -52,7 +52,7 @@ def test_remote_update(remote_call: RemoteCall, mocker: MockerFixture) -> None:
request_mock = mocker.patch("ahriman.core.status.web_client.WebClient.make_request", return_value=response_obj)
assert remote_call.remote_update() == "id"
- request_mock.assert_called_once_with("POST", f"{remote_call.client.address}/api/v1/service/update", json={
+ request_mock.assert_called_once_with("POST", "/api/v1/service/update", json={
"aur": False,
"local": False,
"manual": True,
diff --git a/tests/ahriman/core/status/test_web_client.py b/tests/ahriman/core/status/test_web_client.py
index 7b2b9712..ae47c7a9 100644
--- a/tests/ahriman/core/status/test_web_client.py
+++ b/tests/ahriman/core/status/test_web_client.py
@@ -19,7 +19,6 @@ def test_login_url(web_client: WebClient) -> None:
"""
must generate login url correctly
"""
- assert web_client._login_url.startswith(web_client.address)
assert web_client._login_url.endswith("/api/v1/login")
@@ -27,10 +26,24 @@ def test_status_url(web_client: WebClient) -> None:
"""
must generate package status url correctly
"""
- assert web_client._status_url.startswith(web_client.address)
assert web_client._status_url.endswith("/api/v1/status")
+def test_logs_url(web_client: WebClient, package_ahriman: Package) -> None:
+ """
+ must generate logs url correctly
+ """
+ assert web_client._logs_url(package_ahriman.base).endswith(f"/api/v1/packages/{package_ahriman.base}/logs")
+
+
+def test_package_url(web_client: WebClient, package_ahriman: Package) -> None:
+ """
+ must generate package status url correctly
+ """
+ assert web_client._package_url("").endswith("/api/v1/packages")
+ assert web_client._package_url(package_ahriman.base).endswith(f"/api/v1/packages/{package_ahriman.base}")
+
+
def test_parse_address(configuration: Configuration) -> None:
"""
must extract address correctly
@@ -81,7 +94,8 @@ def test_login(web_client: WebClient, user: User, mocker: MockerFixture) -> None
}
web_client._login(requests.Session())
- requests_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True), params=None, json=payload)
+ requests_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True),
+ params=None, json=payload, files=None)
def test_login_failed(web_client: WebClient, user: User, mocker: MockerFixture) -> None:
@@ -111,49 +125,34 @@ def test_login_skip(web_client: WebClient, mocker: MockerFixture) -> None:
requests_mock.assert_not_called()
-def test_logs_url(web_client: WebClient, package_ahriman: Package) -> None:
- """
- must generate logs url correctly
- """
- assert web_client._logs_url(package_ahriman.base).startswith(web_client.address)
- assert web_client._logs_url(package_ahriman.base).endswith(f"/api/v1/packages/{package_ahriman.base}/logs")
-
-
-def test_package_url(web_client: WebClient, package_ahriman: Package) -> None:
- """
- must generate package status url correctly
- """
- assert web_client._package_url("").startswith(web_client.address)
- assert web_client._package_url("").endswith(f"/api/v1/packages")
-
- assert web_client._package_url(package_ahriman.base).startswith(web_client.address)
- assert web_client._package_url(package_ahriman.base).endswith(f"/api/v1/packages/{package_ahriman.base}")
-
-
def test_make_request(web_client: WebClient, mocker: MockerFixture) -> None:
"""
must make HTTP request
"""
request_mock = mocker.patch("requests.Session.request")
- assert web_client.make_request("GET", "url") is not None
- assert web_client.make_request("GET", "url", params=[("param", "value")]) is not None
+ assert web_client.make_request("GET", "/url1") is not None
+ assert web_client.make_request("GET", "/url2", params=[("param", "value")]) is not None
- assert web_client.make_request("POST", "url") is not None
- assert web_client.make_request("POST", "url", json={"param": "value"}) is not None
+ assert web_client.make_request("POST", "/url3") is not None
+ assert web_client.make_request("POST", "/url4", json={"param": "value"}) is not None
+ # we don't want to put full descriptor here
+ assert web_client.make_request("POST", "/url5", files={"file": "tuple"}) is not None
- assert web_client.make_request("DELETE", "url") is not None
+ assert web_client.make_request("DELETE", "/url6") is not None
request_mock.assert_has_calls([
- MockCall("GET", "url", params=None, json=None),
+ MockCall("GET", f"{web_client.address}/url1", params=None, json=None, files=None),
MockCall().raise_for_status(),
- MockCall("GET", "url", params=[("param", "value")], json=None),
+ MockCall("GET", f"{web_client.address}/url2", params=[("param", "value")], json=None, files=None),
MockCall().raise_for_status(),
- MockCall("POST", "url", params=None, json=None),
+ MockCall("POST", f"{web_client.address}/url3", params=None, json=None, files=None),
MockCall().raise_for_status(),
- MockCall("POST", "url", params=None, json={"param": "value"}),
+ MockCall("POST", f"{web_client.address}/url4", params=None, json={"param": "value"}, files=None),
MockCall().raise_for_status(),
- MockCall("DELETE", "url", params=None, json=None),
+ MockCall("POST", f"{web_client.address}/url5", params=None, json=None, files={"file": "tuple"}),
+ MockCall().raise_for_status(),
+ MockCall("DELETE", f"{web_client.address}/url6", params=None, json=None, files=None),
MockCall().raise_for_status(),
])
@@ -174,7 +173,8 @@ def test_package_add(web_client: WebClient, package_ahriman: Package, mocker: Mo
payload = pytest.helpers.get_package_status(package_ahriman)
web_client.package_add(package_ahriman, BuildStatusEnum.Unknown)
- requests_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True), params=None, json=payload)
+ requests_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True),
+ params=None, json=payload, files=None)
def test_package_add_failed(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None:
@@ -230,7 +230,8 @@ def test_package_get_all(web_client: WebClient, package_ahriman: Package, mocker
requests_mock = mocker.patch("requests.Session.request", return_value=response_obj)
result = web_client.package_get(None)
- requests_mock.assert_called_once_with("GET", web_client._package_url(), params=None, json=None)
+ requests_mock.assert_called_once_with("GET", f"{web_client.address}{web_client._package_url()}",
+ params=None, json=None, files=None)
assert len(result) == len(response)
assert (package_ahriman, BuildStatusEnum.Unknown) in [(package, status.status) for package, status in result]
@@ -263,8 +264,9 @@ def test_package_get_single(web_client: WebClient, package_ahriman: Package, moc
requests_mock = mocker.patch("requests.Session.request", return_value=response_obj)
result = web_client.package_get(package_ahriman.base)
- requests_mock.assert_called_once_with("GET", web_client._package_url(package_ahriman.base),
- params=None, json=None)
+ requests_mock.assert_called_once_with("GET",
+ f"{web_client.address}{web_client._package_url(package_ahriman.base)}",
+ params=None, json=None, files=None)
assert len(result) == len(response)
assert (package_ahriman, BuildStatusEnum.Unknown) in [(package, status.status) for package, status in result]
@@ -282,7 +284,8 @@ def test_package_logs(web_client: WebClient, log_record: logging.LogRecord, pack
}
web_client.package_logs(package_ahriman.base, log_record)
- requests_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True), params=None, json=payload)
+ requests_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True),
+ params=None, json=payload, files=None)
def test_package_logs_failed(web_client: WebClient, log_record: logging.LogRecord, package_ahriman: Package,
@@ -312,7 +315,8 @@ def test_package_remove(web_client: WebClient, package_ahriman: Package, mocker:
requests_mock = mocker.patch("requests.Session.request")
web_client.package_remove(package_ahriman.base)
- requests_mock.assert_called_once_with("DELETE", pytest.helpers.anyvar(str, True), params=None, json=None)
+ requests_mock.assert_called_once_with("DELETE", pytest.helpers.anyvar(str, True),
+ params=None, json=None, files=None)
def test_package_remove_failed(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None:
@@ -341,7 +345,7 @@ def test_package_update(web_client: WebClient, package_ahriman: Package, mocker:
web_client.package_update(package_ahriman.base, BuildStatusEnum.Unknown)
requests_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True), params=None, json={
"status": BuildStatusEnum.Unknown.value
- })
+ }, files=None)
def test_package_update_failed(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None:
@@ -373,7 +377,8 @@ def test_status_get(web_client: WebClient, mocker: MockerFixture) -> None:
requests_mock = mocker.patch("requests.Session.request", return_value=response_obj)
result = web_client.status_get()
- requests_mock.assert_called_once_with("GET", web_client._status_url, params=None, json=None)
+ requests_mock.assert_called_once_with("GET", f"{web_client.address}{web_client._status_url}",
+ params=None, json=None, files=None)
assert result.architecture == "x86_64"
@@ -402,7 +407,7 @@ def test_status_update(web_client: WebClient, mocker: MockerFixture) -> None:
web_client.status_update(BuildStatusEnum.Unknown)
requests_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True), params=None, json={
"status": BuildStatusEnum.Unknown.value
- })
+ }, files=None)
def test_status_update_self_failed(web_client: WebClient, mocker: MockerFixture) -> None:
diff --git a/tests/ahriman/core/upload/conftest.py b/tests/ahriman/core/upload/conftest.py
index 00b4abb2..f3439b07 100644
--- a/tests/ahriman/core/upload/conftest.py
+++ b/tests/ahriman/core/upload/conftest.py
@@ -5,6 +5,7 @@ from unittest.mock import MagicMock
from ahriman.core.configuration import Configuration
from ahriman.core.upload.github import Github
+from ahriman.core.upload.remote_service import RemoteService
from ahriman.core.upload.rsync import Rsync
from ahriman.core.upload.s3 import S3
@@ -45,6 +46,22 @@ def github_release() -> dict[str, Any]:
}
+@pytest.fixture
+def remote_service(configuration: Configuration) -> RemoteService:
+ """
+ fixture for remote service synchronization
+
+ Args:
+ configuration(Configuration): configuration fixture
+
+ Returns:
+ RemoteService: remote service test instance
+ """
+ configuration.set_option("web", "host", "localhost")
+ configuration.set_option("web", "port", "8080")
+ return RemoteService("x86_64", configuration, "remote-service")
+
+
@pytest.fixture
def rsync(configuration: Configuration) -> Rsync:
"""
diff --git a/tests/ahriman/core/upload/test_remote_service.py b/tests/ahriman/core/upload/test_remote_service.py
new file mode 100644
index 00000000..0b4879bd
--- /dev/null
+++ b/tests/ahriman/core/upload/test_remote_service.py
@@ -0,0 +1,49 @@
+import pytest
+
+from pathlib import Path
+from pytest_mock import MockerFixture
+
+from ahriman.core.upload.remote_service import RemoteService
+from ahriman.models.package import Package
+
+
+def test_package_upload(remote_service: RemoteService, package_ahriman: Package, mocker: MockerFixture) -> None:
+ """
+ must upload package to remote host
+ """
+ open_mock = mocker.patch("pathlib.Path.open")
+ upload_mock = mocker.patch("ahriman.core.status.web_client.WebClient.make_request")
+ filename = package_ahriman.packages[package_ahriman.base].filename
+
+ remote_service.sync(Path("local"), [package_ahriman])
+ open_mock.assert_called_once_with("rb")
+ upload_mock.assert_called_once_with("POST", "/api/v1/service/upload", files={
+ "archive": (filename, pytest.helpers.anyvar(int), "application/octet-stream", {})
+ })
+
+
+def test_package_upload_no_filename(
+ remote_service: RemoteService,
+ package_ahriman: Package,
+ mocker: MockerFixture) -> None:
+ """
+ must skip upload if no filename set
+ """
+ open_mock = mocker.patch("pathlib.Path.open")
+ upload_mock = mocker.patch("ahriman.core.status.web_client.WebClient.make_request")
+ package_ahriman.packages[package_ahriman.base].filename = None
+
+ remote_service.sync(Path("local"), [package_ahriman])
+ open_mock.assert_not_called()
+ upload_mock.assert_not_called()
+
+
+def test_sync(remote_service: RemoteService, package_ahriman: Package, mocker: MockerFixture) -> None:
+ """
+ must run sync command
+ """
+ upload_mock = mocker.patch("ahriman.core.upload.remote_service.RemoteService.package_upload")
+ local = Path("local")
+
+ remote_service.sync(local, [package_ahriman])
+ upload_mock.assert_called_once_with(local, package_ahriman)
diff --git a/tests/ahriman/core/upload/test_upload.py b/tests/ahriman/core/upload/test_upload.py
index 03dc6996..3f996a52 100644
--- a/tests/ahriman/core/upload/test_upload.py
+++ b/tests/ahriman/core/upload/test_upload.py
@@ -53,3 +53,15 @@ def test_upload_github(configuration: Configuration, mocker: MockerFixture) -> N
upload_mock = mocker.patch("ahriman.core.upload.github.Github.sync")
Upload.load("x86_64", configuration, "github").run(Path("path"), [])
upload_mock.assert_called_once_with(Path("path"), [])
+
+
+def test_upload_ahriman(configuration: Configuration, mocker: MockerFixture) -> None:
+ """
+ must upload via ahriman
+ """
+ upload_mock = mocker.patch("ahriman.core.upload.remote_service.RemoteService.sync")
+ configuration.set_option("web", "host", "localhost")
+ configuration.set_option("web", "port", "8080")
+
+ Upload.load("x86_64", configuration, "remote-service").run(Path("path"), [])
+ upload_mock.assert_called_once_with(Path("path"), [])
diff --git a/tests/ahriman/models/test_report_settings.py b/tests/ahriman/models/test_report_settings.py
index 527eb1ee..8974cc56 100644
--- a/tests/ahriman/models/test_report_settings.py
+++ b/tests/ahriman/models/test_report_settings.py
@@ -26,3 +26,5 @@ def test_from_option_valid() -> None:
assert ReportSettings.from_option("remote-call") == ReportSettings.RemoteCall
assert ReportSettings.from_option("reMOte-cALL") == ReportSettings.RemoteCall
+ assert ReportSettings.from_option("ahriman") == ReportSettings.RemoteCall
+ assert ReportSettings.from_option("AhRiMAN") == ReportSettings.RemoteCall
diff --git a/tests/ahriman/models/test_upload_settings.py b/tests/ahriman/models/test_upload_settings.py
index d7748edc..02e82fed 100644
--- a/tests/ahriman/models/test_upload_settings.py
+++ b/tests/ahriman/models/test_upload_settings.py
@@ -20,3 +20,8 @@ def test_from_option_valid() -> None:
assert UploadSettings.from_option("github") == UploadSettings.Github
assert UploadSettings.from_option("GitHub") == UploadSettings.Github
+
+ assert UploadSettings.from_option("remote-service") == UploadSettings.RemoteService
+ assert UploadSettings.from_option("Remote-Service") == UploadSettings.RemoteService
+ assert UploadSettings.from_option("ahriman") == UploadSettings.RemoteService
+ assert UploadSettings.from_option("AhRiMAN") == UploadSettings.RemoteService
diff --git a/tests/ahriman/web/schemas/test_file_schema.py b/tests/ahriman/web/schemas/test_file_schema.py
new file mode 100644
index 00000000..1982fb6b
--- /dev/null
+++ b/tests/ahriman/web/schemas/test_file_schema.py
@@ -0,0 +1 @@
+# schema testing goes in view class tests
diff --git a/tests/ahriman/web/views/service/test_views_service_process.py b/tests/ahriman/web/views/service/test_views_service_process.py
index f3ff549f..e9adf12d 100644
--- a/tests/ahriman/web/views/service/test_views_service_process.py
+++ b/tests/ahriman/web/views/service/test_views_service_process.py
@@ -33,7 +33,7 @@ async def test_get(client: TestClient, mocker: MockerFixture) -> None:
assert not response_schema.validate(json)
-async def test_post_empty(client: TestClient, mocker: MockerFixture) -> None:
+async def test_get_empty(client: TestClient, mocker: MockerFixture) -> None:
"""
must call raise 404 on unknown process
"""
diff --git a/tests/ahriman/web/views/service/test_views_service_update.py b/tests/ahriman/web/views/service/test_views_service_update.py
index 1b91bcf9..d60a9cd7 100644
--- a/tests/ahriman/web/views/service/test_views_service_update.py
+++ b/tests/ahriman/web/views/service/test_views_service_update.py
@@ -17,7 +17,7 @@ async def test_get_permission() -> None:
assert await UpdateView.get_permission(request) == UserAccess.Full
-async def test_post_update(client: TestClient, mocker: MockerFixture) -> None:
+async def test_post(client: TestClient, mocker: MockerFixture) -> None:
"""
must call post request correctly for alias
"""
diff --git a/tests/ahriman/web/views/service/test_views_service_upload.py b/tests/ahriman/web/views/service/test_views_service_upload.py
new file mode 100644
index 00000000..ed2fe8c9
--- /dev/null
+++ b/tests/ahriman/web/views/service/test_views_service_upload.py
@@ -0,0 +1,91 @@
+import pytest
+
+from aiohttp import FormData
+from aiohttp.test_utils import TestClient
+from io import BytesIO
+from pytest_mock import MockerFixture
+
+from ahriman.models.user_access import UserAccess
+from ahriman.web.views.service.upload import UploadView
+
+
+async def test_get_permission() -> None:
+ """
+ must return correct permission for the request
+ """
+ for method in ("POST",):
+ request = pytest.helpers.request("", "", method)
+ assert await UploadView.get_permission(request) == UserAccess.Full
+
+
+async def test_post(client: TestClient, mocker: MockerFixture) -> None:
+ """
+ must call post request correctly for alias
+ """
+ open_mock = mocker.patch("pathlib.Path.open")
+ # no content validation here because it has invalid schema
+
+ data = FormData()
+ data.add_field("archive", BytesIO(b"content"), filename="filename", content_type="application/octet-stream")
+
+ response = await client.post("/api/v1/service/upload", data=data)
+ assert response.ok
+ open_mock.assert_called_once_with("wb")
+
+
+async def test_post_not_multipart(client: TestClient) -> None:
+ """
+ must return 400 on invalid payload
+ """
+ response = await client.post("/api/v1/service/upload")
+ assert response.status == 400
+
+
+async def test_post_not_bodypart(client: TestClient, mocker: MockerFixture) -> None:
+ """
+ must return 400 on invalid iterator in multipart
+ """
+ mocker.patch("aiohttp.MultipartReader.next", return_value=42) # surprise, motherfucker
+ data = FormData()
+ data.add_field("archive", BytesIO(b"content"), filename="filename", content_type="application/octet-stream")
+
+ response = await client.post("/api/v1/service/upload", data=data)
+ assert response.status == 400
+
+
+async def test_post_not_archive(client: TestClient) -> None:
+ """
+ must return 400 on invalid multipart key
+ """
+ data = FormData()
+ data.add_field("random", BytesIO(b"content"), filename="filename", content_type="application/octet-stream")
+
+ response = await client.post("/api/v1/service/upload", data=data)
+ assert response.status == 400
+
+
+async def test_post_no_filename(client: TestClient, mocker: MockerFixture) -> None:
+ """
+ must return 400 if filename is not set
+ """
+ mocker.patch("aiohttp.BodyPartReader.filename", return_value=None)
+ data = FormData()
+ data.add_field("random", BytesIO(b"content"), filename="filename", content_type="application/octet-stream")
+
+ response = await client.post("/api/v1/service/upload", data=data)
+ assert response.status == 400
+
+
+async def test_post_filename_invalid(client: TestClient) -> None:
+ """
+ must return 400 if filename is invalid
+ """
+ data = FormData()
+ data.add_field("archive", BytesIO(b"content"), filename="..", content_type="application/octet-stream")
+ response = await client.post("/api/v1/service/upload", data=data)
+ assert response.status == 400
+
+ data = FormData()
+ data.add_field("archive", BytesIO(b"content"), filename="", content_type="application/octet-stream")
+ response = await client.post("/api/v1/service/upload", data=data)
+ assert response.status == 400
diff --git a/tests/testresources/core/ahriman.ini b/tests/testresources/core/ahriman.ini
index cac44f1c..74d87690 100644
--- a/tests/testresources/core/ahriman.ini
+++ b/tests/testresources/core/ahriman.ini
@@ -106,6 +106,8 @@ password =
repository = ahriman
username = arcan1s
+[remote-service]
+
[web]
debug = no
debug_check_host = no