Remote call trigger support (#105)

* add support of remote task tracking
* add remote call trigger implementation
* docs update
* add cross-service upload
* add notes about user
* add more ability to control upload
* multipart upload with signatures as well as safe file save
* configuration reference update
* rename watcher methods
* erase logs based on current package version

Old implementation has used process id instead, but it leads to log
removal in case of remote process trigger

* add --server flag for setup command
* restore behavior of the httploghandler

tests/ahriman/web/schemas/test_file_schema.py

@@ -0,0 +1 @@
+# schema testing goes in view class tests

tests/ahriman/web/schemas/test_process_id_schema.py

@@ -0,0 +1 @@
+# schema testing goes in view class tests

tests/ahriman/web/schemas/test_process_schema.py

@@ -0,0 +1 @@
+# schema testing goes in view class tests

tests/ahriman/web/schemas/test_update_flags_schema.py

@@ -0,0 +1 @@
+# schema testing goes in view class tests

tests/ahriman/web/views/service/test_views_service_add.py

@@ -21,11 +21,12 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
     """
     must call post request correctly
     """
-    add_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_add")
+    add_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_add", return_value="abc")
     user_mock = AsyncMock()
     user_mock.return_value = "username"
     mocker.patch("ahriman.web.views.base.BaseView.username", side_effect=user_mock)
     request_schema = pytest.helpers.schema_request(AddView.post)
+    response_schema = pytest.helpers.schema_response(AddView.post)
 
     payload = {"packages": ["ahriman"]}
     assert not request_schema.validate(payload)
@@ -33,6 +34,10 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
     assert response.ok
     add_mock.assert_called_once_with(["ahriman"], "username", now=True)
 
+    json = await response.json()
+    assert json["process_id"] == "abc"
+    assert not response_schema.validate(json)
+
 
 async def test_post_empty(client: TestClient, mocker: MockerFixture) -> None:
     """

tests/ahriman/web/views/service/test_views_service_pgp.py

@@ -66,8 +66,9 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
     """
     must call post request correctly
     """
-    import_mock = mocker.patch("ahriman.core.spawn.Spawn.key_import")
+    import_mock = mocker.patch("ahriman.core.spawn.Spawn.key_import", return_value="abc")
     request_schema = pytest.helpers.schema_request(PGPView.post)
+    response_schema = pytest.helpers.schema_response(PGPView.post)
 
     payload = {"key": "0xdeadbeaf", "server": "keyserver.ubuntu.com"}
     assert not request_schema.validate(payload)
@@ -75,6 +76,10 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
     assert response.ok
     import_mock.assert_called_once_with("0xdeadbeaf", "keyserver.ubuntu.com")
 
+    json = await response.json()
+    assert json["process_id"] == "abc"
+    assert not response_schema.validate(json)
+
 
 async def test_post_exception(client: TestClient, mocker: MockerFixture) -> None:
     """

tests/ahriman/web/views/service/test_views_service_process.py

@@ -0,0 +1,46 @@
+import pytest
+
+from aiohttp.test_utils import TestClient
+from pytest_mock import MockerFixture
+
+from ahriman.models.user_access import UserAccess
+from ahriman.web.views.service.process import ProcessView
+
+
+async def test_get_permission() -> None:
+    """
+    must return correct permission for the request
+    """
+    for method in ("GET",):
+        request = pytest.helpers.request("", "", method)
+        assert await ProcessView.get_permission(request) == UserAccess.Reporter
+
+
+async def test_get(client: TestClient, mocker: MockerFixture) -> None:
+    """
+    must call post request correctly
+    """
+    process = "abc"
+    process_mock = mocker.patch("ahriman.core.spawn.Spawn.has_process", return_value=True)
+    response_schema = pytest.helpers.schema_response(ProcessView.get)
+
+    response = await client.get(f"/api/v1/service/process/{process}")
+    assert response.ok
+    process_mock.assert_called_once_with(process)
+
+    json = await response.json()
+    assert json["is_alive"]
+    assert not response_schema.validate(json)
+
+
+async def test_get_empty(client: TestClient, mocker: MockerFixture) -> None:
+    """
+    must call raise 404 on unknown process
+    """
+    process = "abc"
+    mocker.patch("ahriman.core.spawn.Spawn.has_process", return_value=False)
+    response_schema = pytest.helpers.schema_response(ProcessView.get, code=404)
+
+    response = await client.get(f"/api/v1/service/process/{process}")
+    assert response.status == 404
+    assert not response_schema.validate(await response.json())

tests/ahriman/web/views/service/test_views_service_rebuild.py

@@ -21,11 +21,12 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
     """
     must call post request correctly
     """
-    rebuild_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_rebuild")
+    rebuild_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_rebuild", return_value="abc")
     user_mock = AsyncMock()
     user_mock.return_value = "username"
     mocker.patch("ahriman.web.views.base.BaseView.username", side_effect=user_mock)
     request_schema = pytest.helpers.schema_request(RebuildView.post)
+    response_schema = pytest.helpers.schema_response(RebuildView.post)
 
     payload = {"packages": ["python", "ahriman"]}
     assert not request_schema.validate(payload)
@@ -33,6 +34,10 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
     assert response.ok
     rebuild_mock.assert_called_once_with("python", "username")
 
+    json = await response.json()
+    assert json["process_id"] == "abc"
+    assert not response_schema.validate(json)
+
 
 async def test_post_exception(client: TestClient, mocker: MockerFixture) -> None:
     """

tests/ahriman/web/views/service/test_views_service_remove.py

@@ -20,8 +20,9 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
     """
     must call post request correctly
     """
-    remove_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_remove")
+    remove_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_remove", return_value="abc")
     request_schema = pytest.helpers.schema_request(RemoveView.post)
+    response_schema = pytest.helpers.schema_response(RemoveView.post)
 
     payload = {"packages": ["ahriman"]}
     assert not request_schema.validate(payload)
@@ -29,6 +30,10 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
     assert response.ok
     remove_mock.assert_called_once_with(["ahriman"])
 
+    json = await response.json()
+    assert json["process_id"] == "abc"
+    assert not response_schema.validate(json)
+
 
 async def test_post_exception(client: TestClient, mocker: MockerFixture) -> None:
     """

tests/ahriman/web/views/service/test_views_service_request.py

@@ -21,11 +21,12 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
     """
     must call post request correctly
     """
-    add_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_add")
+    add_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_add", return_value="abc")
     user_mock = AsyncMock()
     user_mock.return_value = "username"
     mocker.patch("ahriman.web.views.base.BaseView.username", side_effect=user_mock)
     request_schema = pytest.helpers.schema_request(RequestView.post)
+    response_schema = pytest.helpers.schema_response(RequestView.post)
 
     payload = {"packages": ["ahriman"]}
     assert not request_schema.validate(payload)
@@ -33,6 +34,10 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
     assert response.ok
     add_mock.assert_called_once_with(["ahriman"], "username", now=False)
 
+    json = await response.json()
+    assert json["process_id"] == "abc"
+    assert not response_schema.validate(json)
+
 
 async def test_post_exception(client: TestClient, mocker: MockerFixture) -> None:
     """

tests/ahriman/web/views/service/test_views_service_update.py

@@ -1,17 +1,65 @@
+import pytest
+
 from aiohttp.test_utils import TestClient
 from pytest_mock import MockerFixture
 from unittest.mock import AsyncMock
 
+from ahriman.models.user_access import UserAccess
+from ahriman.web.views.service.update import UpdateView
 
-async def test_post_update(client: TestClient, mocker: MockerFixture) -> None:
+
+async def test_get_permission() -> None:
+    """
+    must return correct permission for the request
+    """
+    for method in ("POST",):
+        request = pytest.helpers.request("", "", method)
+        assert await UpdateView.get_permission(request) == UserAccess.Full
+
+
+async def test_post(client: TestClient, mocker: MockerFixture) -> None:
     """
     must call post request correctly for alias
     """
-    update_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_update")
+    update_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_update", return_value="abc")
     user_mock = AsyncMock()
     user_mock.return_value = "username"
     mocker.patch("ahriman.web.views.base.BaseView.username", side_effect=user_mock)
+    request_schema = pytest.helpers.schema_request(UpdateView.post)
+    response_schema = pytest.helpers.schema_response(UpdateView.post)
+
+    defaults = {
+        "aur": True,
+        "local": True,
+        "manual": True,
+    }
+
+    for payload in (
+            {},
+            {"aur": False},
+            {"local": False},
+            {"manual": False},
+    ):
+        assert not request_schema.validate(payload)
+        response = await client.post("/api/v1/service/update", json=payload)
+        assert response.ok
+        update_mock.assert_called_once_with("username", **(defaults | payload))
+        update_mock.reset_mock()
+
+        json = await response.json()
+        assert json["process_id"] == "abc"
+        assert not response_schema.validate(json)
+
+
+async def test_post_empty(client: TestClient, mocker: MockerFixture) -> None:
+    """
+    must call raise 400 on invalid request
+    """
+    mocker.patch("ahriman.web.views.base.BaseView.extract_data", side_effect=Exception())
+    update_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_update")
+    response_schema = pytest.helpers.schema_response(UpdateView.post, code=400)
 
     response = await client.post("/api/v1/service/update")
-    assert response.ok
-    update_mock.assert_called_once_with("username")
+    assert response.status == 400
+    assert not response_schema.validate(await response.json())
+    update_mock.assert_not_called()

tests/ahriman/web/views/service/test_views_service_upload.py

@@ -0,0 +1,179 @@
+import pytest
+
+from aiohttp import FormData
+from aiohttp.test_utils import TestClient
+from aiohttp.web import HTTPBadRequest
+from io import BytesIO
+from pathlib import Path
+from pytest_mock import MockerFixture
+from unittest.mock import AsyncMock, MagicMock, call as MockCall
+
+from ahriman.models.repository_paths import RepositoryPaths
+from ahriman.models.user_access import UserAccess
+from ahriman.web.views.service.upload import UploadView
+
+
+async def test_get_permission() -> None:
+    """
+    must return correct permission for the request
+    """
+    for method in ("POST",):
+        request = pytest.helpers.request("", "", method)
+        assert await UploadView.get_permission(request) == UserAccess.Full
+
+
+async def test_save_file(mocker: MockerFixture) -> None:
+    """
+    must correctly save file
+    """
+    part_mock = MagicMock()
+    part_mock.filename = "filename"
+    part_mock.read_chunk = AsyncMock(side_effect=[b"content", None])
+
+    tempfile_mock = mocker.patch("ahriman.web.views.service.upload.NamedTemporaryFile")
+    file_mock = MagicMock()
+    tempfile_mock.return_value.__enter__.return_value = file_mock
+
+    open_mock = mocker.patch("pathlib.Path.open")
+    copy_mock = mocker.patch("shutil.copyfileobj")
+    local = Path("local")
+
+    assert await UploadView.save_file(part_mock, local, max_body_size=None) == \
+        (part_mock.filename, local / f".{part_mock.filename}")
+    file_mock.write.assert_called_once_with(b"content")
+    open_mock.assert_called_once_with("wb")
+    copy_mock.assert_called_once_with(pytest.helpers.anyvar(int), pytest.helpers.anyvar(int))
+
+
+async def test_save_file_no_filename() -> None:
+    """
+    must raise exception on missing filename
+    """
+    part_mock = MagicMock()
+    part_mock.filename = None
+
+    with pytest.raises(HTTPBadRequest):
+        await UploadView.save_file(part_mock, Path("local"), max_body_size=None)
+
+
+async def test_save_file_invalid_filename() -> None:
+    """
+    must raise exception on invalid filename
+    """
+    part_mock = MagicMock()
+    part_mock.filename = ".."
+
+    with pytest.raises(HTTPBadRequest):
+        await UploadView.save_file(part_mock, Path("local"), max_body_size=None)
+
+
+async def test_save_file_too_big() -> None:
+    """
+    must raise exception on too big file
+    """
+    part_mock = MagicMock()
+    part_mock.filename = "filename"
+    part_mock.read_chunk = AsyncMock(side_effect=[b"content", None])
+
+    with pytest.raises(HTTPBadRequest):
+        await UploadView.save_file(part_mock, Path("local"), max_body_size=0)
+
+
+async def test_post(client: TestClient, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
+    """
+    must process file upload via http
+    """
+    local = Path("local")
+    save_mock = mocker.patch("ahriman.web.views.service.upload.UploadView.save_file",
+                             side_effect=AsyncMock(return_value=("filename", local / ".filename")))
+    rename_mock = mocker.patch("pathlib.Path.rename")
+    # no content validation here because it has invalid schema
+
+    data = FormData()
+    data.add_field("package", BytesIO(b"content"), filename="filename", content_type="application/octet-stream")
+
+    response = await client.post("/api/v1/service/upload", data=data)
+    assert response.ok
+    save_mock.assert_called_once_with(pytest.helpers.anyvar(int), repository_paths.packages, max_body_size=None)
+    rename_mock.assert_called_once_with(local / "filename")
+
+
+async def test_post_with_sig(client: TestClient, repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
+    """
+    must process file upload with signature via http
+    """
+    local = Path("local")
+    save_mock = mocker.patch("ahriman.web.views.service.upload.UploadView.save_file",
+                             side_effect=AsyncMock(side_effect=[
+                                 ("filename", local / ".filename"),
+                                 ("filename.sig", local / ".filename.sig"),
+                             ]))
+    rename_mock = mocker.patch("pathlib.Path.rename")
+    # no content validation here because it has invalid schema
+
+    data = FormData()
+    data.add_field("package", BytesIO(b"content"), filename="filename")
+    data.add_field("signature", BytesIO(b"sig"), filename="filename.sig")
+
+    response = await client.post("/api/v1/service/upload", data=data)
+    assert response.ok
+    save_mock.assert_has_calls([
+        MockCall(pytest.helpers.anyvar(int), repository_paths.packages, max_body_size=None),
+        MockCall(pytest.helpers.anyvar(int), repository_paths.packages, max_body_size=None),
+    ])
+    rename_mock.assert_has_calls([
+        MockCall(local / "filename"),
+        MockCall(local / "filename.sig"),
+    ])
+
+
+async def test_post_not_found(client: TestClient, mocker: MockerFixture) -> None:
+    """
+    must return 404 if request was disabled
+    """
+    mocker.patch("ahriman.core.configuration.Configuration.getboolean", return_value=False)
+    data = FormData()
+    data.add_field("package", BytesIO(b"content"), filename="filename", content_type="application/octet-stream")
+    response_schema = pytest.helpers.schema_response(UploadView.post, code=404)
+
+    response = await client.post("/api/v1/service/upload", data=data)
+    assert response.status == 404
+    assert not response_schema.validate(await response.json())
+
+
+async def test_post_not_multipart(client: TestClient) -> None:
+    """
+    must return 400 on invalid payload
+    """
+    response_schema = pytest.helpers.schema_response(UploadView.post, code=400)
+
+    response = await client.post("/api/v1/service/upload")
+    assert response.status == 400
+    assert not response_schema.validate(await response.json())
+
+
+async def test_post_not_body_part(client: TestClient, mocker: MockerFixture) -> None:
+    """
+    must return 400 on invalid iterator in multipart
+    """
+    response_schema = pytest.helpers.schema_response(UploadView.post, code=400)
+    mocker.patch("aiohttp.MultipartReader.next", return_value=42)  # surprise, motherfucker
+    data = FormData()
+    data.add_field("package", BytesIO(b"content"), filename="filename", content_type="application/octet-stream")
+
+    response = await client.post("/api/v1/service/upload", data=data)
+    assert response.status == 400
+    assert not response_schema.validate(await response.json())
+
+
+async def test_post_not_package(client: TestClient) -> None:
+    """
+    must return 400 on invalid multipart key
+    """
+    response_schema = pytest.helpers.schema_response(UploadView.post, code=400)
+    data = FormData()
+    data.add_field("random", BytesIO(b"content"), filename="filename", content_type="application/octet-stream")
+
+    response = await client.post("/api/v1/service/upload", data=data)
+    assert response.status == 400
+    assert not response_schema.validate(await response.json())

tests/ahriman/web/views/status/test_views_status_logs.py

@@ -30,9 +30,9 @@ async def test_delete(client: TestClient, package_ahriman: Package, package_pyth
                       json={"status": BuildStatusEnum.Success.value, "package": package_python_schedule.view()})
 
     await client.post(f"/api/v1/packages/{package_ahriman.base}/logs",
-                      json={"created": 42.0, "message": "message", "process_id": 42})
+                      json={"created": 42.0, "message": "message", "version": "42"})
     await client.post(f"/api/v1/packages/{package_python_schedule.base}/logs",
-                      json={"created": 42.0, "message": "message", "process_id": 42})
+                      json={"created": 42.0, "message": "message", "version": "42"})
 
     response = await client.delete(f"/api/v1/packages/{package_ahriman.base}/logs")
     assert response.status == 204
@@ -53,7 +53,7 @@ async def test_get(client: TestClient, package_ahriman: Package) -> None:
     await client.post(f"/api/v1/packages/{package_ahriman.base}",
                       json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()})
     await client.post(f"/api/v1/packages/{package_ahriman.base}/logs",
-                      json={"created": 42.0, "message": "message", "process_id": 42})
+                      json={"created": 42.0, "message": "message", "version": "42"})
     response_schema = pytest.helpers.schema_response(LogsView.get)
 
     response = await client.get(f"/api/v1/packages/{package_ahriman.base}/logs")
@@ -83,7 +83,7 @@ async def test_post(client: TestClient, package_ahriman: Package) -> None:
                       json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()})
     request_schema = pytest.helpers.schema_request(LogsView.post)
 
-    payload = {"created": 42.0, "message": "message", "process_id": 42}
+    payload = {"created": 42.0, "message": "message", "version": "42"}
     assert not request_schema.validate(payload)
     response = await client.post(f"/api/v1/packages/{package_ahriman.base}/logs", json=payload)
     assert response.status == 204

tests/ahriman/web/views/status/test_views_status_status.py

@@ -75,7 +75,7 @@ async def test_post_exception_inside(client: TestClient, mocker: MockerFixture)
     exception handler must handle 500 errors
     """
     payload = {"status": BuildStatusEnum.Success.value}
-    mocker.patch("ahriman.core.status.watcher.Watcher.update_self", side_effect=Exception())
+    mocker.patch("ahriman.core.status.watcher.Watcher.status_update", side_effect=Exception())
     response_schema = pytest.helpers.schema_response(StatusView.post, code=500)
 
     response = await client.post("/api/v1/status", json=payload)