From f634f1df582afa9606fd893e9fcca8459f384cbe Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Thu, 8 Apr 2021 01:48:53 +0300 Subject: [PATCH] Add web status route (#13) * add status route * typed status and get status at the start of application --- src/ahriman/application/ahriman.py | 5 +- src/ahriman/application/lock.py | 14 +++- src/ahriman/core/status/client.py | 9 +++ src/ahriman/core/status/web_client.py | 25 +++++++ src/ahriman/models/counters.py | 71 ++++++++++++++++++++ src/ahriman/models/internal_status.py | 60 +++++++++++++++++ src/ahriman/web/routes.py | 5 ++ src/ahriman/web/views/index.py | 3 +- src/ahriman/web/views/status.py | 45 +++++++++++++ tests/ahriman/application/test_lock.py | 28 ++++++++ tests/ahriman/core/status/test_client.py | 8 +++ tests/ahriman/core/status/test_web_client.py | 40 +++++++++++ tests/ahriman/models/conftest.py | 21 ++++++ tests/ahriman/models/test_counters.py | 31 +++++++++ tests/ahriman/models/test_internal_status.py | 8 +++ tests/ahriman/web/views/test_view_status.py | 22 ++++++ 16 files changed, 389 insertions(+), 6 deletions(-) create mode 100644 src/ahriman/models/counters.py create mode 100644 src/ahriman/models/internal_status.py create mode 100644 src/ahriman/web/views/status.py create mode 100644 tests/ahriman/models/test_counters.py create mode 100644 tests/ahriman/models/test_internal_status.py create mode 100644 tests/ahriman/web/views/test_view_status.py diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 4a300b21..e1bfc815 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -22,9 +22,8 @@ import sys from pathlib import Path -import ahriman.application.handlers as handlers -import ahriman.version as version - +from ahriman import version +from ahriman.application import handlers from ahriman.models.build_status import BuildStatusEnum from ahriman.models.sign_settings import SignSettings diff --git a/src/ahriman/application/lock.py b/src/ahriman/application/lock.py index 1cede7f4..b3e321a5 100644 --- a/src/ahriman/application/lock.py +++ b/src/ahriman/application/lock.py @@ -20,12 +20,14 @@ from __future__ import annotations import argparse +import logging import os from pathlib import Path from types import TracebackType from typing import Literal, Optional, Type +from ahriman import version from ahriman.core.configuration import Configuration from ahriman.core.exceptions import DuplicateRun, UnsafeRun from ahriman.core.status.client import Client @@ -61,12 +63,13 @@ class Lock: default workflow is the following: check user UID - remove lock file if force flag is set check if there is lock file + check web status watcher status create lock file report to web if enabled """ self.check_user() + self.check_version() self.create() self.reporter.update_self(BuildStatusEnum.Building) return self @@ -85,6 +88,15 @@ class Lock: self.reporter.update_self(status) return False + def check_version(self) -> None: + """ + check web server version + """ + status = self.reporter.get_internal() + if status.version is not None and status.version != version.__version__: + logging.getLogger("root").warning(f"status watcher version mismatch, " + f"our {version.__version__}, their {status.version}") + def check_user(self) -> None: """ check if current user is actually owner of ahriman root diff --git a/src/ahriman/core/status/client.py b/src/ahriman/core/status/client.py index d7bbfb79..ad1432f7 100644 --- a/src/ahriman/core/status/client.py +++ b/src/ahriman/core/status/client.py @@ -23,6 +23,7 @@ from typing import List, Optional, Tuple, Type from ahriman.core.configuration import Configuration from ahriman.models.build_status import BuildStatus, BuildStatusEnum +from ahriman.models.internal_status import InternalStatus from ahriman.models.package import Package @@ -62,6 +63,14 @@ class Client: del base return [] + # pylint: disable=no-self-use + def get_internal(self) -> InternalStatus: + """ + get internal service status + :return: current internal (web) service status + """ + return InternalStatus() + # pylint: disable=no-self-use def get_self(self) -> BuildStatus: """ diff --git a/src/ahriman/core/status/web_client.py b/src/ahriman/core/status/web_client.py index 531b0890..fbe0e365 100644 --- a/src/ahriman/core/status/web_client.py +++ b/src/ahriman/core/status/web_client.py @@ -24,6 +24,7 @@ from typing import List, Optional, Tuple from ahriman.core.status.client import Client from ahriman.models.build_status import BuildStatusEnum, BuildStatus +from ahriman.models.internal_status import InternalStatus from ahriman.models.package import Package @@ -70,6 +71,13 @@ class WebClient(Client): """ return f"http://{self.host}:{self.port}/api/v1/packages/{base}" + def _status_url(self) -> str: + """ + url generator + :return: full url for web service for status + """ + return f"http://{self.host}:{self.port}/api/v1/status" + def add(self, package: Package, status: BuildStatusEnum) -> None: """ add new package with status @@ -110,6 +118,23 @@ class WebClient(Client): self.logger.exception(f"could not get {base}") return [] + def get_internal(self) -> InternalStatus: + """ + get internal service status + :return: current internal (web) service status + """ + try: + response = requests.get(self._status_url()) + response.raise_for_status() + + status_json = response.json() + return InternalStatus.from_json(status_json) + except requests.exceptions.HTTPError as e: + self.logger.exception(f"could not get web service status: {WebClient._exception_response_text(e)}") + except Exception: + self.logger.exception("could not get web service status") + return InternalStatus() + def get_self(self) -> BuildStatus: """ get ahriman status itself diff --git a/src/ahriman/models/counters.py b/src/ahriman/models/counters.py new file mode 100644 index 00000000..c1ffafbd --- /dev/null +++ b/src/ahriman/models/counters.py @@ -0,0 +1,71 @@ +# +# Copyright (c) 2021 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from __future__ import annotations + +from dataclasses import dataclass, fields +from typing import Any, Dict, List, Tuple, Type + +from ahriman.models.build_status import BuildStatus +from ahriman.models.package import Package + + +@dataclass +class Counters: + """ + package counters + :ivar total: total packages count + :ivar unknown: packages in unknown status count + :ivar pending: packages in pending status count + :ivar building: packages in building status count + :ivar failed: packages in failed status count + :ivar success: packages in success status count + """ + total: int + unknown: int = 0 + pending: int = 0 + building: int = 0 + failed: int = 0 + success: int = 0 + + @classmethod + def from_json(cls: Type[Counters], dump: Dict[str, Any]) -> Counters: + """ + construct counters from json dump + :param dump: json dump body + :return: status counters + """ + # filter to only known fields + known_fields = [pair.name for pair in fields(cls)] + dump = {key: value for key, value in dump.items() if key in known_fields} + return cls(**dump) + + @classmethod + def from_packages(cls: Type[Counters], packages: List[Tuple[Package, BuildStatus]]) -> Counters: + """ + construct counters from packages statuses + :param packages: list of package and their status as per watcher property + :return: status counters + """ + per_status = {"total": len(packages)} + for _, status in packages: + key = status.status.name.lower() + per_status.setdefault(key, 0) + per_status[key] += 1 + return cls(**per_status) diff --git a/src/ahriman/models/internal_status.py b/src/ahriman/models/internal_status.py new file mode 100644 index 00000000..8956e37c --- /dev/null +++ b/src/ahriman/models/internal_status.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2021 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any, Dict, Optional, Type + +from ahriman.models.counters import Counters + + +@dataclass +class InternalStatus: + """ + internal server status + :ivar architecture: repository architecture + :ivar packages: packages statuses counter object + :ivar repository: repository name + :ivar version: service version + """ + architecture: Optional[str] = None + packages: Counters = field(default=Counters(total=0)) + repository: Optional[str] = None + version: Optional[str] = None + + @classmethod + def from_json(cls: Type[InternalStatus], dump: Dict[str, Any]) -> InternalStatus: + """ + construct internal status from json dump + :param dump: json dump body + :return: internal status + """ + counters = Counters.from_json(dump["packages"]) if "packages" in dump else Counters(total=0) + return cls(architecture=dump.get("architecture"), + packages=counters, + repository=dump.get("repository"), + version=dump.get("version")) + + def view(self) -> Dict[str, Any]: + """ + generate json status view + :return: json-friendly dictionary + """ + return asdict(self) diff --git a/src/ahriman/web/routes.py b/src/ahriman/web/routes.py index 48ef29a9..9f50a8fa 100644 --- a/src/ahriman/web/routes.py +++ b/src/ahriman/web/routes.py @@ -23,6 +23,7 @@ from ahriman.web.views.ahriman import AhrimanView from ahriman.web.views.index import IndexView from ahriman.web.views.package import PackageView from ahriman.web.views.packages import PackagesView +from ahriman.web.views.status import StatusView def setup_routes(application: Application) -> None: @@ -44,6 +45,8 @@ def setup_routes(application: Application) -> None: GET /api/v1/package/:base get package base status POST /api/v1/package/:base update package base status + GET /api/v1/status get web service status itself + :param application: web application instance """ application.router.add_get("/", IndexView) @@ -58,3 +61,5 @@ def setup_routes(application: Application) -> None: application.router.add_delete("/api/v1/packages/{package}", PackageView) application.router.add_get("/api/v1/packages/{package}", PackageView) application.router.add_post("/api/v1/packages/{package}", PackageView) + + application.router.add_get("/api/v1/status", StatusView) diff --git a/src/ahriman/web/views/index.py b/src/ahriman/web/views/index.py index 2a1c07ab..57b96811 100644 --- a/src/ahriman/web/views/index.py +++ b/src/ahriman/web/views/index.py @@ -21,8 +21,7 @@ import aiohttp_jinja2 from typing import Any, Dict -import ahriman.version as version - +from ahriman import version from ahriman.core.util import pretty_datetime from ahriman.web.views.base import BaseView diff --git a/src/ahriman/web/views/status.py b/src/ahriman/web/views/status.py new file mode 100644 index 00000000..84aaf937 --- /dev/null +++ b/src/ahriman/web/views/status.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2021 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from aiohttp.web import Response, json_response + +from ahriman import version +from ahriman.models.counters import Counters +from ahriman.models.internal_status import InternalStatus +from ahriman.web.views.base import BaseView + + +class StatusView(BaseView): + """ + web service status web view + """ + + async def get(self) -> Response: + """ + get current service status + :return: 200 with service status object + """ + counters = Counters.from_packages(self.service.packages) + status = InternalStatus( + architecture=self.service.architecture, + packages=counters, + repository=self.service.repository.name, + version=version.__version__) + + return json_response(status.view()) diff --git a/tests/ahriman/application/test_lock.py b/tests/ahriman/application/test_lock.py index 58cf1956..43dcf8f6 100644 --- a/tests/ahriman/application/test_lock.py +++ b/tests/ahriman/application/test_lock.py @@ -5,9 +5,11 @@ from pathlib import Path from pytest_mock import MockerFixture from unittest import mock +from ahriman import version from ahriman.application.lock import Lock from ahriman.core.exceptions import DuplicateRun, UnsafeRun from ahriman.models.build_status import BuildStatusEnum +from ahriman.models.internal_status import InternalStatus def test_enter(lock: Lock, mocker: MockerFixture) -> None: @@ -15,6 +17,7 @@ def test_enter(lock: Lock, mocker: MockerFixture) -> None: must process with context manager """ check_user_mock = mocker.patch("ahriman.application.lock.Lock.check_user") + check_version_mock = mocker.patch("ahriman.application.lock.Lock.check_version") clear_mock = mocker.patch("ahriman.application.lock.Lock.clear") create_mock = mocker.patch("ahriman.application.lock.Lock.create") update_status_mock = mocker.patch("ahriman.core.status.client.Client.update_self") @@ -24,6 +27,7 @@ def test_enter(lock: Lock, mocker: MockerFixture) -> None: check_user_mock.assert_called_once() clear_mock.assert_called_once() create_mock.assert_called_once() + check_version_mock.assert_called_once() update_status_mock.assert_has_calls([ mock.call(BuildStatusEnum.Building), mock.call(BuildStatusEnum.Success) @@ -48,6 +52,30 @@ def test_exit_with_exception(lock: Lock, mocker: MockerFixture) -> None: ]) +def test_check_version(lock: Lock, mocker: MockerFixture) -> None: + """ + must check version correctly + """ + mocker.patch("ahriman.core.status.client.Client.get_internal", + return_value=InternalStatus(version=version.__version__)) + logging_mock = mocker.patch("logging.Logger.warning") + + lock.check_version() + logging_mock.assert_not_called() + + +def test_check_version_mismatch(lock: Lock, mocker: MockerFixture) -> None: + """ + must check version correctly + """ + mocker.patch("ahriman.core.status.client.Client.get_internal", + return_value=InternalStatus(version="version")) + logging_mock = mocker.patch("logging.Logger.warning") + + lock.check_version() + logging_mock.assert_called_once() + + def test_check_user(lock: Lock, mocker: MockerFixture) -> None: """ must check user correctly diff --git a/tests/ahriman/core/status/test_client.py b/tests/ahriman/core/status/test_client.py index 8b1c4e0a..ba677d3b 100644 --- a/tests/ahriman/core/status/test_client.py +++ b/tests/ahriman/core/status/test_client.py @@ -4,6 +4,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.status.client import Client from ahriman.core.status.web_client import WebClient from ahriman.models.build_status import BuildStatusEnum +from ahriman.models.internal_status import InternalStatus from ahriman.models.package import Package @@ -38,6 +39,13 @@ def test_get(client: Client, package_ahriman: Package) -> None: assert client.get(None) == [] +def test_get_internal(client: Client) -> None: + """ + must return dummy status for web service + """ + assert client.get_internal() == InternalStatus() + + def test_get_self(client: Client) -> None: """ must return unknown status for service diff --git a/tests/ahriman/core/status/test_web_client.py b/tests/ahriman/core/status/test_web_client.py index c42f410a..fd8fb2cb 100644 --- a/tests/ahriman/core/status/test_web_client.py +++ b/tests/ahriman/core/status/test_web_client.py @@ -7,6 +7,7 @@ from requests import Response from ahriman.core.status.web_client import WebClient from ahriman.models.build_status import BuildStatus, BuildStatusEnum +from ahriman.models.internal_status import InternalStatus from ahriman.models.package import Package @@ -26,6 +27,14 @@ def test_package_url(web_client: WebClient, package_ahriman: Package) -> None: assert web_client._package_url(package_ahriman.base).endswith(f"/api/v1/packages/{package_ahriman.base}") +def test_status_url(web_client: WebClient) -> None: + """ + must generate service status url correctly + """ + assert web_client._status_url().startswith(f"http://{web_client.host}:{web_client.port}") + assert web_client._status_url().endswith("/api/v1/status") + + def test_add(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None: """ must process package addition @@ -103,6 +112,37 @@ def test_get_single(web_client: WebClient, package_ahriman: Package, mocker: Moc assert (package_ahriman, BuildStatusEnum.Unknown) in [(package, status.status) for package, status in result] +def test_get_internal(web_client: WebClient, mocker: MockerFixture) -> None: + """ + must return web service status + """ + response_obj = Response() + response_obj._content = json.dumps(InternalStatus(architecture="x86_64").view()).encode("utf8") + response_obj.status_code = 200 + + requests_mock = mocker.patch("requests.get", return_value=response_obj) + + result = web_client.get_internal() + requests_mock.assert_called_once() + assert result.architecture == "x86_64" + + +def test_get_internal_failed(web_client: WebClient, mocker: MockerFixture) -> None: + """ + must suppress any exception happened during web service status getting + """ + mocker.patch("requests.get", side_effect=Exception()) + assert web_client.get_internal() == InternalStatus() + + +def test_get_internal_failed_http_error(web_client: WebClient, mocker: MockerFixture) -> None: + """ + must suppress any exception happened during web service status getting + """ + mocker.patch("requests.get", side_effect=requests.exceptions.HTTPError()) + assert web_client.get_internal() == InternalStatus() + + def test_get_self(web_client: WebClient, mocker: MockerFixture) -> None: """ must return service status diff --git a/tests/ahriman/models/conftest.py b/tests/ahriman/models/conftest.py index 191e2505..80a7b8d8 100644 --- a/tests/ahriman/models/conftest.py +++ b/tests/ahriman/models/conftest.py @@ -2,7 +2,10 @@ import pytest from unittest.mock import MagicMock, PropertyMock +from ahriman import version from ahriman.models.build_status import BuildStatus, BuildStatusEnum +from ahriman.models.counters import Counters +from ahriman.models.internal_status import InternalStatus from ahriman.models.package import Package from ahriman.models.package_description import PackageDescription @@ -12,6 +15,24 @@ def build_status_failed() -> BuildStatus: return BuildStatus(BuildStatusEnum.Failed, 42) +@pytest.fixture +def counters() -> Counters: + return Counters(total=10, + unknown=1, + pending=2, + building=3, + failed=4, + success=0) + + +@pytest.fixture +def internal_status(counters: Counters) -> InternalStatus: + return InternalStatus(architecture="x86_64", + packages=counters, + version=version.__version__, + repository="aur-clone") + + @pytest.fixture def package_tpacpi_bat_git() -> Package: return Package( diff --git a/tests/ahriman/models/test_counters.py b/tests/ahriman/models/test_counters.py new file mode 100644 index 00000000..1722f3a3 --- /dev/null +++ b/tests/ahriman/models/test_counters.py @@ -0,0 +1,31 @@ +from dataclasses import asdict + +from ahriman.models.build_status import BuildStatus, BuildStatusEnum +from ahriman.models.counters import Counters +from ahriman.models.package import Package + + +def test_counters_from_json_view(counters: Counters) -> None: + """ + must construct same object from json + """ + assert Counters.from_json(asdict(counters)) == counters + + +def test_counters_from_packages(package_ahriman: Package, package_python_schedule: Package) -> None: + """ + must construct object from list of packages with their statuses + """ + payload = [ + (package_ahriman, BuildStatus(status=BuildStatusEnum.Success)), + (package_python_schedule, BuildStatus(status=BuildStatusEnum.Failed)), + ] + + counters = Counters.from_packages(payload) + assert counters.total == 2 + assert counters.success == 1 + assert counters.failed == 1 + + json = asdict(counters) + total = json.pop("total") + assert total == sum(i for i in json.values()) diff --git a/tests/ahriman/models/test_internal_status.py b/tests/ahriman/models/test_internal_status.py new file mode 100644 index 00000000..f3bf2440 --- /dev/null +++ b/tests/ahriman/models/test_internal_status.py @@ -0,0 +1,8 @@ +from ahriman.models.internal_status import InternalStatus + + +def test_internal_status_from_json_view(internal_status: InternalStatus) -> None: + """ + must construct same object from json + """ + assert InternalStatus.from_json(internal_status.view()) == internal_status diff --git a/tests/ahriman/web/views/test_view_status.py b/tests/ahriman/web/views/test_view_status.py new file mode 100644 index 00000000..5078870b --- /dev/null +++ b/tests/ahriman/web/views/test_view_status.py @@ -0,0 +1,22 @@ +from pytest_aiohttp import TestClient + +import ahriman.version as version + +from ahriman.models.build_status import BuildStatusEnum +from ahriman.models.package import Package + + +async def test_get(client: TestClient, package_ahriman: Package) -> None: + """ + must generate web service status correctly) + """ + await client.post(f"/api/v1/packages/{package_ahriman.base}", + json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()}) + + response = await client.get("/api/v1/status") + assert response.status == 200 + + json = await response.json() + assert json["version"] == version.__version__ + assert json["packages"] + assert json["packages"]["total"] == 1