add support of remote task tracking

This commit is contained in:
Evgenii Alekseev 2023-08-09 00:54:52 +03:00
parent 9ea3a911f7
commit 37d3b9fa83
34 changed files with 612 additions and 98 deletions

View File

@ -115,6 +115,7 @@ Web server settings. If any of ``host``/``port`` is not set, web integration wil
* ``unix_socket`` - path to the listening unix socket, string, optional. If set, server will create the socket on the specified address which can (and will) be used by application. Note, that unlike usual host/port configuration, unix socket allows to perform requests without authorization.
* ``unix_socket_unsafe`` - set unsafe (o+w) permissions to unix socket, boolean, optional, default ``yes``. This option is enabled by default, because it is supposed that unix socket is created in safe environment (only web service is supposed to be used in unsafe), but it can be disabled by configuration.
* ``username`` - username to authorize in web service in order to update service status, string, required in case if authorization enabled.
* ``wait_timeout`` - wait timeout in seconds, maximum amount of time to be waited before lock will be free, int, optional.
``keyring`` group
--------------------

View File

@ -84,6 +84,10 @@ def _parser() -> argparse.ArgumentParser:
parser.add_argument("-q", "--quiet", help="force disable any logging", action="store_true")
parser.add_argument("--unsafe", help="allow to run ahriman as non-ahriman user. Some actions might be unavailable",
action="store_true")
parser.add_argument("--wait-timeout", help="wait for lock to be free. Negative value will lead to "
"immediate application run even if there is lock file. "
"In case of zero value, tthe application will wait infinitely",
type=int, default=-1)
parser.add_argument("-V", "--version", action="version", version=__version__)
subparsers = parser.add_subparsers(title="command", help="command to run", dest="command", required=True)

View File

@ -33,7 +33,6 @@ class Web(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False
ALLOW_MULTI_ARCHITECTURE_RUN = False # required to be able to spawn external processes
COMMAND_ARGS_WHITELIST = ["force", "log_handler", ""]
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
@ -89,3 +88,7 @@ class Web(Handler):
yield "--quiet"
if args.unsafe:
yield "--unsafe"
# arguments from configuration
if (wait_timeout := configuration.getint("web", "wait_timeout", fallback=None)) is not None:
yield from ["--wait-timeout", str(wait_timeout)]

View File

@ -18,7 +18,9 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import argparse
import time
from pathlib import Path
from types import TracebackType
from typing import Literal, Self
@ -41,6 +43,7 @@ class Lock(LazyLogging):
reporter(Client): build status reporter instance
paths(RepositoryPaths): repository paths instance
unsafe(bool): skip user check
wait_timeout(int): wait in seconds until lock will free
Examples:
Instance of this class except for controlling file-based lock is also required for basic applications checks.
@ -65,9 +68,11 @@ class Lock(LazyLogging):
architecture(str): repository architecture
configuration(Configuration): configuration instance
"""
self.path = args.lock.with_stem(f"{args.lock.stem}_{architecture}") if args.lock is not None else None
self.force = args.force
self.unsafe = args.unsafe
self.path: Path | None = \
args.lock.with_stem(f"{args.lock.stem}_{architecture}") if args.lock is not None else None
self.force: bool = args.force
self.unsafe: bool = args.unsafe
self.wait_timeout: int = args.wait_timeout
self.paths = configuration.repository_paths
self.reporter = Client.load(configuration, report=args.report)
@ -110,6 +115,27 @@ class Lock(LazyLogging):
except FileExistsError:
raise DuplicateRunError()
def watch(self, interval: int = 10) -> None:
"""
watch until lock disappear
Args:
interval(int, optional): interval to check in seconds (Default value = 10)
"""
def is_timed_out(start: float) -> bool:
since_start: float = time.monotonic() - start
return self.wait_timeout != 0 and since_start > self.wait_timeout
# there are reasons why we are not using inotify here. First of all, if we would use it, it would bring to
# race conditions because multiple processes will be notified in the same time. Secondly, it is good library,
# but platform-specific, and we only need to check if file exists
if self.path is None:
return
start_time = time.monotonic()
while not is_timed_out(start_time) and self.path.is_file():
time.sleep(interval)
def __enter__(self) -> Self:
"""
default workflow is the following:
@ -117,14 +143,16 @@ class Lock(LazyLogging):
1. Check user UID
2. Check if there is lock file
3. Check web status watcher status
4. Create lock file and directory tree
5. Report to status page if enabled
4. Wait for lock file to be free
5. Create lock file and directory tree
6. Report to status page if enabled
Returns:
Self: always instance of self
"""
self.check_user()
self.check_version()
self.watch()
self.create()
self.reporter.update_self(BuildStatusEnum.Building)
return self

View File

@ -268,6 +268,10 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"username": {
"type": "string",
},
"wait_timeout": {
"type": "integer",
"coerce": "integer",
}
},
},
}

View File

@ -20,6 +20,7 @@
from __future__ import annotations
import argparse
import time
import uuid
from collections.abc import Callable, Iterable
@ -38,7 +39,7 @@ class Spawn(Thread, LazyLogging):
active(dict[str, Process]): map of active child processes required to avoid zombies
architecture(str): repository architecture
command_arguments(list[str]): base command line arguments
queue(Queue[tuple[str, bool]]): multiprocessing queue to read updates from processes
queue(Queue[tuple[str, bool, int]]): multiprocessing queue to read updates from processes
"""
def __init__(self, args_parser: argparse.ArgumentParser, architecture: str, command_arguments: list[str]) -> None:
@ -59,11 +60,25 @@ class Spawn(Thread, LazyLogging):
self.lock = Lock()
self.active: dict[str, Process] = {}
# stupid pylint does not know that it is possible
self.queue: Queue[tuple[str, bool] | None] = Queue() # pylint: disable=unsubscriptable-object
self.queue: Queue[tuple[str, bool, int] | None] = Queue() # pylint: disable=unsubscriptable-object
@staticmethod
def boolean_action_argument(name: str, value: bool) -> str:
"""
convert option of given name with value to boolean action argument
Args:
name(str): command line argument name
value(bool): command line argument value
Returns:
str: if ``value`` is True, then returns positive flag and negative otherwise
"""
return name if value else f"no-{name}"
@staticmethod
def process(callback: Callable[[argparse.Namespace, str], bool], args: argparse.Namespace, architecture: str,
process_id: str, queue: Queue[tuple[str, bool]]) -> None: # pylint: disable=unsubscriptable-object
process_id: str, queue: Queue[tuple[str, bool, int]]) -> None: # pylint: disable=unsubscriptable-object
"""
helper to run external process
@ -72,12 +87,17 @@ class Spawn(Thread, LazyLogging):
args(argparse.Namespace): command line arguments
architecture(str): repository architecture
process_id(str): process unique identifier
queue(Queue[tuple[str, bool]]): output queue
queue(Queue[tuple[str, bool, int]]): output queue
"""
start_time = time.monotonic()
result = callback(args, architecture)
queue.put((process_id, result))
stop_time = time.monotonic()
def _spawn_process(self, command: str, *args: str, **kwargs: str | None) -> None:
consumed_time = int(1000 * (stop_time - start_time))
queue.put((process_id, result, consumed_time))
def _spawn_process(self, command: str, *args: str, **kwargs: str | None) -> str:
"""
spawn external ahriman process with supplied arguments
@ -85,6 +105,9 @@ class Spawn(Thread, LazyLogging):
command(str): subcommand to run
*args(str): positional command arguments
**kwargs(str): named command arguments
Returns:
str: spawned process id
"""
# default arguments
arguments = self.command_arguments[:]
@ -111,19 +134,36 @@ class Spawn(Thread, LazyLogging):
with self.lock:
self.active[process_id] = process
return process_id
def key_import(self, key: str, server: str | None) -> None:
def has_process(self, process_id: str) -> bool:
"""
check if given process is alive
Args:
process_id(str): process id to be checked as returned by ``Spawn._spawn_process``
Returns:
bool: True in case if process still counts as active and False otherwise
"""
with self.lock:
return process_id in self.active
def key_import(self, key: str, server: str | None) -> str:
"""
import key to service cache
Args:
key(str): key to import
server(str | None): PGP key server
Returns:
str: spawned process id
"""
kwargs = {} if server is None else {"key-server": server}
self._spawn_process("service-key-import", key, **kwargs)
return self._spawn_process("service-key-import", key, **kwargs)
def packages_add(self, packages: Iterable[str], username: str | None, *, now: bool) -> None:
def packages_add(self, packages: Iterable[str], username: str | None, *, now: bool) -> str:
"""
add packages
@ -131,48 +171,69 @@ class Spawn(Thread, LazyLogging):
packages(Iterable[str]): packages list to add
username(str | None): optional override of username for build process
now(bool): build packages now
Returns:
str: spawned process id
"""
kwargs = {"username": username}
if now:
kwargs["now"] = ""
self._spawn_process("package-add", *packages, **kwargs)
return self._spawn_process("package-add", *packages, **kwargs)
def packages_rebuild(self, depends_on: str, username: str | None) -> None:
def packages_rebuild(self, depends_on: str, username: str | None) -> str:
"""
rebuild packages which depend on the specified package
Args:
depends_on(str): packages dependency
username(str | None): optional override of username for build process
Returns:
str: spawned process id
"""
kwargs = {"depends-on": depends_on, "username": username}
self._spawn_process("repo-rebuild", **kwargs)
return self._spawn_process("repo-rebuild", **kwargs)
def packages_remove(self, packages: Iterable[str]) -> None:
def packages_remove(self, packages: Iterable[str]) -> str:
"""
remove packages
Args:
packages(Iterable[str]): packages list to remove
"""
self._spawn_process("package-remove", *packages)
def packages_update(self, username: str | None) -> None:
Returns:
str: spawned process id
"""
return self._spawn_process("package-remove", *packages)
def packages_update(self, username: str | None, *, aur: bool, local: bool, manual: bool) -> str:
"""
run full repository update
Args:
username(str | None): optional override of username for build process
aur(bool): check for aur updates
local(bool): check for local packages updates
manual(bool): check for manual packages
Returns:
str: spawned process id
"""
kwargs = {"username": username}
self._spawn_process("repo-update", **kwargs)
kwargs = {
"username": username,
self.boolean_action_argument("aur", aur): "",
self.boolean_action_argument("local", local): "",
self.boolean_action_argument("manual", manual): "",
}
return self._spawn_process("repo-update", **kwargs)
def run(self) -> None:
"""
thread run method
"""
for process_id, status in iter(self.queue.get, None):
self.logger.info("process %s has been terminated with status %s", process_id, status)
for process_id, status, consumed_time in iter(self.queue.get, None):
self.logger.info("process %s has been terminated with status %s, consumed time %s",
process_id, status, consumed_time / 1000)
with self.lock:
process = self.active.pop(process_id, None)

View File

@ -142,7 +142,6 @@ def check_output(*args: str, exception: Exception | None = None, cwd: Path | Non
while selector.get_map(): # while there are unread selectors, keep reading
result.extend(poll(selector))
process.terminate() # make sure that process is terminated
status_code = process.wait()
if status_code != 0:
if exception is not None:

View File

@ -25,6 +25,7 @@ from ahriman.web.views.api.swagger import SwaggerView
from ahriman.web.views.index import IndexView
from ahriman.web.views.service.add import AddView
from ahriman.web.views.service.pgp import PGPView
from ahriman.web.views.service.process import ProcessView
from ahriman.web.views.service.rebuild import RebuildView
from ahriman.web.views.service.remove import RemoveView
from ahriman.web.views.service.request import RequestView
@ -60,6 +61,7 @@ def setup_routes(application: Application, static_path: Path) -> None:
application.router.add_view("/api/v1/service/add", AddView)
application.router.add_view("/api/v1/service/pgp", PGPView)
application.router.add_view("/api/v1/service/rebuild", RebuildView)
application.router.add_view("/api/v1/service/process/{process_id}", ProcessView)
application.router.add_view("/api/v1/service/remove", RemoveView)
application.router.add_view("/api/v1/service/request", RequestView)
application.router.add_view("/api/v1/service/search", SearchView)

View File

@ -33,6 +33,9 @@ from ahriman.web.schemas.package_schema import PackageSchema
from ahriman.web.schemas.package_status_schema import PackageStatusSimplifiedSchema, PackageStatusSchema
from ahriman.web.schemas.pgp_key_id_schema import PGPKeyIdSchema
from ahriman.web.schemas.pgp_key_schema import PGPKeySchema
from ahriman.web.schemas.process_id_schema import ProcessIdSchema
from ahriman.web.schemas.process_schema import ProcessSchema
from ahriman.web.schemas.remote_schema import RemoteSchema
from ahriman.web.schemas.search_schema import SearchSchema
from ahriman.web.schemas.status_schema import StatusSchema
from ahriman.web.schemas.update_flags_schema import UpdateFlagsSchema

View File

@ -0,0 +1,31 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from marshmallow import Schema, fields
class ProcessIdSchema(Schema):
"""
request and response spawned process id schema
"""
process_id = fields.String(required=True, metadata={
"description": "Spawned process unique ID",
"example": "ff456814-5669-4de6-9143-44dbf6f68607",
})

View File

@ -0,0 +1,30 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from marshmallow import Schema, fields
class ProcessSchema(Schema):
"""
process status response schema
"""
is_alive = fields.Bool(required=True, metadata={
"description": "Is process alive or not",
})

View File

@ -0,0 +1,36 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from marshmallow import Schema, fields
class UpdateFlagsSchema(Schema):
"""
update flags request schema
"""
aur = fields.Bool(dump_default=True, metadata={
"description": "Check AUR for updates",
})
local = fields.Bool(dump_default=True, metadata={
"description": "Check local packages for updates",
})
manual = fields.Bool(dump_default=True, metadata={
"description": "Check manually built packages",
})

View File

@ -19,10 +19,10 @@
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPBadRequest, HTTPNoContent
from aiohttp.web import HTTPBadRequest, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
from ahriman.web.views.base import BaseView
@ -41,7 +41,7 @@ class AddView(BaseView):
summary="Add new package",
description="Add new package(s) from AUR",
responses={
204: {"description": "Success response"},
200: {"description": "Success response", "schema": ProcessIdSchema},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
@ -51,13 +51,15 @@ class AddView(BaseView):
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.json_schema(PackageNamesSchema)
async def post(self) -> None:
async def post(self) -> Response:
"""
add new package
Returns:
Response: 200 with spawned process id
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
try:
data = await self.extract_data(["packages"])
@ -66,6 +68,6 @@ class AddView(BaseView):
raise HTTPBadRequest(reason=str(e))
username = await self.username()
self.spawner.packages_add(packages, username, now=True)
process_id = self.spawner.packages_add(packages, username, now=True)
raise HTTPNoContent()
return json_response({"process_id": process_id})

View File

@ -19,10 +19,10 @@
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPBadRequest, HTTPNoContent, HTTPNotFound, Response, json_response
from aiohttp.web import HTTPBadRequest, HTTPNotFound, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, PGPKeyIdSchema, PGPKeySchema
from ahriman.web.schemas import AuthSchema, ErrorSchema, PGPKeyIdSchema, PGPKeySchema, ProcessIdSchema
from ahriman.web.views.base import BaseView
@ -83,7 +83,7 @@ class PGPView(BaseView):
summary="Fetch PGP key",
description="Fetch PGP key from the key server",
responses={
204: {"description": "Success response"},
200: {"description": "Success response", "schema": ProcessIdSchema},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
@ -93,13 +93,15 @@ class PGPView(BaseView):
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.json_schema(PGPKeyIdSchema)
async def post(self) -> None:
async def post(self) -> Response:
"""
store key to the local service environment
Returns:
Response: 200 with spawned process id
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
data = await self.extract_data()
@ -108,6 +110,6 @@ class PGPView(BaseView):
except Exception as e:
raise HTTPBadRequest(reason=str(e))
self.spawner.key_import(key, data.get("server"))
process_id = self.spawner.key_import(key, data.get("server"))
raise HTTPNoContent()
return json_response({"process_id": process_id})

View File

@ -0,0 +1,74 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPNotFound, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, ProcessIdSchema, ProcessSchema
from ahriman.web.views.base import BaseView
class ProcessView(BaseView):
"""
Process information web view
Attributes:
GET_PERMISSION(UserAccess): (class attribute) get permissions of self
"""
GET_PERMISSION = UserAccess.Reporter
@aiohttp_apispec.docs(
tags=["Actions"],
summary="Get process",
description="Get process information",
responses={
200: {"description": "Success response", "schema": ProcessSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
404: {"description": "Process ID is unknown", "schema": ErrorSchema},
500: {"description": "Internal server error", "schema": ErrorSchema},
},
security=[{"token": [GET_PERMISSION]}],
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.match_info_schema(ProcessIdSchema)
async def get(self) -> Response:
"""
get spawned process status
Returns:
Response: 200 with process information
Raises:
HTTPNotFound: if no process found
"""
process_id = self.request.match_info["process_id"]
is_alive = self.spawner.has_process(process_id)
if not is_alive:
raise HTTPNotFound(reason=f"No process {process_id} found")
response = {
"is_alive": is_alive,
}
return json_response(response)

View File

@ -19,10 +19,10 @@
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPBadRequest, HTTPNoContent
from aiohttp.web import HTTPBadRequest, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
from ahriman.web.views.base import BaseView
@ -41,7 +41,7 @@ class RebuildView(BaseView):
summary="Rebuild packages",
description="Rebuild packages which depend on specified one",
responses={
204: {"description": "Success response"},
200: {"description": "Success response", "schema": ProcessIdSchema},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
@ -51,13 +51,15 @@ class RebuildView(BaseView):
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.json_schema(PackageNamesSchema)
async def post(self) -> None:
async def post(self) -> Response:
"""
rebuild packages based on their dependency
Returns:
Response: 200 with spawned process id
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
try:
data = await self.extract_data(["packages"])
@ -67,6 +69,6 @@ class RebuildView(BaseView):
raise HTTPBadRequest(reason=str(e))
username = await self.username()
self.spawner.packages_rebuild(depends_on, username)
process_id = self.spawner.packages_rebuild(depends_on, username)
raise HTTPNoContent()
return json_response({"process_id": process_id})

View File

@ -19,10 +19,10 @@
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPBadRequest, HTTPNoContent
from aiohttp.web import HTTPBadRequest, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
from ahriman.web.views.base import BaseView
@ -41,7 +41,7 @@ class RemoveView(BaseView):
summary="Remove packages",
description="Remove specified packages from the repository",
responses={
204: {"description": "Success response"},
200: {"description": "Success response", "schema": ProcessIdSchema},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
@ -51,13 +51,15 @@ class RemoveView(BaseView):
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.json_schema(PackageNamesSchema)
async def post(self) -> None:
async def post(self) -> Response:
"""
remove existing packages
Returns:
Response: 200 with spawned process id
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
try:
data = await self.extract_data(["packages"])
@ -65,6 +67,6 @@ class RemoveView(BaseView):
except Exception as e:
raise HTTPBadRequest(reason=str(e))
self.spawner.packages_remove(packages)
process_id = self.spawner.packages_remove(packages)
raise HTTPNoContent()
return json_response({"process_id": process_id})

View File

@ -19,10 +19,10 @@
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPBadRequest, HTTPNoContent
from aiohttp.web import HTTPBadRequest, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
from ahriman.web.views.base import BaseView
@ -41,7 +41,7 @@ class RequestView(BaseView):
summary="Request new package",
description="Request new package(s) to be added from AUR",
responses={
204: {"description": "Success response"},
200: {"description": "Success response", "schema": ProcessIdSchema},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
@ -51,13 +51,15 @@ class RequestView(BaseView):
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.json_schema(PackageNamesSchema)
async def post(self) -> None:
async def post(self) -> Response:
"""
request to add new package
Returns:
Response: 200 with spawned process id
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
try:
data = await self.extract_data(["packages"])
@ -66,6 +68,6 @@ class RequestView(BaseView):
raise HTTPBadRequest(reason=str(e))
username = await self.username()
self.spawner.packages_add(packages, username, now=False)
process_id = self.spawner.packages_add(packages, username, now=False)
raise HTTPNoContent()
return json_response({"process_id": process_id})

View File

@ -19,10 +19,10 @@
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPNoContent
from aiohttp.web import HTTPBadRequest, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema
from ahriman.web.schemas import AuthSchema, ErrorSchema, ProcessIdSchema, UpdateFlagsSchema
from ahriman.web.views.base import BaseView
@ -41,7 +41,8 @@ class UpdateView(BaseView):
summary="Update packages",
description="Run repository update process",
responses={
204: {"description": "Success response"},
200: {"description": "Success response", "schema": ProcessIdSchema},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
500: {"description": "Internal server error", "schema": ErrorSchema},
@ -49,14 +50,28 @@ class UpdateView(BaseView):
security=[{"token": [POST_PERMISSION]}],
)
@aiohttp_apispec.cookies_schema(AuthSchema)
async def post(self) -> None:
@aiohttp_apispec.json_schema(UpdateFlagsSchema)
async def post(self) -> Response:
"""
run repository update. No parameters supported here
Raises:
HTTPNoContent: in case of success response
"""
username = await self.username()
self.spawner.packages_update(username)
Returns:
Response: 200 with spawned process id
raise HTTPNoContent()
Raises:
HTTPBadRequest: if bad data is supplied
"""
try:
data = await self.extract_data()
except Exception as e:
raise HTTPBadRequest(reason=str(e))
username = await self.username()
process_id = self.spawner.packages_update(
username,
aur=data.get("aur", True),
local=data.get("local", True),
manual=data.get("manual", True),
)
return json_response({"process_id": process_id})

View File

@ -39,7 +39,7 @@ def args() -> argparse.Namespace:
Returns:
argparse.Namespace: command line arguments test instance
"""
return argparse.Namespace(architecture=None, lock=None, force=False, unsafe=False, report=False)
return argparse.Namespace(architecture=None, lock=None, force=False, unsafe=False, report=False, wait_timeout=-1)
@pytest.fixture

View File

@ -77,6 +77,10 @@ def test_extract_arguments(args: argparse.Namespace, configuration: Configuratio
expected.extend(["--unsafe"])
assert list(Web.extract_arguments(probe, "x86_64", configuration)) == expected
configuration.set_option("web", "wait_timeout", "60")
expected.extend(["--wait-timeout", "60"])
assert list(Web.extract_arguments(probe, "x86_64", configuration)) == expected
def test_extract_arguments_full(parser: argparse.ArgumentParser, configuration: Configuration):
"""
@ -91,6 +95,7 @@ def test_extract_arguments_full(parser: argparse.ArgumentParser, configuration:
value = action.const or \
next(iter(action.choices or []), None) or \
(not action.default if isinstance(action.default, bool) else None) or \
(42 if action.type == int else None) or \
"random string"
if action.type is not None:
value = action.type(value)

View File

@ -47,6 +47,16 @@ def test_parser_option_log_handler(parser: argparse.ArgumentParser) -> None:
assert isinstance(args.log_handler, LogHandler)
def test_parser_option_wait_timeout(parser: argparse.ArgumentParser) -> None:
"""
must convert wait-timeout option to int instance
"""
args = parser.parse_args(["service-config"])
assert isinstance(args.wait_timeout, int)
args = parser.parse_args(["--wait-timeout", "60", "service-config"])
assert isinstance(args.wait_timeout, int)
def test_multiple_architectures(parser: argparse.ArgumentParser) -> None:
"""
must accept multiple architectures

View File

@ -154,12 +154,49 @@ def test_create_unsafe(lock: Lock) -> None:
lock.path.unlink()
def test_watch(lock: Lock, mocker: MockerFixture) -> None:
"""
must check if lock file exists in cycle
"""
mocker.patch("pathlib.Path.is_file", return_value=False)
lock.watch()
def test_watch_wait(lock: Lock, mocker: MockerFixture) -> None:
"""
must wait until file will disappear
"""
mocker.patch("pathlib.Path.is_file", side_effect=[True, False])
lock.path = Path(tempfile.mktemp()) # nosec
lock.wait_timeout = 1
lock.watch(1)
def test_watch_empty_timeout(lock: Lock, mocker: MockerFixture) -> None:
"""
must skip watch on empty timeout
"""
mocker.patch("pathlib.Path.is_file", return_value=True)
lock.path = Path(tempfile.mktemp()) # nosec
lock.watch()
def test_watch_skip(lock: Lock, mocker: MockerFixture) -> None:
"""
must skip watch on empty path
"""
mocker.patch("pathlib.Path.is_file", return_value=True)
lock.watch()
def test_enter(lock: Lock, mocker: MockerFixture) -> None:
"""
must process with context manager
"""
check_user_mock = mocker.patch("ahriman.application.lock.Lock.check_user")
check_version_mock = mocker.patch("ahriman.application.lock.Lock.check_version")
watch_mock = mocker.patch("ahriman.application.lock.Lock.watch")
clear_mock = mocker.patch("ahriman.application.lock.Lock.clear")
create_mock = mocker.patch("ahriman.application.lock.Lock.create")
update_status_mock = mocker.patch("ahriman.core.status.client.Client.update_self")
@ -170,6 +207,7 @@ def test_enter(lock: Lock, mocker: MockerFixture) -> None:
clear_mock.assert_called_once_with()
create_mock.assert_called_once_with()
check_version_mock.assert_called_once_with()
watch_mock.assert_called_once_with()
update_status_mock.assert_has_calls([MockCall(BuildStatusEnum.Building), MockCall(BuildStatusEnum.Success)])

View File

@ -1,9 +1,17 @@
from pytest_mock import MockerFixture
from unittest.mock import MagicMock
from unittest.mock import MagicMock, call as MockCall
from ahriman.core.spawn import Spawn
def test_boolean_action_argument() -> None:
"""
must correctly convert argument to boolean flag
"""
assert Spawn.boolean_action_argument("option", True) == "option"
assert Spawn.boolean_action_argument("option", False) == "no-option"
def test_process(spawner: Spawn) -> None:
"""
must process external process run correctly
@ -15,9 +23,10 @@ def test_process(spawner: Spawn) -> None:
spawner.process(callback, args, spawner.architecture, "id", spawner.queue)
callback.assert_called_once_with(args, spawner.architecture)
(uuid, status) = spawner.queue.get()
(uuid, status, time) = spawner.queue.get()
assert uuid == "id"
assert status
assert time >= 0
assert spawner.queue.empty()
@ -30,9 +39,10 @@ def test_process_error(spawner: Spawn) -> None:
spawner.process(callback, MagicMock(), spawner.architecture, "id", spawner.queue)
(uuid, status) = spawner.queue.get()
(uuid, status, time) = spawner.queue.get()
assert uuid == "id"
assert not status
assert time >= 0
assert spawner.queue.empty()
@ -42,7 +52,7 @@ def test_spawn_process(spawner: Spawn, mocker: MockerFixture) -> None:
"""
start_mock = mocker.patch("multiprocessing.Process.start")
spawner._spawn_process("add", "ahriman", now="", maybe="?", none=None)
assert spawner._spawn_process("add", "ahriman", now="", maybe="?", none=None)
start_mock.assert_called_once_with()
spawner.args_parser.parse_args.assert_called_once_with(
spawner.command_arguments + [
@ -51,12 +61,22 @@ def test_spawn_process(spawner: Spawn, mocker: MockerFixture) -> None:
)
def test_has_process(spawner: Spawn) -> None:
"""
must correctly determine if there is a process
"""
assert not spawner.has_process("id")
spawner.active["id"] = MagicMock()
assert spawner.has_process("id")
def test_key_import(spawner: Spawn, mocker: MockerFixture) -> None:
"""
must call key import
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
spawner.key_import("0xdeadbeaf", None)
assert spawner.key_import("0xdeadbeaf", None)
spawn_mock.assert_called_once_with("service-key-import", "0xdeadbeaf")
@ -65,7 +85,7 @@ def test_key_import_with_server(spawner: Spawn, mocker: MockerFixture) -> None:
must call key import with server specified
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
spawner.key_import("0xdeadbeaf", "keyserver.ubuntu.com")
assert spawner.key_import("0xdeadbeaf", "keyserver.ubuntu.com")
spawn_mock.assert_called_once_with("service-key-import", "0xdeadbeaf", **{"key-server": "keyserver.ubuntu.com"})
@ -74,7 +94,7 @@ def test_packages_add(spawner: Spawn, mocker: MockerFixture) -> None:
must call package addition
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
spawner.packages_add(["ahriman", "linux"], None, now=False)
assert spawner.packages_add(["ahriman", "linux"], None, now=False)
spawn_mock.assert_called_once_with("package-add", "ahriman", "linux", username=None)
@ -83,7 +103,7 @@ def test_packages_add_with_build(spawner: Spawn, mocker: MockerFixture) -> None:
must call package addition with update
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
spawner.packages_add(["ahriman", "linux"], None, now=True)
assert spawner.packages_add(["ahriman", "linux"], None, now=True)
spawn_mock.assert_called_once_with("package-add", "ahriman", "linux", username=None, now="")
@ -92,7 +112,7 @@ def test_packages_add_with_username(spawner: Spawn, mocker: MockerFixture) -> No
must call package addition with username
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
spawner.packages_add(["ahriman", "linux"], "username", now=False)
assert spawner.packages_add(["ahriman", "linux"], "username", now=False)
spawn_mock.assert_called_once_with("package-add", "ahriman", "linux", username="username")
@ -101,7 +121,7 @@ def test_packages_rebuild(spawner: Spawn, mocker: MockerFixture) -> None:
must call package rebuild
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
spawner.packages_rebuild("python", "packager")
assert spawner.packages_rebuild("python", "packager")
spawn_mock.assert_called_once_with("repo-rebuild", **{"depends-on": "python", "username": "packager"})
@ -110,7 +130,7 @@ def test_packages_remove(spawner: Spawn, mocker: MockerFixture) -> None:
must call package removal
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
spawner.packages_remove(["ahriman", "linux"])
assert spawner.packages_remove(["ahriman", "linux"])
spawn_mock.assert_called_once_with("package-remove", "ahriman", "linux")
@ -119,8 +139,26 @@ def test_packages_update(spawner: Spawn, mocker: MockerFixture) -> None:
must call repo update
"""
spawn_mock = mocker.patch("ahriman.core.spawn.Spawn._spawn_process")
spawner.packages_update("packager")
spawn_mock.assert_called_once_with("repo-update", username="packager")
assert spawner.packages_update("packager", aur=True, local=True, manual=True)
args = {"username": "packager", "aur": "", "local": "", "manual": ""}
spawn_mock.assert_called_once_with("repo-update", **args)
spawn_mock.reset_mock()
assert spawner.packages_update("packager", aur=False, local=True, manual=True)
args = {"username": "packager", "no-aur": "", "local": "", "manual": ""}
spawn_mock.assert_called_once_with("repo-update", **args)
spawn_mock.reset_mock()
assert spawner.packages_update("packager", aur=True, local=False, manual=True)
args = {"username": "packager", "aur": "", "no-local": "", "manual": ""}
spawn_mock.assert_called_once_with("repo-update", **args)
spawn_mock.reset_mock()
assert spawner.packages_update("packager", aur=True, local=True, manual=False)
args = {"username": "packager", "aur": "", "local": "", "no-manual": ""}
spawn_mock.assert_called_once_with("repo-update", **args)
spawn_mock.reset_mock()
def test_run(spawner: Spawn, mocker: MockerFixture) -> None:
@ -129,8 +167,8 @@ def test_run(spawner: Spawn, mocker: MockerFixture) -> None:
"""
logging_mock = mocker.patch("logging.Logger.info")
spawner.queue.put(("1", False))
spawner.queue.put(("2", True))
spawner.queue.put(("1", False, 1))
spawner.queue.put(("2", True, 1))
spawner.queue.put(None) # terminate
spawner.run()
@ -144,8 +182,8 @@ def test_run_pop(spawner: Spawn) -> None:
first = spawner.active["1"] = MagicMock()
second = spawner.active["2"] = MagicMock()
spawner.queue.put(("1", False))
spawner.queue.put(("2", True))
spawner.queue.put(("1", False, 1))
spawner.queue.put(("2", True, 1))
spawner.queue.put(None) # terminate
spawner.run()

View File

@ -0,0 +1 @@
# schema testing goes in view class tests

View File

@ -0,0 +1 @@
# schema testing goes in view class tests

View File

@ -0,0 +1 @@
# schema testing goes in view class tests

View File

@ -21,11 +21,12 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
"""
must call post request correctly
"""
add_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_add")
add_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_add", return_value="abc")
user_mock = AsyncMock()
user_mock.return_value = "username"
mocker.patch("ahriman.web.views.base.BaseView.username", side_effect=user_mock)
request_schema = pytest.helpers.schema_request(AddView.post)
response_schema = pytest.helpers.schema_response(AddView.post)
payload = {"packages": ["ahriman"]}
assert not request_schema.validate(payload)
@ -33,6 +34,10 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
assert response.ok
add_mock.assert_called_once_with(["ahriman"], "username", now=True)
json = await response.json()
assert json["process_id"] == "abc"
assert not response_schema.validate(json)
async def test_post_empty(client: TestClient, mocker: MockerFixture) -> None:
"""

View File

@ -66,8 +66,9 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
"""
must call post request correctly
"""
import_mock = mocker.patch("ahriman.core.spawn.Spawn.key_import")
import_mock = mocker.patch("ahriman.core.spawn.Spawn.key_import", return_value="abc")
request_schema = pytest.helpers.schema_request(PGPView.post)
response_schema = pytest.helpers.schema_response(PGPView.post)
payload = {"key": "0xdeadbeaf", "server": "keyserver.ubuntu.com"}
assert not request_schema.validate(payload)
@ -75,6 +76,10 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
assert response.ok
import_mock.assert_called_once_with("0xdeadbeaf", "keyserver.ubuntu.com")
json = await response.json()
assert json["process_id"] == "abc"
assert not response_schema.validate(json)
async def test_post_exception(client: TestClient, mocker: MockerFixture) -> None:
"""

View File

@ -0,0 +1,46 @@
import pytest
from aiohttp.test_utils import TestClient
from pytest_mock import MockerFixture
from ahriman.models.user_access import UserAccess
from ahriman.web.views.service.process import ProcessView
async def test_get_permission() -> None:
"""
must return correct permission for the request
"""
for method in ("GET",):
request = pytest.helpers.request("", "", method)
assert await ProcessView.get_permission(request) == UserAccess.Reporter
async def test_get(client: TestClient, mocker: MockerFixture) -> None:
"""
must call post request correctly
"""
process = "abc"
process_mock = mocker.patch("ahriman.core.spawn.Spawn.has_process", return_value=True)
response_schema = pytest.helpers.schema_response(ProcessView.get)
response = await client.get(f"/api/v1/service/process/{process}")
assert response.ok
process_mock.assert_called_once_with(process)
json = await response.json()
assert json["is_alive"]
assert not response_schema.validate(json)
async def test_post_empty(client: TestClient, mocker: MockerFixture) -> None:
"""
must call raise 404 on unknown process
"""
process = "abc"
mocker.patch("ahriman.core.spawn.Spawn.has_process", return_value=False)
response_schema = pytest.helpers.schema_response(ProcessView.get, code=404)
response = await client.get(f"/api/v1/service/process/{process}")
assert response.status == 404
assert not response_schema.validate(await response.json())

View File

@ -21,11 +21,12 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
"""
must call post request correctly
"""
rebuild_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_rebuild")
rebuild_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_rebuild", return_value="abc")
user_mock = AsyncMock()
user_mock.return_value = "username"
mocker.patch("ahriman.web.views.base.BaseView.username", side_effect=user_mock)
request_schema = pytest.helpers.schema_request(RebuildView.post)
response_schema = pytest.helpers.schema_response(RebuildView.post)
payload = {"packages": ["python", "ahriman"]}
assert not request_schema.validate(payload)
@ -33,6 +34,10 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
assert response.ok
rebuild_mock.assert_called_once_with("python", "username")
json = await response.json()
assert json["process_id"] == "abc"
assert not response_schema.validate(json)
async def test_post_exception(client: TestClient, mocker: MockerFixture) -> None:
"""

View File

@ -20,8 +20,9 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
"""
must call post request correctly
"""
remove_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_remove")
remove_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_remove", return_value="abc")
request_schema = pytest.helpers.schema_request(RemoveView.post)
response_schema = pytest.helpers.schema_response(RemoveView.post)
payload = {"packages": ["ahriman"]}
assert not request_schema.validate(payload)
@ -29,6 +30,10 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
assert response.ok
remove_mock.assert_called_once_with(["ahriman"])
json = await response.json()
assert json["process_id"] == "abc"
assert not response_schema.validate(json)
async def test_post_exception(client: TestClient, mocker: MockerFixture) -> None:
"""

View File

@ -21,11 +21,12 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
"""
must call post request correctly
"""
add_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_add")
add_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_add", return_value="abc")
user_mock = AsyncMock()
user_mock.return_value = "username"
mocker.patch("ahriman.web.views.base.BaseView.username", side_effect=user_mock)
request_schema = pytest.helpers.schema_request(RequestView.post)
response_schema = pytest.helpers.schema_response(RequestView.post)
payload = {"packages": ["ahriman"]}
assert not request_schema.validate(payload)
@ -33,6 +34,10 @@ async def test_post(client: TestClient, mocker: MockerFixture) -> None:
assert response.ok
add_mock.assert_called_once_with(["ahriman"], "username", now=False)
json = await response.json()
assert json["process_id"] == "abc"
assert not response_schema.validate(json)
async def test_post_exception(client: TestClient, mocker: MockerFixture) -> None:
"""

View File

@ -1,17 +1,65 @@
import pytest
from aiohttp.test_utils import TestClient
from pytest_mock import MockerFixture
from unittest.mock import AsyncMock
from ahriman.models.user_access import UserAccess
from ahriman.web.views.service.update import UpdateView
async def test_get_permission() -> None:
"""
must return correct permission for the request
"""
for method in ("POST",):
request = pytest.helpers.request("", "", method)
assert await UpdateView.get_permission(request) == UserAccess.Full
async def test_post_update(client: TestClient, mocker: MockerFixture) -> None:
"""
must call post request correctly for alias
"""
update_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_update")
update_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_update", return_value="abc")
user_mock = AsyncMock()
user_mock.return_value = "username"
mocker.patch("ahriman.web.views.base.BaseView.username", side_effect=user_mock)
request_schema = pytest.helpers.schema_request(UpdateView.post)
response_schema = pytest.helpers.schema_response(UpdateView.post)
defaults = {
"aur": True,
"local": True,
"manual": True,
}
for payload in (
{},
{"aur": False},
{"local": False},
{"manual": False},
):
assert not request_schema.validate(payload)
response = await client.post("/api/v1/service/update", json=payload)
assert response.ok
update_mock.assert_called_once_with("username", **(defaults | payload))
update_mock.reset_mock()
json = await response.json()
assert json["process_id"] == "abc"
assert not response_schema.validate(json)
async def test_post_empty(client: TestClient, mocker: MockerFixture) -> None:
"""
must call raise 400 on invalid request
"""
mocker.patch("ahriman.web.views.base.BaseView.extract_data", side_effect=Exception())
update_mock = mocker.patch("ahriman.core.spawn.Spawn.packages_update")
response_schema = pytest.helpers.schema_response(UpdateView.post, code=400)
response = await client.post("/api/v1/service/update")
assert response.ok
update_mock.assert_called_once_with("username")
assert response.status == 400
assert not response_schema.validate(await response.json())
update_mock.assert_not_called()