mirror of
https://github.com/arcan1s/ahriman.git
synced 2025-07-15 15:05:48 +00:00
Remote call trigger support (#105)
* add support of remote task tracking * add remote call trigger implementation * docs update * add cross-service upload * add notes about user * add more ability to control upload * multipart upload with signatures as well as safe file save * configuration reference update * rename watcher methods * erase logs based on current package version Old implementation has used process id instead, but it leads to log removal in case of remote process trigger * add --server flag for setup command * restore behavior of the httploghandler
This commit is contained in:
@ -84,6 +84,10 @@ def _parser() -> argparse.ArgumentParser:
|
||||
parser.add_argument("-q", "--quiet", help="force disable any logging", action="store_true")
|
||||
parser.add_argument("--unsafe", help="allow to run ahriman as non-ahriman user. Some actions might be unavailable",
|
||||
action="store_true")
|
||||
parser.add_argument("--wait-timeout", help="wait for lock to be free. Negative value will lead to "
|
||||
"immediate application run even if there is lock file. "
|
||||
"In case of zero value, tthe application will wait infinitely",
|
||||
type=int, default=-1)
|
||||
parser.add_argument("-V", "--version", action="version", version=__version__)
|
||||
|
||||
subparsers = parser.add_subparsers(title="command", help="command to run", dest="command", required=True)
|
||||
@ -889,6 +893,7 @@ def _set_service_setup_parser(root: SubParserAction) -> argparse.ArgumentParser:
|
||||
action=argparse.BooleanOptionalAction, default=True)
|
||||
parser.add_argument("--packager", help="packager name and email", required=True)
|
||||
parser.add_argument("--repository", help="repository name", required=True)
|
||||
parser.add_argument("--server", help="server to be used for devtools. If none set, local files will be used")
|
||||
parser.add_argument("--sign-key", help="sign key id")
|
||||
parser.add_argument("--sign-target", help="sign options", action="append",
|
||||
type=SignSettings.from_option, choices=enum_values(SignSettings))
|
||||
|
@ -63,8 +63,9 @@ class Setup(Handler):
|
||||
|
||||
Setup.configuration_create_makepkg(args.packager, args.makeflags_jobs, application.repository.paths)
|
||||
Setup.executable_create(application.repository.paths, args.build_command, architecture)
|
||||
repository_server = f"file://{application.repository.paths.repository}" if args.server is None else args.server
|
||||
Setup.configuration_create_devtools(args.build_command, architecture, args.from_configuration, args.mirror,
|
||||
args.multilib, args.repository, application.repository.paths)
|
||||
args.multilib, args.repository, repository_server)
|
||||
Setup.configuration_create_sudo(application.repository.paths, args.build_command, architecture)
|
||||
|
||||
application.repository.repo.init()
|
||||
@ -134,7 +135,7 @@ class Setup(Handler):
|
||||
|
||||
@staticmethod
|
||||
def configuration_create_devtools(prefix: str, architecture: str, source: Path, mirror: str | None,
|
||||
multilib: bool, repository: str, paths: RepositoryPaths) -> None:
|
||||
multilib: bool, repository: str, repository_server: str) -> None:
|
||||
"""
|
||||
create configuration for devtools based on ``source`` configuration
|
||||
|
||||
@ -148,7 +149,7 @@ class Setup(Handler):
|
||||
mirror(str | None): link to package server mirror
|
||||
multilib(bool): add or do not multilib repository to the configuration
|
||||
repository(str): repository name
|
||||
paths(RepositoryPaths): repository paths instance
|
||||
repository_server(str): url of the repository
|
||||
"""
|
||||
# allow_no_value=True is required because pacman uses boolean configuration in which just keys present
|
||||
# (e.g. NoProgressBar) which will lead to exception
|
||||
@ -178,7 +179,7 @@ class Setup(Handler):
|
||||
|
||||
# add repository itself
|
||||
configuration.set_option(repository, "SigLevel", "Never") # we don't care
|
||||
configuration.set_option(repository, "Server", f"file://{paths.repository}")
|
||||
configuration.set_option(repository, "Server", repository_server)
|
||||
|
||||
target = source.parent / f"{prefix}-{architecture}.conf"
|
||||
with target.open("w") as devtools_configuration:
|
||||
|
@ -50,14 +50,14 @@ class Status(Handler):
|
||||
# we are using reporter here
|
||||
client = Application(architecture, configuration, report=True).repository.reporter
|
||||
if args.ahriman:
|
||||
service_status = client.get_internal()
|
||||
service_status = client.status_get()
|
||||
StatusPrinter(service_status.status).print(verbose=args.info)
|
||||
if args.package:
|
||||
packages: list[tuple[Package, BuildStatus]] = sum(
|
||||
(client.get(base) for base in args.package),
|
||||
(client.package_get(base) for base in args.package),
|
||||
start=[])
|
||||
else:
|
||||
packages = client.get(None)
|
||||
packages = client.package_get(None)
|
||||
|
||||
Status.check_if_empty(args.exit_code, not packages)
|
||||
|
||||
|
@ -49,10 +49,10 @@ class StatusUpdate(Handler):
|
||||
if args.action == Action.Update and args.package:
|
||||
# update packages statuses
|
||||
for package in args.package:
|
||||
client.update(package, args.status)
|
||||
client.package_update(package, args.status)
|
||||
elif args.action == Action.Update:
|
||||
# update service status
|
||||
client.update_self(args.status)
|
||||
client.status_update(args.status)
|
||||
elif args.action == Action.Remove:
|
||||
for package in args.package:
|
||||
client.remove(package)
|
||||
client.package_remove(package)
|
||||
|
@ -33,7 +33,6 @@ class Web(Handler):
|
||||
|
||||
ALLOW_AUTO_ARCHITECTURE_RUN = False
|
||||
ALLOW_MULTI_ARCHITECTURE_RUN = False # required to be able to spawn external processes
|
||||
COMMAND_ARGS_WHITELIST = ["force", "log_handler", ""]
|
||||
|
||||
@classmethod
|
||||
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
|
||||
@ -89,3 +88,7 @@ class Web(Handler):
|
||||
yield "--quiet"
|
||||
if args.unsafe:
|
||||
yield "--unsafe"
|
||||
|
||||
# arguments from configuration
|
||||
if (wait_timeout := configuration.getint("web", "wait_timeout", fallback=None)) is not None:
|
||||
yield from ["--wait-timeout", str(wait_timeout)]
|
||||
|
@ -19,6 +19,7 @@
|
||||
#
|
||||
import argparse
|
||||
|
||||
from pathlib import Path
|
||||
from types import TracebackType
|
||||
from typing import Literal, Self
|
||||
|
||||
@ -29,6 +30,7 @@ from ahriman.core.log import LazyLogging
|
||||
from ahriman.core.status.client import Client
|
||||
from ahriman.core.util import check_user
|
||||
from ahriman.models.build_status import BuildStatusEnum
|
||||
from ahriman.models.waiter import Waiter
|
||||
|
||||
|
||||
class Lock(LazyLogging):
|
||||
@ -41,6 +43,7 @@ class Lock(LazyLogging):
|
||||
reporter(Client): build status reporter instance
|
||||
paths(RepositoryPaths): repository paths instance
|
||||
unsafe(bool): skip user check
|
||||
wait_timeout(int): wait in seconds until lock will free
|
||||
|
||||
Examples:
|
||||
Instance of this class except for controlling file-based lock is also required for basic applications checks.
|
||||
@ -65,9 +68,11 @@ class Lock(LazyLogging):
|
||||
architecture(str): repository architecture
|
||||
configuration(Configuration): configuration instance
|
||||
"""
|
||||
self.path = args.lock.with_stem(f"{args.lock.stem}_{architecture}") if args.lock is not None else None
|
||||
self.force = args.force
|
||||
self.unsafe = args.unsafe
|
||||
self.path: Path | None = \
|
||||
args.lock.with_stem(f"{args.lock.stem}_{architecture}") if args.lock is not None else None
|
||||
self.force: bool = args.force
|
||||
self.unsafe: bool = args.unsafe
|
||||
self.wait_timeout: int = args.wait_timeout
|
||||
|
||||
self.paths = configuration.repository_paths
|
||||
self.reporter = Client.load(configuration, report=args.report)
|
||||
@ -76,7 +81,7 @@ class Lock(LazyLogging):
|
||||
"""
|
||||
check web server version
|
||||
"""
|
||||
status = self.reporter.get_internal()
|
||||
status = self.reporter.status_get()
|
||||
if status.version is not None and status.version != __version__:
|
||||
self.logger.warning("status watcher version mismatch, our %s, their %s",
|
||||
__version__, status.version)
|
||||
@ -110,6 +115,19 @@ class Lock(LazyLogging):
|
||||
except FileExistsError:
|
||||
raise DuplicateRunError()
|
||||
|
||||
def watch(self) -> None:
|
||||
"""
|
||||
watch until lock disappear
|
||||
"""
|
||||
# there are reasons why we are not using inotify here. First of all, if we would use it, it would bring to
|
||||
# race conditions because multiple processes will be notified in the same time. Secondly, it is good library,
|
||||
# but platform-specific, and we only need to check if file exists
|
||||
if self.path is None:
|
||||
return
|
||||
|
||||
waiter = Waiter(self.wait_timeout)
|
||||
waiter.wait(self.path.is_file)
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
"""
|
||||
default workflow is the following:
|
||||
@ -117,16 +135,18 @@ class Lock(LazyLogging):
|
||||
1. Check user UID
|
||||
2. Check if there is lock file
|
||||
3. Check web status watcher status
|
||||
4. Create lock file and directory tree
|
||||
5. Report to status page if enabled
|
||||
4. Wait for lock file to be free
|
||||
5. Create lock file and directory tree
|
||||
6. Report to status page if enabled
|
||||
|
||||
Returns:
|
||||
Self: always instance of self
|
||||
"""
|
||||
self.check_user()
|
||||
self.check_version()
|
||||
self.watch()
|
||||
self.create()
|
||||
self.reporter.update_self(BuildStatusEnum.Building)
|
||||
self.reporter.status_update(BuildStatusEnum.Building)
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type: type[Exception] | None, exc_val: Exception | None,
|
||||
@ -144,5 +164,5 @@ class Lock(LazyLogging):
|
||||
"""
|
||||
self.clear()
|
||||
status = BuildStatusEnum.Success if exc_val is None else BuildStatusEnum.Failed
|
||||
self.reporter.update_self(status)
|
||||
self.reporter.status_update(status)
|
||||
return False
|
||||
|
@ -228,6 +228,10 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
|
||||
"coerce": "list",
|
||||
"schema": {"type": "string"},
|
||||
},
|
||||
"enable_archive_upload": {
|
||||
"type": "boolean",
|
||||
"coerce": "boolean",
|
||||
},
|
||||
"host": {
|
||||
"type": "string",
|
||||
"is_ip_address": ["localhost"],
|
||||
@ -236,6 +240,11 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
|
||||
"type": "string",
|
||||
"is_url": ["http", "https"],
|
||||
},
|
||||
"max_body_size": {
|
||||
"type": "integer",
|
||||
"coerce": "integer",
|
||||
"min": 0,
|
||||
},
|
||||
"password": {
|
||||
"type": "string",
|
||||
},
|
||||
@ -268,6 +277,10 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
|
||||
"username": {
|
||||
"type": "string",
|
||||
},
|
||||
"wait_timeout": {
|
||||
"type": "integer",
|
||||
"coerce": "integer",
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
@ -0,0 +1,36 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
__all__ = ["steps"]
|
||||
|
||||
|
||||
steps = [
|
||||
"""
|
||||
drop index logs_package_base_process_id
|
||||
""",
|
||||
"""
|
||||
alter table logs drop column process_id
|
||||
""",
|
||||
"""
|
||||
alter table logs add column version text not null default ''
|
||||
""",
|
||||
"""
|
||||
create index logs_package_base_version on logs (package_base, version)
|
||||
""",
|
||||
]
|
@ -66,13 +66,13 @@ class LogsOperations(Operations):
|
||||
connection.execute(
|
||||
"""
|
||||
insert into logs
|
||||
(package_base, process_id, created, record)
|
||||
(package_base, version, created, record)
|
||||
values
|
||||
(:package_base, :process_id, :created, :record)
|
||||
(:package_base, :version, :created, :record)
|
||||
""",
|
||||
{
|
||||
"package_base": log_record_id.package_base,
|
||||
"process_id": log_record_id.process_id,
|
||||
"version": log_record_id.version,
|
||||
"created": created,
|
||||
"record": record,
|
||||
}
|
||||
@ -80,22 +80,22 @@ class LogsOperations(Operations):
|
||||
|
||||
return self.with_connection(run, commit=True)
|
||||
|
||||
def logs_remove(self, package_base: str, current_process_id: int | None) -> None:
|
||||
def logs_remove(self, package_base: str, version: str | None) -> None:
|
||||
"""
|
||||
remove log records for the specified package
|
||||
|
||||
Args:
|
||||
package_base(str): package base to remove logs
|
||||
current_process_id(int | None): current process id. If set it will remove only logs belonging to another
|
||||
process
|
||||
version(str): package version. If set it will remove only logs belonging to another
|
||||
version
|
||||
"""
|
||||
def run(connection: Connection) -> None:
|
||||
connection.execute(
|
||||
"""
|
||||
delete from logs
|
||||
where package_base = :package_base and (:process_id is null or process_id <> :process_id)
|
||||
where package_base = :package_base and (:version is null or version <> :version)
|
||||
""",
|
||||
{"package_base": package_base, "process_id": current_process_id}
|
||||
{"package_base": package_base, "version": version}
|
||||
)
|
||||
|
||||
return self.with_connection(run, commit=True)
|
||||
|
@ -30,13 +30,14 @@ class FilteredAccessLogger(AccessLogger):
|
||||
LOG_PATH_REGEX(re.Pattern): (class attribute) regex for logs uri
|
||||
"""
|
||||
|
||||
# official packages have only ``[A-Za-z0-9_.+-]`` regex
|
||||
LOG_PATH_REGEX = re.compile(r"^/api/v1/packages/[A-Za-z0-9_.+%-]+/logs$")
|
||||
LOG_PATH_REGEX = re.compile(r"^/api/v1/packages/[^/]+/logs$")
|
||||
# technically process id is uuid, but we might change it later
|
||||
PROCESS_PATH_REGEX = re.compile(r"^/api/v1/service/process/[^/]+$")
|
||||
|
||||
@staticmethod
|
||||
def is_logs_post(request: BaseRequest) -> bool:
|
||||
"""
|
||||
check if request looks lie logs posting
|
||||
check if request looks like logs posting
|
||||
|
||||
Args:
|
||||
request(BaseRequest): http reqeust descriptor
|
||||
@ -46,6 +47,19 @@ class FilteredAccessLogger(AccessLogger):
|
||||
"""
|
||||
return request.method == "POST" and FilteredAccessLogger.LOG_PATH_REGEX.match(request.path) is not None
|
||||
|
||||
@staticmethod
|
||||
def is_process_get(request: BaseRequest) -> bool:
|
||||
"""
|
||||
check if request looks like process status request
|
||||
|
||||
Args:
|
||||
request(BaseRequest): http reqeust descriptor
|
||||
|
||||
Returns:
|
||||
bool: True in case if request looks like process status request and False otherwise
|
||||
"""
|
||||
return request.method == "GET" and FilteredAccessLogger.PROCESS_PATH_REGEX.match(request.path) is not None
|
||||
|
||||
def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
|
||||
"""
|
||||
access log with enabled filter by request path
|
||||
@ -55,6 +69,7 @@ class FilteredAccessLogger(AccessLogger):
|
||||
response(StreamResponse): streaming response object
|
||||
time(float):
|
||||
"""
|
||||
if self.is_logs_post(request):
|
||||
if self.is_logs_post(request) \
|
||||
or self.is_process_get(request):
|
||||
return
|
||||
AccessLogger.log(self, request, response, time)
|
||||
|
@ -81,12 +81,12 @@ class HttpLogHandler(logging.Handler):
|
||||
Args:
|
||||
record(logging.LogRecord): log record to log
|
||||
"""
|
||||
package_base = getattr(record, "package_base", None)
|
||||
if package_base is None:
|
||||
log_record_id = getattr(record, "package_id", None)
|
||||
if log_record_id is None:
|
||||
return # in case if no package base supplied we need just skip log message
|
||||
|
||||
try:
|
||||
self.reporter.logs(package_base, record)
|
||||
self.reporter.package_logs(log_record_id, record)
|
||||
except Exception:
|
||||
if self.suppress_errors:
|
||||
return
|
||||
|
@ -24,6 +24,8 @@ from collections.abc import Generator
|
||||
from functools import cached_property
|
||||
from typing import Any
|
||||
|
||||
from ahriman.models.log_record_id import LogRecordId
|
||||
|
||||
|
||||
class LazyLogging:
|
||||
"""
|
||||
@ -60,38 +62,40 @@ class LazyLogging:
|
||||
logging.setLogRecordFactory(logging.LogRecord)
|
||||
|
||||
@staticmethod
|
||||
def _package_logger_set(package_base: str) -> None:
|
||||
def _package_logger_set(package_base: str, version: str | None) -> None:
|
||||
"""
|
||||
set package base as extra info to the logger
|
||||
|
||||
Args:
|
||||
package_base(str): package base
|
||||
version(str | None): package version if available
|
||||
"""
|
||||
current_factory = logging.getLogRecordFactory()
|
||||
|
||||
def package_record_factory(*args: Any, **kwargs: Any) -> logging.LogRecord:
|
||||
record = current_factory(*args, **kwargs)
|
||||
record.package_base = package_base
|
||||
record.package_id = LogRecordId(package_base, version or "")
|
||||
return record
|
||||
|
||||
logging.setLogRecordFactory(package_record_factory)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def in_package_context(self, package_base: str) -> Generator[None, None, None]:
|
||||
def in_package_context(self, package_base: str, version: str | None) -> Generator[None, None, None]:
|
||||
"""
|
||||
execute function while setting package context
|
||||
|
||||
Args:
|
||||
package_base(str): package base to set context in
|
||||
version(str | None): package version if available
|
||||
|
||||
Examples:
|
||||
This function is designed to be called as context manager with ``package_base`` argument, e.g.:
|
||||
|
||||
>>> with self.in_package_context(package.base):
|
||||
>>> with self.in_package_context(package.base, package.version):
|
||||
>>> build_package(package)
|
||||
"""
|
||||
try:
|
||||
self._package_logger_set(package_base)
|
||||
self._package_logger_set(package_base, version)
|
||||
yield
|
||||
finally:
|
||||
self._package_logger_reset()
|
||||
|
119
src/ahriman/core/report/remote_call.py
Normal file
119
src/ahriman/core/report/remote_call.py
Normal file
@ -0,0 +1,119 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import requests
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.report.report import Report
|
||||
from ahriman.core.status.web_client import WebClient
|
||||
from ahriman.models.package import Package
|
||||
from ahriman.models.result import Result
|
||||
from ahriman.models.waiter import Waiter
|
||||
|
||||
|
||||
class RemoteCall(Report):
|
||||
"""
|
||||
trigger implementation which call remote service with update
|
||||
|
||||
Attributes:
|
||||
client(WebClient): web client instance
|
||||
update_aur(bool): check for AUR updates
|
||||
update_local(bool): check for local packages update
|
||||
update_manual(bool): check for manually built packages
|
||||
wait_timeout(int): timeout to wait external process
|
||||
"""
|
||||
|
||||
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
|
||||
"""
|
||||
default constructor
|
||||
|
||||
Args:
|
||||
architecture(str): repository architecture
|
||||
configuration(Configuration): configuration instance
|
||||
section(str): settings section name
|
||||
"""
|
||||
Report.__init__(self, architecture, configuration)
|
||||
|
||||
self.client = WebClient(configuration)
|
||||
|
||||
self.update_aur = configuration.getboolean(section, "aur", fallback=False)
|
||||
self.update_local = configuration.getboolean(section, "local", fallback=False)
|
||||
self.update_manual = configuration.getboolean(section, "manual", fallback=False)
|
||||
|
||||
self.wait_timeout = configuration.getint(section, "wait_timeout", fallback=-1)
|
||||
|
||||
def generate(self, packages: list[Package], result: Result) -> None:
|
||||
"""
|
||||
generate report for the specified packages
|
||||
|
||||
Args:
|
||||
packages(list[Package]): list of packages to generate report
|
||||
result(Result): build result
|
||||
"""
|
||||
process_id = self.remote_update()
|
||||
self.remote_wait(process_id)
|
||||
|
||||
def is_process_alive(self, process_id: str) -> bool:
|
||||
"""
|
||||
check if process is alive
|
||||
|
||||
Args:
|
||||
process_id(str): remote process id
|
||||
|
||||
Returns:
|
||||
bool: True in case if remote process is alive and False otherwise
|
||||
"""
|
||||
try:
|
||||
response = self.client.make_request("GET", f"/api/v1/service/process/{process_id}")
|
||||
except requests.RequestException as e:
|
||||
if e.response is not None and e.response.status_code == 404:
|
||||
return False
|
||||
raise
|
||||
|
||||
response_json = response.json()
|
||||
is_alive: bool = response_json["is_alive"]
|
||||
|
||||
return is_alive
|
||||
|
||||
def remote_update(self) -> str:
|
||||
"""
|
||||
call remote server for update
|
||||
|
||||
Returns:
|
||||
str: remote process id
|
||||
"""
|
||||
response = self.client.make_request("POST", "/api/v1/service/update", json={
|
||||
"aur": self.update_aur,
|
||||
"local": self.update_local,
|
||||
"manual": self.update_manual,
|
||||
})
|
||||
response_json = response.json()
|
||||
|
||||
process_id: str = response_json["process_id"]
|
||||
return process_id
|
||||
|
||||
def remote_wait(self, process_id: str) -> None:
|
||||
"""
|
||||
wait for remote process termination
|
||||
|
||||
Args:
|
||||
process_id(str): remote process id
|
||||
"""
|
||||
waiter = Waiter(self.wait_timeout)
|
||||
waiter.wait(self.is_process_alive, process_id)
|
@ -93,6 +93,9 @@ class Report(LazyLogging):
|
||||
if provider == ReportSettings.Telegram:
|
||||
from ahriman.core.report.telegram import Telegram
|
||||
return Telegram(architecture, configuration, section)
|
||||
if provider == ReportSettings.RemoteCall:
|
||||
from ahriman.core.report.remote_call import RemoteCall
|
||||
return RemoteCall(architecture, configuration, section)
|
||||
return Report(architecture, configuration) # should never happen
|
||||
|
||||
def generate(self, packages: list[Package], result: Result) -> None:
|
||||
|
@ -191,6 +191,31 @@ class ReportTrigger(Trigger):
|
||||
},
|
||||
},
|
||||
},
|
||||
"remote-call": {
|
||||
"type": "dict",
|
||||
"schema": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"allowed": ["ahriman", "remote-call"],
|
||||
},
|
||||
"aur": {
|
||||
"type": "boolean",
|
||||
"coerce": "boolean",
|
||||
},
|
||||
"local": {
|
||||
"type": "boolean",
|
||||
"coerce": "boolean",
|
||||
},
|
||||
"manual": {
|
||||
"type": "boolean",
|
||||
"coerce": "boolean",
|
||||
},
|
||||
"wait_timeout": {
|
||||
"type": "integer",
|
||||
"coerce": "integer",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self, architecture: str, configuration: Configuration) -> None:
|
||||
|
@ -93,7 +93,8 @@ class Executor(Cleaner):
|
||||
|
||||
result = Result()
|
||||
for single in updates:
|
||||
with self.in_package_context(single.base), TemporaryDirectory(ignore_cleanup_errors=True) as dir_name:
|
||||
with self.in_package_context(single.base, local_versions.get(single.base)), \
|
||||
TemporaryDirectory(ignore_cleanup_errors=True) as dir_name:
|
||||
try:
|
||||
packager = self.packager(packagers, single.base)
|
||||
build_single(single, Path(dir_name), packager.packager_id)
|
||||
@ -121,7 +122,7 @@ class Executor(Cleaner):
|
||||
self.database.build_queue_clear(package_base)
|
||||
self.database.patches_remove(package_base, [])
|
||||
self.database.logs_remove(package_base, None)
|
||||
self.reporter.remove(package_base) # we only update status page in case of base removal
|
||||
self.reporter.package_remove(package_base) # we only update status page in case of base removal
|
||||
except Exception:
|
||||
self.logger.exception("could not remove base %s", package_base)
|
||||
|
||||
@ -201,14 +202,16 @@ class Executor(Cleaner):
|
||||
package_path = self.paths.repository / safe_filename(name)
|
||||
self.repo.add(package_path)
|
||||
|
||||
current_packages = self.packages()
|
||||
current_packages = {package.base: package for package in self.packages()}
|
||||
local_versions = {package_base: package.version for package_base, package in current_packages.items()}
|
||||
|
||||
removed_packages: list[str] = [] # list of packages which have been removed from the base
|
||||
updates = self.load_archives(packages)
|
||||
packagers = packagers or Packagers()
|
||||
|
||||
result = Result()
|
||||
for local in updates:
|
||||
with self.in_package_context(local.base):
|
||||
with self.in_package_context(local.base, local_versions.get(local.base)):
|
||||
try:
|
||||
packager = self.packager(packagers, local.base)
|
||||
|
||||
@ -218,12 +221,9 @@ class Executor(Cleaner):
|
||||
self.reporter.set_success(local)
|
||||
result.add_success(local)
|
||||
|
||||
current_package_archives = {
|
||||
package
|
||||
for current in current_packages
|
||||
if current.base == local.base
|
||||
for package in current.packages
|
||||
}
|
||||
current_package_archives: set[str] = set()
|
||||
if local.base in current_packages:
|
||||
current_package_archives = set(current_packages[local.base].packages.keys())
|
||||
removed_packages.extend(current_package_archives.difference(local.packages))
|
||||
except Exception:
|
||||
self.reporter.set_failed(local.base)
|
||||
|
@ -66,10 +66,11 @@ class UpdateHandler(Cleaner):
|
||||
continue
|
||||
raise UnknownPackageError(package.base)
|
||||
|
||||
result: list[Package] = []
|
||||
local_versions = {package.base: package.version for package in self.packages()}
|
||||
|
||||
result: list[Package] = []
|
||||
for local in self.packages():
|
||||
with self.in_package_context(local.base):
|
||||
with self.in_package_context(local.base, local_versions.get(local.base)):
|
||||
if not local.remote.is_remote:
|
||||
continue # avoid checking local packages
|
||||
if local.base in self.ignore_list:
|
||||
@ -102,11 +103,12 @@ class UpdateHandler(Cleaner):
|
||||
Returns:
|
||||
list[Package]: list of local packages which are out-of-dated
|
||||
"""
|
||||
result: list[Package] = []
|
||||
packages = {local.base: local for local in self.packages()}
|
||||
local_versions = {package_base: package.version for package_base, package in packages.items()}
|
||||
|
||||
result: list[Package] = []
|
||||
for cache_dir in self.paths.cache.iterdir():
|
||||
with self.in_package_context(cache_dir.name):
|
||||
with self.in_package_context(cache_dir.name, local_versions.get(cache_dir.name)):
|
||||
try:
|
||||
source = RemoteSource(
|
||||
source=PackageSource.Local,
|
||||
|
@ -101,6 +101,19 @@ class GPG(LazyLogging):
|
||||
default_key = configuration.get("sign", "key") if targets else None
|
||||
return targets, default_key
|
||||
|
||||
@staticmethod
|
||||
def signature(filepath: Path) -> Path:
|
||||
"""
|
||||
generate signature name for the file
|
||||
|
||||
Args:
|
||||
filepath(Path): path to the file which will be signed
|
||||
|
||||
Returns:
|
||||
str: path to signature file
|
||||
"""
|
||||
return filepath.parent / f"{filepath.name}.sig"
|
||||
|
||||
def key_download(self, server: str, key: str) -> str:
|
||||
"""
|
||||
download key from public PGP server
|
||||
@ -179,11 +192,11 @@ class GPG(LazyLogging):
|
||||
*GPG.sign_command(path, key),
|
||||
exception=BuildError(path.name),
|
||||
logger=self.logger)
|
||||
return [path, path.parent / f"{path.name}.sig"]
|
||||
return [path, self.signature(path)]
|
||||
|
||||
def process_sign_package(self, path: Path, packager_key: str | None) -> list[Path]:
|
||||
"""
|
||||
sign package if required by configuration
|
||||
sign package if required by configuration and signature doesn't exist
|
||||
|
||||
Args:
|
||||
path(Path): path to file to sign
|
||||
@ -192,6 +205,10 @@ class GPG(LazyLogging):
|
||||
Returns:
|
||||
list[Path]: list of generated files including original file
|
||||
"""
|
||||
if (signature := self.signature(path)).is_file():
|
||||
# the file was already signed before, just use its signature
|
||||
return [path, signature]
|
||||
|
||||
if SignSettings.Packages not in self.targets:
|
||||
return [path]
|
||||
|
||||
|
@ -20,6 +20,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from collections.abc import Callable, Iterable
|
||||
@ -38,7 +39,7 @@ class Spawn(Thread, LazyLogging):
|
||||
active(dict[str, Process]): map of active child processes required to avoid zombies
|
||||
architecture(str): repository architecture
|
||||
command_arguments(list[str]): base command line arguments
|
||||
queue(Queue[tuple[str, bool]]): multiprocessing queue to read updates from processes
|
||||
queue(Queue[tuple[str, bool, int]]): multiprocessing queue to read updates from processes
|
||||
"""
|
||||
|
||||
def __init__(self, args_parser: argparse.ArgumentParser, architecture: str, command_arguments: list[str]) -> None:
|
||||
@ -59,11 +60,25 @@ class Spawn(Thread, LazyLogging):
|
||||
self.lock = Lock()
|
||||
self.active: dict[str, Process] = {}
|
||||
# stupid pylint does not know that it is possible
|
||||
self.queue: Queue[tuple[str, bool] | None] = Queue() # pylint: disable=unsubscriptable-object
|
||||
self.queue: Queue[tuple[str, bool, int] | None] = Queue() # pylint: disable=unsubscriptable-object
|
||||
|
||||
@staticmethod
|
||||
def boolean_action_argument(name: str, value: bool) -> str:
|
||||
"""
|
||||
convert option of given name with value to boolean action argument
|
||||
|
||||
Args:
|
||||
name(str): command line argument name
|
||||
value(bool): command line argument value
|
||||
|
||||
Returns:
|
||||
str: if ``value`` is True, then returns positive flag and negative otherwise
|
||||
"""
|
||||
return name if value else f"no-{name}"
|
||||
|
||||
@staticmethod
|
||||
def process(callback: Callable[[argparse.Namespace, str], bool], args: argparse.Namespace, architecture: str,
|
||||
process_id: str, queue: Queue[tuple[str, bool]]) -> None: # pylint: disable=unsubscriptable-object
|
||||
process_id: str, queue: Queue[tuple[str, bool, int]]) -> None: # pylint: disable=unsubscriptable-object
|
||||
"""
|
||||
helper to run external process
|
||||
|
||||
@ -72,12 +87,17 @@ class Spawn(Thread, LazyLogging):
|
||||
args(argparse.Namespace): command line arguments
|
||||
architecture(str): repository architecture
|
||||
process_id(str): process unique identifier
|
||||
queue(Queue[tuple[str, bool]]): output queue
|
||||
queue(Queue[tuple[str, bool, int]]): output queue
|
||||
"""
|
||||
start_time = time.monotonic()
|
||||
result = callback(args, architecture)
|
||||
queue.put((process_id, result))
|
||||
stop_time = time.monotonic()
|
||||
|
||||
def _spawn_process(self, command: str, *args: str, **kwargs: str | None) -> None:
|
||||
consumed_time = int(1000 * (stop_time - start_time))
|
||||
|
||||
queue.put((process_id, result, consumed_time))
|
||||
|
||||
def _spawn_process(self, command: str, *args: str, **kwargs: str | None) -> str:
|
||||
"""
|
||||
spawn external ahriman process with supplied arguments
|
||||
|
||||
@ -85,6 +105,9 @@ class Spawn(Thread, LazyLogging):
|
||||
command(str): subcommand to run
|
||||
*args(str): positional command arguments
|
||||
**kwargs(str): named command arguments
|
||||
|
||||
Returns:
|
||||
str: spawned process id
|
||||
"""
|
||||
# default arguments
|
||||
arguments = self.command_arguments[:]
|
||||
@ -111,19 +134,36 @@ class Spawn(Thread, LazyLogging):
|
||||
|
||||
with self.lock:
|
||||
self.active[process_id] = process
|
||||
return process_id
|
||||
|
||||
def key_import(self, key: str, server: str | None) -> None:
|
||||
def has_process(self, process_id: str) -> bool:
|
||||
"""
|
||||
check if given process is alive
|
||||
|
||||
Args:
|
||||
process_id(str): process id to be checked as returned by ``Spawn._spawn_process``
|
||||
|
||||
Returns:
|
||||
bool: True in case if process still counts as active and False otherwise
|
||||
"""
|
||||
with self.lock:
|
||||
return process_id in self.active
|
||||
|
||||
def key_import(self, key: str, server: str | None) -> str:
|
||||
"""
|
||||
import key to service cache
|
||||
|
||||
Args:
|
||||
key(str): key to import
|
||||
server(str | None): PGP key server
|
||||
|
||||
Returns:
|
||||
str: spawned process id
|
||||
"""
|
||||
kwargs = {} if server is None else {"key-server": server}
|
||||
self._spawn_process("service-key-import", key, **kwargs)
|
||||
return self._spawn_process("service-key-import", key, **kwargs)
|
||||
|
||||
def packages_add(self, packages: Iterable[str], username: str | None, *, now: bool) -> None:
|
||||
def packages_add(self, packages: Iterable[str], username: str | None, *, now: bool) -> str:
|
||||
"""
|
||||
add packages
|
||||
|
||||
@ -131,48 +171,69 @@ class Spawn(Thread, LazyLogging):
|
||||
packages(Iterable[str]): packages list to add
|
||||
username(str | None): optional override of username for build process
|
||||
now(bool): build packages now
|
||||
|
||||
Returns:
|
||||
str: spawned process id
|
||||
"""
|
||||
kwargs = {"username": username}
|
||||
if now:
|
||||
kwargs["now"] = ""
|
||||
self._spawn_process("package-add", *packages, **kwargs)
|
||||
return self._spawn_process("package-add", *packages, **kwargs)
|
||||
|
||||
def packages_rebuild(self, depends_on: str, username: str | None) -> None:
|
||||
def packages_rebuild(self, depends_on: str, username: str | None) -> str:
|
||||
"""
|
||||
rebuild packages which depend on the specified package
|
||||
|
||||
Args:
|
||||
depends_on(str): packages dependency
|
||||
username(str | None): optional override of username for build process
|
||||
|
||||
Returns:
|
||||
str: spawned process id
|
||||
"""
|
||||
kwargs = {"depends-on": depends_on, "username": username}
|
||||
self._spawn_process("repo-rebuild", **kwargs)
|
||||
return self._spawn_process("repo-rebuild", **kwargs)
|
||||
|
||||
def packages_remove(self, packages: Iterable[str]) -> None:
|
||||
def packages_remove(self, packages: Iterable[str]) -> str:
|
||||
"""
|
||||
remove packages
|
||||
|
||||
Args:
|
||||
packages(Iterable[str]): packages list to remove
|
||||
"""
|
||||
self._spawn_process("package-remove", *packages)
|
||||
|
||||
def packages_update(self, username: str | None) -> None:
|
||||
Returns:
|
||||
str: spawned process id
|
||||
"""
|
||||
return self._spawn_process("package-remove", *packages)
|
||||
|
||||
def packages_update(self, username: str | None, *, aur: bool, local: bool, manual: bool) -> str:
|
||||
"""
|
||||
run full repository update
|
||||
|
||||
Args:
|
||||
username(str | None): optional override of username for build process
|
||||
aur(bool): check for aur updates
|
||||
local(bool): check for local packages updates
|
||||
manual(bool): check for manual packages
|
||||
|
||||
Returns:
|
||||
str: spawned process id
|
||||
"""
|
||||
kwargs = {"username": username}
|
||||
self._spawn_process("repo-update", **kwargs)
|
||||
kwargs = {
|
||||
"username": username,
|
||||
self.boolean_action_argument("aur", aur): "",
|
||||
self.boolean_action_argument("local", local): "",
|
||||
self.boolean_action_argument("manual", manual): "",
|
||||
}
|
||||
return self._spawn_process("repo-update", **kwargs)
|
||||
|
||||
def run(self) -> None:
|
||||
"""
|
||||
thread run method
|
||||
"""
|
||||
for process_id, status in iter(self.queue.get, None):
|
||||
self.logger.info("process %s has been terminated with status %s", process_id, status)
|
||||
for process_id, status, consumed_time in iter(self.queue.get, None):
|
||||
self.logger.info("process %s has been terminated with status %s, consumed time %s",
|
||||
process_id, status, consumed_time / 1000)
|
||||
|
||||
with self.lock:
|
||||
process = self.active.pop(process_id, None)
|
||||
|
@ -24,6 +24,7 @@ import logging
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
|
||||
from ahriman.models.internal_status import InternalStatus
|
||||
from ahriman.models.log_record_id import LogRecordId
|
||||
from ahriman.models.package import Package
|
||||
|
||||
|
||||
@ -60,7 +61,7 @@ class Client:
|
||||
return WebClient(configuration)
|
||||
return Client()
|
||||
|
||||
def add(self, package: Package, status: BuildStatusEnum) -> None:
|
||||
def package_add(self, package: Package, status: BuildStatusEnum) -> None:
|
||||
"""
|
||||
add new package with status
|
||||
|
||||
@ -69,7 +70,7 @@ class Client:
|
||||
status(BuildStatusEnum): current package build status
|
||||
"""
|
||||
|
||||
def get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]:
|
||||
def package_get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]:
|
||||
"""
|
||||
get package status
|
||||
|
||||
@ -82,25 +83,16 @@ class Client:
|
||||
del package_base
|
||||
return []
|
||||
|
||||
def get_internal(self) -> InternalStatus:
|
||||
"""
|
||||
get internal service status
|
||||
|
||||
Returns:
|
||||
InternalStatus: current internal (web) service status
|
||||
"""
|
||||
return InternalStatus(status=BuildStatus())
|
||||
|
||||
def logs(self, package_base: str, record: logging.LogRecord) -> None:
|
||||
def package_logs(self, log_record_id: LogRecordId, record: logging.LogRecord) -> None:
|
||||
"""
|
||||
post log record
|
||||
|
||||
Args:
|
||||
package_base(str) package base
|
||||
log_record_id(LogRecordId): log record id
|
||||
record(logging.LogRecord): log record to post to api
|
||||
"""
|
||||
|
||||
def remove(self, package_base: str) -> None:
|
||||
def package_remove(self, package_base: str) -> None:
|
||||
"""
|
||||
remove packages from watcher
|
||||
|
||||
@ -108,7 +100,7 @@ class Client:
|
||||
package_base(str): package base to remove
|
||||
"""
|
||||
|
||||
def update(self, package_base: str, status: BuildStatusEnum) -> None:
|
||||
def package_update(self, package_base: str, status: BuildStatusEnum) -> None:
|
||||
"""
|
||||
update package build status. Unlike ``add`` it does not update package properties
|
||||
|
||||
@ -117,14 +109,6 @@ class Client:
|
||||
status(BuildStatusEnum): current package build status
|
||||
"""
|
||||
|
||||
def update_self(self, status: BuildStatusEnum) -> None:
|
||||
"""
|
||||
update ahriman status itself
|
||||
|
||||
Args:
|
||||
status(BuildStatusEnum): current ahriman status
|
||||
"""
|
||||
|
||||
def set_building(self, package_base: str) -> None:
|
||||
"""
|
||||
set package status to building
|
||||
@ -132,7 +116,7 @@ class Client:
|
||||
Args:
|
||||
package_base(str): package base to update
|
||||
"""
|
||||
return self.update(package_base, BuildStatusEnum.Building)
|
||||
return self.package_update(package_base, BuildStatusEnum.Building)
|
||||
|
||||
def set_failed(self, package_base: str) -> None:
|
||||
"""
|
||||
@ -141,7 +125,7 @@ class Client:
|
||||
Args:
|
||||
package_base(str): package base to update
|
||||
"""
|
||||
return self.update(package_base, BuildStatusEnum.Failed)
|
||||
return self.package_update(package_base, BuildStatusEnum.Failed)
|
||||
|
||||
def set_pending(self, package_base: str) -> None:
|
||||
"""
|
||||
@ -150,7 +134,7 @@ class Client:
|
||||
Args:
|
||||
package_base(str): package base to update
|
||||
"""
|
||||
return self.update(package_base, BuildStatusEnum.Pending)
|
||||
return self.package_update(package_base, BuildStatusEnum.Pending)
|
||||
|
||||
def set_success(self, package: Package) -> None:
|
||||
"""
|
||||
@ -159,7 +143,7 @@ class Client:
|
||||
Args:
|
||||
package(Package): current package properties
|
||||
"""
|
||||
return self.add(package, BuildStatusEnum.Success)
|
||||
return self.package_add(package, BuildStatusEnum.Success)
|
||||
|
||||
def set_unknown(self, package: Package) -> None:
|
||||
"""
|
||||
@ -168,4 +152,21 @@ class Client:
|
||||
Args:
|
||||
package(Package): current package properties
|
||||
"""
|
||||
return self.add(package, BuildStatusEnum.Unknown)
|
||||
return self.package_add(package, BuildStatusEnum.Unknown)
|
||||
|
||||
def status_get(self) -> InternalStatus:
|
||||
"""
|
||||
get internal service status
|
||||
|
||||
Returns:
|
||||
InternalStatus: current internal (web) service status
|
||||
"""
|
||||
return InternalStatus(status=BuildStatus())
|
||||
|
||||
def status_update(self, status: BuildStatusEnum) -> None:
|
||||
"""
|
||||
update ahriman status itself
|
||||
|
||||
Args:
|
||||
status(BuildStatusEnum): current ahriman status
|
||||
"""
|
||||
|
@ -17,8 +17,6 @@
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import os
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.database import SQLite
|
||||
from ahriman.core.exceptions import UnknownPackageError
|
||||
@ -59,7 +57,7 @@ class Watcher(LazyLogging):
|
||||
self.status = BuildStatus()
|
||||
|
||||
# special variables for updating logs
|
||||
self._last_log_record_id = LogRecordId("", os.getpid())
|
||||
self._last_log_record_id = LogRecordId("", "")
|
||||
|
||||
@property
|
||||
def packages(self) -> list[tuple[Package, BuildStatus]]:
|
||||
@ -71,36 +69,6 @@ class Watcher(LazyLogging):
|
||||
"""
|
||||
return list(self.known.values())
|
||||
|
||||
def get(self, package_base: str) -> tuple[Package, BuildStatus]:
|
||||
"""
|
||||
get current package base build status
|
||||
|
||||
Args:
|
||||
package_base(str): package base
|
||||
|
||||
Returns:
|
||||
tuple[Package, BuildStatus]: package and its status
|
||||
|
||||
Raises:
|
||||
UnknownPackage: if no package found
|
||||
"""
|
||||
try:
|
||||
return self.known[package_base]
|
||||
except KeyError:
|
||||
raise UnknownPackageError(package_base)
|
||||
|
||||
def get_logs(self, package_base: str) -> str:
|
||||
"""
|
||||
extract logs for the package base
|
||||
|
||||
Args:
|
||||
package_base(str): package base
|
||||
|
||||
Returns:
|
||||
str: package logs
|
||||
"""
|
||||
return self.database.logs_get(package_base)
|
||||
|
||||
def load(self) -> None:
|
||||
"""
|
||||
load packages from local repository. In case if last status is known, it will use it
|
||||
@ -117,7 +85,62 @@ class Watcher(LazyLogging):
|
||||
if package.base in self.known:
|
||||
self.known[package.base] = (package, status)
|
||||
|
||||
def remove(self, package_base: str) -> None:
|
||||
def logs_get(self, package_base: str) -> str:
|
||||
"""
|
||||
extract logs for the package base
|
||||
|
||||
Args:
|
||||
package_base(str): package base
|
||||
|
||||
Returns:
|
||||
str: package logs
|
||||
"""
|
||||
return self.database.logs_get(package_base)
|
||||
|
||||
def logs_remove(self, package_base: str, version: str | None) -> None:
|
||||
"""
|
||||
remove package related logs
|
||||
|
||||
Args:
|
||||
package_base(str): package base
|
||||
version(str): package versio
|
||||
"""
|
||||
self.database.logs_remove(package_base, version)
|
||||
|
||||
def logs_update(self, log_record_id: LogRecordId, created: float, record: str) -> None:
|
||||
"""
|
||||
make new log record into database
|
||||
|
||||
Args:
|
||||
log_record_id(LogRecordId): log record id
|
||||
created(float): log created record
|
||||
record(str): log record
|
||||
"""
|
||||
if self._last_log_record_id != log_record_id:
|
||||
# there is new log record, so we remove old ones
|
||||
self.logs_remove(log_record_id.package_base, log_record_id.version)
|
||||
self._last_log_record_id = log_record_id
|
||||
self.database.logs_insert(log_record_id, created, record)
|
||||
|
||||
def package_get(self, package_base: str) -> tuple[Package, BuildStatus]:
|
||||
"""
|
||||
get current package base build status
|
||||
|
||||
Args:
|
||||
package_base(str): package base
|
||||
|
||||
Returns:
|
||||
tuple[Package, BuildStatus]: package and its status
|
||||
|
||||
Raises:
|
||||
UnknownPackage: if no package found
|
||||
"""
|
||||
try:
|
||||
return self.known[package_base]
|
||||
except KeyError:
|
||||
raise UnknownPackageError(package_base)
|
||||
|
||||
def package_remove(self, package_base: str) -> None:
|
||||
"""
|
||||
remove package base from known list if any
|
||||
|
||||
@ -126,19 +149,9 @@ class Watcher(LazyLogging):
|
||||
"""
|
||||
self.known.pop(package_base, None)
|
||||
self.database.package_remove(package_base)
|
||||
self.remove_logs(package_base, None)
|
||||
self.logs_remove(package_base, None)
|
||||
|
||||
def remove_logs(self, package_base: str, current_process_id: int | None) -> None:
|
||||
"""
|
||||
remove package related logs
|
||||
|
||||
Args:
|
||||
package_base(str): package base
|
||||
current_process_id(int | None): current process id
|
||||
"""
|
||||
self.database.logs_remove(package_base, current_process_id)
|
||||
|
||||
def update(self, package_base: str, status: BuildStatusEnum, package: Package | None) -> None:
|
||||
def package_update(self, package_base: str, status: BuildStatusEnum, package: Package | None) -> None:
|
||||
"""
|
||||
update package status and description
|
||||
|
||||
@ -159,22 +172,7 @@ class Watcher(LazyLogging):
|
||||
self.known[package_base] = (package, full_status)
|
||||
self.database.package_update(package, full_status)
|
||||
|
||||
def update_logs(self, log_record_id: LogRecordId, created: float, record: str) -> None:
|
||||
"""
|
||||
make new log record into database
|
||||
|
||||
Args:
|
||||
log_record_id(LogRecordId): log record id
|
||||
created(float): log created record
|
||||
record(str): log record
|
||||
"""
|
||||
if self._last_log_record_id != log_record_id:
|
||||
# there is new log record, so we remove old ones
|
||||
self.remove_logs(log_record_id.package_base, log_record_id.process_id)
|
||||
self._last_log_record_id = log_record_id
|
||||
self.database.logs_insert(log_record_id, created, record)
|
||||
|
||||
def update_self(self, status: BuildStatusEnum) -> None:
|
||||
def status_update(self, status: BuildStatusEnum) -> None:
|
||||
"""
|
||||
update service status
|
||||
|
||||
|
@ -21,7 +21,8 @@ import contextlib
|
||||
import logging
|
||||
import requests
|
||||
|
||||
from collections.abc import Generator
|
||||
from functools import cached_property
|
||||
from typing import Any, IO, Literal
|
||||
from urllib.parse import quote_plus as urlencode
|
||||
|
||||
from ahriman import __version__
|
||||
@ -31,10 +32,15 @@ from ahriman.core.status.client import Client
|
||||
from ahriman.core.util import exception_response_text
|
||||
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
|
||||
from ahriman.models.internal_status import InternalStatus
|
||||
from ahriman.models.log_record_id import LogRecordId
|
||||
from ahriman.models.package import Package
|
||||
from ahriman.models.user import User
|
||||
|
||||
|
||||
# filename, file, content-type, headers
|
||||
MultipartType = tuple[str, IO[bytes], str, dict[str, str]]
|
||||
|
||||
|
||||
class WebClient(Client, LazyLogging):
|
||||
"""
|
||||
build status reporter web client
|
||||
@ -43,8 +49,12 @@ class WebClient(Client, LazyLogging):
|
||||
address(str): address of the web service
|
||||
suppress_errors(bool): suppress logging errors (e.g. if no web server available)
|
||||
user(User | None): web service user descriptor
|
||||
use_unix_socket(bool): use websocket or not
|
||||
"""
|
||||
|
||||
_login_url = "/api/v1/login"
|
||||
_status_url = "/api/v1/status"
|
||||
|
||||
def __init__(self, configuration: Configuration) -> None:
|
||||
"""
|
||||
default constructor
|
||||
@ -52,33 +62,49 @@ class WebClient(Client, LazyLogging):
|
||||
Args:
|
||||
configuration(Configuration): configuration instance
|
||||
"""
|
||||
self.address, use_unix_socket = self.parse_address(configuration)
|
||||
self.address, self.use_unix_socket = self.parse_address(configuration)
|
||||
self.user = User.from_option(
|
||||
configuration.get("web", "username", fallback=None),
|
||||
configuration.get("web", "password", fallback=None))
|
||||
self.suppress_errors = configuration.getboolean("settings", "suppress_http_log_errors", fallback=False)
|
||||
|
||||
self.__session = self._create_session(use_unix_socket=use_unix_socket)
|
||||
|
||||
@property
|
||||
def _login_url(self) -> str:
|
||||
@cached_property
|
||||
def session(self) -> requests.Session:
|
||||
"""
|
||||
get url for the login api
|
||||
get or create session
|
||||
|
||||
Returns:
|
||||
str: full url for web service to log in
|
||||
request.Session: created session object
|
||||
"""
|
||||
return f"{self.address}/api/v1/login"
|
||||
return self._create_session(use_unix_socket=self.use_unix_socket)
|
||||
|
||||
@property
|
||||
def _status_url(self) -> str:
|
||||
@staticmethod
|
||||
def _logs_url(package_base: str) -> str:
|
||||
"""
|
||||
get url for the status api
|
||||
get url for the logs api
|
||||
|
||||
Args:
|
||||
package_base(str): package base
|
||||
|
||||
Returns:
|
||||
str: full url for web service for status
|
||||
str: full url for web service for logs
|
||||
"""
|
||||
return f"{self.address}/api/v1/status"
|
||||
return f"/api/v1/packages/{package_base}/logs"
|
||||
|
||||
@staticmethod
|
||||
def _package_url(package_base: str = "") -> str:
|
||||
"""
|
||||
url generator
|
||||
|
||||
Args:
|
||||
package_base(str, optional): package base to generate url (Default value = "")
|
||||
|
||||
Returns:
|
||||
str: full url of web service for specific package base
|
||||
"""
|
||||
# in case if unix socket is used we need to normalize url
|
||||
suffix = f"/{package_base}" if package_base else ""
|
||||
return f"/api/v1/packages{suffix}"
|
||||
|
||||
@staticmethod
|
||||
def parse_address(configuration: Configuration) -> tuple[str, bool]:
|
||||
@ -102,32 +128,6 @@ class WebClient(Client, LazyLogging):
|
||||
address = f"http://{host}:{port}"
|
||||
return address, False
|
||||
|
||||
@contextlib.contextmanager
|
||||
def __get_session(self, session: requests.Session | None = None) -> Generator[requests.Session, None, None]:
|
||||
"""
|
||||
execute request and handle exceptions
|
||||
|
||||
Args:
|
||||
session(requests.Session | None, optional): session to be used or stored instance property otherwise
|
||||
(Default value = None)
|
||||
|
||||
Yields:
|
||||
requests.Session: session for requests
|
||||
"""
|
||||
try:
|
||||
if session is not None:
|
||||
yield session # use session from arguments
|
||||
else:
|
||||
yield self.__session # use instance generated session
|
||||
except requests.RequestException as e:
|
||||
if self.suppress_errors:
|
||||
return
|
||||
self.logger.exception("could not perform http request: %s", exception_response_text(e))
|
||||
except Exception:
|
||||
if self.suppress_errors:
|
||||
return
|
||||
self.logger.exception("could not perform http request")
|
||||
|
||||
def _create_session(self, *, use_unix_socket: bool) -> requests.Session:
|
||||
"""
|
||||
generate new request session
|
||||
@ -164,38 +164,51 @@ class WebClient(Client, LazyLogging):
|
||||
"username": self.user.username,
|
||||
"password": self.user.password
|
||||
}
|
||||
with contextlib.suppress(Exception):
|
||||
self.make_request("POST", self._login_url, json=payload, session=session)
|
||||
|
||||
with self.__get_session(session):
|
||||
response = session.post(self._login_url, json=payload)
|
||||
def make_request(self, method: Literal["DELETE", "GET", "POST"], url: str, *,
|
||||
params: list[tuple[str, str]] | None = None,
|
||||
json: dict[str, Any] | None = None,
|
||||
files: dict[str, MultipartType] | None = None,
|
||||
session: requests.Session | None = None,
|
||||
suppress_errors: bool | None = None) -> requests.Response:
|
||||
"""
|
||||
perform request with specified parameters
|
||||
|
||||
Args:
|
||||
method(Literal["DELETE", "GET", "POST"]): HTTP method to call
|
||||
url(str): remote url to call
|
||||
params(list[tuple[str, str]] | None, optional): request query parameters (Default value = None)
|
||||
json(dict[str, Any] | None, optional): request json parameters (Default value = None)
|
||||
files(dict[str, MultipartType] | None, optional): multipart upload (Default value = None)
|
||||
session(requests.Session | None, optional): session object if any (Default value = None)
|
||||
suppress_errors(bool | None, optional): suppress logging errors (e.g. if no web server available). If none
|
||||
set, the instance-wide value will be used (Default value = None)
|
||||
|
||||
Returns:
|
||||
requests.Response: response object
|
||||
"""
|
||||
# defaults
|
||||
if suppress_errors is None:
|
||||
suppress_errors = self.suppress_errors
|
||||
if session is None:
|
||||
session = self.session
|
||||
|
||||
try:
|
||||
response = session.request(method, f"{self.address}{url}", params=params, json=json, files=files)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except requests.RequestException as e:
|
||||
if not suppress_errors:
|
||||
self.logger.exception("could not perform http request: %s", exception_response_text(e))
|
||||
raise
|
||||
except Exception:
|
||||
if not suppress_errors:
|
||||
self.logger.exception("could not perform http request")
|
||||
raise
|
||||
|
||||
def _logs_url(self, package_base: str) -> str:
|
||||
"""
|
||||
get url for the logs api
|
||||
|
||||
Args:
|
||||
package_base(str): package base
|
||||
|
||||
Returns:
|
||||
str: full url for web service for logs
|
||||
"""
|
||||
return f"{self.address}/api/v1/packages/{package_base}/logs"
|
||||
|
||||
def _package_url(self, package_base: str = "") -> str:
|
||||
"""
|
||||
url generator
|
||||
|
||||
Args:
|
||||
package_base(str, optional): package base to generate url (Default value = "")
|
||||
|
||||
Returns:
|
||||
str: full url of web service for specific package base
|
||||
"""
|
||||
# in case if unix socket is used we need to normalize url
|
||||
suffix = f"/{package_base}" if package_base else ""
|
||||
return f"{self.address}/api/v1/packages{suffix}"
|
||||
|
||||
def add(self, package: Package, status: BuildStatusEnum) -> None:
|
||||
def package_add(self, package: Package, status: BuildStatusEnum) -> None:
|
||||
"""
|
||||
add new package with status
|
||||
|
||||
@ -207,12 +220,10 @@ class WebClient(Client, LazyLogging):
|
||||
"status": status.value,
|
||||
"package": package.view()
|
||||
}
|
||||
with contextlib.suppress(Exception):
|
||||
self.make_request("POST", self._package_url(package.base), json=payload)
|
||||
|
||||
with self.__get_session() as session:
|
||||
response = session.post(self._package_url(package.base), json=payload)
|
||||
response.raise_for_status()
|
||||
|
||||
def get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]:
|
||||
def package_get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]:
|
||||
"""
|
||||
get package status
|
||||
|
||||
@ -222,66 +233,47 @@ class WebClient(Client, LazyLogging):
|
||||
Returns:
|
||||
list[tuple[Package, BuildStatus]]: list of current package description and status if it has been found
|
||||
"""
|
||||
with self.__get_session() as session:
|
||||
response = session.get(self._package_url(package_base or ""))
|
||||
response.raise_for_status()
|
||||
with contextlib.suppress(Exception):
|
||||
response = self.make_request("GET", self._package_url(package_base or ""))
|
||||
response_json = response.json()
|
||||
|
||||
status_json = response.json()
|
||||
return [
|
||||
(Package.from_json(package["package"]), BuildStatus.from_json(package["status"]))
|
||||
for package in status_json
|
||||
for package in response_json
|
||||
]
|
||||
|
||||
# noinspection PyUnreachableCode
|
||||
return []
|
||||
|
||||
def get_internal(self) -> InternalStatus:
|
||||
"""
|
||||
get internal service status
|
||||
|
||||
Returns:
|
||||
InternalStatus: current internal (web) service status
|
||||
"""
|
||||
with self.__get_session() as session:
|
||||
response = session.get(self._status_url)
|
||||
response.raise_for_status()
|
||||
|
||||
status_json = response.json()
|
||||
return InternalStatus.from_json(status_json)
|
||||
|
||||
# noinspection PyUnreachableCode
|
||||
return InternalStatus(status=BuildStatus())
|
||||
|
||||
def logs(self, package_base: str, record: logging.LogRecord) -> None:
|
||||
def package_logs(self, log_record_id: LogRecordId, record: logging.LogRecord) -> None:
|
||||
"""
|
||||
post log record
|
||||
|
||||
Args:
|
||||
package_base(str) package base
|
||||
log_record_id(LogRecordId): log record id
|
||||
record(logging.LogRecord): log record to post to api
|
||||
"""
|
||||
payload = {
|
||||
"created": record.created,
|
||||
"message": record.getMessage(),
|
||||
"process_id": record.process,
|
||||
"version": log_record_id.version,
|
||||
}
|
||||
|
||||
# in this method exception has to be handled outside in logger handler
|
||||
response = self.__session.post(self._logs_url(package_base), json=payload)
|
||||
response.raise_for_status()
|
||||
# this is special case, because we would like to do not suppress exception here
|
||||
# in case of exception raised it will be handled by upstream HttpLogHandler
|
||||
# In the other hand, we force to suppress all http logs here to avoid cyclic reporting
|
||||
self.make_request("POST", self._logs_url(log_record_id.package_base), json=payload, suppress_errors=True)
|
||||
|
||||
def remove(self, package_base: str) -> None:
|
||||
def package_remove(self, package_base: str) -> None:
|
||||
"""
|
||||
remove packages from watcher
|
||||
|
||||
Args:
|
||||
package_base(str): basename to remove
|
||||
"""
|
||||
with self.__get_session() as session:
|
||||
response = session.delete(self._package_url(package_base))
|
||||
response.raise_for_status()
|
||||
with contextlib.suppress(Exception):
|
||||
self.make_request("DELETE", self._package_url(package_base))
|
||||
|
||||
def update(self, package_base: str, status: BuildStatusEnum) -> None:
|
||||
def package_update(self, package_base: str, status: BuildStatusEnum) -> None:
|
||||
"""
|
||||
update package build status. Unlike ``add`` it does not update package properties
|
||||
|
||||
@ -290,12 +282,25 @@ class WebClient(Client, LazyLogging):
|
||||
status(BuildStatusEnum): current package build status
|
||||
"""
|
||||
payload = {"status": status.value}
|
||||
with contextlib.suppress(Exception):
|
||||
self.make_request("POST", self._package_url(package_base), json=payload)
|
||||
|
||||
with self.__get_session() as session:
|
||||
response = session.post(self._package_url(package_base), json=payload)
|
||||
response.raise_for_status()
|
||||
def status_get(self) -> InternalStatus:
|
||||
"""
|
||||
get internal service status
|
||||
|
||||
def update_self(self, status: BuildStatusEnum) -> None:
|
||||
Returns:
|
||||
InternalStatus: current internal (web) service status
|
||||
"""
|
||||
with contextlib.suppress(Exception):
|
||||
response = self.make_request("GET", self._status_url)
|
||||
response_json = response.json()
|
||||
|
||||
return InternalStatus.from_json(response_json)
|
||||
|
||||
return InternalStatus(status=BuildStatus())
|
||||
|
||||
def status_update(self, status: BuildStatusEnum) -> None:
|
||||
"""
|
||||
update ahriman status itself
|
||||
|
||||
@ -303,7 +308,5 @@ class WebClient(Client, LazyLogging):
|
||||
status(BuildStatusEnum): current ahriman status
|
||||
"""
|
||||
payload = {"status": status.value}
|
||||
|
||||
with self.__get_session() as session:
|
||||
response = session.post(self._status_url, json=payload)
|
||||
response.raise_for_status()
|
||||
with contextlib.suppress(Exception):
|
||||
self.make_request("POST", self._status_url, json=payload)
|
||||
|
@ -20,6 +20,7 @@
|
||||
import hashlib
|
||||
import requests
|
||||
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@ -52,6 +53,16 @@ class HttpUpload(Upload):
|
||||
self.auth = (password, username) if password and username else None
|
||||
self.timeout = configuration.getint(section, "timeout", fallback=30)
|
||||
|
||||
@cached_property
|
||||
def session(self) -> requests.Session:
|
||||
"""
|
||||
get or create session
|
||||
|
||||
Returns:
|
||||
request.Session: created session object
|
||||
"""
|
||||
return requests.Session()
|
||||
|
||||
@staticmethod
|
||||
def calculate_hash(path: Path) -> str:
|
||||
"""
|
||||
@ -110,7 +121,7 @@ class HttpUpload(Upload):
|
||||
requests.Response: request response object
|
||||
"""
|
||||
try:
|
||||
response = requests.request(method, url, auth=self.auth, timeout=self.timeout, **kwargs)
|
||||
response = self.session.request(method, url, auth=self.auth, timeout=self.timeout, **kwargs)
|
||||
response.raise_for_status()
|
||||
except requests.HTTPError as e:
|
||||
self.logger.exception("could not perform %s request to %s: %s", method, url, exception_response_text(e))
|
||||
|
105
src/ahriman/core/upload/remote_service.py
Normal file
105
src/ahriman/core/upload/remote_service.py
Normal file
@ -0,0 +1,105 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import requests
|
||||
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.sign.gpg import GPG
|
||||
from ahriman.core.status.web_client import MultipartType, WebClient
|
||||
from ahriman.core.upload.http_upload import HttpUpload
|
||||
from ahriman.models.package import Package
|
||||
|
||||
|
||||
class RemoteService(HttpUpload):
|
||||
"""
|
||||
upload files to another server instance
|
||||
|
||||
Attributes:
|
||||
client(WebClient): web client instance
|
||||
"""
|
||||
|
||||
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
|
||||
"""
|
||||
default constructor
|
||||
|
||||
Args:
|
||||
architecture(str): repository architecture
|
||||
configuration(Configuration): configuration instance
|
||||
section(str): settings section name
|
||||
"""
|
||||
HttpUpload.__init__(self, architecture, configuration, section)
|
||||
self.client = WebClient(configuration)
|
||||
|
||||
@cached_property
|
||||
def session(self) -> requests.Session:
|
||||
"""
|
||||
get or create session
|
||||
|
||||
Returns:
|
||||
request.Session: created session object
|
||||
"""
|
||||
return self.client.session
|
||||
|
||||
def package_upload(self, path: Path, package: Package) -> None:
|
||||
"""
|
||||
upload single package to remote
|
||||
|
||||
Args:
|
||||
path(Path): local path to sync
|
||||
package(Package): package to upload
|
||||
"""
|
||||
def upload(package_path: Path, signature_path: Path | None) -> None:
|
||||
files: dict[str, MultipartType] = {}
|
||||
|
||||
try:
|
||||
# package part always persists
|
||||
files["package"] = package_path.name, package_path.open("rb"), "application/octet-stream", {}
|
||||
# signature part is optional
|
||||
if signature_path is not None:
|
||||
files["signature"] = signature_path.name, signature_path.open("rb"), "application/octet-stream", {}
|
||||
|
||||
self._request("POST", f"{self.client.address}/api/v1/service/upload", files=files)
|
||||
finally:
|
||||
for _, fd, _, _ in files.values():
|
||||
fd.close()
|
||||
|
||||
for key, descriptor in package.packages.items():
|
||||
if descriptor.filename is None:
|
||||
self.logger.warning("package %s of %s doesn't have filename set", key, package.base)
|
||||
continue
|
||||
|
||||
archive = path / descriptor.filename
|
||||
maybe_signature_path = GPG.signature(archive)
|
||||
signature = maybe_signature_path if maybe_signature_path.is_file() else None
|
||||
|
||||
upload(archive, signature)
|
||||
|
||||
def sync(self, path: Path, built_packages: list[Package]) -> None:
|
||||
"""
|
||||
sync data to remote server
|
||||
|
||||
Args:
|
||||
path(Path): local path to sync
|
||||
built_packages(list[Package]): list of packages which has just been built
|
||||
"""
|
||||
for package in built_packages:
|
||||
self.package_upload(path, package)
|
@ -90,6 +90,9 @@ class Upload(LazyLogging):
|
||||
if provider == UploadSettings.Github:
|
||||
from ahriman.core.upload.github import Github
|
||||
return Github(architecture, configuration, section)
|
||||
if provider == UploadSettings.RemoteService:
|
||||
from ahriman.core.upload.remote_service import RemoteService
|
||||
return RemoteService(architecture, configuration, section)
|
||||
return Upload(architecture, configuration) # should never happen
|
||||
|
||||
def run(self, path: Path, built_packages: list[Package]) -> None:
|
||||
|
@ -92,6 +92,15 @@ class UploadTrigger(Trigger):
|
||||
},
|
||||
},
|
||||
},
|
||||
"remote-service": {
|
||||
"type": "dict",
|
||||
"schema": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"allowed": ["ahriman", "remote-service"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"s3": {
|
||||
"type": "dict",
|
||||
"schema": {
|
||||
|
@ -142,7 +142,6 @@ def check_output(*args: str, exception: Exception | None = None, cwd: Path | Non
|
||||
while selector.get_map(): # while there are unread selectors, keep reading
|
||||
result.extend(poll(selector))
|
||||
|
||||
process.terminate() # make sure that process is terminated
|
||||
status_code = process.wait()
|
||||
if status_code != 0:
|
||||
if exception is not None:
|
||||
@ -280,7 +279,7 @@ def package_like(filename: Path) -> bool:
|
||||
bool: True in case if name contains ``.pkg.`` and not signature, False otherwise
|
||||
"""
|
||||
name = filename.name
|
||||
return ".pkg." in name and not name.endswith(".sig")
|
||||
return not name.startswith(".") and ".pkg." in name and not name.endswith(".sig")
|
||||
|
||||
|
||||
def parse_version(version: str) -> tuple[str | None, str, str]:
|
||||
|
@ -27,8 +27,8 @@ class LogRecordId:
|
||||
|
||||
Attributes:
|
||||
package_base(str): package base for which log record belongs
|
||||
process_id(int): process id from which log record was emitted
|
||||
version(str): package version for which log record belongs
|
||||
"""
|
||||
|
||||
package_base: str
|
||||
process_id: int
|
||||
version: str
|
||||
|
@ -32,6 +32,7 @@ class ReportSettings(str, Enum):
|
||||
Email(ReportSettings): (class attribute) email report generation
|
||||
Console(ReportSettings): (class attribute) print result to console
|
||||
Telegram(ReportSettings): (class attribute) markdown report to telegram channel
|
||||
RemoteCall(ReportSettings): (class attribute) remote ahriman server call
|
||||
"""
|
||||
|
||||
Disabled = "disabled" # for testing purpose
|
||||
@ -39,6 +40,7 @@ class ReportSettings(str, Enum):
|
||||
Email = "email"
|
||||
Console = "console"
|
||||
Telegram = "telegram"
|
||||
RemoteCall = "remote-call"
|
||||
|
||||
@staticmethod
|
||||
def from_option(value: str) -> ReportSettings:
|
||||
@ -59,4 +61,6 @@ class ReportSettings(str, Enum):
|
||||
return ReportSettings.Console
|
||||
if value.lower() in ("telegram",):
|
||||
return ReportSettings.Telegram
|
||||
if value.lower() in ("ahriman", "remote-call",):
|
||||
return ReportSettings.RemoteCall
|
||||
return ReportSettings.Disabled
|
||||
|
@ -31,12 +31,14 @@ class UploadSettings(str, Enum):
|
||||
Rsync(UploadSettings): (class attribute) sync via rsync
|
||||
S3(UploadSettings): (class attribute) sync to Amazon S3
|
||||
Github(UploadSettings): (class attribute) sync to github releases page
|
||||
RemoteService(UploadSettings): (class attribute) sync to another ahriman instance
|
||||
"""
|
||||
|
||||
Disabled = "disabled" # for testing purpose
|
||||
Rsync = "rsync"
|
||||
S3 = "s3"
|
||||
Github = "github"
|
||||
RemoteService = "remote-service"
|
||||
|
||||
@staticmethod
|
||||
def from_option(value: str) -> UploadSettings:
|
||||
@ -55,4 +57,6 @@ class UploadSettings(str, Enum):
|
||||
return UploadSettings.S3
|
||||
if value.lower() in ("github",):
|
||||
return UploadSettings.Github
|
||||
if value.lower() in ("ahriman", "remote-service",):
|
||||
return UploadSettings.RemoteService
|
||||
return UploadSettings.Disabled
|
||||
|
72
src/ahriman/models/waiter.py
Normal file
72
src/ahriman/models/waiter.py
Normal file
@ -0,0 +1,72 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import time
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import ParamSpec
|
||||
|
||||
|
||||
Params = ParamSpec("Params")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Waiter:
|
||||
"""
|
||||
simple waiter implementation
|
||||
|
||||
Attributes:
|
||||
interval(int): interval in seconds between checks
|
||||
start_time(float): monotonic time of the waiter start. More likely must not be assigned explicitly
|
||||
wait_timeout(int): timeout in seconds to wait for. Negative value will result in immediate exit. Zero value
|
||||
means infinite timeout
|
||||
"""
|
||||
|
||||
wait_timeout: int
|
||||
start_time: float = field(default_factory=time.monotonic, kw_only=True)
|
||||
interval: int = field(default=10, kw_only=True)
|
||||
|
||||
def is_timed_out(self) -> bool:
|
||||
"""
|
||||
check if timer is out
|
||||
|
||||
Returns:
|
||||
bool: True in case current monotonic time is more than ``Waiter.start_time`` and
|
||||
``Waiter.wait_timeout`` doesn't equal to 0
|
||||
"""
|
||||
since_start: float = time.monotonic() - self.start_time
|
||||
return self.wait_timeout != 0 and since_start > self.wait_timeout
|
||||
|
||||
def wait(self, in_progress: Callable[Params, bool], *args: Params.args, **kwargs: Params.kwargs) -> float:
|
||||
"""
|
||||
wait until requirements are not met
|
||||
|
||||
Args:
|
||||
in_progress(Callable[Params, bool]): function to check if timer should wait for another cycle
|
||||
*args(Params.args): positional arguments for check call
|
||||
**kwargs(Params.kwargs): keyword arguments for check call
|
||||
|
||||
Returns:
|
||||
float: consumed time in seconds
|
||||
"""
|
||||
while not self.is_timed_out() and in_progress(*args, **kwargs):
|
||||
time.sleep(self.interval)
|
||||
|
||||
return time.monotonic() - self.start_time
|
@ -25,11 +25,13 @@ from ahriman.web.views.api.swagger import SwaggerView
|
||||
from ahriman.web.views.index import IndexView
|
||||
from ahriman.web.views.service.add import AddView
|
||||
from ahriman.web.views.service.pgp import PGPView
|
||||
from ahriman.web.views.service.process import ProcessView
|
||||
from ahriman.web.views.service.rebuild import RebuildView
|
||||
from ahriman.web.views.service.remove import RemoveView
|
||||
from ahriman.web.views.service.request import RequestView
|
||||
from ahriman.web.views.service.search import SearchView
|
||||
from ahriman.web.views.service.update import UpdateView
|
||||
from ahriman.web.views.service.upload import UploadView
|
||||
from ahriman.web.views.status.logs import LogsView
|
||||
from ahriman.web.views.status.package import PackageView
|
||||
from ahriman.web.views.status.packages import PackagesView
|
||||
@ -60,10 +62,12 @@ def setup_routes(application: Application, static_path: Path) -> None:
|
||||
application.router.add_view("/api/v1/service/add", AddView)
|
||||
application.router.add_view("/api/v1/service/pgp", PGPView)
|
||||
application.router.add_view("/api/v1/service/rebuild", RebuildView)
|
||||
application.router.add_view("/api/v1/service/process/{process_id}", ProcessView)
|
||||
application.router.add_view("/api/v1/service/remove", RemoveView)
|
||||
application.router.add_view("/api/v1/service/request", RequestView)
|
||||
application.router.add_view("/api/v1/service/search", SearchView)
|
||||
application.router.add_view("/api/v1/service/update", UpdateView)
|
||||
application.router.add_view("/api/v1/service/upload", UploadView)
|
||||
|
||||
application.router.add_view("/api/v1/packages", PackagesView)
|
||||
application.router.add_view("/api/v1/packages/{package}", PackageView)
|
||||
|
@ -21,6 +21,7 @@ from ahriman.web.schemas.aur_package_schema import AURPackageSchema
|
||||
from ahriman.web.schemas.auth_schema import AuthSchema
|
||||
from ahriman.web.schemas.counters_schema import CountersSchema
|
||||
from ahriman.web.schemas.error_schema import ErrorSchema
|
||||
from ahriman.web.schemas.file_schema import FileSchema
|
||||
from ahriman.web.schemas.internal_status_schema import InternalStatusSchema
|
||||
from ahriman.web.schemas.log_schema import LogSchema
|
||||
from ahriman.web.schemas.login_schema import LoginSchema
|
||||
@ -33,6 +34,9 @@ from ahriman.web.schemas.package_schema import PackageSchema
|
||||
from ahriman.web.schemas.package_status_schema import PackageStatusSimplifiedSchema, PackageStatusSchema
|
||||
from ahriman.web.schemas.pgp_key_id_schema import PGPKeyIdSchema
|
||||
from ahriman.web.schemas.pgp_key_schema import PGPKeySchema
|
||||
from ahriman.web.schemas.process_id_schema import ProcessIdSchema
|
||||
from ahriman.web.schemas.process_schema import ProcessSchema
|
||||
from ahriman.web.schemas.remote_schema import RemoteSchema
|
||||
from ahriman.web.schemas.search_schema import SearchSchema
|
||||
from ahriman.web.schemas.status_schema import StatusSchema
|
||||
from ahriman.web.schemas.update_flags_schema import UpdateFlagsSchema
|
||||
|
30
src/ahriman/web/schemas/file_schema.py
Normal file
30
src/ahriman/web/schemas/file_schema.py
Normal file
@ -0,0 +1,30 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from marshmallow import Schema, fields
|
||||
|
||||
|
||||
class FileSchema(Schema):
|
||||
"""
|
||||
request file upload schema
|
||||
"""
|
||||
|
||||
archive = fields.Field(required=True, metadata={
|
||||
"description": "Package archive to be uploaded",
|
||||
})
|
@ -19,6 +19,8 @@
|
||||
#
|
||||
from marshmallow import Schema, fields
|
||||
|
||||
from ahriman import __version__
|
||||
|
||||
|
||||
class LogSchema(Schema):
|
||||
"""
|
||||
@ -29,9 +31,9 @@ class LogSchema(Schema):
|
||||
"description": "Log record timestamp",
|
||||
"example": 1680537091.233495,
|
||||
})
|
||||
process_id = fields.Integer(required=True, metadata={
|
||||
"description": "Current process id",
|
||||
"example": 42,
|
||||
version = fields.Integer(required=True, metadata={
|
||||
"description": "Package version to tag",
|
||||
"example": __version__,
|
||||
})
|
||||
message = fields.String(required=True, metadata={
|
||||
"description": "Log message",
|
||||
|
31
src/ahriman/web/schemas/process_id_schema.py
Normal file
31
src/ahriman/web/schemas/process_id_schema.py
Normal file
@ -0,0 +1,31 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from marshmallow import Schema, fields
|
||||
|
||||
|
||||
class ProcessIdSchema(Schema):
|
||||
"""
|
||||
request and response spawned process id schema
|
||||
"""
|
||||
|
||||
process_id = fields.String(required=True, metadata={
|
||||
"description": "Spawned process unique ID",
|
||||
"example": "ff456814-5669-4de6-9143-44dbf6f68607",
|
||||
})
|
30
src/ahriman/web/schemas/process_schema.py
Normal file
30
src/ahriman/web/schemas/process_schema.py
Normal file
@ -0,0 +1,30 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from marshmallow import Schema, fields
|
||||
|
||||
|
||||
class ProcessSchema(Schema):
|
||||
"""
|
||||
process status response schema
|
||||
"""
|
||||
|
||||
is_alive = fields.Bool(required=True, metadata={
|
||||
"description": "Is process alive or not",
|
||||
})
|
36
src/ahriman/web/schemas/update_flags_schema.py
Normal file
36
src/ahriman/web/schemas/update_flags_schema.py
Normal file
@ -0,0 +1,36 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from marshmallow import Schema, fields
|
||||
|
||||
|
||||
class UpdateFlagsSchema(Schema):
|
||||
"""
|
||||
update flags request schema
|
||||
"""
|
||||
|
||||
aur = fields.Bool(dump_default=True, metadata={
|
||||
"description": "Check AUR for updates",
|
||||
})
|
||||
local = fields.Bool(dump_default=True, metadata={
|
||||
"description": "Check local packages for updates",
|
||||
})
|
||||
manual = fields.Bool(dump_default=True, metadata={
|
||||
"description": "Check manually built packages",
|
||||
})
|
@ -43,7 +43,7 @@ class SwaggerView(BaseView):
|
||||
Response: 200 with json api specification
|
||||
"""
|
||||
spec = self.request.app["swagger_dict"]
|
||||
is_body_parameter: Callable[[dict[str, str]], bool] = lambda p: p["in"] == "body"
|
||||
is_body_parameter: Callable[[dict[str, str]], bool] = lambda p: p["in"] == "body" or p["in"] == "formData"
|
||||
|
||||
# special workaround because it writes request body to parameters section
|
||||
paths = spec["paths"]
|
||||
@ -56,11 +56,14 @@ class SwaggerView(BaseView):
|
||||
if not body:
|
||||
continue # there were no ``body`` parameters found
|
||||
|
||||
schema = next(iter(body))
|
||||
content_type = "multipart/form-data" if schema["in"] == "formData" else "application/json"
|
||||
|
||||
# there should be only one body parameters
|
||||
method["requestBody"] = {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": next(iter(body))["schema"]
|
||||
content_type: {
|
||||
"schema": schema["schema"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -19,10 +19,10 @@
|
||||
#
|
||||
import aiohttp_apispec # type: ignore[import]
|
||||
|
||||
from aiohttp.web import HTTPBadRequest, HTTPNoContent
|
||||
from aiohttp.web import HTTPBadRequest, Response, json_response
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
|
||||
from ahriman.web.views.base import BaseView
|
||||
|
||||
|
||||
@ -41,7 +41,7 @@ class AddView(BaseView):
|
||||
summary="Add new package",
|
||||
description="Add new package(s) from AUR",
|
||||
responses={
|
||||
204: {"description": "Success response"},
|
||||
200: {"description": "Success response", "schema": ProcessIdSchema},
|
||||
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
@ -51,13 +51,15 @@ class AddView(BaseView):
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
@aiohttp_apispec.json_schema(PackageNamesSchema)
|
||||
async def post(self) -> None:
|
||||
async def post(self) -> Response:
|
||||
"""
|
||||
add new package
|
||||
|
||||
Returns:
|
||||
Response: 200 with spawned process id
|
||||
|
||||
Raises:
|
||||
HTTPBadRequest: if bad data is supplied
|
||||
HTTPNoContent: in case of success response
|
||||
"""
|
||||
try:
|
||||
data = await self.extract_data(["packages"])
|
||||
@ -66,6 +68,6 @@ class AddView(BaseView):
|
||||
raise HTTPBadRequest(reason=str(e))
|
||||
|
||||
username = await self.username()
|
||||
self.spawner.packages_add(packages, username, now=True)
|
||||
process_id = self.spawner.packages_add(packages, username, now=True)
|
||||
|
||||
raise HTTPNoContent()
|
||||
return json_response({"process_id": process_id})
|
||||
|
@ -19,10 +19,10 @@
|
||||
#
|
||||
import aiohttp_apispec # type: ignore[import]
|
||||
|
||||
from aiohttp.web import HTTPBadRequest, HTTPNoContent, HTTPNotFound, Response, json_response
|
||||
from aiohttp.web import HTTPBadRequest, HTTPNotFound, Response, json_response
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, PGPKeyIdSchema, PGPKeySchema
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, PGPKeyIdSchema, PGPKeySchema, ProcessIdSchema
|
||||
from ahriman.web.views.base import BaseView
|
||||
|
||||
|
||||
@ -83,7 +83,7 @@ class PGPView(BaseView):
|
||||
summary="Fetch PGP key",
|
||||
description="Fetch PGP key from the key server",
|
||||
responses={
|
||||
204: {"description": "Success response"},
|
||||
200: {"description": "Success response", "schema": ProcessIdSchema},
|
||||
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
@ -93,13 +93,15 @@ class PGPView(BaseView):
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
@aiohttp_apispec.json_schema(PGPKeyIdSchema)
|
||||
async def post(self) -> None:
|
||||
async def post(self) -> Response:
|
||||
"""
|
||||
store key to the local service environment
|
||||
|
||||
Returns:
|
||||
Response: 200 with spawned process id
|
||||
|
||||
Raises:
|
||||
HTTPBadRequest: if bad data is supplied
|
||||
HTTPNoContent: in case of success response
|
||||
"""
|
||||
data = await self.extract_data()
|
||||
|
||||
@ -108,6 +110,6 @@ class PGPView(BaseView):
|
||||
except Exception as e:
|
||||
raise HTTPBadRequest(reason=str(e))
|
||||
|
||||
self.spawner.key_import(key, data.get("server"))
|
||||
process_id = self.spawner.key_import(key, data.get("server"))
|
||||
|
||||
raise HTTPNoContent()
|
||||
return json_response({"process_id": process_id})
|
||||
|
74
src/ahriman/web/views/service/process.py
Normal file
74
src/ahriman/web/views/service/process.py
Normal file
@ -0,0 +1,74 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import aiohttp_apispec # type: ignore[import]
|
||||
|
||||
from aiohttp.web import HTTPNotFound, Response, json_response
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, ProcessIdSchema, ProcessSchema
|
||||
from ahriman.web.views.base import BaseView
|
||||
|
||||
|
||||
class ProcessView(BaseView):
|
||||
"""
|
||||
Process information web view
|
||||
|
||||
Attributes:
|
||||
GET_PERMISSION(UserAccess): (class attribute) get permissions of self
|
||||
"""
|
||||
|
||||
GET_PERMISSION = UserAccess.Reporter
|
||||
|
||||
@aiohttp_apispec.docs(
|
||||
tags=["Actions"],
|
||||
summary="Get process",
|
||||
description="Get process information",
|
||||
responses={
|
||||
200: {"description": "Success response", "schema": ProcessSchema},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
404: {"description": "Not found", "schema": ErrorSchema},
|
||||
500: {"description": "Internal server error", "schema": ErrorSchema},
|
||||
},
|
||||
security=[{"token": [GET_PERMISSION]}],
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
@aiohttp_apispec.match_info_schema(ProcessIdSchema)
|
||||
async def get(self) -> Response:
|
||||
"""
|
||||
get spawned process status
|
||||
|
||||
Returns:
|
||||
Response: 200 with process information
|
||||
|
||||
Raises:
|
||||
HTTPNotFound: if no process found
|
||||
"""
|
||||
process_id = self.request.match_info["process_id"]
|
||||
|
||||
is_alive = self.spawner.has_process(process_id)
|
||||
if not is_alive:
|
||||
raise HTTPNotFound(reason=f"No process {process_id} found")
|
||||
|
||||
response = {
|
||||
"is_alive": is_alive,
|
||||
}
|
||||
|
||||
return json_response(response)
|
@ -19,10 +19,10 @@
|
||||
#
|
||||
import aiohttp_apispec # type: ignore[import]
|
||||
|
||||
from aiohttp.web import HTTPBadRequest, HTTPNoContent
|
||||
from aiohttp.web import HTTPBadRequest, Response, json_response
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
|
||||
from ahriman.web.views.base import BaseView
|
||||
|
||||
|
||||
@ -41,7 +41,7 @@ class RebuildView(BaseView):
|
||||
summary="Rebuild packages",
|
||||
description="Rebuild packages which depend on specified one",
|
||||
responses={
|
||||
204: {"description": "Success response"},
|
||||
200: {"description": "Success response", "schema": ProcessIdSchema},
|
||||
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
@ -51,13 +51,15 @@ class RebuildView(BaseView):
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
@aiohttp_apispec.json_schema(PackageNamesSchema)
|
||||
async def post(self) -> None:
|
||||
async def post(self) -> Response:
|
||||
"""
|
||||
rebuild packages based on their dependency
|
||||
|
||||
Returns:
|
||||
Response: 200 with spawned process id
|
||||
|
||||
Raises:
|
||||
HTTPBadRequest: if bad data is supplied
|
||||
HTTPNoContent: in case of success response
|
||||
"""
|
||||
try:
|
||||
data = await self.extract_data(["packages"])
|
||||
@ -67,6 +69,6 @@ class RebuildView(BaseView):
|
||||
raise HTTPBadRequest(reason=str(e))
|
||||
|
||||
username = await self.username()
|
||||
self.spawner.packages_rebuild(depends_on, username)
|
||||
process_id = self.spawner.packages_rebuild(depends_on, username)
|
||||
|
||||
raise HTTPNoContent()
|
||||
return json_response({"process_id": process_id})
|
||||
|
@ -19,10 +19,10 @@
|
||||
#
|
||||
import aiohttp_apispec # type: ignore[import]
|
||||
|
||||
from aiohttp.web import HTTPBadRequest, HTTPNoContent
|
||||
from aiohttp.web import HTTPBadRequest, Response, json_response
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
|
||||
from ahriman.web.views.base import BaseView
|
||||
|
||||
|
||||
@ -41,7 +41,7 @@ class RemoveView(BaseView):
|
||||
summary="Remove packages",
|
||||
description="Remove specified packages from the repository",
|
||||
responses={
|
||||
204: {"description": "Success response"},
|
||||
200: {"description": "Success response", "schema": ProcessIdSchema},
|
||||
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
@ -51,13 +51,15 @@ class RemoveView(BaseView):
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
@aiohttp_apispec.json_schema(PackageNamesSchema)
|
||||
async def post(self) -> None:
|
||||
async def post(self) -> Response:
|
||||
"""
|
||||
remove existing packages
|
||||
|
||||
Returns:
|
||||
Response: 200 with spawned process id
|
||||
|
||||
Raises:
|
||||
HTTPBadRequest: if bad data is supplied
|
||||
HTTPNoContent: in case of success response
|
||||
"""
|
||||
try:
|
||||
data = await self.extract_data(["packages"])
|
||||
@ -65,6 +67,6 @@ class RemoveView(BaseView):
|
||||
except Exception as e:
|
||||
raise HTTPBadRequest(reason=str(e))
|
||||
|
||||
self.spawner.packages_remove(packages)
|
||||
process_id = self.spawner.packages_remove(packages)
|
||||
|
||||
raise HTTPNoContent()
|
||||
return json_response({"process_id": process_id})
|
||||
|
@ -19,10 +19,10 @@
|
||||
#
|
||||
import aiohttp_apispec # type: ignore[import]
|
||||
|
||||
from aiohttp.web import HTTPBadRequest, HTTPNoContent
|
||||
from aiohttp.web import HTTPBadRequest, Response, json_response
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
|
||||
from ahriman.web.views.base import BaseView
|
||||
|
||||
|
||||
@ -41,7 +41,7 @@ class RequestView(BaseView):
|
||||
summary="Request new package",
|
||||
description="Request new package(s) to be added from AUR",
|
||||
responses={
|
||||
204: {"description": "Success response"},
|
||||
200: {"description": "Success response", "schema": ProcessIdSchema},
|
||||
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
@ -51,13 +51,15 @@ class RequestView(BaseView):
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
@aiohttp_apispec.json_schema(PackageNamesSchema)
|
||||
async def post(self) -> None:
|
||||
async def post(self) -> Response:
|
||||
"""
|
||||
request to add new package
|
||||
|
||||
Returns:
|
||||
Response: 200 with spawned process id
|
||||
|
||||
Raises:
|
||||
HTTPBadRequest: if bad data is supplied
|
||||
HTTPNoContent: in case of success response
|
||||
"""
|
||||
try:
|
||||
data = await self.extract_data(["packages"])
|
||||
@ -66,6 +68,6 @@ class RequestView(BaseView):
|
||||
raise HTTPBadRequest(reason=str(e))
|
||||
|
||||
username = await self.username()
|
||||
self.spawner.packages_add(packages, username, now=False)
|
||||
process_id = self.spawner.packages_add(packages, username, now=False)
|
||||
|
||||
raise HTTPNoContent()
|
||||
return json_response({"process_id": process_id})
|
||||
|
@ -19,10 +19,10 @@
|
||||
#
|
||||
import aiohttp_apispec # type: ignore[import]
|
||||
|
||||
from aiohttp.web import HTTPNoContent
|
||||
from aiohttp.web import HTTPBadRequest, Response, json_response
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, ProcessIdSchema, UpdateFlagsSchema
|
||||
from ahriman.web.views.base import BaseView
|
||||
|
||||
|
||||
@ -41,7 +41,8 @@ class UpdateView(BaseView):
|
||||
summary="Update packages",
|
||||
description="Run repository update process",
|
||||
responses={
|
||||
204: {"description": "Success response"},
|
||||
200: {"description": "Success response", "schema": ProcessIdSchema},
|
||||
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
500: {"description": "Internal server error", "schema": ErrorSchema},
|
||||
@ -49,14 +50,28 @@ class UpdateView(BaseView):
|
||||
security=[{"token": [POST_PERMISSION]}],
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
async def post(self) -> None:
|
||||
@aiohttp_apispec.json_schema(UpdateFlagsSchema)
|
||||
async def post(self) -> Response:
|
||||
"""
|
||||
run repository update. No parameters supported here
|
||||
|
||||
Raises:
|
||||
HTTPNoContent: in case of success response
|
||||
"""
|
||||
username = await self.username()
|
||||
self.spawner.packages_update(username)
|
||||
Returns:
|
||||
Response: 200 with spawned process id
|
||||
|
||||
raise HTTPNoContent()
|
||||
Raises:
|
||||
HTTPBadRequest: if bad data is supplied
|
||||
"""
|
||||
try:
|
||||
data = await self.extract_data()
|
||||
except Exception as e:
|
||||
raise HTTPBadRequest(reason=str(e))
|
||||
|
||||
username = await self.username()
|
||||
process_id = self.spawner.packages_update(
|
||||
username,
|
||||
aur=data.get("aur", True),
|
||||
local=data.get("local", True),
|
||||
manual=data.get("manual", True),
|
||||
)
|
||||
|
||||
return json_response({"process_id": process_id})
|
||||
|
144
src/ahriman/web/views/service/upload.py
Normal file
144
src/ahriman/web/views/service/upload.py
Normal file
@ -0,0 +1,144 @@
|
||||
#
|
||||
# Copyright (c) 2021-2023 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import aiohttp_apispec # type: ignore[import]
|
||||
import shutil
|
||||
|
||||
from aiohttp import BodyPartReader
|
||||
from aiohttp.web import HTTPBadRequest, HTTPCreated, HTTPNotFound
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
from ahriman.models.user_access import UserAccess
|
||||
from ahriman.web.schemas import AuthSchema, ErrorSchema, FileSchema
|
||||
from ahriman.web.views.base import BaseView
|
||||
|
||||
|
||||
class UploadView(BaseView):
|
||||
"""
|
||||
upload file to repository
|
||||
|
||||
Attributes:
|
||||
POST_PERMISSION(UserAccess): (class attribute) post permissions of self
|
||||
"""
|
||||
|
||||
POST_PERMISSION = UserAccess.Full
|
||||
|
||||
@staticmethod
|
||||
async def save_file(part: BodyPartReader, target: Path, *, max_body_size: int | None = None) -> tuple[str, Path]:
|
||||
"""
|
||||
save file to local cache
|
||||
|
||||
Args:
|
||||
part(BodyPartReader): multipart part to be saved
|
||||
target(Path): path to directory to which file should be saved
|
||||
max_body_size(int | None, optional): max body size in bytes (Default value = None)
|
||||
|
||||
Returns:
|
||||
tuple[str, Path]: map of received filename to its local path
|
||||
|
||||
Raises:
|
||||
HTTPBadRequest: if bad data is supplied
|
||||
"""
|
||||
archive_name = part.filename
|
||||
if archive_name is None:
|
||||
raise HTTPBadRequest(reason="Filename must be set")
|
||||
# some magic inside. We would like to make sure that passed filename is filename
|
||||
# without slashes, dots, etc
|
||||
if Path(archive_name).resolve().name != archive_name:
|
||||
raise HTTPBadRequest(reason="Filename must be valid archive name")
|
||||
|
||||
current_size = 0
|
||||
|
||||
# in order to handle errors automatically we create temporary file for long operation (transfer)
|
||||
# and then copy it to valid location
|
||||
with NamedTemporaryFile() as cache:
|
||||
while True:
|
||||
chunk = await part.read_chunk()
|
||||
if not chunk:
|
||||
break
|
||||
|
||||
current_size += len(chunk)
|
||||
if max_body_size is not None and current_size > max_body_size:
|
||||
raise HTTPBadRequest(reason="Body part is too large")
|
||||
|
||||
cache.write(chunk)
|
||||
|
||||
cache.seek(0) # reset file position
|
||||
|
||||
# and now copy temporary file to target location as hidden file
|
||||
# we put it as hidden in order to make sure that it will not be handled during some random process
|
||||
temporary_output = target / f".{archive_name}"
|
||||
with temporary_output.open("wb") as archive:
|
||||
shutil.copyfileobj(cache, archive)
|
||||
|
||||
return archive_name, temporary_output
|
||||
|
||||
@aiohttp_apispec.docs(
|
||||
tags=["Actions"],
|
||||
summary="Upload package",
|
||||
description="Upload package to local filesystem",
|
||||
responses={
|
||||
201: {"description": "Success response"},
|
||||
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
|
||||
401: {"description": "Authorization required", "schema": ErrorSchema},
|
||||
403: {"description": "Access is forbidden", "schema": ErrorSchema},
|
||||
404: {"description": "Not found", "schema": ErrorSchema},
|
||||
500: {"description": "Internal server error", "schema": ErrorSchema},
|
||||
},
|
||||
security=[{"token": [POST_PERMISSION]}],
|
||||
)
|
||||
@aiohttp_apispec.cookies_schema(AuthSchema)
|
||||
@aiohttp_apispec.form_schema(FileSchema)
|
||||
async def post(self) -> None:
|
||||
"""
|
||||
upload file from another instance to the server
|
||||
|
||||
Raises:
|
||||
HTTPBadRequest: if bad data is supplied
|
||||
HTTPCreated: on success response
|
||||
"""
|
||||
if not self.configuration.getboolean("web", "enable_archive_upload", fallback=False):
|
||||
raise HTTPNotFound()
|
||||
|
||||
try:
|
||||
reader = await self.request.multipart()
|
||||
except Exception as e:
|
||||
raise HTTPBadRequest(reason=str(e))
|
||||
|
||||
max_body_size = self.configuration.getint("web", "max_body_size", fallback=None)
|
||||
target = self.configuration.repository_paths.packages
|
||||
|
||||
files = []
|
||||
while (part := await reader.next()) is not None:
|
||||
if not isinstance(part, BodyPartReader):
|
||||
raise HTTPBadRequest(reason="Invalid multipart message received")
|
||||
|
||||
if part.name not in ("package", "signature"):
|
||||
raise HTTPBadRequest(reason="Multipart field isn't package or signature")
|
||||
|
||||
files.append(await self.save_file(part, target, max_body_size=max_body_size))
|
||||
|
||||
# and now we can rename files, which is relatively fast operation
|
||||
# it is probably good way to call lock here, however
|
||||
for filename, current_location in files:
|
||||
target_location = current_location.parent / filename
|
||||
current_location.rename(target_location)
|
||||
|
||||
raise HTTPCreated()
|
@ -63,7 +63,7 @@ class LogsView(BaseView):
|
||||
HTTPNoContent: on success response
|
||||
"""
|
||||
package_base = self.request.match_info["package"]
|
||||
self.service.remove_logs(package_base, None)
|
||||
self.service.logs_remove(package_base, None)
|
||||
|
||||
raise HTTPNoContent()
|
||||
|
||||
@ -95,10 +95,10 @@ class LogsView(BaseView):
|
||||
package_base = self.request.match_info["package"]
|
||||
|
||||
try:
|
||||
_, status = self.service.get(package_base)
|
||||
_, status = self.service.package_get(package_base)
|
||||
except UnknownPackageError:
|
||||
raise HTTPNotFound()
|
||||
logs = self.service.get_logs(package_base)
|
||||
logs = self.service.logs_get(package_base)
|
||||
|
||||
response = {
|
||||
"package_base": package_base,
|
||||
@ -137,10 +137,10 @@ class LogsView(BaseView):
|
||||
try:
|
||||
created = data["created"]
|
||||
record = data["message"]
|
||||
process_id = data["process_id"]
|
||||
version = data["version"]
|
||||
except Exception as e:
|
||||
raise HTTPBadRequest(reason=str(e))
|
||||
|
||||
self.service.update_logs(LogRecordId(package_base, process_id), created, record)
|
||||
self.service.logs_update(LogRecordId(package_base, version), created, record)
|
||||
|
||||
raise HTTPNoContent()
|
||||
|
@ -64,7 +64,7 @@ class PackageView(BaseView):
|
||||
HTTPNoContent: on success response
|
||||
"""
|
||||
package_base = self.request.match_info["package"]
|
||||
self.service.remove(package_base)
|
||||
self.service.package_remove(package_base)
|
||||
|
||||
raise HTTPNoContent()
|
||||
|
||||
@ -96,7 +96,7 @@ class PackageView(BaseView):
|
||||
package_base = self.request.match_info["package"]
|
||||
|
||||
try:
|
||||
package, status = self.service.get(package_base)
|
||||
package, status = self.service.package_get(package_base)
|
||||
except UnknownPackageError:
|
||||
raise HTTPNotFound()
|
||||
|
||||
@ -142,7 +142,7 @@ class PackageView(BaseView):
|
||||
raise HTTPBadRequest(reason=str(e))
|
||||
|
||||
try:
|
||||
self.service.update(package_base, status, package)
|
||||
self.service.package_update(package_base, status, package)
|
||||
except UnknownPackageError:
|
||||
raise HTTPBadRequest(reason=f"Package {package_base} is unknown, but no package body set")
|
||||
|
||||
|
@ -102,6 +102,6 @@ class StatusView(BaseView):
|
||||
except Exception as e:
|
||||
raise HTTPBadRequest(reason=str(e))
|
||||
|
||||
self.service.update_self(status)
|
||||
self.service.status_update(status)
|
||||
|
||||
raise HTTPNoContent()
|
||||
|
Reference in New Issue
Block a user