Remote call trigger support (#105)

* add support of remote task tracking
* add remote call trigger implementation
* docs update
* add cross-service upload
* add notes about user
* add more ability to control upload
* multipart upload with signatures as well as safe file save
* configuration reference update
* rename watcher methods
* erase logs based on current package version

Old implementation has used process id instead, but it leads to log
removal in case of remote process trigger

* add --server flag for setup command
* restore behavior of the httploghandler
This commit is contained in:
2023-08-20 03:44:31 +03:00
committed by GitHub
parent 5b172ad20b
commit ad1c0051c4
111 changed files with 2774 additions and 712 deletions

View File

@ -84,6 +84,10 @@ def _parser() -> argparse.ArgumentParser:
parser.add_argument("-q", "--quiet", help="force disable any logging", action="store_true")
parser.add_argument("--unsafe", help="allow to run ahriman as non-ahriman user. Some actions might be unavailable",
action="store_true")
parser.add_argument("--wait-timeout", help="wait for lock to be free. Negative value will lead to "
"immediate application run even if there is lock file. "
"In case of zero value, tthe application will wait infinitely",
type=int, default=-1)
parser.add_argument("-V", "--version", action="version", version=__version__)
subparsers = parser.add_subparsers(title="command", help="command to run", dest="command", required=True)
@ -889,6 +893,7 @@ def _set_service_setup_parser(root: SubParserAction) -> argparse.ArgumentParser:
action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("--packager", help="packager name and email", required=True)
parser.add_argument("--repository", help="repository name", required=True)
parser.add_argument("--server", help="server to be used for devtools. If none set, local files will be used")
parser.add_argument("--sign-key", help="sign key id")
parser.add_argument("--sign-target", help="sign options", action="append",
type=SignSettings.from_option, choices=enum_values(SignSettings))

View File

@ -63,8 +63,9 @@ class Setup(Handler):
Setup.configuration_create_makepkg(args.packager, args.makeflags_jobs, application.repository.paths)
Setup.executable_create(application.repository.paths, args.build_command, architecture)
repository_server = f"file://{application.repository.paths.repository}" if args.server is None else args.server
Setup.configuration_create_devtools(args.build_command, architecture, args.from_configuration, args.mirror,
args.multilib, args.repository, application.repository.paths)
args.multilib, args.repository, repository_server)
Setup.configuration_create_sudo(application.repository.paths, args.build_command, architecture)
application.repository.repo.init()
@ -134,7 +135,7 @@ class Setup(Handler):
@staticmethod
def configuration_create_devtools(prefix: str, architecture: str, source: Path, mirror: str | None,
multilib: bool, repository: str, paths: RepositoryPaths) -> None:
multilib: bool, repository: str, repository_server: str) -> None:
"""
create configuration for devtools based on ``source`` configuration
@ -148,7 +149,7 @@ class Setup(Handler):
mirror(str | None): link to package server mirror
multilib(bool): add or do not multilib repository to the configuration
repository(str): repository name
paths(RepositoryPaths): repository paths instance
repository_server(str): url of the repository
"""
# allow_no_value=True is required because pacman uses boolean configuration in which just keys present
# (e.g. NoProgressBar) which will lead to exception
@ -178,7 +179,7 @@ class Setup(Handler):
# add repository itself
configuration.set_option(repository, "SigLevel", "Never") # we don't care
configuration.set_option(repository, "Server", f"file://{paths.repository}")
configuration.set_option(repository, "Server", repository_server)
target = source.parent / f"{prefix}-{architecture}.conf"
with target.open("w") as devtools_configuration:

View File

@ -50,14 +50,14 @@ class Status(Handler):
# we are using reporter here
client = Application(architecture, configuration, report=True).repository.reporter
if args.ahriman:
service_status = client.get_internal()
service_status = client.status_get()
StatusPrinter(service_status.status).print(verbose=args.info)
if args.package:
packages: list[tuple[Package, BuildStatus]] = sum(
(client.get(base) for base in args.package),
(client.package_get(base) for base in args.package),
start=[])
else:
packages = client.get(None)
packages = client.package_get(None)
Status.check_if_empty(args.exit_code, not packages)

View File

@ -49,10 +49,10 @@ class StatusUpdate(Handler):
if args.action == Action.Update and args.package:
# update packages statuses
for package in args.package:
client.update(package, args.status)
client.package_update(package, args.status)
elif args.action == Action.Update:
# update service status
client.update_self(args.status)
client.status_update(args.status)
elif args.action == Action.Remove:
for package in args.package:
client.remove(package)
client.package_remove(package)

View File

@ -33,7 +33,6 @@ class Web(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False
ALLOW_MULTI_ARCHITECTURE_RUN = False # required to be able to spawn external processes
COMMAND_ARGS_WHITELIST = ["force", "log_handler", ""]
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
@ -89,3 +88,7 @@ class Web(Handler):
yield "--quiet"
if args.unsafe:
yield "--unsafe"
# arguments from configuration
if (wait_timeout := configuration.getint("web", "wait_timeout", fallback=None)) is not None:
yield from ["--wait-timeout", str(wait_timeout)]

View File

@ -19,6 +19,7 @@
#
import argparse
from pathlib import Path
from types import TracebackType
from typing import Literal, Self
@ -29,6 +30,7 @@ from ahriman.core.log import LazyLogging
from ahriman.core.status.client import Client
from ahriman.core.util import check_user
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.waiter import Waiter
class Lock(LazyLogging):
@ -41,6 +43,7 @@ class Lock(LazyLogging):
reporter(Client): build status reporter instance
paths(RepositoryPaths): repository paths instance
unsafe(bool): skip user check
wait_timeout(int): wait in seconds until lock will free
Examples:
Instance of this class except for controlling file-based lock is also required for basic applications checks.
@ -65,9 +68,11 @@ class Lock(LazyLogging):
architecture(str): repository architecture
configuration(Configuration): configuration instance
"""
self.path = args.lock.with_stem(f"{args.lock.stem}_{architecture}") if args.lock is not None else None
self.force = args.force
self.unsafe = args.unsafe
self.path: Path | None = \
args.lock.with_stem(f"{args.lock.stem}_{architecture}") if args.lock is not None else None
self.force: bool = args.force
self.unsafe: bool = args.unsafe
self.wait_timeout: int = args.wait_timeout
self.paths = configuration.repository_paths
self.reporter = Client.load(configuration, report=args.report)
@ -76,7 +81,7 @@ class Lock(LazyLogging):
"""
check web server version
"""
status = self.reporter.get_internal()
status = self.reporter.status_get()
if status.version is not None and status.version != __version__:
self.logger.warning("status watcher version mismatch, our %s, their %s",
__version__, status.version)
@ -110,6 +115,19 @@ class Lock(LazyLogging):
except FileExistsError:
raise DuplicateRunError()
def watch(self) -> None:
"""
watch until lock disappear
"""
# there are reasons why we are not using inotify here. First of all, if we would use it, it would bring to
# race conditions because multiple processes will be notified in the same time. Secondly, it is good library,
# but platform-specific, and we only need to check if file exists
if self.path is None:
return
waiter = Waiter(self.wait_timeout)
waiter.wait(self.path.is_file)
def __enter__(self) -> Self:
"""
default workflow is the following:
@ -117,16 +135,18 @@ class Lock(LazyLogging):
1. Check user UID
2. Check if there is lock file
3. Check web status watcher status
4. Create lock file and directory tree
5. Report to status page if enabled
4. Wait for lock file to be free
5. Create lock file and directory tree
6. Report to status page if enabled
Returns:
Self: always instance of self
"""
self.check_user()
self.check_version()
self.watch()
self.create()
self.reporter.update_self(BuildStatusEnum.Building)
self.reporter.status_update(BuildStatusEnum.Building)
return self
def __exit__(self, exc_type: type[Exception] | None, exc_val: Exception | None,
@ -144,5 +164,5 @@ class Lock(LazyLogging):
"""
self.clear()
status = BuildStatusEnum.Success if exc_val is None else BuildStatusEnum.Failed
self.reporter.update_self(status)
self.reporter.status_update(status)
return False

View File

@ -228,6 +228,10 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"coerce": "list",
"schema": {"type": "string"},
},
"enable_archive_upload": {
"type": "boolean",
"coerce": "boolean",
},
"host": {
"type": "string",
"is_ip_address": ["localhost"],
@ -236,6 +240,11 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"type": "string",
"is_url": ["http", "https"],
},
"max_body_size": {
"type": "integer",
"coerce": "integer",
"min": 0,
},
"password": {
"type": "string",
},
@ -268,6 +277,10 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"username": {
"type": "string",
},
"wait_timeout": {
"type": "integer",
"coerce": "integer",
}
},
},
}

View File

@ -0,0 +1,36 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ["steps"]
steps = [
"""
drop index logs_package_base_process_id
""",
"""
alter table logs drop column process_id
""",
"""
alter table logs add column version text not null default ''
""",
"""
create index logs_package_base_version on logs (package_base, version)
""",
]

View File

@ -66,13 +66,13 @@ class LogsOperations(Operations):
connection.execute(
"""
insert into logs
(package_base, process_id, created, record)
(package_base, version, created, record)
values
(:package_base, :process_id, :created, :record)
(:package_base, :version, :created, :record)
""",
{
"package_base": log_record_id.package_base,
"process_id": log_record_id.process_id,
"version": log_record_id.version,
"created": created,
"record": record,
}
@ -80,22 +80,22 @@ class LogsOperations(Operations):
return self.with_connection(run, commit=True)
def logs_remove(self, package_base: str, current_process_id: int | None) -> None:
def logs_remove(self, package_base: str, version: str | None) -> None:
"""
remove log records for the specified package
Args:
package_base(str): package base to remove logs
current_process_id(int | None): current process id. If set it will remove only logs belonging to another
process
version(str): package version. If set it will remove only logs belonging to another
version
"""
def run(connection: Connection) -> None:
connection.execute(
"""
delete from logs
where package_base = :package_base and (:process_id is null or process_id <> :process_id)
where package_base = :package_base and (:version is null or version <> :version)
""",
{"package_base": package_base, "process_id": current_process_id}
{"package_base": package_base, "version": version}
)
return self.with_connection(run, commit=True)

View File

@ -30,13 +30,14 @@ class FilteredAccessLogger(AccessLogger):
LOG_PATH_REGEX(re.Pattern): (class attribute) regex for logs uri
"""
# official packages have only ``[A-Za-z0-9_.+-]`` regex
LOG_PATH_REGEX = re.compile(r"^/api/v1/packages/[A-Za-z0-9_.+%-]+/logs$")
LOG_PATH_REGEX = re.compile(r"^/api/v1/packages/[^/]+/logs$")
# technically process id is uuid, but we might change it later
PROCESS_PATH_REGEX = re.compile(r"^/api/v1/service/process/[^/]+$")
@staticmethod
def is_logs_post(request: BaseRequest) -> bool:
"""
check if request looks lie logs posting
check if request looks like logs posting
Args:
request(BaseRequest): http reqeust descriptor
@ -46,6 +47,19 @@ class FilteredAccessLogger(AccessLogger):
"""
return request.method == "POST" and FilteredAccessLogger.LOG_PATH_REGEX.match(request.path) is not None
@staticmethod
def is_process_get(request: BaseRequest) -> bool:
"""
check if request looks like process status request
Args:
request(BaseRequest): http reqeust descriptor
Returns:
bool: True in case if request looks like process status request and False otherwise
"""
return request.method == "GET" and FilteredAccessLogger.PROCESS_PATH_REGEX.match(request.path) is not None
def log(self, request: BaseRequest, response: StreamResponse, time: float) -> None:
"""
access log with enabled filter by request path
@ -55,6 +69,7 @@ class FilteredAccessLogger(AccessLogger):
response(StreamResponse): streaming response object
time(float):
"""
if self.is_logs_post(request):
if self.is_logs_post(request) \
or self.is_process_get(request):
return
AccessLogger.log(self, request, response, time)

View File

@ -81,12 +81,12 @@ class HttpLogHandler(logging.Handler):
Args:
record(logging.LogRecord): log record to log
"""
package_base = getattr(record, "package_base", None)
if package_base is None:
log_record_id = getattr(record, "package_id", None)
if log_record_id is None:
return # in case if no package base supplied we need just skip log message
try:
self.reporter.logs(package_base, record)
self.reporter.package_logs(log_record_id, record)
except Exception:
if self.suppress_errors:
return

View File

@ -24,6 +24,8 @@ from collections.abc import Generator
from functools import cached_property
from typing import Any
from ahriman.models.log_record_id import LogRecordId
class LazyLogging:
"""
@ -60,38 +62,40 @@ class LazyLogging:
logging.setLogRecordFactory(logging.LogRecord)
@staticmethod
def _package_logger_set(package_base: str) -> None:
def _package_logger_set(package_base: str, version: str | None) -> None:
"""
set package base as extra info to the logger
Args:
package_base(str): package base
version(str | None): package version if available
"""
current_factory = logging.getLogRecordFactory()
def package_record_factory(*args: Any, **kwargs: Any) -> logging.LogRecord:
record = current_factory(*args, **kwargs)
record.package_base = package_base
record.package_id = LogRecordId(package_base, version or "")
return record
logging.setLogRecordFactory(package_record_factory)
@contextlib.contextmanager
def in_package_context(self, package_base: str) -> Generator[None, None, None]:
def in_package_context(self, package_base: str, version: str | None) -> Generator[None, None, None]:
"""
execute function while setting package context
Args:
package_base(str): package base to set context in
version(str | None): package version if available
Examples:
This function is designed to be called as context manager with ``package_base`` argument, e.g.:
>>> with self.in_package_context(package.base):
>>> with self.in_package_context(package.base, package.version):
>>> build_package(package)
"""
try:
self._package_logger_set(package_base)
self._package_logger_set(package_base, version)
yield
finally:
self._package_logger_reset()

View File

@ -0,0 +1,119 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import requests
from ahriman.core.configuration import Configuration
from ahriman.core.report.report import Report
from ahriman.core.status.web_client import WebClient
from ahriman.models.package import Package
from ahriman.models.result import Result
from ahriman.models.waiter import Waiter
class RemoteCall(Report):
"""
trigger implementation which call remote service with update
Attributes:
client(WebClient): web client instance
update_aur(bool): check for AUR updates
update_local(bool): check for local packages update
update_manual(bool): check for manually built packages
wait_timeout(int): timeout to wait external process
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
configuration(Configuration): configuration instance
section(str): settings section name
"""
Report.__init__(self, architecture, configuration)
self.client = WebClient(configuration)
self.update_aur = configuration.getboolean(section, "aur", fallback=False)
self.update_local = configuration.getboolean(section, "local", fallback=False)
self.update_manual = configuration.getboolean(section, "manual", fallback=False)
self.wait_timeout = configuration.getint(section, "wait_timeout", fallback=-1)
def generate(self, packages: list[Package], result: Result) -> None:
"""
generate report for the specified packages
Args:
packages(list[Package]): list of packages to generate report
result(Result): build result
"""
process_id = self.remote_update()
self.remote_wait(process_id)
def is_process_alive(self, process_id: str) -> bool:
"""
check if process is alive
Args:
process_id(str): remote process id
Returns:
bool: True in case if remote process is alive and False otherwise
"""
try:
response = self.client.make_request("GET", f"/api/v1/service/process/{process_id}")
except requests.RequestException as e:
if e.response is not None and e.response.status_code == 404:
return False
raise
response_json = response.json()
is_alive: bool = response_json["is_alive"]
return is_alive
def remote_update(self) -> str:
"""
call remote server for update
Returns:
str: remote process id
"""
response = self.client.make_request("POST", "/api/v1/service/update", json={
"aur": self.update_aur,
"local": self.update_local,
"manual": self.update_manual,
})
response_json = response.json()
process_id: str = response_json["process_id"]
return process_id
def remote_wait(self, process_id: str) -> None:
"""
wait for remote process termination
Args:
process_id(str): remote process id
"""
waiter = Waiter(self.wait_timeout)
waiter.wait(self.is_process_alive, process_id)

View File

@ -93,6 +93,9 @@ class Report(LazyLogging):
if provider == ReportSettings.Telegram:
from ahriman.core.report.telegram import Telegram
return Telegram(architecture, configuration, section)
if provider == ReportSettings.RemoteCall:
from ahriman.core.report.remote_call import RemoteCall
return RemoteCall(architecture, configuration, section)
return Report(architecture, configuration) # should never happen
def generate(self, packages: list[Package], result: Result) -> None:

View File

@ -191,6 +191,31 @@ class ReportTrigger(Trigger):
},
},
},
"remote-call": {
"type": "dict",
"schema": {
"type": {
"type": "string",
"allowed": ["ahriman", "remote-call"],
},
"aur": {
"type": "boolean",
"coerce": "boolean",
},
"local": {
"type": "boolean",
"coerce": "boolean",
},
"manual": {
"type": "boolean",
"coerce": "boolean",
},
"wait_timeout": {
"type": "integer",
"coerce": "integer",
},
},
}
}
def __init__(self, architecture: str, configuration: Configuration) -> None:

View File

@ -93,7 +93,8 @@ class Executor(Cleaner):
result = Result()
for single in updates:
with self.in_package_context(single.base), TemporaryDirectory(ignore_cleanup_errors=True) as dir_name:
with self.in_package_context(single.base, local_versions.get(single.base)), \
TemporaryDirectory(ignore_cleanup_errors=True) as dir_name:
try:
packager = self.packager(packagers, single.base)
build_single(single, Path(dir_name), packager.packager_id)
@ -121,7 +122,7 @@ class Executor(Cleaner):
self.database.build_queue_clear(package_base)
self.database.patches_remove(package_base, [])
self.database.logs_remove(package_base, None)
self.reporter.remove(package_base) # we only update status page in case of base removal
self.reporter.package_remove(package_base) # we only update status page in case of base removal
except Exception:
self.logger.exception("could not remove base %s", package_base)
@ -201,14 +202,16 @@ class Executor(Cleaner):
package_path = self.paths.repository / safe_filename(name)
self.repo.add(package_path)
current_packages = self.packages()
current_packages = {package.base: package for package in self.packages()}
local_versions = {package_base: package.version for package_base, package in current_packages.items()}
removed_packages: list[str] = [] # list of packages which have been removed from the base
updates = self.load_archives(packages)
packagers = packagers or Packagers()
result = Result()
for local in updates:
with self.in_package_context(local.base):
with self.in_package_context(local.base, local_versions.get(local.base)):
try:
packager = self.packager(packagers, local.base)
@ -218,12 +221,9 @@ class Executor(Cleaner):
self.reporter.set_success(local)
result.add_success(local)
current_package_archives = {
package
for current in current_packages
if current.base == local.base
for package in current.packages
}
current_package_archives: set[str] = set()
if local.base in current_packages:
current_package_archives = set(current_packages[local.base].packages.keys())
removed_packages.extend(current_package_archives.difference(local.packages))
except Exception:
self.reporter.set_failed(local.base)

View File

@ -66,10 +66,11 @@ class UpdateHandler(Cleaner):
continue
raise UnknownPackageError(package.base)
result: list[Package] = []
local_versions = {package.base: package.version for package in self.packages()}
result: list[Package] = []
for local in self.packages():
with self.in_package_context(local.base):
with self.in_package_context(local.base, local_versions.get(local.base)):
if not local.remote.is_remote:
continue # avoid checking local packages
if local.base in self.ignore_list:
@ -102,11 +103,12 @@ class UpdateHandler(Cleaner):
Returns:
list[Package]: list of local packages which are out-of-dated
"""
result: list[Package] = []
packages = {local.base: local for local in self.packages()}
local_versions = {package_base: package.version for package_base, package in packages.items()}
result: list[Package] = []
for cache_dir in self.paths.cache.iterdir():
with self.in_package_context(cache_dir.name):
with self.in_package_context(cache_dir.name, local_versions.get(cache_dir.name)):
try:
source = RemoteSource(
source=PackageSource.Local,

View File

@ -101,6 +101,19 @@ class GPG(LazyLogging):
default_key = configuration.get("sign", "key") if targets else None
return targets, default_key
@staticmethod
def signature(filepath: Path) -> Path:
"""
generate signature name for the file
Args:
filepath(Path): path to the file which will be signed
Returns:
str: path to signature file
"""
return filepath.parent / f"{filepath.name}.sig"
def key_download(self, server: str, key: str) -> str:
"""
download key from public PGP server
@ -179,11 +192,11 @@ class GPG(LazyLogging):
*GPG.sign_command(path, key),
exception=BuildError(path.name),
logger=self.logger)
return [path, path.parent / f"{path.name}.sig"]
return [path, self.signature(path)]
def process_sign_package(self, path: Path, packager_key: str | None) -> list[Path]:
"""
sign package if required by configuration
sign package if required by configuration and signature doesn't exist
Args:
path(Path): path to file to sign
@ -192,6 +205,10 @@ class GPG(LazyLogging):
Returns:
list[Path]: list of generated files including original file
"""
if (signature := self.signature(path)).is_file():
# the file was already signed before, just use its signature
return [path, signature]
if SignSettings.Packages not in self.targets:
return [path]

View File

@ -20,6 +20,7 @@
from __future__ import annotations
import argparse
import time
import uuid
from collections.abc import Callable, Iterable
@ -38,7 +39,7 @@ class Spawn(Thread, LazyLogging):
active(dict[str, Process]): map of active child processes required to avoid zombies
architecture(str): repository architecture
command_arguments(list[str]): base command line arguments
queue(Queue[tuple[str, bool]]): multiprocessing queue to read updates from processes
queue(Queue[tuple[str, bool, int]]): multiprocessing queue to read updates from processes
"""
def __init__(self, args_parser: argparse.ArgumentParser, architecture: str, command_arguments: list[str]) -> None:
@ -59,11 +60,25 @@ class Spawn(Thread, LazyLogging):
self.lock = Lock()
self.active: dict[str, Process] = {}
# stupid pylint does not know that it is possible
self.queue: Queue[tuple[str, bool] | None] = Queue() # pylint: disable=unsubscriptable-object
self.queue: Queue[tuple[str, bool, int] | None] = Queue() # pylint: disable=unsubscriptable-object
@staticmethod
def boolean_action_argument(name: str, value: bool) -> str:
"""
convert option of given name with value to boolean action argument
Args:
name(str): command line argument name
value(bool): command line argument value
Returns:
str: if ``value`` is True, then returns positive flag and negative otherwise
"""
return name if value else f"no-{name}"
@staticmethod
def process(callback: Callable[[argparse.Namespace, str], bool], args: argparse.Namespace, architecture: str,
process_id: str, queue: Queue[tuple[str, bool]]) -> None: # pylint: disable=unsubscriptable-object
process_id: str, queue: Queue[tuple[str, bool, int]]) -> None: # pylint: disable=unsubscriptable-object
"""
helper to run external process
@ -72,12 +87,17 @@ class Spawn(Thread, LazyLogging):
args(argparse.Namespace): command line arguments
architecture(str): repository architecture
process_id(str): process unique identifier
queue(Queue[tuple[str, bool]]): output queue
queue(Queue[tuple[str, bool, int]]): output queue
"""
start_time = time.monotonic()
result = callback(args, architecture)
queue.put((process_id, result))
stop_time = time.monotonic()
def _spawn_process(self, command: str, *args: str, **kwargs: str | None) -> None:
consumed_time = int(1000 * (stop_time - start_time))
queue.put((process_id, result, consumed_time))
def _spawn_process(self, command: str, *args: str, **kwargs: str | None) -> str:
"""
spawn external ahriman process with supplied arguments
@ -85,6 +105,9 @@ class Spawn(Thread, LazyLogging):
command(str): subcommand to run
*args(str): positional command arguments
**kwargs(str): named command arguments
Returns:
str: spawned process id
"""
# default arguments
arguments = self.command_arguments[:]
@ -111,19 +134,36 @@ class Spawn(Thread, LazyLogging):
with self.lock:
self.active[process_id] = process
return process_id
def key_import(self, key: str, server: str | None) -> None:
def has_process(self, process_id: str) -> bool:
"""
check if given process is alive
Args:
process_id(str): process id to be checked as returned by ``Spawn._spawn_process``
Returns:
bool: True in case if process still counts as active and False otherwise
"""
with self.lock:
return process_id in self.active
def key_import(self, key: str, server: str | None) -> str:
"""
import key to service cache
Args:
key(str): key to import
server(str | None): PGP key server
Returns:
str: spawned process id
"""
kwargs = {} if server is None else {"key-server": server}
self._spawn_process("service-key-import", key, **kwargs)
return self._spawn_process("service-key-import", key, **kwargs)
def packages_add(self, packages: Iterable[str], username: str | None, *, now: bool) -> None:
def packages_add(self, packages: Iterable[str], username: str | None, *, now: bool) -> str:
"""
add packages
@ -131,48 +171,69 @@ class Spawn(Thread, LazyLogging):
packages(Iterable[str]): packages list to add
username(str | None): optional override of username for build process
now(bool): build packages now
Returns:
str: spawned process id
"""
kwargs = {"username": username}
if now:
kwargs["now"] = ""
self._spawn_process("package-add", *packages, **kwargs)
return self._spawn_process("package-add", *packages, **kwargs)
def packages_rebuild(self, depends_on: str, username: str | None) -> None:
def packages_rebuild(self, depends_on: str, username: str | None) -> str:
"""
rebuild packages which depend on the specified package
Args:
depends_on(str): packages dependency
username(str | None): optional override of username for build process
Returns:
str: spawned process id
"""
kwargs = {"depends-on": depends_on, "username": username}
self._spawn_process("repo-rebuild", **kwargs)
return self._spawn_process("repo-rebuild", **kwargs)
def packages_remove(self, packages: Iterable[str]) -> None:
def packages_remove(self, packages: Iterable[str]) -> str:
"""
remove packages
Args:
packages(Iterable[str]): packages list to remove
"""
self._spawn_process("package-remove", *packages)
def packages_update(self, username: str | None) -> None:
Returns:
str: spawned process id
"""
return self._spawn_process("package-remove", *packages)
def packages_update(self, username: str | None, *, aur: bool, local: bool, manual: bool) -> str:
"""
run full repository update
Args:
username(str | None): optional override of username for build process
aur(bool): check for aur updates
local(bool): check for local packages updates
manual(bool): check for manual packages
Returns:
str: spawned process id
"""
kwargs = {"username": username}
self._spawn_process("repo-update", **kwargs)
kwargs = {
"username": username,
self.boolean_action_argument("aur", aur): "",
self.boolean_action_argument("local", local): "",
self.boolean_action_argument("manual", manual): "",
}
return self._spawn_process("repo-update", **kwargs)
def run(self) -> None:
"""
thread run method
"""
for process_id, status in iter(self.queue.get, None):
self.logger.info("process %s has been terminated with status %s", process_id, status)
for process_id, status, consumed_time in iter(self.queue.get, None):
self.logger.info("process %s has been terminated with status %s, consumed time %s",
process_id, status, consumed_time / 1000)
with self.lock:
process = self.active.pop(process_id, None)

View File

@ -24,6 +24,7 @@ import logging
from ahriman.core.configuration import Configuration
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.internal_status import InternalStatus
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
@ -60,7 +61,7 @@ class Client:
return WebClient(configuration)
return Client()
def add(self, package: Package, status: BuildStatusEnum) -> None:
def package_add(self, package: Package, status: BuildStatusEnum) -> None:
"""
add new package with status
@ -69,7 +70,7 @@ class Client:
status(BuildStatusEnum): current package build status
"""
def get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]:
def package_get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]:
"""
get package status
@ -82,25 +83,16 @@ class Client:
del package_base
return []
def get_internal(self) -> InternalStatus:
"""
get internal service status
Returns:
InternalStatus: current internal (web) service status
"""
return InternalStatus(status=BuildStatus())
def logs(self, package_base: str, record: logging.LogRecord) -> None:
def package_logs(self, log_record_id: LogRecordId, record: logging.LogRecord) -> None:
"""
post log record
Args:
package_base(str) package base
log_record_id(LogRecordId): log record id
record(logging.LogRecord): log record to post to api
"""
def remove(self, package_base: str) -> None:
def package_remove(self, package_base: str) -> None:
"""
remove packages from watcher
@ -108,7 +100,7 @@ class Client:
package_base(str): package base to remove
"""
def update(self, package_base: str, status: BuildStatusEnum) -> None:
def package_update(self, package_base: str, status: BuildStatusEnum) -> None:
"""
update package build status. Unlike ``add`` it does not update package properties
@ -117,14 +109,6 @@ class Client:
status(BuildStatusEnum): current package build status
"""
def update_self(self, status: BuildStatusEnum) -> None:
"""
update ahriman status itself
Args:
status(BuildStatusEnum): current ahriman status
"""
def set_building(self, package_base: str) -> None:
"""
set package status to building
@ -132,7 +116,7 @@ class Client:
Args:
package_base(str): package base to update
"""
return self.update(package_base, BuildStatusEnum.Building)
return self.package_update(package_base, BuildStatusEnum.Building)
def set_failed(self, package_base: str) -> None:
"""
@ -141,7 +125,7 @@ class Client:
Args:
package_base(str): package base to update
"""
return self.update(package_base, BuildStatusEnum.Failed)
return self.package_update(package_base, BuildStatusEnum.Failed)
def set_pending(self, package_base: str) -> None:
"""
@ -150,7 +134,7 @@ class Client:
Args:
package_base(str): package base to update
"""
return self.update(package_base, BuildStatusEnum.Pending)
return self.package_update(package_base, BuildStatusEnum.Pending)
def set_success(self, package: Package) -> None:
"""
@ -159,7 +143,7 @@ class Client:
Args:
package(Package): current package properties
"""
return self.add(package, BuildStatusEnum.Success)
return self.package_add(package, BuildStatusEnum.Success)
def set_unknown(self, package: Package) -> None:
"""
@ -168,4 +152,21 @@ class Client:
Args:
package(Package): current package properties
"""
return self.add(package, BuildStatusEnum.Unknown)
return self.package_add(package, BuildStatusEnum.Unknown)
def status_get(self) -> InternalStatus:
"""
get internal service status
Returns:
InternalStatus: current internal (web) service status
"""
return InternalStatus(status=BuildStatus())
def status_update(self, status: BuildStatusEnum) -> None:
"""
update ahriman status itself
Args:
status(BuildStatusEnum): current ahriman status
"""

View File

@ -17,8 +17,6 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import os
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.exceptions import UnknownPackageError
@ -59,7 +57,7 @@ class Watcher(LazyLogging):
self.status = BuildStatus()
# special variables for updating logs
self._last_log_record_id = LogRecordId("", os.getpid())
self._last_log_record_id = LogRecordId("", "")
@property
def packages(self) -> list[tuple[Package, BuildStatus]]:
@ -71,36 +69,6 @@ class Watcher(LazyLogging):
"""
return list(self.known.values())
def get(self, package_base: str) -> tuple[Package, BuildStatus]:
"""
get current package base build status
Args:
package_base(str): package base
Returns:
tuple[Package, BuildStatus]: package and its status
Raises:
UnknownPackage: if no package found
"""
try:
return self.known[package_base]
except KeyError:
raise UnknownPackageError(package_base)
def get_logs(self, package_base: str) -> str:
"""
extract logs for the package base
Args:
package_base(str): package base
Returns:
str: package logs
"""
return self.database.logs_get(package_base)
def load(self) -> None:
"""
load packages from local repository. In case if last status is known, it will use it
@ -117,7 +85,62 @@ class Watcher(LazyLogging):
if package.base in self.known:
self.known[package.base] = (package, status)
def remove(self, package_base: str) -> None:
def logs_get(self, package_base: str) -> str:
"""
extract logs for the package base
Args:
package_base(str): package base
Returns:
str: package logs
"""
return self.database.logs_get(package_base)
def logs_remove(self, package_base: str, version: str | None) -> None:
"""
remove package related logs
Args:
package_base(str): package base
version(str): package versio
"""
self.database.logs_remove(package_base, version)
def logs_update(self, log_record_id: LogRecordId, created: float, record: str) -> None:
"""
make new log record into database
Args:
log_record_id(LogRecordId): log record id
created(float): log created record
record(str): log record
"""
if self._last_log_record_id != log_record_id:
# there is new log record, so we remove old ones
self.logs_remove(log_record_id.package_base, log_record_id.version)
self._last_log_record_id = log_record_id
self.database.logs_insert(log_record_id, created, record)
def package_get(self, package_base: str) -> tuple[Package, BuildStatus]:
"""
get current package base build status
Args:
package_base(str): package base
Returns:
tuple[Package, BuildStatus]: package and its status
Raises:
UnknownPackage: if no package found
"""
try:
return self.known[package_base]
except KeyError:
raise UnknownPackageError(package_base)
def package_remove(self, package_base: str) -> None:
"""
remove package base from known list if any
@ -126,19 +149,9 @@ class Watcher(LazyLogging):
"""
self.known.pop(package_base, None)
self.database.package_remove(package_base)
self.remove_logs(package_base, None)
self.logs_remove(package_base, None)
def remove_logs(self, package_base: str, current_process_id: int | None) -> None:
"""
remove package related logs
Args:
package_base(str): package base
current_process_id(int | None): current process id
"""
self.database.logs_remove(package_base, current_process_id)
def update(self, package_base: str, status: BuildStatusEnum, package: Package | None) -> None:
def package_update(self, package_base: str, status: BuildStatusEnum, package: Package | None) -> None:
"""
update package status and description
@ -159,22 +172,7 @@ class Watcher(LazyLogging):
self.known[package_base] = (package, full_status)
self.database.package_update(package, full_status)
def update_logs(self, log_record_id: LogRecordId, created: float, record: str) -> None:
"""
make new log record into database
Args:
log_record_id(LogRecordId): log record id
created(float): log created record
record(str): log record
"""
if self._last_log_record_id != log_record_id:
# there is new log record, so we remove old ones
self.remove_logs(log_record_id.package_base, log_record_id.process_id)
self._last_log_record_id = log_record_id
self.database.logs_insert(log_record_id, created, record)
def update_self(self, status: BuildStatusEnum) -> None:
def status_update(self, status: BuildStatusEnum) -> None:
"""
update service status

View File

@ -21,7 +21,8 @@ import contextlib
import logging
import requests
from collections.abc import Generator
from functools import cached_property
from typing import Any, IO, Literal
from urllib.parse import quote_plus as urlencode
from ahriman import __version__
@ -31,10 +32,15 @@ from ahriman.core.status.client import Client
from ahriman.core.util import exception_response_text
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.internal_status import InternalStatus
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
from ahriman.models.user import User
# filename, file, content-type, headers
MultipartType = tuple[str, IO[bytes], str, dict[str, str]]
class WebClient(Client, LazyLogging):
"""
build status reporter web client
@ -43,8 +49,12 @@ class WebClient(Client, LazyLogging):
address(str): address of the web service
suppress_errors(bool): suppress logging errors (e.g. if no web server available)
user(User | None): web service user descriptor
use_unix_socket(bool): use websocket or not
"""
_login_url = "/api/v1/login"
_status_url = "/api/v1/status"
def __init__(self, configuration: Configuration) -> None:
"""
default constructor
@ -52,33 +62,49 @@ class WebClient(Client, LazyLogging):
Args:
configuration(Configuration): configuration instance
"""
self.address, use_unix_socket = self.parse_address(configuration)
self.address, self.use_unix_socket = self.parse_address(configuration)
self.user = User.from_option(
configuration.get("web", "username", fallback=None),
configuration.get("web", "password", fallback=None))
self.suppress_errors = configuration.getboolean("settings", "suppress_http_log_errors", fallback=False)
self.__session = self._create_session(use_unix_socket=use_unix_socket)
@property
def _login_url(self) -> str:
@cached_property
def session(self) -> requests.Session:
"""
get url for the login api
get or create session
Returns:
str: full url for web service to log in
request.Session: created session object
"""
return f"{self.address}/api/v1/login"
return self._create_session(use_unix_socket=self.use_unix_socket)
@property
def _status_url(self) -> str:
@staticmethod
def _logs_url(package_base: str) -> str:
"""
get url for the status api
get url for the logs api
Args:
package_base(str): package base
Returns:
str: full url for web service for status
str: full url for web service for logs
"""
return f"{self.address}/api/v1/status"
return f"/api/v1/packages/{package_base}/logs"
@staticmethod
def _package_url(package_base: str = "") -> str:
"""
url generator
Args:
package_base(str, optional): package base to generate url (Default value = "")
Returns:
str: full url of web service for specific package base
"""
# in case if unix socket is used we need to normalize url
suffix = f"/{package_base}" if package_base else ""
return f"/api/v1/packages{suffix}"
@staticmethod
def parse_address(configuration: Configuration) -> tuple[str, bool]:
@ -102,32 +128,6 @@ class WebClient(Client, LazyLogging):
address = f"http://{host}:{port}"
return address, False
@contextlib.contextmanager
def __get_session(self, session: requests.Session | None = None) -> Generator[requests.Session, None, None]:
"""
execute request and handle exceptions
Args:
session(requests.Session | None, optional): session to be used or stored instance property otherwise
(Default value = None)
Yields:
requests.Session: session for requests
"""
try:
if session is not None:
yield session # use session from arguments
else:
yield self.__session # use instance generated session
except requests.RequestException as e:
if self.suppress_errors:
return
self.logger.exception("could not perform http request: %s", exception_response_text(e))
except Exception:
if self.suppress_errors:
return
self.logger.exception("could not perform http request")
def _create_session(self, *, use_unix_socket: bool) -> requests.Session:
"""
generate new request session
@ -164,38 +164,51 @@ class WebClient(Client, LazyLogging):
"username": self.user.username,
"password": self.user.password
}
with contextlib.suppress(Exception):
self.make_request("POST", self._login_url, json=payload, session=session)
with self.__get_session(session):
response = session.post(self._login_url, json=payload)
def make_request(self, method: Literal["DELETE", "GET", "POST"], url: str, *,
params: list[tuple[str, str]] | None = None,
json: dict[str, Any] | None = None,
files: dict[str, MultipartType] | None = None,
session: requests.Session | None = None,
suppress_errors: bool | None = None) -> requests.Response:
"""
perform request with specified parameters
Args:
method(Literal["DELETE", "GET", "POST"]): HTTP method to call
url(str): remote url to call
params(list[tuple[str, str]] | None, optional): request query parameters (Default value = None)
json(dict[str, Any] | None, optional): request json parameters (Default value = None)
files(dict[str, MultipartType] | None, optional): multipart upload (Default value = None)
session(requests.Session | None, optional): session object if any (Default value = None)
suppress_errors(bool | None, optional): suppress logging errors (e.g. if no web server available). If none
set, the instance-wide value will be used (Default value = None)
Returns:
requests.Response: response object
"""
# defaults
if suppress_errors is None:
suppress_errors = self.suppress_errors
if session is None:
session = self.session
try:
response = session.request(method, f"{self.address}{url}", params=params, json=json, files=files)
response.raise_for_status()
return response
except requests.RequestException as e:
if not suppress_errors:
self.logger.exception("could not perform http request: %s", exception_response_text(e))
raise
except Exception:
if not suppress_errors:
self.logger.exception("could not perform http request")
raise
def _logs_url(self, package_base: str) -> str:
"""
get url for the logs api
Args:
package_base(str): package base
Returns:
str: full url for web service for logs
"""
return f"{self.address}/api/v1/packages/{package_base}/logs"
def _package_url(self, package_base: str = "") -> str:
"""
url generator
Args:
package_base(str, optional): package base to generate url (Default value = "")
Returns:
str: full url of web service for specific package base
"""
# in case if unix socket is used we need to normalize url
suffix = f"/{package_base}" if package_base else ""
return f"{self.address}/api/v1/packages{suffix}"
def add(self, package: Package, status: BuildStatusEnum) -> None:
def package_add(self, package: Package, status: BuildStatusEnum) -> None:
"""
add new package with status
@ -207,12 +220,10 @@ class WebClient(Client, LazyLogging):
"status": status.value,
"package": package.view()
}
with contextlib.suppress(Exception):
self.make_request("POST", self._package_url(package.base), json=payload)
with self.__get_session() as session:
response = session.post(self._package_url(package.base), json=payload)
response.raise_for_status()
def get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]:
def package_get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]:
"""
get package status
@ -222,66 +233,47 @@ class WebClient(Client, LazyLogging):
Returns:
list[tuple[Package, BuildStatus]]: list of current package description and status if it has been found
"""
with self.__get_session() as session:
response = session.get(self._package_url(package_base or ""))
response.raise_for_status()
with contextlib.suppress(Exception):
response = self.make_request("GET", self._package_url(package_base or ""))
response_json = response.json()
status_json = response.json()
return [
(Package.from_json(package["package"]), BuildStatus.from_json(package["status"]))
for package in status_json
for package in response_json
]
# noinspection PyUnreachableCode
return []
def get_internal(self) -> InternalStatus:
"""
get internal service status
Returns:
InternalStatus: current internal (web) service status
"""
with self.__get_session() as session:
response = session.get(self._status_url)
response.raise_for_status()
status_json = response.json()
return InternalStatus.from_json(status_json)
# noinspection PyUnreachableCode
return InternalStatus(status=BuildStatus())
def logs(self, package_base: str, record: logging.LogRecord) -> None:
def package_logs(self, log_record_id: LogRecordId, record: logging.LogRecord) -> None:
"""
post log record
Args:
package_base(str) package base
log_record_id(LogRecordId): log record id
record(logging.LogRecord): log record to post to api
"""
payload = {
"created": record.created,
"message": record.getMessage(),
"process_id": record.process,
"version": log_record_id.version,
}
# in this method exception has to be handled outside in logger handler
response = self.__session.post(self._logs_url(package_base), json=payload)
response.raise_for_status()
# this is special case, because we would like to do not suppress exception here
# in case of exception raised it will be handled by upstream HttpLogHandler
# In the other hand, we force to suppress all http logs here to avoid cyclic reporting
self.make_request("POST", self._logs_url(log_record_id.package_base), json=payload, suppress_errors=True)
def remove(self, package_base: str) -> None:
def package_remove(self, package_base: str) -> None:
"""
remove packages from watcher
Args:
package_base(str): basename to remove
"""
with self.__get_session() as session:
response = session.delete(self._package_url(package_base))
response.raise_for_status()
with contextlib.suppress(Exception):
self.make_request("DELETE", self._package_url(package_base))
def update(self, package_base: str, status: BuildStatusEnum) -> None:
def package_update(self, package_base: str, status: BuildStatusEnum) -> None:
"""
update package build status. Unlike ``add`` it does not update package properties
@ -290,12 +282,25 @@ class WebClient(Client, LazyLogging):
status(BuildStatusEnum): current package build status
"""
payload = {"status": status.value}
with contextlib.suppress(Exception):
self.make_request("POST", self._package_url(package_base), json=payload)
with self.__get_session() as session:
response = session.post(self._package_url(package_base), json=payload)
response.raise_for_status()
def status_get(self) -> InternalStatus:
"""
get internal service status
def update_self(self, status: BuildStatusEnum) -> None:
Returns:
InternalStatus: current internal (web) service status
"""
with contextlib.suppress(Exception):
response = self.make_request("GET", self._status_url)
response_json = response.json()
return InternalStatus.from_json(response_json)
return InternalStatus(status=BuildStatus())
def status_update(self, status: BuildStatusEnum) -> None:
"""
update ahriman status itself
@ -303,7 +308,5 @@ class WebClient(Client, LazyLogging):
status(BuildStatusEnum): current ahriman status
"""
payload = {"status": status.value}
with self.__get_session() as session:
response = session.post(self._status_url, json=payload)
response.raise_for_status()
with contextlib.suppress(Exception):
self.make_request("POST", self._status_url, json=payload)

View File

@ -20,6 +20,7 @@
import hashlib
import requests
from functools import cached_property
from pathlib import Path
from typing import Any
@ -52,6 +53,16 @@ class HttpUpload(Upload):
self.auth = (password, username) if password and username else None
self.timeout = configuration.getint(section, "timeout", fallback=30)
@cached_property
def session(self) -> requests.Session:
"""
get or create session
Returns:
request.Session: created session object
"""
return requests.Session()
@staticmethod
def calculate_hash(path: Path) -> str:
"""
@ -110,7 +121,7 @@ class HttpUpload(Upload):
requests.Response: request response object
"""
try:
response = requests.request(method, url, auth=self.auth, timeout=self.timeout, **kwargs)
response = self.session.request(method, url, auth=self.auth, timeout=self.timeout, **kwargs)
response.raise_for_status()
except requests.HTTPError as e:
self.logger.exception("could not perform %s request to %s: %s", method, url, exception_response_text(e))

View File

@ -0,0 +1,105 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import requests
from functools import cached_property
from pathlib import Path
from ahriman.core.configuration import Configuration
from ahriman.core.sign.gpg import GPG
from ahriman.core.status.web_client import MultipartType, WebClient
from ahriman.core.upload.http_upload import HttpUpload
from ahriman.models.package import Package
class RemoteService(HttpUpload):
"""
upload files to another server instance
Attributes:
client(WebClient): web client instance
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
configuration(Configuration): configuration instance
section(str): settings section name
"""
HttpUpload.__init__(self, architecture, configuration, section)
self.client = WebClient(configuration)
@cached_property
def session(self) -> requests.Session:
"""
get or create session
Returns:
request.Session: created session object
"""
return self.client.session
def package_upload(self, path: Path, package: Package) -> None:
"""
upload single package to remote
Args:
path(Path): local path to sync
package(Package): package to upload
"""
def upload(package_path: Path, signature_path: Path | None) -> None:
files: dict[str, MultipartType] = {}
try:
# package part always persists
files["package"] = package_path.name, package_path.open("rb"), "application/octet-stream", {}
# signature part is optional
if signature_path is not None:
files["signature"] = signature_path.name, signature_path.open("rb"), "application/octet-stream", {}
self._request("POST", f"{self.client.address}/api/v1/service/upload", files=files)
finally:
for _, fd, _, _ in files.values():
fd.close()
for key, descriptor in package.packages.items():
if descriptor.filename is None:
self.logger.warning("package %s of %s doesn't have filename set", key, package.base)
continue
archive = path / descriptor.filename
maybe_signature_path = GPG.signature(archive)
signature = maybe_signature_path if maybe_signature_path.is_file() else None
upload(archive, signature)
def sync(self, path: Path, built_packages: list[Package]) -> None:
"""
sync data to remote server
Args:
path(Path): local path to sync
built_packages(list[Package]): list of packages which has just been built
"""
for package in built_packages:
self.package_upload(path, package)

View File

@ -90,6 +90,9 @@ class Upload(LazyLogging):
if provider == UploadSettings.Github:
from ahriman.core.upload.github import Github
return Github(architecture, configuration, section)
if provider == UploadSettings.RemoteService:
from ahriman.core.upload.remote_service import RemoteService
return RemoteService(architecture, configuration, section)
return Upload(architecture, configuration) # should never happen
def run(self, path: Path, built_packages: list[Package]) -> None:

View File

@ -92,6 +92,15 @@ class UploadTrigger(Trigger):
},
},
},
"remote-service": {
"type": "dict",
"schema": {
"type": {
"type": "string",
"allowed": ["ahriman", "remote-service"],
},
},
},
"s3": {
"type": "dict",
"schema": {

View File

@ -142,7 +142,6 @@ def check_output(*args: str, exception: Exception | None = None, cwd: Path | Non
while selector.get_map(): # while there are unread selectors, keep reading
result.extend(poll(selector))
process.terminate() # make sure that process is terminated
status_code = process.wait()
if status_code != 0:
if exception is not None:
@ -280,7 +279,7 @@ def package_like(filename: Path) -> bool:
bool: True in case if name contains ``.pkg.`` and not signature, False otherwise
"""
name = filename.name
return ".pkg." in name and not name.endswith(".sig")
return not name.startswith(".") and ".pkg." in name and not name.endswith(".sig")
def parse_version(version: str) -> tuple[str | None, str, str]:

View File

@ -27,8 +27,8 @@ class LogRecordId:
Attributes:
package_base(str): package base for which log record belongs
process_id(int): process id from which log record was emitted
version(str): package version for which log record belongs
"""
package_base: str
process_id: int
version: str

View File

@ -32,6 +32,7 @@ class ReportSettings(str, Enum):
Email(ReportSettings): (class attribute) email report generation
Console(ReportSettings): (class attribute) print result to console
Telegram(ReportSettings): (class attribute) markdown report to telegram channel
RemoteCall(ReportSettings): (class attribute) remote ahriman server call
"""
Disabled = "disabled" # for testing purpose
@ -39,6 +40,7 @@ class ReportSettings(str, Enum):
Email = "email"
Console = "console"
Telegram = "telegram"
RemoteCall = "remote-call"
@staticmethod
def from_option(value: str) -> ReportSettings:
@ -59,4 +61,6 @@ class ReportSettings(str, Enum):
return ReportSettings.Console
if value.lower() in ("telegram",):
return ReportSettings.Telegram
if value.lower() in ("ahriman", "remote-call",):
return ReportSettings.RemoteCall
return ReportSettings.Disabled

View File

@ -31,12 +31,14 @@ class UploadSettings(str, Enum):
Rsync(UploadSettings): (class attribute) sync via rsync
S3(UploadSettings): (class attribute) sync to Amazon S3
Github(UploadSettings): (class attribute) sync to github releases page
RemoteService(UploadSettings): (class attribute) sync to another ahriman instance
"""
Disabled = "disabled" # for testing purpose
Rsync = "rsync"
S3 = "s3"
Github = "github"
RemoteService = "remote-service"
@staticmethod
def from_option(value: str) -> UploadSettings:
@ -55,4 +57,6 @@ class UploadSettings(str, Enum):
return UploadSettings.S3
if value.lower() in ("github",):
return UploadSettings.Github
if value.lower() in ("ahriman", "remote-service",):
return UploadSettings.RemoteService
return UploadSettings.Disabled

View File

@ -0,0 +1,72 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import time
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import ParamSpec
Params = ParamSpec("Params")
@dataclass(frozen=True)
class Waiter:
"""
simple waiter implementation
Attributes:
interval(int): interval in seconds between checks
start_time(float): monotonic time of the waiter start. More likely must not be assigned explicitly
wait_timeout(int): timeout in seconds to wait for. Negative value will result in immediate exit. Zero value
means infinite timeout
"""
wait_timeout: int
start_time: float = field(default_factory=time.monotonic, kw_only=True)
interval: int = field(default=10, kw_only=True)
def is_timed_out(self) -> bool:
"""
check if timer is out
Returns:
bool: True in case current monotonic time is more than ``Waiter.start_time`` and
``Waiter.wait_timeout`` doesn't equal to 0
"""
since_start: float = time.monotonic() - self.start_time
return self.wait_timeout != 0 and since_start > self.wait_timeout
def wait(self, in_progress: Callable[Params, bool], *args: Params.args, **kwargs: Params.kwargs) -> float:
"""
wait until requirements are not met
Args:
in_progress(Callable[Params, bool]): function to check if timer should wait for another cycle
*args(Params.args): positional arguments for check call
**kwargs(Params.kwargs): keyword arguments for check call
Returns:
float: consumed time in seconds
"""
while not self.is_timed_out() and in_progress(*args, **kwargs):
time.sleep(self.interval)
return time.monotonic() - self.start_time

View File

@ -25,11 +25,13 @@ from ahriman.web.views.api.swagger import SwaggerView
from ahriman.web.views.index import IndexView
from ahriman.web.views.service.add import AddView
from ahriman.web.views.service.pgp import PGPView
from ahriman.web.views.service.process import ProcessView
from ahriman.web.views.service.rebuild import RebuildView
from ahriman.web.views.service.remove import RemoveView
from ahriman.web.views.service.request import RequestView
from ahriman.web.views.service.search import SearchView
from ahriman.web.views.service.update import UpdateView
from ahriman.web.views.service.upload import UploadView
from ahriman.web.views.status.logs import LogsView
from ahriman.web.views.status.package import PackageView
from ahriman.web.views.status.packages import PackagesView
@ -60,10 +62,12 @@ def setup_routes(application: Application, static_path: Path) -> None:
application.router.add_view("/api/v1/service/add", AddView)
application.router.add_view("/api/v1/service/pgp", PGPView)
application.router.add_view("/api/v1/service/rebuild", RebuildView)
application.router.add_view("/api/v1/service/process/{process_id}", ProcessView)
application.router.add_view("/api/v1/service/remove", RemoveView)
application.router.add_view("/api/v1/service/request", RequestView)
application.router.add_view("/api/v1/service/search", SearchView)
application.router.add_view("/api/v1/service/update", UpdateView)
application.router.add_view("/api/v1/service/upload", UploadView)
application.router.add_view("/api/v1/packages", PackagesView)
application.router.add_view("/api/v1/packages/{package}", PackageView)

View File

@ -21,6 +21,7 @@ from ahriman.web.schemas.aur_package_schema import AURPackageSchema
from ahriman.web.schemas.auth_schema import AuthSchema
from ahriman.web.schemas.counters_schema import CountersSchema
from ahriman.web.schemas.error_schema import ErrorSchema
from ahriman.web.schemas.file_schema import FileSchema
from ahriman.web.schemas.internal_status_schema import InternalStatusSchema
from ahriman.web.schemas.log_schema import LogSchema
from ahriman.web.schemas.login_schema import LoginSchema
@ -33,6 +34,9 @@ from ahriman.web.schemas.package_schema import PackageSchema
from ahriman.web.schemas.package_status_schema import PackageStatusSimplifiedSchema, PackageStatusSchema
from ahriman.web.schemas.pgp_key_id_schema import PGPKeyIdSchema
from ahriman.web.schemas.pgp_key_schema import PGPKeySchema
from ahriman.web.schemas.process_id_schema import ProcessIdSchema
from ahriman.web.schemas.process_schema import ProcessSchema
from ahriman.web.schemas.remote_schema import RemoteSchema
from ahriman.web.schemas.search_schema import SearchSchema
from ahriman.web.schemas.status_schema import StatusSchema
from ahriman.web.schemas.update_flags_schema import UpdateFlagsSchema

View File

@ -0,0 +1,30 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from marshmallow import Schema, fields
class FileSchema(Schema):
"""
request file upload schema
"""
archive = fields.Field(required=True, metadata={
"description": "Package archive to be uploaded",
})

View File

@ -19,6 +19,8 @@
#
from marshmallow import Schema, fields
from ahriman import __version__
class LogSchema(Schema):
"""
@ -29,9 +31,9 @@ class LogSchema(Schema):
"description": "Log record timestamp",
"example": 1680537091.233495,
})
process_id = fields.Integer(required=True, metadata={
"description": "Current process id",
"example": 42,
version = fields.Integer(required=True, metadata={
"description": "Package version to tag",
"example": __version__,
})
message = fields.String(required=True, metadata={
"description": "Log message",

View File

@ -0,0 +1,31 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from marshmallow import Schema, fields
class ProcessIdSchema(Schema):
"""
request and response spawned process id schema
"""
process_id = fields.String(required=True, metadata={
"description": "Spawned process unique ID",
"example": "ff456814-5669-4de6-9143-44dbf6f68607",
})

View File

@ -0,0 +1,30 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from marshmallow import Schema, fields
class ProcessSchema(Schema):
"""
process status response schema
"""
is_alive = fields.Bool(required=True, metadata={
"description": "Is process alive or not",
})

View File

@ -0,0 +1,36 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from marshmallow import Schema, fields
class UpdateFlagsSchema(Schema):
"""
update flags request schema
"""
aur = fields.Bool(dump_default=True, metadata={
"description": "Check AUR for updates",
})
local = fields.Bool(dump_default=True, metadata={
"description": "Check local packages for updates",
})
manual = fields.Bool(dump_default=True, metadata={
"description": "Check manually built packages",
})

View File

@ -43,7 +43,7 @@ class SwaggerView(BaseView):
Response: 200 with json api specification
"""
spec = self.request.app["swagger_dict"]
is_body_parameter: Callable[[dict[str, str]], bool] = lambda p: p["in"] == "body"
is_body_parameter: Callable[[dict[str, str]], bool] = lambda p: p["in"] == "body" or p["in"] == "formData"
# special workaround because it writes request body to parameters section
paths = spec["paths"]
@ -56,11 +56,14 @@ class SwaggerView(BaseView):
if not body:
continue # there were no ``body`` parameters found
schema = next(iter(body))
content_type = "multipart/form-data" if schema["in"] == "formData" else "application/json"
# there should be only one body parameters
method["requestBody"] = {
"content": {
"application/json": {
"schema": next(iter(body))["schema"]
content_type: {
"schema": schema["schema"]
}
}
}

View File

@ -19,10 +19,10 @@
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPBadRequest, HTTPNoContent
from aiohttp.web import HTTPBadRequest, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
from ahriman.web.views.base import BaseView
@ -41,7 +41,7 @@ class AddView(BaseView):
summary="Add new package",
description="Add new package(s) from AUR",
responses={
204: {"description": "Success response"},
200: {"description": "Success response", "schema": ProcessIdSchema},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
@ -51,13 +51,15 @@ class AddView(BaseView):
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.json_schema(PackageNamesSchema)
async def post(self) -> None:
async def post(self) -> Response:
"""
add new package
Returns:
Response: 200 with spawned process id
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
try:
data = await self.extract_data(["packages"])
@ -66,6 +68,6 @@ class AddView(BaseView):
raise HTTPBadRequest(reason=str(e))
username = await self.username()
self.spawner.packages_add(packages, username, now=True)
process_id = self.spawner.packages_add(packages, username, now=True)
raise HTTPNoContent()
return json_response({"process_id": process_id})

View File

@ -19,10 +19,10 @@
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPBadRequest, HTTPNoContent, HTTPNotFound, Response, json_response
from aiohttp.web import HTTPBadRequest, HTTPNotFound, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, PGPKeyIdSchema, PGPKeySchema
from ahriman.web.schemas import AuthSchema, ErrorSchema, PGPKeyIdSchema, PGPKeySchema, ProcessIdSchema
from ahriman.web.views.base import BaseView
@ -83,7 +83,7 @@ class PGPView(BaseView):
summary="Fetch PGP key",
description="Fetch PGP key from the key server",
responses={
204: {"description": "Success response"},
200: {"description": "Success response", "schema": ProcessIdSchema},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
@ -93,13 +93,15 @@ class PGPView(BaseView):
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.json_schema(PGPKeyIdSchema)
async def post(self) -> None:
async def post(self) -> Response:
"""
store key to the local service environment
Returns:
Response: 200 with spawned process id
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
data = await self.extract_data()
@ -108,6 +110,6 @@ class PGPView(BaseView):
except Exception as e:
raise HTTPBadRequest(reason=str(e))
self.spawner.key_import(key, data.get("server"))
process_id = self.spawner.key_import(key, data.get("server"))
raise HTTPNoContent()
return json_response({"process_id": process_id})

View File

@ -0,0 +1,74 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPNotFound, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, ProcessIdSchema, ProcessSchema
from ahriman.web.views.base import BaseView
class ProcessView(BaseView):
"""
Process information web view
Attributes:
GET_PERMISSION(UserAccess): (class attribute) get permissions of self
"""
GET_PERMISSION = UserAccess.Reporter
@aiohttp_apispec.docs(
tags=["Actions"],
summary="Get process",
description="Get process information",
responses={
200: {"description": "Success response", "schema": ProcessSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
404: {"description": "Not found", "schema": ErrorSchema},
500: {"description": "Internal server error", "schema": ErrorSchema},
},
security=[{"token": [GET_PERMISSION]}],
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.match_info_schema(ProcessIdSchema)
async def get(self) -> Response:
"""
get spawned process status
Returns:
Response: 200 with process information
Raises:
HTTPNotFound: if no process found
"""
process_id = self.request.match_info["process_id"]
is_alive = self.spawner.has_process(process_id)
if not is_alive:
raise HTTPNotFound(reason=f"No process {process_id} found")
response = {
"is_alive": is_alive,
}
return json_response(response)

View File

@ -19,10 +19,10 @@
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPBadRequest, HTTPNoContent
from aiohttp.web import HTTPBadRequest, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
from ahriman.web.views.base import BaseView
@ -41,7 +41,7 @@ class RebuildView(BaseView):
summary="Rebuild packages",
description="Rebuild packages which depend on specified one",
responses={
204: {"description": "Success response"},
200: {"description": "Success response", "schema": ProcessIdSchema},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
@ -51,13 +51,15 @@ class RebuildView(BaseView):
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.json_schema(PackageNamesSchema)
async def post(self) -> None:
async def post(self) -> Response:
"""
rebuild packages based on their dependency
Returns:
Response: 200 with spawned process id
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
try:
data = await self.extract_data(["packages"])
@ -67,6 +69,6 @@ class RebuildView(BaseView):
raise HTTPBadRequest(reason=str(e))
username = await self.username()
self.spawner.packages_rebuild(depends_on, username)
process_id = self.spawner.packages_rebuild(depends_on, username)
raise HTTPNoContent()
return json_response({"process_id": process_id})

View File

@ -19,10 +19,10 @@
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPBadRequest, HTTPNoContent
from aiohttp.web import HTTPBadRequest, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
from ahriman.web.views.base import BaseView
@ -41,7 +41,7 @@ class RemoveView(BaseView):
summary="Remove packages",
description="Remove specified packages from the repository",
responses={
204: {"description": "Success response"},
200: {"description": "Success response", "schema": ProcessIdSchema},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
@ -51,13 +51,15 @@ class RemoveView(BaseView):
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.json_schema(PackageNamesSchema)
async def post(self) -> None:
async def post(self) -> Response:
"""
remove existing packages
Returns:
Response: 200 with spawned process id
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
try:
data = await self.extract_data(["packages"])
@ -65,6 +67,6 @@ class RemoveView(BaseView):
except Exception as e:
raise HTTPBadRequest(reason=str(e))
self.spawner.packages_remove(packages)
process_id = self.spawner.packages_remove(packages)
raise HTTPNoContent()
return json_response({"process_id": process_id})

View File

@ -19,10 +19,10 @@
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPBadRequest, HTTPNoContent
from aiohttp.web import HTTPBadRequest, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema
from ahriman.web.schemas import AuthSchema, ErrorSchema, PackageNamesSchema, ProcessIdSchema
from ahriman.web.views.base import BaseView
@ -41,7 +41,7 @@ class RequestView(BaseView):
summary="Request new package",
description="Request new package(s) to be added from AUR",
responses={
204: {"description": "Success response"},
200: {"description": "Success response", "schema": ProcessIdSchema},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
@ -51,13 +51,15 @@ class RequestView(BaseView):
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.json_schema(PackageNamesSchema)
async def post(self) -> None:
async def post(self) -> Response:
"""
request to add new package
Returns:
Response: 200 with spawned process id
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
try:
data = await self.extract_data(["packages"])
@ -66,6 +68,6 @@ class RequestView(BaseView):
raise HTTPBadRequest(reason=str(e))
username = await self.username()
self.spawner.packages_add(packages, username, now=False)
process_id = self.spawner.packages_add(packages, username, now=False)
raise HTTPNoContent()
return json_response({"process_id": process_id})

View File

@ -19,10 +19,10 @@
#
import aiohttp_apispec # type: ignore[import]
from aiohttp.web import HTTPNoContent
from aiohttp.web import HTTPBadRequest, Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema
from ahriman.web.schemas import AuthSchema, ErrorSchema, ProcessIdSchema, UpdateFlagsSchema
from ahriman.web.views.base import BaseView
@ -41,7 +41,8 @@ class UpdateView(BaseView):
summary="Update packages",
description="Run repository update process",
responses={
204: {"description": "Success response"},
200: {"description": "Success response", "schema": ProcessIdSchema},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
500: {"description": "Internal server error", "schema": ErrorSchema},
@ -49,14 +50,28 @@ class UpdateView(BaseView):
security=[{"token": [POST_PERMISSION]}],
)
@aiohttp_apispec.cookies_schema(AuthSchema)
async def post(self) -> None:
@aiohttp_apispec.json_schema(UpdateFlagsSchema)
async def post(self) -> Response:
"""
run repository update. No parameters supported here
Raises:
HTTPNoContent: in case of success response
"""
username = await self.username()
self.spawner.packages_update(username)
Returns:
Response: 200 with spawned process id
raise HTTPNoContent()
Raises:
HTTPBadRequest: if bad data is supplied
"""
try:
data = await self.extract_data()
except Exception as e:
raise HTTPBadRequest(reason=str(e))
username = await self.username()
process_id = self.spawner.packages_update(
username,
aur=data.get("aur", True),
local=data.get("local", True),
manual=data.get("manual", True),
)
return json_response({"process_id": process_id})

View File

@ -0,0 +1,144 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import aiohttp_apispec # type: ignore[import]
import shutil
from aiohttp import BodyPartReader
from aiohttp.web import HTTPBadRequest, HTTPCreated, HTTPNotFound
from pathlib import Path
from tempfile import NamedTemporaryFile
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, FileSchema
from ahriman.web.views.base import BaseView
class UploadView(BaseView):
"""
upload file to repository
Attributes:
POST_PERMISSION(UserAccess): (class attribute) post permissions of self
"""
POST_PERMISSION = UserAccess.Full
@staticmethod
async def save_file(part: BodyPartReader, target: Path, *, max_body_size: int | None = None) -> tuple[str, Path]:
"""
save file to local cache
Args:
part(BodyPartReader): multipart part to be saved
target(Path): path to directory to which file should be saved
max_body_size(int | None, optional): max body size in bytes (Default value = None)
Returns:
tuple[str, Path]: map of received filename to its local path
Raises:
HTTPBadRequest: if bad data is supplied
"""
archive_name = part.filename
if archive_name is None:
raise HTTPBadRequest(reason="Filename must be set")
# some magic inside. We would like to make sure that passed filename is filename
# without slashes, dots, etc
if Path(archive_name).resolve().name != archive_name:
raise HTTPBadRequest(reason="Filename must be valid archive name")
current_size = 0
# in order to handle errors automatically we create temporary file for long operation (transfer)
# and then copy it to valid location
with NamedTemporaryFile() as cache:
while True:
chunk = await part.read_chunk()
if not chunk:
break
current_size += len(chunk)
if max_body_size is not None and current_size > max_body_size:
raise HTTPBadRequest(reason="Body part is too large")
cache.write(chunk)
cache.seek(0) # reset file position
# and now copy temporary file to target location as hidden file
# we put it as hidden in order to make sure that it will not be handled during some random process
temporary_output = target / f".{archive_name}"
with temporary_output.open("wb") as archive:
shutil.copyfileobj(cache, archive)
return archive_name, temporary_output
@aiohttp_apispec.docs(
tags=["Actions"],
summary="Upload package",
description="Upload package to local filesystem",
responses={
201: {"description": "Success response"},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
404: {"description": "Not found", "schema": ErrorSchema},
500: {"description": "Internal server error", "schema": ErrorSchema},
},
security=[{"token": [POST_PERMISSION]}],
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.form_schema(FileSchema)
async def post(self) -> None:
"""
upload file from another instance to the server
Raises:
HTTPBadRequest: if bad data is supplied
HTTPCreated: on success response
"""
if not self.configuration.getboolean("web", "enable_archive_upload", fallback=False):
raise HTTPNotFound()
try:
reader = await self.request.multipart()
except Exception as e:
raise HTTPBadRequest(reason=str(e))
max_body_size = self.configuration.getint("web", "max_body_size", fallback=None)
target = self.configuration.repository_paths.packages
files = []
while (part := await reader.next()) is not None:
if not isinstance(part, BodyPartReader):
raise HTTPBadRequest(reason="Invalid multipart message received")
if part.name not in ("package", "signature"):
raise HTTPBadRequest(reason="Multipart field isn't package or signature")
files.append(await self.save_file(part, target, max_body_size=max_body_size))
# and now we can rename files, which is relatively fast operation
# it is probably good way to call lock here, however
for filename, current_location in files:
target_location = current_location.parent / filename
current_location.rename(target_location)
raise HTTPCreated()

View File

@ -63,7 +63,7 @@ class LogsView(BaseView):
HTTPNoContent: on success response
"""
package_base = self.request.match_info["package"]
self.service.remove_logs(package_base, None)
self.service.logs_remove(package_base, None)
raise HTTPNoContent()
@ -95,10 +95,10 @@ class LogsView(BaseView):
package_base = self.request.match_info["package"]
try:
_, status = self.service.get(package_base)
_, status = self.service.package_get(package_base)
except UnknownPackageError:
raise HTTPNotFound()
logs = self.service.get_logs(package_base)
logs = self.service.logs_get(package_base)
response = {
"package_base": package_base,
@ -137,10 +137,10 @@ class LogsView(BaseView):
try:
created = data["created"]
record = data["message"]
process_id = data["process_id"]
version = data["version"]
except Exception as e:
raise HTTPBadRequest(reason=str(e))
self.service.update_logs(LogRecordId(package_base, process_id), created, record)
self.service.logs_update(LogRecordId(package_base, version), created, record)
raise HTTPNoContent()

View File

@ -64,7 +64,7 @@ class PackageView(BaseView):
HTTPNoContent: on success response
"""
package_base = self.request.match_info["package"]
self.service.remove(package_base)
self.service.package_remove(package_base)
raise HTTPNoContent()
@ -96,7 +96,7 @@ class PackageView(BaseView):
package_base = self.request.match_info["package"]
try:
package, status = self.service.get(package_base)
package, status = self.service.package_get(package_base)
except UnknownPackageError:
raise HTTPNotFound()
@ -142,7 +142,7 @@ class PackageView(BaseView):
raise HTTPBadRequest(reason=str(e))
try:
self.service.update(package_base, status, package)
self.service.package_update(package_base, status, package)
except UnknownPackageError:
raise HTTPBadRequest(reason=f"Package {package_base} is unknown, but no package body set")

View File

@ -102,6 +102,6 @@ class StatusView(BaseView):
except Exception as e:
raise HTTPBadRequest(reason=str(e))
self.service.update_self(status)
self.service.status_update(status)
raise HTTPNoContent()