From 3d50ea46869ddc07fc72baae0266359c9397a6fd Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Fri, 23 Feb 2024 17:59:44 +0200 Subject: [PATCH] implement local reporter mode --- .../application/application_properties.py | 11 + .../application/handlers/status_update.py | 2 +- .../database/operations/package_operations.py | 62 +++--- src/ahriman/core/log/http_log_handler.py | 2 +- .../core/repository/repository_properties.py | 2 +- src/ahriman/core/status/client.py | 170 ++++++++++++-- src/ahriman/core/status/local_client.py | 208 ++++++++++++++++++ src/ahriman/core/status/watcher.py | 65 +++--- src/ahriman/core/status/web_client.py | 152 ++++++++++++- src/ahriman/core/support/package_creator.py | 4 +- src/ahriman/models/dependencies.py | 29 ++- src/ahriman/models/pkgbuild_patch.py | 19 +- src/ahriman/web/schemas/__init__.py | 2 + .../web/schemas/dependencies_schema.py | 35 +++ .../web/schemas/package_version_schema.py | 34 +++ src/ahriman/web/views/v1/packages/changes.py | 3 +- .../web/views/v1/packages/dependencies.py | 66 ++++++ .../web/views/v1/packages/dependency.py | 117 ++++++++++ src/ahriman/web/views/v1/packages/logs.py | 9 +- src/ahriman/web/web.py | 4 +- 20 files changed, 897 insertions(+), 99 deletions(-) create mode 100644 src/ahriman/core/status/local_client.py create mode 100644 src/ahriman/web/schemas/dependencies_schema.py create mode 100644 src/ahriman/web/schemas/package_version_schema.py create mode 100644 src/ahriman/web/views/v1/packages/dependencies.py create mode 100644 src/ahriman/web/views/v1/packages/dependency.py diff --git a/src/ahriman/application/application/application_properties.py b/src/ahriman/application/application/application_properties.py index 9d5ba208..55a88d09 100644 --- a/src/ahriman/application/application/application_properties.py +++ b/src/ahriman/application/application/application_properties.py @@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.database import SQLite from ahriman.core.log import LazyLogging from ahriman.core.repository import Repository +from ahriman.core.status.client import Client from ahriman.models.pacman_synchronization import PacmanSynchronization from ahriman.models.repository_id import RepositoryId @@ -63,3 +64,13 @@ class ApplicationProperties(LazyLogging): str: repository architecture """ return self.repository_id.architecture + + @property + def reporter(self) -> Client: + """ + instance of the web/database client + + Returns: + Client: repository reposter + """ + return self.repository.reporter diff --git a/src/ahriman/application/handlers/status_update.py b/src/ahriman/application/handlers/status_update.py index 3b1637b0..161e8772 100644 --- a/src/ahriman/application/handlers/status_update.py +++ b/src/ahriman/application/handlers/status_update.py @@ -56,7 +56,7 @@ class StatusUpdate(Handler): if (local := next((package for package in packages if package.base == base), None)) is not None: client.package_add(local, args.status) else: - client.package_update(base, args.status) + client.package_set(base, args.status) case Action.Update: # update service status client.status_update(args.status) diff --git a/src/ahriman/core/database/operations/package_operations.py b/src/ahriman/core/database/operations/package_operations.py index 812d0009..78c2149d 100644 --- a/src/ahriman/core/database/operations/package_operations.py +++ b/src/ahriman/core/database/operations/package_operations.py @@ -150,34 +150,6 @@ class PackageOperations(Operations): """, package_list) - @staticmethod - def _package_update_insert_status(connection: Connection, package_base: str, status: BuildStatus, - repository_id: RepositoryId) -> None: - """ - insert base package status into table - - Args: - connection(Connection): database connection - package_base(str): package base name - status(BuildStatus): new build status - repository_id(RepositoryId): repository unique identifier - """ - connection.execute( - """ - insert into package_statuses - (package_base, status, last_updated, repository) - values - (:package_base, :status, :last_updated, :repository) - on conflict (package_base, repository) do update set - status = :status, last_updated = :last_updated - """, - { - "package_base": package_base, - "status": status.status.value, - "last_updated": status.timestamp, - "repository": repository_id.id, - }) - @staticmethod def _packages_get_select_package_bases(connection: Connection, repository_id: RepositoryId) -> dict[str, Package]: """ @@ -277,20 +249,18 @@ class PackageOperations(Operations): return self.with_connection(run, commit=True) - def package_update(self, package: Package, status: BuildStatus, repository_id: RepositoryId | None = None) -> None: + def package_update(self, package: Package, repository_id: RepositoryId | None = None) -> None: """ update package status Args: package(Package): package properties - status(BuildStatus): new build status repository_id(RepositoryId, optional): repository unique identifier override (Default value = None) """ repository_id = repository_id or self._repository_id def run(connection: Connection) -> None: self._package_update_insert_base(connection, package, repository_id) - self._package_update_insert_status(connection, package.base, status, repository_id) self._package_update_insert_packages(connection, package, repository_id) self._package_remove_packages(connection, package.base, package.packages.keys(), repository_id) @@ -336,3 +306,33 @@ class PackageOperations(Operations): package_base: package.remote for package_base, package in self.with_connection(run).items() } + + def status_update(self, package_base: str, status: BuildStatus, repository_id: RepositoryId | None = None) -> None: + """ + insert base package status into table + + Args: + package_base(str): package base name + status(BuildStatus): new build status + repository_id(RepositoryId, optional): repository unique identifier override (Default value = None) + """ + repository_id = repository_id or self._repository_id + + def run(connection: Connection) -> None: + connection.execute( + """ + insert into package_statuses + (package_base, status, last_updated, repository) + values + (:package_base, :status, :last_updated, :repository) + on conflict (package_base, repository) do update set + status = :status, last_updated = :last_updated + """, + { + "package_base": package_base, + "status": status.status.value, + "last_updated": status.timestamp, + "repository": repository_id.id, + }) + + return self.with_connection(run, commit=True) diff --git a/src/ahriman/core/log/http_log_handler.py b/src/ahriman/core/log/http_log_handler.py index f2b0fde3..dba4245b 100644 --- a/src/ahriman/core/log/http_log_handler.py +++ b/src/ahriman/core/log/http_log_handler.py @@ -92,7 +92,7 @@ class HttpLogHandler(logging.Handler): return # in case if no package base supplied we need just skip log message try: - self.reporter.package_logs(log_record_id, record) + self.reporter.package_logs_add(log_record_id, record.created, record.getMessage()) except Exception: if self.suppress_errors: return diff --git a/src/ahriman/core/repository/repository_properties.py b/src/ahriman/core/repository/repository_properties.py index 714b3c6d..42f30a15 100644 --- a/src/ahriman/core/repository/repository_properties.py +++ b/src/ahriman/core/repository/repository_properties.py @@ -75,7 +75,7 @@ class RepositoryProperties(LazyLogging): self.pacman = Pacman(repository_id, configuration, refresh_database=refresh_pacman_database) self.sign = GPG(configuration) self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args) - self.reporter = Client.load(repository_id, configuration, report=report) + self.reporter = Client.load(repository_id, configuration, database, report=report) self.triggers = TriggerLoader.load(repository_id, configuration) @property diff --git a/src/ahriman/core/status/client.py b/src/ahriman/core/status/client.py index 585f2aec..8a7a0362 100644 --- a/src/ahriman/core/status/client.py +++ b/src/ahriman/core/status/client.py @@ -17,16 +17,18 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +# pylint: disable=too-many-public-methods from __future__ import annotations -import logging - from ahriman.core.configuration import Configuration +from ahriman.core.database import SQLite from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.changes import Changes +from ahriman.models.dependencies import Dependencies from ahriman.models.internal_status import InternalStatus from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package +from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.repository_id import RepositoryId @@ -36,22 +38,31 @@ class Client: """ @staticmethod - def load(repository_id: RepositoryId, configuration: Configuration, *, report: bool) -> Client: + def load(repository_id: RepositoryId, configuration: Configuration, database: SQLite | None = None, *, + report: bool = True) -> Client: """ load client from settings Args: repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance - report(bool): force enable or disable reporting + database(SQLite | None, optional): database instance (Default value = None) + report(bool, optional): force enable or disable reporting (Default value = True) Returns: Client: client according to current settings """ + def make_local_client() -> Client: + if database is None: + return Client() + + from ahriman.core.status.local_client import LocalClient + return LocalClient(repository_id, database) + if not report: - return Client() + return make_local_client() if not configuration.getboolean("status", "enabled", fallback=True): # global switch - return Client() + return make_local_client() # new-style section address = configuration.get("status", "address", fallback=None) @@ -65,7 +76,8 @@ class Client: if address or legacy_address or (host and port) or socket: from ahriman.core.status.web_client import WebClient return WebClient(repository_id, configuration) - return Client() + + return make_local_client() def package_add(self, package: Package, status: BuildStatusEnum) -> None: """ @@ -74,7 +86,11 @@ class Client: Args: package(Package): package properties status(BuildStatusEnum): current package build status + + Raises: + NotImplementedError: not implemented method """ + raise NotImplementedError def package_changes_get(self, package_base: str) -> Changes: """ @@ -85,9 +101,11 @@ class Client: Returns: Changes: package changes if available and empty object otherwise + + Raises: + NotImplementedError: not implemented method """ - del package_base - return Changes() + raise NotImplementedError def package_changes_set(self, package_base: str, changes: Changes) -> None: """ @@ -96,7 +114,38 @@ class Client: Args: package_base(str): package base to update changes(Changes): changes descriptor + + Raises: + NotImplementedError: not implemented method """ + raise NotImplementedError + + def package_dependencies_get(self, package_base: str | None) -> list[Dependencies]: + """ + get package dependencies + + Args: + package_base(str | None): package base to retrieve + + Returns: + list[Dependencies]: package implicit dependencies if available + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_dependencies_set(self, dependencies: Dependencies) -> None: + """ + update package dependencies + + Args: + dependencies(Dependencies): dependencies descriptor + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError def package_get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]: """ @@ -107,35 +156,118 @@ class Client: Returns: list[tuple[Package, BuildStatus]]: list of current package description and status if it has been found - """ - del package_base - return [] - def package_logs(self, log_record_id: LogRecordId, record: logging.LogRecord) -> None: + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: """ post log record Args: log_record_id(LogRecordId): log record id - record(logging.LogRecord): log record to post to api + created(float): log created timestamp + message(str): log message """ + def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: + """ + get package logs + + Args: + package_base(str): package base + limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1) + offset(int, optional): records offset (Default value = 0) + + Returns: + list[tuple[float, str]]: package logs + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_logs_remove(self, package_base: str, version: str | None) -> None: + """ + remove package logs + + Args: + package_base(str): package base + version(str | None): package version to remove logs. If None set, all logs will be removed + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_patches_add(self, package_base: str, patch: PkgbuildPatch) -> None: + """ + create or update package patch + + Args: + package_base(str): package base to update + patch(PkgbuildPatch): package patch + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_patches_get(self, package_base: str, variable: str | None) -> list[PkgbuildPatch]: + """ + get package patches + + Args: + package_base(str): package base to retrieve + variable(str | None): optional filter by patch variable + + Returns: + list[PkgbuildPatch]: list of patches for the specified package + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_patches_remove(self, package_base: str, variable: str | None) -> None: + """ + remove package patch + + Args: + package_base(str): package base to update + variable(str | None): patch name. If None set, all patches will be removed + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + def package_remove(self, package_base: str) -> None: """ remove packages from watcher Args: package_base(str): package base to remove - """ - def package_update(self, package_base: str, status: BuildStatusEnum) -> None: + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_set(self, package_base: str, status: BuildStatusEnum) -> None: """ update package build status. Unlike :func:`package_add()` it does not update package properties Args: package_base(str): package base to update status(BuildStatusEnum): current package build status + + Raises: + NotImplementedError: not implemented method """ + raise NotImplementedError def set_building(self, package_base: str) -> None: """ @@ -144,7 +276,7 @@ class Client: Args: package_base(str): package base to update """ - return self.package_update(package_base, BuildStatusEnum.Building) + return self.package_set(package_base, BuildStatusEnum.Building) def set_failed(self, package_base: str) -> None: """ @@ -153,7 +285,7 @@ class Client: Args: package_base(str): package base to update """ - return self.package_update(package_base, BuildStatusEnum.Failed) + return self.package_set(package_base, BuildStatusEnum.Failed) def set_pending(self, package_base: str) -> None: """ @@ -162,7 +294,7 @@ class Client: Args: package_base(str): package base to update """ - return self.package_update(package_base, BuildStatusEnum.Pending) + return self.package_set(package_base, BuildStatusEnum.Pending) def set_success(self, package: Package) -> None: """ diff --git a/src/ahriman/core/status/local_client.py b/src/ahriman/core/status/local_client.py new file mode 100644 index 00000000..6b117401 --- /dev/null +++ b/src/ahriman/core/status/local_client.py @@ -0,0 +1,208 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from ahriman.core.database import SQLite +from ahriman.core.status.client import Client +from ahriman.models.build_status import BuildStatus, BuildStatusEnum +from ahriman.models.changes import Changes +from ahriman.models.dependencies import Dependencies +from ahriman.models.log_record_id import LogRecordId +from ahriman.models.package import Package +from ahriman.models.pkgbuild_patch import PkgbuildPatch +from ahriman.models.repository_id import RepositoryId + + +class LocalClient(Client): + """ + local database handler + + Attributes: + database(SQLite): database instance + repository_id(RepositoryId): repository unique identifier + """ + + def __init__(self, repository_id: RepositoryId, database: SQLite) -> None: + """ + default constructor + + Args: + repository_id(RepositoryId): repository unique identifier + database(SQLite): database instance: + """ + self.database = database + self.repository_id = repository_id + + def package_add(self, package: Package, status: BuildStatusEnum) -> None: + """ + add new package with status + + Args: + package(Package): package properties + status(BuildStatusEnum): current package build status + """ + self.database.package_update(package, self.repository_id) + self.database.status_update(package.base, BuildStatus(status), self.repository_id) + + def package_changes_get(self, package_base: str) -> Changes: + """ + get package changes + + Args: + package_base(str): package base to retrieve + + Returns: + Changes: package changes if available and empty object otherwise + """ + return self.database.changes_get(package_base, self.repository_id) + + def package_changes_set(self, package_base: str, changes: Changes) -> None: + """ + update package changes + + Args: + package_base(str): package base to update + changes(Changes): changes descriptor + """ + self.database.changes_insert(package_base, changes, self.repository_id) + + def package_dependencies_get(self, package_base: str | None) -> list[Dependencies]: + """ + get package dependencies + + Args: + package_base(str | None): package base to retrieve + + Returns: + list[Dependencies]: package implicit dependencies if available + """ + return self.database.dependencies_get(package_base, self.repository_id) + + def package_dependencies_set(self, dependencies: Dependencies) -> None: + """ + update package dependencies + + Args: + dependencies(Dependencies): dependencies descriptor + """ + self.database.dependencies_insert(dependencies, self.repository_id) + + def package_get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]: + """ + get package status + + Args: + package_base(str | None): package base to get + + Returns: + list[tuple[Package, BuildStatus]]: list of current package description and status if it has been found + """ + packages = self.database.packages_get() + if package_base is None: + return packages + return [(package, status) for package, status in packages if package.base == package_base] + + def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: + """ + post log record + + Args: + log_record_id(LogRecordId): log record id + created(float): log created timestamp + message(str): log message + """ + self.database.logs_insert(log_record_id, created, message, self.repository_id) + + def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: + """ + get package logs + + Args: + package_base(str): package base + limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1) + offset(int, optional): records offset (Default value = 0) + + Returns: + list[tuple[float, str]]: package logs + """ + return self.database.logs_get(package_base, limit, offset, self.repository_id) + + def package_logs_remove(self, package_base: str, version: str | None) -> None: + """ + remove package logs + + Args: + package_base(str): package base + version(str | None): package version to remove logs. If None set, all logs will be removed + """ + self.database.logs_remove(package_base, version, self.repository_id) + + def package_patches_add(self, package_base: str, patch: PkgbuildPatch) -> None: + """ + create or update package patch + + Args: + package_base(str): package base to update + patch(PkgbuildPatch): package patch + """ + self.database.patches_insert(package_base, [patch]) + + def package_patches_get(self, package_base: str, variable: str | None) -> list[PkgbuildPatch]: + """ + get package patches + + Args: + package_base(str): package base to retrieve + variable(str | None): optional filter by patch variable + + Returns: + list[PkgbuildPatch]: list of patches for the specified package + """ + variables = [variable] if variable is not None else None + return self.database.patches_list(package_base, variables).get(package_base, []) + + def package_patches_remove(self, package_base: str, variable: str | None) -> None: + """ + remove package patch + + Args: + package_base(str): package base to update + variable(str | None): patch name. If None set, all patches will be removed + """ + variables = [variable] if variable is not None else None + self.database.patches_remove(package_base, variables) + + def package_remove(self, package_base: str) -> None: + """ + remove packages from watcher + + Args: + package_base(str): package base to remove + """ + self.database.package_remove(package_base, self.repository_id) + self.package_logs_remove(package_base, None) + + def package_set(self, package_base: str, status: BuildStatusEnum) -> None: + """ + update package build status. Unlike :func:`package_add()` it does not update package properties + + Args: + package_base(str): package base to update + status(BuildStatusEnum): current package build status + """ + self.database.status_update(package_base, BuildStatus(status), self.repository_id) diff --git a/src/ahriman/core/status/watcher.py b/src/ahriman/core/status/watcher.py index 7b36dae8..ce5219ff 100644 --- a/src/ahriman/core/status/watcher.py +++ b/src/ahriman/core/status/watcher.py @@ -19,15 +19,15 @@ # from threading import Lock -from ahriman.core.database import SQLite from ahriman.core.exceptions import UnknownPackageError from ahriman.core.log import LazyLogging +from ahriman.core.status.client import Client from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.changes import Changes +from ahriman.models.dependencies import Dependencies from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch -from ahriman.models.repository_id import RepositoryId class Watcher(LazyLogging): @@ -35,21 +35,18 @@ class Watcher(LazyLogging): package status watcher Attributes: - database(SQLite): database instance - repository_id(RepositoryId): repository unique identifier + client(Client): reporter instance status(BuildStatus): daemon status """ - def __init__(self, repository_id: RepositoryId, database: SQLite) -> None: + def __init__(self, client: Client) -> None: """ default constructor Args: - repository_id(RepositoryId): repository unique identifier - database(SQLite): database instance + client(Client): reporter instance """ - self.repository_id = repository_id - self.database = database + self.client = client self._lock = Lock() self._known: dict[str, tuple[Package, BuildStatus]] = {} @@ -76,7 +73,7 @@ class Watcher(LazyLogging): with self._lock: self._known = { package.base: (package, status) - for package, status in self.database.packages_get(self.repository_id) + for package, status in self.client.package_get(None) } def logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: @@ -91,8 +88,8 @@ class Watcher(LazyLogging): Returns: list[tuple[float, str]]: package logs """ - self.package_get(package_base) - return self.database.logs_get(package_base, limit, offset, self.repository_id) + _ = self.package_get(package_base) + return self.client.package_logs_get(package_base, limit, offset) def logs_remove(self, package_base: str, version: str | None) -> None: """ @@ -100,24 +97,24 @@ class Watcher(LazyLogging): Args: package_base(str): package base - version(str): package versio + version(str): package version """ - self.database.logs_remove(package_base, version, self.repository_id) + self.client.package_logs_remove(package_base, version) - def logs_update(self, log_record_id: LogRecordId, created: float, record: str) -> None: + def logs_update(self, log_record_id: LogRecordId, created: float, message: str) -> None: """ make new log record into database Args: log_record_id(LogRecordId): log record id created(float): log created timestamp - record(str): log record + message(str): log message """ if self._last_log_record_id != log_record_id: # there is new log record, so we remove old ones self.logs_remove(log_record_id.package_base, log_record_id.version) self._last_log_record_id = log_record_id - self.database.logs_insert(log_record_id, created, record, self.repository_id) + self.client.package_logs_add(log_record_id, created, message) def package_changes_get(self, package_base: str) -> Changes: """ @@ -129,8 +126,24 @@ class Watcher(LazyLogging): Returns: Changes: package changes if available """ - self.package_get(package_base) - return self.database.changes_get(package_base, self.repository_id) + _ = self.package_get(package_base) + return self.client.package_changes_get(package_base) + + def package_dependencies_get(self, package_base: str) -> Dependencies: + """ + retrieve package dependencies + + Args: + package_base(str): package base + + Returns: + Dependencies: package dependencies if available + """ + _ = self.package_get(package_base) + try: + return next(iter(self.client.package_dependencies_get(package_base))) + except StopIteration: + return Dependencies(package_base) def package_get(self, package_base: str) -> tuple[Package, BuildStatus]: """ @@ -160,7 +173,7 @@ class Watcher(LazyLogging): """ with self._lock: self._known.pop(package_base, None) - self.database.package_remove(package_base, self.repository_id) + self.client.package_remove(package_base) self.logs_remove(package_base, None) def package_update(self, package_base: str, status: BuildStatusEnum, package: Package | None) -> None: @@ -174,10 +187,9 @@ class Watcher(LazyLogging): """ if package is None: package, _ = self.package_get(package_base) - full_status = BuildStatus(status) with self._lock: - self._known[package_base] = (package, full_status) - self.database.package_update(package, full_status, self.repository_id) + self._known[package_base] = (package, BuildStatus(status)) + self.client.package_set(package_base, status) def patches_get(self, package_base: str, variable: str | None) -> list[PkgbuildPatch]: """ @@ -192,8 +204,7 @@ class Watcher(LazyLogging): """ # patches are package base based, we don't know (and don't differentiate) to which package does them belong # so here we skip checking if package exists or not - variables = [variable] if variable is not None else None - return self.database.patches_list(package_base, variables).get(package_base, []) + return self.client.package_patches_get(package_base, variable) def patches_remove(self, package_base: str, variable: str) -> None: """ @@ -203,7 +214,7 @@ class Watcher(LazyLogging): package_base(str): package base variable(str): patch variable name """ - self.database.patches_remove(package_base, [variable]) + self.client.package_patches_remove(package_base, variable) def patches_update(self, package_base: str, patch: PkgbuildPatch) -> None: """ @@ -213,7 +224,7 @@ class Watcher(LazyLogging): package_base(str): package base patch(PkgbuildPatch): package patch """ - self.database.patches_insert(package_base, [patch]) + self.client.package_patches_add(package_base, patch) def status_update(self, status: BuildStatusEnum) -> None: """ diff --git a/src/ahriman/core/status/web_client.py b/src/ahriman/core/status/web_client.py index 45a05a90..14738caa 100644 --- a/src/ahriman/core/status/web_client.py +++ b/src/ahriman/core/status/web_client.py @@ -18,7 +18,6 @@ # along with this program. If not, see . # import contextlib -import logging from urllib.parse import quote_plus as urlencode @@ -27,9 +26,11 @@ from ahriman.core.http import SyncAhrimanClient from ahriman.core.status.client import Client from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.changes import Changes +from ahriman.models.dependencies import Dependencies from ahriman.models.internal_status import InternalStatus from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package +from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.repository_id import RepositoryId @@ -92,10 +93,22 @@ class WebClient(Client, SyncAhrimanClient): package_base(str): package base Returns: - str: full url for web service for logs + str: full url for web service for changes """ return f"{self.address}/api/v1/packages/{urlencode(package_base)}/changes" + def _dependencies_url(self, package_base: str = "") -> str: + """ + get url for the dependencies api + + Args: + package_base(str, optional): package base (Default value = "") + + Returns: + str: full url for web service for dependencies + """ + return f"{self.address}/api/v1/packages/{urlencode(package_base)}/dependencies" + def _logs_url(self, package_base: str) -> str: """ get url for the logs api @@ -110,7 +123,7 @@ class WebClient(Client, SyncAhrimanClient): def _package_url(self, package_base: str = "") -> str: """ - url generator + package url generator Args: package_base(str, optional): package base to generate url (Default value = "") @@ -121,6 +134,20 @@ class WebClient(Client, SyncAhrimanClient): suffix = f"/{urlencode(package_base)}" if package_base else "" return f"{self.address}/api/v1/packages{suffix}" + def _patches_url(self, package_base: str, variable: str = "") -> str: + """ + patches url generator + + Args: + package_base(str): package base + variable(str, optional): patch variable name to generate url (Default value = "") + + Returns: + str: full url of web service for the package patch + """ + suffix = f"/{urlencode(variable)}" if variable else "" + return f"{self.address}/api/v1/packages/{urlencode(package_base)}/patches{suffix}" + def _status_url(self) -> str: """ get url for the status api @@ -177,6 +204,37 @@ class WebClient(Client, SyncAhrimanClient): self.make_request("POST", self._changes_url(package_base), params=self.repository_id.query(), json=changes.view()) + def package_dependencies_get(self, package_base: str | None) -> list[Dependencies]: + """ + get package dependencies + + Args: + package_base(str | None): package base to retrieve + + Returns: + list[Dependencies]: package implicit dependencies if available + """ + with contextlib.suppress(Exception): + response = self.make_request("GET", self._dependencies_url(package_base or ""), + params=self.repository_id.query()) + response_json = response.json() + + dependencies = response_json if package_base is None else [response_json] + return [Dependencies.from_json(dependencies) for dependencies in dependencies] + + return [] + + def package_dependencies_set(self, dependencies: Dependencies) -> None: + """ + update package dependencies + + Args: + dependencies(Dependencies): dependencies descriptor + """ + with contextlib.suppress(Exception): + self.make_request("POST", self._dependencies_url(dependencies.package_base), + params=self.repository_id.query(), json=dependencies.view()) + def package_get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]: """ get package status @@ -199,17 +257,18 @@ class WebClient(Client, SyncAhrimanClient): return [] - def package_logs(self, log_record_id: LogRecordId, record: logging.LogRecord) -> None: + def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: """ post log record Args: log_record_id(LogRecordId): log record id - record(logging.LogRecord): log record to post to api + created(float): log created timestamp + message(str): log message """ payload = { - "created": record.created, - "message": record.getMessage(), + "created": created, + "message": message, "version": log_record_id.version, } @@ -219,6 +278,83 @@ class WebClient(Client, SyncAhrimanClient): self.make_request("POST", self._logs_url(log_record_id.package_base), params=self.repository_id.query(), json=payload, suppress_errors=True) + def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: + """ + get package logs + + Args: + package_base(str): package base + limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1) + offset(int, optional): records offset (Default value = 0) + + Returns: + list[tuple[float, str]]: package logs + """ + with contextlib.suppress(Exception): + query = self.repository_id.query() + [("limit", str(limit)), ("offset", str(offset))] + response = self.make_request("GET", self._logs_url(package_base), params=query) + response_json = response.json() + + return [(record["created"], record["message"]) for record in response_json] + + return [] + + def package_logs_remove(self, package_base: str, version: str | None) -> None: + """ + remove package logs + + Args: + package_base(str): package base + version(str | None): package version to remove logs. If None set, all logs will be removed + """ + with contextlib.suppress(Exception): + query = self.repository_id.query() + if version is not None: + query += [("version", version)] + self.make_request("DELETE", self._logs_url(package_base), params=query) + + def package_patches_add(self, package_base: str, patch: PkgbuildPatch) -> None: + """ + create or update package patch + + Args: + package_base(str): package base to update + patch(PkgbuildPatch): package patch + """ + with contextlib.suppress(Exception): + self.make_request("POST", self._patches_url(package_base), json=patch.view()) + + def package_patches_get(self, package_base: str, variable: str | None) -> list[PkgbuildPatch]: + """ + get package patches + + Args: + package_base(str): package base to retrieve + variable(str | None): optional filter by patch variable + + Returns: + list[PkgbuildPatch]: list of patches for the specified package + """ + with contextlib.suppress(Exception): + response = self.make_request("GET", self._patches_url(package_base, variable or "")) + response_json = response.json() + + patches = response_json if variable is None else [response_json] + return [PkgbuildPatch.from_json(patch) for patch in patches] + + return [] + + def package_patches_remove(self, package_base: str, variable: str | None) -> None: + """ + remove package patch + + Args: + package_base(str): package base to update + variable(str | None): patch name. If None set, all patches will be removed + """ + with contextlib.suppress(Exception): + self.make_request("DELETE", self._patches_url(package_base, variable or "")) + def package_remove(self, package_base: str) -> None: """ remove packages from watcher @@ -229,7 +365,7 @@ class WebClient(Client, SyncAhrimanClient): with contextlib.suppress(Exception): self.make_request("DELETE", self._package_url(package_base), params=self.repository_id.query()) - def package_update(self, package_base: str, status: BuildStatusEnum) -> None: + def package_set(self, package_base: str, status: BuildStatusEnum) -> None: """ update package build status. Unlike :func:`package_add()` it does not update package properties diff --git a/src/ahriman/core/support/package_creator.py b/src/ahriman/core/support/package_creator.py index a95cbcba..83d0885f 100644 --- a/src/ahriman/core/support/package_creator.py +++ b/src/ahriman/core/support/package_creator.py @@ -68,4 +68,6 @@ class PackageCreator: database: SQLite = ctx.get(ContextKey("database", SQLite)) _, repository_id = self.configuration.check_loaded() package = Package.from_build(local_path, repository_id.architecture, None) - database.package_update(package, BuildStatus()) + + database.package_update(package) + database.status_update(package.base, BuildStatus()) diff --git a/src/ahriman/models/dependencies.py b/src/ahriman/models/dependencies.py index fed89841..685e12b7 100644 --- a/src/ahriman/models/dependencies.py +++ b/src/ahriman/models/dependencies.py @@ -17,8 +17,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields from pathlib import Path +from typing import Any, Self + +from ahriman.core.util import dataclass_view, filter_json @dataclass(frozen=True) @@ -33,3 +36,27 @@ class Dependencies: package_base: str paths: dict[Path, list[str]] = field(default_factory=dict) + + @classmethod + def from_json(cls, dump: dict[str, Any]) -> Self: + """ + construct dependencies from the json dump + + Args: + dump(dict[str, Any]): json dump body + + Returns: + Self: dependencies object + """ + # filter to only known fields + known_fields = [pair.name for pair in fields(cls)] + return cls(**filter_json(dump, known_fields)) + + def view(self) -> dict[str, Any]: + """ + generate json dependencies view + + Returns: + dict[str, Any]: json-friendly dictionary + """ + return dataclass_view(self) diff --git a/src/ahriman/models/pkgbuild_patch.py b/src/ahriman/models/pkgbuild_patch.py index 68dd9cea..e8193425 100644 --- a/src/ahriman/models/pkgbuild_patch.py +++ b/src/ahriman/models/pkgbuild_patch.py @@ -19,11 +19,11 @@ # import shlex -from dataclasses import dataclass +from dataclasses import dataclass, fields from pathlib import Path from typing import Any, Self -from ahriman.core.util import dataclass_view, unquote +from ahriman.core.util import dataclass_view, filter_json, unquote @dataclass(frozen=True) @@ -90,6 +90,21 @@ class PkgbuildPatch: return cls(key, value) + @classmethod + def from_json(cls, dump: dict[str, Any]) -> Self: + """ + construct patch descriptor from the json dump + + Args: + dump(dict[str, Any]): json dump body + + Returns: + Self: patch object + """ + # filter to only known fields + known_fields = [pair.name for pair in fields(cls)] + return cls(**filter_json(dump, known_fields)) + def serialize(self) -> str: """ serialize key-value pair into PKGBUILD string. List values will be put inside parentheses. All string diff --git a/src/ahriman/web/schemas/__init__.py b/src/ahriman/web/schemas/__init__.py index 3b3e4bd6..aac967f8 100644 --- a/src/ahriman/web/schemas/__init__.py +++ b/src/ahriman/web/schemas/__init__.py @@ -22,6 +22,7 @@ from ahriman.web.schemas.auth_schema import AuthSchema from ahriman.web.schemas.build_options_schema import BuildOptionsSchema from ahriman.web.schemas.changes_schema import ChangesSchema from ahriman.web.schemas.counters_schema import CountersSchema +from ahriman.web.schemas.dependencies_schema import DependenciesSchema from ahriman.web.schemas.error_schema import ErrorSchema from ahriman.web.schemas.file_schema import FileSchema from ahriman.web.schemas.info_schema import InfoSchema @@ -36,6 +37,7 @@ from ahriman.web.schemas.package_patch_schema import PackagePatchSchema from ahriman.web.schemas.package_properties_schema import PackagePropertiesSchema from ahriman.web.schemas.package_schema import PackageSchema from ahriman.web.schemas.package_status_schema import PackageStatusSchema, PackageStatusSimplifiedSchema +from ahriman.web.schemas.package_version_schema import PackageVersionSchema from ahriman.web.schemas.pagination_schema import PaginationSchema from ahriman.web.schemas.patch_name_schema import PatchNameSchema from ahriman.web.schemas.patch_schema import PatchSchema diff --git a/src/ahriman/web/schemas/dependencies_schema.py b/src/ahriman/web/schemas/dependencies_schema.py new file mode 100644 index 00000000..87cd496b --- /dev/null +++ b/src/ahriman/web/schemas/dependencies_schema.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from marshmallow import Schema, fields + + +class DependenciesSchema(Schema): + """ + request/response package dependencies schema + """ + + package_base = fields.String(metadata={ + "description": "Package base name", + "example": "ahriman", + }) + paths = fields.Dict( + keys=fields.String(), values=fields.List(fields.String()), required=True, metadata={ + "description": "Map of filesystem paths to packages which contain this path", + }) diff --git a/src/ahriman/web/schemas/package_version_schema.py b/src/ahriman/web/schemas/package_version_schema.py new file mode 100644 index 00000000..4b997b48 --- /dev/null +++ b/src/ahriman/web/schemas/package_version_schema.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from marshmallow import fields + +from ahriman import __version__ +from ahriman.web.schemas.repository_id_schema import RepositoryIdSchema + + +class PackageVersionSchema(RepositoryIdSchema): + """ + request package name schema + """ + + version = fields.String(required=True, metadata={ + "description": "Package version", + "example": __version__, + }) diff --git a/src/ahriman/web/views/v1/packages/changes.py b/src/ahriman/web/views/v1/packages/changes.py index b14eb282..7d113228 100644 --- a/src/ahriman/web/views/v1/packages/changes.py +++ b/src/ahriman/web/views/v1/packages/changes.py @@ -113,7 +113,6 @@ class ChangesView(StatusViewGuard, BaseView): raise HTTPBadRequest(reason=str(ex)) changes = Changes(last_commit_sha, change) - repository_id = self.repository_id() - self.service(repository_id).database.changes_insert(package_base, changes, repository_id) + self.service().client.package_changes_set(package_base, changes) raise HTTPNoContent diff --git a/src/ahriman/web/views/v1/packages/dependencies.py b/src/ahriman/web/views/v1/packages/dependencies.py new file mode 100644 index 00000000..777eba13 --- /dev/null +++ b/src/ahriman/web/views/v1/packages/dependencies.py @@ -0,0 +1,66 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import aiohttp_apispec # type: ignore[import-untyped] + +from aiohttp.web import Response, json_response + +from ahriman.models.user_access import UserAccess +from ahriman.web.schemas import AuthSchema, DependenciesSchema, ErrorSchema, RepositoryIdSchema +from ahriman.web.views.base import BaseView +from ahriman.web.views.status_view_guard import StatusViewGuard + + +class DependenciesView(StatusViewGuard, BaseView): + """ + packages dependencies web view + + Attributes: + GET_PERMISSION(UserAccess): (class attribute) get permissions of self + """ + + GET_PERMISSION = UserAccess.Reporter + POST_PERMISSION = UserAccess.Full + ROUTES = ["/api/v1/packages/dependencies"] + + @aiohttp_apispec.docs( + tags=["Packages"], + summary="Get dependencies for all packages", + description="Retrieve implicit dependencies for all known packages", + responses={ + 200: {"description": "Success response", "schema": DependenciesSchema(many=True)}, + 401: {"description": "Authorization required", "schema": ErrorSchema}, + 403: {"description": "Access is forbidden", "schema": ErrorSchema}, + 404: {"description": "Package base and/or repository are unknown", "schema": ErrorSchema}, + 500: {"description": "Internal server error", "schema": ErrorSchema}, + }, + security=[{"token": [GET_PERMISSION]}], + ) + @aiohttp_apispec.cookies_schema(AuthSchema) + @aiohttp_apispec.querystring_schema(RepositoryIdSchema) + async def get(self) -> Response: + """ + get dependencies for all packages + + Returns: + Response: 200 with package implicit dependencies on success + """ + dependencies = self.service().client.package_dependencies_get(None) + + return json_response([dependency.view() for dependency in dependencies]) diff --git a/src/ahriman/web/views/v1/packages/dependency.py b/src/ahriman/web/views/v1/packages/dependency.py new file mode 100644 index 00000000..595ba5ae --- /dev/null +++ b/src/ahriman/web/views/v1/packages/dependency.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import aiohttp_apispec # type: ignore[import-untyped] + +from aiohttp.web import HTTPBadRequest, HTTPNoContent, HTTPNotFound, Response, json_response + +from ahriman.core.exceptions import UnknownPackageError +from ahriman.models.dependencies import Dependencies +from ahriman.models.user_access import UserAccess +from ahriman.web.schemas import AuthSchema, DependenciesSchema, ErrorSchema, PackageNameSchema, RepositoryIdSchema +from ahriman.web.views.base import BaseView +from ahriman.web.views.status_view_guard import StatusViewGuard + + +class DependencyView(StatusViewGuard, BaseView): + """ + package dependencies web view + + Attributes: + GET_PERMISSION(UserAccess): (class attribute) get permissions of self + POST_PERMISSION(UserAccess): (class attribute) post permissions of self + """ + + GET_PERMISSION = UserAccess.Reporter + POST_PERMISSION = UserAccess.Full + ROUTES = ["/api/v1/packages/{package}/dependencies"] + + @aiohttp_apispec.docs( + tags=["Packages"], + summary="Get package dependencies", + description="Retrieve package implicit dependencies", + responses={ + 200: {"description": "Success response", "schema": DependenciesSchema}, + 401: {"description": "Authorization required", "schema": ErrorSchema}, + 403: {"description": "Access is forbidden", "schema": ErrorSchema}, + 404: {"description": "Package base and/or repository are unknown", "schema": ErrorSchema}, + 500: {"description": "Internal server error", "schema": ErrorSchema}, + }, + security=[{"token": [GET_PERMISSION]}], + ) + @aiohttp_apispec.cookies_schema(AuthSchema) + @aiohttp_apispec.match_info_schema(PackageNameSchema) + @aiohttp_apispec.querystring_schema(RepositoryIdSchema) + async def get(self) -> Response: + """ + get package dependencies + + Returns: + Response: 200 with package implicit dependencies on success + + Raises: + HTTPNotFound: if package base is unknown + """ + package_base = self.request.match_info["package"] + + try: + dependencies = self.service().package_dependencies_get(package_base) + except UnknownPackageError: + raise HTTPNotFound(reason=f"Package {package_base} is unknown") + + return json_response(dependencies.view()) + + @aiohttp_apispec.docs( + tags=["Packages"], + summary="Update package dependencies", + description="Set package implicit dependencies", + responses={ + 204: {"description": "Success response"}, + 400: {"description": "Bad data is supplied", "schema": ErrorSchema}, + 401: {"description": "Authorization required", "schema": ErrorSchema}, + 403: {"description": "Access is forbidden", "schema": ErrorSchema}, + 404: {"description": "Repository is unknown", "schema": ErrorSchema}, + 500: {"description": "Internal server error", "schema": ErrorSchema}, + }, + security=[{"token": [POST_PERMISSION]}], + ) + @aiohttp_apispec.cookies_schema(AuthSchema) + @aiohttp_apispec.match_info_schema(PackageNameSchema) + @aiohttp_apispec.querystring_schema(RepositoryIdSchema) + @aiohttp_apispec.json_schema(DependenciesSchema) + async def post(self) -> None: + """ + insert new package dependencies + + Raises: + HTTPBadRequest: if bad data is supplied + HTTPNoContent: in case of success response + """ + package_base = self.request.match_info["package"] + + try: + data = await self.request.json() + data["package_base"] = package_base # read from path instead of object + dependencies = Dependencies.from_json(data) + except Exception as ex: + raise HTTPBadRequest(reason=str(ex)) + + self.service().client.package_dependencies_set(dependencies) + + raise HTTPNoContent diff --git a/src/ahriman/web/views/v1/packages/logs.py b/src/ahriman/web/views/v1/packages/logs.py index af527c46..68d61f4b 100644 --- a/src/ahriman/web/views/v1/packages/logs.py +++ b/src/ahriman/web/views/v1/packages/logs.py @@ -25,8 +25,8 @@ from ahriman.core.exceptions import UnknownPackageError from ahriman.core.util import pretty_datetime from ahriman.models.log_record_id import LogRecordId from ahriman.models.user_access import UserAccess -from ahriman.web.schemas import AuthSchema, ErrorSchema, LogsSchema, PackageNameSchema, RepositoryIdSchema, \ - VersionedLogSchema +from ahriman.web.schemas import AuthSchema, ErrorSchema, LogsSchema, PackageNameSchema, PackageVersionSchema, \ + RepositoryIdSchema, VersionedLogSchema from ahriman.web.views.base import BaseView from ahriman.web.views.status_view_guard import StatusViewGuard @@ -60,7 +60,7 @@ class LogsView(StatusViewGuard, BaseView): ) @aiohttp_apispec.cookies_schema(AuthSchema) @aiohttp_apispec.match_info_schema(PackageNameSchema) - @aiohttp_apispec.querystring_schema(RepositoryIdSchema) + @aiohttp_apispec.querystring_schema(PackageVersionSchema) async def delete(self) -> None: """ delete package logs @@ -69,7 +69,8 @@ class LogsView(StatusViewGuard, BaseView): HTTPNoContent: on success response """ package_base = self.request.match_info["package"] - self.service().logs_remove(package_base, None) + version = self.request.query.get("version") + self.service().logs_remove(package_base, version) raise HTTPNoContent diff --git a/src/ahriman/web/web.py b/src/ahriman/web/web.py index 64b48ae8..6d7fd315 100644 --- a/src/ahriman/web/web.py +++ b/src/ahriman/web/web.py @@ -30,6 +30,7 @@ from ahriman.core.database import SQLite from ahriman.core.distributed import WorkersCache from ahriman.core.exceptions import InitializeError from ahriman.core.spawn import Spawn +from ahriman.core.status.client import Client from ahriman.core.status.watcher import Watcher from ahriman.models.repository_id import RepositoryId from ahriman.web.apispec import setup_apispec @@ -167,7 +168,8 @@ def setup_server(configuration: Configuration, spawner: Spawn, repositories: lis watchers: dict[RepositoryId, Watcher] = {} for repository_id in repositories: application.logger.info("load repository %s", repository_id) - watchers[repository_id] = Watcher(repository_id, database) + client = Client.load(repository_id, configuration, database, report=False) # explicitly load local client + watchers[repository_id] = Watcher(client) application[WatcherKey] = watchers # workers cache application[WorkersKey] = WorkersCache(configuration)