From 6f57ed550b1d5eedd817ec160128eeccead9fae3 Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Sun, 9 Mar 2025 14:46:33 +0200 Subject: [PATCH] feat: refine log system (#142) * refine package logging * add interface * revert version selection * replace tuple with model * rename column in logs table, add coverters * generate process identifier for child proocesses --- package/share/ahriman/settings/ahriman.ini | 2 + .../ahriman/templates/build-status.jinja2 | 4 +- .../build-status/package-info-modal.jinja2 | 54 +++++++++-- .../ahriman/templates/utils/style.jinja2 | 6 ++ src/ahriman/core/configuration/schema.py | 5 + .../migrations/m015_logs_process_id.py | 30 ++++++ .../database/operations/logs_operations.py | 81 ++++++++++++---- src/ahriman/core/log/http_log_handler.py | 17 +++- src/ahriman/core/log/lazy_logging.py | 23 +---- src/ahriman/core/repository/update_handler.py | 3 +- src/ahriman/core/status/client.py | 20 ++-- src/ahriman/core/status/local_client.py | 23 +++-- src/ahriman/core/status/watcher.py | 24 +---- src/ahriman/core/status/web_client.py | 35 ++++--- src/ahriman/models/event.py | 2 +- src/ahriman/models/log_record.py | 76 +++++++++++++++ src/ahriman/models/log_record_id.py | 15 +++ src/ahriman/models/package.py | 9 +- src/ahriman/web/apispec/decorators.py | 3 +- src/ahriman/web/schemas/__init__.py | 2 +- src/ahriman/web/schemas/log_schema.py | 10 +- ...ed_log_schema.py => logs_rotate_schema.py} | 14 +-- src/ahriman/web/views/v1/packages/logs.py | 15 ++- src/ahriman/web/views/v1/service/logs.py | 63 +++++++++++++ src/ahriman/web/views/v2/packages/logs.py | 7 +- .../migrations/test_m015_logs_process_id.py | 8 ++ .../operations/test_logs_operations.py | 93 ++++++++++++++----- .../ahriman/core/log/test_http_log_handler.py | 18 +++- tests/ahriman/core/log/test_lazy_logging.py | 10 -- tests/ahriman/core/status/test_client.py | 10 +- .../ahriman/core/status/test_local_client.py | 16 +++- tests/ahriman/core/status/test_watcher.py | 33 ------- tests/ahriman/core/status/test_web_client.py | 81 ++++++++++++++-- tests/ahriman/models/test_log_record.py | 13 +++ tests/ahriman/models/test_log_record_id.py | 9 ++ ...g_schema.py => test_logs_rotate_schema.py} | 0 .../v1/service/test_view_v1_service_logs.py | 77 +++++++++++++++ .../v2/packages/test_view_v2_packages_logs.py | 9 +- 38 files changed, 711 insertions(+), 209 deletions(-) create mode 100644 src/ahriman/core/database/migrations/m015_logs_process_id.py create mode 100644 src/ahriman/models/log_record.py rename src/ahriman/web/schemas/{versioned_log_schema.py => logs_rotate_schema.py} (65%) create mode 100644 src/ahriman/web/views/v1/service/logs.py create mode 100644 tests/ahriman/core/database/migrations/test_m015_logs_process_id.py create mode 100644 tests/ahriman/models/test_log_record.py rename tests/ahriman/web/schemas/{test_versioned_log_schema.py => test_logs_rotate_schema.py} (100%) create mode 100644 tests/ahriman/web/views/v1/service/test_view_v1_service_logs.py diff --git a/package/share/ahriman/settings/ahriman.ini b/package/share/ahriman/settings/ahriman.ini index d7e2f328..e53a1fa2 100644 --- a/package/share/ahriman/settings/ahriman.ini +++ b/package/share/ahriman/settings/ahriman.ini @@ -7,6 +7,8 @@ logging = ahriman.ini.d/logging.ini ;apply_migrations = yes ; Path to the application SQLite database. database = ${repository:root}/ahriman.db +; Keep last build logs for each package +keep_last_logs = 5 [alpm] ; Path to pacman system database cache. diff --git a/package/share/ahriman/templates/build-status.jinja2 b/package/share/ahriman/templates/build-status.jinja2 index 86b52e8b..0309ba7d 100644 --- a/package/share/ahriman/templates/build-status.jinja2 +++ b/package/share/ahriman/templates/build-status.jinja2 @@ -16,11 +16,11 @@
-
+
+
+ +
+
+
+
+
@@ -100,6 +107,7 @@ const packageInfoModalHeader = document.getElementById("package-info-modal-header"); const packageInfo = document.getElementById("package-info"); + const packageInfoLogsVersions = document.getElementById("package-info-logs-versions"); const packageInfoLogsInput = document.getElementById("package-info-logs-input"); const packageInfoLogsCopyButton = document.getElementById("package-info-logs-copy-button"); @@ -285,11 +293,45 @@ convert: response => response.json(), }, data => { - const logs = data.map(log_record => { - return `[${new Date(1000 * log_record.created).toISOString()}] ${log_record.message}`; - }); - packageInfoLogsInput.textContent = logs.join("\n"); - highlight(packageInfoLogsInput); + const selectors = Object + .values( + data.reduce((acc, log_record) => { + const id = `${log_record.version}-${log_record.process_id}`; + if (acc[id]) + acc[id].created = Math.min(log_record.created, acc[id].created); + else + acc[id] = log_record; + return acc; + }, {}) + ) + .sort(({created: left}, {created: right}) => + right - left + ) + .map(version => { + const link = document.createElement("a"); + link.classList.add("nav-link"); + + link.textContent = version.version; + link.href = "#"; + link.onclick = _ => { + const logs = data + .filter(log_record => log_record.version === version.version && log_record.process_id === version.process_id) + .map(log_record => `[${new Date(1000 * log_record.created).toISOString()}] ${log_record.message}`); + + packageInfoLogsInput.textContent = logs.join("\n"); + highlight(packageInfoLogsInput); + + Array.from(packageInfoLogsVersions.children).forEach(el => el.classList.remove("active")); + link.classList.add("active"); + + return false; + }; + + return link; + }); + + packageInfoLogsVersions.replaceChildren(...selectors); + selectors.find(Boolean)?.click(); }, onFailure, ); diff --git a/package/share/ahriman/templates/utils/style.jinja2 b/package/share/ahriman/templates/utils/style.jinja2 index 5641adf1..d736dcee 100644 --- a/package/share/ahriman/templates/utils/style.jinja2 +++ b/package/share/ahriman/templates/utils/style.jinja2 @@ -27,4 +27,10 @@ top: 0; right: 5px; } + + .nav-link.active { + pointer-events: none; + cursor: default; + color: black !important; + } diff --git a/src/ahriman/core/configuration/schema.py b/src/ahriman/core/configuration/schema.py index 8c93fc43..008db9b4 100644 --- a/src/ahriman/core/configuration/schema.py +++ b/src/ahriman/core/configuration/schema.py @@ -45,6 +45,11 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = { "path_exists": True, "path_type": "dir", }, + "keep_last_logs": { + "type": "integer", + "coerce": "integer", + "min": 0, + }, "logging": { "type": "path", "coerce": "absolute_path", diff --git a/src/ahriman/core/database/migrations/m015_logs_process_id.py b/src/ahriman/core/database/migrations/m015_logs_process_id.py new file mode 100644 index 00000000..81f29481 --- /dev/null +++ b/src/ahriman/core/database/migrations/m015_logs_process_id.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2021-2025 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +__all__ = ["steps"] + + +steps = [ + """ + alter table logs add column process_id text not null default '' + """, + """ + alter table logs rename column record to message + """, +] diff --git a/src/ahriman/core/database/operations/logs_operations.py b/src/ahriman/core/database/operations/logs_operations.py index 8704df49..4dc5d0e6 100644 --- a/src/ahriman/core/database/operations/logs_operations.py +++ b/src/ahriman/core/database/operations/logs_operations.py @@ -20,7 +20,7 @@ from sqlite3 import Connection from ahriman.core.database.operations.operations import Operations -from ahriman.models.log_record_id import LogRecordId +from ahriman.models.log_record import LogRecord from ahriman.models.repository_id import RepositoryId @@ -30,7 +30,7 @@ class LogsOperations(Operations): """ def logs_get(self, package_base: str, limit: int = -1, offset: int = 0, - repository_id: RepositoryId | None = None) -> list[tuple[float, str]]: + repository_id: RepositoryId | None = None) -> list[LogRecord]: """ extract logs for specified package base @@ -41,16 +41,16 @@ class LogsOperations(Operations): repository_id(RepositoryId, optional): repository unique identifier override (Default value = None) Return: - list[tuple[float, str]]: sorted package log records and their timestamps + list[LogRecord]: sorted package log records """ repository_id = repository_id or self._repository_id - def run(connection: Connection) -> list[tuple[float, str]]: + def run(connection: Connection) -> list[LogRecord]: return [ - (row["created"], row["record"]) + LogRecord.from_json(package_base, row) for row in connection.execute( """ - select created, record from ( + select created, message, version, process_id from ( select * from logs where package_base = :package_base and repository = :repository order by created desc limit :limit offset :offset @@ -66,15 +66,12 @@ class LogsOperations(Operations): return self.with_connection(run) - def logs_insert(self, log_record_id: LogRecordId, created: float, record: str, - repository_id: RepositoryId | None = None) -> None: + def logs_insert(self, log_record: LogRecord, repository_id: RepositoryId | None = None) -> None: """ write new log record to database Args: - log_record_id(LogRecordId): current log record id - created(float): log created timestamp from log record attribute - record(str): log record + log_record(LogRecord): log record object repository_id(RepositoryId, optional): repository unique identifier override (Default value = None) """ repository_id = repository_id or self._repository_id @@ -83,17 +80,14 @@ class LogsOperations(Operations): connection.execute( """ insert into logs - (package_base, version, created, record, repository) + (package_base, version, created, message, repository, process_id) values - (:package_base, :version, :created, :record, :repository) + (:package_base, :version, :created, :message, :repository, :process_id) """, { - "package_base": log_record_id.package_base, - "version": log_record_id.version, - "created": created, - "record": record, + "package_base": log_record.log_record_id.package_base, "repository": repository_id.id, - } + } | log_record.view() ) return self.with_connection(run, commit=True) @@ -125,3 +119,54 @@ class LogsOperations(Operations): ) return self.with_connection(run, commit=True) + + def logs_rotate(self, keep_last_records: int, repository_id: RepositoryId | None = None) -> None: + """ + rotate logs in storage. This method will remove old logs, keeping only the last N records for each package + + Args: + keep_last_records(int): number of last records to keep + repository_id(RepositoryId, optional): repository unique identifier override (Default value = None) + """ + repository_id = repository_id or self._repository_id + + def remove_duplicates(connection: Connection) -> None: + connection.execute( + """ + delete from logs + where (package_base, version, repository, process_id) not in ( + select package_base, version, repository, process_id from logs + where (package_base, version, repository, created) in ( + select package_base, version, repository, max(created) from logs + where repository = :repository + group by package_base, version, repository + ) + ) + """, + { + "repository": repository_id.id, + } + ) + + def remove_older(connection: Connection) -> None: + connection.execute( + """ + delete from logs + where (package_base, repository, process_id) in ( + select package_base, repository, process_id from logs + where repository = :repository + group by package_base, repository, process_id + order by min(created) desc limit -1 offset :offset + ) + """, + { + "offset": keep_last_records, + "repository": repository_id.id, + } + ) + + def run(connection: Connection) -> None: + remove_duplicates(connection) + remove_older(connection) + + return self.with_connection(run, commit=True) diff --git a/src/ahriman/core/log/http_log_handler.py b/src/ahriman/core/log/http_log_handler.py index 769692dc..bb90f1a8 100644 --- a/src/ahriman/core/log/http_log_handler.py +++ b/src/ahriman/core/log/http_log_handler.py @@ -17,12 +17,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +import atexit import logging +import uuid from typing import Self from ahriman.core.configuration import Configuration from ahriman.core.status import Client +from ahriman.models.log_record import LogRecord +from ahriman.models.log_record_id import LogRecordId from ahriman.models.repository_id import RepositoryId @@ -33,6 +37,7 @@ class HttpLogHandler(logging.Handler): method Attributes: + keep_last_records(int): number of last records to keep reporter(Client): build status reporter instance suppress_errors(bool): suppress logging errors (e.g. if no web server available) """ @@ -51,6 +56,7 @@ class HttpLogHandler(logging.Handler): self.reporter = Client.load(repository_id, configuration, report=report) self.suppress_errors = suppress_errors + self.keep_last_records = configuration.getint("settings", "keep_last_logs", fallback=0) @classmethod def load(cls, repository_id: RepositoryId, configuration: Configuration, *, report: bool) -> Self: @@ -76,6 +82,9 @@ class HttpLogHandler(logging.Handler): handler = cls(repository_id, configuration, report=report, suppress_errors=suppress_errors) root.addHandler(handler) + LogRecordId.DEFAULT_PROCESS_ID = str(uuid.uuid4()) # assign default process identifier for log records + atexit.register(handler.rotate) + return handler def emit(self, record: logging.LogRecord) -> None: @@ -90,8 +99,14 @@ class HttpLogHandler(logging.Handler): return # in case if no package base supplied we need just skip log message try: - self.reporter.package_logs_add(log_record_id, record.created, record.getMessage()) + self.reporter.package_logs_add(LogRecord(log_record_id, record.created, record.getMessage())) except Exception: if self.suppress_errors: return self.handleError(record) + + def rotate(self) -> None: + """ + rotate log records, removing older ones + """ + self.reporter.logs_rotate(self.keep_last_records) diff --git a/src/ahriman/core/log/lazy_logging.py b/src/ahriman/core/log/lazy_logging.py index 2e6b4b02..8a69fd6c 100644 --- a/src/ahriman/core/log/lazy_logging.py +++ b/src/ahriman/core/log/lazy_logging.py @@ -74,7 +74,7 @@ class LazyLogging: def package_record_factory(*args: Any, **kwargs: Any) -> logging.LogRecord: record = current_factory(*args, **kwargs) - record.package_id = LogRecordId(package_base, version or "") + record.package_id = LogRecordId(package_base, version or "") return record logging.setLogRecordFactory(package_record_factory) @@ -99,24 +99,3 @@ class LazyLogging: yield finally: self._package_logger_reset() - - @contextlib.contextmanager - def suppress_logging(self, log_level: int = logging.WARNING) -> Generator[None, None, None]: - """ - silence log messages in context - - Args: - log_level(int, optional): the highest log level to keep (Default value = logging.WARNING) - - Examples: - This function is designed to be used to suppress all log messages in context, e.g.: - - >>> with self.suppress_logging(): - >>> do_some_noisy_actions() - """ - current_level = self.logger.manager.disable - try: - logging.disable(log_level) - yield - finally: - logging.disable(current_level) diff --git a/src/ahriman/core/repository/update_handler.py b/src/ahriman/core/repository/update_handler.py index 83d3d989..225bd150 100644 --- a/src/ahriman/core/repository/update_handler.py +++ b/src/ahriman/core/repository/update_handler.py @@ -144,8 +144,7 @@ class UpdateHandler(PackageInfo, Cleaner): branch="master", ) - with self.suppress_logging(): - Sources.fetch(cache_dir, source) + Sources.fetch(cache_dir, source) remote = Package.from_build(cache_dir, self.architecture, None) local = packages.get(remote.base) diff --git a/src/ahriman/core/status/client.py b/src/ahriman/core/status/client.py index b7de1b65..948b0a91 100644 --- a/src/ahriman/core/status/client.py +++ b/src/ahriman/core/status/client.py @@ -27,7 +27,7 @@ from ahriman.models.changes import Changes from ahriman.models.dependencies import Dependencies from ahriman.models.event import Event, EventType from ahriman.models.internal_status import InternalStatus -from ahriman.models.log_record_id import LogRecordId +from ahriman.models.log_record import LogRecord from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.repository_id import RepositoryId @@ -115,6 +115,14 @@ class Client: """ raise NotImplementedError + def logs_rotate(self, keep_last_records: int) -> None: + """ + remove older logs from storage + + Args: + keep_last_records(int): number of last records to keep + """ + def package_changes_get(self, package_base: str) -> Changes: """ get package changes @@ -186,18 +194,16 @@ class Client: """ raise NotImplementedError - def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: + def package_logs_add(self, log_record: LogRecord) -> None: """ post log record Args: - log_record_id(LogRecordId): log record id - created(float): log created timestamp - message(str): log message + log_record(LogRecord): log record """ # this method does not raise NotImplementedError because it is actively used as dummy client for http log - def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: + def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[LogRecord]: """ get package logs @@ -207,7 +213,7 @@ class Client: offset(int, optional): records offset (Default value = 0) Returns: - list[tuple[float, str]]: package logs + list[LogRecord]: package logs Raises: NotImplementedError: not implemented method diff --git a/src/ahriman/core/status/local_client.py b/src/ahriman/core/status/local_client.py index 6c0608ee..c3266ae4 100644 --- a/src/ahriman/core/status/local_client.py +++ b/src/ahriman/core/status/local_client.py @@ -23,7 +23,7 @@ from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.changes import Changes from ahriman.models.dependencies import Dependencies from ahriman.models.event import Event, EventType -from ahriman.models.log_record_id import LogRecordId +from ahriman.models.log_record import LogRecord from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.repository_id import RepositoryId @@ -75,6 +75,15 @@ class LocalClient(Client): """ return self.database.event_get(event, object_id, from_date, to_date, limit, offset, self.repository_id) + def logs_rotate(self, keep_last_records: int) -> None: + """ + remove older logs from storage + + Args: + keep_last_records(int): number of last records to keep + """ + self.database.logs_rotate(keep_last_records, self.repository_id) + def package_changes_get(self, package_base: str) -> Changes: """ get package changes @@ -134,18 +143,16 @@ class LocalClient(Client): return packages return [(package, status) for package, status in packages if package.base == package_base] - def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: + def package_logs_add(self, log_record: LogRecord) -> None: """ post log record Args: - log_record_id(LogRecordId): log record id - created(float): log created timestamp - message(str): log message + log_record(LogRecord): log record """ - self.database.logs_insert(log_record_id, created, message, self.repository_id) + self.database.logs_insert(log_record, self.repository_id) - def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: + def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[LogRecord]: """ get package logs @@ -155,7 +162,7 @@ class LocalClient(Client): offset(int, optional): records offset (Default value = 0) Returns: - list[tuple[float, str]]: package logs + list[LogRecord]: package logs """ return self.database.logs_get(package_base, limit, offset, self.repository_id) diff --git a/src/ahriman/core/status/watcher.py b/src/ahriman/core/status/watcher.py index ee541094..525fd05d 100644 --- a/src/ahriman/core/status/watcher.py +++ b/src/ahriman/core/status/watcher.py @@ -28,7 +28,7 @@ from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.changes import Changes from ahriman.models.dependencies import Dependencies from ahriman.models.event import Event, EventType -from ahriman.models.log_record_id import LogRecordId +from ahriman.models.log_record import LogRecord from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch @@ -53,9 +53,6 @@ class Watcher(LazyLogging): self._known: dict[str, tuple[Package, BuildStatus]] = {} self.status = BuildStatus() - # special variables for updating logs - self._last_log_record_id = LogRecordId("", "") - @property def packages(self) -> list[tuple[Package, BuildStatus]]: """ @@ -81,6 +78,8 @@ class Watcher(LazyLogging): for package, status in self.client.package_get(None) } + logs_rotate: Callable[[int], None] + package_changes_get: Callable[[str], Changes] package_changes_update: Callable[[str, Changes], None] @@ -108,22 +107,9 @@ class Watcher(LazyLogging): except KeyError: raise UnknownPackageError(package_base) from None - def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: - """ - make new log record into database + package_logs_add: Callable[[LogRecord], None] - Args: - log_record_id(LogRecordId): log record id - created(float): log created timestamp - message(str): log message - """ - if self._last_log_record_id != log_record_id: - # there is new log record, so we remove old ones - self.package_logs_remove(log_record_id.package_base, log_record_id.version) - self._last_log_record_id = log_record_id - self.client.package_logs_add(log_record_id, created, message) - - package_logs_get: Callable[[str, int, int], list[tuple[float, str]]] + package_logs_get: Callable[[str, int, int], list[LogRecord]] package_logs_remove: Callable[[str, str | None], None] diff --git a/src/ahriman/core/status/web_client.py b/src/ahriman/core/status/web_client.py index 158cf9cd..e492de18 100644 --- a/src/ahriman/core/status/web_client.py +++ b/src/ahriman/core/status/web_client.py @@ -29,7 +29,7 @@ from ahriman.models.changes import Changes from ahriman.models.dependencies import Dependencies from ahriman.models.event import Event, EventType from ahriman.models.internal_status import InternalStatus -from ahriman.models.log_record_id import LogRecordId +from ahriman.models.log_record import LogRecord from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.repository_id import RepositoryId @@ -210,6 +210,18 @@ class WebClient(Client, SyncAhrimanClient): return [] + def logs_rotate(self, keep_last_records: int) -> None: + """ + remove older logs from storage + + Args: + keep_last_records(int): number of last records to keep + """ + query = self.repository_id.query() + [("keep_last_records", str(keep_last_records))] + + with contextlib.suppress(Exception): + self.make_request("DELETE", f"{self.address}/api/v1/service/logs", params=query) + def package_changes_get(self, package_base: str) -> Changes: """ get package changes @@ -294,28 +306,27 @@ class WebClient(Client, SyncAhrimanClient): return [] - def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: + def package_logs_add(self, log_record: LogRecord) -> None: """ post log record Args: - log_record_id(LogRecordId): log record id - created(float): log created timestamp - message(str): log message + log_record(LogRecord): log record """ payload = { - "created": created, - "message": message, - "version": log_record_id.version, + "created": log_record.created, + "message": log_record.message, + "process_id": log_record.log_record_id.process_id, + "version": log_record.log_record_id.version, } # this is special case, because we would like to do not suppress exception here # in case of exception raised it will be handled by upstream HttpLogHandler # In the other hand, we force to suppress all http logs here to avoid cyclic reporting - self.make_request("POST", self._logs_url(log_record_id.package_base), + self.make_request("POST", self._logs_url(log_record.log_record_id.package_base), params=self.repository_id.query(), json=payload, suppress_errors=True) - def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: + def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[LogRecord]: """ get package logs @@ -325,7 +336,7 @@ class WebClient(Client, SyncAhrimanClient): offset(int, optional): records offset (Default value = 0) Returns: - list[tuple[float, str]]: package logs + list[LogRecord]: package logs """ query = self.repository_id.query() + [("limit", str(limit)), ("offset", str(offset))] @@ -333,7 +344,7 @@ class WebClient(Client, SyncAhrimanClient): response = self.make_request("GET", self._logs_url(package_base), params=query) response_json = response.json() - return [(record["created"], record["message"]) for record in response_json] + return [LogRecord.from_json(package_base, record) for record in response_json] return [] diff --git a/src/ahriman/models/event.py b/src/ahriman/models/event.py index 3c3f4ea4..2f1281e7 100644 --- a/src/ahriman/models/event.py +++ b/src/ahriman/models/event.py @@ -78,7 +78,7 @@ class Event: dump(dict[str, Any]): json dump body Returns: - Self: dependencies object + Self: event object """ return cls( event=dump["event"], diff --git a/src/ahriman/models/log_record.py b/src/ahriman/models/log_record.py new file mode 100644 index 00000000..1f6d9a25 --- /dev/null +++ b/src/ahriman/models/log_record.py @@ -0,0 +1,76 @@ +# +# Copyright (c) 2021-2025 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from dataclasses import dataclass +from typing import Any, Self + +from ahriman.models.log_record_id import LogRecordId + + +@dataclass(frozen=True) +class LogRecord: + """ + log record + + Attributes: + log_record_id(LogRecordId): log record identifier + created(float): log record creation timestamp + message(str): log record message + """ + + log_record_id: LogRecordId + created: float + message: str + + @classmethod + def from_json(cls, package_base: str, dump: dict[str, Any]) -> Self: + """ + construct log record from the json dump + + Args: + package_base(str): package base for which log record belongs + dump(dict[str, Any]): json dump body + + Returns: + Self: log record object + """ + if "process_id" in dump: + log_record_id = LogRecordId(package_base, dump["version"], dump["process_id"]) + else: + log_record_id = LogRecordId(package_base, dump["version"]) + + return cls( + log_record_id=log_record_id, + created=dump["created"], + message=dump["message"], + ) + + def view(self) -> dict[str, Any]: + """ + generate json log record view + + Returns: + dict[str, Any]: json-friendly dictionary + """ + return { + "created": self.created, + "message": self.message, + "version": self.log_record_id.version, + "process_id": self.log_record_id.process_id, + } diff --git a/src/ahriman/models/log_record_id.py b/src/ahriman/models/log_record_id.py index bff9b6ee..26d43b40 100644 --- a/src/ahriman/models/log_record_id.py +++ b/src/ahriman/models/log_record_id.py @@ -17,7 +17,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +import uuid + from dataclasses import dataclass +from typing import ClassVar @dataclass(frozen=True) @@ -26,9 +29,21 @@ class LogRecordId: log record process identifier Attributes: + DEFAULT_PROCESS_ID(str): (class attribute) default process identifier package_base(str): package base for which log record belongs version(str): package version for which log record belongs + process_id(str, optional): unique process identifier """ package_base: str version: str + process_id: str = "" + + DEFAULT_PROCESS_ID: ClassVar[str] = str(uuid.uuid4()) + + def __post_init__(self) -> None: + """ + assign process identifier from default if not set + """ + if not self.process_id: + object.__setattr__(self, "process_id", self.DEFAULT_PROCESS_ID) diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index ee1ca8df..cfad22cc 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -429,12 +429,11 @@ class Package(LazyLogging): task = Task(self, configuration, repository_id.architecture, paths) try: - with self.suppress_logging(): - # create fresh chroot environment, fetch sources and - automagically - update PKGBUILD - task.init(paths.cache_for(self.base), [], None) - pkgbuild = Pkgbuild.from_file(paths.cache_for(self.base) / "PKGBUILD") + # create fresh chroot environment, fetch sources and - automagically - update PKGBUILD + task.init(paths.cache_for(self.base), [], None) + pkgbuild = Pkgbuild.from_file(paths.cache_for(self.base) / "PKGBUILD") - return full_version(pkgbuild.get("epoch"), pkgbuild["pkgver"], pkgbuild["pkgrel"]) + return full_version(pkgbuild.get("epoch"), pkgbuild["pkgver"], pkgbuild["pkgrel"]) except Exception: self.logger.exception("cannot determine version of VCS package") finally: diff --git a/src/ahriman/web/apispec/decorators.py b/src/ahriman/web/apispec/decorators.py index 88778944..e3ef3955 100644 --- a/src/ahriman/web/apispec/decorators.py +++ b/src/ahriman/web/apispec/decorators.py @@ -18,7 +18,8 @@ # along with this program. If not, see . # from aiohttp.web import HTTPException -from typing import Any, Callable +from collections.abc import Callable +from typing import Any from ahriman.models.user_access import UserAccess from ahriman.web.apispec import Schema, aiohttp_apispec diff --git a/src/ahriman/web/schemas/__init__.py b/src/ahriman/web/schemas/__init__.py index ceb68421..998ec790 100644 --- a/src/ahriman/web/schemas/__init__.py +++ b/src/ahriman/web/schemas/__init__.py @@ -31,6 +31,7 @@ from ahriman.web.schemas.info_schema import InfoSchema from ahriman.web.schemas.internal_status_schema import InternalStatusSchema from ahriman.web.schemas.log_schema import LogSchema from ahriman.web.schemas.login_schema import LoginSchema +from ahriman.web.schemas.logs_rotate_schema import LogsRotateSchema from ahriman.web.schemas.logs_schema import LogsSchema from ahriman.web.schemas.oauth2_schema import OAuth2Schema from ahriman.web.schemas.package_name_schema import PackageNameSchema @@ -53,5 +54,4 @@ from ahriman.web.schemas.repository_stats_schema import RepositoryStatsSchema from ahriman.web.schemas.search_schema import SearchSchema from ahriman.web.schemas.status_schema import StatusSchema from ahriman.web.schemas.update_flags_schema import UpdateFlagsSchema -from ahriman.web.schemas.versioned_log_schema import VersionedLogSchema from ahriman.web.schemas.worker_schema import WorkerSchema diff --git a/src/ahriman/web/schemas/log_schema.py b/src/ahriman/web/schemas/log_schema.py index 374c4809..5a4bd49f 100644 --- a/src/ahriman/web/schemas/log_schema.py +++ b/src/ahriman/web/schemas/log_schema.py @@ -17,12 +17,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +from ahriman import __version__ from ahriman.web.apispec import Schema, fields class LogSchema(Schema): """ - request package log schema + request and response package log schema """ created = fields.Float(required=True, metadata={ @@ -32,3 +33,10 @@ class LogSchema(Schema): message = fields.String(required=True, metadata={ "description": "Log message", }) + version = fields.String(required=True, metadata={ + "description": "Package version to tag", + "example": __version__, + }) + process_id = fields.String(metadata={ + "description": "Process unique identifier", + }) diff --git a/src/ahriman/web/schemas/versioned_log_schema.py b/src/ahriman/web/schemas/logs_rotate_schema.py similarity index 65% rename from src/ahriman/web/schemas/versioned_log_schema.py rename to src/ahriman/web/schemas/logs_rotate_schema.py index 2e08571d..09d5872a 100644 --- a/src/ahriman/web/schemas/versioned_log_schema.py +++ b/src/ahriman/web/schemas/logs_rotate_schema.py @@ -17,18 +17,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from ahriman import __version__ -from ahriman.web.apispec import fields -from ahriman.web.schemas.log_schema import LogSchema -from ahriman.web.schemas.repository_id_schema import RepositoryIdSchema +from ahriman.web.apispec import Schema, fields -class VersionedLogSchema(LogSchema, RepositoryIdSchema): +class LogsRotateSchema(Schema): """ - request package log schema + request logs rotate schema """ - version = fields.Integer(required=True, metadata={ - "description": "Package version to tag", - "example": __version__, + keep_last_records = fields.Integer(metadata={ + "description": "Keep the specified amount of records", }) diff --git a/src/ahriman/web/views/v1/packages/logs.py b/src/ahriman/web/views/v1/packages/logs.py index 87383571..cb82ff53 100644 --- a/src/ahriman/web/views/v1/packages/logs.py +++ b/src/ahriman/web/views/v1/packages/logs.py @@ -21,11 +21,10 @@ from aiohttp.web import HTTPBadRequest, HTTPNoContent, HTTPNotFound, Response, j from ahriman.core.exceptions import UnknownPackageError from ahriman.core.utils import pretty_datetime -from ahriman.models.log_record_id import LogRecordId +from ahriman.models.log_record import LogRecord from ahriman.models.user_access import UserAccess from ahriman.web.apispec.decorators import apidocs -from ahriman.web.schemas import LogsSchema, PackageNameSchema, PackageVersionSchema, RepositoryIdSchema, \ - VersionedLogSchema +from ahriman.web.schemas import LogSchema, LogsSchema, PackageNameSchema, PackageVersionSchema, RepositoryIdSchema from ahriman.web.views.base import BaseView from ahriman.web.views.status_view_guard import StatusViewGuard @@ -97,7 +96,7 @@ class LogsView(StatusViewGuard, BaseView): response = { "package_base": package_base, "status": status.view(), - "logs": "\n".join(f"[{pretty_datetime(created)}] {message}" for created, message in logs) + "logs": "\n".join(f"[{pretty_datetime(log_record.created)}] {log_record.message}" for log_record in logs) } return json_response(response) @@ -109,7 +108,7 @@ class LogsView(StatusViewGuard, BaseView): error_400_enabled=True, error_404_description="Repository is unknown", match_schema=PackageNameSchema, - body_schema=VersionedLogSchema, + body_schema=LogSchema, ) async def post(self) -> None: """ @@ -123,12 +122,10 @@ class LogsView(StatusViewGuard, BaseView): try: data = await self.request.json() - created = data["created"] - record = data["message"] - version = data["version"] + log_record = LogRecord.from_json(package_base, data) except Exception as ex: raise HTTPBadRequest(reason=str(ex)) - self.service().package_logs_add(LogRecordId(package_base, version), created, record) + self.service().package_logs_add(log_record) raise HTTPNoContent diff --git a/src/ahriman/web/views/v1/service/logs.py b/src/ahriman/web/views/v1/service/logs.py new file mode 100644 index 00000000..552912fb --- /dev/null +++ b/src/ahriman/web/views/v1/service/logs.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2021-2025 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from aiohttp.web import HTTPBadRequest, HTTPNoContent + +from ahriman.models.user_access import UserAccess +from ahriman.web.apispec.decorators import apidocs +from ahriman.web.schemas import LogsRotateSchema +from ahriman.web.views.base import BaseView + + +class LogsView(BaseView): + """ + logs management web view + + Attributes: + DELETE_PERMISSION(UserAccess): (class attribute) delete permissions of self + """ + + DELETE_PERMISSION = UserAccess.Full + ROUTES = ["/api/v1/service/logs"] + + @apidocs( + tags=["Actions"], + summary="Rotate logs", + description="Remove older logs from system", + permission=DELETE_PERMISSION, + error_400_enabled=True, + error_404_description="Repository is unknown", + query_schema=LogsRotateSchema, + ) + async def delete(self) -> None: + """ + rotate logs from system + + Raises: + HTTPBadRequest: if bad data is supplied + HTTPNoContent: on success response + """ + try: + keep_last_records = int(self.request.query.get("keep_last_records", 0)) + except Exception as ex: + raise HTTPBadRequest(reason=str(ex)) + + self.service().logs_rotate(keep_last_records) + + raise HTTPNoContent diff --git a/src/ahriman/web/views/v2/packages/logs.py b/src/ahriman/web/views/v2/packages/logs.py index f1fda731..0e9bfe40 100644 --- a/src/ahriman/web/views/v2/packages/logs.py +++ b/src/ahriman/web/views/v2/packages/logs.py @@ -63,10 +63,5 @@ class LogsView(StatusViewGuard, BaseView): logs = self.service(package_base=package_base).package_logs_get(package_base, limit, offset) - response = [ - { - "created": created, - "message": message, - } for created, message in logs - ] + response = [log_record.view() for log_record in logs] return json_response(response) diff --git a/tests/ahriman/core/database/migrations/test_m015_logs_process_id.py b/tests/ahriman/core/database/migrations/test_m015_logs_process_id.py new file mode 100644 index 00000000..0149c728 --- /dev/null +++ b/tests/ahriman/core/database/migrations/test_m015_logs_process_id.py @@ -0,0 +1,8 @@ +from ahriman.core.database.migrations.m015_logs_process_id import steps + + +def test_migration_logs_process_id() -> None: + """ + migration must not be empty + """ + assert steps diff --git a/tests/ahriman/core/database/operations/test_logs_operations.py b/tests/ahriman/core/database/operations/test_logs_operations.py index 3779a5de..5257955d 100644 --- a/tests/ahriman/core/database/operations/test_logs_operations.py +++ b/tests/ahriman/core/database/operations/test_logs_operations.py @@ -1,4 +1,5 @@ from ahriman.core.database import SQLite +from ahriman.models.log_record import LogRecord from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package from ahriman.models.repository_id import RepositoryId @@ -9,67 +10,115 @@ def test_logs_insert_remove_version(database: SQLite, package_ahriman: Package, """ must clear version specific package logs """ - database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") - database.logs_insert(LogRecordId(package_ahriman.base, "2"), 43.0, "message 2") - database.logs_insert(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3") + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1")) + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "2"), 43.0, "message 2")) + database.logs_insert(LogRecord(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3")) database.logs_remove(package_ahriman.base, "1") - assert database.logs_get(package_ahriman.base) == [(42.0, "message 1")] - assert database.logs_get(package_python_schedule.base) == [(42.0, "message 3")] + assert database.logs_get(package_ahriman.base) == [ + LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"), + ] + assert database.logs_get(package_python_schedule.base) == [ + LogRecord(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3"), + ] def test_logs_insert_remove_multi(database: SQLite, package_ahriman: Package) -> None: """ must clear logs for specified repository """ - database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") - database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2", + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1")) + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"), RepositoryId("i686", database._repository_id.name)) database.logs_remove(package_ahriman.base, None, RepositoryId("i686", database._repository_id.name)) assert not database.logs_get(package_ahriman.base, repository_id=RepositoryId("i686", database._repository_id.name)) - assert database.logs_get(package_ahriman.base) == [(42.0, "message 1")] + assert database.logs_get(package_ahriman.base) == [ + LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"), + ] def test_logs_insert_remove_full(database: SQLite, package_ahriman: Package, package_python_schedule: Package) -> None: """ must clear full package logs """ - database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") - database.logs_insert(LogRecordId(package_ahriman.base, "2"), 43.0, "message 2") - database.logs_insert(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3") + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1")) + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "2"), 43.0, "message 2")) + database.logs_insert(LogRecord(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3")) database.logs_remove(package_ahriman.base, None) assert not database.logs_get(package_ahriman.base) - assert database.logs_get(package_python_schedule.base) == [(42.0, "message 3")] + assert database.logs_get(package_python_schedule.base) == [ + LogRecord(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3"), + ] def test_logs_insert_get(database: SQLite, package_ahriman: Package) -> None: """ must insert and get package logs """ - database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2") - database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") - assert database.logs_get(package_ahriman.base) == [(42.0, "message 1"), (43.0, "message 2")] + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2")) + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1")) + assert database.logs_get(package_ahriman.base) == [ + LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"), + LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"), + ] def test_logs_insert_get_pagination(database: SQLite, package_ahriman: Package) -> None: """ must insert and get package logs with pagination """ - database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") - database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2") - assert database.logs_get(package_ahriman.base, 1, 1) == [(42.0, "message 1")] + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1")) + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2")) + assert database.logs_get(package_ahriman.base, 1, 1) == [ + LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"), + ] def test_logs_insert_get_multi(database: SQLite, package_ahriman: Package) -> None: """ must insert and get package logs for multiple repositories """ - database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") - database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2", + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1")) + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"), RepositoryId("i686", database._repository_id.name)) assert database.logs_get(package_ahriman.base, - repository_id=RepositoryId("i686", database._repository_id.name)) == [(43.0, "message 2")] - assert database.logs_get(package_ahriman.base) == [(42.0, "message 1")] + repository_id=RepositoryId("i686", database._repository_id.name)) == [ + LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"), + ] + assert database.logs_get(package_ahriman.base) == [ + LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"), + ] + + +def test_logs_rotate_remove_all(database: SQLite, package_ahriman: Package) -> None: + """ + must remove all records when rotating with keep_last_records is 0 + """ + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1")) + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2")) + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "2"), 44.0, "message 3")) + + database.logs_rotate(0) + assert not database.logs_get(package_ahriman.base) + + +def test_logs_rotate_remove_duplicates(database: SQLite, package_ahriman: Package) -> None: + """ + must remove duplicate records while preserving the most recent one for each package version + """ + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1", "p1"), 42.0, "message 1")) + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1", "p2"), 43.0, "message 2")) + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1", "p3"), 44.0, "message 3")) + database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "2", "p1"), 45.0, "message 4")) + + database.logs_rotate(2) + + logs = database.logs_get(package_ahriman.base) + assert len(logs) == 2 + assert logs == [ + LogRecord(LogRecordId(package_ahriman.base, "1", "p3"), 44.0, "message 3"), + LogRecord(LogRecordId(package_ahriman.base, "2", "p1"), 45.0, "message 4"), + ] diff --git a/tests/ahriman/core/log/test_http_log_handler.py b/tests/ahriman/core/log/test_http_log_handler.py index 40395c30..d8ba6d3e 100644 --- a/tests/ahriman/core/log/test_http_log_handler.py +++ b/tests/ahriman/core/log/test_http_log_handler.py @@ -4,6 +4,7 @@ from pytest_mock import MockerFixture from ahriman.core.configuration import Configuration from ahriman.core.log.http_log_handler import HttpLogHandler +from ahriman.models.log_record import LogRecord from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package @@ -19,12 +20,14 @@ def test_load(configuration: Configuration, mocker: MockerFixture) -> None: add_mock = mocker.patch("logging.Logger.addHandler") load_mock = mocker.patch("ahriman.core.status.Client.load") + atexit_mock = mocker.patch("atexit.register") _, repository_id = configuration.check_loaded() handler = HttpLogHandler.load(repository_id, configuration, report=False) assert handler add_mock.assert_called_once_with(handler) load_mock.assert_called_once_with(repository_id, configuration, report=False) + atexit_mock.assert_called_once_with(handler.rotate) def test_load_exist(configuration: Configuration) -> None: @@ -49,7 +52,7 @@ def test_emit(configuration: Configuration, log_record: logging.LogRecord, packa handler = HttpLogHandler(repository_id, configuration, report=False, suppress_errors=False) handler.emit(log_record) - log_mock.assert_called_once_with(log_record_id, log_record.created, log_record.getMessage()) + log_mock.assert_called_once_with(LogRecord(log_record_id, log_record.created, log_record.getMessage())) def test_emit_failed(configuration: Configuration, log_record: logging.LogRecord, package_ahriman: Package, @@ -93,3 +96,16 @@ def test_emit_skip(configuration: Configuration, log_record: logging.LogRecord, handler.emit(log_record) log_mock.assert_not_called() + + +def test_rotate(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must rotate logs + """ + rotate_mock = mocker.patch("ahriman.core.status.Client.logs_rotate") + + _, repository_id = configuration.check_loaded() + handler = HttpLogHandler(repository_id, configuration, report=False, suppress_errors=False) + + handler.rotate() + rotate_mock.assert_called_once_with(handler.keep_last_records) diff --git a/tests/ahriman/core/log/test_lazy_logging.py b/tests/ahriman/core/log/test_lazy_logging.py index 9e0ce317..a3f416e2 100644 --- a/tests/ahriman/core/log/test_lazy_logging.py +++ b/tests/ahriman/core/log/test_lazy_logging.py @@ -87,13 +87,3 @@ def test_in_package_context_failed(database: SQLite, package_ahriman: Package, m raise ValueError() reset_mock.assert_called_once_with() - - -def test_suppress_logging(database: SQLite, mocker: MockerFixture) -> None: - """ - must temporary disable log messages - """ - disable_mock = mocker.patch("ahriman.core.log.lazy_logging.logging.disable") - with database.suppress_logging(): - pass - disable_mock.assert_has_calls([MockCall(logging.WARNING), MockCall(logging.NOTSET)]) diff --git a/tests/ahriman/core/status/test_client.py b/tests/ahriman/core/status/test_client.py index b152f621..3462855f 100644 --- a/tests/ahriman/core/status/test_client.py +++ b/tests/ahriman/core/status/test_client.py @@ -13,6 +13,7 @@ from ahriman.models.changes import Changes from ahriman.models.dependencies import Dependencies from ahriman.models.event import Event from ahriman.models.internal_status import InternalStatus +from ahriman.models.log_record import LogRecord from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch @@ -112,6 +113,13 @@ def test_event_get(client: Client) -> None: client.event_get(None, None) +def test_logs_rotate(client: Client) -> None: + """ + must do not raise exception on logs rotation call + """ + client.logs_rotate(1) + + def test_package_changes_get(client: Client, package_ahriman: Package) -> None: """ must raise not implemented on package changes request @@ -157,7 +165,7 @@ def test_package_logs_add(client: Client, package_ahriman: Package, log_record: must process log record addition without exception """ log_record_id = LogRecordId(package_ahriman.base, package_ahriman.version) - client.package_logs_add(log_record_id, log_record.created, log_record.getMessage()) + client.package_logs_add(LogRecord(log_record_id, log_record.created, log_record.getMessage())) def test_package_logs_get(client: Client, package_ahriman: Package) -> None: diff --git a/tests/ahriman/core/status/test_local_client.py b/tests/ahriman/core/status/test_local_client.py index 255983a7..87326e1a 100644 --- a/tests/ahriman/core/status/test_local_client.py +++ b/tests/ahriman/core/status/test_local_client.py @@ -8,6 +8,7 @@ from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.changes import Changes from ahriman.models.dependencies import Dependencies from ahriman.models.event import Event, EventType +from ahriman.models.log_record import LogRecord from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch @@ -34,6 +35,15 @@ def test_event_get(local_client: LocalClient, package_ahriman: Package, mocker: local_client.repository_id) +def test_logs_rotate(local_client: LocalClient, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must rotate logs + """ + rotate_mock = mocker.patch("ahriman.core.database.SQLite.logs_rotate") + local_client.logs_rotate(42) + rotate_mock.assert_called_once_with(42, local_client.repository_id) + + def test_package_changes_get(local_client: LocalClient, package_ahriman: Package, mocker: MockerFixture) -> None: """ must retrieve package changes @@ -103,10 +113,10 @@ def test_package_logs_add(local_client: LocalClient, package_ahriman: Package, l """ logs_mock = mocker.patch("ahriman.core.database.SQLite.logs_insert") log_record_id = LogRecordId(package_ahriman.base, package_ahriman.version) + record = LogRecord(log_record_id, log_record.created, log_record.getMessage()) - local_client.package_logs_add(log_record_id, log_record.created, log_record.getMessage()) - logs_mock.assert_called_once_with(log_record_id, log_record.created, log_record.getMessage(), - local_client.repository_id) + local_client.package_logs_add(record) + logs_mock.assert_called_once_with(record, local_client.repository_id) def test_package_logs_get(local_client: LocalClient, package_ahriman: Package, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/core/status/test_watcher.py b/tests/ahriman/core/status/test_watcher.py index 620e918f..ff99d3da 100644 --- a/tests/ahriman/core/status/test_watcher.py +++ b/tests/ahriman/core/status/test_watcher.py @@ -5,7 +5,6 @@ from pytest_mock import MockerFixture from ahriman.core.exceptions import UnknownPackageError from ahriman.core.status.watcher import Watcher from ahriman.models.build_status import BuildStatus, BuildStatusEnum -from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package @@ -64,38 +63,6 @@ def test_package_get_failed(watcher: Watcher, package_ahriman: Package) -> None: watcher.package_get(package_ahriman.base) -def test_package_logs_add_new(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: - """ - must create package logs record for new package - """ - delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.package_logs_remove", create=True) - insert_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_logs_add") - - log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.version) - assert watcher._last_log_record_id != log_record_id - - watcher.package_logs_add(log_record_id, 42.01, "log record") - delete_mock.assert_called_once_with(package_ahriman.base, log_record_id.version) - insert_mock.assert_called_once_with(log_record_id, 42.01, "log record") - - assert watcher._last_log_record_id == log_record_id - - -def test_package_logs_add_update(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: - """ - must create package logs record for current package - """ - delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.package_logs_remove", create=True) - insert_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_logs_add") - - log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.version) - watcher._last_log_record_id = log_record_id - - watcher.package_logs_add(log_record_id, 42.01, "log record") - delete_mock.assert_not_called() - insert_mock.assert_called_once_with(log_record_id, 42.01, "log record") - - def test_package_remove(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: """ must remove package base diff --git a/tests/ahriman/core/status/test_web_client.py b/tests/ahriman/core/status/test_web_client.py index c6ec36ba..f925049a 100644 --- a/tests/ahriman/core/status/test_web_client.py +++ b/tests/ahriman/core/status/test_web_client.py @@ -12,6 +12,7 @@ from ahriman.models.changes import Changes from ahriman.models.dependencies import Dependencies from ahriman.models.event import Event, EventType from ahriman.models.internal_status import InternalStatus +from ahriman.models.log_record import LogRecord from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch @@ -257,6 +258,57 @@ def test_event_get_failed_http_error_suppress(web_client: WebClient, mocker: Moc logging_mock.assert_not_called() +def test_logs_rotate(web_client: WebClient, mocker: MockerFixture) -> None: + """ + must rotate logs + """ + requests_mock = mocker.patch("ahriman.core.status.web_client.WebClient.make_request") + + web_client.logs_rotate(42) + requests_mock.assert_called_once_with("DELETE", pytest.helpers.anyvar(str, True), + params=web_client.repository_id.query() + [("keep_last_records", "42")]) + + +def test_logs_rotate_failed(web_client: WebClient, mocker: MockerFixture) -> None: + """ + must suppress any exception happened during logs rotation + """ + mocker.patch("requests.Session.request", side_effect=Exception()) + web_client.logs_rotate(42) + + +def test_logs_rotate_failed_http_error(web_client: WebClient, mocker: MockerFixture) -> None: + """ + must suppress HTTP exception happened during logs rotation + """ + mocker.patch("requests.Session.request", side_effect=requests.HTTPError()) + web_client.logs_rotate(42) + + +def test_logs_rotate_failed_suppress(web_client: WebClient, mocker: MockerFixture) -> None: + """ + must suppress any exception happened during logs rotation and don't log + """ + web_client.suppress_errors = True + mocker.patch("requests.Session.request", side_effect=Exception()) + logging_mock = mocker.patch("logging.exception") + + web_client.logs_rotate(42) + logging_mock.assert_not_called() + + +def test_logs_rotate_failed_http_error_suppress(web_client: WebClient, mocker: MockerFixture) -> None: + """ + must suppress HTTP exception happened during logs rotation and don't log + """ + web_client.suppress_errors = True + mocker.patch("requests.Session.request", side_effect=requests.HTTPError()) + logging_mock = mocker.patch("logging.exception") + + web_client.logs_rotate(42) + logging_mock.assert_not_called() + + def test_package_changes_get(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None: """ must get changes @@ -551,11 +603,13 @@ def test_package_logs_add(web_client: WebClient, log_record: logging.LogRecord, payload = { "created": log_record.created, "message": log_record.getMessage(), + "process_id": LogRecordId.DEFAULT_PROCESS_ID, "version": package_ahriman.version, } + record = LogRecord(LogRecordId(package_ahriman.base, package_ahriman.version), + log_record.created, log_record.getMessage()) - web_client.package_logs_add(LogRecordId(package_ahriman.base, package_ahriman.version), - log_record.created, log_record.getMessage()) + web_client.package_logs_add(record) requests_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True), params=web_client.repository_id.query(), json=payload, suppress_errors=True) @@ -567,9 +621,11 @@ def test_package_logs_add_failed(web_client: WebClient, log_record: logging.LogR """ mocker.patch("requests.Session.request", side_effect=Exception()) log_record.package_base = package_ahriman.base + record = LogRecord(LogRecordId(package_ahriman.base, package_ahriman.version), + log_record.created, log_record.getMessage()) + with pytest.raises(Exception): - web_client.package_logs_add(LogRecordId(package_ahriman.base, package_ahriman.version), - log_record.created, log_record.getMessage()) + web_client.package_logs_add(record) def test_package_logs_add_failed_http_error(web_client: WebClient, log_record: logging.LogRecord, @@ -579,16 +635,23 @@ def test_package_logs_add_failed_http_error(web_client: WebClient, log_record: l """ mocker.patch("requests.Session.request", side_effect=requests.HTTPError()) log_record.package_base = package_ahriman.base + record = LogRecord(LogRecordId(package_ahriman.base, package_ahriman.version), + log_record.created, log_record.getMessage()) + with pytest.raises(Exception): - web_client.package_logs_add(LogRecordId(package_ahriman.base, package_ahriman.version), - log_record.created, log_record.getMessage()) + web_client.package_logs_add(record) def test_package_logs_get(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None: """ must get logs """ - message = {"created": 42.0, "message": "log"} + message = { + "created": 42.0, + "message": "log", + "version": package_ahriman.version, + "process_id": LogRecordId.DEFAULT_PROCESS_ID, + } response_obj = requests.Response() response_obj._content = json.dumps([message]).encode("utf8") response_obj.status_code = 200 @@ -598,7 +661,9 @@ def test_package_logs_get(web_client: WebClient, package_ahriman: Package, mocke result = web_client.package_logs_get(package_ahriman.base, 1, 2) requests_mock.assert_called_once_with("GET", pytest.helpers.anyvar(str, True), params=web_client.repository_id.query() + [("limit", "1"), ("offset", "2")]) - assert result == [(message["created"], message["message"])] + assert result == [ + LogRecord(LogRecordId(package_ahriman.base, package_ahriman.version), message["created"], message["message"]), + ] def test_package_logs_get_failed(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/models/test_log_record.py b/tests/ahriman/models/test_log_record.py new file mode 100644 index 00000000..730463c3 --- /dev/null +++ b/tests/ahriman/models/test_log_record.py @@ -0,0 +1,13 @@ +from ahriman.models.log_record import LogRecord +from ahriman.models.log_record_id import LogRecordId + + +def test_log_record_from_json_view() -> None: + """ + must construct same object from json + """ + log_record = LogRecord(LogRecordId("base", "version"), 0, "message") + assert LogRecord.from_json(log_record.log_record_id.package_base, log_record.view()) == log_record + + log_record = LogRecord(LogRecordId("base", "version", "process_id"), 0, "message") + assert LogRecord.from_json(log_record.log_record_id.package_base, log_record.view()) == log_record diff --git a/tests/ahriman/models/test_log_record_id.py b/tests/ahriman/models/test_log_record_id.py index e69de29b..4876d561 100644 --- a/tests/ahriman/models/test_log_record_id.py +++ b/tests/ahriman/models/test_log_record_id.py @@ -0,0 +1,9 @@ +from ahriman.models.log_record_id import LogRecordId + + +def test_init() -> None: + """ + must correctly assign proces identifier if not set + """ + assert LogRecordId("1", "2").process_id == LogRecordId.DEFAULT_PROCESS_ID + assert LogRecordId("1", "2", "3").process_id == "3" diff --git a/tests/ahriman/web/schemas/test_versioned_log_schema.py b/tests/ahriman/web/schemas/test_logs_rotate_schema.py similarity index 100% rename from tests/ahriman/web/schemas/test_versioned_log_schema.py rename to tests/ahriman/web/schemas/test_logs_rotate_schema.py diff --git a/tests/ahriman/web/views/v1/service/test_view_v1_service_logs.py b/tests/ahriman/web/views/v1/service/test_view_v1_service_logs.py new file mode 100644 index 00000000..131c4032 --- /dev/null +++ b/tests/ahriman/web/views/v1/service/test_view_v1_service_logs.py @@ -0,0 +1,77 @@ +import pytest + +from aiohttp.test_utils import TestClient + +from ahriman.models.build_status import BuildStatusEnum +from ahriman.models.package import Package +from ahriman.models.user_access import UserAccess +from ahriman.web.views.v1.service.logs import LogsView + + +async def test_get_permission() -> None: + """ + must return correct permission for the request + """ + for method in ("DELETE",): + request = pytest.helpers.request("", "", method) + assert await LogsView.get_permission(request) == UserAccess.Full + + +def test_routes() -> None: + """ + must return correct routes + """ + assert LogsView.ROUTES == ["/api/v1/service/logs"] + + +async def test_delete(client: TestClient, package_ahriman: Package) -> None: + """ + must delete all logs + """ + await client.post(f"/api/v1/packages/{package_ahriman.base}", + json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()}) + await client.post(f"/api/v1/packages/{package_ahriman.base}/logs", + json={"created": 42.0, "message": "message 1", "version": "42"}) + await client.post(f"/api/v1/packages/{package_ahriman.base}/logs", + json={"created": 43.0, "message": "message 2", "version": "43"}) + + response = await client.delete("/api/v1/service/logs") + assert response.status == 204 + + response = await client.get(f"/api/v2/packages/{package_ahriman.base}/logs") + json = await response.json() + assert not json + + +async def test_delete_partially(client: TestClient, package_ahriman: Package) -> None: + """ + must delete logs based on count input + """ + await client.post(f"/api/v1/packages/{package_ahriman.base}", + json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()}) + await client.post(f"/api/v1/packages/{package_ahriman.base}/logs", + json={"created": 42.0, "message": "message 1", "version": "42"}) + await client.post(f"/api/v1/packages/{package_ahriman.base}/logs", + json={"created": 43.0, "message": "message 2", "version": "43"}) + request_schema = pytest.helpers.schema_request(LogsView.delete, location="querystring") + + payload = {"keep_last_records": 1} + assert not request_schema.validate(payload) + + response = await client.delete("/api/v1/service/logs", params=payload) + assert response.status == 204 + + response = await client.get(f"/api/v2/packages/{package_ahriman.base}/logs") + json = await response.json() + assert json + + +async def test_delete_exception(client: TestClient, package_ahriman: Package) -> None: + """ + must raise exception on invalid payload + """ + response_schema = pytest.helpers.schema_response(LogsView.delete, code=400) + + response = await client.delete("/api/v1/service/logs", params={"keep_last_records": "string"}) + assert response.status == 400 + assert not response_schema.validate(await response.json()) diff --git a/tests/ahriman/web/views/v2/packages/test_view_v2_packages_logs.py b/tests/ahriman/web/views/v2/packages/test_view_v2_packages_logs.py index b40010f6..4a3cb949 100644 --- a/tests/ahriman/web/views/v2/packages/test_view_v2_packages_logs.py +++ b/tests/ahriman/web/views/v2/packages/test_view_v2_packages_logs.py @@ -3,6 +3,7 @@ import pytest from aiohttp.test_utils import TestClient from ahriman.models.build_status import BuildStatusEnum +from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package from ahriman.models.user_access import UserAccess from ahriman.web.views.v2.packages.logs import LogsView @@ -48,10 +49,14 @@ async def test_get(client: TestClient, package_ahriman: Package) -> None: { "created": 42.0, "message": "message 1", + "version": "42", + "process_id": LogRecordId.DEFAULT_PROCESS_ID, }, { "created": 43.0, "message": "message 2", + "version": "42", + "process_id": LogRecordId.DEFAULT_PROCESS_ID, }, ] @@ -76,7 +81,9 @@ async def test_get_with_pagination(client: TestClient, package_ahriman: Package) logs = await response.json() assert not response_schema.validate(logs) - assert logs == [{"created": 42.0, "message": "message 1"}] + assert logs == [ + {"created": 42.0, "message": "message 1", "version": "42", "process_id": LogRecordId.DEFAULT_PROCESS_ID}, + ] async def test_get_bad_request(client: TestClient, package_ahriman: Package) -> None: