diff --git a/package/share/ahriman/settings/ahriman.ini b/package/share/ahriman/settings/ahriman.ini index d7e2f328..e53a1fa2 100644 --- a/package/share/ahriman/settings/ahriman.ini +++ b/package/share/ahriman/settings/ahriman.ini @@ -7,6 +7,8 @@ logging = ahriman.ini.d/logging.ini ;apply_migrations = yes ; Path to the application SQLite database. database = ${repository:root}/ahriman.db +; Keep last build logs for each package +keep_last_logs = 5 [alpm] ; Path to pacman system database cache. diff --git a/src/ahriman/core/configuration/schema.py b/src/ahriman/core/configuration/schema.py index 8c93fc43..008db9b4 100644 --- a/src/ahriman/core/configuration/schema.py +++ b/src/ahriman/core/configuration/schema.py @@ -45,6 +45,11 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = { "path_exists": True, "path_type": "dir", }, + "keep_last_logs": { + "type": "integer", + "coerce": "integer", + "min": 0, + }, "logging": { "type": "path", "coerce": "absolute_path", diff --git a/src/ahriman/core/database/migrations/m015_logs_process_id.py b/src/ahriman/core/database/migrations/m015_logs_process_id.py new file mode 100644 index 00000000..b0a9001b --- /dev/null +++ b/src/ahriman/core/database/migrations/m015_logs_process_id.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2021-2025 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +__all__ = ["steps"] + + +steps = [ + """ + alter table logs add column process_id text not null default '' + """, +] diff --git a/src/ahriman/core/database/operations/logs_operations.py b/src/ahriman/core/database/operations/logs_operations.py index 8704df49..50645b00 100644 --- a/src/ahriman/core/database/operations/logs_operations.py +++ b/src/ahriman/core/database/operations/logs_operations.py @@ -30,7 +30,7 @@ class LogsOperations(Operations): """ def logs_get(self, package_base: str, limit: int = -1, offset: int = 0, - repository_id: RepositoryId | None = None) -> list[tuple[float, str]]: + repository_id: RepositoryId | None = None) -> list[tuple[LogRecordId, float, str]]: """ extract logs for specified package base @@ -41,16 +41,16 @@ class LogsOperations(Operations): repository_id(RepositoryId, optional): repository unique identifier override (Default value = None) Return: - list[tuple[float, str]]: sorted package log records and their timestamps + list[tuple[LogRecordId, float, str]]: sorted package log records and their timestamps """ repository_id = repository_id or self._repository_id - def run(connection: Connection) -> list[tuple[float, str]]: + def run(connection: Connection) -> list[tuple[LogRecordId, float, str]]: return [ - (row["created"], row["record"]) + (LogRecordId(package_base, row["version"], row["process_id"]), row["created"], row["record"]) for row in connection.execute( """ - select created, record from ( + select created, record, version, process_id from ( select * from logs where package_base = :package_base and repository = :repository order by created desc limit :limit offset :offset @@ -83,9 +83,9 @@ class LogsOperations(Operations): connection.execute( """ insert into logs - (package_base, version, created, record, repository) + (package_base, version, created, record, repository, process_id) values - (:package_base, :version, :created, :record, :repository) + (:package_base, :version, :created, :record, :repository, :process_id) """, { "package_base": log_record_id.package_base, @@ -93,6 +93,7 @@ class LogsOperations(Operations): "created": created, "record": record, "repository": repository_id.id, + "process_id": log_record_id.process_id, } ) @@ -125,3 +126,54 @@ class LogsOperations(Operations): ) return self.with_connection(run, commit=True) + + def logs_rotate(self, keep_last_records: int, repository_id: RepositoryId | None = None) -> None: + """ + rotate logs in storage. This method will remove old logs, keeping only the last N records for each package + + Args: + keep_last_records(int): number of last records to keep + repository_id(RepositoryId, optional): repository unique identifier override (Default value = None) + """ + repository_id = repository_id or self._repository_id + + def remove_duplicates(connection: Connection) -> None: + connection.execute( + """ + delete from logs + where (package_base, version, repository, process_id) not in ( + select package_base, version, repository, process_id from logs + where (package_base, version, repository, created) in ( + select package_base, version, repository, max(created) from logs + where repository = :repository + group by package_base, version, repository + ) + ) + """, + { + "repository": repository_id.id, + } + ) + + def remove_older(connection: Connection) -> None: + connection.execute( + """ + delete from logs + where (package_base, repository, process_id) in ( + select package_base, repository, process_id from logs + where repository = :repository + group by package_base, repository, process_id + order by min(created) desc limit -1 offset :offset + ) + """, + { + "offset": keep_last_records, + "repository": repository_id.id, + } + ) + + def run(connection: Connection) -> None: + remove_duplicates(connection) + remove_older(connection) + + return self.with_connection(run, commit=True) diff --git a/src/ahriman/core/log/http_log_handler.py b/src/ahriman/core/log/http_log_handler.py index 769692dc..2377478f 100644 --- a/src/ahriman/core/log/http_log_handler.py +++ b/src/ahriman/core/log/http_log_handler.py @@ -17,6 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +import atexit import logging from typing import Self @@ -33,6 +34,7 @@ class HttpLogHandler(logging.Handler): method Attributes: + keep_last_records(int): number of last records to keep reporter(Client): build status reporter instance suppress_errors(bool): suppress logging errors (e.g. if no web server available) """ @@ -51,6 +53,7 @@ class HttpLogHandler(logging.Handler): self.reporter = Client.load(repository_id, configuration, report=report) self.suppress_errors = suppress_errors + self.keep_last_records = configuration.getint("settings", "keep_last_logs", fallback=0) @classmethod def load(cls, repository_id: RepositoryId, configuration: Configuration, *, report: bool) -> Self: @@ -76,6 +79,8 @@ class HttpLogHandler(logging.Handler): handler = cls(repository_id, configuration, report=report, suppress_errors=suppress_errors) root.addHandler(handler) + atexit.register(handler.rotate) + return handler def emit(self, record: logging.LogRecord) -> None: @@ -95,3 +100,9 @@ class HttpLogHandler(logging.Handler): if self.suppress_errors: return self.handleError(record) + + def rotate(self) -> None: + """ + rotate log records, removing older ones + """ + self.reporter.logs_rotate(self.keep_last_records) diff --git a/src/ahriman/core/log/lazy_logging.py b/src/ahriman/core/log/lazy_logging.py index 2e6b4b02..ff7db2cf 100644 --- a/src/ahriman/core/log/lazy_logging.py +++ b/src/ahriman/core/log/lazy_logging.py @@ -99,24 +99,3 @@ class LazyLogging: yield finally: self._package_logger_reset() - - @contextlib.contextmanager - def suppress_logging(self, log_level: int = logging.WARNING) -> Generator[None, None, None]: - """ - silence log messages in context - - Args: - log_level(int, optional): the highest log level to keep (Default value = logging.WARNING) - - Examples: - This function is designed to be used to suppress all log messages in context, e.g.: - - >>> with self.suppress_logging(): - >>> do_some_noisy_actions() - """ - current_level = self.logger.manager.disable - try: - logging.disable(log_level) - yield - finally: - logging.disable(current_level) diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py index 41755e79..6f9f0f8f 100644 --- a/src/ahriman/core/repository/executor.py +++ b/src/ahriman/core/repository/executor.py @@ -75,7 +75,7 @@ class Executor(PackageInfo, Cleaner): result = Result() for single in updates: - with self.in_package_context(single.base, local_versions.get(single.base)), \ + with self.in_package_context(single.base, single.version), \ TemporaryDirectory(ignore_cleanup_errors=True) as dir_name: try: with self.in_event(single.base, EventType.PackageUpdated, failure=EventType.PackageUpdateFailed): @@ -194,7 +194,6 @@ class Executor(PackageInfo, Cleaner): self.repo.add(package_path) current_packages = {package.base: package for package in self.packages()} - local_versions = {package_base: package.version for package_base, package in current_packages.items()} removed_packages: list[str] = [] # list of packages which have been removed from the base updates = self.load_archives(packages) @@ -202,7 +201,7 @@ class Executor(PackageInfo, Cleaner): result = Result() for local in updates: - with self.in_package_context(local.base, local_versions.get(local.base)): + with self.in_package_context(local.base, local.version): try: packager = self.packager(packagers, local.base) diff --git a/src/ahriman/core/repository/update_handler.py b/src/ahriman/core/repository/update_handler.py index 83d3d989..225bd150 100644 --- a/src/ahriman/core/repository/update_handler.py +++ b/src/ahriman/core/repository/update_handler.py @@ -144,8 +144,7 @@ class UpdateHandler(PackageInfo, Cleaner): branch="master", ) - with self.suppress_logging(): - Sources.fetch(cache_dir, source) + Sources.fetch(cache_dir, source) remote = Package.from_build(cache_dir, self.architecture, None) local = packages.get(remote.base) diff --git a/src/ahriman/core/status/client.py b/src/ahriman/core/status/client.py index b7de1b65..abc78a15 100644 --- a/src/ahriman/core/status/client.py +++ b/src/ahriman/core/status/client.py @@ -115,6 +115,14 @@ class Client: """ raise NotImplementedError + def logs_rotate(self, keep_last_records: int) -> None: + """ + remove older logs from storage + + Args: + keep_last_records(int): number of last records to keep + """ + def package_changes_get(self, package_base: str) -> Changes: """ get package changes @@ -197,7 +205,8 @@ class Client: """ # this method does not raise NotImplementedError because it is actively used as dummy client for http log - def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: + def package_logs_get(self, package_base: str, limit: int = -1, + offset: int = 0) -> list[tuple[LogRecordId, float, str]]: """ get package logs @@ -207,7 +216,7 @@ class Client: offset(int, optional): records offset (Default value = 0) Returns: - list[tuple[float, str]]: package logs + list[tuple[LogRecordId, float, str]]: package logs Raises: NotImplementedError: not implemented method diff --git a/src/ahriman/core/status/local_client.py b/src/ahriman/core/status/local_client.py index 6c0608ee..b58a7a60 100644 --- a/src/ahriman/core/status/local_client.py +++ b/src/ahriman/core/status/local_client.py @@ -75,6 +75,15 @@ class LocalClient(Client): """ return self.database.event_get(event, object_id, from_date, to_date, limit, offset, self.repository_id) + def logs_rotate(self, keep_last_records: int) -> None: + """ + remove older logs from storage + + Args: + keep_last_records(int): number of last records to keep + """ + self.database.logs_rotate(keep_last_records, self.repository_id) + def package_changes_get(self, package_base: str) -> Changes: """ get package changes @@ -145,7 +154,8 @@ class LocalClient(Client): """ self.database.logs_insert(log_record_id, created, message, self.repository_id) - def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: + def package_logs_get(self, package_base: str, limit: int = -1, + offset: int = 0) -> list[tuple[LogRecordId, float, str]]: """ get package logs @@ -155,7 +165,7 @@ class LocalClient(Client): offset(int, optional): records offset (Default value = 0) Returns: - list[tuple[float, str]]: package logs + list[tuple[LogRecordId, float, str]]: package logs """ return self.database.logs_get(package_base, limit, offset, self.repository_id) diff --git a/src/ahriman/core/status/watcher.py b/src/ahriman/core/status/watcher.py index ee541094..04014cf9 100644 --- a/src/ahriman/core/status/watcher.py +++ b/src/ahriman/core/status/watcher.py @@ -53,9 +53,6 @@ class Watcher(LazyLogging): self._known: dict[str, tuple[Package, BuildStatus]] = {} self.status = BuildStatus() - # special variables for updating logs - self._last_log_record_id = LogRecordId("", "") - @property def packages(self) -> list[tuple[Package, BuildStatus]]: """ @@ -81,6 +78,8 @@ class Watcher(LazyLogging): for package, status in self.client.package_get(None) } + logs_rotate: Callable[[int], None] + package_changes_get: Callable[[str], Changes] package_changes_update: Callable[[str, Changes], None] @@ -108,22 +107,9 @@ class Watcher(LazyLogging): except KeyError: raise UnknownPackageError(package_base) from None - def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: - """ - make new log record into database + package_logs_add: Callable[[LogRecordId, float, str], None] - Args: - log_record_id(LogRecordId): log record id - created(float): log created timestamp - message(str): log message - """ - if self._last_log_record_id != log_record_id: - # there is new log record, so we remove old ones - self.package_logs_remove(log_record_id.package_base, log_record_id.version) - self._last_log_record_id = log_record_id - self.client.package_logs_add(log_record_id, created, message) - - package_logs_get: Callable[[str, int, int], list[tuple[float, str]]] + package_logs_get: Callable[[str, int, int], list[tuple[LogRecordId, float, str]]] package_logs_remove: Callable[[str, str | None], None] diff --git a/src/ahriman/core/status/web_client.py b/src/ahriman/core/status/web_client.py index 158cf9cd..e29f5c9b 100644 --- a/src/ahriman/core/status/web_client.py +++ b/src/ahriman/core/status/web_client.py @@ -210,6 +210,18 @@ class WebClient(Client, SyncAhrimanClient): return [] + def logs_rotate(self, keep_last_records: int) -> None: + """ + remove older logs from storage + + Args: + keep_last_records(int): number of last records to keep + """ + query = self.repository_id.query() + [("keep_last_records", str(keep_last_records))] + + with contextlib.suppress(Exception): + self.make_request("DELETE", f"{self.address}/api/v1/service/logs", params=query) + def package_changes_get(self, package_base: str) -> Changes: """ get package changes @@ -306,6 +318,7 @@ class WebClient(Client, SyncAhrimanClient): payload = { "created": created, "message": message, + "process_id": log_record_id.process_id, "version": log_record_id.version, } @@ -315,7 +328,8 @@ class WebClient(Client, SyncAhrimanClient): self.make_request("POST", self._logs_url(log_record_id.package_base), params=self.repository_id.query(), json=payload, suppress_errors=True) - def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: + def package_logs_get(self, package_base: str, limit: int = -1, + offset: int = 0) -> list[tuple[LogRecordId, float, str]]: """ get package logs @@ -325,7 +339,7 @@ class WebClient(Client, SyncAhrimanClient): offset(int, optional): records offset (Default value = 0) Returns: - list[tuple[float, str]]: package logs + list[tuple[LogRecordId, float, str]]: package logs """ query = self.repository_id.query() + [("limit", str(limit)), ("offset", str(offset))] @@ -333,7 +347,13 @@ class WebClient(Client, SyncAhrimanClient): response = self.make_request("GET", self._logs_url(package_base), params=query) response_json = response.json() - return [(record["created"], record["message"]) for record in response_json] + return [ + ( + LogRecordId(package_base, record["version"], record["process_id"]), + record["created"], + record["message"] + ) for record in response_json + ] return [] diff --git a/src/ahriman/models/log_record_id.py b/src/ahriman/models/log_record_id.py index bff9b6ee..c7399ef8 100644 --- a/src/ahriman/models/log_record_id.py +++ b/src/ahriman/models/log_record_id.py @@ -17,7 +17,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from dataclasses import dataclass +import uuid + +from dataclasses import dataclass, field @dataclass(frozen=True) @@ -28,7 +30,12 @@ class LogRecordId: Attributes: package_base(str): package base for which log record belongs version(str): package version for which log record belongs + process_id(str, optional): unique process identifier """ package_base: str version: str + + # this is not mistake, this value is kind of global identifier, which is generated + # upon the process start + process_id: str = field(default=str(uuid.uuid4())) diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index ee1ca8df..cfad22cc 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -429,12 +429,11 @@ class Package(LazyLogging): task = Task(self, configuration, repository_id.architecture, paths) try: - with self.suppress_logging(): - # create fresh chroot environment, fetch sources and - automagically - update PKGBUILD - task.init(paths.cache_for(self.base), [], None) - pkgbuild = Pkgbuild.from_file(paths.cache_for(self.base) / "PKGBUILD") + # create fresh chroot environment, fetch sources and - automagically - update PKGBUILD + task.init(paths.cache_for(self.base), [], None) + pkgbuild = Pkgbuild.from_file(paths.cache_for(self.base) / "PKGBUILD") - return full_version(pkgbuild.get("epoch"), pkgbuild["pkgver"], pkgbuild["pkgrel"]) + return full_version(pkgbuild.get("epoch"), pkgbuild["pkgver"], pkgbuild["pkgrel"]) except Exception: self.logger.exception("cannot determine version of VCS package") finally: diff --git a/src/ahriman/web/apispec/decorators.py b/src/ahriman/web/apispec/decorators.py index 88778944..e3ef3955 100644 --- a/src/ahriman/web/apispec/decorators.py +++ b/src/ahriman/web/apispec/decorators.py @@ -18,7 +18,8 @@ # along with this program. If not, see . # from aiohttp.web import HTTPException -from typing import Any, Callable +from collections.abc import Callable +from typing import Any from ahriman.models.user_access import UserAccess from ahriman.web.apispec import Schema, aiohttp_apispec diff --git a/src/ahriman/web/schemas/__init__.py b/src/ahriman/web/schemas/__init__.py index ceb68421..998ec790 100644 --- a/src/ahriman/web/schemas/__init__.py +++ b/src/ahriman/web/schemas/__init__.py @@ -31,6 +31,7 @@ from ahriman.web.schemas.info_schema import InfoSchema from ahriman.web.schemas.internal_status_schema import InternalStatusSchema from ahriman.web.schemas.log_schema import LogSchema from ahriman.web.schemas.login_schema import LoginSchema +from ahriman.web.schemas.logs_rotate_schema import LogsRotateSchema from ahriman.web.schemas.logs_schema import LogsSchema from ahriman.web.schemas.oauth2_schema import OAuth2Schema from ahriman.web.schemas.package_name_schema import PackageNameSchema @@ -53,5 +54,4 @@ from ahriman.web.schemas.repository_stats_schema import RepositoryStatsSchema from ahriman.web.schemas.search_schema import SearchSchema from ahriman.web.schemas.status_schema import StatusSchema from ahriman.web.schemas.update_flags_schema import UpdateFlagsSchema -from ahriman.web.schemas.versioned_log_schema import VersionedLogSchema from ahriman.web.schemas.worker_schema import WorkerSchema diff --git a/src/ahriman/web/schemas/log_schema.py b/src/ahriman/web/schemas/log_schema.py index 374c4809..5a4bd49f 100644 --- a/src/ahriman/web/schemas/log_schema.py +++ b/src/ahriman/web/schemas/log_schema.py @@ -17,12 +17,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +from ahriman import __version__ from ahriman.web.apispec import Schema, fields class LogSchema(Schema): """ - request package log schema + request and response package log schema """ created = fields.Float(required=True, metadata={ @@ -32,3 +33,10 @@ class LogSchema(Schema): message = fields.String(required=True, metadata={ "description": "Log message", }) + version = fields.String(required=True, metadata={ + "description": "Package version to tag", + "example": __version__, + }) + process_id = fields.String(metadata={ + "description": "Process unique identifier", + }) diff --git a/src/ahriman/web/schemas/versioned_log_schema.py b/src/ahriman/web/schemas/logs_rotate_schema.py similarity index 65% rename from src/ahriman/web/schemas/versioned_log_schema.py rename to src/ahriman/web/schemas/logs_rotate_schema.py index 2e08571d..09d5872a 100644 --- a/src/ahriman/web/schemas/versioned_log_schema.py +++ b/src/ahriman/web/schemas/logs_rotate_schema.py @@ -17,18 +17,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from ahriman import __version__ -from ahriman.web.apispec import fields -from ahriman.web.schemas.log_schema import LogSchema -from ahriman.web.schemas.repository_id_schema import RepositoryIdSchema +from ahriman.web.apispec import Schema, fields -class VersionedLogSchema(LogSchema, RepositoryIdSchema): +class LogsRotateSchema(Schema): """ - request package log schema + request logs rotate schema """ - version = fields.Integer(required=True, metadata={ - "description": "Package version to tag", - "example": __version__, + keep_last_records = fields.Integer(metadata={ + "description": "Keep the specified amount of records", }) diff --git a/src/ahriman/web/views/v1/packages/logs.py b/src/ahriman/web/views/v1/packages/logs.py index 87383571..1e253fa2 100644 --- a/src/ahriman/web/views/v1/packages/logs.py +++ b/src/ahriman/web/views/v1/packages/logs.py @@ -24,8 +24,7 @@ from ahriman.core.utils import pretty_datetime from ahriman.models.log_record_id import LogRecordId from ahriman.models.user_access import UserAccess from ahriman.web.apispec.decorators import apidocs -from ahriman.web.schemas import LogsSchema, PackageNameSchema, PackageVersionSchema, RepositoryIdSchema, \ - VersionedLogSchema +from ahriman.web.schemas import LogSchema, LogsSchema, PackageNameSchema, PackageVersionSchema, RepositoryIdSchema from ahriman.web.views.base import BaseView from ahriman.web.views.status_view_guard import StatusViewGuard @@ -97,7 +96,7 @@ class LogsView(StatusViewGuard, BaseView): response = { "package_base": package_base, "status": status.view(), - "logs": "\n".join(f"[{pretty_datetime(created)}] {message}" for created, message in logs) + "logs": "\n".join(f"[{pretty_datetime(created)}] {message}" for _, created, message in logs) } return json_response(response) @@ -109,7 +108,7 @@ class LogsView(StatusViewGuard, BaseView): error_400_enabled=True, error_404_description="Repository is unknown", match_schema=PackageNameSchema, - body_schema=VersionedLogSchema, + body_schema=LogSchema, ) async def post(self) -> None: """ @@ -129,6 +128,8 @@ class LogsView(StatusViewGuard, BaseView): except Exception as ex: raise HTTPBadRequest(reason=str(ex)) - self.service().package_logs_add(LogRecordId(package_base, version), created, record) + # either read from process identifier from payload or assign to the current process identifier + process_id = data.get("process_id", LogRecordId("", "").process_id) + self.service().package_logs_add(LogRecordId(package_base, version, process_id), created, record) raise HTTPNoContent diff --git a/src/ahriman/web/views/v1/service/logs.py b/src/ahriman/web/views/v1/service/logs.py new file mode 100644 index 00000000..552912fb --- /dev/null +++ b/src/ahriman/web/views/v1/service/logs.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2021-2025 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from aiohttp.web import HTTPBadRequest, HTTPNoContent + +from ahriman.models.user_access import UserAccess +from ahriman.web.apispec.decorators import apidocs +from ahriman.web.schemas import LogsRotateSchema +from ahriman.web.views.base import BaseView + + +class LogsView(BaseView): + """ + logs management web view + + Attributes: + DELETE_PERMISSION(UserAccess): (class attribute) delete permissions of self + """ + + DELETE_PERMISSION = UserAccess.Full + ROUTES = ["/api/v1/service/logs"] + + @apidocs( + tags=["Actions"], + summary="Rotate logs", + description="Remove older logs from system", + permission=DELETE_PERMISSION, + error_400_enabled=True, + error_404_description="Repository is unknown", + query_schema=LogsRotateSchema, + ) + async def delete(self) -> None: + """ + rotate logs from system + + Raises: + HTTPBadRequest: if bad data is supplied + HTTPNoContent: on success response + """ + try: + keep_last_records = int(self.request.query.get("keep_last_records", 0)) + except Exception as ex: + raise HTTPBadRequest(reason=str(ex)) + + self.service().logs_rotate(keep_last_records) + + raise HTTPNoContent diff --git a/src/ahriman/web/views/v2/packages/logs.py b/src/ahriman/web/views/v2/packages/logs.py index f1fda731..feae6f2e 100644 --- a/src/ahriman/web/views/v2/packages/logs.py +++ b/src/ahriman/web/views/v2/packages/logs.py @@ -67,6 +67,8 @@ class LogsView(StatusViewGuard, BaseView): { "created": created, "message": message, - } for created, message in logs + "version": log_record_id.version, + "process_id": log_record_id.process_id, + } for log_record_id, created, message in logs ] return json_response(response) diff --git a/tests/ahriman/core/database/migrations/test_m015_logs_process_id.py b/tests/ahriman/core/database/migrations/test_m015_logs_process_id.py new file mode 100644 index 00000000..0149c728 --- /dev/null +++ b/tests/ahriman/core/database/migrations/test_m015_logs_process_id.py @@ -0,0 +1,8 @@ +from ahriman.core.database.migrations.m015_logs_process_id import steps + + +def test_migration_logs_process_id() -> None: + """ + migration must not be empty + """ + assert steps diff --git a/tests/ahriman/core/database/operations/test_logs_operations.py b/tests/ahriman/core/database/operations/test_logs_operations.py index 3779a5de..601cff99 100644 --- a/tests/ahriman/core/database/operations/test_logs_operations.py +++ b/tests/ahriman/core/database/operations/test_logs_operations.py @@ -14,8 +14,12 @@ def test_logs_insert_remove_version(database: SQLite, package_ahriman: Package, database.logs_insert(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3") database.logs_remove(package_ahriman.base, "1") - assert database.logs_get(package_ahriman.base) == [(42.0, "message 1")] - assert database.logs_get(package_python_schedule.base) == [(42.0, "message 3")] + assert database.logs_get(package_ahriman.base) == [ + (LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"), + ] + assert database.logs_get(package_python_schedule.base) == [ + (LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3"), + ] def test_logs_insert_remove_multi(database: SQLite, package_ahriman: Package) -> None: @@ -28,7 +32,7 @@ def test_logs_insert_remove_multi(database: SQLite, package_ahriman: Package) -> database.logs_remove(package_ahriman.base, None, RepositoryId("i686", database._repository_id.name)) assert not database.logs_get(package_ahriman.base, repository_id=RepositoryId("i686", database._repository_id.name)) - assert database.logs_get(package_ahriman.base) == [(42.0, "message 1")] + assert database.logs_get(package_ahriman.base) == [(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1")] def test_logs_insert_remove_full(database: SQLite, package_ahriman: Package, package_python_schedule: Package) -> None: @@ -41,7 +45,9 @@ def test_logs_insert_remove_full(database: SQLite, package_ahriman: Package, pac database.logs_remove(package_ahriman.base, None) assert not database.logs_get(package_ahriman.base) - assert database.logs_get(package_python_schedule.base) == [(42.0, "message 3")] + assert database.logs_get(package_python_schedule.base) == [ + (LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3"), + ] def test_logs_insert_get(database: SQLite, package_ahriman: Package) -> None: @@ -50,7 +56,10 @@ def test_logs_insert_get(database: SQLite, package_ahriman: Package) -> None: """ database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2") database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") - assert database.logs_get(package_ahriman.base) == [(42.0, "message 1"), (43.0, "message 2")] + assert database.logs_get(package_ahriman.base) == [ + (LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"), + (LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"), + ] def test_logs_insert_get_pagination(database: SQLite, package_ahriman: Package) -> None: @@ -59,7 +68,9 @@ def test_logs_insert_get_pagination(database: SQLite, package_ahriman: Package) """ database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2") - assert database.logs_get(package_ahriman.base, 1, 1) == [(42.0, "message 1")] + assert database.logs_get(package_ahriman.base, 1, 1) == [ + (LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"), + ] def test_logs_insert_get_multi(database: SQLite, package_ahriman: Package) -> None: @@ -71,5 +82,40 @@ def test_logs_insert_get_multi(database: SQLite, package_ahriman: Package) -> No RepositoryId("i686", database._repository_id.name)) assert database.logs_get(package_ahriman.base, - repository_id=RepositoryId("i686", database._repository_id.name)) == [(43.0, "message 2")] - assert database.logs_get(package_ahriman.base) == [(42.0, "message 1")] + repository_id=RepositoryId("i686", database._repository_id.name)) == [ + (LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"), + ] + assert database.logs_get(package_ahriman.base) == [ + (LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"), + ] + + +def test_logs_rotate_remove_all(database: SQLite, package_ahriman: Package) -> None: + """ + must remove all records when rotating with keep_last_records is 0 + """ + database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") + database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2") + database.logs_insert(LogRecordId(package_ahriman.base, "2"), 44.0, "message 3") + + database.logs_rotate(0) + assert not database.logs_get(package_ahriman.base) + + +def test_logs_rotate_remove_duplicates(database: SQLite, package_ahriman: Package) -> None: + """ + must remove duplicate records while preserving the most recent one for each package version + """ + database.logs_insert(LogRecordId(package_ahriman.base, "1", "p1"), 42.0, "message 1") + database.logs_insert(LogRecordId(package_ahriman.base, "1", "p2"), 43.0, "message 2") + database.logs_insert(LogRecordId(package_ahriman.base, "1", "p3"), 44.0, "message 3") + database.logs_insert(LogRecordId(package_ahriman.base, "2", "p1"), 45.0, "message 4") + + database.logs_rotate(2) + + logs = database.logs_get(package_ahriman.base) + assert len(logs) == 2 + assert logs == [ + (LogRecordId(package_ahriman.base, "1", "p3"), 44.0, "message 3"), + (LogRecordId(package_ahriman.base, "2", "p1"), 45.0, "message 4"), + ] \ No newline at end of file diff --git a/tests/ahriman/core/log/test_http_log_handler.py b/tests/ahriman/core/log/test_http_log_handler.py index 40395c30..224f5881 100644 --- a/tests/ahriman/core/log/test_http_log_handler.py +++ b/tests/ahriman/core/log/test_http_log_handler.py @@ -19,12 +19,14 @@ def test_load(configuration: Configuration, mocker: MockerFixture) -> None: add_mock = mocker.patch("logging.Logger.addHandler") load_mock = mocker.patch("ahriman.core.status.Client.load") + atexit_mock = mocker.patch("atexit.register") _, repository_id = configuration.check_loaded() handler = HttpLogHandler.load(repository_id, configuration, report=False) assert handler add_mock.assert_called_once_with(handler) load_mock.assert_called_once_with(repository_id, configuration, report=False) + atexit_mock.assert_called_once_with(handler.rotate) def test_load_exist(configuration: Configuration) -> None: @@ -93,3 +95,16 @@ def test_emit_skip(configuration: Configuration, log_record: logging.LogRecord, handler.emit(log_record) log_mock.assert_not_called() + + +def test_rotate(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must rotate logs + """ + rotate_mock = mocker.patch("ahriman.core.status.Client.logs_rotate") + + _, repository_id = configuration.check_loaded() + handler = HttpLogHandler(repository_id, configuration, report=False, suppress_errors=False) + + handler.rotate() + rotate_mock.assert_called_once_with(handler.keep_last_records) diff --git a/tests/ahriman/core/log/test_lazy_logging.py b/tests/ahriman/core/log/test_lazy_logging.py index 9e0ce317..a3f416e2 100644 --- a/tests/ahriman/core/log/test_lazy_logging.py +++ b/tests/ahriman/core/log/test_lazy_logging.py @@ -87,13 +87,3 @@ def test_in_package_context_failed(database: SQLite, package_ahriman: Package, m raise ValueError() reset_mock.assert_called_once_with() - - -def test_suppress_logging(database: SQLite, mocker: MockerFixture) -> None: - """ - must temporary disable log messages - """ - disable_mock = mocker.patch("ahriman.core.log.lazy_logging.logging.disable") - with database.suppress_logging(): - pass - disable_mock.assert_has_calls([MockCall(logging.WARNING), MockCall(logging.NOTSET)]) diff --git a/tests/ahriman/core/status/test_client.py b/tests/ahriman/core/status/test_client.py index b152f621..ff2f0589 100644 --- a/tests/ahriman/core/status/test_client.py +++ b/tests/ahriman/core/status/test_client.py @@ -112,6 +112,13 @@ def test_event_get(client: Client) -> None: client.event_get(None, None) +def test_logs_rotate(client: Client, package_ahriman: Package) -> None: + """ + must do not raise exception on logs rotation call + """ + client.logs_rotate(1) + + def test_package_changes_get(client: Client, package_ahriman: Package) -> None: """ must raise not implemented on package changes request diff --git a/tests/ahriman/core/status/test_local_client.py b/tests/ahriman/core/status/test_local_client.py index 255983a7..082126a8 100644 --- a/tests/ahriman/core/status/test_local_client.py +++ b/tests/ahriman/core/status/test_local_client.py @@ -34,6 +34,15 @@ def test_event_get(local_client: LocalClient, package_ahriman: Package, mocker: local_client.repository_id) +def test_logs_rotate(local_client: LocalClient, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must rotate logs + """ + rotate_mock = mocker.patch("ahriman.core.database.SQLite.logs_rotate") + local_client.logs_rotate(42) + rotate_mock.assert_called_once_with(42, local_client.repository_id) + + def test_package_changes_get(local_client: LocalClient, package_ahriman: Package, mocker: MockerFixture) -> None: """ must retrieve package changes diff --git a/tests/ahriman/core/status/test_watcher.py b/tests/ahriman/core/status/test_watcher.py index 620e918f..ff99d3da 100644 --- a/tests/ahriman/core/status/test_watcher.py +++ b/tests/ahriman/core/status/test_watcher.py @@ -5,7 +5,6 @@ from pytest_mock import MockerFixture from ahriman.core.exceptions import UnknownPackageError from ahriman.core.status.watcher import Watcher from ahriman.models.build_status import BuildStatus, BuildStatusEnum -from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package @@ -64,38 +63,6 @@ def test_package_get_failed(watcher: Watcher, package_ahriman: Package) -> None: watcher.package_get(package_ahriman.base) -def test_package_logs_add_new(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: - """ - must create package logs record for new package - """ - delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.package_logs_remove", create=True) - insert_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_logs_add") - - log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.version) - assert watcher._last_log_record_id != log_record_id - - watcher.package_logs_add(log_record_id, 42.01, "log record") - delete_mock.assert_called_once_with(package_ahriman.base, log_record_id.version) - insert_mock.assert_called_once_with(log_record_id, 42.01, "log record") - - assert watcher._last_log_record_id == log_record_id - - -def test_package_logs_add_update(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: - """ - must create package logs record for current package - """ - delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.package_logs_remove", create=True) - insert_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_logs_add") - - log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.version) - watcher._last_log_record_id = log_record_id - - watcher.package_logs_add(log_record_id, 42.01, "log record") - delete_mock.assert_not_called() - insert_mock.assert_called_once_with(log_record_id, 42.01, "log record") - - def test_package_remove(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: """ must remove package base diff --git a/tests/ahriman/core/status/test_web_client.py b/tests/ahriman/core/status/test_web_client.py index c6ec36ba..05ce6516 100644 --- a/tests/ahriman/core/status/test_web_client.py +++ b/tests/ahriman/core/status/test_web_client.py @@ -257,6 +257,57 @@ def test_event_get_failed_http_error_suppress(web_client: WebClient, mocker: Moc logging_mock.assert_not_called() +def test_logs_rotate(web_client: WebClient, mocker: MockerFixture) -> None: + """ + must rotate logs + """ + requests_mock = mocker.patch("ahriman.core.status.web_client.WebClient.make_request") + + web_client.logs_rotate(42) + requests_mock.assert_called_once_with("DELETE", pytest.helpers.anyvar(str, True), + params=web_client.repository_id.query() + [("keep_last_records", "42")]) + + +def test_logs_rotate_failed(web_client: WebClient, mocker: MockerFixture) -> None: + """ + must suppress any exception happened during logs rotation + """ + mocker.patch("requests.Session.request", side_effect=Exception()) + web_client.logs_rotate(42) + + +def test_logs_rotate_failed_http_error(web_client: WebClient, mocker: MockerFixture) -> None: + """ + must suppress HTTP exception happened during logs rotation + """ + mocker.patch("requests.Session.request", side_effect=requests.HTTPError()) + web_client.logs_rotate(42) + + +def test_logs_rotate_failed_suppress(web_client: WebClient, mocker: MockerFixture) -> None: + """ + must suppress any exception happened during logs rotation and don't log + """ + web_client.suppress_errors = True + mocker.patch("requests.Session.request", side_effect=Exception()) + logging_mock = mocker.patch("logging.exception") + + web_client.logs_rotate(42) + logging_mock.assert_not_called() + + +def test_logs_rotate_failed_http_error_suppress(web_client: WebClient, mocker: MockerFixture) -> None: + """ + must suppress HTTP exception happened during logs rotation and don't log + """ + web_client.suppress_errors = True + mocker.patch("requests.Session.request", side_effect=requests.HTTPError()) + logging_mock = mocker.patch("logging.exception") + + web_client.logs_rotate(42) + logging_mock.assert_not_called() + + def test_package_changes_get(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None: """ must get changes @@ -551,6 +602,7 @@ def test_package_logs_add(web_client: WebClient, log_record: logging.LogRecord, payload = { "created": log_record.created, "message": log_record.getMessage(), + "process_id": LogRecordId.process_id, "version": package_ahriman.version, } @@ -588,7 +640,12 @@ def test_package_logs_get(web_client: WebClient, package_ahriman: Package, mocke """ must get logs """ - message = {"created": 42.0, "message": "log"} + message = { + "created": 42.0, + "message": "log", + "version": package_ahriman.version, + "process_id": LogRecordId.process_id, + } response_obj = requests.Response() response_obj._content = json.dumps([message]).encode("utf8") response_obj.status_code = 200 @@ -598,7 +655,9 @@ def test_package_logs_get(web_client: WebClient, package_ahriman: Package, mocke result = web_client.package_logs_get(package_ahriman.base, 1, 2) requests_mock.assert_called_once_with("GET", pytest.helpers.anyvar(str, True), params=web_client.repository_id.query() + [("limit", "1"), ("offset", "2")]) - assert result == [(message["created"], message["message"])] + assert result == [ + (LogRecordId(package_ahriman.base, package_ahriman.version), message["created"], message["message"]), + ] def test_package_logs_get_failed(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/web/schemas/test_versioned_log_schema.py b/tests/ahriman/web/schemas/test_logs_rotate_schema.py similarity index 100% rename from tests/ahriman/web/schemas/test_versioned_log_schema.py rename to tests/ahriman/web/schemas/test_logs_rotate_schema.py