diff --git a/package/share/ahriman/settings/ahriman.ini b/package/share/ahriman/settings/ahriman.ini
index d7e2f328..e53a1fa2 100644
--- a/package/share/ahriman/settings/ahriman.ini
+++ b/package/share/ahriman/settings/ahriman.ini
@@ -7,6 +7,8 @@ logging = ahriman.ini.d/logging.ini
;apply_migrations = yes
; Path to the application SQLite database.
database = ${repository:root}/ahriman.db
+; Keep last build logs for each package
+keep_last_logs = 5
[alpm]
; Path to pacman system database cache.
diff --git a/src/ahriman/core/configuration/schema.py b/src/ahriman/core/configuration/schema.py
index 8c93fc43..008db9b4 100644
--- a/src/ahriman/core/configuration/schema.py
+++ b/src/ahriman/core/configuration/schema.py
@@ -45,6 +45,11 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"path_exists": True,
"path_type": "dir",
},
+ "keep_last_logs": {
+ "type": "integer",
+ "coerce": "integer",
+ "min": 0,
+ },
"logging": {
"type": "path",
"coerce": "absolute_path",
diff --git a/src/ahriman/core/database/migrations/m015_logs_process_id.py b/src/ahriman/core/database/migrations/m015_logs_process_id.py
new file mode 100644
index 00000000..b0a9001b
--- /dev/null
+++ b/src/ahriman/core/database/migrations/m015_logs_process_id.py
@@ -0,0 +1,27 @@
+#
+# Copyright (c) 2021-2025 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+__all__ = ["steps"]
+
+
+steps = [
+ """
+ alter table logs add column process_id text not null default ''
+ """,
+]
diff --git a/src/ahriman/core/database/operations/logs_operations.py b/src/ahriman/core/database/operations/logs_operations.py
index 8704df49..50645b00 100644
--- a/src/ahriman/core/database/operations/logs_operations.py
+++ b/src/ahriman/core/database/operations/logs_operations.py
@@ -30,7 +30,7 @@ class LogsOperations(Operations):
"""
def logs_get(self, package_base: str, limit: int = -1, offset: int = 0,
- repository_id: RepositoryId | None = None) -> list[tuple[float, str]]:
+ repository_id: RepositoryId | None = None) -> list[tuple[LogRecordId, float, str]]:
"""
extract logs for specified package base
@@ -41,16 +41,16 @@ class LogsOperations(Operations):
repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
Return:
- list[tuple[float, str]]: sorted package log records and their timestamps
+ list[tuple[LogRecordId, float, str]]: sorted package log records and their timestamps
"""
repository_id = repository_id or self._repository_id
- def run(connection: Connection) -> list[tuple[float, str]]:
+ def run(connection: Connection) -> list[tuple[LogRecordId, float, str]]:
return [
- (row["created"], row["record"])
+ (LogRecordId(package_base, row["version"], row["process_id"]), row["created"], row["record"])
for row in connection.execute(
"""
- select created, record from (
+ select created, record, version, process_id from (
select * from logs
where package_base = :package_base and repository = :repository
order by created desc limit :limit offset :offset
@@ -83,9 +83,9 @@ class LogsOperations(Operations):
connection.execute(
"""
insert into logs
- (package_base, version, created, record, repository)
+ (package_base, version, created, record, repository, process_id)
values
- (:package_base, :version, :created, :record, :repository)
+ (:package_base, :version, :created, :record, :repository, :process_id)
""",
{
"package_base": log_record_id.package_base,
@@ -93,6 +93,7 @@ class LogsOperations(Operations):
"created": created,
"record": record,
"repository": repository_id.id,
+ "process_id": log_record_id.process_id,
}
)
@@ -125,3 +126,54 @@ class LogsOperations(Operations):
)
return self.with_connection(run, commit=True)
+
+ def logs_rotate(self, keep_last_records: int, repository_id: RepositoryId | None = None) -> None:
+ """
+ rotate logs in storage. This method will remove old logs, keeping only the last N records for each package
+
+ Args:
+ keep_last_records(int): number of last records to keep
+ repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
+ """
+ repository_id = repository_id or self._repository_id
+
+ def remove_duplicates(connection: Connection) -> None:
+ connection.execute(
+ """
+ delete from logs
+ where (package_base, version, repository, process_id) not in (
+ select package_base, version, repository, process_id from logs
+ where (package_base, version, repository, created) in (
+ select package_base, version, repository, max(created) from logs
+ where repository = :repository
+ group by package_base, version, repository
+ )
+ )
+ """,
+ {
+ "repository": repository_id.id,
+ }
+ )
+
+ def remove_older(connection: Connection) -> None:
+ connection.execute(
+ """
+ delete from logs
+ where (package_base, repository, process_id) in (
+ select package_base, repository, process_id from logs
+ where repository = :repository
+ group by package_base, repository, process_id
+ order by min(created) desc limit -1 offset :offset
+ )
+ """,
+ {
+ "offset": keep_last_records,
+ "repository": repository_id.id,
+ }
+ )
+
+ def run(connection: Connection) -> None:
+ remove_duplicates(connection)
+ remove_older(connection)
+
+ return self.with_connection(run, commit=True)
diff --git a/src/ahriman/core/log/http_log_handler.py b/src/ahriman/core/log/http_log_handler.py
index 769692dc..2377478f 100644
--- a/src/ahriman/core/log/http_log_handler.py
+++ b/src/ahriman/core/log/http_log_handler.py
@@ -17,6 +17,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#
+import atexit
import logging
from typing import Self
@@ -33,6 +34,7 @@ class HttpLogHandler(logging.Handler):
method
Attributes:
+ keep_last_records(int): number of last records to keep
reporter(Client): build status reporter instance
suppress_errors(bool): suppress logging errors (e.g. if no web server available)
"""
@@ -51,6 +53,7 @@ class HttpLogHandler(logging.Handler):
self.reporter = Client.load(repository_id, configuration, report=report)
self.suppress_errors = suppress_errors
+ self.keep_last_records = configuration.getint("settings", "keep_last_logs", fallback=0)
@classmethod
def load(cls, repository_id: RepositoryId, configuration: Configuration, *, report: bool) -> Self:
@@ -76,6 +79,8 @@ class HttpLogHandler(logging.Handler):
handler = cls(repository_id, configuration, report=report, suppress_errors=suppress_errors)
root.addHandler(handler)
+ atexit.register(handler.rotate)
+
return handler
def emit(self, record: logging.LogRecord) -> None:
@@ -95,3 +100,9 @@ class HttpLogHandler(logging.Handler):
if self.suppress_errors:
return
self.handleError(record)
+
+ def rotate(self) -> None:
+ """
+ rotate log records, removing older ones
+ """
+ self.reporter.logs_rotate(self.keep_last_records)
diff --git a/src/ahriman/core/log/lazy_logging.py b/src/ahriman/core/log/lazy_logging.py
index 2e6b4b02..ff7db2cf 100644
--- a/src/ahriman/core/log/lazy_logging.py
+++ b/src/ahriman/core/log/lazy_logging.py
@@ -99,24 +99,3 @@ class LazyLogging:
yield
finally:
self._package_logger_reset()
-
- @contextlib.contextmanager
- def suppress_logging(self, log_level: int = logging.WARNING) -> Generator[None, None, None]:
- """
- silence log messages in context
-
- Args:
- log_level(int, optional): the highest log level to keep (Default value = logging.WARNING)
-
- Examples:
- This function is designed to be used to suppress all log messages in context, e.g.:
-
- >>> with self.suppress_logging():
- >>> do_some_noisy_actions()
- """
- current_level = self.logger.manager.disable
- try:
- logging.disable(log_level)
- yield
- finally:
- logging.disable(current_level)
diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py
index 41755e79..6f9f0f8f 100644
--- a/src/ahriman/core/repository/executor.py
+++ b/src/ahriman/core/repository/executor.py
@@ -75,7 +75,7 @@ class Executor(PackageInfo, Cleaner):
result = Result()
for single in updates:
- with self.in_package_context(single.base, local_versions.get(single.base)), \
+ with self.in_package_context(single.base, single.version), \
TemporaryDirectory(ignore_cleanup_errors=True) as dir_name:
try:
with self.in_event(single.base, EventType.PackageUpdated, failure=EventType.PackageUpdateFailed):
@@ -194,7 +194,6 @@ class Executor(PackageInfo, Cleaner):
self.repo.add(package_path)
current_packages = {package.base: package for package in self.packages()}
- local_versions = {package_base: package.version for package_base, package in current_packages.items()}
removed_packages: list[str] = [] # list of packages which have been removed from the base
updates = self.load_archives(packages)
@@ -202,7 +201,7 @@ class Executor(PackageInfo, Cleaner):
result = Result()
for local in updates:
- with self.in_package_context(local.base, local_versions.get(local.base)):
+ with self.in_package_context(local.base, local.version):
try:
packager = self.packager(packagers, local.base)
diff --git a/src/ahriman/core/repository/update_handler.py b/src/ahriman/core/repository/update_handler.py
index 83d3d989..225bd150 100644
--- a/src/ahriman/core/repository/update_handler.py
+++ b/src/ahriman/core/repository/update_handler.py
@@ -144,8 +144,7 @@ class UpdateHandler(PackageInfo, Cleaner):
branch="master",
)
- with self.suppress_logging():
- Sources.fetch(cache_dir, source)
+ Sources.fetch(cache_dir, source)
remote = Package.from_build(cache_dir, self.architecture, None)
local = packages.get(remote.base)
diff --git a/src/ahriman/core/status/client.py b/src/ahriman/core/status/client.py
index b7de1b65..abc78a15 100644
--- a/src/ahriman/core/status/client.py
+++ b/src/ahriman/core/status/client.py
@@ -115,6 +115,14 @@ class Client:
"""
raise NotImplementedError
+ def logs_rotate(self, keep_last_records: int) -> None:
+ """
+ remove older logs from storage
+
+ Args:
+ keep_last_records(int): number of last records to keep
+ """
+
def package_changes_get(self, package_base: str) -> Changes:
"""
get package changes
@@ -197,7 +205,8 @@ class Client:
"""
# this method does not raise NotImplementedError because it is actively used as dummy client for http log
- def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]:
+ def package_logs_get(self, package_base: str, limit: int = -1,
+ offset: int = 0) -> list[tuple[LogRecordId, float, str]]:
"""
get package logs
@@ -207,7 +216,7 @@ class Client:
offset(int, optional): records offset (Default value = 0)
Returns:
- list[tuple[float, str]]: package logs
+ list[tuple[LogRecordId, float, str]]: package logs
Raises:
NotImplementedError: not implemented method
diff --git a/src/ahriman/core/status/local_client.py b/src/ahriman/core/status/local_client.py
index 6c0608ee..b58a7a60 100644
--- a/src/ahriman/core/status/local_client.py
+++ b/src/ahriman/core/status/local_client.py
@@ -75,6 +75,15 @@ class LocalClient(Client):
"""
return self.database.event_get(event, object_id, from_date, to_date, limit, offset, self.repository_id)
+ def logs_rotate(self, keep_last_records: int) -> None:
+ """
+ remove older logs from storage
+
+ Args:
+ keep_last_records(int): number of last records to keep
+ """
+ self.database.logs_rotate(keep_last_records, self.repository_id)
+
def package_changes_get(self, package_base: str) -> Changes:
"""
get package changes
@@ -145,7 +154,8 @@ class LocalClient(Client):
"""
self.database.logs_insert(log_record_id, created, message, self.repository_id)
- def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]:
+ def package_logs_get(self, package_base: str, limit: int = -1,
+ offset: int = 0) -> list[tuple[LogRecordId, float, str]]:
"""
get package logs
@@ -155,7 +165,7 @@ class LocalClient(Client):
offset(int, optional): records offset (Default value = 0)
Returns:
- list[tuple[float, str]]: package logs
+ list[tuple[LogRecordId, float, str]]: package logs
"""
return self.database.logs_get(package_base, limit, offset, self.repository_id)
diff --git a/src/ahriman/core/status/watcher.py b/src/ahriman/core/status/watcher.py
index ee541094..04014cf9 100644
--- a/src/ahriman/core/status/watcher.py
+++ b/src/ahriman/core/status/watcher.py
@@ -53,9 +53,6 @@ class Watcher(LazyLogging):
self._known: dict[str, tuple[Package, BuildStatus]] = {}
self.status = BuildStatus()
- # special variables for updating logs
- self._last_log_record_id = LogRecordId("", "")
-
@property
def packages(self) -> list[tuple[Package, BuildStatus]]:
"""
@@ -81,6 +78,8 @@ class Watcher(LazyLogging):
for package, status in self.client.package_get(None)
}
+ logs_rotate: Callable[[int], None]
+
package_changes_get: Callable[[str], Changes]
package_changes_update: Callable[[str, Changes], None]
@@ -108,22 +107,9 @@ class Watcher(LazyLogging):
except KeyError:
raise UnknownPackageError(package_base) from None
- def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None:
- """
- make new log record into database
+ package_logs_add: Callable[[LogRecordId, float, str], None]
- Args:
- log_record_id(LogRecordId): log record id
- created(float): log created timestamp
- message(str): log message
- """
- if self._last_log_record_id != log_record_id:
- # there is new log record, so we remove old ones
- self.package_logs_remove(log_record_id.package_base, log_record_id.version)
- self._last_log_record_id = log_record_id
- self.client.package_logs_add(log_record_id, created, message)
-
- package_logs_get: Callable[[str, int, int], list[tuple[float, str]]]
+ package_logs_get: Callable[[str, int, int], list[tuple[LogRecordId, float, str]]]
package_logs_remove: Callable[[str, str | None], None]
diff --git a/src/ahriman/core/status/web_client.py b/src/ahriman/core/status/web_client.py
index 158cf9cd..e29f5c9b 100644
--- a/src/ahriman/core/status/web_client.py
+++ b/src/ahriman/core/status/web_client.py
@@ -210,6 +210,18 @@ class WebClient(Client, SyncAhrimanClient):
return []
+ def logs_rotate(self, keep_last_records: int) -> None:
+ """
+ remove older logs from storage
+
+ Args:
+ keep_last_records(int): number of last records to keep
+ """
+ query = self.repository_id.query() + [("keep_last_records", str(keep_last_records))]
+
+ with contextlib.suppress(Exception):
+ self.make_request("DELETE", f"{self.address}/api/v1/service/logs", params=query)
+
def package_changes_get(self, package_base: str) -> Changes:
"""
get package changes
@@ -306,6 +318,7 @@ class WebClient(Client, SyncAhrimanClient):
payload = {
"created": created,
"message": message,
+ "process_id": log_record_id.process_id,
"version": log_record_id.version,
}
@@ -315,7 +328,8 @@ class WebClient(Client, SyncAhrimanClient):
self.make_request("POST", self._logs_url(log_record_id.package_base),
params=self.repository_id.query(), json=payload, suppress_errors=True)
- def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]:
+ def package_logs_get(self, package_base: str, limit: int = -1,
+ offset: int = 0) -> list[tuple[LogRecordId, float, str]]:
"""
get package logs
@@ -325,7 +339,7 @@ class WebClient(Client, SyncAhrimanClient):
offset(int, optional): records offset (Default value = 0)
Returns:
- list[tuple[float, str]]: package logs
+ list[tuple[LogRecordId, float, str]]: package logs
"""
query = self.repository_id.query() + [("limit", str(limit)), ("offset", str(offset))]
@@ -333,7 +347,13 @@ class WebClient(Client, SyncAhrimanClient):
response = self.make_request("GET", self._logs_url(package_base), params=query)
response_json = response.json()
- return [(record["created"], record["message"]) for record in response_json]
+ return [
+ (
+ LogRecordId(package_base, record["version"], record["process_id"]),
+ record["created"],
+ record["message"]
+ ) for record in response_json
+ ]
return []
diff --git a/src/ahriman/models/log_record_id.py b/src/ahriman/models/log_record_id.py
index bff9b6ee..c7399ef8 100644
--- a/src/ahriman/models/log_record_id.py
+++ b/src/ahriman/models/log_record_id.py
@@ -17,7 +17,9 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#
-from dataclasses import dataclass
+import uuid
+
+from dataclasses import dataclass, field
@dataclass(frozen=True)
@@ -28,7 +30,12 @@ class LogRecordId:
Attributes:
package_base(str): package base for which log record belongs
version(str): package version for which log record belongs
+ process_id(str, optional): unique process identifier
"""
package_base: str
version: str
+
+ # this is not mistake, this value is kind of global identifier, which is generated
+ # upon the process start
+ process_id: str = field(default=str(uuid.uuid4()))
diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py
index ee1ca8df..cfad22cc 100644
--- a/src/ahriman/models/package.py
+++ b/src/ahriman/models/package.py
@@ -429,12 +429,11 @@ class Package(LazyLogging):
task = Task(self, configuration, repository_id.architecture, paths)
try:
- with self.suppress_logging():
- # create fresh chroot environment, fetch sources and - automagically - update PKGBUILD
- task.init(paths.cache_for(self.base), [], None)
- pkgbuild = Pkgbuild.from_file(paths.cache_for(self.base) / "PKGBUILD")
+ # create fresh chroot environment, fetch sources and - automagically - update PKGBUILD
+ task.init(paths.cache_for(self.base), [], None)
+ pkgbuild = Pkgbuild.from_file(paths.cache_for(self.base) / "PKGBUILD")
- return full_version(pkgbuild.get("epoch"), pkgbuild["pkgver"], pkgbuild["pkgrel"])
+ return full_version(pkgbuild.get("epoch"), pkgbuild["pkgver"], pkgbuild["pkgrel"])
except Exception:
self.logger.exception("cannot determine version of VCS package")
finally:
diff --git a/src/ahriman/web/apispec/decorators.py b/src/ahriman/web/apispec/decorators.py
index 88778944..e3ef3955 100644
--- a/src/ahriman/web/apispec/decorators.py
+++ b/src/ahriman/web/apispec/decorators.py
@@ -18,7 +18,8 @@
# along with this program. If not, see .
#
from aiohttp.web import HTTPException
-from typing import Any, Callable
+from collections.abc import Callable
+from typing import Any
from ahriman.models.user_access import UserAccess
from ahriman.web.apispec import Schema, aiohttp_apispec
diff --git a/src/ahriman/web/schemas/__init__.py b/src/ahriman/web/schemas/__init__.py
index ceb68421..998ec790 100644
--- a/src/ahriman/web/schemas/__init__.py
+++ b/src/ahriman/web/schemas/__init__.py
@@ -31,6 +31,7 @@ from ahriman.web.schemas.info_schema import InfoSchema
from ahriman.web.schemas.internal_status_schema import InternalStatusSchema
from ahriman.web.schemas.log_schema import LogSchema
from ahriman.web.schemas.login_schema import LoginSchema
+from ahriman.web.schemas.logs_rotate_schema import LogsRotateSchema
from ahriman.web.schemas.logs_schema import LogsSchema
from ahriman.web.schemas.oauth2_schema import OAuth2Schema
from ahriman.web.schemas.package_name_schema import PackageNameSchema
@@ -53,5 +54,4 @@ from ahriman.web.schemas.repository_stats_schema import RepositoryStatsSchema
from ahriman.web.schemas.search_schema import SearchSchema
from ahriman.web.schemas.status_schema import StatusSchema
from ahriman.web.schemas.update_flags_schema import UpdateFlagsSchema
-from ahriman.web.schemas.versioned_log_schema import VersionedLogSchema
from ahriman.web.schemas.worker_schema import WorkerSchema
diff --git a/src/ahriman/web/schemas/log_schema.py b/src/ahriman/web/schemas/log_schema.py
index 374c4809..5a4bd49f 100644
--- a/src/ahriman/web/schemas/log_schema.py
+++ b/src/ahriman/web/schemas/log_schema.py
@@ -17,12 +17,13 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#
+from ahriman import __version__
from ahriman.web.apispec import Schema, fields
class LogSchema(Schema):
"""
- request package log schema
+ request and response package log schema
"""
created = fields.Float(required=True, metadata={
@@ -32,3 +33,10 @@ class LogSchema(Schema):
message = fields.String(required=True, metadata={
"description": "Log message",
})
+ version = fields.String(required=True, metadata={
+ "description": "Package version to tag",
+ "example": __version__,
+ })
+ process_id = fields.String(metadata={
+ "description": "Process unique identifier",
+ })
diff --git a/src/ahriman/web/schemas/versioned_log_schema.py b/src/ahriman/web/schemas/logs_rotate_schema.py
similarity index 65%
rename from src/ahriman/web/schemas/versioned_log_schema.py
rename to src/ahriman/web/schemas/logs_rotate_schema.py
index 2e08571d..09d5872a 100644
--- a/src/ahriman/web/schemas/versioned_log_schema.py
+++ b/src/ahriman/web/schemas/logs_rotate_schema.py
@@ -17,18 +17,14 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see .
#
-from ahriman import __version__
-from ahriman.web.apispec import fields
-from ahriman.web.schemas.log_schema import LogSchema
-from ahriman.web.schemas.repository_id_schema import RepositoryIdSchema
+from ahriman.web.apispec import Schema, fields
-class VersionedLogSchema(LogSchema, RepositoryIdSchema):
+class LogsRotateSchema(Schema):
"""
- request package log schema
+ request logs rotate schema
"""
- version = fields.Integer(required=True, metadata={
- "description": "Package version to tag",
- "example": __version__,
+ keep_last_records = fields.Integer(metadata={
+ "description": "Keep the specified amount of records",
})
diff --git a/src/ahriman/web/views/v1/packages/logs.py b/src/ahriman/web/views/v1/packages/logs.py
index 87383571..1e253fa2 100644
--- a/src/ahriman/web/views/v1/packages/logs.py
+++ b/src/ahriman/web/views/v1/packages/logs.py
@@ -24,8 +24,7 @@ from ahriman.core.utils import pretty_datetime
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.user_access import UserAccess
from ahriman.web.apispec.decorators import apidocs
-from ahriman.web.schemas import LogsSchema, PackageNameSchema, PackageVersionSchema, RepositoryIdSchema, \
- VersionedLogSchema
+from ahriman.web.schemas import LogSchema, LogsSchema, PackageNameSchema, PackageVersionSchema, RepositoryIdSchema
from ahriman.web.views.base import BaseView
from ahriman.web.views.status_view_guard import StatusViewGuard
@@ -97,7 +96,7 @@ class LogsView(StatusViewGuard, BaseView):
response = {
"package_base": package_base,
"status": status.view(),
- "logs": "\n".join(f"[{pretty_datetime(created)}] {message}" for created, message in logs)
+ "logs": "\n".join(f"[{pretty_datetime(created)}] {message}" for _, created, message in logs)
}
return json_response(response)
@@ -109,7 +108,7 @@ class LogsView(StatusViewGuard, BaseView):
error_400_enabled=True,
error_404_description="Repository is unknown",
match_schema=PackageNameSchema,
- body_schema=VersionedLogSchema,
+ body_schema=LogSchema,
)
async def post(self) -> None:
"""
@@ -129,6 +128,8 @@ class LogsView(StatusViewGuard, BaseView):
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
- self.service().package_logs_add(LogRecordId(package_base, version), created, record)
+ # either read from process identifier from payload or assign to the current process identifier
+ process_id = data.get("process_id", LogRecordId("", "").process_id)
+ self.service().package_logs_add(LogRecordId(package_base, version, process_id), created, record)
raise HTTPNoContent
diff --git a/src/ahriman/web/views/v1/service/logs.py b/src/ahriman/web/views/v1/service/logs.py
new file mode 100644
index 00000000..552912fb
--- /dev/null
+++ b/src/ahriman/web/views/v1/service/logs.py
@@ -0,0 +1,63 @@
+#
+# Copyright (c) 2021-2025 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+from aiohttp.web import HTTPBadRequest, HTTPNoContent
+
+from ahriman.models.user_access import UserAccess
+from ahriman.web.apispec.decorators import apidocs
+from ahriman.web.schemas import LogsRotateSchema
+from ahriman.web.views.base import BaseView
+
+
+class LogsView(BaseView):
+ """
+ logs management web view
+
+ Attributes:
+ DELETE_PERMISSION(UserAccess): (class attribute) delete permissions of self
+ """
+
+ DELETE_PERMISSION = UserAccess.Full
+ ROUTES = ["/api/v1/service/logs"]
+
+ @apidocs(
+ tags=["Actions"],
+ summary="Rotate logs",
+ description="Remove older logs from system",
+ permission=DELETE_PERMISSION,
+ error_400_enabled=True,
+ error_404_description="Repository is unknown",
+ query_schema=LogsRotateSchema,
+ )
+ async def delete(self) -> None:
+ """
+ rotate logs from system
+
+ Raises:
+ HTTPBadRequest: if bad data is supplied
+ HTTPNoContent: on success response
+ """
+ try:
+ keep_last_records = int(self.request.query.get("keep_last_records", 0))
+ except Exception as ex:
+ raise HTTPBadRequest(reason=str(ex))
+
+ self.service().logs_rotate(keep_last_records)
+
+ raise HTTPNoContent
diff --git a/src/ahriman/web/views/v2/packages/logs.py b/src/ahriman/web/views/v2/packages/logs.py
index f1fda731..feae6f2e 100644
--- a/src/ahriman/web/views/v2/packages/logs.py
+++ b/src/ahriman/web/views/v2/packages/logs.py
@@ -67,6 +67,8 @@ class LogsView(StatusViewGuard, BaseView):
{
"created": created,
"message": message,
- } for created, message in logs
+ "version": log_record_id.version,
+ "process_id": log_record_id.process_id,
+ } for log_record_id, created, message in logs
]
return json_response(response)
diff --git a/tests/ahriman/core/database/migrations/test_m015_logs_process_id.py b/tests/ahriman/core/database/migrations/test_m015_logs_process_id.py
new file mode 100644
index 00000000..0149c728
--- /dev/null
+++ b/tests/ahriman/core/database/migrations/test_m015_logs_process_id.py
@@ -0,0 +1,8 @@
+from ahriman.core.database.migrations.m015_logs_process_id import steps
+
+
+def test_migration_logs_process_id() -> None:
+ """
+ migration must not be empty
+ """
+ assert steps
diff --git a/tests/ahriman/core/database/operations/test_logs_operations.py b/tests/ahriman/core/database/operations/test_logs_operations.py
index 3779a5de..601cff99 100644
--- a/tests/ahriman/core/database/operations/test_logs_operations.py
+++ b/tests/ahriman/core/database/operations/test_logs_operations.py
@@ -14,8 +14,12 @@ def test_logs_insert_remove_version(database: SQLite, package_ahriman: Package,
database.logs_insert(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3")
database.logs_remove(package_ahriman.base, "1")
- assert database.logs_get(package_ahriman.base) == [(42.0, "message 1")]
- assert database.logs_get(package_python_schedule.base) == [(42.0, "message 3")]
+ assert database.logs_get(package_ahriman.base) == [
+ (LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"),
+ ]
+ assert database.logs_get(package_python_schedule.base) == [
+ (LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3"),
+ ]
def test_logs_insert_remove_multi(database: SQLite, package_ahriman: Package) -> None:
@@ -28,7 +32,7 @@ def test_logs_insert_remove_multi(database: SQLite, package_ahriman: Package) ->
database.logs_remove(package_ahriman.base, None, RepositoryId("i686", database._repository_id.name))
assert not database.logs_get(package_ahriman.base, repository_id=RepositoryId("i686", database._repository_id.name))
- assert database.logs_get(package_ahriman.base) == [(42.0, "message 1")]
+ assert database.logs_get(package_ahriman.base) == [(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1")]
def test_logs_insert_remove_full(database: SQLite, package_ahriman: Package, package_python_schedule: Package) -> None:
@@ -41,7 +45,9 @@ def test_logs_insert_remove_full(database: SQLite, package_ahriman: Package, pac
database.logs_remove(package_ahriman.base, None)
assert not database.logs_get(package_ahriman.base)
- assert database.logs_get(package_python_schedule.base) == [(42.0, "message 3")]
+ assert database.logs_get(package_python_schedule.base) == [
+ (LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3"),
+ ]
def test_logs_insert_get(database: SQLite, package_ahriman: Package) -> None:
@@ -50,7 +56,10 @@ def test_logs_insert_get(database: SQLite, package_ahriman: Package) -> None:
"""
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2")
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1")
- assert database.logs_get(package_ahriman.base) == [(42.0, "message 1"), (43.0, "message 2")]
+ assert database.logs_get(package_ahriman.base) == [
+ (LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"),
+ (LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"),
+ ]
def test_logs_insert_get_pagination(database: SQLite, package_ahriman: Package) -> None:
@@ -59,7 +68,9 @@ def test_logs_insert_get_pagination(database: SQLite, package_ahriman: Package)
"""
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1")
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2")
- assert database.logs_get(package_ahriman.base, 1, 1) == [(42.0, "message 1")]
+ assert database.logs_get(package_ahriman.base, 1, 1) == [
+ (LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"),
+ ]
def test_logs_insert_get_multi(database: SQLite, package_ahriman: Package) -> None:
@@ -71,5 +82,40 @@ def test_logs_insert_get_multi(database: SQLite, package_ahriman: Package) -> No
RepositoryId("i686", database._repository_id.name))
assert database.logs_get(package_ahriman.base,
- repository_id=RepositoryId("i686", database._repository_id.name)) == [(43.0, "message 2")]
- assert database.logs_get(package_ahriman.base) == [(42.0, "message 1")]
+ repository_id=RepositoryId("i686", database._repository_id.name)) == [
+ (LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"),
+ ]
+ assert database.logs_get(package_ahriman.base) == [
+ (LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"),
+ ]
+
+
+def test_logs_rotate_remove_all(database: SQLite, package_ahriman: Package) -> None:
+ """
+ must remove all records when rotating with keep_last_records is 0
+ """
+ database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1")
+ database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2")
+ database.logs_insert(LogRecordId(package_ahriman.base, "2"), 44.0, "message 3")
+
+ database.logs_rotate(0)
+ assert not database.logs_get(package_ahriman.base)
+
+
+def test_logs_rotate_remove_duplicates(database: SQLite, package_ahriman: Package) -> None:
+ """
+ must remove duplicate records while preserving the most recent one for each package version
+ """
+ database.logs_insert(LogRecordId(package_ahriman.base, "1", "p1"), 42.0, "message 1")
+ database.logs_insert(LogRecordId(package_ahriman.base, "1", "p2"), 43.0, "message 2")
+ database.logs_insert(LogRecordId(package_ahriman.base, "1", "p3"), 44.0, "message 3")
+ database.logs_insert(LogRecordId(package_ahriman.base, "2", "p1"), 45.0, "message 4")
+
+ database.logs_rotate(2)
+
+ logs = database.logs_get(package_ahriman.base)
+ assert len(logs) == 2
+ assert logs == [
+ (LogRecordId(package_ahriman.base, "1", "p3"), 44.0, "message 3"),
+ (LogRecordId(package_ahriman.base, "2", "p1"), 45.0, "message 4"),
+ ]
\ No newline at end of file
diff --git a/tests/ahriman/core/log/test_http_log_handler.py b/tests/ahriman/core/log/test_http_log_handler.py
index 40395c30..224f5881 100644
--- a/tests/ahriman/core/log/test_http_log_handler.py
+++ b/tests/ahriman/core/log/test_http_log_handler.py
@@ -19,12 +19,14 @@ def test_load(configuration: Configuration, mocker: MockerFixture) -> None:
add_mock = mocker.patch("logging.Logger.addHandler")
load_mock = mocker.patch("ahriman.core.status.Client.load")
+ atexit_mock = mocker.patch("atexit.register")
_, repository_id = configuration.check_loaded()
handler = HttpLogHandler.load(repository_id, configuration, report=False)
assert handler
add_mock.assert_called_once_with(handler)
load_mock.assert_called_once_with(repository_id, configuration, report=False)
+ atexit_mock.assert_called_once_with(handler.rotate)
def test_load_exist(configuration: Configuration) -> None:
@@ -93,3 +95,16 @@ def test_emit_skip(configuration: Configuration, log_record: logging.LogRecord,
handler.emit(log_record)
log_mock.assert_not_called()
+
+
+def test_rotate(configuration: Configuration, mocker: MockerFixture) -> None:
+ """
+ must rotate logs
+ """
+ rotate_mock = mocker.patch("ahriman.core.status.Client.logs_rotate")
+
+ _, repository_id = configuration.check_loaded()
+ handler = HttpLogHandler(repository_id, configuration, report=False, suppress_errors=False)
+
+ handler.rotate()
+ rotate_mock.assert_called_once_with(handler.keep_last_records)
diff --git a/tests/ahriman/core/log/test_lazy_logging.py b/tests/ahriman/core/log/test_lazy_logging.py
index 9e0ce317..a3f416e2 100644
--- a/tests/ahriman/core/log/test_lazy_logging.py
+++ b/tests/ahriman/core/log/test_lazy_logging.py
@@ -87,13 +87,3 @@ def test_in_package_context_failed(database: SQLite, package_ahriman: Package, m
raise ValueError()
reset_mock.assert_called_once_with()
-
-
-def test_suppress_logging(database: SQLite, mocker: MockerFixture) -> None:
- """
- must temporary disable log messages
- """
- disable_mock = mocker.patch("ahriman.core.log.lazy_logging.logging.disable")
- with database.suppress_logging():
- pass
- disable_mock.assert_has_calls([MockCall(logging.WARNING), MockCall(logging.NOTSET)])
diff --git a/tests/ahriman/core/status/test_client.py b/tests/ahriman/core/status/test_client.py
index b152f621..ff2f0589 100644
--- a/tests/ahriman/core/status/test_client.py
+++ b/tests/ahriman/core/status/test_client.py
@@ -112,6 +112,13 @@ def test_event_get(client: Client) -> None:
client.event_get(None, None)
+def test_logs_rotate(client: Client, package_ahriman: Package) -> None:
+ """
+ must do not raise exception on logs rotation call
+ """
+ client.logs_rotate(1)
+
+
def test_package_changes_get(client: Client, package_ahriman: Package) -> None:
"""
must raise not implemented on package changes request
diff --git a/tests/ahriman/core/status/test_local_client.py b/tests/ahriman/core/status/test_local_client.py
index 255983a7..082126a8 100644
--- a/tests/ahriman/core/status/test_local_client.py
+++ b/tests/ahriman/core/status/test_local_client.py
@@ -34,6 +34,15 @@ def test_event_get(local_client: LocalClient, package_ahriman: Package, mocker:
local_client.repository_id)
+def test_logs_rotate(local_client: LocalClient, package_ahriman: Package, mocker: MockerFixture) -> None:
+ """
+ must rotate logs
+ """
+ rotate_mock = mocker.patch("ahriman.core.database.SQLite.logs_rotate")
+ local_client.logs_rotate(42)
+ rotate_mock.assert_called_once_with(42, local_client.repository_id)
+
+
def test_package_changes_get(local_client: LocalClient, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must retrieve package changes
diff --git a/tests/ahriman/core/status/test_watcher.py b/tests/ahriman/core/status/test_watcher.py
index 620e918f..ff99d3da 100644
--- a/tests/ahriman/core/status/test_watcher.py
+++ b/tests/ahriman/core/status/test_watcher.py
@@ -5,7 +5,6 @@ from pytest_mock import MockerFixture
from ahriman.core.exceptions import UnknownPackageError
from ahriman.core.status.watcher import Watcher
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
-from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
@@ -64,38 +63,6 @@ def test_package_get_failed(watcher: Watcher, package_ahriman: Package) -> None:
watcher.package_get(package_ahriman.base)
-def test_package_logs_add_new(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
- """
- must create package logs record for new package
- """
- delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.package_logs_remove", create=True)
- insert_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_logs_add")
-
- log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.version)
- assert watcher._last_log_record_id != log_record_id
-
- watcher.package_logs_add(log_record_id, 42.01, "log record")
- delete_mock.assert_called_once_with(package_ahriman.base, log_record_id.version)
- insert_mock.assert_called_once_with(log_record_id, 42.01, "log record")
-
- assert watcher._last_log_record_id == log_record_id
-
-
-def test_package_logs_add_update(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
- """
- must create package logs record for current package
- """
- delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.package_logs_remove", create=True)
- insert_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_logs_add")
-
- log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.version)
- watcher._last_log_record_id = log_record_id
-
- watcher.package_logs_add(log_record_id, 42.01, "log record")
- delete_mock.assert_not_called()
- insert_mock.assert_called_once_with(log_record_id, 42.01, "log record")
-
-
def test_package_remove(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must remove package base
diff --git a/tests/ahriman/core/status/test_web_client.py b/tests/ahriman/core/status/test_web_client.py
index c6ec36ba..05ce6516 100644
--- a/tests/ahriman/core/status/test_web_client.py
+++ b/tests/ahriman/core/status/test_web_client.py
@@ -257,6 +257,57 @@ def test_event_get_failed_http_error_suppress(web_client: WebClient, mocker: Moc
logging_mock.assert_not_called()
+def test_logs_rotate(web_client: WebClient, mocker: MockerFixture) -> None:
+ """
+ must rotate logs
+ """
+ requests_mock = mocker.patch("ahriman.core.status.web_client.WebClient.make_request")
+
+ web_client.logs_rotate(42)
+ requests_mock.assert_called_once_with("DELETE", pytest.helpers.anyvar(str, True),
+ params=web_client.repository_id.query() + [("keep_last_records", "42")])
+
+
+def test_logs_rotate_failed(web_client: WebClient, mocker: MockerFixture) -> None:
+ """
+ must suppress any exception happened during logs rotation
+ """
+ mocker.patch("requests.Session.request", side_effect=Exception())
+ web_client.logs_rotate(42)
+
+
+def test_logs_rotate_failed_http_error(web_client: WebClient, mocker: MockerFixture) -> None:
+ """
+ must suppress HTTP exception happened during logs rotation
+ """
+ mocker.patch("requests.Session.request", side_effect=requests.HTTPError())
+ web_client.logs_rotate(42)
+
+
+def test_logs_rotate_failed_suppress(web_client: WebClient, mocker: MockerFixture) -> None:
+ """
+ must suppress any exception happened during logs rotation and don't log
+ """
+ web_client.suppress_errors = True
+ mocker.patch("requests.Session.request", side_effect=Exception())
+ logging_mock = mocker.patch("logging.exception")
+
+ web_client.logs_rotate(42)
+ logging_mock.assert_not_called()
+
+
+def test_logs_rotate_failed_http_error_suppress(web_client: WebClient, mocker: MockerFixture) -> None:
+ """
+ must suppress HTTP exception happened during logs rotation and don't log
+ """
+ web_client.suppress_errors = True
+ mocker.patch("requests.Session.request", side_effect=requests.HTTPError())
+ logging_mock = mocker.patch("logging.exception")
+
+ web_client.logs_rotate(42)
+ logging_mock.assert_not_called()
+
+
def test_package_changes_get(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must get changes
@@ -551,6 +602,7 @@ def test_package_logs_add(web_client: WebClient, log_record: logging.LogRecord,
payload = {
"created": log_record.created,
"message": log_record.getMessage(),
+ "process_id": LogRecordId.process_id,
"version": package_ahriman.version,
}
@@ -588,7 +640,12 @@ def test_package_logs_get(web_client: WebClient, package_ahriman: Package, mocke
"""
must get logs
"""
- message = {"created": 42.0, "message": "log"}
+ message = {
+ "created": 42.0,
+ "message": "log",
+ "version": package_ahriman.version,
+ "process_id": LogRecordId.process_id,
+ }
response_obj = requests.Response()
response_obj._content = json.dumps([message]).encode("utf8")
response_obj.status_code = 200
@@ -598,7 +655,9 @@ def test_package_logs_get(web_client: WebClient, package_ahriman: Package, mocke
result = web_client.package_logs_get(package_ahriman.base, 1, 2)
requests_mock.assert_called_once_with("GET", pytest.helpers.anyvar(str, True),
params=web_client.repository_id.query() + [("limit", "1"), ("offset", "2")])
- assert result == [(message["created"], message["message"])]
+ assert result == [
+ (LogRecordId(package_ahriman.base, package_ahriman.version), message["created"], message["message"]),
+ ]
def test_package_logs_get_failed(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None:
diff --git a/tests/ahriman/web/schemas/test_versioned_log_schema.py b/tests/ahriman/web/schemas/test_logs_rotate_schema.py
similarity index 100%
rename from tests/ahriman/web/schemas/test_versioned_log_schema.py
rename to tests/ahriman/web/schemas/test_logs_rotate_schema.py