refine package logging

package/share/ahriman/settings/ahriman.ini

@@ -7,6 +7,8 @@ logging = ahriman.ini.d/logging.ini
 ;apply_migrations = yes
 ; Path to the application SQLite database.
 database = ${repository:root}/ahriman.db
+; Keep last build logs for each package
+keep_last_logs = 5
 
 [alpm]
 ; Path to pacman system database cache.

src/ahriman/core/configuration/schema.py

@@ -45,6 +45,11 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
                 "path_exists": True,
                 "path_type": "dir",
             },
+            "keep_last_logs": {
+                "type": "integer",
+                "coerce": "integer",
+                "min": 0,
+            },
             "logging": {
                 "type": "path",
                 "coerce": "absolute_path",

src/ahriman/core/database/migrations/m015_logs_process_id.py

@@ -0,0 +1,27 @@
+#
+# Copyright (c) 2021-2025 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+__all__ = ["steps"]
+
+
+steps = [
+    """
+    alter table logs add column process_id text not null default ''
+    """,
+]

src/ahriman/core/database/operations/logs_operations.py

@@ -30,7 +30,7 @@ class LogsOperations(Operations):
     """
 
     def logs_get(self, package_base: str, limit: int = -1, offset: int = 0,
-                 repository_id: RepositoryId | None = None) -> list[tuple[float, str]]:
+                 repository_id: RepositoryId | None = None) -> list[tuple[LogRecordId, float, str]]:
         """
         extract logs for specified package base
 
@@ -41,16 +41,16 @@ class LogsOperations(Operations):
             repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
 
         Return:
-            list[tuple[float, str]]: sorted package log records and their timestamps
+            list[tuple[LogRecordId, float, str]]: sorted package log records and their timestamps
         """
         repository_id = repository_id or self._repository_id
 
-        def run(connection: Connection) -> list[tuple[float, str]]:
+        def run(connection: Connection) -> list[tuple[LogRecordId, float, str]]:
             return [
-                (row["created"], row["record"])
+                (LogRecordId(package_base, row["version"], row["process_id"]), row["created"], row["record"])
                 for row in connection.execute(
                     """
-                    select created, record from (
+                    select created, record, version, process_id from (
                         select * from logs
                         where package_base = :package_base and repository = :repository
                         order by created desc limit :limit offset :offset
@@ -83,9 +83,9 @@ class LogsOperations(Operations):
             connection.execute(
                 """
                 insert into logs
-                (package_base, version, created, record, repository)
+                (package_base, version, created, record, repository, process_id)
                 values
-                (:package_base, :version, :created, :record, :repository)
+                (:package_base, :version, :created, :record, :repository, :process_id)
                 """,
                 {
                     "package_base": log_record_id.package_base,
@@ -93,6 +93,7 @@ class LogsOperations(Operations):
                     "created": created,
                     "record": record,
                     "repository": repository_id.id,
+                    "process_id": log_record_id.process_id,
                 }
             )
 
@@ -125,3 +126,54 @@ class LogsOperations(Operations):
             )
 
         return self.with_connection(run, commit=True)
+
+    def logs_rotate(self, keep_last_records: int, repository_id: RepositoryId | None = None) -> None:
+        """
+        rotate logs in storage. This method will remove old logs, keeping only the last N records for each package
+
+        Args:
+            keep_last_records(int): number of last records to keep
+            repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
+        """
+        repository_id = repository_id or self._repository_id
+
+        def remove_duplicates(connection: Connection) -> None:
+            connection.execute(
+                """
+                delete from logs
+                where (package_base, version, repository, process_id) not in (
+                  select package_base, version, repository, process_id from logs
+                  where (package_base, version, repository, created) in (
+                    select package_base, version, repository, max(created) from logs
+                    where repository = :repository
+                    group by package_base, version, repository
+                  )
+                )
+                """,
+                {
+                    "repository": repository_id.id,
+                }
+            )
+
+        def remove_older(connection: Connection) -> None:
+            connection.execute(
+                """
+                delete from logs
+                where (package_base, repository, process_id) in (
+                  select package_base, repository, process_id from logs
+                  where repository = :repository
+                  group by package_base, repository, process_id
+                  order by min(created) desc limit -1 offset :offset
+                )
+                """,
+                {
+                    "offset": keep_last_records,
+                    "repository": repository_id.id,
+                }
+            )
+
+        def run(connection: Connection) -> None:
+            remove_duplicates(connection)
+            remove_older(connection)
+
+        return self.with_connection(run, commit=True)

src/ahriman/core/log/http_log_handler.py

@@ -17,6 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import atexit
 import logging
 
 from typing import Self
@@ -33,6 +34,7 @@ class HttpLogHandler(logging.Handler):
     method
 
     Attributes:
+        keep_last_records(int): number of last records to keep
         reporter(Client): build status reporter instance
         suppress_errors(bool): suppress logging errors (e.g. if no web server available)
     """
@@ -51,6 +53,7 @@ class HttpLogHandler(logging.Handler):
 
         self.reporter = Client.load(repository_id, configuration, report=report)
         self.suppress_errors = suppress_errors
+        self.keep_last_records = configuration.getint("settings", "keep_last_logs", fallback=0)
 
     @classmethod
     def load(cls, repository_id: RepositoryId, configuration: Configuration, *, report: bool) -> Self:
@@ -76,6 +79,8 @@ class HttpLogHandler(logging.Handler):
         handler = cls(repository_id, configuration, report=report, suppress_errors=suppress_errors)
         root.addHandler(handler)
 
+        atexit.register(handler.rotate)
+
         return handler
 
     def emit(self, record: logging.LogRecord) -> None:
@@ -95,3 +100,9 @@ class HttpLogHandler(logging.Handler):
             if self.suppress_errors:
                 return
             self.handleError(record)
+
+    def rotate(self) -> None:
+        """
+        rotate log records, removing older ones
+        """
+        self.reporter.logs_rotate(self.keep_last_records)

src/ahriman/core/log/lazy_logging.py

@@ -99,24 +99,3 @@ class LazyLogging:
             yield
         finally:
             self._package_logger_reset()
-
-    @contextlib.contextmanager
-    def suppress_logging(self, log_level: int = logging.WARNING) -> Generator[None, None, None]:
-        """
-        silence log messages in context
-
-        Args:
-            log_level(int, optional): the highest log level to keep (Default value = logging.WARNING)
-
-        Examples:
-             This function is designed to be used to suppress all log messages in context, e.g.:
-
-                 >>> with self.suppress_logging():
-                 >>>     do_some_noisy_actions()
-        """
-        current_level = self.logger.manager.disable
-        try:
-            logging.disable(log_level)
-            yield
-        finally:
-            logging.disable(current_level)

src/ahriman/core/repository/executor.py

@@ -75,7 +75,7 @@ class Executor(PackageInfo, Cleaner):
 
         result = Result()
         for single in updates:
-            with self.in_package_context(single.base, local_versions.get(single.base)), \
+            with self.in_package_context(single.base, single.version), \
                     TemporaryDirectory(ignore_cleanup_errors=True) as dir_name:
                 try:
                     with self.in_event(single.base, EventType.PackageUpdated, failure=EventType.PackageUpdateFailed):
@@ -194,7 +194,6 @@ class Executor(PackageInfo, Cleaner):
             self.repo.add(package_path)
 
         current_packages = {package.base: package for package in self.packages()}
-        local_versions = {package_base: package.version for package_base, package in current_packages.items()}
 
         removed_packages: list[str] = []  # list of packages which have been removed from the base
         updates = self.load_archives(packages)
@@ -202,7 +201,7 @@ class Executor(PackageInfo, Cleaner):
 
         result = Result()
         for local in updates:
-            with self.in_package_context(local.base, local_versions.get(local.base)):
+            with self.in_package_context(local.base, local.version):
                 try:
                     packager = self.packager(packagers, local.base)
 

src/ahriman/core/repository/update_handler.py

@@ -144,7 +144,6 @@ class UpdateHandler(PackageInfo, Cleaner):
                         branch="master",
                     )
 
-                    with self.suppress_logging():
                     Sources.fetch(cache_dir, source)
                     remote = Package.from_build(cache_dir, self.architecture, None)
 

src/ahriman/core/status/client.py

@@ -115,6 +115,14 @@ class Client:
         """
         raise NotImplementedError
 
+    def logs_rotate(self, keep_last_records: int) -> None:
+        """
+        remove older logs from storage
+
+        Args:
+            keep_last_records(int): number of last records to keep
+        """
+
     def package_changes_get(self, package_base: str) -> Changes:
         """
         get package changes
@@ -197,7 +205,8 @@ class Client:
         """
         # this method does not raise NotImplementedError because it is actively used as dummy client for http log
 
-    def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]:
+    def package_logs_get(self, package_base: str, limit: int = -1,
+                         offset: int = 0) -> list[tuple[LogRecordId, float, str]]:
         """
         get package logs
 
@@ -207,7 +216,7 @@ class Client:
             offset(int, optional): records offset (Default value = 0)
 
         Returns:
-            list[tuple[float, str]]: package logs
+            list[tuple[LogRecordId, float, str]]: package logs
 
         Raises:
             NotImplementedError: not implemented method

src/ahriman/core/status/local_client.py

@@ -75,6 +75,15 @@ class LocalClient(Client):
         """
         return self.database.event_get(event, object_id, from_date, to_date, limit, offset, self.repository_id)
 
+    def logs_rotate(self, keep_last_records: int) -> None:
+        """
+        remove older logs from storage
+
+        Args:
+            keep_last_records(int): number of last records to keep
+        """
+        self.database.logs_rotate(keep_last_records, self.repository_id)
+
     def package_changes_get(self, package_base: str) -> Changes:
         """
         get package changes
@@ -145,7 +154,8 @@ class LocalClient(Client):
         """
         self.database.logs_insert(log_record_id, created, message, self.repository_id)
 
-    def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]:
+    def package_logs_get(self, package_base: str, limit: int = -1,
+                         offset: int = 0) -> list[tuple[LogRecordId, float, str]]:
         """
         get package logs
 
@@ -155,7 +165,7 @@ class LocalClient(Client):
             offset(int, optional): records offset (Default value = 0)
 
         Returns:
-            list[tuple[float, str]]: package logs
+            list[tuple[LogRecordId, float, str]]: package logs
         """
         return self.database.logs_get(package_base, limit, offset, self.repository_id)
 

src/ahriman/core/status/watcher.py

@@ -53,9 +53,6 @@ class Watcher(LazyLogging):
         self._known: dict[str, tuple[Package, BuildStatus]] = {}
         self.status = BuildStatus()
 
-        # special variables for updating logs
-        self._last_log_record_id = LogRecordId("", "")
-
     @property
     def packages(self) -> list[tuple[Package, BuildStatus]]:
         """
@@ -81,6 +78,8 @@ class Watcher(LazyLogging):
                 for package, status in self.client.package_get(None)
             }
 
+    logs_rotate: Callable[[int], None]
+
     package_changes_get: Callable[[str], Changes]
 
     package_changes_update: Callable[[str, Changes], None]
@@ -108,22 +107,9 @@ class Watcher(LazyLogging):
         except KeyError:
             raise UnknownPackageError(package_base) from None
 
-    def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None:
-        """
-        make new log record into database
+    package_logs_add: Callable[[LogRecordId, float, str], None]
 
-        Args:
-            log_record_id(LogRecordId): log record id
-            created(float): log created timestamp
-            message(str): log message
-        """
-        if self._last_log_record_id != log_record_id:
-            # there is new log record, so we remove old ones
-            self.package_logs_remove(log_record_id.package_base, log_record_id.version)
-        self._last_log_record_id = log_record_id
-        self.client.package_logs_add(log_record_id, created, message)
-
-    package_logs_get: Callable[[str, int, int], list[tuple[float, str]]]
+    package_logs_get: Callable[[str, int, int], list[tuple[LogRecordId, float, str]]]
 
     package_logs_remove: Callable[[str, str | None], None]
 

src/ahriman/core/status/web_client.py

@@ -210,6 +210,18 @@ class WebClient(Client, SyncAhrimanClient):
 
         return []
 
+    def logs_rotate(self, keep_last_records: int) -> None:
+        """
+        remove older logs from storage
+
+        Args:
+            keep_last_records(int): number of last records to keep
+        """
+        query = self.repository_id.query() + [("keep_last_records", str(keep_last_records))]
+
+        with contextlib.suppress(Exception):
+            self.make_request("DELETE", f"{self.address}/api/v1/service/logs", params=query)
+
     def package_changes_get(self, package_base: str) -> Changes:
         """
         get package changes
@@ -306,6 +318,7 @@ class WebClient(Client, SyncAhrimanClient):
         payload = {
             "created": created,
             "message": message,
+            "process_id": log_record_id.process_id,
             "version": log_record_id.version,
         }
 
@@ -315,7 +328,8 @@ class WebClient(Client, SyncAhrimanClient):
         self.make_request("POST", self._logs_url(log_record_id.package_base),
                           params=self.repository_id.query(), json=payload, suppress_errors=True)
 
-    def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]:
+    def package_logs_get(self, package_base: str, limit: int = -1,
+                         offset: int = 0) -> list[tuple[LogRecordId, float, str]]:
         """
         get package logs
 
@@ -325,7 +339,7 @@ class WebClient(Client, SyncAhrimanClient):
             offset(int, optional): records offset (Default value = 0)
 
         Returns:
-            list[tuple[float, str]]: package logs
+            list[tuple[LogRecordId, float, str]]: package logs
         """
         query = self.repository_id.query() + [("limit", str(limit)), ("offset", str(offset))]
 
@@ -333,7 +347,13 @@ class WebClient(Client, SyncAhrimanClient):
             response = self.make_request("GET", self._logs_url(package_base), params=query)
             response_json = response.json()
 
-            return [(record["created"], record["message"]) for record in response_json]
+            return [
+                (
+                    LogRecordId(package_base, record["version"], record["process_id"]),
+                    record["created"],
+                    record["message"]
+                ) for record in response_json
+            ]
 
         return []
 

src/ahriman/models/log_record_id.py

@@ -17,7 +17,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from dataclasses import dataclass
+import uuid
+
+from dataclasses import dataclass, field
 
 
 @dataclass(frozen=True)
@@ -28,7 +30,12 @@ class LogRecordId:
     Attributes:
         package_base(str): package base for which log record belongs
         version(str): package version for which log record belongs
+        process_id(str, optional): unique process identifier
     """
 
     package_base: str
     version: str
+
+    # this is not mistake, this value is kind of global identifier, which is generated
+    # upon the process start
+    process_id: str = field(default=str(uuid.uuid4()))

src/ahriman/models/package.py

@@ -429,7 +429,6 @@ class Package(LazyLogging):
         task = Task(self, configuration, repository_id.architecture, paths)
 
         try:
-            with self.suppress_logging():
             # create fresh chroot environment, fetch sources and - automagically - update PKGBUILD
             task.init(paths.cache_for(self.base), [], None)
             pkgbuild = Pkgbuild.from_file(paths.cache_for(self.base) / "PKGBUILD")

src/ahriman/web/apispec/decorators.py

@@ -18,7 +18,8 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from aiohttp.web import HTTPException
-from typing import Any, Callable
+from collections.abc import Callable
+from typing import Any
 
 from ahriman.models.user_access import UserAccess
 from ahriman.web.apispec import Schema, aiohttp_apispec

src/ahriman/web/schemas/__init__.py

@@ -31,6 +31,7 @@ from ahriman.web.schemas.info_schema import InfoSchema
 from ahriman.web.schemas.internal_status_schema import InternalStatusSchema
 from ahriman.web.schemas.log_schema import LogSchema
 from ahriman.web.schemas.login_schema import LoginSchema
+from ahriman.web.schemas.logs_rotate_schema import LogsRotateSchema
 from ahriman.web.schemas.logs_schema import LogsSchema
 from ahriman.web.schemas.oauth2_schema import OAuth2Schema
 from ahriman.web.schemas.package_name_schema import PackageNameSchema
@@ -53,5 +54,4 @@ from ahriman.web.schemas.repository_stats_schema import RepositoryStatsSchema
 from ahriman.web.schemas.search_schema import SearchSchema
 from ahriman.web.schemas.status_schema import StatusSchema
 from ahriman.web.schemas.update_flags_schema import UpdateFlagsSchema
-from ahriman.web.schemas.versioned_log_schema import VersionedLogSchema
 from ahriman.web.schemas.worker_schema import WorkerSchema

src/ahriman/web/schemas/log_schema.py

@@ -17,12 +17,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+from ahriman import __version__
 from ahriman.web.apispec import Schema, fields
 
 
 class LogSchema(Schema):
     """
-    request package log schema
+    request and response package log schema
     """
 
     created = fields.Float(required=True, metadata={
@@ -32,3 +33,10 @@ class LogSchema(Schema):
     message = fields.String(required=True, metadata={
         "description": "Log message",
     })
+    version = fields.String(required=True, metadata={
+        "description": "Package version to tag",
+        "example": __version__,
+    })
+    process_id = fields.String(metadata={
+        "description": "Process unique identifier",
+    })

src/ahriman/web/schemas/logs_rotate_schema.py

@@ -17,18 +17,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from ahriman import __version__
-from ahriman.web.apispec import fields
-from ahriman.web.schemas.log_schema import LogSchema
-from ahriman.web.schemas.repository_id_schema import RepositoryIdSchema
+from ahriman.web.apispec import Schema, fields
 
 
-class VersionedLogSchema(LogSchema, RepositoryIdSchema):
+class LogsRotateSchema(Schema):
     """
-    request package log schema
+    request logs rotate schema
     """
 
-    version = fields.Integer(required=True, metadata={
-        "description": "Package version to tag",
-        "example": __version__,
+    keep_last_records = fields.Integer(metadata={
+        "description": "Keep the specified amount of records",
     })

src/ahriman/web/views/v1/packages/logs.py

@@ -24,8 +24,7 @@ from ahriman.core.utils import pretty_datetime
 from ahriman.models.log_record_id import LogRecordId
 from ahriman.models.user_access import UserAccess
 from ahriman.web.apispec.decorators import apidocs
-from ahriman.web.schemas import LogsSchema, PackageNameSchema, PackageVersionSchema, RepositoryIdSchema, \
-    VersionedLogSchema
+from ahriman.web.schemas import LogSchema, LogsSchema, PackageNameSchema, PackageVersionSchema, RepositoryIdSchema
 from ahriman.web.views.base import BaseView
 from ahriman.web.views.status_view_guard import StatusViewGuard
 
@@ -97,7 +96,7 @@ class LogsView(StatusViewGuard, BaseView):
         response = {
             "package_base": package_base,
             "status": status.view(),
-            "logs": "\n".join(f"[{pretty_datetime(created)}] {message}" for created, message in logs)
+            "logs": "\n".join(f"[{pretty_datetime(created)}] {message}" for _, created, message in logs)
         }
         return json_response(response)
 
@@ -109,7 +108,7 @@ class LogsView(StatusViewGuard, BaseView):
         error_400_enabled=True,
         error_404_description="Repository is unknown",
         match_schema=PackageNameSchema,
-        body_schema=VersionedLogSchema,
+        body_schema=LogSchema,
     )
     async def post(self) -> None:
         """
@@ -129,6 +128,8 @@ class LogsView(StatusViewGuard, BaseView):
         except Exception as ex:
             raise HTTPBadRequest(reason=str(ex))
 
-        self.service().package_logs_add(LogRecordId(package_base, version), created, record)
+        # either read from process identifier from payload or assign to the current process identifier
+        process_id = data.get("process_id", LogRecordId("", "").process_id)
+        self.service().package_logs_add(LogRecordId(package_base, version, process_id), created, record)
 
         raise HTTPNoContent

src/ahriman/web/views/v1/service/logs.py

@@ -0,0 +1,63 @@
+#
+# Copyright (c) 2021-2025 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from aiohttp.web import HTTPBadRequest, HTTPNoContent
+
+from ahriman.models.user_access import UserAccess
+from ahriman.web.apispec.decorators import apidocs
+from ahriman.web.schemas import LogsRotateSchema
+from ahriman.web.views.base import BaseView
+
+
+class LogsView(BaseView):
+    """
+    logs management web view
+
+    Attributes:
+        DELETE_PERMISSION(UserAccess): (class attribute) delete permissions of self
+    """
+
+    DELETE_PERMISSION = UserAccess.Full
+    ROUTES = ["/api/v1/service/logs"]
+
+    @apidocs(
+        tags=["Actions"],
+        summary="Rotate logs",
+        description="Remove older logs from system",
+        permission=DELETE_PERMISSION,
+        error_400_enabled=True,
+        error_404_description="Repository is unknown",
+        query_schema=LogsRotateSchema,
+    )
+    async def delete(self) -> None:
+        """
+        rotate logs from system
+
+        Raises:
+            HTTPBadRequest: if bad data is supplied
+            HTTPNoContent: on success response
+        """
+        try:
+            keep_last_records = int(self.request.query.get("keep_last_records", 0))
+        except Exception as ex:
+            raise HTTPBadRequest(reason=str(ex))
+
+        self.service().logs_rotate(keep_last_records)
+
+        raise HTTPNoContent

src/ahriman/web/views/v2/packages/logs.py

@@ -67,6 +67,8 @@ class LogsView(StatusViewGuard, BaseView):
             {
                 "created": created,
                 "message": message,
-            } for created, message in logs
+                "version": log_record_id.version,
+                "process_id": log_record_id.process_id,
+            } for log_record_id, created, message in logs
         ]
         return json_response(response)