This commit is contained in:
Evgenii Alekseev 2025-03-07 02:24:32 +02:00 committed by GitHub
commit f13c25e50c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
36 changed files with 650 additions and 199 deletions

View File

@ -7,6 +7,8 @@ logging = ahriman.ini.d/logging.ini
;apply_migrations = yes ;apply_migrations = yes
; Path to the application SQLite database. ; Path to the application SQLite database.
database = ${repository:root}/ahriman.db database = ${repository:root}/ahriman.db
; Keep last build logs for each package
keep_last_logs = 5
[alpm] [alpm]
; Path to pacman system database cache. ; Path to pacman system database cache.

View File

@ -16,11 +16,11 @@
<div class="container"> <div class="container">
<nav class="navbar navbar-expand-lg"> <nav class="navbar navbar-expand-lg">
<div class="navbar-brand"><a href="https://github.com/arcan1s/ahriman" title="logo"><img src="/static/logo.svg" width="30" height="30" alt=""></a></div> <div class="navbar-brand"><a href="https://github.com/arcan1s/ahriman" title="logo"><img src="/static/logo.svg" width="30" height="30" alt=""></a></div>
<button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#repositories-navbar-supported-content" aria-controls="repositories-navbar-supported-content" aria-expanded="false" aria-label="Toggle navigation"> <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#repositories-navbar" aria-controls="repositories-navbar" aria-expanded="false" aria-label="Toggle navigation">
<span class="navbar-toggler-icon"></span> <span class="navbar-toggler-icon"></span>
</button> </button>
<div id="repositories-navbar-supported-content" class="collapse navbar-collapse"> <div id="repositories-navbar" class="collapse navbar-collapse">
<ul id="repositories" class="nav nav-tabs"> <ul id="repositories" class="nav nav-tabs">
{% for repository in repositories %} {% for repository in repositories %}
<li class="nav-item"> <li class="nav-item">

View File

@ -59,8 +59,15 @@
</nav> </nav>
<div class="tab-content" id="nav-tabContent"> <div class="tab-content" id="nav-tabContent">
<div id="package-info-logs" class="tab-pane fade show active" role="tabpanel" aria-labelledby="package-info-logs-button" tabindex="0"> <div id="package-info-logs" class="tab-pane fade show active" role="tabpanel" aria-labelledby="package-info-logs-button" tabindex="0">
<div class="row">
<div class="col-2">
<nav id="package-info-logs-versions" class="nav flex-column"></nav>
</div>
<div class="col-10">
<pre class="language-console"><code id="package-info-logs-input" class="pre-scrollable language-console"></code><button id="package-info-logs-copy-button" type="button" class="btn language-console" onclick="copyLogs()"><i class="bi bi-clipboard"></i> copy</button></pre> <pre class="language-console"><code id="package-info-logs-input" class="pre-scrollable language-console"></code><button id="package-info-logs-copy-button" type="button" class="btn language-console" onclick="copyLogs()"><i class="bi bi-clipboard"></i> copy</button></pre>
</div> </div>
</div>
</div>
<div id="package-info-changes" class="tab-pane fade" role="tabpanel" aria-labelledby="package-info-changes-button" tabindex="0"> <div id="package-info-changes" class="tab-pane fade" role="tabpanel" aria-labelledby="package-info-changes-button" tabindex="0">
<pre class="language-diff"><code id="package-info-changes-input" class="pre-scrollable language-diff"></code><button id="package-info-changes-copy-button" type="button" class="btn language-diff" onclick="copyChanges()"><i class="bi bi-clipboard"></i> copy</button></pre> <pre class="language-diff"><code id="package-info-changes-input" class="pre-scrollable language-diff"></code><button id="package-info-changes-copy-button" type="button" class="btn language-diff" onclick="copyChanges()"><i class="bi bi-clipboard"></i> copy</button></pre>
</div> </div>
@ -100,6 +107,7 @@
const packageInfoModalHeader = document.getElementById("package-info-modal-header"); const packageInfoModalHeader = document.getElementById("package-info-modal-header");
const packageInfo = document.getElementById("package-info"); const packageInfo = document.getElementById("package-info");
const packageInfoLogsVersions = document.getElementById("package-info-logs-versions");
const packageInfoLogsInput = document.getElementById("package-info-logs-input"); const packageInfoLogsInput = document.getElementById("package-info-logs-input");
const packageInfoLogsCopyButton = document.getElementById("package-info-logs-copy-button"); const packageInfoLogsCopyButton = document.getElementById("package-info-logs-copy-button");
@ -285,11 +293,45 @@
convert: response => response.json(), convert: response => response.json(),
}, },
data => { data => {
const logs = data.map(log_record => { const selectors = Object
return `[${new Date(1000 * log_record.created).toISOString()}] ${log_record.message}`; .values(
}); data.reduce((acc, log_record) => {
const id = `${log_record.version}-${log_record.process_id}`;
if (acc[id])
acc[id].created = Math.min(log_record.created, acc[id].created);
else
acc[id] = log_record;
return acc;
}, {})
)
.sort(({created: left}, {created: right}) =>
right - left
)
.map(version => {
const link = document.createElement("a");
link.classList.add("nav-link");
link.textContent = version.version;
link.href = "#";
link.onclick = _ => {
const logs = data
.filter(log_record => log_record.version === version.version && log_record.process_id === version.process_id)
.map(log_record => `[${new Date(1000 * log_record.created).toISOString()}] ${log_record.message}`);
packageInfoLogsInput.textContent = logs.join("\n"); packageInfoLogsInput.textContent = logs.join("\n");
highlight(packageInfoLogsInput); highlight(packageInfoLogsInput);
Array.from(packageInfoLogsVersions.children).forEach(el => el.classList.remove("active"));
link.classList.add("active");
return false;
};
return link;
});
packageInfoLogsVersions.replaceChildren(...selectors);
selectors.find(Boolean)?.click();
}, },
onFailure, onFailure,
); );

View File

@ -27,4 +27,10 @@
top: 0; top: 0;
right: 5px; right: 5px;
} }
.nav-link.active {
pointer-events: none;
cursor: default;
color: black !important;
}
</style> </style>

View File

@ -45,6 +45,11 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"path_exists": True, "path_exists": True,
"path_type": "dir", "path_type": "dir",
}, },
"keep_last_logs": {
"type": "integer",
"coerce": "integer",
"min": 0,
},
"logging": { "logging": {
"type": "path", "type": "path",
"coerce": "absolute_path", "coerce": "absolute_path",

View File

@ -0,0 +1,27 @@
#
# Copyright (c) 2021-2025 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ["steps"]
steps = [
"""
alter table logs add column process_id text not null default ''
""",
]

View File

@ -20,6 +20,7 @@
from sqlite3 import Connection from sqlite3 import Connection
from ahriman.core.database.operations.operations import Operations from ahriman.core.database.operations.operations import Operations
from ahriman.models.log_record import LogRecord
from ahriman.models.log_record_id import LogRecordId from ahriman.models.log_record_id import LogRecordId
from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_id import RepositoryId
@ -30,7 +31,7 @@ class LogsOperations(Operations):
""" """
def logs_get(self, package_base: str, limit: int = -1, offset: int = 0, def logs_get(self, package_base: str, limit: int = -1, offset: int = 0,
repository_id: RepositoryId | None = None) -> list[tuple[float, str]]: repository_id: RepositoryId | None = None) -> list[LogRecord]:
""" """
extract logs for specified package base extract logs for specified package base
@ -41,16 +42,16 @@ class LogsOperations(Operations):
repository_id(RepositoryId, optional): repository unique identifier override (Default value = None) repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
Return: Return:
list[tuple[float, str]]: sorted package log records and their timestamps list[LogRecord]: sorted package log records
""" """
repository_id = repository_id or self._repository_id repository_id = repository_id or self._repository_id
def run(connection: Connection) -> list[tuple[float, str]]: def run(connection: Connection) -> list[LogRecord]:
return [ return [
(row["created"], row["record"]) LogRecord(LogRecordId(package_base, row["version"], row["process_id"]), row["created"], row["record"])
for row in connection.execute( for row in connection.execute(
""" """
select created, record from ( select created, record, version, process_id from (
select * from logs select * from logs
where package_base = :package_base and repository = :repository where package_base = :package_base and repository = :repository
order by created desc limit :limit offset :offset order by created desc limit :limit offset :offset
@ -66,15 +67,12 @@ class LogsOperations(Operations):
return self.with_connection(run) return self.with_connection(run)
def logs_insert(self, log_record_id: LogRecordId, created: float, record: str, def logs_insert(self, log_record: LogRecord, repository_id: RepositoryId | None = None) -> None:
repository_id: RepositoryId | None = None) -> None:
""" """
write new log record to database write new log record to database
Args: Args:
log_record_id(LogRecordId): current log record id log_record(LogRecord): log record object
created(float): log created timestamp from log record attribute
record(str): log record
repository_id(RepositoryId, optional): repository unique identifier override (Default value = None) repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
""" """
repository_id = repository_id or self._repository_id repository_id = repository_id or self._repository_id
@ -83,16 +81,17 @@ class LogsOperations(Operations):
connection.execute( connection.execute(
""" """
insert into logs insert into logs
(package_base, version, created, record, repository) (package_base, version, created, record, repository, process_id)
values values
(:package_base, :version, :created, :record, :repository) (:package_base, :version, :created, :record, :repository, :process_id)
""", """,
{ {
"package_base": log_record_id.package_base, "package_base": log_record.log_record_id.package_base,
"version": log_record_id.version, "version": log_record.log_record_id.version,
"created": created, "created": log_record.created,
"record": record, "record": log_record.message,
"repository": repository_id.id, "repository": repository_id.id,
"process_id": log_record.log_record_id.process_id,
} }
) )
@ -125,3 +124,54 @@ class LogsOperations(Operations):
) )
return self.with_connection(run, commit=True) return self.with_connection(run, commit=True)
def logs_rotate(self, keep_last_records: int, repository_id: RepositoryId | None = None) -> None:
"""
rotate logs in storage. This method will remove old logs, keeping only the last N records for each package
Args:
keep_last_records(int): number of last records to keep
repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
"""
repository_id = repository_id or self._repository_id
def remove_duplicates(connection: Connection) -> None:
connection.execute(
"""
delete from logs
where (package_base, version, repository, process_id) not in (
select package_base, version, repository, process_id from logs
where (package_base, version, repository, created) in (
select package_base, version, repository, max(created) from logs
where repository = :repository
group by package_base, version, repository
)
)
""",
{
"repository": repository_id.id,
}
)
def remove_older(connection: Connection) -> None:
connection.execute(
"""
delete from logs
where (package_base, repository, process_id) in (
select package_base, repository, process_id from logs
where repository = :repository
group by package_base, repository, process_id
order by min(created) desc limit -1 offset :offset
)
""",
{
"offset": keep_last_records,
"repository": repository_id.id,
}
)
def run(connection: Connection) -> None:
remove_duplicates(connection)
remove_older(connection)
return self.with_connection(run, commit=True)

View File

@ -17,12 +17,14 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
import atexit
import logging import logging
from typing import Self from typing import Self
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.status import Client from ahriman.core.status import Client
from ahriman.models.log_record import LogRecord
from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_id import RepositoryId
@ -33,6 +35,7 @@ class HttpLogHandler(logging.Handler):
method method
Attributes: Attributes:
keep_last_records(int): number of last records to keep
reporter(Client): build status reporter instance reporter(Client): build status reporter instance
suppress_errors(bool): suppress logging errors (e.g. if no web server available) suppress_errors(bool): suppress logging errors (e.g. if no web server available)
""" """
@ -51,6 +54,7 @@ class HttpLogHandler(logging.Handler):
self.reporter = Client.load(repository_id, configuration, report=report) self.reporter = Client.load(repository_id, configuration, report=report)
self.suppress_errors = suppress_errors self.suppress_errors = suppress_errors
self.keep_last_records = configuration.getint("settings", "keep_last_logs", fallback=0)
@classmethod @classmethod
def load(cls, repository_id: RepositoryId, configuration: Configuration, *, report: bool) -> Self: def load(cls, repository_id: RepositoryId, configuration: Configuration, *, report: bool) -> Self:
@ -76,6 +80,8 @@ class HttpLogHandler(logging.Handler):
handler = cls(repository_id, configuration, report=report, suppress_errors=suppress_errors) handler = cls(repository_id, configuration, report=report, suppress_errors=suppress_errors)
root.addHandler(handler) root.addHandler(handler)
atexit.register(handler.rotate)
return handler return handler
def emit(self, record: logging.LogRecord) -> None: def emit(self, record: logging.LogRecord) -> None:
@ -90,8 +96,14 @@ class HttpLogHandler(logging.Handler):
return # in case if no package base supplied we need just skip log message return # in case if no package base supplied we need just skip log message
try: try:
self.reporter.package_logs_add(log_record_id, record.created, record.getMessage()) self.reporter.package_logs_add(LogRecord(log_record_id, record.created, record.getMessage()))
except Exception: except Exception:
if self.suppress_errors: if self.suppress_errors:
return return
self.handleError(record) self.handleError(record)
def rotate(self) -> None:
"""
rotate log records, removing older ones
"""
self.reporter.logs_rotate(self.keep_last_records)

View File

@ -74,7 +74,7 @@ class LazyLogging:
def package_record_factory(*args: Any, **kwargs: Any) -> logging.LogRecord: def package_record_factory(*args: Any, **kwargs: Any) -> logging.LogRecord:
record = current_factory(*args, **kwargs) record = current_factory(*args, **kwargs)
record.package_id = LogRecordId(package_base, version or "") record.package_id = LogRecordId(package_base, version or "<unknown>")
return record return record
logging.setLogRecordFactory(package_record_factory) logging.setLogRecordFactory(package_record_factory)
@ -99,24 +99,3 @@ class LazyLogging:
yield yield
finally: finally:
self._package_logger_reset() self._package_logger_reset()
@contextlib.contextmanager
def suppress_logging(self, log_level: int = logging.WARNING) -> Generator[None, None, None]:
"""
silence log messages in context
Args:
log_level(int, optional): the highest log level to keep (Default value = logging.WARNING)
Examples:
This function is designed to be used to suppress all log messages in context, e.g.:
>>> with self.suppress_logging():
>>> do_some_noisy_actions()
"""
current_level = self.logger.manager.disable
try:
logging.disable(log_level)
yield
finally:
logging.disable(current_level)

View File

@ -144,7 +144,6 @@ class UpdateHandler(PackageInfo, Cleaner):
branch="master", branch="master",
) )
with self.suppress_logging():
Sources.fetch(cache_dir, source) Sources.fetch(cache_dir, source)
remote = Package.from_build(cache_dir, self.architecture, None) remote = Package.from_build(cache_dir, self.architecture, None)

View File

@ -27,7 +27,7 @@ from ahriman.models.changes import Changes
from ahriman.models.dependencies import Dependencies from ahriman.models.dependencies import Dependencies
from ahriman.models.event import Event, EventType from ahriman.models.event import Event, EventType
from ahriman.models.internal_status import InternalStatus from ahriman.models.internal_status import InternalStatus
from ahriman.models.log_record_id import LogRecordId from ahriman.models.log_record import LogRecord
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_id import RepositoryId
@ -115,6 +115,14 @@ class Client:
""" """
raise NotImplementedError raise NotImplementedError
def logs_rotate(self, keep_last_records: int) -> None:
"""
remove older logs from storage
Args:
keep_last_records(int): number of last records to keep
"""
def package_changes_get(self, package_base: str) -> Changes: def package_changes_get(self, package_base: str) -> Changes:
""" """
get package changes get package changes
@ -186,18 +194,16 @@ class Client:
""" """
raise NotImplementedError raise NotImplementedError
def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: def package_logs_add(self, log_record: LogRecord) -> None:
""" """
post log record post log record
Args: Args:
log_record_id(LogRecordId): log record id log_record(LogRecord): log record
created(float): log created timestamp
message(str): log message
""" """
# this method does not raise NotImplementedError because it is actively used as dummy client for http log # this method does not raise NotImplementedError because it is actively used as dummy client for http log
def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[LogRecord]:
""" """
get package logs get package logs
@ -207,7 +213,7 @@ class Client:
offset(int, optional): records offset (Default value = 0) offset(int, optional): records offset (Default value = 0)
Returns: Returns:
list[tuple[float, str]]: package logs list[LogRecord]: package logs
Raises: Raises:
NotImplementedError: not implemented method NotImplementedError: not implemented method

View File

@ -23,7 +23,7 @@ from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.changes import Changes from ahriman.models.changes import Changes
from ahriman.models.dependencies import Dependencies from ahriman.models.dependencies import Dependencies
from ahriman.models.event import Event, EventType from ahriman.models.event import Event, EventType
from ahriman.models.log_record_id import LogRecordId from ahriman.models.log_record import LogRecord
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_id import RepositoryId
@ -75,6 +75,15 @@ class LocalClient(Client):
""" """
return self.database.event_get(event, object_id, from_date, to_date, limit, offset, self.repository_id) return self.database.event_get(event, object_id, from_date, to_date, limit, offset, self.repository_id)
def logs_rotate(self, keep_last_records: int) -> None:
"""
remove older logs from storage
Args:
keep_last_records(int): number of last records to keep
"""
self.database.logs_rotate(keep_last_records, self.repository_id)
def package_changes_get(self, package_base: str) -> Changes: def package_changes_get(self, package_base: str) -> Changes:
""" """
get package changes get package changes
@ -134,18 +143,16 @@ class LocalClient(Client):
return packages return packages
return [(package, status) for package, status in packages if package.base == package_base] return [(package, status) for package, status in packages if package.base == package_base]
def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: def package_logs_add(self, log_record: LogRecord) -> None:
""" """
post log record post log record
Args: Args:
log_record_id(LogRecordId): log record id log_record(LogRecord): log record
created(float): log created timestamp
message(str): log message
""" """
self.database.logs_insert(log_record_id, created, message, self.repository_id) self.database.logs_insert(log_record, self.repository_id)
def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[LogRecord]:
""" """
get package logs get package logs
@ -155,7 +162,7 @@ class LocalClient(Client):
offset(int, optional): records offset (Default value = 0) offset(int, optional): records offset (Default value = 0)
Returns: Returns:
list[tuple[float, str]]: package logs list[LogRecord]: package logs
""" """
return self.database.logs_get(package_base, limit, offset, self.repository_id) return self.database.logs_get(package_base, limit, offset, self.repository_id)

View File

@ -28,7 +28,7 @@ from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.changes import Changes from ahriman.models.changes import Changes
from ahriman.models.dependencies import Dependencies from ahriman.models.dependencies import Dependencies
from ahriman.models.event import Event, EventType from ahriman.models.event import Event, EventType
from ahriman.models.log_record_id import LogRecordId from ahriman.models.log_record import LogRecord
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.pkgbuild_patch import PkgbuildPatch
@ -53,9 +53,6 @@ class Watcher(LazyLogging):
self._known: dict[str, tuple[Package, BuildStatus]] = {} self._known: dict[str, tuple[Package, BuildStatus]] = {}
self.status = BuildStatus() self.status = BuildStatus()
# special variables for updating logs
self._last_log_record_id = LogRecordId("", "")
@property @property
def packages(self) -> list[tuple[Package, BuildStatus]]: def packages(self) -> list[tuple[Package, BuildStatus]]:
""" """
@ -81,6 +78,8 @@ class Watcher(LazyLogging):
for package, status in self.client.package_get(None) for package, status in self.client.package_get(None)
} }
logs_rotate: Callable[[int], None]
package_changes_get: Callable[[str], Changes] package_changes_get: Callable[[str], Changes]
package_changes_update: Callable[[str, Changes], None] package_changes_update: Callable[[str, Changes], None]
@ -108,22 +107,9 @@ class Watcher(LazyLogging):
except KeyError: except KeyError:
raise UnknownPackageError(package_base) from None raise UnknownPackageError(package_base) from None
def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: package_logs_add: Callable[[LogRecord], None]
"""
make new log record into database
Args: package_logs_get: Callable[[str, int, int], list[LogRecord]]
log_record_id(LogRecordId): log record id
created(float): log created timestamp
message(str): log message
"""
if self._last_log_record_id != log_record_id:
# there is new log record, so we remove old ones
self.package_logs_remove(log_record_id.package_base, log_record_id.version)
self._last_log_record_id = log_record_id
self.client.package_logs_add(log_record_id, created, message)
package_logs_get: Callable[[str, int, int], list[tuple[float, str]]]
package_logs_remove: Callable[[str, str | None], None] package_logs_remove: Callable[[str, str | None], None]

View File

@ -29,6 +29,7 @@ from ahriman.models.changes import Changes
from ahriman.models.dependencies import Dependencies from ahriman.models.dependencies import Dependencies
from ahriman.models.event import Event, EventType from ahriman.models.event import Event, EventType
from ahriman.models.internal_status import InternalStatus from ahriman.models.internal_status import InternalStatus
from ahriman.models.log_record import LogRecord
from ahriman.models.log_record_id import LogRecordId from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.pkgbuild_patch import PkgbuildPatch
@ -210,6 +211,18 @@ class WebClient(Client, SyncAhrimanClient):
return [] return []
def logs_rotate(self, keep_last_records: int) -> None:
"""
remove older logs from storage
Args:
keep_last_records(int): number of last records to keep
"""
query = self.repository_id.query() + [("keep_last_records", str(keep_last_records))]
with contextlib.suppress(Exception):
self.make_request("DELETE", f"{self.address}/api/v1/service/logs", params=query)
def package_changes_get(self, package_base: str) -> Changes: def package_changes_get(self, package_base: str) -> Changes:
""" """
get package changes get package changes
@ -294,28 +307,27 @@ class WebClient(Client, SyncAhrimanClient):
return [] return []
def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: def package_logs_add(self, log_record: LogRecord) -> None:
""" """
post log record post log record
Args: Args:
log_record_id(LogRecordId): log record id log_record(LogRecord): log record
created(float): log created timestamp
message(str): log message
""" """
payload = { payload = {
"created": created, "created": log_record.created,
"message": message, "message": log_record.message,
"version": log_record_id.version, "process_id": log_record.log_record_id.process_id,
"version": log_record.log_record_id.version,
} }
# this is special case, because we would like to do not suppress exception here # this is special case, because we would like to do not suppress exception here
# in case of exception raised it will be handled by upstream HttpLogHandler # in case of exception raised it will be handled by upstream HttpLogHandler
# In the other hand, we force to suppress all http logs here to avoid cyclic reporting # In the other hand, we force to suppress all http logs here to avoid cyclic reporting
self.make_request("POST", self._logs_url(log_record_id.package_base), self.make_request("POST", self._logs_url(log_record.log_record_id.package_base),
params=self.repository_id.query(), json=payload, suppress_errors=True) params=self.repository_id.query(), json=payload, suppress_errors=True)
def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[LogRecord]:
""" """
get package logs get package logs
@ -325,7 +337,7 @@ class WebClient(Client, SyncAhrimanClient):
offset(int, optional): records offset (Default value = 0) offset(int, optional): records offset (Default value = 0)
Returns: Returns:
list[tuple[float, str]]: package logs list[LogRecord]: package logs
""" """
query = self.repository_id.query() + [("limit", str(limit)), ("offset", str(offset))] query = self.repository_id.query() + [("limit", str(limit)), ("offset", str(offset))]
@ -333,7 +345,13 @@ class WebClient(Client, SyncAhrimanClient):
response = self.make_request("GET", self._logs_url(package_base), params=query) response = self.make_request("GET", self._logs_url(package_base), params=query)
response_json = response.json() response_json = response.json()
return [(record["created"], record["message"]) for record in response_json] return [
LogRecord(
LogRecordId(package_base, record["version"], record["process_id"]),
record["created"],
record["message"]
) for record in response_json
]
return [] return []

View File

@ -0,0 +1,38 @@
#
# Copyright (c) 2021-2025 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass
from ahriman.models.log_record_id import LogRecordId
@dataclass(frozen=True)
class LogRecord:
"""
log record
Attributes:
log_record_id(LogRecordId): log record identifier
created(float): log record creation timestamp
message(str): log record message
"""
log_record_id: LogRecordId
created: float
message: str

View File

@ -17,7 +17,9 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
from dataclasses import dataclass import uuid
from dataclasses import dataclass, field
@dataclass(frozen=True) @dataclass(frozen=True)
@ -28,7 +30,12 @@ class LogRecordId:
Attributes: Attributes:
package_base(str): package base for which log record belongs package_base(str): package base for which log record belongs
version(str): package version for which log record belongs version(str): package version for which log record belongs
process_id(str, optional): unique process identifier
""" """
package_base: str package_base: str
version: str version: str
# this is not mistake, this value is kind of global identifier, which is generated
# upon the process start
process_id: str = field(default=str(uuid.uuid4()))

View File

@ -429,7 +429,6 @@ class Package(LazyLogging):
task = Task(self, configuration, repository_id.architecture, paths) task = Task(self, configuration, repository_id.architecture, paths)
try: try:
with self.suppress_logging():
# create fresh chroot environment, fetch sources and - automagically - update PKGBUILD # create fresh chroot environment, fetch sources and - automagically - update PKGBUILD
task.init(paths.cache_for(self.base), [], None) task.init(paths.cache_for(self.base), [], None)
pkgbuild = Pkgbuild.from_file(paths.cache_for(self.base) / "PKGBUILD") pkgbuild = Pkgbuild.from_file(paths.cache_for(self.base) / "PKGBUILD")

View File

@ -18,7 +18,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
from aiohttp.web import HTTPException from aiohttp.web import HTTPException
from typing import Any, Callable from collections.abc import Callable
from typing import Any
from ahriman.models.user_access import UserAccess from ahriman.models.user_access import UserAccess
from ahriman.web.apispec import Schema, aiohttp_apispec from ahriman.web.apispec import Schema, aiohttp_apispec

View File

@ -31,6 +31,7 @@ from ahriman.web.schemas.info_schema import InfoSchema
from ahriman.web.schemas.internal_status_schema import InternalStatusSchema from ahriman.web.schemas.internal_status_schema import InternalStatusSchema
from ahriman.web.schemas.log_schema import LogSchema from ahriman.web.schemas.log_schema import LogSchema
from ahriman.web.schemas.login_schema import LoginSchema from ahriman.web.schemas.login_schema import LoginSchema
from ahriman.web.schemas.logs_rotate_schema import LogsRotateSchema
from ahriman.web.schemas.logs_schema import LogsSchema from ahriman.web.schemas.logs_schema import LogsSchema
from ahriman.web.schemas.oauth2_schema import OAuth2Schema from ahriman.web.schemas.oauth2_schema import OAuth2Schema
from ahriman.web.schemas.package_name_schema import PackageNameSchema from ahriman.web.schemas.package_name_schema import PackageNameSchema
@ -53,5 +54,4 @@ from ahriman.web.schemas.repository_stats_schema import RepositoryStatsSchema
from ahriman.web.schemas.search_schema import SearchSchema from ahriman.web.schemas.search_schema import SearchSchema
from ahriman.web.schemas.status_schema import StatusSchema from ahriman.web.schemas.status_schema import StatusSchema
from ahriman.web.schemas.update_flags_schema import UpdateFlagsSchema from ahriman.web.schemas.update_flags_schema import UpdateFlagsSchema
from ahriman.web.schemas.versioned_log_schema import VersionedLogSchema
from ahriman.web.schemas.worker_schema import WorkerSchema from ahriman.web.schemas.worker_schema import WorkerSchema

View File

@ -17,12 +17,13 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
from ahriman import __version__
from ahriman.web.apispec import Schema, fields from ahriman.web.apispec import Schema, fields
class LogSchema(Schema): class LogSchema(Schema):
""" """
request package log schema request and response package log schema
""" """
created = fields.Float(required=True, metadata={ created = fields.Float(required=True, metadata={
@ -32,3 +33,10 @@ class LogSchema(Schema):
message = fields.String(required=True, metadata={ message = fields.String(required=True, metadata={
"description": "Log message", "description": "Log message",
}) })
version = fields.String(required=True, metadata={
"description": "Package version to tag",
"example": __version__,
})
process_id = fields.String(metadata={
"description": "Process unique identifier",
})

View File

@ -17,18 +17,14 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
from ahriman import __version__ from ahriman.web.apispec import Schema, fields
from ahriman.web.apispec import fields
from ahriman.web.schemas.log_schema import LogSchema
from ahriman.web.schemas.repository_id_schema import RepositoryIdSchema
class VersionedLogSchema(LogSchema, RepositoryIdSchema): class LogsRotateSchema(Schema):
""" """
request package log schema request logs rotate schema
""" """
version = fields.Integer(required=True, metadata={ keep_last_records = fields.Integer(metadata={
"description": "Package version to tag", "description": "Keep the specified amount of records",
"example": __version__,
}) })

View File

@ -21,11 +21,11 @@ from aiohttp.web import HTTPBadRequest, HTTPNoContent, HTTPNotFound, Response, j
from ahriman.core.exceptions import UnknownPackageError from ahriman.core.exceptions import UnknownPackageError
from ahriman.core.utils import pretty_datetime from ahriman.core.utils import pretty_datetime
from ahriman.models.log_record import LogRecord
from ahriman.models.log_record_id import LogRecordId from ahriman.models.log_record_id import LogRecordId
from ahriman.models.user_access import UserAccess from ahriman.models.user_access import UserAccess
from ahriman.web.apispec.decorators import apidocs from ahriman.web.apispec.decorators import apidocs
from ahriman.web.schemas import LogsSchema, PackageNameSchema, PackageVersionSchema, RepositoryIdSchema, \ from ahriman.web.schemas import LogSchema, LogsSchema, PackageNameSchema, PackageVersionSchema, RepositoryIdSchema
VersionedLogSchema
from ahriman.web.views.base import BaseView from ahriman.web.views.base import BaseView
from ahriman.web.views.status_view_guard import StatusViewGuard from ahriman.web.views.status_view_guard import StatusViewGuard
@ -97,7 +97,7 @@ class LogsView(StatusViewGuard, BaseView):
response = { response = {
"package_base": package_base, "package_base": package_base,
"status": status.view(), "status": status.view(),
"logs": "\n".join(f"[{pretty_datetime(created)}] {message}" for created, message in logs) "logs": "\n".join(f"[{pretty_datetime(log_record.created)}] {log_record.message}" for log_record in logs)
} }
return json_response(response) return json_response(response)
@ -109,7 +109,7 @@ class LogsView(StatusViewGuard, BaseView):
error_400_enabled=True, error_400_enabled=True,
error_404_description="Repository is unknown", error_404_description="Repository is unknown",
match_schema=PackageNameSchema, match_schema=PackageNameSchema,
body_schema=VersionedLogSchema, body_schema=LogSchema,
) )
async def post(self) -> None: async def post(self) -> None:
""" """
@ -129,6 +129,9 @@ class LogsView(StatusViewGuard, BaseView):
except Exception as ex: except Exception as ex:
raise HTTPBadRequest(reason=str(ex)) raise HTTPBadRequest(reason=str(ex))
self.service().package_logs_add(LogRecordId(package_base, version), created, record) # either read from process identifier from payload or assign to the current process identifier
process_id = data.get("process_id", LogRecordId("", "").process_id)
log_record = LogRecord(LogRecordId(package_base, version, process_id), created, record)
self.service().package_logs_add(log_record)
raise HTTPNoContent raise HTTPNoContent

View File

@ -0,0 +1,63 @@
#
# Copyright (c) 2021-2025 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from aiohttp.web import HTTPBadRequest, HTTPNoContent
from ahriman.models.user_access import UserAccess
from ahriman.web.apispec.decorators import apidocs
from ahriman.web.schemas import LogsRotateSchema
from ahriman.web.views.base import BaseView
class LogsView(BaseView):
"""
logs management web view
Attributes:
DELETE_PERMISSION(UserAccess): (class attribute) delete permissions of self
"""
DELETE_PERMISSION = UserAccess.Full
ROUTES = ["/api/v1/service/logs"]
@apidocs(
tags=["Actions"],
summary="Rotate logs",
description="Remove older logs from system",
permission=DELETE_PERMISSION,
error_400_enabled=True,
error_404_description="Repository is unknown",
query_schema=LogsRotateSchema,
)
async def delete(self) -> None:
"""
rotate logs from system
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: on success response
"""
try:
keep_last_records = int(self.request.query.get("keep_last_records", 0))
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
self.service().logs_rotate(keep_last_records)
raise HTTPNoContent

View File

@ -65,8 +65,10 @@ class LogsView(StatusViewGuard, BaseView):
response = [ response = [
{ {
"created": created, "created": log_record.created,
"message": message, "message": log_record.message,
} for created, message in logs "version": log_record.log_record_id.version,
"process_id": log_record.log_record_id.process_id,
} for log_record in logs
] ]
return json_response(response) return json_response(response)

View File

@ -0,0 +1,8 @@
from ahriman.core.database.migrations.m015_logs_process_id import steps
def test_migration_logs_process_id() -> None:
"""
migration must not be empty
"""
assert steps

View File

@ -1,4 +1,5 @@
from ahriman.core.database import SQLite from ahriman.core.database import SQLite
from ahriman.models.log_record import LogRecord
from ahriman.models.log_record_id import LogRecordId from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_id import RepositoryId
@ -9,67 +10,115 @@ def test_logs_insert_remove_version(database: SQLite, package_ahriman: Package,
""" """
must clear version specific package logs must clear version specific package logs
""" """
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"))
database.logs_insert(LogRecordId(package_ahriman.base, "2"), 43.0, "message 2") database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "2"), 43.0, "message 2"))
database.logs_insert(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3") database.logs_insert(LogRecord(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3"))
database.logs_remove(package_ahriman.base, "1") database.logs_remove(package_ahriman.base, "1")
assert database.logs_get(package_ahriman.base) == [(42.0, "message 1")] assert database.logs_get(package_ahriman.base) == [
assert database.logs_get(package_python_schedule.base) == [(42.0, "message 3")] LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"),
]
assert database.logs_get(package_python_schedule.base) == [
LogRecord(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3"),
]
def test_logs_insert_remove_multi(database: SQLite, package_ahriman: Package) -> None: def test_logs_insert_remove_multi(database: SQLite, package_ahriman: Package) -> None:
""" """
must clear logs for specified repository must clear logs for specified repository
""" """
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"))
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2", database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"),
RepositoryId("i686", database._repository_id.name)) RepositoryId("i686", database._repository_id.name))
database.logs_remove(package_ahriman.base, None, RepositoryId("i686", database._repository_id.name)) database.logs_remove(package_ahriman.base, None, RepositoryId("i686", database._repository_id.name))
assert not database.logs_get(package_ahriman.base, repository_id=RepositoryId("i686", database._repository_id.name)) assert not database.logs_get(package_ahriman.base, repository_id=RepositoryId("i686", database._repository_id.name))
assert database.logs_get(package_ahriman.base) == [(42.0, "message 1")] assert database.logs_get(package_ahriman.base) == [
LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"),
]
def test_logs_insert_remove_full(database: SQLite, package_ahriman: Package, package_python_schedule: Package) -> None: def test_logs_insert_remove_full(database: SQLite, package_ahriman: Package, package_python_schedule: Package) -> None:
""" """
must clear full package logs must clear full package logs
""" """
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"))
database.logs_insert(LogRecordId(package_ahriman.base, "2"), 43.0, "message 2") database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "2"), 43.0, "message 2"))
database.logs_insert(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3") database.logs_insert(LogRecord(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3"))
database.logs_remove(package_ahriman.base, None) database.logs_remove(package_ahriman.base, None)
assert not database.logs_get(package_ahriman.base) assert not database.logs_get(package_ahriman.base)
assert database.logs_get(package_python_schedule.base) == [(42.0, "message 3")] assert database.logs_get(package_python_schedule.base) == [
LogRecord(LogRecordId(package_python_schedule.base, "1"), 42.0, "message 3"),
]
def test_logs_insert_get(database: SQLite, package_ahriman: Package) -> None: def test_logs_insert_get(database: SQLite, package_ahriman: Package) -> None:
""" """
must insert and get package logs must insert and get package logs
""" """
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2") database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"))
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"))
assert database.logs_get(package_ahriman.base) == [(42.0, "message 1"), (43.0, "message 2")] assert database.logs_get(package_ahriman.base) == [
LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"),
LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"),
]
def test_logs_insert_get_pagination(database: SQLite, package_ahriman: Package) -> None: def test_logs_insert_get_pagination(database: SQLite, package_ahriman: Package) -> None:
""" """
must insert and get package logs with pagination must insert and get package logs with pagination
""" """
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"))
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2") database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"))
assert database.logs_get(package_ahriman.base, 1, 1) == [(42.0, "message 1")] assert database.logs_get(package_ahriman.base, 1, 1) == [
LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"),
]
def test_logs_insert_get_multi(database: SQLite, package_ahriman: Package) -> None: def test_logs_insert_get_multi(database: SQLite, package_ahriman: Package) -> None:
""" """
must insert and get package logs for multiple repositories must insert and get package logs for multiple repositories
""" """
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1") database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"))
database.logs_insert(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2", database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"),
RepositoryId("i686", database._repository_id.name)) RepositoryId("i686", database._repository_id.name))
assert database.logs_get(package_ahriman.base, assert database.logs_get(package_ahriman.base,
repository_id=RepositoryId("i686", database._repository_id.name)) == [(43.0, "message 2")] repository_id=RepositoryId("i686", database._repository_id.name)) == [
assert database.logs_get(package_ahriman.base) == [(42.0, "message 1")] LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"),
]
assert database.logs_get(package_ahriman.base) == [
LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"),
]
def test_logs_rotate_remove_all(database: SQLite, package_ahriman: Package) -> None:
"""
must remove all records when rotating with keep_last_records is 0
"""
database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 42.0, "message 1"))
database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1"), 43.0, "message 2"))
database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "2"), 44.0, "message 3"))
database.logs_rotate(0)
assert not database.logs_get(package_ahriman.base)
def test_logs_rotate_remove_duplicates(database: SQLite, package_ahriman: Package) -> None:
"""
must remove duplicate records while preserving the most recent one for each package version
"""
database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1", "p1"), 42.0, "message 1"))
database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1", "p2"), 43.0, "message 2"))
database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "1", "p3"), 44.0, "message 3"))
database.logs_insert(LogRecord(LogRecordId(package_ahriman.base, "2", "p1"), 45.0, "message 4"))
database.logs_rotate(2)
logs = database.logs_get(package_ahriman.base)
assert len(logs) == 2
assert logs == [
LogRecord(LogRecordId(package_ahriman.base, "1", "p3"), 44.0, "message 3"),
LogRecord(LogRecordId(package_ahriman.base, "2", "p1"), 45.0, "message 4"),
]

View File

@ -4,6 +4,7 @@ from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.log.http_log_handler import HttpLogHandler from ahriman.core.log.http_log_handler import HttpLogHandler
from ahriman.models.log_record import LogRecord
from ahriman.models.log_record_id import LogRecordId from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package from ahriman.models.package import Package
@ -19,12 +20,14 @@ def test_load(configuration: Configuration, mocker: MockerFixture) -> None:
add_mock = mocker.patch("logging.Logger.addHandler") add_mock = mocker.patch("logging.Logger.addHandler")
load_mock = mocker.patch("ahriman.core.status.Client.load") load_mock = mocker.patch("ahriman.core.status.Client.load")
atexit_mock = mocker.patch("atexit.register")
_, repository_id = configuration.check_loaded() _, repository_id = configuration.check_loaded()
handler = HttpLogHandler.load(repository_id, configuration, report=False) handler = HttpLogHandler.load(repository_id, configuration, report=False)
assert handler assert handler
add_mock.assert_called_once_with(handler) add_mock.assert_called_once_with(handler)
load_mock.assert_called_once_with(repository_id, configuration, report=False) load_mock.assert_called_once_with(repository_id, configuration, report=False)
atexit_mock.assert_called_once_with(handler.rotate)
def test_load_exist(configuration: Configuration) -> None: def test_load_exist(configuration: Configuration) -> None:
@ -49,7 +52,7 @@ def test_emit(configuration: Configuration, log_record: logging.LogRecord, packa
handler = HttpLogHandler(repository_id, configuration, report=False, suppress_errors=False) handler = HttpLogHandler(repository_id, configuration, report=False, suppress_errors=False)
handler.emit(log_record) handler.emit(log_record)
log_mock.assert_called_once_with(log_record_id, log_record.created, log_record.getMessage()) log_mock.assert_called_once_with(LogRecord(log_record_id, log_record.created, log_record.getMessage()))
def test_emit_failed(configuration: Configuration, log_record: logging.LogRecord, package_ahriman: Package, def test_emit_failed(configuration: Configuration, log_record: logging.LogRecord, package_ahriman: Package,
@ -93,3 +96,16 @@ def test_emit_skip(configuration: Configuration, log_record: logging.LogRecord,
handler.emit(log_record) handler.emit(log_record)
log_mock.assert_not_called() log_mock.assert_not_called()
def test_rotate(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must rotate logs
"""
rotate_mock = mocker.patch("ahriman.core.status.Client.logs_rotate")
_, repository_id = configuration.check_loaded()
handler = HttpLogHandler(repository_id, configuration, report=False, suppress_errors=False)
handler.rotate()
rotate_mock.assert_called_once_with(handler.keep_last_records)

View File

@ -87,13 +87,3 @@ def test_in_package_context_failed(database: SQLite, package_ahriman: Package, m
raise ValueError() raise ValueError()
reset_mock.assert_called_once_with() reset_mock.assert_called_once_with()
def test_suppress_logging(database: SQLite, mocker: MockerFixture) -> None:
"""
must temporary disable log messages
"""
disable_mock = mocker.patch("ahriman.core.log.lazy_logging.logging.disable")
with database.suppress_logging():
pass
disable_mock.assert_has_calls([MockCall(logging.WARNING), MockCall(logging.NOTSET)])

View File

@ -13,6 +13,7 @@ from ahriman.models.changes import Changes
from ahriman.models.dependencies import Dependencies from ahriman.models.dependencies import Dependencies
from ahriman.models.event import Event from ahriman.models.event import Event
from ahriman.models.internal_status import InternalStatus from ahriman.models.internal_status import InternalStatus
from ahriman.models.log_record import LogRecord
from ahriman.models.log_record_id import LogRecordId from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.pkgbuild_patch import PkgbuildPatch
@ -112,6 +113,13 @@ def test_event_get(client: Client) -> None:
client.event_get(None, None) client.event_get(None, None)
def test_logs_rotate(client: Client) -> None:
"""
must do not raise exception on logs rotation call
"""
client.logs_rotate(1)
def test_package_changes_get(client: Client, package_ahriman: Package) -> None: def test_package_changes_get(client: Client, package_ahriman: Package) -> None:
""" """
must raise not implemented on package changes request must raise not implemented on package changes request
@ -157,7 +165,7 @@ def test_package_logs_add(client: Client, package_ahriman: Package, log_record:
must process log record addition without exception must process log record addition without exception
""" """
log_record_id = LogRecordId(package_ahriman.base, package_ahriman.version) log_record_id = LogRecordId(package_ahriman.base, package_ahriman.version)
client.package_logs_add(log_record_id, log_record.created, log_record.getMessage()) client.package_logs_add(LogRecord(log_record_id, log_record.created, log_record.getMessage()))
def test_package_logs_get(client: Client, package_ahriman: Package) -> None: def test_package_logs_get(client: Client, package_ahriman: Package) -> None:

View File

@ -8,6 +8,7 @@ from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.changes import Changes from ahriman.models.changes import Changes
from ahriman.models.dependencies import Dependencies from ahriman.models.dependencies import Dependencies
from ahriman.models.event import Event, EventType from ahriman.models.event import Event, EventType
from ahriman.models.log_record import LogRecord
from ahriman.models.log_record_id import LogRecordId from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.pkgbuild_patch import PkgbuildPatch
@ -34,6 +35,15 @@ def test_event_get(local_client: LocalClient, package_ahriman: Package, mocker:
local_client.repository_id) local_client.repository_id)
def test_logs_rotate(local_client: LocalClient, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must rotate logs
"""
rotate_mock = mocker.patch("ahriman.core.database.SQLite.logs_rotate")
local_client.logs_rotate(42)
rotate_mock.assert_called_once_with(42, local_client.repository_id)
def test_package_changes_get(local_client: LocalClient, package_ahriman: Package, mocker: MockerFixture) -> None: def test_package_changes_get(local_client: LocalClient, package_ahriman: Package, mocker: MockerFixture) -> None:
""" """
must retrieve package changes must retrieve package changes
@ -103,10 +113,10 @@ def test_package_logs_add(local_client: LocalClient, package_ahriman: Package, l
""" """
logs_mock = mocker.patch("ahriman.core.database.SQLite.logs_insert") logs_mock = mocker.patch("ahriman.core.database.SQLite.logs_insert")
log_record_id = LogRecordId(package_ahriman.base, package_ahriman.version) log_record_id = LogRecordId(package_ahriman.base, package_ahriman.version)
record = LogRecord(log_record_id, log_record.created, log_record.getMessage())
local_client.package_logs_add(log_record_id, log_record.created, log_record.getMessage()) local_client.package_logs_add(record)
logs_mock.assert_called_once_with(log_record_id, log_record.created, log_record.getMessage(), logs_mock.assert_called_once_with(record, local_client.repository_id)
local_client.repository_id)
def test_package_logs_get(local_client: LocalClient, package_ahriman: Package, mocker: MockerFixture) -> None: def test_package_logs_get(local_client: LocalClient, package_ahriman: Package, mocker: MockerFixture) -> None:

View File

@ -5,7 +5,6 @@ from pytest_mock import MockerFixture
from ahriman.core.exceptions import UnknownPackageError from ahriman.core.exceptions import UnknownPackageError
from ahriman.core.status.watcher import Watcher from ahriman.core.status.watcher import Watcher
from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package from ahriman.models.package import Package
@ -64,38 +63,6 @@ def test_package_get_failed(watcher: Watcher, package_ahriman: Package) -> None:
watcher.package_get(package_ahriman.base) watcher.package_get(package_ahriman.base)
def test_package_logs_add_new(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must create package logs record for new package
"""
delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.package_logs_remove", create=True)
insert_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_logs_add")
log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.version)
assert watcher._last_log_record_id != log_record_id
watcher.package_logs_add(log_record_id, 42.01, "log record")
delete_mock.assert_called_once_with(package_ahriman.base, log_record_id.version)
insert_mock.assert_called_once_with(log_record_id, 42.01, "log record")
assert watcher._last_log_record_id == log_record_id
def test_package_logs_add_update(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must create package logs record for current package
"""
delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.package_logs_remove", create=True)
insert_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.package_logs_add")
log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.version)
watcher._last_log_record_id = log_record_id
watcher.package_logs_add(log_record_id, 42.01, "log record")
delete_mock.assert_not_called()
insert_mock.assert_called_once_with(log_record_id, 42.01, "log record")
def test_package_remove(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: def test_package_remove(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
""" """
must remove package base must remove package base

View File

@ -12,6 +12,7 @@ from ahriman.models.changes import Changes
from ahriman.models.dependencies import Dependencies from ahriman.models.dependencies import Dependencies
from ahriman.models.event import Event, EventType from ahriman.models.event import Event, EventType
from ahriman.models.internal_status import InternalStatus from ahriman.models.internal_status import InternalStatus
from ahriman.models.log_record import LogRecord
from ahriman.models.log_record_id import LogRecordId from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.pkgbuild_patch import PkgbuildPatch
@ -257,6 +258,57 @@ def test_event_get_failed_http_error_suppress(web_client: WebClient, mocker: Moc
logging_mock.assert_not_called() logging_mock.assert_not_called()
def test_logs_rotate(web_client: WebClient, mocker: MockerFixture) -> None:
"""
must rotate logs
"""
requests_mock = mocker.patch("ahriman.core.status.web_client.WebClient.make_request")
web_client.logs_rotate(42)
requests_mock.assert_called_once_with("DELETE", pytest.helpers.anyvar(str, True),
params=web_client.repository_id.query() + [("keep_last_records", "42")])
def test_logs_rotate_failed(web_client: WebClient, mocker: MockerFixture) -> None:
"""
must suppress any exception happened during logs rotation
"""
mocker.patch("requests.Session.request", side_effect=Exception())
web_client.logs_rotate(42)
def test_logs_rotate_failed_http_error(web_client: WebClient, mocker: MockerFixture) -> None:
"""
must suppress HTTP exception happened during logs rotation
"""
mocker.patch("requests.Session.request", side_effect=requests.HTTPError())
web_client.logs_rotate(42)
def test_logs_rotate_failed_suppress(web_client: WebClient, mocker: MockerFixture) -> None:
"""
must suppress any exception happened during logs rotation and don't log
"""
web_client.suppress_errors = True
mocker.patch("requests.Session.request", side_effect=Exception())
logging_mock = mocker.patch("logging.exception")
web_client.logs_rotate(42)
logging_mock.assert_not_called()
def test_logs_rotate_failed_http_error_suppress(web_client: WebClient, mocker: MockerFixture) -> None:
"""
must suppress HTTP exception happened during logs rotation and don't log
"""
web_client.suppress_errors = True
mocker.patch("requests.Session.request", side_effect=requests.HTTPError())
logging_mock = mocker.patch("logging.exception")
web_client.logs_rotate(42)
logging_mock.assert_not_called()
def test_package_changes_get(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None: def test_package_changes_get(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None:
""" """
must get changes must get changes
@ -551,11 +603,13 @@ def test_package_logs_add(web_client: WebClient, log_record: logging.LogRecord,
payload = { payload = {
"created": log_record.created, "created": log_record.created,
"message": log_record.getMessage(), "message": log_record.getMessage(),
"process_id": LogRecordId.process_id,
"version": package_ahriman.version, "version": package_ahriman.version,
} }
record = LogRecord(LogRecordId(package_ahriman.base, package_ahriman.version),
web_client.package_logs_add(LogRecordId(package_ahriman.base, package_ahriman.version),
log_record.created, log_record.getMessage()) log_record.created, log_record.getMessage())
web_client.package_logs_add(record)
requests_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True), requests_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True),
params=web_client.repository_id.query(), json=payload, suppress_errors=True) params=web_client.repository_id.query(), json=payload, suppress_errors=True)
@ -567,10 +621,12 @@ def test_package_logs_add_failed(web_client: WebClient, log_record: logging.LogR
""" """
mocker.patch("requests.Session.request", side_effect=Exception()) mocker.patch("requests.Session.request", side_effect=Exception())
log_record.package_base = package_ahriman.base log_record.package_base = package_ahriman.base
with pytest.raises(Exception): record = LogRecord(LogRecordId(package_ahriman.base, package_ahriman.version),
web_client.package_logs_add(LogRecordId(package_ahriman.base, package_ahriman.version),
log_record.created, log_record.getMessage()) log_record.created, log_record.getMessage())
with pytest.raises(Exception):
web_client.package_logs_add(record)
def test_package_logs_add_failed_http_error(web_client: WebClient, log_record: logging.LogRecord, def test_package_logs_add_failed_http_error(web_client: WebClient, log_record: logging.LogRecord,
package_ahriman: Package, mocker: MockerFixture) -> None: package_ahriman: Package, mocker: MockerFixture) -> None:
@ -579,16 +635,23 @@ def test_package_logs_add_failed_http_error(web_client: WebClient, log_record: l
""" """
mocker.patch("requests.Session.request", side_effect=requests.HTTPError()) mocker.patch("requests.Session.request", side_effect=requests.HTTPError())
log_record.package_base = package_ahriman.base log_record.package_base = package_ahriman.base
with pytest.raises(Exception): record = LogRecord(LogRecordId(package_ahriman.base, package_ahriman.version),
web_client.package_logs_add(LogRecordId(package_ahriman.base, package_ahriman.version),
log_record.created, log_record.getMessage()) log_record.created, log_record.getMessage())
with pytest.raises(Exception):
web_client.package_logs_add(record)
def test_package_logs_get(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None: def test_package_logs_get(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None:
""" """
must get logs must get logs
""" """
message = {"created": 42.0, "message": "log"} message = {
"created": 42.0,
"message": "log",
"version": package_ahriman.version,
"process_id": LogRecordId.process_id,
}
response_obj = requests.Response() response_obj = requests.Response()
response_obj._content = json.dumps([message]).encode("utf8") response_obj._content = json.dumps([message]).encode("utf8")
response_obj.status_code = 200 response_obj.status_code = 200
@ -598,7 +661,9 @@ def test_package_logs_get(web_client: WebClient, package_ahriman: Package, mocke
result = web_client.package_logs_get(package_ahriman.base, 1, 2) result = web_client.package_logs_get(package_ahriman.base, 1, 2)
requests_mock.assert_called_once_with("GET", pytest.helpers.anyvar(str, True), requests_mock.assert_called_once_with("GET", pytest.helpers.anyvar(str, True),
params=web_client.repository_id.query() + [("limit", "1"), ("offset", "2")]) params=web_client.repository_id.query() + [("limit", "1"), ("offset", "2")])
assert result == [(message["created"], message["message"])] assert result == [
LogRecord(LogRecordId(package_ahriman.base, package_ahriman.version), message["created"], message["message"]),
]
def test_package_logs_get_failed(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None: def test_package_logs_get_failed(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None:

View File

View File

@ -0,0 +1,77 @@
import pytest
from aiohttp.test_utils import TestClient
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package
from ahriman.models.user_access import UserAccess
from ahriman.web.views.v1.service.logs import LogsView
async def test_get_permission() -> None:
"""
must return correct permission for the request
"""
for method in ("DELETE",):
request = pytest.helpers.request("", "", method)
assert await LogsView.get_permission(request) == UserAccess.Full
def test_routes() -> None:
"""
must return correct routes
"""
assert LogsView.ROUTES == ["/api/v1/service/logs"]
async def test_delete(client: TestClient, package_ahriman: Package) -> None:
"""
must delete all logs
"""
await client.post(f"/api/v1/packages/{package_ahriman.base}",
json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()})
await client.post(f"/api/v1/packages/{package_ahriman.base}/logs",
json={"created": 42.0, "message": "message 1", "version": "42"})
await client.post(f"/api/v1/packages/{package_ahriman.base}/logs",
json={"created": 43.0, "message": "message 2", "version": "43"})
response = await client.delete("/api/v1/service/logs")
assert response.status == 204
response = await client.get(f"/api/v2/packages/{package_ahriman.base}/logs")
json = await response.json()
assert not json
async def test_delete_partially(client: TestClient, package_ahriman: Package) -> None:
"""
must delete logs based on count input
"""
await client.post(f"/api/v1/packages/{package_ahriman.base}",
json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()})
await client.post(f"/api/v1/packages/{package_ahriman.base}/logs",
json={"created": 42.0, "message": "message 1", "version": "42"})
await client.post(f"/api/v1/packages/{package_ahriman.base}/logs",
json={"created": 43.0, "message": "message 2", "version": "43"})
request_schema = pytest.helpers.schema_request(LogsView.delete, location="querystring")
payload = {"keep_last_records": 1}
assert not request_schema.validate(payload)
response = await client.delete("/api/v1/service/logs", params=payload)
assert response.status == 204
response = await client.get(f"/api/v2/packages/{package_ahriman.base}/logs")
json = await response.json()
assert json
async def test_delete_exception(client: TestClient, package_ahriman: Package) -> None:
"""
must raise exception on invalid payload
"""
response_schema = pytest.helpers.schema_response(LogsView.delete, code=400)
response = await client.delete("/api/v1/service/logs", params={"keep_last_records": "string"})
assert response.status == 400
assert not response_schema.validate(await response.json())

View File

@ -3,6 +3,7 @@ import pytest
from aiohttp.test_utils import TestClient from aiohttp.test_utils import TestClient
from ahriman.models.build_status import BuildStatusEnum from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package from ahriman.models.package import Package
from ahriman.models.user_access import UserAccess from ahriman.models.user_access import UserAccess
from ahriman.web.views.v2.packages.logs import LogsView from ahriman.web.views.v2.packages.logs import LogsView
@ -48,10 +49,14 @@ async def test_get(client: TestClient, package_ahriman: Package) -> None:
{ {
"created": 42.0, "created": 42.0,
"message": "message 1", "message": "message 1",
"version": "42",
"process_id": LogRecordId.process_id,
}, },
{ {
"created": 43.0, "created": 43.0,
"message": "message 2", "message": "message 2",
"version": "42",
"process_id": LogRecordId.process_id,
}, },
] ]
@ -76,7 +81,7 @@ async def test_get_with_pagination(client: TestClient, package_ahriman: Package)
logs = await response.json() logs = await response.json()
assert not response_schema.validate(logs) assert not response_schema.validate(logs)
assert logs == [{"created": 42.0, "message": "message 1"}] assert logs == [{"created": 42.0, "message": "message 1", "version": "42", "process_id": LogRecordId.process_id}]
async def test_get_bad_request(client: TestClient, package_ahriman: Package) -> None: async def test_get_bad_request(client: TestClient, package_ahriman: Package) -> None: