From e22e57bf08128ebfc7bc761dcc0124cab3aff4f9 Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Sun, 20 Nov 2022 12:37:51 +0200 Subject: [PATCH] handle process id during removal During one process we can write logs from different packages in different times (e.g. check and update laster) and we would like to store all logs belong to the same process --- .../core/database/migrations/m004_logs.py | 3 +- .../database/operations/logs_operations.py | 49 +++++++++++-------- src/ahriman/core/status/watcher.py | 10 ++-- src/ahriman/web/views/status/logs.py | 2 +- .../operations/test_logs_operations.py | 32 +++++++++--- tests/ahriman/core/status/test_watcher.py | 18 ++++--- 6 files changed, 71 insertions(+), 43 deletions(-) diff --git a/src/ahriman/core/database/migrations/m004_logs.py b/src/ahriman/core/database/migrations/m004_logs.py index bc09c434..1d26af04 100644 --- a/src/ahriman/core/database/migrations/m004_logs.py +++ b/src/ahriman/core/database/migrations/m004_logs.py @@ -24,11 +24,12 @@ steps = [ """ create table logs ( package_base text not null, + process_id integer not null, created real not null, record text ) """, """ - create index logs_package_base on logs (package_base) + create index logs_package_base_process_id on logs (package_base, process_id) """, ] diff --git a/src/ahriman/core/database/operations/logs_operations.py b/src/ahriman/core/database/operations/logs_operations.py index cd14d943..f6e61cf3 100644 --- a/src/ahriman/core/database/operations/logs_operations.py +++ b/src/ahriman/core/database/operations/logs_operations.py @@ -18,9 +18,10 @@ # along with this program. If not, see . # from sqlite3 import Connection -from typing import List +from typing import List, Optional from ahriman.core.database.operations import Operations +from ahriman.models.log_record_id import LogRecordId class LogsOperations(Operations): @@ -28,21 +29,6 @@ class LogsOperations(Operations): logs operations """ - def logs_delete(self, package_base: str) -> None: - """ - delete log records for the specified package - - Args: - package_base(str): package base to remove logs - """ - def run(connection: Connection) -> None: - connection.execute( - """delete from logs where package_base = :package_base""", - {"package_base": package_base} - ) - - return self.with_connection(run, commit=True) - def logs_get(self, package_base: str) -> str: """ extract logs for specified package base @@ -67,12 +53,12 @@ class LogsOperations(Operations): records = self.with_connection(run) return "\n".join(records) - def logs_insert(self, package_base: str, created: float, record: str) -> None: + def logs_insert(self, log_record_id: LogRecordId, created: float, record: str) -> None: """ write new log record to database Args: - package_base(str): package base + log_record_id(LogRecordId): current log record id created(float): log created timestamp from log record attribute record(str): log record """ @@ -80,15 +66,36 @@ class LogsOperations(Operations): connection.execute( """ insert into logs - (package_base, created, record) + (package_base, process_id, created, record) values - (:package_base, :created, :record) + (:package_base, :process_id, :created, :record) """, dict( - package_base=package_base, + package_base=log_record_id.package_base, + process_id=log_record_id.process_id, created=created, record=record ) ) return self.with_connection(run, commit=True) + + def logs_remove(self, package_base: str, current_process_id: Optional[int]) -> None: + """ + remove log records for the specified package + + Args: + package_base(str): package base to remove logs + current_process_id(Optional[int]): current process id. If set it will remove only logs belonging to another + process + """ + def run(connection: Connection) -> None: + connection.execute( + """ + delete from logs + where package_base = :package_base and (:process_id is null or process_id <> :process_id) + """, + {"package_base": package_base, "process_id": current_process_id} + ) + + return self.with_connection(run, commit=True) diff --git a/src/ahriman/core/status/watcher.py b/src/ahriman/core/status/watcher.py index 8e059f59..328661c3 100644 --- a/src/ahriman/core/status/watcher.py +++ b/src/ahriman/core/status/watcher.py @@ -128,15 +128,17 @@ class Watcher(LazyLogging): """ self.known.pop(package_base, None) self.database.package_remove(package_base) + self.remove_logs(package_base, None) - def remove_logs(self, package_base: str) -> None: + def remove_logs(self, package_base: str, current_process_id: Optional[int]) -> None: """ remove package related logs Args: package_base(str): package base + current_process_id(int): current process id """ - self.database.logs_delete(package_base) + self.database.logs_remove(package_base, current_process_id) def update(self, package_base: str, status: BuildStatusEnum, package: Optional[Package]) -> None: """ @@ -170,9 +172,9 @@ class Watcher(LazyLogging): """ if self._last_log_record_id != log_record_id: # there is new log record, so we remove old ones - self.database.logs_delete(log_record_id.package_base) + self.remove_logs(log_record_id.package_base, log_record_id.process_id) self._last_log_record_id = log_record_id - self.database.logs_insert(log_record_id.package_base, created, record) + self.database.logs_insert(log_record_id, created, record) def update_self(self, status: BuildStatusEnum) -> None: """ diff --git a/src/ahriman/web/views/status/logs.py b/src/ahriman/web/views/status/logs.py index 8614305b..8bf7cb76 100644 --- a/src/ahriman/web/views/status/logs.py +++ b/src/ahriman/web/views/status/logs.py @@ -46,7 +46,7 @@ class LogsView(BaseView): HTTPNoContent: on success response """ package_base = self.request.match_info["package"] - self.service.remove_logs(package_base) + self.service.remove_logs(package_base, None) raise HTTPNoContent() diff --git a/tests/ahriman/core/database/operations/test_logs_operations.py b/tests/ahriman/core/database/operations/test_logs_operations.py index 456176c5..2168d25f 100644 --- a/tests/ahriman/core/database/operations/test_logs_operations.py +++ b/tests/ahriman/core/database/operations/test_logs_operations.py @@ -1,23 +1,39 @@ from ahriman.core.database import SQLite +from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package -def test_logs_insert_delete(database: SQLite, package_ahriman: Package, package_python_schedule: Package) -> None: +def test_logs_insert_remove_process(database: SQLite, package_ahriman: Package, + package_python_schedule: Package) -> None: """ - must clear all packages + must clear process specific package logs """ - database.logs_insert(package_ahriman.base, 0.001, "message 1") - database.logs_insert(package_python_schedule.base, 0.002, "message 2") + database.logs_insert(LogRecordId(package_ahriman.base, 1), 0.001, "message 1") + database.logs_insert(LogRecordId(package_ahriman.base, 2), 0.001, "message 2") + database.logs_insert(LogRecordId(package_python_schedule.base, 1), 0.002, "message 3") - database.logs_delete(package_ahriman.base) + database.logs_remove(package_ahriman.base, 1) + assert database.logs_get(package_ahriman.base) == "message 1" + assert database.logs_get(package_python_schedule.base) == "message 3" + + +def test_logs_insert_remove_full(database: SQLite, package_ahriman: Package, package_python_schedule: Package) -> None: + """ + must clear full package logs + """ + database.logs_insert(LogRecordId(package_ahriman.base, 1), 0.001, "message 1") + database.logs_insert(LogRecordId(package_ahriman.base, 2), 0.001, "message 2") + database.logs_insert(LogRecordId(package_python_schedule.base, 1), 0.002, "message 3") + + database.logs_remove(package_ahriman.base, None) assert not database.logs_get(package_ahriman.base) - assert database.logs_get(package_python_schedule.base) + assert database.logs_get(package_python_schedule.base) == "message 3" def test_logs_insert_get(database: SQLite, package_ahriman: Package) -> None: """ must insert and get package logs """ - database.logs_insert(package_ahriman.base, 0.002, "message 2") - database.logs_insert(package_ahriman.base, 0.001, "message 1") + database.logs_insert(LogRecordId(package_ahriman.base, 1), 0.002, "message 2") + database.logs_insert(LogRecordId(package_ahriman.base, 1), 0.001, "message 1") assert database.logs_get(package_ahriman.base) == "message 1\nmessage 2" diff --git a/tests/ahriman/core/status/test_watcher.py b/tests/ahriman/core/status/test_watcher.py index 1eada003..a35b9b12 100644 --- a/tests/ahriman/core/status/test_watcher.py +++ b/tests/ahriman/core/status/test_watcher.py @@ -83,20 +83,22 @@ def test_remove(watcher: Watcher, package_ahriman: Package, mocker: MockerFixtur must remove package base """ cache_mock = mocker.patch("ahriman.core.database.SQLite.package_remove") + logs_mock = mocker.patch("ahriman.core.status.watcher.Watcher.remove_logs") watcher.known = {package_ahriman.base: (package_ahriman, BuildStatus())} watcher.remove(package_ahriman.base) assert not watcher.known cache_mock.assert_called_once_with(package_ahriman.base) + logs_mock.assert_called_once_with(package_ahriman.base, None) def test_remove_logs(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: """ must remove package logs """ - logs_mock = mocker.patch("ahriman.core.database.SQLite.logs_delete") - watcher.remove_logs(package_ahriman.base) - logs_mock.assert_called_once_with(package_ahriman.base) + logs_mock = mocker.patch("ahriman.core.database.SQLite.logs_remove") + watcher.remove_logs(package_ahriman.base, 42) + logs_mock.assert_called_once_with(package_ahriman.base, 42) def test_remove_unknown(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: @@ -148,15 +150,15 @@ def test_update_logs_new(watcher: Watcher, package_ahriman: Package, mocker: Moc """ must create package logs record for new package """ - delete_mock = mocker.patch("ahriman.core.database.SQLite.logs_delete") + delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.remove_logs") insert_mock = mocker.patch("ahriman.core.database.SQLite.logs_insert") log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.process_id) assert watcher._last_log_record_id != log_record_id watcher.update_logs(log_record_id, 42.01, "log record") - delete_mock.assert_called_once_with(package_ahriman.base) - insert_mock.assert_called_once_with(package_ahriman.base, 42.01, "log record") + delete_mock.assert_called_once_with(package_ahriman.base, log_record_id.process_id) + insert_mock.assert_called_once_with(log_record_id, 42.01, "log record") assert watcher._last_log_record_id == log_record_id @@ -165,7 +167,7 @@ def test_update_logs_update(watcher: Watcher, package_ahriman: Package, mocker: """ must create package logs record for current package """ - delete_mock = mocker.patch("ahriman.core.database.SQLite.logs_delete") + delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.remove_logs") insert_mock = mocker.patch("ahriman.core.database.SQLite.logs_insert") log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.process_id) @@ -173,7 +175,7 @@ def test_update_logs_update(watcher: Watcher, package_ahriman: Package, mocker: watcher.update_logs(log_record_id, 42.01, "log record") delete_mock.assert_not_called() - insert_mock.assert_called_once_with(package_ahriman.base, 42.01, "log record") + insert_mock.assert_called_once_with(log_record_id, 42.01, "log record") def test_update_self(watcher: Watcher) -> None: