handle process id during removal

During one process we can write logs from different packages in
different times (e.g. check and update laster) and we would like
to store all logs belong to the same process

src/ahriman/core/database/migrations/m004_logs.py

@@ -24,11 +24,12 @@ steps = [
     """
     create table logs (
         package_base text not null,
+        process_id integer not null,
         created real not null,
         record text
     )
     """,
     """
-    create index logs_package_base on logs (package_base)
+    create index logs_package_base_process_id on logs (package_base, process_id)
     """,
 ]

src/ahriman/core/database/operations/logs_operations.py

@@ -18,9 +18,10 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from sqlite3 import Connection
-from typing import List
+from typing import List, Optional
 
 from ahriman.core.database.operations import Operations
+from ahriman.models.log_record_id import LogRecordId
 
 
 class LogsOperations(Operations):
@@ -28,21 +29,6 @@ class LogsOperations(Operations):
     logs operations
     """
 
-    def logs_delete(self, package_base: str) -> None:
-        """
-        delete log records for the specified package
-
-        Args:
-            package_base(str): package base to remove logs
-        """
-        def run(connection: Connection) -> None:
-            connection.execute(
-                """delete from logs where package_base = :package_base""",
-                {"package_base": package_base}
-            )
-
-        return self.with_connection(run, commit=True)
-
     def logs_get(self, package_base: str) -> str:
         """
         extract logs for specified package base
@@ -67,12 +53,12 @@ class LogsOperations(Operations):
         records = self.with_connection(run)
         return "\n".join(records)
 
-    def logs_insert(self, package_base: str, created: float, record: str) -> None:
+    def logs_insert(self, log_record_id: LogRecordId, created: float, record: str) -> None:
         """
         write new log record to database
 
         Args:
-            package_base(str): package base
+            log_record_id(LogRecordId): current log record id
             created(float): log created timestamp from log record attribute
             record(str): log record
         """
@@ -80,15 +66,36 @@ class LogsOperations(Operations):
             connection.execute(
                 """
                 insert into logs
-                (package_base, created, record)
+                (package_base, process_id, created, record)
                 values
-                (:package_base, :created, :record)
+                (:package_base, :process_id, :created, :record)
                 """,
                 dict(
-                    package_base=package_base,
+                    package_base=log_record_id.package_base,
+                    process_id=log_record_id.process_id,
                     created=created,
                     record=record
                 )
             )
 
         return self.with_connection(run, commit=True)
+
+    def logs_remove(self, package_base: str, current_process_id: Optional[int]) -> None:
+        """
+        remove log records for the specified package
+
+        Args:
+            package_base(str): package base to remove logs
+            current_process_id(Optional[int]): current process id. If set it will remove only logs belonging to another
+                process
+        """
+        def run(connection: Connection) -> None:
+            connection.execute(
+                """
+                delete from logs
+                where package_base = :package_base and (:process_id is null or process_id <> :process_id)
+                """,
+                {"package_base": package_base, "process_id": current_process_id}
+            )
+
+        return self.with_connection(run, commit=True)

src/ahriman/core/status/watcher.py

@@ -128,15 +128,17 @@ class Watcher(LazyLogging):
         """
         self.known.pop(package_base, None)
         self.database.package_remove(package_base)
+        self.remove_logs(package_base, None)
 
-    def remove_logs(self, package_base: str) -> None:
+    def remove_logs(self, package_base: str, current_process_id: Optional[int]) -> None:
         """
         remove package related logs
 
         Args:
             package_base(str): package base
+            current_process_id(int): current process id
         """
-        self.database.logs_delete(package_base)
+        self.database.logs_remove(package_base, current_process_id)
 
     def update(self, package_base: str, status: BuildStatusEnum, package: Optional[Package]) -> None:
         """
@@ -170,9 +172,9 @@ class Watcher(LazyLogging):
         """
         if self._last_log_record_id != log_record_id:
             # there is new log record, so we remove old ones
-            self.database.logs_delete(log_record_id.package_base)
+            self.remove_logs(log_record_id.package_base, log_record_id.process_id)
         self._last_log_record_id = log_record_id
-        self.database.logs_insert(log_record_id.package_base, created, record)
+        self.database.logs_insert(log_record_id, created, record)
 
     def update_self(self, status: BuildStatusEnum) -> None:
         """

src/ahriman/web/views/status/logs.py

@@ -46,7 +46,7 @@ class LogsView(BaseView):
             HTTPNoContent: on success response
         """
         package_base = self.request.match_info["package"]
-        self.service.remove_logs(package_base)
+        self.service.remove_logs(package_base, None)
 
         raise HTTPNoContent()
 

tests/ahriman/core/database/operations/test_logs_operations.py

@@ -1,23 +1,39 @@
 from ahriman.core.database import SQLite
+from ahriman.models.log_record_id import LogRecordId
 from ahriman.models.package import Package
 
 
-def test_logs_insert_delete(database: SQLite, package_ahriman: Package, package_python_schedule: Package) -> None:
+def test_logs_insert_remove_process(database: SQLite, package_ahriman: Package,
+                                    package_python_schedule: Package) -> None:
     """
-    must clear all packages
+    must clear process specific package logs
     """
-    database.logs_insert(package_ahriman.base, 0.001, "message 1")
+    database.logs_insert(LogRecordId(package_ahriman.base, 1), 0.001, "message 1")
-    database.logs_insert(package_python_schedule.base, 0.002, "message 2")
+    database.logs_insert(LogRecordId(package_ahriman.base, 2), 0.001, "message 2")
+    database.logs_insert(LogRecordId(package_python_schedule.base, 1), 0.002, "message 3")
 
-    database.logs_delete(package_ahriman.base)
+    database.logs_remove(package_ahriman.base, 1)
+    assert database.logs_get(package_ahriman.base) == "message 1"
+    assert database.logs_get(package_python_schedule.base) == "message 3"
+
+
+def test_logs_insert_remove_full(database: SQLite, package_ahriman: Package, package_python_schedule: Package) -> None:
+    """
+    must clear full package logs
+    """
+    database.logs_insert(LogRecordId(package_ahriman.base, 1), 0.001, "message 1")
+    database.logs_insert(LogRecordId(package_ahriman.base, 2), 0.001, "message 2")
+    database.logs_insert(LogRecordId(package_python_schedule.base, 1), 0.002, "message 3")
+
+    database.logs_remove(package_ahriman.base, None)
     assert not database.logs_get(package_ahriman.base)
-    assert database.logs_get(package_python_schedule.base)
+    assert database.logs_get(package_python_schedule.base) == "message 3"
 
 
 def test_logs_insert_get(database: SQLite, package_ahriman: Package) -> None:
     """
     must insert and get package logs
     """
-    database.logs_insert(package_ahriman.base, 0.002, "message 2")
+    database.logs_insert(LogRecordId(package_ahriman.base, 1), 0.002, "message 2")
-    database.logs_insert(package_ahriman.base, 0.001, "message 1")
+    database.logs_insert(LogRecordId(package_ahriman.base, 1), 0.001, "message 1")
     assert database.logs_get(package_ahriman.base) == "message 1\nmessage 2"

tests/ahriman/core/status/test_watcher.py

@@ -83,20 +83,22 @@ def test_remove(watcher: Watcher, package_ahriman: Package, mocker: MockerFixtur
     must remove package base
     """
     cache_mock = mocker.patch("ahriman.core.database.SQLite.package_remove")
+    logs_mock = mocker.patch("ahriman.core.status.watcher.Watcher.remove_logs")
     watcher.known = {package_ahriman.base: (package_ahriman, BuildStatus())}
 
     watcher.remove(package_ahriman.base)
     assert not watcher.known
     cache_mock.assert_called_once_with(package_ahriman.base)
+    logs_mock.assert_called_once_with(package_ahriman.base, None)
 
 
 def test_remove_logs(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
     """
     must remove package logs
     """
-    logs_mock = mocker.patch("ahriman.core.database.SQLite.logs_delete")
+    logs_mock = mocker.patch("ahriman.core.database.SQLite.logs_remove")
-    watcher.remove_logs(package_ahriman.base)
+    watcher.remove_logs(package_ahriman.base, 42)
-    logs_mock.assert_called_once_with(package_ahriman.base)
+    logs_mock.assert_called_once_with(package_ahriman.base, 42)
 
 
 def test_remove_unknown(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None:
@@ -148,15 +150,15 @@ def test_update_logs_new(watcher: Watcher, package_ahriman: Package, mocker: Moc
     """
     must create package logs record for new package
     """
-    delete_mock = mocker.patch("ahriman.core.database.SQLite.logs_delete")
+    delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.remove_logs")
     insert_mock = mocker.patch("ahriman.core.database.SQLite.logs_insert")
 
     log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.process_id)
     assert watcher._last_log_record_id != log_record_id
 
     watcher.update_logs(log_record_id, 42.01, "log record")
-    delete_mock.assert_called_once_with(package_ahriman.base)
+    delete_mock.assert_called_once_with(package_ahriman.base, log_record_id.process_id)
-    insert_mock.assert_called_once_with(package_ahriman.base, 42.01, "log record")
+    insert_mock.assert_called_once_with(log_record_id, 42.01, "log record")
 
     assert watcher._last_log_record_id == log_record_id
 
@@ -165,7 +167,7 @@ def test_update_logs_update(watcher: Watcher, package_ahriman: Package, mocker:
     """
     must create package logs record for current package
     """
-    delete_mock = mocker.patch("ahriman.core.database.SQLite.logs_delete")
+    delete_mock = mocker.patch("ahriman.core.status.watcher.Watcher.remove_logs")
     insert_mock = mocker.patch("ahriman.core.database.SQLite.logs_insert")
 
     log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.process_id)
@@ -173,7 +175,7 @@ def test_update_logs_update(watcher: Watcher, package_ahriman: Package, mocker:
 
     watcher.update_logs(log_record_id, 42.01, "log record")
     delete_mock.assert_not_called()
-    insert_mock.assert_called_once_with(package_ahriman.base, 42.01, "log record")
+    insert_mock.assert_called_once_with(log_record_id, 42.01, "log record")
 
 
 def test_update_self(watcher: Watcher) -> None: