feat: allow to run daemon mode with split packages check (#120)

docs/ahriman.application.application.rst

@@ -44,6 +44,14 @@ ahriman.application.application.application\_repository module
    :no-undoc-members:
    :show-inheritance:
 
+ahriman.application.application.updates\_iterator module
+--------------------------------------------------------
+
+.. automodule:: ahriman.application.application.updates_iterator
+   :members:
+   :no-undoc-members:
+   :show-inheritance:
+
 Module contents
 ---------------
 

docs/ahriman.web.schemas.rst

@@ -60,6 +60,14 @@ ahriman.web.schemas.file\_schema module
    :no-undoc-members:
    :show-inheritance:
 
+ahriman.web.schemas.info\_schema module
+---------------------------------------
+
+.. automodule:: ahriman.web.schemas.info_schema
+   :members:
+   :no-undoc-members:
+   :show-inheritance:
+
 ahriman.web.schemas.internal\_status\_schema module
 ---------------------------------------------------
 

docs/ahriman.web.views.v1.status.rst

@@ -12,6 +12,14 @@ ahriman.web.views.v1.status.changes module
    :no-undoc-members:
    :show-inheritance:
 
+ahriman.web.views.v1.status.info module
+---------------------------------------
+
+.. automodule:: ahriman.web.views.v1.status.info
+   :members:
+   :no-undoc-members:
+   :show-inheritance:
+
 ahriman.web.views.v1.status.logs module
 ---------------------------------------
 

package/share/bash-completion/completions/_ahriman

@@ -31,8 +31,8 @@ _shtab_ahriman_repo_check_option_strings=('-h' '--help' '--changes' '--no-change
 _shtab_ahriman_check_option_strings=('-h' '--help' '--changes' '--no-changes' '-e' '--exit-code' '--vcs' '--no-vcs' '-y' '--refresh')
 _shtab_ahriman_repo_create_keyring_option_strings=('-h' '--help')
 _shtab_ahriman_repo_create_mirrorlist_option_strings=('-h' '--help')
-_shtab_ahriman_repo_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--changes' '--no-changes' '--dependencies' '--no-dependencies' '--dry-run' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh')
+_shtab_ahriman_repo_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--changes' '--no-changes' '--dependencies' '--no-dependencies' '--dry-run' '--local' '--no-local' '--manual' '--no-manual' '--partitions' '--no-partitions' '--vcs' '--no-vcs' '-y' '--refresh')
-_shtab_ahriman_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--changes' '--no-changes' '--dependencies' '--no-dependencies' '--dry-run' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh')
+_shtab_ahriman_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--changes' '--no-changes' '--dependencies' '--no-dependencies' '--dry-run' '--local' '--no-local' '--manual' '--no-manual' '--partitions' '--no-partitions' '--vcs' '--no-vcs' '-y' '--refresh')
 _shtab_ahriman_repo_rebuild_option_strings=('-h' '--help' '--depends-on' '--dry-run' '--from-database' '--increment' '--no-increment' '-e' '--exit-code' '-s' '--status' '-u' '--username')
 _shtab_ahriman_rebuild_option_strings=('-h' '--help' '--depends-on' '--dry-run' '--from-database' '--increment' '--no-increment' '-e' '--exit-code' '-s' '--status' '-u' '--username')
 _shtab_ahriman_repo_remove_unknown_option_strings=('-h' '--help' '--dry-run')
@@ -277,6 +277,8 @@ _shtab_ahriman_repo_daemon___local_nargs=0
 _shtab_ahriman_repo_daemon___no_local_nargs=0
 _shtab_ahriman_repo_daemon___manual_nargs=0
 _shtab_ahriman_repo_daemon___no_manual_nargs=0
+_shtab_ahriman_repo_daemon___partitions_nargs=0
+_shtab_ahriman_repo_daemon___no_partitions_nargs=0
 _shtab_ahriman_repo_daemon___vcs_nargs=0
 _shtab_ahriman_repo_daemon___no_vcs_nargs=0
 _shtab_ahriman_repo_daemon__y_nargs=0
@@ -294,6 +296,8 @@ _shtab_ahriman_daemon___local_nargs=0
 _shtab_ahriman_daemon___no_local_nargs=0
 _shtab_ahriman_daemon___manual_nargs=0
 _shtab_ahriman_daemon___no_manual_nargs=0
+_shtab_ahriman_daemon___partitions_nargs=0
+_shtab_ahriman_daemon___no_partitions_nargs=0
 _shtab_ahriman_daemon___vcs_nargs=0
 _shtab_ahriman_daemon___no_vcs_nargs=0
 _shtab_ahriman_daemon__y_nargs=0

package/share/man/man1/ahriman.1

@@ -1,4 +1,4 @@
-.TH AHRIMAN "1" "2023\-12\-08" "ahriman" "Generated Python Manual"
+.TH AHRIMAN "1" "2023\-12\-26" "ahriman" "Generated Python Manual"
 .SH NAME
 ahriman
 .SH SYNOPSIS
@@ -485,7 +485,7 @@ create package which contains list of available mirrors as set by configuration.
 .SH COMMAND \fI\,'ahriman repo\-daemon'\/\fR
 usage: ahriman repo\-daemon [\-h] [\-i INTERVAL] [\-\-aur | \-\-no\-aur] [\-\-changes | \-\-no\-changes]
                            [\-\-dependencies | \-\-no\-dependencies] [\-\-dry\-run] [\-\-local | \-\-no\-local]
-                           [\-\-manual | \-\-no\-manual] [\-\-vcs | \-\-no\-vcs] [\-y]
+                           [\-\-manual | \-\-no\-manual] [\-\-partitions | \-\-no\-partitions] [\-\-vcs | \-\-no\-vcs] [\-y]
 
 start process which periodically will run update process
 
@@ -518,6 +518,10 @@ enable or disable checking of local packages for updates
 \fB\-\-manual\fR, \fB\-\-no\-manual\fR
 include or exclude manual updates
 
+.TP
+\fB\-\-partitions\fR, \fB\-\-no\-partitions\fR
+instead of updating whole repository, split updates into chunks
+
 .TP
 \fB\-\-vcs\fR, \fB\-\-no\-vcs\fR
 fetch actual version of VCS packages

package/share/zsh/site-functions/_ahriman

@@ -157,6 +157,7 @@ _shtab_ahriman_daemon_options=(
   "--dry-run[just perform check for updates, same as check command (default\: False)]"
   {--local,--no-local}"[enable or disable checking of local packages for updates (default\: True)]:local:"
   {--manual,--no-manual}"[include or exclude manual updates (default\: True)]:manual:"
+  {--partitions,--no-partitions}"[instead of updating whole repository, split updates into chunks (default\: True)]:partitions:"
   {--vcs,--no-vcs}"[fetch actual version of VCS packages (default\: True)]:vcs:"
   "*"{-y,--refresh}"[download fresh package databases from the mirror before actions, -yy to force refresh even if up to date (default\: False)]"
 )
@@ -364,6 +365,7 @@ _shtab_ahriman_repo_daemon_options=(
   "--dry-run[just perform check for updates, same as check command (default\: False)]"
   {--local,--no-local}"[enable or disable checking of local packages for updates (default\: True)]:local:"
   {--manual,--no-manual}"[include or exclude manual updates (default\: True)]:manual:"
+  {--partitions,--no-partitions}"[instead of updating whole repository, split updates into chunks (default\: True)]:partitions:"
   {--vcs,--no-vcs}"[fetch actual version of VCS packages (default\: True)]:vcs:"
   "*"{-y,--refresh}"[download fresh package databases from the mirror before actions, -yy to force refresh even if up to date (default\: False)]"
 )

src/ahriman/application/ahriman.py

@@ -607,16 +607,22 @@ def _set_repo_daemon_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser.add_argument("--dependencies", help="process missing package dependencies",
                         action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("--dry-run", help="just perform check for updates, same as check command", action="store_true")
+    parser.add_argument("--increment", help="increment package release (pkgrel) on duplicate",
+                        action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("--local", help="enable or disable checking of local packages for updates",
                         action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("--manual", help="include or exclude manual updates",
                         action=argparse.BooleanOptionalAction, default=True)
+    parser.add_argument("--partitions", help="instead of updating whole repository, split updates into chunks",
+                        action=argparse.BooleanOptionalAction, default=True)
+    parser.add_argument("-u", "--username", help="build as user", default=extract_user())
     parser.add_argument("--vcs", help="fetch actual version of VCS packages",
                         action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("-y", "--refresh", help="download fresh package databases from the mirror before actions, "
                                                 "-yy to force refresh even if up to date",
                         action="count", default=False)
-    parser.set_defaults(handler=handlers.Daemon, exit_code=False, package=[])
+    parser.set_defaults(handler=handlers.Daemon, exit_code=False,
+                        lock=Path(tempfile.gettempdir()) / "ahriman-daemon.lock", package=[])
     return parser
 
 

src/ahriman/application/application/updates_iterator.py

@@ -0,0 +1,133 @@
+#
+# Copyright (c) 2021-2023 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import time
+
+from collections.abc import Iterator
+from typing import Self
+
+from ahriman.application.application import Application
+from ahriman.core.tree import Tree
+
+
+class UpdatesIterator(Iterator[list[str] | None]):
+    """
+    class-helper for iteration over packages to check for updates. It yields list of packages which were not yet
+    updated
+
+    Attributes:
+        application(Application): application instance
+        interval(int): predefined interval for updates. The updates will be split into chunks in the way in which all
+            packages will be updated in the specified interval
+        updated_packages(set[str]): list of packages which have been already updated
+
+    Examples:
+        Typical usage of this class is something like:
+
+            >>> application = ...
+            >>> iterator = UpdatesIterator(application, None)
+            >>>
+            >>> for updates in iterator:
+            >>>     print(updates)
+    """
+
+    def __init__(self, application: Application, interval: int) -> None:
+        """
+        default constructor
+
+        Args:
+            application(Application): application instance
+            interval(int): predefined interval for updates
+        """
+        self.application = application
+        self.interval = interval
+
+        self.updated_packages: set[str] = set()
+
+    def select_packages(self) -> tuple[list[str] | None, int]:
+        """
+        select next packages partition for updates
+
+        Returns:
+            tuple[list[str] | None, int]: packages partition for updates if any and total amount of partitions.
+        """
+        packages = self.application.repository.packages()
+        if not packages:  # empty repository case
+            return None, 1
+
+        # split packages to the maximal available amount of chunks
+        partitions = Tree.partition(packages, count=len(packages))
+        frequency = len(partitions)  # must be always not-empty
+
+        for partition in partitions:
+            bases = [package.base for package in partition]
+            # check if all packages from this partition have been already updated
+            if self.updated_packages.issuperset(bases):
+                continue
+            # there are packages which were not checked yet, return them
+            return bases, frequency
+
+        # in this case there is nothing to update or repository is empty
+        self.updated_packages.clear()
+
+        # extract bases from the first partition and return them
+        bases = [package.base for package in next(iter(partitions))]
+        return bases, frequency
+
+    def __iter__(self) -> Self:
+        """
+        base iterator method
+
+        Returns:
+            Self: iterator instance
+        """
+        return self
+
+    def __next__(self) -> list[str] | None:
+        """
+        retrieve next element in the iterator. This method will delay result for the amount of time equals
+        :attr:`interval` divided by the amount of chunks
+
+        Returns:
+            list[str] | None: next packages chunk to be updated. ``None`` means no updates
+        """
+        to_update, frequency = self.select_packages()
+        if to_update is not None:
+            # update cached built packages
+            self.updated_packages.update(to_update)
+
+        # wait for update before emit
+        time.sleep(self.interval / frequency)
+
+        return to_update
+
+
+class FixedUpdatesIterator(UpdatesIterator):
+    """
+    implementation of the :class:`UpdatesIterator` which always emits empty list, which is the same as update all
+    """
+
+    def select_packages(self) -> tuple[list[str] | None, int]:
+        """
+        select next packages partition for updates
+
+        Returns:
+            tuple[list[str] | None, int]: packages partition for updates if any and total amount of partitions.
+        """
+        return [], 1

src/ahriman/application/handlers/daemon.py

@@ -18,8 +18,9 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import argparse
-import threading
 
+from ahriman.application.application import Application
+from ahriman.application.application.updates_iterator import FixedUpdatesIterator, UpdatesIterator
 from ahriman.application.handlers import Handler
 from ahriman.core.configuration import Configuration
 from ahriman.models.repository_id import RepositoryId
@@ -44,9 +45,15 @@ class Daemon(Handler):
         """
         from ahriman.application.handlers import Update
 
-        event = threading.Event()
+        application = Application(repository_id, configuration, report=report, refresh_pacman_database=args.refresh)
-        try:
+        if args.partitions:
-            while not event.wait(args.interval):
+            iterator = UpdatesIterator(application, args.interval)
+        else:
+            iterator = FixedUpdatesIterator(application, args.interval)
+
+        for packages in iterator:
+            if packages is None:
+                continue  # nothing to check case
+
+            args.package = packages
             Update.run(args, repository_id, configuration, report=report)
-        except KeyboardInterrupt:
-            pass  # normal exit

tests/ahriman/application/application/conftest.py

@@ -2,9 +2,11 @@ import pytest
 
 from pytest_mock import MockerFixture
 
+from ahriman.application.application import Application
 from ahriman.application.application.application_packages import ApplicationPackages
 from ahriman.application.application.application_properties import ApplicationProperties
 from ahriman.application.application.application_repository import ApplicationRepository
+from ahriman.application.application.updates_iterator import FixedUpdatesIterator, UpdatesIterator
 from ahriman.core.configuration import Configuration
 from ahriman.core.database import SQLite
 from ahriman.core.repository import Repository
@@ -71,3 +73,31 @@ def application_repository(configuration: Configuration, database: SQLite, repos
     mocker.patch("ahriman.core.database.SQLite.load", return_value=database)
     _, repository_id = configuration.check_loaded()
     return ApplicationRepository(repository_id, configuration, report=False)
+
+
+@pytest.fixture
+def fixed_updates_iterator(application: Application) -> FixedUpdatesIterator:
+    """
+    fixture for fixed updates iterator
+
+    Args:
+        application(Application): application fixture
+
+    Returns:
+        FixedUpdatesIterator: fixed updates iterator test instance
+    """
+    return FixedUpdatesIterator(application, 1)
+
+
+@pytest.fixture
+def updates_iterator(application: Application) -> UpdatesIterator:
+    """
+    fixture for chunk bases updates iterator
+
+    Args:
+        application(Application): application fixture
+
+    Returns:
+        UpdatesIterator: updates iterator test instance
+    """
+    return UpdatesIterator(application, 1)

tests/ahriman/application/application/test_updates_iterator.py

@@ -0,0 +1,74 @@
+from pytest_mock import MockerFixture
+from unittest.mock import call as MockCall
+
+from ahriman.application.application.updates_iterator import FixedUpdatesIterator, UpdatesIterator
+from ahriman.models.package import Package
+
+
+def test_select_packages(updates_iterator: UpdatesIterator, package_ahriman: Package,
+                         package_python_schedule: Package, mocker: MockerFixture) -> None:
+    """
+    must return next partition
+    """
+    mocker.patch("ahriman.core.repository.Repository.packages",
+                 return_value=[package_ahriman, package_python_schedule])
+
+    assert updates_iterator.select_packages() == ([package_python_schedule.base], 2)
+    assert updates_iterator.select_packages() == ([package_python_schedule.base], 2)
+
+
+def test_select_packages_empty(updates_iterator: UpdatesIterator, mocker: MockerFixture) -> None:
+    """
+    must return None for empty repository
+    """
+    mocker.patch("ahriman.core.repository.Repository.packages", return_value=[])
+    assert updates_iterator.select_packages() == (None, 1)
+
+
+def test_select_packages_cycle(updates_iterator: UpdatesIterator, package_ahriman: Package,
+                               package_python_schedule: Package, mocker: MockerFixture) -> None:
+    """
+    must cycle over partitions
+    """
+    mocker.patch("ahriman.core.repository.Repository.packages",
+                 return_value=[package_ahriman, package_python_schedule])
+
+    assert updates_iterator.select_packages() == ([package_python_schedule.base], 2)
+    updates_iterator.updated_packages.add(package_python_schedule.base)
+
+    assert updates_iterator.select_packages() == ([package_ahriman.base], 2)
+    updates_iterator.updated_packages.add(package_ahriman.base)
+
+    assert updates_iterator.select_packages() == ([package_python_schedule.base], 2)
+    assert not updates_iterator.updated_packages
+
+
+def test_iter(updates_iterator: UpdatesIterator) -> None:
+    """
+    must return self as iterator
+    """
+    assert updates_iterator.__iter__() == updates_iterator
+
+
+def test_next(updates_iterator: UpdatesIterator, package_ahriman: Package, mocker: MockerFixture) -> None:
+    """
+    must return next chunk to update
+    """
+    mocker.patch("ahriman.application.application.updates_iterator.UpdatesIterator.select_packages",
+                 side_effect=[([package_ahriman.base], 2), (None, 2), StopIteration])
+    sleep_mock = mocker.patch("time.sleep")
+
+    updates = [packages for packages in updates_iterator]
+    assert updates == [[package_ahriman.base], None]
+    sleep_mock.assert_has_calls([MockCall(0.5), MockCall(0.5)])
+
+
+def test_fixed_updates_iterator(fixed_updates_iterator: FixedUpdatesIterator, package_ahriman: Package,
+                                mocker: MockerFixture) -> None:
+    """
+    must always return empty package list
+    """
+    assert fixed_updates_iterator.select_packages() == ([], 1)
+
+    mocker.patch("ahriman.core.repository.Repository.packages", return_value=[package_ahriman])
+    assert fixed_updates_iterator.select_packages() == ([], 1)

tests/ahriman/application/handlers/test_handler_daemon.py

@@ -1,10 +1,11 @@
 import argparse
 
 from pytest_mock import MockerFixture
-from unittest.mock import call as MockCall
 
 from ahriman.application.handlers import Daemon
 from ahriman.core.configuration import Configuration
+from ahriman.core.repository import Repository
+from ahriman.models.package import Package
 
 
 def _default_args(args: argparse.Namespace) -> argparse.Namespace:
@@ -18,35 +19,60 @@ def _default_args(args: argparse.Namespace) -> argparse.Namespace:
         argparse.Namespace: generated arguments for these test cases
     """
     args.interval = 60 * 60 * 12
-    args.aur = True
+    args.partitions = True
-    args.local = True
+    args.refresh = 0
-    args.manual = True
-    args.vcs = True
     return args
 
 
-def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
+def test_run(args: argparse.Namespace, configuration: Configuration, package_ahriman: Package, repository: Repository,
+             mocker: MockerFixture) -> None:
     """
     must run command
     """
     args = _default_args(args)
+    mocker.patch("ahriman.core.repository.Repository.load", return_value=repository)
     run_mock = mocker.patch("ahriman.application.handlers.Update.run")
-    wait_mock = mocker.patch("threading.Event.wait", side_effect=[False, True])
+    iter_mock = mocker.patch("ahriman.application.application.updates_iterator.UpdatesIterator.__iter__",
+                             return_value=iter([[package_ahriman.base]]))
+
+    _, repository_id = configuration.check_loaded()
+    Daemon.run(args, repository_id, configuration, report=True)
+    args.package = [package_ahriman.base]
+    run_mock.assert_called_once_with(args, repository_id, configuration, report=True)
+    iter_mock.assert_called_once_with()
+
+
+def test_run_no_partitions(args: argparse.Namespace, configuration: Configuration, repository: Repository,
+                           mocker: MockerFixture) -> None:
+    """
+    must run command without partitioning
+    """
+    args = _default_args(args)
+    args.partitions = False
+    mocker.patch("ahriman.core.repository.Repository.load", return_value=repository)
+    run_mock = mocker.patch("ahriman.application.handlers.Update.run")
+    iter_mock = mocker.patch("ahriman.application.application.updates_iterator.UpdatesIterator.__iter__",
+                             return_value=iter([[]]))
 
     _, repository_id = configuration.check_loaded()
     Daemon.run(args, repository_id, configuration, report=True)
     run_mock.assert_called_once_with(args, repository_id, configuration, report=True)
-    wait_mock.assert_has_calls([MockCall(args.interval), MockCall(args.interval)])
+    iter_mock.assert_called_once_with()
 
 
-def test_run_keyboard_interrupt(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
+def test_run_no_updates(args: argparse.Namespace, configuration: Configuration, package_ahriman: Package,
+                        repository: Repository, mocker: MockerFixture) -> None:
     """
-    must handle KeyboardInterrupt exception
+    must skip empty update list
     """
     args = _default_args(args)
-    mocker.patch("ahriman.application.handlers.Update.run", side_effect=KeyboardInterrupt)
+    mocker.patch("ahriman.core.repository.Repository.load", return_value=repository)
-    wait_mock = mocker.patch("threading.Event.wait", return_value=False)
+    run_mock = mocker.patch("ahriman.application.handlers.Update.run")
+    iter_mock = mocker.patch("ahriman.application.application.updates_iterator.UpdatesIterator.__iter__",
+                             return_value=iter([[package_ahriman.base], None]))
 
     _, repository_id = configuration.check_loaded()
     Daemon.run(args, repository_id, configuration, report=True)
-    wait_mock.assert_called_once_with(args.interval)
+    args.package = [package_ahriman.base]
+    run_mock.assert_called_once_with(args, repository_id, configuration, report=True)
+    iter_mock.assert_called_once_with()