From b4fa10781b180332d5a9ba973382f0faf965f78b Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Wed, 27 Dec 2023 03:05:44 +0200 Subject: [PATCH] feat: allow to run daemon mode with split packages check (#120) --- docs/ahriman.application.application.rst | 8 ++ docs/ahriman.web.schemas.rst | 8 ++ docs/ahriman.web.views.v1.status.rst | 8 ++ .../bash-completion/completions/_ahriman | 8 +- package/share/man/man1/ahriman.1 | 8 +- package/share/zsh/site-functions/_ahriman | 2 + src/ahriman/application/ahriman.py | 8 +- .../application/updates_iterator.py | 133 ++++++++++++++++++ src/ahriman/application/handlers/daemon.py | 21 ++- .../application/application/conftest.py | 30 ++++ .../application/test_updates_iterator.py | 74 ++++++++++ .../handlers/test_handler_daemon.py | 52 +++++-- 12 files changed, 335 insertions(+), 25 deletions(-) create mode 100644 src/ahriman/application/application/updates_iterator.py create mode 100644 tests/ahriman/application/application/test_updates_iterator.py diff --git a/docs/ahriman.application.application.rst b/docs/ahriman.application.application.rst index c3672dde..5a78d134 100644 --- a/docs/ahriman.application.application.rst +++ b/docs/ahriman.application.application.rst @@ -44,6 +44,14 @@ ahriman.application.application.application\_repository module :no-undoc-members: :show-inheritance: +ahriman.application.application.updates\_iterator module +-------------------------------------------------------- + +.. automodule:: ahriman.application.application.updates_iterator + :members: + :no-undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/ahriman.web.schemas.rst b/docs/ahriman.web.schemas.rst index fa3bc8b0..a3b9fcd6 100644 --- a/docs/ahriman.web.schemas.rst +++ b/docs/ahriman.web.schemas.rst @@ -60,6 +60,14 @@ ahriman.web.schemas.file\_schema module :no-undoc-members: :show-inheritance: +ahriman.web.schemas.info\_schema module +--------------------------------------- + +.. automodule:: ahriman.web.schemas.info_schema + :members: + :no-undoc-members: + :show-inheritance: + ahriman.web.schemas.internal\_status\_schema module --------------------------------------------------- diff --git a/docs/ahriman.web.views.v1.status.rst b/docs/ahriman.web.views.v1.status.rst index 54ae2539..e5cced9e 100644 --- a/docs/ahriman.web.views.v1.status.rst +++ b/docs/ahriman.web.views.v1.status.rst @@ -12,6 +12,14 @@ ahriman.web.views.v1.status.changes module :no-undoc-members: :show-inheritance: +ahriman.web.views.v1.status.info module +--------------------------------------- + +.. automodule:: ahriman.web.views.v1.status.info + :members: + :no-undoc-members: + :show-inheritance: + ahriman.web.views.v1.status.logs module --------------------------------------- diff --git a/package/share/bash-completion/completions/_ahriman b/package/share/bash-completion/completions/_ahriman index a62c9ec5..2c9225a5 100644 --- a/package/share/bash-completion/completions/_ahriman +++ b/package/share/bash-completion/completions/_ahriman @@ -31,8 +31,8 @@ _shtab_ahriman_repo_check_option_strings=('-h' '--help' '--changes' '--no-change _shtab_ahriman_check_option_strings=('-h' '--help' '--changes' '--no-changes' '-e' '--exit-code' '--vcs' '--no-vcs' '-y' '--refresh') _shtab_ahriman_repo_create_keyring_option_strings=('-h' '--help') _shtab_ahriman_repo_create_mirrorlist_option_strings=('-h' '--help') -_shtab_ahriman_repo_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--changes' '--no-changes' '--dependencies' '--no-dependencies' '--dry-run' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh') -_shtab_ahriman_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--changes' '--no-changes' '--dependencies' '--no-dependencies' '--dry-run' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh') +_shtab_ahriman_repo_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--changes' '--no-changes' '--dependencies' '--no-dependencies' '--dry-run' '--local' '--no-local' '--manual' '--no-manual' '--partitions' '--no-partitions' '--vcs' '--no-vcs' '-y' '--refresh') +_shtab_ahriman_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--changes' '--no-changes' '--dependencies' '--no-dependencies' '--dry-run' '--local' '--no-local' '--manual' '--no-manual' '--partitions' '--no-partitions' '--vcs' '--no-vcs' '-y' '--refresh') _shtab_ahriman_repo_rebuild_option_strings=('-h' '--help' '--depends-on' '--dry-run' '--from-database' '--increment' '--no-increment' '-e' '--exit-code' '-s' '--status' '-u' '--username') _shtab_ahriman_rebuild_option_strings=('-h' '--help' '--depends-on' '--dry-run' '--from-database' '--increment' '--no-increment' '-e' '--exit-code' '-s' '--status' '-u' '--username') _shtab_ahriman_repo_remove_unknown_option_strings=('-h' '--help' '--dry-run') @@ -277,6 +277,8 @@ _shtab_ahriman_repo_daemon___local_nargs=0 _shtab_ahriman_repo_daemon___no_local_nargs=0 _shtab_ahriman_repo_daemon___manual_nargs=0 _shtab_ahriman_repo_daemon___no_manual_nargs=0 +_shtab_ahriman_repo_daemon___partitions_nargs=0 +_shtab_ahriman_repo_daemon___no_partitions_nargs=0 _shtab_ahriman_repo_daemon___vcs_nargs=0 _shtab_ahriman_repo_daemon___no_vcs_nargs=0 _shtab_ahriman_repo_daemon__y_nargs=0 @@ -294,6 +296,8 @@ _shtab_ahriman_daemon___local_nargs=0 _shtab_ahriman_daemon___no_local_nargs=0 _shtab_ahriman_daemon___manual_nargs=0 _shtab_ahriman_daemon___no_manual_nargs=0 +_shtab_ahriman_daemon___partitions_nargs=0 +_shtab_ahriman_daemon___no_partitions_nargs=0 _shtab_ahriman_daemon___vcs_nargs=0 _shtab_ahriman_daemon___no_vcs_nargs=0 _shtab_ahriman_daemon__y_nargs=0 diff --git a/package/share/man/man1/ahriman.1 b/package/share/man/man1/ahriman.1 index b8f350ab..82bb4b7b 100644 --- a/package/share/man/man1/ahriman.1 +++ b/package/share/man/man1/ahriman.1 @@ -1,4 +1,4 @@ -.TH AHRIMAN "1" "2023\-12\-08" "ahriman" "Generated Python Manual" +.TH AHRIMAN "1" "2023\-12\-26" "ahriman" "Generated Python Manual" .SH NAME ahriman .SH SYNOPSIS @@ -485,7 +485,7 @@ create package which contains list of available mirrors as set by configuration. .SH COMMAND \fI\,'ahriman repo\-daemon'\/\fR usage: ahriman repo\-daemon [\-h] [\-i INTERVAL] [\-\-aur | \-\-no\-aur] [\-\-changes | \-\-no\-changes] [\-\-dependencies | \-\-no\-dependencies] [\-\-dry\-run] [\-\-local | \-\-no\-local] - [\-\-manual | \-\-no\-manual] [\-\-vcs | \-\-no\-vcs] [\-y] + [\-\-manual | \-\-no\-manual] [\-\-partitions | \-\-no\-partitions] [\-\-vcs | \-\-no\-vcs] [\-y] start process which periodically will run update process @@ -518,6 +518,10 @@ enable or disable checking of local packages for updates \fB\-\-manual\fR, \fB\-\-no\-manual\fR include or exclude manual updates +.TP +\fB\-\-partitions\fR, \fB\-\-no\-partitions\fR +instead of updating whole repository, split updates into chunks + .TP \fB\-\-vcs\fR, \fB\-\-no\-vcs\fR fetch actual version of VCS packages diff --git a/package/share/zsh/site-functions/_ahriman b/package/share/zsh/site-functions/_ahriman index bc703b71..79e98171 100644 --- a/package/share/zsh/site-functions/_ahriman +++ b/package/share/zsh/site-functions/_ahriman @@ -157,6 +157,7 @@ _shtab_ahriman_daemon_options=( "--dry-run[just perform check for updates, same as check command (default\: False)]" {--local,--no-local}"[enable or disable checking of local packages for updates (default\: True)]:local:" {--manual,--no-manual}"[include or exclude manual updates (default\: True)]:manual:" + {--partitions,--no-partitions}"[instead of updating whole repository, split updates into chunks (default\: True)]:partitions:" {--vcs,--no-vcs}"[fetch actual version of VCS packages (default\: True)]:vcs:" "*"{-y,--refresh}"[download fresh package databases from the mirror before actions, -yy to force refresh even if up to date (default\: False)]" ) @@ -364,6 +365,7 @@ _shtab_ahriman_repo_daemon_options=( "--dry-run[just perform check for updates, same as check command (default\: False)]" {--local,--no-local}"[enable or disable checking of local packages for updates (default\: True)]:local:" {--manual,--no-manual}"[include or exclude manual updates (default\: True)]:manual:" + {--partitions,--no-partitions}"[instead of updating whole repository, split updates into chunks (default\: True)]:partitions:" {--vcs,--no-vcs}"[fetch actual version of VCS packages (default\: True)]:vcs:" "*"{-y,--refresh}"[download fresh package databases from the mirror before actions, -yy to force refresh even if up to date (default\: False)]" ) diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 9b54fa3c..1865c5d7 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -607,16 +607,22 @@ def _set_repo_daemon_parser(root: SubParserAction) -> argparse.ArgumentParser: parser.add_argument("--dependencies", help="process missing package dependencies", action=argparse.BooleanOptionalAction, default=True) parser.add_argument("--dry-run", help="just perform check for updates, same as check command", action="store_true") + parser.add_argument("--increment", help="increment package release (pkgrel) on duplicate", + action=argparse.BooleanOptionalAction, default=True) parser.add_argument("--local", help="enable or disable checking of local packages for updates", action=argparse.BooleanOptionalAction, default=True) parser.add_argument("--manual", help="include or exclude manual updates", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--partitions", help="instead of updating whole repository, split updates into chunks", + action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("-u", "--username", help="build as user", default=extract_user()) parser.add_argument("--vcs", help="fetch actual version of VCS packages", action=argparse.BooleanOptionalAction, default=True) parser.add_argument("-y", "--refresh", help="download fresh package databases from the mirror before actions, " "-yy to force refresh even if up to date", action="count", default=False) - parser.set_defaults(handler=handlers.Daemon, exit_code=False, package=[]) + parser.set_defaults(handler=handlers.Daemon, exit_code=False, + lock=Path(tempfile.gettempdir()) / "ahriman-daemon.lock", package=[]) return parser diff --git a/src/ahriman/application/application/updates_iterator.py b/src/ahriman/application/application/updates_iterator.py new file mode 100644 index 00000000..e85604bc --- /dev/null +++ b/src/ahriman/application/application/updates_iterator.py @@ -0,0 +1,133 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import time + +from collections.abc import Iterator +from typing import Self + +from ahriman.application.application import Application +from ahriman.core.tree import Tree + + +class UpdatesIterator(Iterator[list[str] | None]): + """ + class-helper for iteration over packages to check for updates. It yields list of packages which were not yet + updated + + Attributes: + application(Application): application instance + interval(int): predefined interval for updates. The updates will be split into chunks in the way in which all + packages will be updated in the specified interval + updated_packages(set[str]): list of packages which have been already updated + + Examples: + Typical usage of this class is something like: + + >>> application = ... + >>> iterator = UpdatesIterator(application, None) + >>> + >>> for updates in iterator: + >>> print(updates) + """ + + def __init__(self, application: Application, interval: int) -> None: + """ + default constructor + + Args: + application(Application): application instance + interval(int): predefined interval for updates + """ + self.application = application + self.interval = interval + + self.updated_packages: set[str] = set() + + def select_packages(self) -> tuple[list[str] | None, int]: + """ + select next packages partition for updates + + Returns: + tuple[list[str] | None, int]: packages partition for updates if any and total amount of partitions. + """ + packages = self.application.repository.packages() + if not packages: # empty repository case + return None, 1 + + # split packages to the maximal available amount of chunks + partitions = Tree.partition(packages, count=len(packages)) + frequency = len(partitions) # must be always not-empty + + for partition in partitions: + bases = [package.base for package in partition] + # check if all packages from this partition have been already updated + if self.updated_packages.issuperset(bases): + continue + # there are packages which were not checked yet, return them + return bases, frequency + + # in this case there is nothing to update or repository is empty + self.updated_packages.clear() + + # extract bases from the first partition and return them + bases = [package.base for package in next(iter(partitions))] + return bases, frequency + + def __iter__(self) -> Self: + """ + base iterator method + + Returns: + Self: iterator instance + """ + return self + + def __next__(self) -> list[str] | None: + """ + retrieve next element in the iterator. This method will delay result for the amount of time equals + :attr:`interval` divided by the amount of chunks + + Returns: + list[str] | None: next packages chunk to be updated. ``None`` means no updates + """ + to_update, frequency = self.select_packages() + if to_update is not None: + # update cached built packages + self.updated_packages.update(to_update) + + # wait for update before emit + time.sleep(self.interval / frequency) + + return to_update + + +class FixedUpdatesIterator(UpdatesIterator): + """ + implementation of the :class:`UpdatesIterator` which always emits empty list, which is the same as update all + """ + + def select_packages(self) -> tuple[list[str] | None, int]: + """ + select next packages partition for updates + + Returns: + tuple[list[str] | None, int]: packages partition for updates if any and total amount of partitions. + """ + return [], 1 diff --git a/src/ahriman/application/handlers/daemon.py b/src/ahriman/application/handlers/daemon.py index 665a8994..0b350cd9 100644 --- a/src/ahriman/application/handlers/daemon.py +++ b/src/ahriman/application/handlers/daemon.py @@ -18,8 +18,9 @@ # along with this program. If not, see . # import argparse -import threading +from ahriman.application.application import Application +from ahriman.application.application.updates_iterator import FixedUpdatesIterator, UpdatesIterator from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.models.repository_id import RepositoryId @@ -44,9 +45,15 @@ class Daemon(Handler): """ from ahriman.application.handlers import Update - event = threading.Event() - try: - while not event.wait(args.interval): - Update.run(args, repository_id, configuration, report=report) - except KeyboardInterrupt: - pass # normal exit + application = Application(repository_id, configuration, report=report, refresh_pacman_database=args.refresh) + if args.partitions: + iterator = UpdatesIterator(application, args.interval) + else: + iterator = FixedUpdatesIterator(application, args.interval) + + for packages in iterator: + if packages is None: + continue # nothing to check case + + args.package = packages + Update.run(args, repository_id, configuration, report=report) diff --git a/tests/ahriman/application/application/conftest.py b/tests/ahriman/application/application/conftest.py index 9b607430..276614a0 100644 --- a/tests/ahriman/application/application/conftest.py +++ b/tests/ahriman/application/application/conftest.py @@ -2,9 +2,11 @@ import pytest from pytest_mock import MockerFixture +from ahriman.application.application import Application from ahriman.application.application.application_packages import ApplicationPackages from ahriman.application.application.application_properties import ApplicationProperties from ahriman.application.application.application_repository import ApplicationRepository +from ahriman.application.application.updates_iterator import FixedUpdatesIterator, UpdatesIterator from ahriman.core.configuration import Configuration from ahriman.core.database import SQLite from ahriman.core.repository import Repository @@ -71,3 +73,31 @@ def application_repository(configuration: Configuration, database: SQLite, repos mocker.patch("ahriman.core.database.SQLite.load", return_value=database) _, repository_id = configuration.check_loaded() return ApplicationRepository(repository_id, configuration, report=False) + + +@pytest.fixture +def fixed_updates_iterator(application: Application) -> FixedUpdatesIterator: + """ + fixture for fixed updates iterator + + Args: + application(Application): application fixture + + Returns: + FixedUpdatesIterator: fixed updates iterator test instance + """ + return FixedUpdatesIterator(application, 1) + + +@pytest.fixture +def updates_iterator(application: Application) -> UpdatesIterator: + """ + fixture for chunk bases updates iterator + + Args: + application(Application): application fixture + + Returns: + UpdatesIterator: updates iterator test instance + """ + return UpdatesIterator(application, 1) diff --git a/tests/ahriman/application/application/test_updates_iterator.py b/tests/ahriman/application/application/test_updates_iterator.py new file mode 100644 index 00000000..7e6bc74b --- /dev/null +++ b/tests/ahriman/application/application/test_updates_iterator.py @@ -0,0 +1,74 @@ +from pytest_mock import MockerFixture +from unittest.mock import call as MockCall + +from ahriman.application.application.updates_iterator import FixedUpdatesIterator, UpdatesIterator +from ahriman.models.package import Package + + +def test_select_packages(updates_iterator: UpdatesIterator, package_ahriman: Package, + package_python_schedule: Package, mocker: MockerFixture) -> None: + """ + must return next partition + """ + mocker.patch("ahriman.core.repository.Repository.packages", + return_value=[package_ahriman, package_python_schedule]) + + assert updates_iterator.select_packages() == ([package_python_schedule.base], 2) + assert updates_iterator.select_packages() == ([package_python_schedule.base], 2) + + +def test_select_packages_empty(updates_iterator: UpdatesIterator, mocker: MockerFixture) -> None: + """ + must return None for empty repository + """ + mocker.patch("ahriman.core.repository.Repository.packages", return_value=[]) + assert updates_iterator.select_packages() == (None, 1) + + +def test_select_packages_cycle(updates_iterator: UpdatesIterator, package_ahriman: Package, + package_python_schedule: Package, mocker: MockerFixture) -> None: + """ + must cycle over partitions + """ + mocker.patch("ahriman.core.repository.Repository.packages", + return_value=[package_ahriman, package_python_schedule]) + + assert updates_iterator.select_packages() == ([package_python_schedule.base], 2) + updates_iterator.updated_packages.add(package_python_schedule.base) + + assert updates_iterator.select_packages() == ([package_ahriman.base], 2) + updates_iterator.updated_packages.add(package_ahriman.base) + + assert updates_iterator.select_packages() == ([package_python_schedule.base], 2) + assert not updates_iterator.updated_packages + + +def test_iter(updates_iterator: UpdatesIterator) -> None: + """ + must return self as iterator + """ + assert updates_iterator.__iter__() == updates_iterator + + +def test_next(updates_iterator: UpdatesIterator, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must return next chunk to update + """ + mocker.patch("ahriman.application.application.updates_iterator.UpdatesIterator.select_packages", + side_effect=[([package_ahriman.base], 2), (None, 2), StopIteration]) + sleep_mock = mocker.patch("time.sleep") + + updates = [packages for packages in updates_iterator] + assert updates == [[package_ahriman.base], None] + sleep_mock.assert_has_calls([MockCall(0.5), MockCall(0.5)]) + + +def test_fixed_updates_iterator(fixed_updates_iterator: FixedUpdatesIterator, package_ahriman: Package, + mocker: MockerFixture) -> None: + """ + must always return empty package list + """ + assert fixed_updates_iterator.select_packages() == ([], 1) + + mocker.patch("ahriman.core.repository.Repository.packages", return_value=[package_ahriman]) + assert fixed_updates_iterator.select_packages() == ([], 1) diff --git a/tests/ahriman/application/handlers/test_handler_daemon.py b/tests/ahriman/application/handlers/test_handler_daemon.py index 8c470690..ee897bea 100644 --- a/tests/ahriman/application/handlers/test_handler_daemon.py +++ b/tests/ahriman/application/handlers/test_handler_daemon.py @@ -1,10 +1,11 @@ import argparse from pytest_mock import MockerFixture -from unittest.mock import call as MockCall from ahriman.application.handlers import Daemon from ahriman.core.configuration import Configuration +from ahriman.core.repository import Repository +from ahriman.models.package import Package def _default_args(args: argparse.Namespace) -> argparse.Namespace: @@ -18,35 +19,60 @@ def _default_args(args: argparse.Namespace) -> argparse.Namespace: argparse.Namespace: generated arguments for these test cases """ args.interval = 60 * 60 * 12 - args.aur = True - args.local = True - args.manual = True - args.vcs = True + args.partitions = True + args.refresh = 0 return args -def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None: +def test_run(args: argparse.Namespace, configuration: Configuration, package_ahriman: Package, repository: Repository, + mocker: MockerFixture) -> None: """ must run command """ args = _default_args(args) + mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) run_mock = mocker.patch("ahriman.application.handlers.Update.run") - wait_mock = mocker.patch("threading.Event.wait", side_effect=[False, True]) + iter_mock = mocker.patch("ahriman.application.application.updates_iterator.UpdatesIterator.__iter__", + return_value=iter([[package_ahriman.base]])) + + _, repository_id = configuration.check_loaded() + Daemon.run(args, repository_id, configuration, report=True) + args.package = [package_ahriman.base] + run_mock.assert_called_once_with(args, repository_id, configuration, report=True) + iter_mock.assert_called_once_with() + + +def test_run_no_partitions(args: argparse.Namespace, configuration: Configuration, repository: Repository, + mocker: MockerFixture) -> None: + """ + must run command without partitioning + """ + args = _default_args(args) + args.partitions = False + mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) + run_mock = mocker.patch("ahriman.application.handlers.Update.run") + iter_mock = mocker.patch("ahriman.application.application.updates_iterator.UpdatesIterator.__iter__", + return_value=iter([[]])) _, repository_id = configuration.check_loaded() Daemon.run(args, repository_id, configuration, report=True) run_mock.assert_called_once_with(args, repository_id, configuration, report=True) - wait_mock.assert_has_calls([MockCall(args.interval), MockCall(args.interval)]) + iter_mock.assert_called_once_with() -def test_run_keyboard_interrupt(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None: +def test_run_no_updates(args: argparse.Namespace, configuration: Configuration, package_ahriman: Package, + repository: Repository, mocker: MockerFixture) -> None: """ - must handle KeyboardInterrupt exception + must skip empty update list """ args = _default_args(args) - mocker.patch("ahriman.application.handlers.Update.run", side_effect=KeyboardInterrupt) - wait_mock = mocker.patch("threading.Event.wait", return_value=False) + mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) + run_mock = mocker.patch("ahriman.application.handlers.Update.run") + iter_mock = mocker.patch("ahriman.application.application.updates_iterator.UpdatesIterator.__iter__", + return_value=iter([[package_ahriman.base], None])) _, repository_id = configuration.check_loaded() Daemon.run(args, repository_id, configuration, report=True) - wait_mock.assert_called_once_with(args.interval) + args.package = [package_ahriman.base] + run_mock.assert_called_once_with(args, repository_id, configuration, report=True) + iter_mock.assert_called_once_with()