From 5b9f35220fae06a21234d91c549f8351c31d1780 Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Tue, 3 Sep 2024 02:42:29 +0300 Subject: [PATCH] feat: implement stats subcommand (#132) --- .github/workflows/setup.sh | 2 +- Dockerfile | 2 +- docs/ahriman.application.handlers.rst | 8 + docs/ahriman.core.formatters.rst | 16 ++ package/archlinux/PKGBUILD | 1 + pyproject.toml | 3 + src/ahriman/application/ahriman.py | 26 +++ src/ahriman/application/handlers/__init__.py | 1 + .../application/handlers/statistics.py | 170 +++++++++++++++ .../database/operations/event_operations.py | 6 +- src/ahriman/core/formatters/__init__.py | 2 + .../core/formatters/event_stats_printer.py | 74 +++++++ .../core/formatters/package_stats_printer.py | 58 +++++ src/ahriman/core/status/client.py | 6 +- src/ahriman/core/status/local_client.py | 6 +- src/ahriman/core/status/web_client.py | 6 +- .../handlers/test_handler_statistics.py | 199 ++++++++++++++++++ tests/ahriman/application/test_ahriman.py | 67 +++++- tests/ahriman/core/formatters/conftest.py | 46 +++- .../formatters/test_event_stats_printer.py | 29 +++ .../formatters/test_package_stats_printer.py | 30 +++ tox.ini | 2 +- 22 files changed, 740 insertions(+), 20 deletions(-) create mode 100644 src/ahriman/application/handlers/statistics.py create mode 100644 src/ahriman/core/formatters/event_stats_printer.py create mode 100644 src/ahriman/core/formatters/package_stats_printer.py create mode 100644 tests/ahriman/application/handlers/test_handler_statistics.py create mode 100644 tests/ahriman/core/formatters/test_event_stats_printer.py create mode 100644 tests/ahriman/core/formatters/test_package_stats_printer.py diff --git a/.github/workflows/setup.sh b/.github/workflows/setup.sh index 67822439..4a42f2e7 100755 --- a/.github/workflows/setup.sh +++ b/.github/workflows/setup.sh @@ -20,7 +20,7 @@ if [[ -z $MINIMAL_INSTALL ]]; then # web server pacman -Sy --noconfirm python-aioauth-client python-aiohttp python-aiohttp-apispec-git python-aiohttp-cors python-aiohttp-jinja2 python-aiohttp-security python-aiohttp-session python-cryptography python-jinja # additional features - pacman -Sy --noconfirm gnupg python-boto3 rsync + pacman -Sy --noconfirm gnupg python-boto3 python-matplotlib rsync fi # FIXME since 1.0.4 devtools requires dbus to be run, which doesn't work now in container cp "docker/systemd-nspawn.sh" "/usr/local/bin/systemd-nspawn" diff --git a/Dockerfile b/Dockerfile index 4607f1dc..416d98de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ COPY "docker/install-aur-package.sh" "/usr/local/bin/install-aur-package" ## darcs is not installed by reasons, because it requires a lot haskell packages which dramatically increase image size RUN pacman -Sy --noconfirm --asdeps devtools git pyalpm python-cerberus python-inflection python-passlib python-pyelftools python-requests python-srcinfo && \ pacman -Sy --noconfirm --asdeps base-devel python-build python-flit python-installer python-wheel && \ - pacman -Sy --noconfirm --asdeps breezy git mercurial python-aiohttp python-boto3 python-cryptography python-jinja python-systemd rsync subversion && \ + pacman -Sy --noconfirm --asdeps breezy git mercurial python-aiohttp python-boto3 python-cryptography python-jinja python-matplotlib python-systemd rsync subversion && \ runuser -u build -- install-aur-package python-aioauth-client python-webargs python-aiohttp-apispec-git python-aiohttp-cors \ python-aiohttp-jinja2 python-aiohttp-session python-aiohttp-security python-requests-unixsocket2 diff --git a/docs/ahriman.application.handlers.rst b/docs/ahriman.application.handlers.rst index 006262a0..3c312ef0 100644 --- a/docs/ahriman.application.handlers.rst +++ b/docs/ahriman.application.handlers.rst @@ -172,6 +172,14 @@ ahriman.application.handlers.sign module :no-undoc-members: :show-inheritance: +ahriman.application.handlers.statistics module +---------------------------------------------- + +.. automodule:: ahriman.application.handlers.statistics + :members: + :no-undoc-members: + :show-inheritance: + ahriman.application.handlers.status module ------------------------------------------ diff --git a/docs/ahriman.core.formatters.rst b/docs/ahriman.core.formatters.rst index d1a84336..dbcf8649 100644 --- a/docs/ahriman.core.formatters.rst +++ b/docs/ahriman.core.formatters.rst @@ -44,6 +44,14 @@ ahriman.core.formatters.configuration\_printer module :no-undoc-members: :show-inheritance: +ahriman.core.formatters.event\_stats\_printer module +---------------------------------------------------- + +.. automodule:: ahriman.core.formatters.event_stats_printer + :members: + :no-undoc-members: + :show-inheritance: + ahriman.core.formatters.package\_printer module ----------------------------------------------- @@ -52,6 +60,14 @@ ahriman.core.formatters.package\_printer module :no-undoc-members: :show-inheritance: +ahriman.core.formatters.package\_stats\_printer module +------------------------------------------------------ + +.. automodule:: ahriman.core.formatters.package_stats_printer + :members: + :no-undoc-members: + :show-inheritance: + ahriman.core.formatters.patch\_printer module --------------------------------------------- diff --git a/package/archlinux/PKGBUILD b/package/archlinux/PKGBUILD index 9f212964..2fa9b686 100644 --- a/package/archlinux/PKGBUILD +++ b/package/archlinux/PKGBUILD @@ -21,6 +21,7 @@ optdepends=('breezy: -bzr packages support' 'python-aiohttp-session: web server with authorization' 'python-boto3: sync to s3' 'python-cryptography: web server with authorization' + 'python-matplotlib: usage statistics chart' 'python-requests-unixsocket2: client report to web server by unix socket' 'python-jinja: html report generation' 'python-systemd: journal support' diff --git a/pyproject.toml b/pyproject.toml index 7cba70fd..d45d4ada 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,9 @@ pacman = [ s3 = [ "boto3", ] +stats = [ + "matplotlib", +] tests = [ "pytest", "pytest-aiohttp", diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index e8212f22..f401dd0e 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -28,6 +28,7 @@ from ahriman.application import handlers from ahriman.core.utils import enum_values, extract_user from ahriman.models.action import Action from ahriman.models.build_status import BuildStatusEnum +from ahriman.models.event import EventType from ahriman.models.log_handler import LogHandler from ahriman.models.package_source import PackageSource from ahriman.models.sign_settings import SignSettings @@ -119,6 +120,7 @@ def _parser() -> argparse.ArgumentParser: _set_repo_report_parser(subparsers) _set_repo_restore_parser(subparsers) _set_repo_sign_parser(subparsers) + _set_repo_statistics_parser(subparsers) _set_repo_status_update_parser(subparsers) _set_repo_sync_parser(subparsers) _set_repo_tree_parser(subparsers) @@ -735,6 +737,30 @@ def _set_repo_sign_parser(root: SubParserAction) -> argparse.ArgumentParser: return parser +def _set_repo_statistics_parser(root: SubParserAction) -> argparse.ArgumentParser: + """ + add parser for repository statistics subcommand + + Args: + root(SubParserAction): subparsers for the commands + + Returns: + argparse.ArgumentParser: created argument parser + """ + parser = root.add_parser("repo-statistics", help="repository statistics", + description="fetch repository statistics", formatter_class=_formatter) + parser.add_argument("package", help="fetch only events for the specified package", nargs="?") + parser.add_argument("--chart", help="create updates chart and save it to the specified path", type=Path) + parser.add_argument("-e", "--event", help="event type filter", + type=EventType, choices=enum_values(EventType), default=EventType.PackageUpdated) + parser.add_argument("--from-date", help="only fetch events which are newer than the date") + parser.add_argument("--limit", help="limit response by specified amount of events", type=int, default=-1) + parser.add_argument("--offset", help="skip specified amount of events", type=int, default=0) + parser.add_argument("--to-date", help="only fetch events which are older than the date") + parser.set_defaults(handler=handlers.Statistics, lock=None, quiet=True, report=False, unsafe=True) + return parser + + def _set_repo_status_update_parser(root: SubParserAction) -> argparse.ArgumentParser: """ add parser for repository status update subcommand diff --git a/src/ahriman/application/handlers/__init__.py b/src/ahriman/application/handlers/__init__.py index 0c731648..e29723cc 100644 --- a/src/ahriman/application/handlers/__init__.py +++ b/src/ahriman/application/handlers/__init__.py @@ -38,6 +38,7 @@ from ahriman.application.handlers.service_updates import ServiceUpdates from ahriman.application.handlers.setup import Setup from ahriman.application.handlers.shell import Shell from ahriman.application.handlers.sign import Sign +from ahriman.application.handlers.statistics import Statistics from ahriman.application.handlers.status import Status from ahriman.application.handlers.status_update import StatusUpdate from ahriman.application.handlers.structure import Structure diff --git a/src/ahriman/application/handlers/statistics.py b/src/ahriman/application/handlers/statistics.py new file mode 100644 index 00000000..aef8320c --- /dev/null +++ b/src/ahriman/application/handlers/statistics.py @@ -0,0 +1,170 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import argparse +import datetime +import itertools + +from collections.abc import Callable +from pathlib import Path + +from ahriman.application.application import Application +from ahriman.application.handlers.handler import Handler +from ahriman.core.configuration import Configuration +from ahriman.core.formatters import EventStatsPrinter, PackageStatsPrinter +from ahriman.core.utils import pretty_datetime +from ahriman.models.event import Event +from ahriman.models.repository_id import RepositoryId + + +class Statistics(Handler): + """ + repository statistics handler + """ + + ALLOW_MULTI_ARCHITECTURE_RUN = False # conflicting io + + @classmethod + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: + """ + callback for command line + + Args: + args(argparse.Namespace): command line args + repository_id(RepositoryId): repository unique identifier + configuration(Configuration): configuration instance + report(bool): force enable or disable reporting + """ + application = Application(repository_id, configuration, report=True) + + from_date = to_date = None + if (value := args.from_date) is not None: + from_date = datetime.datetime.fromisoformat(value).timestamp() + if (value := args.to_date) is not None: + to_date = datetime.datetime.fromisoformat(value).timestamp() + + events = application.reporter.event_get(args.event, args.package, from_date, to_date, args.limit, args.offset) + + match args.package: + case None: + Statistics.stats_per_package(args.event, events, args.chart) + case _: + Statistics.stats_for_package(args.event, events, args.chart) + + @staticmethod + def event_stats(event_type: str, events: list[Event]) -> None: + """ + calculate event stats + + Args: + event_type(str): event type + events(list[Event]): list of events + """ + times = [event.get("took") for event in events if event.get("took") is not None] + EventStatsPrinter(f"{event_type} duration, s", times)(verbose=True) + + @staticmethod + def plot_packages(event_type: str, events: dict[str, int], path: Path) -> None: + """ + plot packages frequency + + Args: + event_type(str): event type + events(dict[str, int]): list of events + path(Path): path to save plot + """ + from matplotlib import pyplot as plt + + x, y = list(events.keys()), list(events.values()) + plt.bar(x, y) + + plt.xlabel("Package base") + plt.ylabel("Frequency") + plt.title(f"Frequency of the {event_type} event per package") + + plt.savefig(path) + + @staticmethod + def plot_times(event_type: str, events: list[Event], path: Path) -> None: + """ + plot events timeline + + Args: + event_type(str): event type + events(list[Event]): list of events + path(Path): path to save plot + """ + from matplotlib import pyplot as plt + + figure = plt.figure() + + x, y = zip(*[(pretty_datetime(event.created), event.get("took")) for event in events]) + plt.plot(x, y) + + plt.xlabel("Event timestamp") + plt.ylabel("Duration, s") + plt.title(f"Duration of the {event_type} event") + figure.autofmt_xdate() + + plt.savefig(path) + + @staticmethod + def stats_for_package(event_type: str, events: list[Event], chart_path: Path | None) -> None: + """ + calculate statistics for a package + + Args: + event_type(str): event type + events(list[Event]): list of events + chart_path(Path): path to save plot if any + """ + # event statistics + Statistics.event_stats(event_type, events) + + # chart if enabled + if chart_path is not None: + Statistics.plot_times(event_type, events, chart_path) + + @staticmethod + def stats_per_package(event_type: str, events: list[Event], chart_path: Path | None) -> None: + """ + calculate overall statistics + + Args: + event_type(str): event type + events(list[Event]): list of events + chart_path(Path): path to save plot if any + """ + key: Callable[[Event], str] = lambda event: event.object_id + by_object_id = { + object_id: len(list(related)) + for object_id, related in itertools.groupby(sorted(events, key=key), key=key) + } + + # distribution per package + PackageStatsPrinter(by_object_id)(verbose=True) + EventStatsPrinter(f"{event_type} frequency", list(by_object_id.values()))(verbose=True) + + # event statistics + Statistics.event_stats(event_type, events) + + # chart if enabled + if chart_path is not None: + Statistics.plot_packages(event_type, by_object_id, chart_path) diff --git a/src/ahriman/core/database/operations/event_operations.py b/src/ahriman/core/database/operations/event_operations.py index 023d6dc9..fc52080d 100644 --- a/src/ahriman/core/database/operations/event_operations.py +++ b/src/ahriman/core/database/operations/event_operations.py @@ -30,7 +30,7 @@ class EventOperations(Operations): """ def event_get(self, event: str | EventType | None = None, object_id: str | None = None, - from_date: int | None = None, to_date: int | None = None, + from_date: int | float | None = None, to_date: int | float | None = None, limit: int = -1, offset: int = 0, repository_id: RepositoryId | None = None) -> list[Event]: """ get list of events with filters applied @@ -38,8 +38,8 @@ class EventOperations(Operations): Args: event(str | EventType | None, optional): filter by event type (Default value = None) object_id(str | None, optional): filter by event object (Default value = None) - from_date(int | None, optional): minimal creation date, inclusive (Default value = None) - to_date(int | None, optional): maximal creation date, exclusive (Default value = None) + from_date(int | float | None, optional): minimal creation date, inclusive (Default value = None) + to_date(int | float | None, optional): maximal creation date, exclusive (Default value = None) limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1) offset(int, optional): records offset (Default value = 0) repository_id(RepositoryId, optional): repository unique identifier override (Default value = None) diff --git a/src/ahriman/core/formatters/__init__.py b/src/ahriman/core/formatters/__init__.py index dfcee20a..76d1b731 100644 --- a/src/ahriman/core/formatters/__init__.py +++ b/src/ahriman/core/formatters/__init__.py @@ -22,7 +22,9 @@ from ahriman.core.formatters.build_printer import BuildPrinter from ahriman.core.formatters.changes_printer import ChangesPrinter from ahriman.core.formatters.configuration_paths_printer import ConfigurationPathsPrinter from ahriman.core.formatters.configuration_printer import ConfigurationPrinter +from ahriman.core.formatters.event_stats_printer import EventStatsPrinter from ahriman.core.formatters.package_printer import PackagePrinter +from ahriman.core.formatters.package_stats_printer import PackageStatsPrinter from ahriman.core.formatters.patch_printer import PatchPrinter from ahriman.core.formatters.printer import Printer from ahriman.core.formatters.repository_printer import RepositoryPrinter diff --git a/src/ahriman/core/formatters/event_stats_printer.py b/src/ahriman/core/formatters/event_stats_printer.py new file mode 100644 index 00000000..8ded399f --- /dev/null +++ b/src/ahriman/core/formatters/event_stats_printer.py @@ -0,0 +1,74 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import statistics + +from ahriman.core.formatters.string_printer import StringPrinter +from ahriman.core.utils import minmax +from ahriman.models.property import Property + + +class EventStatsPrinter(StringPrinter): + """ + print event statistics + + Attributes: + events(list[float | int]): event values to build statistics + """ + + def __init__(self, event_type: str, events: list[float | int]) -> None: + """ + default constructor + + Args: + event_type(str): event type used for this statistics + events(list[float | int]): event values to build statistics + """ + StringPrinter.__init__(self, event_type) + self.events = events + + def properties(self) -> list[Property]: + """ + convert content into printable data + + Returns: + list[Property]: list of content properties + """ + properties = [ + Property("total", len(self.events)), + ] + + # time statistics + if self.events: + min_time, max_time = minmax(self.events) + mean = statistics.mean(self.events) + + if len(self.events) > 1: + stdev = statistics.stdev(self.events) + average = f"{mean:.3f} ± {stdev:.3f}" + else: + average = f"{mean:.3f}" + + properties.extend([ + Property("min", min_time), + Property("average", average), + Property("max", max_time), + ]) + + return properties diff --git a/src/ahriman/core/formatters/package_stats_printer.py b/src/ahriman/core/formatters/package_stats_printer.py new file mode 100644 index 00000000..3040f5c8 --- /dev/null +++ b/src/ahriman/core/formatters/package_stats_printer.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from ahriman.core.formatters.string_printer import StringPrinter +from ahriman.models.property import Property + + +class PackageStatsPrinter(StringPrinter): + """ + print packages statistics + + Attributes: + events(dict[str, int]): map of package to its event frequency + """ + + MAX_COUNT = 10 + + def __init__(self, events: dict[str, int]) -> None: + """ + default constructor + + Args: + events(dict[str, int]): map of package to its event frequency + """ + StringPrinter.__init__(self, "The most frequent packages") + self.events = events + + def properties(self) -> list[Property]: + """ + convert content into printable data + + Returns: + list[Property]: list of content properties + """ + if not self.events: + return [] # no events found, discard any stats + + properties = [] + for object_id, count in sorted(self.events.items(), key=lambda pair: pair[1], reverse=True)[:self.MAX_COUNT]: + properties.append(Property(object_id, count)) + + return properties diff --git a/src/ahriman/core/status/client.py b/src/ahriman/core/status/client.py index 08abc5cd..9fc7c19e 100644 --- a/src/ahriman/core/status/client.py +++ b/src/ahriman/core/status/client.py @@ -93,7 +93,7 @@ class Client: raise NotImplementedError def event_get(self, event: str | EventType | None, object_id: str | None, - from_date: int | None = None, to_date: int | None = None, + from_date: int | float | None = None, to_date: int | float | None = None, limit: int = -1, offset: int = 0) -> list[Event]: """ retrieve list of events @@ -101,8 +101,8 @@ class Client: Args: event(str | EventType | None): filter by event type object_id(str | None): filter by event object - from_date(int | None, optional): minimal creation date, inclusive (Default value = None) - to_date(int | None, optional): maximal creation date, exclusive (Default value = None) + from_date(int | float | None, optional): minimal creation date, inclusive (Default value = None) + to_date(int | float | None, optional): maximal creation date, exclusive (Default value = None) limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1) offset(int, optional): records offset (Default value = 0) diff --git a/src/ahriman/core/status/local_client.py b/src/ahriman/core/status/local_client.py index bd0d2a3e..f07fe3ca 100644 --- a/src/ahriman/core/status/local_client.py +++ b/src/ahriman/core/status/local_client.py @@ -59,7 +59,7 @@ class LocalClient(Client): self.database.event_insert(event, self.repository_id) def event_get(self, event: str | EventType | None, object_id: str | None, - from_date: int | None = None, to_date: int | None = None, + from_date: int | float | None = None, to_date: int | float | None = None, limit: int = -1, offset: int = 0) -> list[Event]: """ retrieve list of events @@ -67,8 +67,8 @@ class LocalClient(Client): Args: event(str | EventType | None): filter by event type object_id(str | None): filter by event object - from_date(int | None, optional): minimal creation date, inclusive (Default value = None) - to_date(int | None, optional): maximal creation date, exclusive (Default value = None) + from_date(int | float | None, optional): minimal creation date, inclusive (Default value = None) + to_date(int | float | None, optional): maximal creation date, exclusive (Default value = None) limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1) offset(int, optional): records offset (Default value = 0) diff --git a/src/ahriman/core/status/web_client.py b/src/ahriman/core/status/web_client.py index 6b890278..6e5f834b 100644 --- a/src/ahriman/core/status/web_client.py +++ b/src/ahriman/core/status/web_client.py @@ -178,7 +178,7 @@ class WebClient(Client, SyncAhrimanClient): self.make_request("POST", self._events_url(), params=self.repository_id.query(), json=event.view()) def event_get(self, event: str | EventType | None, object_id: str | None, - from_date: int | None = None, to_date: int | None = None, + from_date: int | float | None = None, to_date: int | float | None = None, limit: int = -1, offset: int = 0) -> list[Event]: """ retrieve list of events @@ -186,8 +186,8 @@ class WebClient(Client, SyncAhrimanClient): Args: event(str | EventType | None): filter by event type object_id(str | None): filter by event object - from_date(int | None, optional): minimal creation date, inclusive (Default value = None) - to_date(int | None, optional): maximal creation date, exclusive (Default value = None) + from_date(int | float | None, optional): minimal creation date, inclusive (Default value = None) + to_date(int | float | None, optional): maximal creation date, exclusive (Default value = None) limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1) offset(int, optional): records offset (Default value = 0) diff --git a/tests/ahriman/application/handlers/test_handler_statistics.py b/tests/ahriman/application/handlers/test_handler_statistics.py new file mode 100644 index 00000000..9d4dede9 --- /dev/null +++ b/tests/ahriman/application/handlers/test_handler_statistics.py @@ -0,0 +1,199 @@ +import argparse +import pytest + +from pathlib import Path +from pytest_mock import MockerFixture +from unittest.mock import call as MockCall + +from ahriman.application.handlers import Statistics +from ahriman.core.configuration import Configuration +from ahriman.core.repository import Repository +from ahriman.core.utils import pretty_datetime, utcnow +from ahriman.models.event import Event, EventType +from ahriman.models.package import Package + + +def _default_args(args: argparse.Namespace) -> argparse.Namespace: + """ + default arguments for these test cases + + Args: + args(argparse.Namespace): command line arguments fixture + + Returns: + argparse.Namespace: generated arguments for these test cases + """ + args.chart = None + args.event = EventType.PackageUpdated + args.from_date = None + args.limit = -1 + args.offset = 0 + args.package = None + args.to_date = None + return args + + +def test_run(args: argparse.Namespace, configuration: Configuration, repository: Repository, + mocker: MockerFixture) -> None: + """ + must run command + """ + args = _default_args(args) + events = [Event("1", "1"), Event("2", "2")] + mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) + events_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.event_get", return_value=events) + application_mock = mocker.patch("ahriman.application.handlers.Statistics.stats_per_package") + + _, repository_id = configuration.check_loaded() + Statistics.run(args, repository_id, configuration, report=False) + events_mock.assert_called_once_with(args.event, args.package, None, None, args.limit, args.offset) + application_mock.assert_called_once_with(args.event, events, args.chart) + + +def test_run_for_package(args: argparse.Namespace, configuration: Configuration, repository: Repository, + package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must run command for specific package + """ + args = _default_args(args) + args.package = package_ahriman.base + events = [Event("1", "1"), Event("2", "2")] + mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) + events_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.event_get", return_value=events) + application_mock = mocker.patch("ahriman.application.handlers.Statistics.stats_for_package") + + _, repository_id = configuration.check_loaded() + Statistics.run(args, repository_id, configuration, report=False) + events_mock.assert_called_once_with(args.event, args.package, None, None, args.limit, args.offset) + application_mock.assert_called_once_with(args.event, events, args.chart) + + +def test_run_convert_from_date(args: argparse.Namespace, configuration: Configuration, repository: Repository, + mocker: MockerFixture) -> None: + """ + must convert from date + """ + args = _default_args(args) + date = utcnow() + args.from_date = date.isoformat() + mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) + mocker.patch("ahriman.application.handlers.Statistics.stats_per_package") + events_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.event_get", return_value=[]) + + _, repository_id = configuration.check_loaded() + Statistics.run(args, repository_id, configuration, report=False) + events_mock.assert_called_once_with(args.event, args.package, date.timestamp(), None, args.limit, args.offset) + + +def test_run_convert_to_date(args: argparse.Namespace, configuration: Configuration, repository: Repository, + mocker: MockerFixture) -> None: + """ + must convert to date + """ + args = _default_args(args) + date = utcnow() + args.to_date = date.isoformat() + mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) + mocker.patch("ahriman.application.handlers.Statistics.stats_per_package") + events_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.event_get", return_value=[]) + + _, repository_id = configuration.check_loaded() + Statistics.run(args, repository_id, configuration, report=False) + events_mock.assert_called_once_with(args.event, args.package, None, date.timestamp(), args.limit, args.offset) + + +def test_event_stats(mocker: MockerFixture) -> None: + """ + must print event stats + """ + print_mock = mocker.patch("ahriman.core.formatters.Printer.print") + events = [Event("event", "1"), Event("event", "2", took=42.0)] + + Statistics.event_stats("event", events) + print_mock.assert_called_once_with(verbose=True, log_fn=pytest.helpers.anyvar(int), separator=": ") + + +def test_plot_packages(mocker: MockerFixture) -> None: + """ + must plot chart for packages + """ + plot_mock = mocker.patch("matplotlib.pyplot.bar") + save_mock = mocker.patch("matplotlib.pyplot.savefig") + local = Path("local") + + Statistics.plot_packages("event", {"1": 1, "2": 2}, local) + plot_mock.assert_called_once_with(["1", "2"], [1, 2]) + save_mock.assert_called_once_with(local) + + +def test_plot_times(mocker: MockerFixture) -> None: + """ + must plot chart for durations + """ + plot_mock = mocker.patch("matplotlib.pyplot.plot") + save_mock = mocker.patch("matplotlib.pyplot.savefig") + local = Path("local") + + Statistics.plot_times("event", [ + Event("", "", created=1, took=2), + Event("", "", created=3, took=4), + ], local) + plot_mock.assert_called_once_with((pretty_datetime(1), pretty_datetime(3)), (2, 4)) + save_mock.assert_called_once_with(local) + + +def test_stats_for_package(mocker: MockerFixture) -> None: + """ + must print statistics for the package + """ + events = [Event("event", "1"), Event("event", "1")] + events_mock = mocker.patch("ahriman.application.handlers.Statistics.event_stats") + chart_plot = mocker.patch("ahriman.application.handlers.Statistics.plot_times") + + Statistics.stats_for_package("event", events, None) + events_mock.assert_called_once_with("event", events) + chart_plot.assert_not_called() + + +def test_stats_for_package_with_chart(mocker: MockerFixture) -> None: + """ + must generate chart for package stats + """ + local = Path("local") + events = [Event("event", "1"), Event("event", "1")] + mocker.patch("ahriman.application.handlers.Statistics.event_stats") + chart_plot = mocker.patch("ahriman.application.handlers.Statistics.plot_times") + + Statistics.stats_for_package("event", events, local) + chart_plot.assert_called_once_with("event", events, local) + + +def test_stats_per_package(mocker: MockerFixture) -> None: + """ + must print statistics per package + """ + events = [Event("event", "1"), Event("event", "2"), Event("event", "1")] + print_mock = mocker.patch("ahriman.core.formatters.Printer.print") + events_mock = mocker.patch("ahriman.application.handlers.Statistics.event_stats") + chart_plot = mocker.patch("ahriman.application.handlers.Statistics.plot_packages") + + Statistics.stats_per_package("event", events, None) + print_mock.assert_has_calls([ + MockCall(verbose=True, log_fn=pytest.helpers.anyvar(int), separator=": "), + MockCall(verbose=True, log_fn=pytest.helpers.anyvar(int), separator=": "), + ]) + events_mock.assert_called_once_with("event", events) + chart_plot.assert_not_called() + + +def test_stats_per_package_with_chart(mocker: MockerFixture) -> None: + """ + must print statistics per package with chart + """ + local = Path("local") + events = [Event("event", "1"), Event("event", "2"), Event("event", "1")] + mocker.patch("ahriman.application.handlers.Statistics.event_stats") + chart_plot = mocker.patch("ahriman.application.handlers.Statistics.plot_packages") + + Statistics.stats_per_package("event", events, local) + chart_plot.assert_called_once_with("event", {"1": 2, "2": 1}, local) diff --git a/tests/ahriman/application/test_ahriman.py b/tests/ahriman/application/test_ahriman.py index 06acc232..04bf465b 100644 --- a/tests/ahriman/application/test_ahriman.py +++ b/tests/ahriman/application/test_ahriman.py @@ -9,6 +9,7 @@ from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.models.action import Action from ahriman.models.build_status import BuildStatusEnum +from ahriman.models.event import EventType from ahriman.models.log_handler import LogHandler from ahriman.models.sign_settings import SignSettings from ahriman.models.user_access import UserAccess @@ -931,11 +932,73 @@ def test_subparsers_repo_sign_option_repository(parser: argparse.ArgumentParser) assert args.repository == "repo" +def test_subparsers_repo_statistics(parser: argparse.ArgumentParser) -> None: + """ + repo-statistics command must imply lock, quiet, report and unsafe + """ + args = parser.parse_args(["-a", "x86_64", "-r", "repo", "repo-statistics"]) + assert args.architecture == "x86_64" + assert args.lock is None + assert args.quiet + assert not args.report + assert args.repository == "repo" + assert args.unsafe + + +def test_subparsers_repo_statistics_option_event(parser: argparse.ArgumentParser) -> None: + """ + repo-statistics command must convert event option to EventType instance + """ + args = parser.parse_args(["-a", "x86_64", "-r", "repo", "repo-statistics"]) + assert isinstance(args.event, EventType) + args = parser.parse_args(["-a", "x86_64", "-r", "repo", "repo-statistics", "--event", "package-removed"]) + assert isinstance(args.event, EventType) + + +def test_subparsers_repo_statistics_option_package(parser: argparse.ArgumentParser) -> None: + """ + repo-statistics command must parse optional package argument + """ + args = parser.parse_args(["-a", "x86_64", "-r", "repo", "repo-statistics"]) + assert args.package is None + + args = parser.parse_args(["-a", "x86_64", "-r", "repo", "repo-statistics", "package"]) + assert args.package == "package" + + +def test_subparsers_repo_statistics_option_chart(parser: argparse.ArgumentParser) -> None: + """ + repo-statistics command must convert chart option to Path instance + """ + args = parser.parse_args(["-a", "x86_64", "-r", "repo", "repo-statistics", "--chart", "path"]) + assert isinstance(args.chart, Path) + + +def test_subparsers_repo_statistics_option_limit(parser: argparse.ArgumentParser) -> None: + """ + repo-statistics command must convert chart option to Path instance + """ + args = parser.parse_args(["-a", "x86_64", "-r", "repo", "repo-statistics"]) + assert isinstance(args.limit, int) + args = parser.parse_args(["-a", "x86_64", "-r", "repo", "repo-statistics", "--limit", "42"]) + assert isinstance(args.limit, int) + + +def test_subparsers_repo_statistics_option_offset(parser: argparse.ArgumentParser) -> None: + """ + repo-statistics command must convert chart option to Path instance + """ + args = parser.parse_args(["-a", "x86_64", "-r", "repo", "repo-statistics"]) + assert isinstance(args.offset, int) + args = parser.parse_args(["-a", "x86_64", "-r", "repo", "repo-statistics", "--offset", "42"]) + assert isinstance(args.offset, int) + + def test_subparsers_repo_status_update(parser: argparse.ArgumentParser) -> None: """ - re[p-status-update command must imply action, lock, quiet, report, package and unsafe + repo-status-update command must imply action, lock, quiet, report, package and unsafe """ - args = parser.parse_args(["-a", "x86_64", "-r", "repo", "package-status-update"]) + args = parser.parse_args(["-a", "x86_64", "-r", "repo", "repo-status-update"]) assert args.architecture == "x86_64" assert args.action == Action.Update assert args.lock is None diff --git a/tests/ahriman/core/formatters/conftest.py b/tests/ahriman/core/formatters/conftest.py index 90402d76..5e995f6c 100644 --- a/tests/ahriman/core/formatters/conftest.py +++ b/tests/ahriman/core/formatters/conftest.py @@ -2,9 +2,23 @@ import pytest from pathlib import Path -from ahriman.core.formatters import AurPrinter, ChangesPrinter, ConfigurationPathsPrinter, ConfigurationPrinter, \ - PackagePrinter, PatchPrinter, RepositoryPrinter, StatusPrinter, StringPrinter, TreePrinter, UpdatePrinter, \ - UserPrinter, ValidationPrinter, VersionPrinter +from ahriman.core.formatters import \ + AurPrinter, \ + ChangesPrinter, \ + ConfigurationPathsPrinter, \ + ConfigurationPrinter, \ + EventStatsPrinter, \ + PackagePrinter, \ + PackageStatsPrinter, \ + PatchPrinter, \ + RepositoryPrinter, \ + StatusPrinter, \ + StringPrinter, \ + TreePrinter, \ + UpdatePrinter, \ + UserPrinter, \ + ValidationPrinter, \ + VersionPrinter from ahriman.models.aur_package import AURPackage from ahriman.models.build_status import BuildStatus from ahriman.models.changes import Changes @@ -61,6 +75,17 @@ def configuration_printer() -> ConfigurationPrinter: return ConfigurationPrinter("section", {"key_one": "value_one", "key_two": "value_two"}) +@pytest.fixture +def event_stats_printer() -> EventStatsPrinter: + """ + fixture for event stats printer + + Returns: + EventStatsPrinter: event stats printer test instance + """ + return EventStatsPrinter("event", [5, 2, 7, 9, 8, 0, 4, 1, 6, 3]) + + @pytest.fixture def package_ahriman_printer(package_ahriman: Package) -> PackagePrinter: """ @@ -75,6 +100,21 @@ def package_ahriman_printer(package_ahriman: Package) -> PackagePrinter: return PackagePrinter(package_ahriman, BuildStatus()) +@pytest.fixture +def package_stats_printer(package_ahriman: Package, package_python_schedule: Package) -> PackageStatsPrinter: + """ + fixture for package stats printer + + Args: + package_ahriman(Package): package fixture + package_python_schedule(Package): schedule package fixture + + Returns: + PackageStatsPrinter: package stats printer test instance + """ + return PackageStatsPrinter({package_ahriman.base: 4, package_python_schedule.base: 5}) + + @pytest.fixture def patch_printer(package_ahriman: Package) -> PatchPrinter: """ diff --git a/tests/ahriman/core/formatters/test_event_stats_printer.py b/tests/ahriman/core/formatters/test_event_stats_printer.py new file mode 100644 index 00000000..c2591a0a --- /dev/null +++ b/tests/ahriman/core/formatters/test_event_stats_printer.py @@ -0,0 +1,29 @@ +from ahriman.core.formatters import EventStatsPrinter + + +def test_properties(event_stats_printer: EventStatsPrinter) -> None: + """ + must return empty properties list + """ + assert event_stats_printer.properties() + + +def test_properties_empty() -> None: + """ + must correctly generate properties for empty events list + """ + assert EventStatsPrinter("event", []).properties() + + +def test_properties_single() -> None: + """ + must skip calculation of the standard deviation for single event + """ + assert EventStatsPrinter("event", [1]).properties() + + +def test_title(event_stats_printer: EventStatsPrinter) -> None: + """ + must return non-empty title + """ + assert event_stats_printer.title() is not None diff --git a/tests/ahriman/core/formatters/test_package_stats_printer.py b/tests/ahriman/core/formatters/test_package_stats_printer.py new file mode 100644 index 00000000..79b6d7aa --- /dev/null +++ b/tests/ahriman/core/formatters/test_package_stats_printer.py @@ -0,0 +1,30 @@ +from ahriman.core.formatters import PackageStatsPrinter + + +def test_properties(package_stats_printer: PackageStatsPrinter) -> None: + """ + must return non-empty properties list + """ + assert package_stats_printer.properties() + + +def test_properties_sorted(package_stats_printer: PackageStatsPrinter) -> None: + """ + properties list must be sorted in descending order + """ + prop1, prop2 = package_stats_printer.properties() + assert prop1.value > prop2.value + + +def test_properties_empty() -> None: + """ + must return empty properties list for the empty events list + """ + assert not PackageStatsPrinter({}).properties() + + +def test_title(package_stats_printer: PackageStatsPrinter) -> None: + """ + must return non-empty title + """ + assert package_stats_printer.title() is not None diff --git a/tox.ini b/tox.ini index a01ad397..a7c8c7f4 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ envlist = check, tests isolated_build = True labels = release = version, docs, publish -dependencies = -e .[journald,pacman,s3,web] +dependencies = -e .[journald,pacman,s3,stats,web] project_name = ahriman [mypy]