mirror of
https://github.com/arcan1s/ahriman.git
synced 2025-07-15 06:55:48 +00:00
feat: implement stats subcommand (#132)
This commit is contained in:
@ -28,6 +28,7 @@ from ahriman.application import handlers
|
||||
from ahriman.core.utils import enum_values, extract_user
|
||||
from ahriman.models.action import Action
|
||||
from ahriman.models.build_status import BuildStatusEnum
|
||||
from ahriman.models.event import EventType
|
||||
from ahriman.models.log_handler import LogHandler
|
||||
from ahriman.models.package_source import PackageSource
|
||||
from ahriman.models.sign_settings import SignSettings
|
||||
@ -119,6 +120,7 @@ def _parser() -> argparse.ArgumentParser:
|
||||
_set_repo_report_parser(subparsers)
|
||||
_set_repo_restore_parser(subparsers)
|
||||
_set_repo_sign_parser(subparsers)
|
||||
_set_repo_statistics_parser(subparsers)
|
||||
_set_repo_status_update_parser(subparsers)
|
||||
_set_repo_sync_parser(subparsers)
|
||||
_set_repo_tree_parser(subparsers)
|
||||
@ -735,6 +737,30 @@ def _set_repo_sign_parser(root: SubParserAction) -> argparse.ArgumentParser:
|
||||
return parser
|
||||
|
||||
|
||||
def _set_repo_statistics_parser(root: SubParserAction) -> argparse.ArgumentParser:
|
||||
"""
|
||||
add parser for repository statistics subcommand
|
||||
|
||||
Args:
|
||||
root(SubParserAction): subparsers for the commands
|
||||
|
||||
Returns:
|
||||
argparse.ArgumentParser: created argument parser
|
||||
"""
|
||||
parser = root.add_parser("repo-statistics", help="repository statistics",
|
||||
description="fetch repository statistics", formatter_class=_formatter)
|
||||
parser.add_argument("package", help="fetch only events for the specified package", nargs="?")
|
||||
parser.add_argument("--chart", help="create updates chart and save it to the specified path", type=Path)
|
||||
parser.add_argument("-e", "--event", help="event type filter",
|
||||
type=EventType, choices=enum_values(EventType), default=EventType.PackageUpdated)
|
||||
parser.add_argument("--from-date", help="only fetch events which are newer than the date")
|
||||
parser.add_argument("--limit", help="limit response by specified amount of events", type=int, default=-1)
|
||||
parser.add_argument("--offset", help="skip specified amount of events", type=int, default=0)
|
||||
parser.add_argument("--to-date", help="only fetch events which are older than the date")
|
||||
parser.set_defaults(handler=handlers.Statistics, lock=None, quiet=True, report=False, unsafe=True)
|
||||
return parser
|
||||
|
||||
|
||||
def _set_repo_status_update_parser(root: SubParserAction) -> argparse.ArgumentParser:
|
||||
"""
|
||||
add parser for repository status update subcommand
|
||||
|
||||
@ -38,6 +38,7 @@ from ahriman.application.handlers.service_updates import ServiceUpdates
|
||||
from ahriman.application.handlers.setup import Setup
|
||||
from ahriman.application.handlers.shell import Shell
|
||||
from ahriman.application.handlers.sign import Sign
|
||||
from ahriman.application.handlers.statistics import Statistics
|
||||
from ahriman.application.handlers.status import Status
|
||||
from ahriman.application.handlers.status_update import StatusUpdate
|
||||
from ahriman.application.handlers.structure import Structure
|
||||
|
||||
170
src/ahriman/application/handlers/statistics.py
Normal file
170
src/ahriman/application/handlers/statistics.py
Normal file
@ -0,0 +1,170 @@
|
||||
#
|
||||
# Copyright (c) 2021-2024 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import argparse
|
||||
import datetime
|
||||
import itertools
|
||||
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from ahriman.application.application import Application
|
||||
from ahriman.application.handlers.handler import Handler
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.formatters import EventStatsPrinter, PackageStatsPrinter
|
||||
from ahriman.core.utils import pretty_datetime
|
||||
from ahriman.models.event import Event
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
|
||||
|
||||
class Statistics(Handler):
|
||||
"""
|
||||
repository statistics handler
|
||||
"""
|
||||
|
||||
ALLOW_MULTI_ARCHITECTURE_RUN = False # conflicting io
|
||||
|
||||
@classmethod
|
||||
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
|
||||
report: bool) -> None:
|
||||
"""
|
||||
callback for command line
|
||||
|
||||
Args:
|
||||
args(argparse.Namespace): command line args
|
||||
repository_id(RepositoryId): repository unique identifier
|
||||
configuration(Configuration): configuration instance
|
||||
report(bool): force enable or disable reporting
|
||||
"""
|
||||
application = Application(repository_id, configuration, report=True)
|
||||
|
||||
from_date = to_date = None
|
||||
if (value := args.from_date) is not None:
|
||||
from_date = datetime.datetime.fromisoformat(value).timestamp()
|
||||
if (value := args.to_date) is not None:
|
||||
to_date = datetime.datetime.fromisoformat(value).timestamp()
|
||||
|
||||
events = application.reporter.event_get(args.event, args.package, from_date, to_date, args.limit, args.offset)
|
||||
|
||||
match args.package:
|
||||
case None:
|
||||
Statistics.stats_per_package(args.event, events, args.chart)
|
||||
case _:
|
||||
Statistics.stats_for_package(args.event, events, args.chart)
|
||||
|
||||
@staticmethod
|
||||
def event_stats(event_type: str, events: list[Event]) -> None:
|
||||
"""
|
||||
calculate event stats
|
||||
|
||||
Args:
|
||||
event_type(str): event type
|
||||
events(list[Event]): list of events
|
||||
"""
|
||||
times = [event.get("took") for event in events if event.get("took") is not None]
|
||||
EventStatsPrinter(f"{event_type} duration, s", times)(verbose=True)
|
||||
|
||||
@staticmethod
|
||||
def plot_packages(event_type: str, events: dict[str, int], path: Path) -> None:
|
||||
"""
|
||||
plot packages frequency
|
||||
|
||||
Args:
|
||||
event_type(str): event type
|
||||
events(dict[str, int]): list of events
|
||||
path(Path): path to save plot
|
||||
"""
|
||||
from matplotlib import pyplot as plt
|
||||
|
||||
x, y = list(events.keys()), list(events.values())
|
||||
plt.bar(x, y)
|
||||
|
||||
plt.xlabel("Package base")
|
||||
plt.ylabel("Frequency")
|
||||
plt.title(f"Frequency of the {event_type} event per package")
|
||||
|
||||
plt.savefig(path)
|
||||
|
||||
@staticmethod
|
||||
def plot_times(event_type: str, events: list[Event], path: Path) -> None:
|
||||
"""
|
||||
plot events timeline
|
||||
|
||||
Args:
|
||||
event_type(str): event type
|
||||
events(list[Event]): list of events
|
||||
path(Path): path to save plot
|
||||
"""
|
||||
from matplotlib import pyplot as plt
|
||||
|
||||
figure = plt.figure()
|
||||
|
||||
x, y = zip(*[(pretty_datetime(event.created), event.get("took")) for event in events])
|
||||
plt.plot(x, y)
|
||||
|
||||
plt.xlabel("Event timestamp")
|
||||
plt.ylabel("Duration, s")
|
||||
plt.title(f"Duration of the {event_type} event")
|
||||
figure.autofmt_xdate()
|
||||
|
||||
plt.savefig(path)
|
||||
|
||||
@staticmethod
|
||||
def stats_for_package(event_type: str, events: list[Event], chart_path: Path | None) -> None:
|
||||
"""
|
||||
calculate statistics for a package
|
||||
|
||||
Args:
|
||||
event_type(str): event type
|
||||
events(list[Event]): list of events
|
||||
chart_path(Path): path to save plot if any
|
||||
"""
|
||||
# event statistics
|
||||
Statistics.event_stats(event_type, events)
|
||||
|
||||
# chart if enabled
|
||||
if chart_path is not None:
|
||||
Statistics.plot_times(event_type, events, chart_path)
|
||||
|
||||
@staticmethod
|
||||
def stats_per_package(event_type: str, events: list[Event], chart_path: Path | None) -> None:
|
||||
"""
|
||||
calculate overall statistics
|
||||
|
||||
Args:
|
||||
event_type(str): event type
|
||||
events(list[Event]): list of events
|
||||
chart_path(Path): path to save plot if any
|
||||
"""
|
||||
key: Callable[[Event], str] = lambda event: event.object_id
|
||||
by_object_id = {
|
||||
object_id: len(list(related))
|
||||
for object_id, related in itertools.groupby(sorted(events, key=key), key=key)
|
||||
}
|
||||
|
||||
# distribution per package
|
||||
PackageStatsPrinter(by_object_id)(verbose=True)
|
||||
EventStatsPrinter(f"{event_type} frequency", list(by_object_id.values()))(verbose=True)
|
||||
|
||||
# event statistics
|
||||
Statistics.event_stats(event_type, events)
|
||||
|
||||
# chart if enabled
|
||||
if chart_path is not None:
|
||||
Statistics.plot_packages(event_type, by_object_id, chart_path)
|
||||
@ -30,7 +30,7 @@ class EventOperations(Operations):
|
||||
"""
|
||||
|
||||
def event_get(self, event: str | EventType | None = None, object_id: str | None = None,
|
||||
from_date: int | None = None, to_date: int | None = None,
|
||||
from_date: int | float | None = None, to_date: int | float | None = None,
|
||||
limit: int = -1, offset: int = 0, repository_id: RepositoryId | None = None) -> list[Event]:
|
||||
"""
|
||||
get list of events with filters applied
|
||||
@ -38,8 +38,8 @@ class EventOperations(Operations):
|
||||
Args:
|
||||
event(str | EventType | None, optional): filter by event type (Default value = None)
|
||||
object_id(str | None, optional): filter by event object (Default value = None)
|
||||
from_date(int | None, optional): minimal creation date, inclusive (Default value = None)
|
||||
to_date(int | None, optional): maximal creation date, exclusive (Default value = None)
|
||||
from_date(int | float | None, optional): minimal creation date, inclusive (Default value = None)
|
||||
to_date(int | float | None, optional): maximal creation date, exclusive (Default value = None)
|
||||
limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1)
|
||||
offset(int, optional): records offset (Default value = 0)
|
||||
repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
|
||||
|
||||
@ -22,7 +22,9 @@ from ahriman.core.formatters.build_printer import BuildPrinter
|
||||
from ahriman.core.formatters.changes_printer import ChangesPrinter
|
||||
from ahriman.core.formatters.configuration_paths_printer import ConfigurationPathsPrinter
|
||||
from ahriman.core.formatters.configuration_printer import ConfigurationPrinter
|
||||
from ahriman.core.formatters.event_stats_printer import EventStatsPrinter
|
||||
from ahriman.core.formatters.package_printer import PackagePrinter
|
||||
from ahriman.core.formatters.package_stats_printer import PackageStatsPrinter
|
||||
from ahriman.core.formatters.patch_printer import PatchPrinter
|
||||
from ahriman.core.formatters.printer import Printer
|
||||
from ahriman.core.formatters.repository_printer import RepositoryPrinter
|
||||
|
||||
74
src/ahriman/core/formatters/event_stats_printer.py
Normal file
74
src/ahriman/core/formatters/event_stats_printer.py
Normal file
@ -0,0 +1,74 @@
|
||||
#
|
||||
# Copyright (c) 2021-2024 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import statistics
|
||||
|
||||
from ahriman.core.formatters.string_printer import StringPrinter
|
||||
from ahriman.core.utils import minmax
|
||||
from ahriman.models.property import Property
|
||||
|
||||
|
||||
class EventStatsPrinter(StringPrinter):
|
||||
"""
|
||||
print event statistics
|
||||
|
||||
Attributes:
|
||||
events(list[float | int]): event values to build statistics
|
||||
"""
|
||||
|
||||
def __init__(self, event_type: str, events: list[float | int]) -> None:
|
||||
"""
|
||||
default constructor
|
||||
|
||||
Args:
|
||||
event_type(str): event type used for this statistics
|
||||
events(list[float | int]): event values to build statistics
|
||||
"""
|
||||
StringPrinter.__init__(self, event_type)
|
||||
self.events = events
|
||||
|
||||
def properties(self) -> list[Property]:
|
||||
"""
|
||||
convert content into printable data
|
||||
|
||||
Returns:
|
||||
list[Property]: list of content properties
|
||||
"""
|
||||
properties = [
|
||||
Property("total", len(self.events)),
|
||||
]
|
||||
|
||||
# time statistics
|
||||
if self.events:
|
||||
min_time, max_time = minmax(self.events)
|
||||
mean = statistics.mean(self.events)
|
||||
|
||||
if len(self.events) > 1:
|
||||
stdev = statistics.stdev(self.events)
|
||||
average = f"{mean:.3f} ± {stdev:.3f}"
|
||||
else:
|
||||
average = f"{mean:.3f}"
|
||||
|
||||
properties.extend([
|
||||
Property("min", min_time),
|
||||
Property("average", average),
|
||||
Property("max", max_time),
|
||||
])
|
||||
|
||||
return properties
|
||||
58
src/ahriman/core/formatters/package_stats_printer.py
Normal file
58
src/ahriman/core/formatters/package_stats_printer.py
Normal file
@ -0,0 +1,58 @@
|
||||
#
|
||||
# Copyright (c) 2021-2024 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from ahriman.core.formatters.string_printer import StringPrinter
|
||||
from ahriman.models.property import Property
|
||||
|
||||
|
||||
class PackageStatsPrinter(StringPrinter):
|
||||
"""
|
||||
print packages statistics
|
||||
|
||||
Attributes:
|
||||
events(dict[str, int]): map of package to its event frequency
|
||||
"""
|
||||
|
||||
MAX_COUNT = 10
|
||||
|
||||
def __init__(self, events: dict[str, int]) -> None:
|
||||
"""
|
||||
default constructor
|
||||
|
||||
Args:
|
||||
events(dict[str, int]): map of package to its event frequency
|
||||
"""
|
||||
StringPrinter.__init__(self, "The most frequent packages")
|
||||
self.events = events
|
||||
|
||||
def properties(self) -> list[Property]:
|
||||
"""
|
||||
convert content into printable data
|
||||
|
||||
Returns:
|
||||
list[Property]: list of content properties
|
||||
"""
|
||||
if not self.events:
|
||||
return [] # no events found, discard any stats
|
||||
|
||||
properties = []
|
||||
for object_id, count in sorted(self.events.items(), key=lambda pair: pair[1], reverse=True)[:self.MAX_COUNT]:
|
||||
properties.append(Property(object_id, count))
|
||||
|
||||
return properties
|
||||
@ -93,7 +93,7 @@ class Client:
|
||||
raise NotImplementedError
|
||||
|
||||
def event_get(self, event: str | EventType | None, object_id: str | None,
|
||||
from_date: int | None = None, to_date: int | None = None,
|
||||
from_date: int | float | None = None, to_date: int | float | None = None,
|
||||
limit: int = -1, offset: int = 0) -> list[Event]:
|
||||
"""
|
||||
retrieve list of events
|
||||
@ -101,8 +101,8 @@ class Client:
|
||||
Args:
|
||||
event(str | EventType | None): filter by event type
|
||||
object_id(str | None): filter by event object
|
||||
from_date(int | None, optional): minimal creation date, inclusive (Default value = None)
|
||||
to_date(int | None, optional): maximal creation date, exclusive (Default value = None)
|
||||
from_date(int | float | None, optional): minimal creation date, inclusive (Default value = None)
|
||||
to_date(int | float | None, optional): maximal creation date, exclusive (Default value = None)
|
||||
limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1)
|
||||
offset(int, optional): records offset (Default value = 0)
|
||||
|
||||
|
||||
@ -59,7 +59,7 @@ class LocalClient(Client):
|
||||
self.database.event_insert(event, self.repository_id)
|
||||
|
||||
def event_get(self, event: str | EventType | None, object_id: str | None,
|
||||
from_date: int | None = None, to_date: int | None = None,
|
||||
from_date: int | float | None = None, to_date: int | float | None = None,
|
||||
limit: int = -1, offset: int = 0) -> list[Event]:
|
||||
"""
|
||||
retrieve list of events
|
||||
@ -67,8 +67,8 @@ class LocalClient(Client):
|
||||
Args:
|
||||
event(str | EventType | None): filter by event type
|
||||
object_id(str | None): filter by event object
|
||||
from_date(int | None, optional): minimal creation date, inclusive (Default value = None)
|
||||
to_date(int | None, optional): maximal creation date, exclusive (Default value = None)
|
||||
from_date(int | float | None, optional): minimal creation date, inclusive (Default value = None)
|
||||
to_date(int | float | None, optional): maximal creation date, exclusive (Default value = None)
|
||||
limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1)
|
||||
offset(int, optional): records offset (Default value = 0)
|
||||
|
||||
|
||||
@ -178,7 +178,7 @@ class WebClient(Client, SyncAhrimanClient):
|
||||
self.make_request("POST", self._events_url(), params=self.repository_id.query(), json=event.view())
|
||||
|
||||
def event_get(self, event: str | EventType | None, object_id: str | None,
|
||||
from_date: int | None = None, to_date: int | None = None,
|
||||
from_date: int | float | None = None, to_date: int | float | None = None,
|
||||
limit: int = -1, offset: int = 0) -> list[Event]:
|
||||
"""
|
||||
retrieve list of events
|
||||
@ -186,8 +186,8 @@ class WebClient(Client, SyncAhrimanClient):
|
||||
Args:
|
||||
event(str | EventType | None): filter by event type
|
||||
object_id(str | None): filter by event object
|
||||
from_date(int | None, optional): minimal creation date, inclusive (Default value = None)
|
||||
to_date(int | None, optional): maximal creation date, exclusive (Default value = None)
|
||||
from_date(int | float | None, optional): minimal creation date, inclusive (Default value = None)
|
||||
to_date(int | float | None, optional): maximal creation date, exclusive (Default value = None)
|
||||
limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1)
|
||||
offset(int, optional): records offset (Default value = 0)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user