feat: add counters to repository stats overview

This commit is contained in:
Evgenii Alekseev 2025-01-09 15:51:10 +02:00
parent ed67898012
commit 65324633b4
26 changed files with 519 additions and 28 deletions

View File

@ -92,6 +92,14 @@ ahriman.core.formatters.repository\_printer module
:no-undoc-members:
:show-inheritance:
ahriman.core.formatters.repository\_stats\_printer module
---------------------------------------------------------
.. automodule:: ahriman.core.formatters.repository_stats_printer
:members:
:no-undoc-members:
:show-inheritance:
ahriman.core.formatters.status\_printer module
----------------------------------------------

View File

@ -236,6 +236,14 @@ ahriman.models.repository\_paths module
:no-undoc-members:
:show-inheritance:
ahriman.models.repository\_stats module
---------------------------------------
.. automodule:: ahriman.models.repository_stats
:members:
:no-undoc-members:
:show-inheritance:
ahriman.models.result module
----------------------------
@ -252,6 +260,14 @@ ahriman.models.scan\_paths module
:no-undoc-members:
:show-inheritance:
ahriman.models.series\_statistics module
----------------------------------------
.. automodule:: ahriman.models.series_statistics
:members:
:no-undoc-members:
:show-inheritance:
ahriman.models.sign\_settings module
------------------------------------

View File

@ -260,6 +260,14 @@ ahriman.web.schemas.repository\_id\_schema module
:no-undoc-members:
:show-inheritance:
ahriman.web.schemas.repository\_stats\_schema module
----------------------------------------------------
.. automodule:: ahriman.web.schemas.repository_stats_schema
:members:
:no-undoc-members:
:show-inheritance:
ahriman.web.schemas.search\_schema module
-----------------------------------------

View File

@ -27,7 +27,7 @@ from pathlib import Path
from ahriman.application.application import Application
from ahriman.application.handlers.handler import Handler, SubParserAction
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import EventStatsPrinter, PackageStatsPrinter
from ahriman.core.formatters import EventStatsPrinter, PackageStatsPrinter, RepositoryStatsPrinter
from ahriman.core.utils import enum_values, pretty_datetime
from ahriman.models.event import Event, EventType
from ahriman.models.repository_id import RepositoryId
@ -64,6 +64,7 @@ class Statistics(Handler):
match args.package:
case None:
RepositoryStatsPrinter(repository_id, application.reporter.statistics())(verbose=True)
Statistics.stats_per_package(args.event, events, args.chart)
case _:
Statistics.stats_for_package(args.event, events, args.chart)

View File

@ -28,6 +28,7 @@ from ahriman.core.formatters.package_stats_printer import PackageStatsPrinter
from ahriman.core.formatters.patch_printer import PatchPrinter
from ahriman.core.formatters.printer import Printer
from ahriman.core.formatters.repository_printer import RepositoryPrinter
from ahriman.core.formatters.repository_stats_printer import RepositoryStatsPrinter
from ahriman.core.formatters.status_printer import StatusPrinter
from ahriman.core.formatters.string_printer import StringPrinter
from ahriman.core.formatters.tree_printer import TreePrinter

View File

@ -17,11 +17,9 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import statistics
from ahriman.core.formatters.string_printer import StringPrinter
from ahriman.core.utils import minmax
from ahriman.models.property import Property
from ahriman.models.series_statistics import SeriesStatistics
class EventStatsPrinter(StringPrinter):
@ -29,7 +27,7 @@ class EventStatsPrinter(StringPrinter):
print event statistics
Attributes:
events(list[float | int]): event values to build statistics
statistics(SeriesStatistics): statistics object
"""
def __init__(self, event_type: str, events: list[float | int]) -> None:
@ -39,7 +37,7 @@ class EventStatsPrinter(StringPrinter):
events(list[float | int]): event values to build statistics
"""
StringPrinter.__init__(self, event_type)
self.events = events
self.statistics = SeriesStatistics(events)
def properties(self) -> list[Property]:
"""
@ -49,24 +47,17 @@ class EventStatsPrinter(StringPrinter):
list[Property]: list of content properties
"""
properties = [
Property("total", len(self.events)),
Property("total", self.statistics.total),
]
# time statistics
if self.events:
min_time, max_time = minmax(self.events)
mean = statistics.mean(self.events)
if len(self.events) > 1:
st_dev = statistics.stdev(self.events)
average = f"{mean:.3f} ± {st_dev:.3f}"
else:
average = f"{mean:.3f}"
if self.statistics:
mean = self.statistics.mean
properties.extend([
Property("min", min_time),
Property("average", average),
Property("max", max_time),
Property("min", self.statistics.min),
Property("average", f"{mean:.3f} ± {self.statistics.st_dev:.3f}"),
Property("max", self.statistics.max),
])
return properties

View File

@ -0,0 +1,53 @@
#
# Copyright (c) 2021-2025 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.formatters.string_printer import StringPrinter
from ahriman.core.utils import pretty_size
from ahriman.models.property import Property
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_stats import RepositoryStats
class RepositoryStatsPrinter(StringPrinter):
"""
print repository statistics
Attributes:
statistics(RepositoryStats): repository statistics
"""
def __init__(self, repository_id: RepositoryId, statistics: RepositoryStats) -> None:
"""
Args:
statistics(RepositoryStats): repository statistics
"""
StringPrinter.__init__(self, str(repository_id))
self.statistics = statistics
def properties(self) -> list[Property]:
"""
convert content into printable data
Returns:
list[Property]: list of content properties
"""
return [
Property("Packages", self.statistics.bases),
Property("Repository size", pretty_size(self.statistics.archive_size)),
]

View File

@ -31,6 +31,7 @@ from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_stats import RepositoryStats
class Client:
@ -354,6 +355,16 @@ class Client:
return # skip update in case if package is already known
self.package_update(package, BuildStatusEnum.Unknown)
def statistics(self) -> RepositoryStats:
"""
get repository statistics
Returns:
RepositoryStats: repository statistics object
"""
packages = [package for package, _ in self.package_get(None)]
return RepositoryStats.from_packages(packages)
def status_get(self) -> InternalStatus:
"""
get internal service status

View File

@ -23,6 +23,7 @@ from typing import Any, Self
from ahriman.core.utils import dataclass_view
from ahriman.models.build_status import BuildStatus
from ahriman.models.counters import Counters
from ahriman.models.repository_stats import RepositoryStats
@dataclass(frozen=True, kw_only=True)
@ -35,6 +36,7 @@ class InternalStatus:
architecture(str | None): repository architecture
packages(Counters): packages statuses counter object
repository(str | None): repository name
stats(RepositoryStats | None): repository stats
version(str | None): service version
"""
@ -42,6 +44,7 @@ class InternalStatus:
architecture: str | None = None
packages: Counters = field(default=Counters(total=0))
repository: str | None = None
stats: RepositoryStats | None = None
version: str | None = None
@classmethod
@ -56,11 +59,13 @@ class InternalStatus:
Self: internal status
"""
counters = Counters.from_json(dump["packages"]) if "packages" in dump else Counters(total=0)
stats = RepositoryStats.from_json(dump["stats"]) if "stats" in dump else None
build_status = dump.get("status") or {}
return cls(status=BuildStatus.from_json(build_status),
architecture=dump.get("architecture"),
packages=counters,
repository=dump.get("repository"),
stats=stats,
version=dump.get("version"))
def view(self) -> dict[str, Any]:

View File

@ -97,3 +97,12 @@ class RepositoryId:
raise ValueError(f"'<' not supported between instances of '{type(self)}' and '{type(other)}'")
return (self.name, self.architecture) < (other.name, other.architecture)
def __str__(self) -> str:
"""
string representation of the repository identifier
Returns:
str: string view of the repository identifier
"""
return f"{self.name} ({self.architecture})"

View File

@ -0,0 +1,77 @@
#
# Copyright (c) 2021-2025 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass, fields
from typing import Any, Self
from ahriman.core.utils import filter_json
from ahriman.models.package import Package
@dataclass(frozen=True, kw_only=True)
class RepositoryStats:
"""
repository stats representation
"""
bases: int
packages: int
archive_size: int
installed_size: int
@classmethod
def from_json(cls, dump: dict[str, Any]) -> Self:
"""
construct counters from json dump
Args:
dump(dict[str, Any]): json dump body
Returns:
Self: status counters
"""
# filter to only known fields
known_fields = [pair.name for pair in fields(cls)]
return cls(**filter_json(dump, known_fields))
@classmethod
def from_packages(cls, packages: list[Package]) -> Self:
"""
construct statistics from list of repository packages
Args:
packages(list[Packages]): list of repository packages
Returns:
Self: constructed statistics object
"""
return cls(
bases=len(packages),
packages=sum(len(package.packages) for package in packages),
archive_size=sum(
archive.archive_size or 0
for package in packages
for archive in package.packages.values()
),
installed_size=sum(
archive.installed_size or 0
for package in packages
for archive in package.packages.values()
),
)

View File

@ -0,0 +1,104 @@
#
# Copyright (c) 2021-2025 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import statistics
from dataclasses import dataclass
@dataclass(frozen=True)
class SeriesStatistics:
"""
series statistics helper
Attributes:
series(list[float | int]): list of values to be processed
"""
series: list[float | int]
@property
def max(self) -> float | int | None:
"""
get max value in series
Returns:
float | int | None: ``None`` if series is empty and maximal value otherwise``
"""
if self:
return max(self.series)
return None
@property
def mean(self) -> float | int | None:
"""
get mean value in series
Returns:
float | int | None: ``None`` if series is empty and mean value otherwise
"""
if self:
return statistics.mean(self.series)
return None
@property
def min(self) -> float | int | None:
"""
get min value in series
Returns:
float | int | None: ``None`` if series is empty and minimal value otherwise
"""
if self:
return min(self.series)
return None
@property
def st_dev(self) -> float | None:
"""
get standard deviation in series
Returns:
float | None: ``None`` if series size is less than 1, 0 if series contains single element and standard
deviation otherwise
"""
if not self:
return None
if len(self.series) > 1:
return statistics.stdev(self.series)
return 0.0
@property
def total(self) -> int:
"""
retrieve amount of elements
Returns:
int: the series collection size
"""
return len(self.series)
def __bool__(self) -> bool:
"""
check if series is empty or not
Returns:
bool: ``True`` if series contains elements and ``False`` otherwise
"""
return bool(self.total)

View File

@ -49,6 +49,7 @@ from ahriman.web.schemas.process_id_schema import ProcessIdSchema
from ahriman.web.schemas.process_schema import ProcessSchema
from ahriman.web.schemas.remote_schema import RemoteSchema
from ahriman.web.schemas.repository_id_schema import RepositoryIdSchema
from ahriman.web.schemas.repository_stats_schema import RepositoryStatsSchema
from ahriman.web.schemas.search_schema import SearchSchema
from ahriman.web.schemas.status_schema import StatusSchema
from ahriman.web.schemas.update_flags_schema import UpdateFlagsSchema

View File

@ -21,6 +21,7 @@ from ahriman import __version__
from ahriman.web.apispec import fields
from ahriman.web.schemas.counters_schema import CountersSchema
from ahriman.web.schemas.repository_id_schema import RepositoryIdSchema
from ahriman.web.schemas.repository_stats_schema import RepositoryStatsSchema
from ahriman.web.schemas.status_schema import StatusSchema
@ -32,6 +33,9 @@ class InternalStatusSchema(RepositoryIdSchema):
packages = fields.Nested(CountersSchema(), required=True, metadata={
"description": "Repository package counters",
})
stats = fields.Nested(RepositoryStatsSchema(), required=True, metadata={
"description": "Repository stats",
})
status = fields.Nested(StatusSchema(), required=True, metadata={
"description": "Repository status as stored by web service",
})

View File

@ -0,0 +1,43 @@
#
# Copyright (c) 2021-2025 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.web.apispec import Schema, fields
class RepositoryStatsSchema(Schema):
"""
response repository stats schema
"""
bases = fields.Int(metadata={
"description": "Amount of unique packages bases",
"example": 2,
})
packages = fields.Int(metadata={
"description": "Amount of unique packages",
"example": 4,
})
archive_size = fields.Int(metadata={
"description": "Total archive size of the packages in bytes",
"example": 42000,
})
installed_size = fields.Int(metadata={
"description": "Total installed size of the packages in bytes",
"example": 42000000,
})

View File

@ -23,6 +23,7 @@ from ahriman import __version__
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.counters import Counters
from ahriman.models.internal_status import InternalStatus
from ahriman.models.repository_stats import RepositoryStats
from ahriman.models.user_access import UserAccess
from ahriman.web.apispec.decorators import apidocs
from ahriman.web.schemas import InternalStatusSchema, RepositoryIdSchema, StatusSchema
@ -60,12 +61,16 @@ class StatusView(StatusViewGuard, BaseView):
Response: 200 with service status object
"""
repository_id = self.repository_id()
counters = Counters.from_packages(self.service(repository_id).packages)
packages = self.service(repository_id).packages
counters = Counters.from_packages(packages)
stats = RepositoryStats.from_packages([package for package, _ in packages])
status = InternalStatus(
status=self.service(repository_id).status,
architecture=repository_id.architecture,
packages=counters,
repository=repository_id.name,
stats=stats,
version=__version__,
)

View File

@ -11,6 +11,7 @@ from ahriman.core.repository import Repository
from ahriman.core.utils import pretty_datetime, utcnow
from ahriman.models.event import Event, EventType
from ahriman.models.package import Package
from ahriman.models.repository_stats import RepositoryStats
def _default_args(args: argparse.Namespace) -> argparse.Namespace:
@ -40,13 +41,16 @@ def test_run(args: argparse.Namespace, configuration: Configuration, repository:
"""
args = _default_args(args)
events = [Event("1", "1"), Event("2", "2")]
stats = RepositoryStats(bases=1, packages=2, archive_size=3, installed_size=4)
mocker.patch("ahriman.core.repository.Repository.load", return_value=repository)
events_mock = mocker.patch("ahriman.core.status.local_client.LocalClient.event_get", return_value=events)
stats_mock = mocker.patch("ahriman.core.status.client.Client.statistics", return_value=stats)
application_mock = mocker.patch("ahriman.application.handlers.statistics.Statistics.stats_per_package")
_, repository_id = configuration.check_loaded()
Statistics.run(args, repository_id, configuration, report=False)
events_mock.assert_called_once_with(args.event, args.package, None, None, args.limit, args.offset)
stats_mock.assert_called_once_with()
application_mock.assert_called_once_with(args.event, events, args.chart)

View File

@ -12,6 +12,7 @@ from ahriman.core.formatters import \
PackageStatsPrinter, \
PatchPrinter, \
RepositoryPrinter, \
RepositoryStatsPrinter, \
StatusPrinter, \
StringPrinter, \
TreePrinter, \
@ -25,6 +26,7 @@ from ahriman.models.changes import Changes
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_stats import RepositoryStats
from ahriman.models.user import User
@ -134,12 +136,29 @@ def repository_printer(repository_id: RepositoryId) -> RepositoryPrinter:
"""
fixture for repository printer
Args:
repository_id(RepositoryId): repository identifier fixture
Returns:
RepositoryPrinter: repository printer test instance
"""
return RepositoryPrinter(repository_id)
@pytest.fixture
def repository_stats_printer(repository_id: RepositoryId) -> RepositoryStatsPrinter:
"""
fixture for repository stats printer
Args:
repository_id(RepositoryId): repository identifier fixture
Returns:
RepositoryStatsPrinter: repository stats printer test instance
"""
return RepositoryStatsPrinter(repository_id, RepositoryStats(bases=1, packages=2, archive_size=3, installed_size=4))
@pytest.fixture
def status_printer() -> StatusPrinter:
"""

View File

@ -15,13 +15,6 @@ def test_properties_empty() -> None:
assert EventStatsPrinter("event", []).properties()
def test_properties_single() -> None:
"""
must skip calculation of the standard deviation for single event
"""
assert EventStatsPrinter("event", [1]).properties()
def test_title(event_stats_printer: EventStatsPrinter) -> None:
"""
must return non-empty title

View File

@ -0,0 +1,15 @@
from ahriman.core.formatters import RepositoryStatsPrinter
def test_properties(repository_stats_printer: RepositoryStatsPrinter) -> None:
"""
must return non-empty properties list
"""
assert repository_stats_printer.properties()
def test_title(repository_stats_printer: RepositoryStatsPrinter) -> None:
"""
must return non-empty title
"""
assert repository_stats_printer.title()

View File

@ -16,6 +16,7 @@ from ahriman.models.internal_status import InternalStatus
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_stats import RepositoryStats
def test_load_dummy_client(configuration: Configuration) -> None:
@ -285,6 +286,14 @@ def test_set_unknown_skip(client: Client, package_ahriman: Package, mocker: Mock
update_mock.assert_not_called()
def test_statistics(client: Client, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must correctly fetch statistics
"""
mocker.patch("ahriman.core.status.Client.package_get", return_value=[(package_ahriman, None)])
assert client.statistics() == RepositoryStats(bases=1, packages=1, archive_size=4200, installed_size=4200000)
def test_status_get(client: Client) -> None:
"""
must return dummy status for web service

View File

@ -14,6 +14,7 @@ from ahriman.models.package_description import PackageDescription
from ahriman.models.package_source import PackageSource
from ahriman.models.pkgbuild import Pkgbuild
from ahriman.models.remote_source import RemoteSource
from ahriman.models.repository_stats import RepositoryStats
@pytest.fixture
@ -71,8 +72,9 @@ def internal_status(counters: Counters) -> InternalStatus:
status=BuildStatus(),
architecture="x86_64",
packages=counters,
version=__version__,
repository="aur",
stats=RepositoryStats(bases=1, packages=2, archive_size=3, installed_size=4),
version=__version__,
)

View File

@ -68,3 +68,10 @@ def test_lt_invalid() -> None:
"""
with pytest.raises(ValueError):
assert RepositoryId("x86_64", "a") < 42
def test_str() -> None:
"""
must convert identifier to string
"""
assert str(RepositoryId("x86_64", "a")) == "a (x86_64)"

View File

@ -0,0 +1,24 @@
from dataclasses import asdict
from ahriman.models.package import Package
from ahriman.models.repository_stats import RepositoryStats
def test_repository_stats_from_json_view(package_ahriman: Package, package_python_schedule: Package) -> None:
"""
must construct same object from json
"""
stats = RepositoryStats.from_packages([package_ahriman, package_python_schedule])
assert RepositoryStats.from_json(asdict(stats)) == stats
def test_from_packages(package_ahriman: Package, package_python_schedule: Package) -> None:
"""
must generate stats from packages list
"""
assert RepositoryStats.from_packages([package_ahriman, package_python_schedule]) == RepositoryStats(
bases=2,
packages=3,
archive_size=12603,
installed_size=12600003,
)

View File

@ -0,0 +1,80 @@
from ahriman.models.series_statistics import SeriesStatistics
def test_max() -> None:
"""
must return maximal value
"""
assert SeriesStatistics([1, 3, 2]).max == 3
def test_max_empty() -> None:
"""
must return None as maximal value if series is empty
"""
assert SeriesStatistics([]).max is None
def test_mean() -> None:
"""
must return mean value
"""
assert SeriesStatistics([1, 3, 2]).mean == 2
def test_mean_empty() -> None:
"""
must return None as mean value if series is empty
"""
assert SeriesStatistics([]).mean is None
def test_min() -> None:
"""
must return minimal value
"""
assert SeriesStatistics([1, 3, 2]).min == 1
def test_min_empty() -> None:
"""
must return None as minimal value if series is empty
"""
assert SeriesStatistics([]).min is None
def test_st_dev() -> None:
"""
must return standard deviation
"""
assert SeriesStatistics([1, 3, 2]).st_dev == 1
def test_st_dev_empty() -> None:
"""
must return None as standard deviation if series is empty
"""
assert SeriesStatistics([]).st_dev is None
def test_st_dev_single() -> None:
"""
must return 0 as standard deviation if series contains only one element
"""
assert SeriesStatistics([1]).st_dev == 0
def test_total() -> None:
"""
must return size of collection
"""
assert SeriesStatistics([1]).total == 1
assert SeriesStatistics([]).total == 0
def test_bool() -> None:
"""
must correctly define empty collection
"""
assert SeriesStatistics([1])
assert not SeriesStatistics([])

View File

@ -0,0 +1 @@
# schema testing goes in view class tests