From af2269c64a54e9322679e1111f892fce3550ece9 Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Fri, 16 Aug 2024 16:24:11 +0300 Subject: [PATCH] fix: print current and updated version correctly The issue appears in case if versions ar the same (e.g. rebuild); in this case printer doesn't increment version as builder does. Also util has been renamed to utils, keeping backward compatibiltiy --- docs/ahriman.core.rst | 6 +- docs/ahriman.core.status.rst | 8 + docs/ahriman.models.rst | 8 + docs/ahriman.web.schemas.rst | 16 + docs/ahriman.web.views.v1.packages.rst | 8 + src/ahriman/application/ahriman.py | 2 +- .../application/application_packages.py | 2 +- src/ahriman/application/lock.py | 2 +- src/ahriman/core/alpm/pacman.py | 2 +- src/ahriman/core/alpm/repo.py | 2 +- src/ahriman/core/build_tools/sources.py | 2 +- src/ahriman/core/build_tools/task.py | 2 +- .../migrations/m005_make_opt_depends.py | 2 +- .../database/migrations/m007_check_depends.py | 2 +- .../database/migrations/m008_packagers.py | 2 +- src/ahriman/core/formatters/aur_printer.py | 2 +- src/ahriman/core/formatters/update_printer.py | 10 +- src/ahriman/core/gitremote/remote_pull.py | 2 +- src/ahriman/core/report/email.py | 2 +- src/ahriman/core/report/jinja_template.py | 2 +- src/ahriman/core/repository/executor.py | 2 +- src/ahriman/core/repository/package_info.py | 2 +- src/ahriman/core/sign/gpg.py | 2 +- .../support/pkgbuild/pkgbuild_generator.py | 2 +- src/ahriman/core/tree.py | 2 +- src/ahriman/core/upload/github.py | 2 +- src/ahriman/core/upload/rsync.py | 2 +- src/ahriman/core/upload/s3.py | 2 +- src/ahriman/core/util.py | 487 +---------------- src/ahriman/core/utils.py | 504 ++++++++++++++++++ src/ahriman/models/aur_package.py | 2 +- src/ahriman/models/build_status.py | 2 +- src/ahriman/models/changes.py | 2 +- src/ahriman/models/counters.py | 2 +- src/ahriman/models/dependencies.py | 2 +- src/ahriman/models/filesystem_package.py | 2 +- src/ahriman/models/internal_status.py | 2 +- src/ahriman/models/package.py | 9 +- src/ahriman/models/package_archive.py | 2 +- src/ahriman/models/package_description.py | 2 +- src/ahriman/models/package_source.py | 2 +- src/ahriman/models/pkgbuild_patch.py | 2 +- src/ahriman/models/remote_source.py | 2 +- src/ahriman/models/worker.py | 2 +- src/ahriman/web/views/api/swagger.py | 2 +- src/ahriman/web/views/v1/packages/logs.py | 2 +- .../core/formatters/test_update_printer.py | 26 + tests/ahriman/core/test_util.py | 497 +---------------- tests/ahriman/core/test_utils.py | 496 +++++++++++++++++ .../core/triggers/test_trigger_loader.py | 2 +- tests/ahriman/core/upload/test_github.py | 2 +- tests/ahriman/core/upload/test_s3.py | 2 +- tests/ahriman/models/test_package.py | 4 +- tests/ahriman/test_tests.py | 2 +- tests/ahriman/web/test_routes.py | 2 +- 55 files changed, 1136 insertions(+), 1027 deletions(-) create mode 100644 src/ahriman/core/utils.py create mode 100644 tests/ahriman/core/test_utils.py diff --git a/docs/ahriman.core.rst b/docs/ahriman.core.rst index 31ca571f..624f394d 100644 --- a/docs/ahriman.core.rst +++ b/docs/ahriman.core.rst @@ -52,10 +52,10 @@ ahriman.core.tree module :no-undoc-members: :show-inheritance: -ahriman.core.util module ------------------------- +ahriman.core.utils module +------------------------- -.. automodule:: ahriman.core.util +.. automodule:: ahriman.core.utils :members: :no-undoc-members: :show-inheritance: diff --git a/docs/ahriman.core.status.rst b/docs/ahriman.core.status.rst index 34bea3f9..b0096df3 100644 --- a/docs/ahriman.core.status.rst +++ b/docs/ahriman.core.status.rst @@ -12,6 +12,14 @@ ahriman.core.status.client module :no-undoc-members: :show-inheritance: +ahriman.core.status.local\_client module +---------------------------------------- + +.. automodule:: ahriman.core.status.local_client + :members: + :no-undoc-members: + :show-inheritance: + ahriman.core.status.watcher module ---------------------------------- diff --git a/docs/ahriman.models.rst b/docs/ahriman.models.rst index 06626aa9..40901e66 100644 --- a/docs/ahriman.models.rst +++ b/docs/ahriman.models.rst @@ -68,6 +68,14 @@ ahriman.models.dependencies module :no-undoc-members: :show-inheritance: +ahriman.models.filesystem\_package module +----------------------------------------- + +.. automodule:: ahriman.models.filesystem_package + :members: + :no-undoc-members: + :show-inheritance: + ahriman.models.internal\_status module -------------------------------------- diff --git a/docs/ahriman.web.schemas.rst b/docs/ahriman.web.schemas.rst index ef02c15a..bd4b5000 100644 --- a/docs/ahriman.web.schemas.rst +++ b/docs/ahriman.web.schemas.rst @@ -44,6 +44,14 @@ ahriman.web.schemas.counters\_schema module :no-undoc-members: :show-inheritance: +ahriman.web.schemas.dependencies\_schema module +----------------------------------------------- + +.. automodule:: ahriman.web.schemas.dependencies_schema + :members: + :no-undoc-members: + :show-inheritance: + ahriman.web.schemas.error\_schema module ---------------------------------------- @@ -156,6 +164,14 @@ ahriman.web.schemas.package\_status\_schema module :no-undoc-members: :show-inheritance: +ahriman.web.schemas.package\_version\_schema module +--------------------------------------------------- + +.. automodule:: ahriman.web.schemas.package_version_schema + :members: + :no-undoc-members: + :show-inheritance: + ahriman.web.schemas.pagination\_schema module --------------------------------------------- diff --git a/docs/ahriman.web.views.v1.packages.rst b/docs/ahriman.web.views.v1.packages.rst index 40364345..48f2b916 100644 --- a/docs/ahriman.web.views.v1.packages.rst +++ b/docs/ahriman.web.views.v1.packages.rst @@ -12,6 +12,14 @@ ahriman.web.views.v1.packages.changes module :no-undoc-members: :show-inheritance: +ahriman.web.views.v1.packages.dependencies module +------------------------------------------------- + +.. automodule:: ahriman.web.views.v1.packages.dependencies + :members: + :no-undoc-members: + :show-inheritance: + ahriman.web.views.v1.packages.logs module ----------------------------------------- diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 510d63db..e8212f22 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -25,7 +25,7 @@ from typing import TypeVar from ahriman import __version__ from ahriman.application import handlers -from ahriman.core.util import enum_values, extract_user +from ahriman.core.utils import enum_values, extract_user from ahriman.models.action import Action from ahriman.models.build_status import BuildStatusEnum from ahriman.models.log_handler import LogHandler diff --git a/src/ahriman/application/application/application_packages.py b/src/ahriman/application/application/application_packages.py index 45e62a89..c2b2ef4d 100644 --- a/src/ahriman/application/application/application_packages.py +++ b/src/ahriman/application/application/application_packages.py @@ -27,7 +27,7 @@ from typing import Any from ahriman.application.application.application_properties import ApplicationProperties from ahriman.core.build_tools.sources import Sources from ahriman.core.exceptions import UnknownPackageError -from ahriman.core.util import package_like +from ahriman.core.utils import package_like from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.result import Result diff --git a/src/ahriman/application/lock.py b/src/ahriman/application/lock.py index 526dbd56..56aab2d9 100644 --- a/src/ahriman/application/lock.py +++ b/src/ahriman/application/lock.py @@ -31,7 +31,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.exceptions import DuplicateRunError from ahriman.core.log import LazyLogging from ahriman.core.status import Client -from ahriman.core.util import check_user +from ahriman.core.utils import check_user from ahriman.models.build_status import BuildStatusEnum from ahriman.models.repository_id import RepositoryId from ahriman.models.waiter import Waiter diff --git a/src/ahriman/core/alpm/pacman.py b/src/ahriman/core/alpm/pacman.py index aa309471..d8911b55 100644 --- a/src/ahriman/core/alpm/pacman.py +++ b/src/ahriman/core/alpm/pacman.py @@ -30,7 +30,7 @@ from string import Template from ahriman.core.alpm.pacman_database import PacmanDatabase from ahriman.core.configuration import Configuration from ahriman.core.log import LazyLogging -from ahriman.core.util import trim_package +from ahriman.core.utils import trim_package from ahriman.models.pacman_synchronization import PacmanSynchronization from ahriman.models.repository_id import RepositoryId diff --git a/src/ahriman/core/alpm/repo.py b/src/ahriman/core/alpm/repo.py index 049073c1..9d13d468 100644 --- a/src/ahriman/core/alpm/repo.py +++ b/src/ahriman/core/alpm/repo.py @@ -21,7 +21,7 @@ from pathlib import Path from ahriman.core.exceptions import BuildError from ahriman.core.log import LazyLogging -from ahriman.core.util import check_output +from ahriman.core.utils import check_output from ahriman.models.repository_paths import RepositoryPaths diff --git a/src/ahriman/core/build_tools/sources.py b/src/ahriman/core/build_tools/sources.py index 4179b9de..d32d90c0 100644 --- a/src/ahriman/core/build_tools/sources.py +++ b/src/ahriman/core/build_tools/sources.py @@ -23,7 +23,7 @@ from pathlib import Path from ahriman.core.exceptions import CalledProcessError from ahriman.core.log import LazyLogging -from ahriman.core.util import check_output, utcnow, walk +from ahriman.core.utils import check_output, utcnow, walk from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.remote_source import RemoteSource diff --git a/src/ahriman/core/build_tools/task.py b/src/ahriman/core/build_tools/task.py index 12bf1aa6..17fec01c 100644 --- a/src/ahriman/core/build_tools/task.py +++ b/src/ahriman/core/build_tools/task.py @@ -23,7 +23,7 @@ from ahriman.core.build_tools.sources import Sources from ahriman.core.configuration import Configuration from ahriman.core.exceptions import BuildError from ahriman.core.log import LazyLogging -from ahriman.core.util import check_output +from ahriman.core.utils import check_output from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.repository_paths import RepositoryPaths diff --git a/src/ahriman/core/database/migrations/m005_make_opt_depends.py b/src/ahriman/core/database/migrations/m005_make_opt_depends.py index 7efcca84..23efa031 100644 --- a/src/ahriman/core/database/migrations/m005_make_opt_depends.py +++ b/src/ahriman/core/database/migrations/m005_make_opt_depends.py @@ -21,7 +21,7 @@ from sqlite3 import Connection from ahriman.core.alpm.pacman import Pacman from ahriman.core.configuration import Configuration -from ahriman.core.util import package_like +from ahriman.core.utils import package_like from ahriman.models.package import Package from ahriman.models.pacman_synchronization import PacmanSynchronization diff --git a/src/ahriman/core/database/migrations/m007_check_depends.py b/src/ahriman/core/database/migrations/m007_check_depends.py index 3d6f4978..b3cf77b6 100644 --- a/src/ahriman/core/database/migrations/m007_check_depends.py +++ b/src/ahriman/core/database/migrations/m007_check_depends.py @@ -21,7 +21,7 @@ from sqlite3 import Connection from ahriman.core.alpm.pacman import Pacman from ahriman.core.configuration import Configuration -from ahriman.core.util import package_like +from ahriman.core.utils import package_like from ahriman.models.package import Package from ahriman.models.pacman_synchronization import PacmanSynchronization diff --git a/src/ahriman/core/database/migrations/m008_packagers.py b/src/ahriman/core/database/migrations/m008_packagers.py index 46927d5e..ad693e5d 100644 --- a/src/ahriman/core/database/migrations/m008_packagers.py +++ b/src/ahriman/core/database/migrations/m008_packagers.py @@ -21,7 +21,7 @@ from sqlite3 import Connection from ahriman.core.alpm.pacman import Pacman from ahriman.core.configuration import Configuration -from ahriman.core.util import package_like +from ahriman.core.utils import package_like from ahriman.models.package import Package from ahriman.models.pacman_synchronization import PacmanSynchronization diff --git a/src/ahriman/core/formatters/aur_printer.py b/src/ahriman/core/formatters/aur_printer.py index fb1ec07d..a1293a31 100644 --- a/src/ahriman/core/formatters/aur_printer.py +++ b/src/ahriman/core/formatters/aur_printer.py @@ -18,7 +18,7 @@ # along with this program. If not, see . # from ahriman.core.formatters.string_printer import StringPrinter -from ahriman.core.util import pretty_datetime +from ahriman.core.utils import pretty_datetime from ahriman.models.aur_package import AURPackage from ahriman.models.property import Property diff --git a/src/ahriman/core/formatters/update_printer.py b/src/ahriman/core/formatters/update_printer.py index a861fe4b..7e149608 100644 --- a/src/ahriman/core/formatters/update_printer.py +++ b/src/ahriman/core/formatters/update_printer.py @@ -18,6 +18,7 @@ # along with this program. If not, see . # from ahriman.core.formatters.string_printer import StringPrinter +from ahriman.core.utils import full_version, parse_version from ahriman.models.package import Package from ahriman.models.property import Property @@ -41,7 +42,7 @@ class UpdatePrinter(StringPrinter): """ StringPrinter.__init__(self, remote.base) self.package = remote - self.local_version = local_version or "N/A" + self.local_version = local_version def properties(self) -> list[Property]: """ @@ -50,4 +51,9 @@ class UpdatePrinter(StringPrinter): Returns: list[Property]: list of content properties """ - return [Property(self.local_version, self.package.version, is_required=True)] + if (pkgrel := self.package.next_pkgrel(self.local_version)) is not None: + epoch, pkgver, _ = parse_version(self.package.version) + effective_new_version = full_version(epoch, pkgver, pkgrel) + else: + effective_new_version = self.package.version + return [Property(self.local_version or "N/A", effective_new_version, is_required=True)] diff --git a/src/ahriman/core/gitremote/remote_pull.py b/src/ahriman/core/gitremote/remote_pull.py index a00aff04..68ed9be3 100644 --- a/src/ahriman/core/gitremote/remote_pull.py +++ b/src/ahriman/core/gitremote/remote_pull.py @@ -26,7 +26,7 @@ from ahriman.core.build_tools.sources import Sources from ahriman.core.configuration import Configuration from ahriman.core.exceptions import GitRemoteError from ahriman.core.log import LazyLogging -from ahriman.core.util import walk +from ahriman.core.utils import walk from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.remote_source import RemoteSource diff --git a/src/ahriman/core/report/email.py b/src/ahriman/core/report/email.py index e215c868..58c3d781 100644 --- a/src/ahriman/core/report/email.py +++ b/src/ahriman/core/report/email.py @@ -25,7 +25,7 @@ from email.mime.text import MIMEText from ahriman.core.configuration import Configuration from ahriman.core.report.jinja_template import JinjaTemplate from ahriman.core.report.report import Report -from ahriman.core.util import pretty_datetime, utcnow +from ahriman.core.utils import pretty_datetime, utcnow from ahriman.models.package import Package from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result diff --git a/src/ahriman/core/report/jinja_template.py b/src/ahriman/core/report/jinja_template.py index a87d948f..ab51990c 100644 --- a/src/ahriman/core/report/jinja_template.py +++ b/src/ahriman/core/report/jinja_template.py @@ -24,7 +24,7 @@ from pathlib import Path from ahriman.core.configuration import Configuration from ahriman.core.sign.gpg import GPG -from ahriman.core.util import pretty_datetime, pretty_size +from ahriman.core.utils import pretty_datetime, pretty_size from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result from ahriman.models.sign_settings import SignSettings diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py index a8af5a29..9b9e44f0 100644 --- a/src/ahriman/core/repository/executor.py +++ b/src/ahriman/core/repository/executor.py @@ -26,7 +26,7 @@ from tempfile import TemporaryDirectory from ahriman.core.build_tools.task import Task from ahriman.core.repository.cleaner import Cleaner from ahriman.core.repository.package_info import PackageInfo -from ahriman.core.util import safe_filename +from ahriman.core.utils import safe_filename from ahriman.models.changes import Changes from ahriman.models.package import Package from ahriman.models.package_archive import PackageArchive diff --git a/src/ahriman/core/repository/package_info.py b/src/ahriman/core/repository/package_info.py index 28ba659f..1ab0b3f0 100644 --- a/src/ahriman/core/repository/package_info.py +++ b/src/ahriman/core/repository/package_info.py @@ -23,7 +23,7 @@ from tempfile import TemporaryDirectory from ahriman.core.build_tools.sources import Sources from ahriman.core.repository.repository_properties import RepositoryProperties -from ahriman.core.util import package_like +from ahriman.core.utils import package_like from ahriman.models.changes import Changes from ahriman.models.package import Package diff --git a/src/ahriman/core/sign/gpg.py b/src/ahriman/core/sign/gpg.py index 82acb5f2..f583a697 100644 --- a/src/ahriman/core/sign/gpg.py +++ b/src/ahriman/core/sign/gpg.py @@ -22,7 +22,7 @@ from pathlib import Path from ahriman.core.configuration import Configuration from ahriman.core.exceptions import BuildError from ahriman.core.http import SyncHttpClient -from ahriman.core.util import check_output +from ahriman.core.utils import check_output from ahriman.models.sign_settings import SignSettings diff --git a/src/ahriman/core/support/pkgbuild/pkgbuild_generator.py b/src/ahriman/core/support/pkgbuild/pkgbuild_generator.py index 149386b2..83f7770d 100644 --- a/src/ahriman/core/support/pkgbuild/pkgbuild_generator.py +++ b/src/ahriman/core/support/pkgbuild/pkgbuild_generator.py @@ -23,7 +23,7 @@ import itertools from collections.abc import Callable, Generator from pathlib import Path -from ahriman.core.util import utcnow +from ahriman.core.utils import utcnow from ahriman.models.pkgbuild_patch import PkgbuildPatch diff --git a/src/ahriman/core/tree.py b/src/ahriman/core/tree.py index 4a3a8585..28e2fb97 100644 --- a/src/ahriman/core/tree.py +++ b/src/ahriman/core/tree.py @@ -23,7 +23,7 @@ from collections.abc import Iterable from functools import partial from ahriman.core.exceptions import PartitionError -from ahriman.core.util import minmax, partition +from ahriman.core.utils import minmax, partition from ahriman.models.package import Package diff --git a/src/ahriman/core/upload/github.py b/src/ahriman/core/upload/github.py index e10cbb6d..a0916385 100644 --- a/src/ahriman/core/upload/github.py +++ b/src/ahriman/core/upload/github.py @@ -26,7 +26,7 @@ from typing import Any from ahriman.core.configuration import Configuration from ahriman.core.upload.http_upload import HttpUpload from ahriman.core.upload.upload import Upload -from ahriman.core.util import walk +from ahriman.core.utils import walk from ahriman.models.package import Package from ahriman.models.repository_id import RepositoryId diff --git a/src/ahriman/core/upload/rsync.py b/src/ahriman/core/upload/rsync.py index c81f3c87..2486dc1d 100644 --- a/src/ahriman/core/upload/rsync.py +++ b/src/ahriman/core/upload/rsync.py @@ -21,7 +21,7 @@ from pathlib import Path from ahriman.core.configuration import Configuration from ahriman.core.upload.upload import Upload -from ahriman.core.util import check_output +from ahriman.core.utils import check_output from ahriman.models.package import Package from ahriman.models.repository_id import RepositoryId diff --git a/src/ahriman/core/upload/s3.py b/src/ahriman/core/upload/s3.py index 372655cb..5a60c934 100644 --- a/src/ahriman/core/upload/s3.py +++ b/src/ahriman/core/upload/s3.py @@ -26,7 +26,7 @@ from typing import Any from ahriman.core.configuration import Configuration from ahriman.core.upload.upload import Upload -from ahriman.core.util import walk +from ahriman.core.utils import walk from ahriman.models.package import Package from ahriman.models.repository_id import RepositoryId diff --git a/src/ahriman/core/util.py b/src/ahriman/core/util.py index fac8e6c0..085e64d6 100644 --- a/src/ahriman/core/util.py +++ b/src/ahriman/core/util.py @@ -17,488 +17,5 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -# pylint: disable=too-many-lines -import datetime -import io -import itertools -import logging -import os -import re -import selectors -import subprocess - -from collections.abc import Callable, Generator, Iterable -from dataclasses import asdict -from enum import Enum -from pathlib import Path -from pwd import getpwuid -from typing import Any, IO, TypeVar - -from ahriman.core.exceptions import CalledProcessError, OptionError, UnsafeRunError -from ahriman.models.repository_paths import RepositoryPaths - - -__all__ = [ - "check_output", - "check_user", - "dataclass_view", - "enum_values", - "extract_user", - "filter_json", - "full_version", - "minmax", - "package_like", - "parse_version", - "partition", - "pretty_datetime", - "pretty_size", - "safe_filename", - "srcinfo_property", - "srcinfo_property_list", - "trim_package", - "utcnow", - "walk", -] - - -T = TypeVar("T") - - -# pylint: disable=too-many-locals -def check_output(*args: str, exception: Exception | Callable[[int, list[str], str, str], Exception] | None = None, - cwd: Path | None = None, input_data: str | None = None, - logger: logging.Logger | None = None, user: int | None = None, - environment: dict[str, str] | None = None) -> str: - """ - subprocess wrapper - - Args: - *args(str): command line arguments - exception(Exception | Callable[[int, list[str], str, str]] | None, optional): exception which has to be raised - instead of default subprocess exception. If callable us is supplied, the - :exc:`subprocess.CalledProcessError` arguments will be passed (Default value = None) - cwd(Path | None, optional): current working directory (Default value = None) - input_data(str | None, optional): data which will be written to command stdin (Default value = None) - logger(logging.Logger | None, optional): logger to log command result if required (Default value = None) - user(int | None, optional): run process as specified user (Default value = None) - environment(dict[str, str] | None, optional): optional environment variables if any (Default value = None) - - Returns: - str: command output - - Raises: - CalledProcessError: if subprocess ended with status code different from 0 and no exception supplied - - Examples: - Simply call the function:: - - >>> check_output("echo", "hello world") - - The more complicated calls which include result logging and input data are also possible:: - - >>> import logging - >>> - >>> logger = logging.getLogger() - >>> check_output("python", "-c", "greeting = input('say hello: '); print(); print(greeting)", - >>> input_data="hello world", logger=logger) - - An additional argument ``exception`` can be supplied in order to override the default exception:: - - >>> check_output("false", exception=RuntimeError("An exception occurred")) - """ - # hack for IO[str] handle - def get_io(proc: subprocess.Popen[str], channel_name: str) -> IO[str]: - channel: IO[str] | None = getattr(proc, channel_name, None) - return channel if channel is not None else io.StringIO() - - # wrapper around selectors polling - def poll(sel: selectors.BaseSelector) -> Generator[tuple[str, str], None, None]: - for key, _ in sel.select(): # we don't need to check mask here because we have only subscribed on reading - line = key.fileobj.readline() # type: ignore[union-attr] - if not line: # in case of empty line we remove selector as there is no data here anymore - sel.unregister(key.fileobj) - continue - line = line.rstrip() - - if logger is not None: - logger.debug(line) - - yield key.data, line - - # build system environment based on args and current environment - environment = environment or {} - if user is not None: - environment["HOME"] = getpwuid(user).pw_dir - full_environment = { - key: value - for key, value in os.environ.items() - if key in ("PATH",) # whitelisted variables only - } | environment - - with subprocess.Popen(args, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - user=user, env=full_environment, text=True, encoding="utf8", bufsize=1) as process: - if input_data is not None: - input_channel = get_io(process, "stdin") - input_channel.write(input_data) - input_channel.close() - - selector = selectors.DefaultSelector() - selector.register(get_io(process, "stdout"), selectors.EVENT_READ, data="stdout") - selector.register(get_io(process, "stderr"), selectors.EVENT_READ, data="stderr") - - result: dict[str, list[str]] = { - "stdout": [], - "stderr": [], - } - while selector.get_map(): # while there are unread selectors, keep reading - for key_data, output in poll(selector): - result[key_data].append(output) - - stdout = "\n".join(result["stdout"]).rstrip("\n") # remove newline at the end of any - stderr = "\n".join(result["stderr"]).rstrip("\n") - - status_code = process.wait() - if status_code != 0: - if isinstance(exception, Exception): - raise exception - if callable(exception): - raise exception(status_code, list(args), stdout, stderr) - raise CalledProcessError(status_code, list(args), stderr) - - return stdout - - -def check_user(paths: RepositoryPaths, *, unsafe: bool) -> None: - """ - check if current user is the owner of the root - - Args: - paths(RepositoryPaths): repository paths object - unsafe(bool): if set no user check will be performed before path creation - - Raises: - UnsafeRunError: if root uid differs from current uid and check is enabled - - Examples: - Simply run function with arguments:: - - >>> check_user(paths, unsafe=False) - """ - if not paths.root.exists(): - return # no directory found, skip check - if unsafe: - return # unsafe flag is enabled, no check performed - current_uid = os.getuid() - root_uid, _ = paths.root_owner - if current_uid != root_uid: - raise UnsafeRunError(current_uid, root_uid) - - -def dataclass_view(instance: Any) -> dict[str, Any]: - """ - convert dataclass instance to json object - - Args: - instance(Any): dataclass instance - - Returns: - dict[str, Any]: json representation of the dataclass with empty field removed - """ - return asdict(instance, dict_factory=lambda fields: {key: value for key, value in fields if value is not None}) - - -def enum_values(enum: type[Enum]) -> list[str]: - """ - generate list of enumeration values from the source - - Args: - enum(type[Enum]): source enumeration class - - Returns: - list[str]: available enumeration values as string - """ - return [str(key.value) for key in enum] # explicit str conversion for typing - - -def extract_user() -> str | None: - """ - extract user from system environment - - Returns: - str | None: SUDO_USER in case if set and USER otherwise. It can return None in case if environment has been - cleared before application start - """ - return os.getenv("SUDO_USER") or os.getenv("DOAS_USER") or os.getenv("USER") - - -def filter_json(source: dict[str, Any], known_fields: Iterable[str]) -> dict[str, Any]: - """ - filter json object by fields used for json-to-object conversion - - Args: - source(dict[str, Any]): raw json object - known_fields(Iterable[str]): list of fields which have to be known for the target object - - Returns: - dict[str, Any]: json object without unknown and empty fields - - Examples: - This wrapper is mainly used for the dataclasses, thus the flow must be something like this:: - - >>> from dataclasses import fields - >>> from ahriman.models.package import Package - >>> - >>> known_fields = [pair.name for pair in fields(Package)] - >>> properties = filter_json(dump, known_fields) - >>> package = Package(**properties) - """ - return {key: value for key, value in source.items() if key in known_fields and value is not None} - - -def full_version(epoch: str | int | None, pkgver: str, pkgrel: str) -> str: - """ - generate full version from components - - Args: - epoch(str | int | None): package epoch if any - pkgver(str): package version - pkgrel(str): package release version (arch linux specific) - - Returns: - str: generated version - """ - prefix = f"{epoch}:" if epoch else "" - return f"{prefix}{pkgver}-{pkgrel}" - - -def minmax(source: Iterable[T], *, key: Callable[[T], Any] | None = None) -> tuple[T, T]: - """ - get min and max value from iterable - - Args: - source(Iterable[T]): source list to find min and max values - key(Callable[[T], Any] | None, optional): key to sort (Default value = None) - - Returns: - tuple[T, T]: min and max values for sequence - """ - first_iter, second_iter = itertools.tee(source) - # typing doesn't expose SupportLessThan, so we just ignore this in typecheck - return min(first_iter, key=key), max(second_iter, key=key) # type: ignore - - -def package_like(filename: Path) -> bool: - """ - check if file looks like package - - Args: - filename(Path): name of file to check - - Returns: - bool: True in case if name contains ``.pkg.`` and not signature, False otherwise - """ - name = filename.name - return not name.startswith(".") and ".pkg." in name and not name.endswith(".sig") - - -def parse_version(version: str) -> tuple[str | None, str, str]: - """ - parse version and returns its components - - Args: - version(str): full version string - - Returns: - tuple[str | None, str, str]: epoch if any, pkgver and pkgrel variables - """ - if ":" in version: - epoch, version = version.split(":", maxsplit=1) - else: - epoch = None - pkgver, pkgrel = version.rsplit("-", maxsplit=1) - - return epoch, pkgver, pkgrel - - -def partition(source: Iterable[T], predicate: Callable[[T], bool]) -> tuple[list[T], list[T]]: - """ - partition list into two based on predicate, based on https://docs.python.org/dev/library/itertools.html#itertools-recipes - - Args: - source(Iterable[T]): source list to be partitioned - predicate(Callable[[T], bool]): filter function - - Returns: - tuple[list[T], list[T]]: two lists, first is which ``predicate`` is ``True``, second is ``False`` - """ - first_iter, second_iter = itertools.tee(source) - return list(filter(predicate, first_iter)), list(itertools.filterfalse(predicate, second_iter)) - - -def pretty_datetime(timestamp: datetime.datetime | float | int | None) -> str: - """ - convert datetime object to string - - Args: - timestamp(datetime.datetime | float | int | None): datetime to convert - - Returns: - str: pretty printable datetime as string - """ - if timestamp is None: - return "" - if isinstance(timestamp, (int, float)): - timestamp = datetime.datetime.fromtimestamp(timestamp, datetime.UTC) - return timestamp.strftime("%Y-%m-%d %H:%M:%S") - - -def pretty_size(size: float | None, level: int = 0) -> str: - """ - convert size to string - - Args: - size(float | None): size to convert - level(int, optional): represents current units, 0 is B, 1 is KiB, etc. (Default value = 0) - - Returns: - str: pretty printable size as string - - Raises: - OptionError: if size is more than 1TiB - """ - def str_level() -> str: - match level: - case 0: - return "B" - case 1: - return "KiB" - case 2: - return "MiB" - case 3: - return "GiB" - case _: - raise OptionError(level) # must never happen actually - - if size is None: - return "" - if size < 1024 or level >= 3: - return f"{size:.1f} {str_level()}" - return pretty_size(size / 1024, level + 1) - - -def safe_filename(source: str) -> str: - """ - convert source string to its safe representation - - Args: - source(str): string to convert - - Returns: - str: result string in which all unsafe characters are replaced by dash - """ - # RFC-3986 https://datatracker.ietf.org/doc/html/rfc3986 states that unreserved characters are - # https://datatracker.ietf.org/doc/html/rfc3986#section-2.3 - # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" - # however we would like to allow some gen-delims characters in filename, because those characters are used - # as delimiter in other URI parts. The ones we allow to are: - # ":" - used as separator in schema and userinfo - # "[" and "]" - used for host part - # "@" - used as separator between host and userinfo - return re.sub(r"[^A-Za-z\d\-._~:\[\]@]", "-", source) - - -def srcinfo_property(key: str, srcinfo: dict[str, Any], package_srcinfo: dict[str, Any], *, - default: Any = None) -> Any: - """ - extract property from SRCINFO. This method extracts property from package if this property is presented in - ``srcinfo``. Otherwise, it looks for the same property in root srcinfo. If none found, the default value will be - returned - - Args: - key(str): key to extract - srcinfo(dict[str, Any]): root structure of SRCINFO - package_srcinfo(dict[str, Any]): package specific SRCINFO - default(Any, optional): the default value for the specified key (Default value = None) - - Returns: - Any: extracted value from SRCINFO - """ - return package_srcinfo.get(key) or srcinfo.get(key) or default - - -def srcinfo_property_list(key: str, srcinfo: dict[str, Any], package_srcinfo: dict[str, Any], *, - architecture: str | None = None) -> list[Any]: - """ - extract list property from SRCINFO. Unlike :func:`srcinfo_property()` it supposes that default return value is - always empty list. If ``architecture`` is supplied, then it will try to lookup for architecture specific values and - will append it at the end of result - - Args: - key(str): key to extract - srcinfo(dict[str, Any]): root structure of SRCINFO - package_srcinfo(dict[str, Any]): package specific SRCINFO - architecture(str | None, optional): package architecture if set (Default value = None) - - Returns: - list[Any]: list of extracted properties from SRCINFO - """ - values: list[Any] = srcinfo_property(key, srcinfo, package_srcinfo, default=[]) - if architecture is not None: - values.extend(srcinfo_property(f"{key}_{architecture}", srcinfo, package_srcinfo, default=[])) - return values - - -def trim_package(package_name: str) -> str: - """ - remove version bound and description from package name. Pacman allows to specify version bound (=, <=, >= etc.) for - packages in dependencies and also allows to specify description (via ``:``); this function removes trailing parts - and return exact package name - - Args: - package_name(str): source package name - - Returns: - str: package name without description or version bound - """ - for symbol in ("<", "=", ">", ":"): - package_name = package_name.partition(symbol)[0] - return package_name - - -def utcnow() -> datetime.datetime: - """ - get current time - - Returns: - datetime.datetime: current time in UTC - """ - return datetime.datetime.now(datetime.UTC) - - -def walk(directory_path: Path) -> Generator[Path, None, None]: - """ - list all file paths in given directory - Credits to https://stackoverflow.com/a/64915960 - - Args: - directory_path(Path): root directory path - - Yields: - Path: all found files in given directory with full path - - Examples: - Since the :mod:`pathlib` module does not provide an alternative to :func:`os.walk()`, this wrapper - can be used instead:: - - >>> from pathlib import Path - >>> - >>> for file_path in walk(Path.cwd()): - >>> print(file_path) - - Note, however, that unlike the original method, it does not yield directories. - """ - for element in directory_path.iterdir(): - if element.is_dir(): - yield from walk(element) - continue - yield element +# backward compatibility wrapper +from ahriman.core.utils import * # pylint: disable=wildcard-import,unused-wildcard-import diff --git a/src/ahriman/core/utils.py b/src/ahriman/core/utils.py new file mode 100644 index 00000000..fac8e6c0 --- /dev/null +++ b/src/ahriman/core/utils.py @@ -0,0 +1,504 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# pylint: disable=too-many-lines +import datetime +import io +import itertools +import logging +import os +import re +import selectors +import subprocess + +from collections.abc import Callable, Generator, Iterable +from dataclasses import asdict +from enum import Enum +from pathlib import Path +from pwd import getpwuid +from typing import Any, IO, TypeVar + +from ahriman.core.exceptions import CalledProcessError, OptionError, UnsafeRunError +from ahriman.models.repository_paths import RepositoryPaths + + +__all__ = [ + "check_output", + "check_user", + "dataclass_view", + "enum_values", + "extract_user", + "filter_json", + "full_version", + "minmax", + "package_like", + "parse_version", + "partition", + "pretty_datetime", + "pretty_size", + "safe_filename", + "srcinfo_property", + "srcinfo_property_list", + "trim_package", + "utcnow", + "walk", +] + + +T = TypeVar("T") + + +# pylint: disable=too-many-locals +def check_output(*args: str, exception: Exception | Callable[[int, list[str], str, str], Exception] | None = None, + cwd: Path | None = None, input_data: str | None = None, + logger: logging.Logger | None = None, user: int | None = None, + environment: dict[str, str] | None = None) -> str: + """ + subprocess wrapper + + Args: + *args(str): command line arguments + exception(Exception | Callable[[int, list[str], str, str]] | None, optional): exception which has to be raised + instead of default subprocess exception. If callable us is supplied, the + :exc:`subprocess.CalledProcessError` arguments will be passed (Default value = None) + cwd(Path | None, optional): current working directory (Default value = None) + input_data(str | None, optional): data which will be written to command stdin (Default value = None) + logger(logging.Logger | None, optional): logger to log command result if required (Default value = None) + user(int | None, optional): run process as specified user (Default value = None) + environment(dict[str, str] | None, optional): optional environment variables if any (Default value = None) + + Returns: + str: command output + + Raises: + CalledProcessError: if subprocess ended with status code different from 0 and no exception supplied + + Examples: + Simply call the function:: + + >>> check_output("echo", "hello world") + + The more complicated calls which include result logging and input data are also possible:: + + >>> import logging + >>> + >>> logger = logging.getLogger() + >>> check_output("python", "-c", "greeting = input('say hello: '); print(); print(greeting)", + >>> input_data="hello world", logger=logger) + + An additional argument ``exception`` can be supplied in order to override the default exception:: + + >>> check_output("false", exception=RuntimeError("An exception occurred")) + """ + # hack for IO[str] handle + def get_io(proc: subprocess.Popen[str], channel_name: str) -> IO[str]: + channel: IO[str] | None = getattr(proc, channel_name, None) + return channel if channel is not None else io.StringIO() + + # wrapper around selectors polling + def poll(sel: selectors.BaseSelector) -> Generator[tuple[str, str], None, None]: + for key, _ in sel.select(): # we don't need to check mask here because we have only subscribed on reading + line = key.fileobj.readline() # type: ignore[union-attr] + if not line: # in case of empty line we remove selector as there is no data here anymore + sel.unregister(key.fileobj) + continue + line = line.rstrip() + + if logger is not None: + logger.debug(line) + + yield key.data, line + + # build system environment based on args and current environment + environment = environment or {} + if user is not None: + environment["HOME"] = getpwuid(user).pw_dir + full_environment = { + key: value + for key, value in os.environ.items() + if key in ("PATH",) # whitelisted variables only + } | environment + + with subprocess.Popen(args, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + user=user, env=full_environment, text=True, encoding="utf8", bufsize=1) as process: + if input_data is not None: + input_channel = get_io(process, "stdin") + input_channel.write(input_data) + input_channel.close() + + selector = selectors.DefaultSelector() + selector.register(get_io(process, "stdout"), selectors.EVENT_READ, data="stdout") + selector.register(get_io(process, "stderr"), selectors.EVENT_READ, data="stderr") + + result: dict[str, list[str]] = { + "stdout": [], + "stderr": [], + } + while selector.get_map(): # while there are unread selectors, keep reading + for key_data, output in poll(selector): + result[key_data].append(output) + + stdout = "\n".join(result["stdout"]).rstrip("\n") # remove newline at the end of any + stderr = "\n".join(result["stderr"]).rstrip("\n") + + status_code = process.wait() + if status_code != 0: + if isinstance(exception, Exception): + raise exception + if callable(exception): + raise exception(status_code, list(args), stdout, stderr) + raise CalledProcessError(status_code, list(args), stderr) + + return stdout + + +def check_user(paths: RepositoryPaths, *, unsafe: bool) -> None: + """ + check if current user is the owner of the root + + Args: + paths(RepositoryPaths): repository paths object + unsafe(bool): if set no user check will be performed before path creation + + Raises: + UnsafeRunError: if root uid differs from current uid and check is enabled + + Examples: + Simply run function with arguments:: + + >>> check_user(paths, unsafe=False) + """ + if not paths.root.exists(): + return # no directory found, skip check + if unsafe: + return # unsafe flag is enabled, no check performed + current_uid = os.getuid() + root_uid, _ = paths.root_owner + if current_uid != root_uid: + raise UnsafeRunError(current_uid, root_uid) + + +def dataclass_view(instance: Any) -> dict[str, Any]: + """ + convert dataclass instance to json object + + Args: + instance(Any): dataclass instance + + Returns: + dict[str, Any]: json representation of the dataclass with empty field removed + """ + return asdict(instance, dict_factory=lambda fields: {key: value for key, value in fields if value is not None}) + + +def enum_values(enum: type[Enum]) -> list[str]: + """ + generate list of enumeration values from the source + + Args: + enum(type[Enum]): source enumeration class + + Returns: + list[str]: available enumeration values as string + """ + return [str(key.value) for key in enum] # explicit str conversion for typing + + +def extract_user() -> str | None: + """ + extract user from system environment + + Returns: + str | None: SUDO_USER in case if set and USER otherwise. It can return None in case if environment has been + cleared before application start + """ + return os.getenv("SUDO_USER") or os.getenv("DOAS_USER") or os.getenv("USER") + + +def filter_json(source: dict[str, Any], known_fields: Iterable[str]) -> dict[str, Any]: + """ + filter json object by fields used for json-to-object conversion + + Args: + source(dict[str, Any]): raw json object + known_fields(Iterable[str]): list of fields which have to be known for the target object + + Returns: + dict[str, Any]: json object without unknown and empty fields + + Examples: + This wrapper is mainly used for the dataclasses, thus the flow must be something like this:: + + >>> from dataclasses import fields + >>> from ahriman.models.package import Package + >>> + >>> known_fields = [pair.name for pair in fields(Package)] + >>> properties = filter_json(dump, known_fields) + >>> package = Package(**properties) + """ + return {key: value for key, value in source.items() if key in known_fields and value is not None} + + +def full_version(epoch: str | int | None, pkgver: str, pkgrel: str) -> str: + """ + generate full version from components + + Args: + epoch(str | int | None): package epoch if any + pkgver(str): package version + pkgrel(str): package release version (arch linux specific) + + Returns: + str: generated version + """ + prefix = f"{epoch}:" if epoch else "" + return f"{prefix}{pkgver}-{pkgrel}" + + +def minmax(source: Iterable[T], *, key: Callable[[T], Any] | None = None) -> tuple[T, T]: + """ + get min and max value from iterable + + Args: + source(Iterable[T]): source list to find min and max values + key(Callable[[T], Any] | None, optional): key to sort (Default value = None) + + Returns: + tuple[T, T]: min and max values for sequence + """ + first_iter, second_iter = itertools.tee(source) + # typing doesn't expose SupportLessThan, so we just ignore this in typecheck + return min(first_iter, key=key), max(second_iter, key=key) # type: ignore + + +def package_like(filename: Path) -> bool: + """ + check if file looks like package + + Args: + filename(Path): name of file to check + + Returns: + bool: True in case if name contains ``.pkg.`` and not signature, False otherwise + """ + name = filename.name + return not name.startswith(".") and ".pkg." in name and not name.endswith(".sig") + + +def parse_version(version: str) -> tuple[str | None, str, str]: + """ + parse version and returns its components + + Args: + version(str): full version string + + Returns: + tuple[str | None, str, str]: epoch if any, pkgver and pkgrel variables + """ + if ":" in version: + epoch, version = version.split(":", maxsplit=1) + else: + epoch = None + pkgver, pkgrel = version.rsplit("-", maxsplit=1) + + return epoch, pkgver, pkgrel + + +def partition(source: Iterable[T], predicate: Callable[[T], bool]) -> tuple[list[T], list[T]]: + """ + partition list into two based on predicate, based on https://docs.python.org/dev/library/itertools.html#itertools-recipes + + Args: + source(Iterable[T]): source list to be partitioned + predicate(Callable[[T], bool]): filter function + + Returns: + tuple[list[T], list[T]]: two lists, first is which ``predicate`` is ``True``, second is ``False`` + """ + first_iter, second_iter = itertools.tee(source) + return list(filter(predicate, first_iter)), list(itertools.filterfalse(predicate, second_iter)) + + +def pretty_datetime(timestamp: datetime.datetime | float | int | None) -> str: + """ + convert datetime object to string + + Args: + timestamp(datetime.datetime | float | int | None): datetime to convert + + Returns: + str: pretty printable datetime as string + """ + if timestamp is None: + return "" + if isinstance(timestamp, (int, float)): + timestamp = datetime.datetime.fromtimestamp(timestamp, datetime.UTC) + return timestamp.strftime("%Y-%m-%d %H:%M:%S") + + +def pretty_size(size: float | None, level: int = 0) -> str: + """ + convert size to string + + Args: + size(float | None): size to convert + level(int, optional): represents current units, 0 is B, 1 is KiB, etc. (Default value = 0) + + Returns: + str: pretty printable size as string + + Raises: + OptionError: if size is more than 1TiB + """ + def str_level() -> str: + match level: + case 0: + return "B" + case 1: + return "KiB" + case 2: + return "MiB" + case 3: + return "GiB" + case _: + raise OptionError(level) # must never happen actually + + if size is None: + return "" + if size < 1024 or level >= 3: + return f"{size:.1f} {str_level()}" + return pretty_size(size / 1024, level + 1) + + +def safe_filename(source: str) -> str: + """ + convert source string to its safe representation + + Args: + source(str): string to convert + + Returns: + str: result string in which all unsafe characters are replaced by dash + """ + # RFC-3986 https://datatracker.ietf.org/doc/html/rfc3986 states that unreserved characters are + # https://datatracker.ietf.org/doc/html/rfc3986#section-2.3 + # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + # however we would like to allow some gen-delims characters in filename, because those characters are used + # as delimiter in other URI parts. The ones we allow to are: + # ":" - used as separator in schema and userinfo + # "[" and "]" - used for host part + # "@" - used as separator between host and userinfo + return re.sub(r"[^A-Za-z\d\-._~:\[\]@]", "-", source) + + +def srcinfo_property(key: str, srcinfo: dict[str, Any], package_srcinfo: dict[str, Any], *, + default: Any = None) -> Any: + """ + extract property from SRCINFO. This method extracts property from package if this property is presented in + ``srcinfo``. Otherwise, it looks for the same property in root srcinfo. If none found, the default value will be + returned + + Args: + key(str): key to extract + srcinfo(dict[str, Any]): root structure of SRCINFO + package_srcinfo(dict[str, Any]): package specific SRCINFO + default(Any, optional): the default value for the specified key (Default value = None) + + Returns: + Any: extracted value from SRCINFO + """ + return package_srcinfo.get(key) or srcinfo.get(key) or default + + +def srcinfo_property_list(key: str, srcinfo: dict[str, Any], package_srcinfo: dict[str, Any], *, + architecture: str | None = None) -> list[Any]: + """ + extract list property from SRCINFO. Unlike :func:`srcinfo_property()` it supposes that default return value is + always empty list. If ``architecture`` is supplied, then it will try to lookup for architecture specific values and + will append it at the end of result + + Args: + key(str): key to extract + srcinfo(dict[str, Any]): root structure of SRCINFO + package_srcinfo(dict[str, Any]): package specific SRCINFO + architecture(str | None, optional): package architecture if set (Default value = None) + + Returns: + list[Any]: list of extracted properties from SRCINFO + """ + values: list[Any] = srcinfo_property(key, srcinfo, package_srcinfo, default=[]) + if architecture is not None: + values.extend(srcinfo_property(f"{key}_{architecture}", srcinfo, package_srcinfo, default=[])) + return values + + +def trim_package(package_name: str) -> str: + """ + remove version bound and description from package name. Pacman allows to specify version bound (=, <=, >= etc.) for + packages in dependencies and also allows to specify description (via ``:``); this function removes trailing parts + and return exact package name + + Args: + package_name(str): source package name + + Returns: + str: package name without description or version bound + """ + for symbol in ("<", "=", ">", ":"): + package_name = package_name.partition(symbol)[0] + return package_name + + +def utcnow() -> datetime.datetime: + """ + get current time + + Returns: + datetime.datetime: current time in UTC + """ + return datetime.datetime.now(datetime.UTC) + + +def walk(directory_path: Path) -> Generator[Path, None, None]: + """ + list all file paths in given directory + Credits to https://stackoverflow.com/a/64915960 + + Args: + directory_path(Path): root directory path + + Yields: + Path: all found files in given directory with full path + + Examples: + Since the :mod:`pathlib` module does not provide an alternative to :func:`os.walk()`, this wrapper + can be used instead:: + + >>> from pathlib import Path + >>> + >>> for file_path in walk(Path.cwd()): + >>> print(file_path) + + Note, however, that unlike the original method, it does not yield directories. + """ + for element in directory_path.iterdir(): + if element.is_dir(): + yield from walk(element) + continue + yield element diff --git a/src/ahriman/models/aur_package.py b/src/ahriman/models/aur_package.py index 6f143955..ebc38825 100644 --- a/src/ahriman/models/aur_package.py +++ b/src/ahriman/models/aur_package.py @@ -25,7 +25,7 @@ from dataclasses import dataclass, field, fields from pyalpm import Package # type: ignore[import-not-found] from typing import Any, Self -from ahriman.core.util import filter_json, full_version +from ahriman.core.utils import filter_json, full_version @dataclass(frozen=True, kw_only=True) diff --git a/src/ahriman/models/build_status.py b/src/ahriman/models/build_status.py index 7621c344..e0a55ba4 100644 --- a/src/ahriman/models/build_status.py +++ b/src/ahriman/models/build_status.py @@ -21,7 +21,7 @@ from dataclasses import dataclass, field, fields from enum import StrEnum from typing import Any, Self -from ahriman.core.util import filter_json, pretty_datetime, utcnow +from ahriman.core.utils import filter_json, pretty_datetime, utcnow class BuildStatusEnum(StrEnum): diff --git a/src/ahriman/models/changes.py b/src/ahriman/models/changes.py index f74fdd77..bd6de1fa 100644 --- a/src/ahriman/models/changes.py +++ b/src/ahriman/models/changes.py @@ -20,7 +20,7 @@ from dataclasses import dataclass, fields from typing import Any, Self -from ahriman.core.util import dataclass_view, filter_json +from ahriman.core.utils import dataclass_view, filter_json @dataclass(frozen=True) diff --git a/src/ahriman/models/counters.py b/src/ahriman/models/counters.py index 1f9bca63..c17b9001 100644 --- a/src/ahriman/models/counters.py +++ b/src/ahriman/models/counters.py @@ -20,7 +20,7 @@ from dataclasses import dataclass, fields from typing import Any, Self -from ahriman.core.util import filter_json +from ahriman.core.utils import filter_json from ahriman.models.build_status import BuildStatus from ahriman.models.package import Package diff --git a/src/ahriman/models/dependencies.py b/src/ahriman/models/dependencies.py index 65b62875..b9fd84ee 100644 --- a/src/ahriman/models/dependencies.py +++ b/src/ahriman/models/dependencies.py @@ -20,7 +20,7 @@ from dataclasses import dataclass, field, fields from typing import Any, Self -from ahriman.core.util import dataclass_view, filter_json +from ahriman.core.utils import dataclass_view, filter_json @dataclass(frozen=True) diff --git a/src/ahriman/models/filesystem_package.py b/src/ahriman/models/filesystem_package.py index 633484e6..11416d37 100644 --- a/src/ahriman/models/filesystem_package.py +++ b/src/ahriman/models/filesystem_package.py @@ -23,7 +23,7 @@ from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path -from ahriman.core.util import trim_package +from ahriman.core.utils import trim_package @dataclass(frozen=True, kw_only=True) diff --git a/src/ahriman/models/internal_status.py b/src/ahriman/models/internal_status.py index b014529d..204760f1 100644 --- a/src/ahriman/models/internal_status.py +++ b/src/ahriman/models/internal_status.py @@ -20,7 +20,7 @@ from dataclasses import dataclass, field from typing import Any, Self -from ahriman.core.util import dataclass_view +from ahriman.core.utils import dataclass_view from ahriman.models.build_status import BuildStatus from ahriman.models.counters import Counters diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 9afdbc30..adfd7296 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -34,7 +34,7 @@ from ahriman.core.alpm.pacman import Pacman from ahriman.core.alpm.remote import AUR, Official, OfficialSyncdb from ahriman.core.exceptions import PackageInfoError from ahriman.core.log import LazyLogging -from ahriman.core.util import check_output, dataclass_view, full_version, parse_version, srcinfo_property_list, utcnow +from ahriman.core.utils import check_output, dataclass_view, full_version, parse_version, srcinfo_property_list, utcnow from ahriman.models.package_description import PackageDescription from ahriman.models.package_source import PackageSource from ahriman.models.remote_source import RemoteSource @@ -539,20 +539,23 @@ class Package(LazyLogging): result: int = vercmp(self.version, remote_version) return result < 0 - def next_pkgrel(self, local_version: str) -> str | None: + def next_pkgrel(self, local_version: str | None) -> str | None: """ generate next pkgrel variable. The package release will be incremented if ``local_version`` is more or equal to the :attr:`version`; in this case the function will return new pkgrel value, otherwise ``None`` will be returned Args: - local_version(str): locally stored package version + local_version(str | None): locally stored package version if available Returns: str | None: new generated package release version if any. In case if the release contains dot (e.g. 1.2), the minor part will be incremented by 1. If the release does not contain major.minor notation, the minor version equals to 1 will be appended """ + if local_version is None: + return None # local version not found, keep upstream pkgrel + epoch, pkgver, _ = parse_version(self.version) local_epoch, local_pkgver, local_pkgrel = parse_version(local_version) diff --git a/src/ahriman/models/package_archive.py b/src/ahriman/models/package_archive.py index 81bbab04..8e831983 100644 --- a/src/ahriman/models/package_archive.py +++ b/src/ahriman/models/package_archive.py @@ -26,7 +26,7 @@ from typing import IO from ahriman.core.alpm.pacman import Pacman from ahriman.core.alpm.remote import OfficialSyncdb from ahriman.core.exceptions import UnknownPackageError -from ahriman.core.util import walk +from ahriman.core.utils import walk from ahriman.models.dependencies import Dependencies from ahriman.models.filesystem_package import FilesystemPackage from ahriman.models.package import Package diff --git a/src/ahriman/models/package_description.py b/src/ahriman/models/package_description.py index 8e5388a0..baa25b3a 100644 --- a/src/ahriman/models/package_description.py +++ b/src/ahriman/models/package_description.py @@ -22,7 +22,7 @@ from pathlib import Path from pyalpm import Package # type: ignore[import-not-found] from typing import Any, Self -from ahriman.core.util import dataclass_view, filter_json, trim_package +from ahriman.core.utils import dataclass_view, filter_json, trim_package from ahriman.models.aur_package import AURPackage diff --git a/src/ahriman/models/package_source.py b/src/ahriman/models/package_source.py index 51f63ee9..2269ae42 100644 --- a/src/ahriman/models/package_source.py +++ b/src/ahriman/models/package_source.py @@ -23,7 +23,7 @@ from enum import StrEnum from pathlib import Path from urllib.parse import urlparse -from ahriman.core.util import package_like +from ahriman.core.utils import package_like from ahriman.models.repository_paths import RepositoryPaths diff --git a/src/ahriman/models/pkgbuild_patch.py b/src/ahriman/models/pkgbuild_patch.py index def9b14e..3a809b12 100644 --- a/src/ahriman/models/pkgbuild_patch.py +++ b/src/ahriman/models/pkgbuild_patch.py @@ -23,7 +23,7 @@ from dataclasses import dataclass, fields from pathlib import Path from typing import Any, Generator, Self -from ahriman.core.util import dataclass_view, filter_json +from ahriman.core.utils import dataclass_view, filter_json @dataclass(frozen=True) diff --git a/src/ahriman/models/remote_source.py b/src/ahriman/models/remote_source.py index 8f5a6b4f..1446a969 100644 --- a/src/ahriman/models/remote_source.py +++ b/src/ahriman/models/remote_source.py @@ -22,7 +22,7 @@ from pathlib import Path from typing import Any, Self from ahriman.core.exceptions import InitializeError -from ahriman.core.util import dataclass_view, filter_json +from ahriman.core.utils import dataclass_view, filter_json from ahriman.models.package_source import PackageSource diff --git a/src/ahriman/models/worker.py b/src/ahriman/models/worker.py index c6159789..c4800a4c 100644 --- a/src/ahriman/models/worker.py +++ b/src/ahriman/models/worker.py @@ -21,7 +21,7 @@ from dataclasses import dataclass, field from typing import Any from urllib.parse import urlparse -from ahriman.core.util import dataclass_view +from ahriman.core.utils import dataclass_view @dataclass(frozen=True) diff --git a/src/ahriman/web/views/api/swagger.py b/src/ahriman/web/views/api/swagger.py index aecaed12..ae16c57c 100644 --- a/src/ahriman/web/views/api/swagger.py +++ b/src/ahriman/web/views/api/swagger.py @@ -20,7 +20,7 @@ from aiohttp.web import Response, json_response from collections.abc import Callable -from ahriman.core.util import partition +from ahriman.core.utils import partition from ahriman.models.user_access import UserAccess from ahriman.web.views.base import BaseView diff --git a/src/ahriman/web/views/v1/packages/logs.py b/src/ahriman/web/views/v1/packages/logs.py index be59685e..3db6754f 100644 --- a/src/ahriman/web/views/v1/packages/logs.py +++ b/src/ahriman/web/views/v1/packages/logs.py @@ -22,7 +22,7 @@ import aiohttp_apispec # type: ignore[import-untyped] from aiohttp.web import HTTPBadRequest, HTTPNoContent, HTTPNotFound, Response, json_response from ahriman.core.exceptions import UnknownPackageError -from ahriman.core.util import pretty_datetime +from ahriman.core.utils import pretty_datetime from ahriman.models.log_record_id import LogRecordId from ahriman.models.user_access import UserAccess from ahriman.web.schemas import AuthSchema, ErrorSchema, LogsSchema, PackageNameSchema, PackageVersionSchema, \ diff --git a/tests/ahriman/core/formatters/test_update_printer.py b/tests/ahriman/core/formatters/test_update_printer.py index 286d655b..37dd0e9a 100644 --- a/tests/ahriman/core/formatters/test_update_printer.py +++ b/tests/ahriman/core/formatters/test_update_printer.py @@ -1,4 +1,5 @@ from ahriman.core.formatters import UpdatePrinter +from ahriman.models.property import Property def test_properties(update_printer: UpdatePrinter) -> None: @@ -8,6 +9,31 @@ def test_properties(update_printer: UpdatePrinter) -> None: assert update_printer.properties() +def test_properties_na(update_printer: UpdatePrinter) -> None: + """ + must return N/A if local version is unknown + """ + assert update_printer.properties() == [Property("N/A", update_printer.package.version, is_required=True)] + + +def test_properties_bump_pkgrel(update_printer: UpdatePrinter) -> None: + """ + must bump pkgrel if local version is the same + """ + update_printer.local_version = update_printer.package.version + assert update_printer.properties() == [Property(update_printer.package.version, "2.6.0-1.1", is_required=True)] + + +def test_properties_keep_pkgrel(update_printer: UpdatePrinter) -> None: + """ + must keep pkgrel if local version is not the same + """ + update_printer.local_version = "2.5.0-1" + assert update_printer.properties() == [ + Property(update_printer.local_version, update_printer.package.version, is_required=True), + ] + + def test_title(update_printer: UpdatePrinter) -> None: """ must return non-empty title diff --git a/tests/ahriman/core/test_util.py b/tests/ahriman/core/test_util.py index b26303fc..2dd3cc0d 100644 --- a/tests/ahriman/core/test_util.py +++ b/tests/ahriman/core/test_util.py @@ -1,496 +1,11 @@ -import datetime -import logging -import os -import pytest +import ahriman.core.utils -from pathlib import Path -from pytest_mock import MockerFixture -from typing import Any -from unittest.mock import call as MockCall +from ahriman.core.util import * -from ahriman.core.exceptions import BuildError, CalledProcessError, OptionError, UnsafeRunError -from ahriman.core.util import check_output, check_user, dataclass_view, enum_values, extract_user, filter_json, \ - full_version, minmax, package_like, parse_version, partition, pretty_datetime, pretty_size, safe_filename, \ - srcinfo_property, srcinfo_property_list, trim_package, utcnow, walk -from ahriman.models.package import Package -from ahriman.models.package_source import PackageSource -from ahriman.models.repository_id import RepositoryId -from ahriman.models.repository_paths import RepositoryPaths - -def test_check_output(mocker: MockerFixture) -> None: - """ - must run command and log result - """ - logger_mock = mocker.patch("logging.Logger.debug") - - assert check_output("echo", "hello") == "hello" - logger_mock.assert_not_called() - - assert check_output("echo", "hello", logger=logging.getLogger("")) == "hello" - logger_mock.assert_called_once_with("hello") - - -def test_check_output_stderr(mocker: MockerFixture) -> None: - """ - must run command and log stderr output - """ - logger_mock = mocker.patch("logging.Logger.debug") - - assert check_output("python", "-c", """import sys; print("hello", file=sys.stderr)""") == "" - logger_mock.assert_not_called() - - assert check_output("python", "-c", """import sys; print("hello", file=sys.stderr)""", - logger=logging.getLogger("")) == "" - logger_mock.assert_called_once_with("hello") - - -def test_check_output_with_stdin() -> None: - """ - must run command and put string to stdin - """ - assert check_output("python", "-c", "import sys; value = sys.stdin.read(); print(value)", - input_data="single line") == "single line" - - -def test_check_output_with_stdin_newline() -> None: - """ - must run command and put string to stdin ending with new line - """ - assert check_output("python", "-c", "import sys; value = sys.stdin.read(); print(value)", - input_data="single line\n") == "single line" - - -def test_check_output_multiple_with_stdin() -> None: - """ - must run command and put multiple lines to stdin - """ - assert check_output("python", "-c", "import sys; value = sys.stdin.read(); print(value)", - input_data="multiple\nlines") == "multiple\nlines" - - -def test_check_output_multiple_with_stdin_newline() -> None: - """ - must run command and put multiple lines to stdin with new line at the end - """ - assert check_output("python", "-c", "import sys; value = sys.stdin.read(); print(value)", - input_data="multiple\nlines\n") == "multiple\nlines" - - -def test_check_output_with_user(passwd: Any, mocker: MockerFixture) -> None: - """ - must run command as specified user and set its homedir - """ - assert check_output("python", "-c", """import os; print(os.getenv("HOME"))""") != passwd.pw_dir - - getpwuid_mock = mocker.patch("ahriman.core.util.getpwuid", return_value=passwd) - user = os.getuid() - - assert check_output("python", "-c", """import os; print(os.getenv("HOME"))""", user=user) == passwd.pw_dir - getpwuid_mock.assert_called_once_with(user) - - -def test_check_output_with_user_and_environment(passwd: Any, mocker: MockerFixture) -> None: - """ - must run set environment if both environment and user are set - """ - mocker.patch("ahriman.core.util.getpwuid", return_value=passwd) - user = os.getuid() - assert check_output("python", "-c", """import os; print(os.getenv("HOME"), os.getenv("VAR"))""", - environment={"VAR": "VALUE"}, user=user) == f"{passwd.pw_dir} VALUE" - - -def test_check_output_failure(mocker: MockerFixture) -> None: - """ - must process exception correctly - """ - mocker.patch("subprocess.Popen.wait", return_value=1) - - with pytest.raises(CalledProcessError): - check_output("echo", "hello") - - with pytest.raises(CalledProcessError): - check_output("echo", "hello", logger=logging.getLogger("")) - - -def test_check_output_failure_exception(mocker: MockerFixture) -> None: - """ - must raise exception provided instead of default - """ - mocker.patch("subprocess.Popen.wait", return_value=1) - exception = BuildError("") - - with pytest.raises(BuildError): - check_output("echo", "hello", exception=exception) - - with pytest.raises(BuildError): - check_output("echo", "hello", exception=exception, logger=logging.getLogger("")) - - -def test_check_output_failure_exception_callable(mocker: MockerFixture) -> None: - """ - must raise exception from callable provided instead of default - """ - mocker.patch("subprocess.Popen.wait", return_value=1) - exception = BuildError.from_process("") - - with pytest.raises(BuildError): - check_output("echo", "hello", exception=exception) - - with pytest.raises(BuildError): - check_output("echo", "hello", exception=exception, logger=logging.getLogger("")) - - -def test_check_output_empty_line(mocker: MockerFixture) -> None: - """ - must correctly process empty lines in command output - """ - logger_mock = mocker.patch("logging.Logger.debug") - assert check_output("python", "-c", """print(); print("hello")""", logger=logging.getLogger("")) == "\nhello" - logger_mock.assert_has_calls([MockCall(""), MockCall("hello")]) - - -def test_check_user(repository_id: RepositoryId, mocker: MockerFixture) -> None: - """ - must check user correctly - """ - paths = RepositoryPaths(Path.cwd(), repository_id) - mocker.patch("os.getuid", return_value=paths.root_owner[0]) - check_user(paths, unsafe=False) - - -def test_check_user_no_directory(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: - """ - must not fail in case if no directory found - """ - mocker.patch("pathlib.Path.exists", return_value=False) - check_user(repository_paths, unsafe=False) - - -def test_check_user_exception(repository_id: RepositoryId, mocker: MockerFixture) -> None: - """ - must raise exception if user differs - """ - paths = RepositoryPaths(Path.cwd(), repository_id) - mocker.patch("os.getuid", return_value=paths.root_owner[0] + 1) - - with pytest.raises(UnsafeRunError): - check_user(paths, unsafe=False) - - -def test_check_user_unsafe(repository_id: RepositoryId, mocker: MockerFixture) -> None: - """ - must skip check if unsafe flag is set - """ - paths = RepositoryPaths(Path.cwd(), repository_id) - mocker.patch("os.getuid", return_value=paths.root_owner[0] + 1) - check_user(paths, unsafe=True) - - -def test_dataclass_view(package_ahriman: Package) -> None: - """ - must serialize dataclasses - """ - assert Package.from_json(dataclass_view(package_ahriman)) == package_ahriman - - -def test_dataclass_view_without_none(package_ahriman: Package) -> None: - """ - must serialize dataclasses with None fields removed - """ - package_ahriman.packager = None - result = dataclass_view(package_ahriman) - assert "packager" not in result - assert Package.from_json(result) == package_ahriman - - -def test_enum_values() -> None: - """ - must correctly generate choices from enumeration classes - """ - values = enum_values(PackageSource) - for value in values: - assert PackageSource(value).value == value - - -def test_extract_user() -> None: - """ - must extract user from system environment - """ - os.environ["USER"] = "user" - assert extract_user() == "user" - - os.environ["SUDO_USER"] = "sudo" - assert extract_user() == "sudo" - - os.environ["DOAS_USER"] = "doas" - assert extract_user() == "sudo" - - del os.environ["SUDO_USER"] - assert extract_user() == "doas" - - -def test_filter_json(package_ahriman: Package) -> None: - """ - must filter fields by known list - """ - expected = package_ahriman.view() - probe = package_ahriman.view() - probe["unknown_field"] = "value" - - assert expected == filter_json(probe, expected.keys()) - - -def test_filter_json_empty_value(package_ahriman: Package) -> None: - """ - must filter empty values from object - """ - probe = package_ahriman.view() - probe["base"] = None - assert "base" not in filter_json(probe, probe.keys()) - - -def test_full_version() -> None: - """ - must construct full version - """ - assert full_version("1", "r2388.d30e3201", "1") == "1:r2388.d30e3201-1" - assert full_version(None, "0.12.1", "1") == "0.12.1-1" - assert full_version(0, "0.12.1", "1") == "0.12.1-1" - assert full_version(1, "0.12.1", "1") == "1:0.12.1-1" - - -def test_minmax() -> None: - """ - must correctly define minimal and maximal value - """ - assert minmax([1, 4, 3, 2]) == (1, 4) - assert minmax([[1, 2, 3], [4, 5], [6, 7, 8, 9]], key=len) == ([4, 5], [6, 7, 8, 9]) - - -def test_package_like(package_ahriman: Package) -> None: - """ - package_like must return true for archives - """ - assert package_like(package_ahriman.packages[package_ahriman.base].filepath) - - -def test_package_like_hidden(package_ahriman: Package) -> None: - """ - package_like must return false for hidden files - """ - package_file = package_ahriman.packages[package_ahriman.base].filepath - hidden_file = package_file.parent / f".{package_file.name}" - assert not package_like(hidden_file) - - -def test_package_like_sig(package_ahriman: Package) -> None: - """ - package_like must return false for signature files - """ - package_file = package_ahriman.packages[package_ahriman.base].filepath - sig_file = package_file.parent / f"{package_file.name}.sig" - assert not package_like(sig_file) - - -def test_parse_version() -> None: - """ - must correctly parse version into components - """ - assert parse_version("1.2.3-4") == (None, "1.2.3", "4") - assert parse_version("5:1.2.3-4") == ("5", "1.2.3", "4") - assert parse_version("1.2.3-4.2") == (None, "1.2.3", "4.2") - assert parse_version("0:1.2.3-4.2") == ("0", "1.2.3", "4.2") - assert parse_version("0:1.2.3-4") == ("0", "1.2.3", "4") - - -def test_partition() -> None: - """ - must partition list based on predicate - """ - even, odd = partition([1, 4, 2, 1, 3, 4], lambda i: i % 2 == 0) - assert even == [4, 2, 4] - assert odd == [1, 1, 3] - - -def test_pretty_datetime() -> None: - """ - must generate string from timestamp value - """ - assert pretty_datetime(0) == "1970-01-01 00:00:00" - - -def test_pretty_datetime_datetime() -> None: - """ - must generate string from datetime object - """ - assert pretty_datetime(datetime.datetime(1970, 1, 1, 0, 0, 0)) == "1970-01-01 00:00:00" - - -def test_pretty_datetime_empty() -> None: - """ - must generate empty string from None timestamp - """ - assert pretty_datetime(None) == "" - - -def test_pretty_size_bytes() -> None: - """ - must generate bytes string for bytes value - """ - value, abbrev = pretty_size(42).split() - assert value == "42.0" - assert abbrev == "B" - - -def test_pretty_size_kbytes() -> None: - """ - must generate kibibytes string for kibibytes value - """ - value, abbrev = pretty_size(42 * 1024).split() - assert value == "42.0" - assert abbrev == "KiB" - - -def test_pretty_size_mbytes() -> None: - """ - must generate mebibytes string for mebibytes value - """ - value, abbrev = pretty_size(42 * 1024 * 1024).split() - assert value == "42.0" - assert abbrev == "MiB" - - -def test_pretty_size_gbytes() -> None: - """ - must generate gibibytes string for gibibytes value - """ - value, abbrev = pretty_size(42 * 1024 * 1024 * 1024).split() - assert value == "42.0" - assert abbrev == "GiB" - - -def test_pretty_size_pbytes() -> None: - """ - must generate pebibytes string for pebibytes value - """ - value, abbrev = pretty_size(42 * 1024 * 1024 * 1024 * 1024).split() - assert value == "43008.0" - assert abbrev == "GiB" - - -def test_pretty_size_pbytes_failure() -> None: - """ - must raise exception if level >= 4 supplied - """ - with pytest.raises(OptionError): - pretty_size(42 * 1024 * 1024 * 1024 * 1024, 4).split() - - -def test_pretty_size_empty() -> None: - """ - must generate empty string for None value - """ - assert pretty_size(None) == "" - - -def test_safe_filename() -> None: - """ - must replace unsafe characters by dashes - """ - # so far I found only plus sign - assert safe_filename( - "gconf-3.2.6+11+g07808097-10-x86_64.pkg.tar.zst") == "gconf-3.2.6-11-g07808097-10-x86_64.pkg.tar.zst" - assert safe_filename( - "netkit-telnet-ssl-0.17.41+0.2-6-x86_64.pkg.tar.zst") == "netkit-telnet-ssl-0.17.41-0.2-6-x86_64.pkg.tar.zst" - assert safe_filename("spotify-1:1.1.84.716-2-x86_64.pkg.tar.zst") == "spotify-1:1.1.84.716-2-x86_64.pkg.tar.zst" - assert safe_filename("tolua++-1.0.93-4-x86_64.pkg.tar.zst") == "tolua---1.0.93-4-x86_64.pkg.tar.zst" - - -def test_srcinfo_property() -> None: - """ - must correctly extract properties - """ - assert srcinfo_property("key", {"key": "root"}, {"key": "overrides"}, default="default") == "overrides" - assert srcinfo_property("key", {"key": "root"}, {}, default="default") == "root" - assert srcinfo_property("key", {}, {"key": "overrides"}, default="default") == "overrides" - assert srcinfo_property("key", {}, {}, default="default") == "default" - assert srcinfo_property("key", {}, {}) is None - - -def test_srcinfo_property_list() -> None: - """ - must correctly extract property list - """ - assert srcinfo_property_list("key", {"key": ["root"]}, {"key": ["overrides"]}) == ["overrides"] - assert srcinfo_property_list("key", {"key": ["root"]}, {"key_x86_64": ["overrides"]}, architecture="x86_64") == [ - "root", "overrides" - ] - assert srcinfo_property_list("key", {"key": ["root"], "key_x86_64": ["overrides"]}, {}, architecture="x86_64") == [ - "root", "overrides" - ] - assert srcinfo_property_list("key", {"key_x86_64": ["overrides"]}, {}, architecture="x86_64") == ["overrides"] - - -def test_trim_package() -> None: - """ - must trim package version - """ - assert trim_package("package=1") == "package" - assert trim_package("package>=1") == "package" - assert trim_package("package>1") == "package" - assert trim_package("package<1") == "package" - assert trim_package("package<=1") == "package" - assert trim_package("package: a description") == "package" - - -def test_utcnow() -> None: - """ - must generate correct timestamp - """ - ts1 = utcnow() - ts2 = utcnow() - assert 1 > (ts2 - ts1).total_seconds() > 0 - - -def test_walk(resource_path_root: Path) -> None: +def test_import() -> None: """ - must traverse directory recursively + ahriman.core.util must provide same methods as ahriman.core.utils module """ - expected = sorted([ - resource_path_root / "core" / "ahriman.ini", - resource_path_root / "core" / "arcanisrepo.files.tar.gz", - resource_path_root / "core" / "logging.ini", - resource_path_root / "models" / "aur_error", - resource_path_root / "models" / "big_file_checksum", - resource_path_root / "models" / "empty_file_checksum", - resource_path_root / "models" / "official_error", - resource_path_root / "models" / "package_ahriman_aur", - resource_path_root / "models" / "package_akonadi_aur", - resource_path_root / "models" / "package_ahriman_files", - resource_path_root / "models" / "package_ahriman_srcinfo", - resource_path_root / "models" / "package_gcc10_srcinfo", - resource_path_root / "models" / "package_jellyfin-ffmpeg5-bin_srcinfo", - resource_path_root / "models" / "package_tpacpi-bat-git_srcinfo", - resource_path_root / "models" / "package_yay_srcinfo", - resource_path_root / "web" / "templates" / "build-status" / "alerts.jinja2", - resource_path_root / "web" / "templates" / "build-status" / "key-import-modal.jinja2", - resource_path_root / "web" / "templates" / "build-status" / "login-modal.jinja2", - resource_path_root / "web" / "templates" / "build-status" / "package-add-modal.jinja2", - resource_path_root / "web" / "templates" / "build-status" / "package-info-modal.jinja2", - resource_path_root / "web" / "templates" / "build-status" / "package-rebuild-modal.jinja2", - resource_path_root / "web" / "templates" / "build-status" / "table.jinja2", - resource_path_root / "web" / "templates" / "static" / "favicon.ico", - resource_path_root / "web" / "templates" / "static" / "logo.svg", - resource_path_root / "web" / "templates" / "utils" / "bootstrap-scripts.jinja2", - resource_path_root / "web" / "templates" / "utils" / "style.jinja2", - resource_path_root / "web" / "templates" / "api.jinja2", - resource_path_root / "web" / "templates" / "build-status.jinja2", - resource_path_root / "web" / "templates" / "email-index.jinja2", - resource_path_root / "web" / "templates" / "error.jinja2", - resource_path_root / "web" / "templates" / "repo-index.jinja2", - resource_path_root / "web" / "templates" / "shell", - resource_path_root / "web" / "templates" / "telegram-index.jinja2", - ]) - local_files = list(sorted(walk(resource_path_root))) - assert local_files == expected + for method in dir(): + assert hasattr(ahriman.core.utils, method) diff --git a/tests/ahriman/core/test_utils.py b/tests/ahriman/core/test_utils.py new file mode 100644 index 00000000..ecf2b9b6 --- /dev/null +++ b/tests/ahriman/core/test_utils.py @@ -0,0 +1,496 @@ +import datetime +import logging +import os +import pytest + +from pathlib import Path +from pytest_mock import MockerFixture +from typing import Any +from unittest.mock import call as MockCall + +from ahriman.core.exceptions import BuildError, CalledProcessError, OptionError, UnsafeRunError +from ahriman.core.utils import check_output, check_user, dataclass_view, enum_values, extract_user, filter_json, \ + full_version, minmax, package_like, parse_version, partition, pretty_datetime, pretty_size, safe_filename, \ + srcinfo_property, srcinfo_property_list, trim_package, utcnow, walk +from ahriman.models.package import Package +from ahriman.models.package_source import PackageSource +from ahriman.models.repository_id import RepositoryId +from ahriman.models.repository_paths import RepositoryPaths + + +def test_check_output(mocker: MockerFixture) -> None: + """ + must run command and log result + """ + logger_mock = mocker.patch("logging.Logger.debug") + + assert check_output("echo", "hello") == "hello" + logger_mock.assert_not_called() + + assert check_output("echo", "hello", logger=logging.getLogger("")) == "hello" + logger_mock.assert_called_once_with("hello") + + +def test_check_output_stderr(mocker: MockerFixture) -> None: + """ + must run command and log stderr output + """ + logger_mock = mocker.patch("logging.Logger.debug") + + assert check_output("python", "-c", """import sys; print("hello", file=sys.stderr)""") == "" + logger_mock.assert_not_called() + + assert check_output("python", "-c", """import sys; print("hello", file=sys.stderr)""", + logger=logging.getLogger("")) == "" + logger_mock.assert_called_once_with("hello") + + +def test_check_output_with_stdin() -> None: + """ + must run command and put string to stdin + """ + assert check_output("python", "-c", "import sys; value = sys.stdin.read(); print(value)", + input_data="single line") == "single line" + + +def test_check_output_with_stdin_newline() -> None: + """ + must run command and put string to stdin ending with new line + """ + assert check_output("python", "-c", "import sys; value = sys.stdin.read(); print(value)", + input_data="single line\n") == "single line" + + +def test_check_output_multiple_with_stdin() -> None: + """ + must run command and put multiple lines to stdin + """ + assert check_output("python", "-c", "import sys; value = sys.stdin.read(); print(value)", + input_data="multiple\nlines") == "multiple\nlines" + + +def test_check_output_multiple_with_stdin_newline() -> None: + """ + must run command and put multiple lines to stdin with new line at the end + """ + assert check_output("python", "-c", "import sys; value = sys.stdin.read(); print(value)", + input_data="multiple\nlines\n") == "multiple\nlines" + + +def test_check_output_with_user(passwd: Any, mocker: MockerFixture) -> None: + """ + must run command as specified user and set its homedir + """ + assert check_output("python", "-c", """import os; print(os.getenv("HOME"))""") != passwd.pw_dir + + getpwuid_mock = mocker.patch("ahriman.core.utils.getpwuid", return_value=passwd) + user = os.getuid() + + assert check_output("python", "-c", """import os; print(os.getenv("HOME"))""", user=user) == passwd.pw_dir + getpwuid_mock.assert_called_once_with(user) + + +def test_check_output_with_user_and_environment(passwd: Any, mocker: MockerFixture) -> None: + """ + must run set environment if both environment and user are set + """ + mocker.patch("ahriman.core.utils.getpwuid", return_value=passwd) + user = os.getuid() + assert check_output("python", "-c", """import os; print(os.getenv("HOME"), os.getenv("VAR"))""", + environment={"VAR": "VALUE"}, user=user) == f"{passwd.pw_dir} VALUE" + + +def test_check_output_failure(mocker: MockerFixture) -> None: + """ + must process exception correctly + """ + mocker.patch("subprocess.Popen.wait", return_value=1) + + with pytest.raises(CalledProcessError): + check_output("echo", "hello") + + with pytest.raises(CalledProcessError): + check_output("echo", "hello", logger=logging.getLogger("")) + + +def test_check_output_failure_exception(mocker: MockerFixture) -> None: + """ + must raise exception provided instead of default + """ + mocker.patch("subprocess.Popen.wait", return_value=1) + exception = BuildError("") + + with pytest.raises(BuildError): + check_output("echo", "hello", exception=exception) + + with pytest.raises(BuildError): + check_output("echo", "hello", exception=exception, logger=logging.getLogger("")) + + +def test_check_output_failure_exception_callable(mocker: MockerFixture) -> None: + """ + must raise exception from callable provided instead of default + """ + mocker.patch("subprocess.Popen.wait", return_value=1) + exception = BuildError.from_process("") + + with pytest.raises(BuildError): + check_output("echo", "hello", exception=exception) + + with pytest.raises(BuildError): + check_output("echo", "hello", exception=exception, logger=logging.getLogger("")) + + +def test_check_output_empty_line(mocker: MockerFixture) -> None: + """ + must correctly process empty lines in command output + """ + logger_mock = mocker.patch("logging.Logger.debug") + assert check_output("python", "-c", """print(); print("hello")""", logger=logging.getLogger("")) == "\nhello" + logger_mock.assert_has_calls([MockCall(""), MockCall("hello")]) + + +def test_check_user(repository_id: RepositoryId, mocker: MockerFixture) -> None: + """ + must check user correctly + """ + paths = RepositoryPaths(Path.cwd(), repository_id) + mocker.patch("os.getuid", return_value=paths.root_owner[0]) + check_user(paths, unsafe=False) + + +def test_check_user_no_directory(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: + """ + must not fail in case if no directory found + """ + mocker.patch("pathlib.Path.exists", return_value=False) + check_user(repository_paths, unsafe=False) + + +def test_check_user_exception(repository_id: RepositoryId, mocker: MockerFixture) -> None: + """ + must raise exception if user differs + """ + paths = RepositoryPaths(Path.cwd(), repository_id) + mocker.patch("os.getuid", return_value=paths.root_owner[0] + 1) + + with pytest.raises(UnsafeRunError): + check_user(paths, unsafe=False) + + +def test_check_user_unsafe(repository_id: RepositoryId, mocker: MockerFixture) -> None: + """ + must skip check if unsafe flag is set + """ + paths = RepositoryPaths(Path.cwd(), repository_id) + mocker.patch("os.getuid", return_value=paths.root_owner[0] + 1) + check_user(paths, unsafe=True) + + +def test_dataclass_view(package_ahriman: Package) -> None: + """ + must serialize dataclasses + """ + assert Package.from_json(dataclass_view(package_ahriman)) == package_ahriman + + +def test_dataclass_view_without_none(package_ahriman: Package) -> None: + """ + must serialize dataclasses with None fields removed + """ + package_ahriman.packager = None + result = dataclass_view(package_ahriman) + assert "packager" not in result + assert Package.from_json(result) == package_ahriman + + +def test_enum_values() -> None: + """ + must correctly generate choices from enumeration classes + """ + values = enum_values(PackageSource) + for value in values: + assert PackageSource(value).value == value + + +def test_extract_user() -> None: + """ + must extract user from system environment + """ + os.environ["USER"] = "user" + assert extract_user() == "user" + + os.environ["SUDO_USER"] = "sudo" + assert extract_user() == "sudo" + + os.environ["DOAS_USER"] = "doas" + assert extract_user() == "sudo" + + del os.environ["SUDO_USER"] + assert extract_user() == "doas" + + +def test_filter_json(package_ahriman: Package) -> None: + """ + must filter fields by known list + """ + expected = package_ahriman.view() + probe = package_ahriman.view() + probe["unknown_field"] = "value" + + assert expected == filter_json(probe, expected.keys()) + + +def test_filter_json_empty_value(package_ahriman: Package) -> None: + """ + must filter empty values from object + """ + probe = package_ahriman.view() + probe["base"] = None + assert "base" not in filter_json(probe, probe.keys()) + + +def test_full_version() -> None: + """ + must construct full version + """ + assert full_version("1", "r2388.d30e3201", "1") == "1:r2388.d30e3201-1" + assert full_version(None, "0.12.1", "1") == "0.12.1-1" + assert full_version(0, "0.12.1", "1") == "0.12.1-1" + assert full_version(1, "0.12.1", "1") == "1:0.12.1-1" + + +def test_minmax() -> None: + """ + must correctly define minimal and maximal value + """ + assert minmax([1, 4, 3, 2]) == (1, 4) + assert minmax([[1, 2, 3], [4, 5], [6, 7, 8, 9]], key=len) == ([4, 5], [6, 7, 8, 9]) + + +def test_package_like(package_ahriman: Package) -> None: + """ + package_like must return true for archives + """ + assert package_like(package_ahriman.packages[package_ahriman.base].filepath) + + +def test_package_like_hidden(package_ahriman: Package) -> None: + """ + package_like must return false for hidden files + """ + package_file = package_ahriman.packages[package_ahriman.base].filepath + hidden_file = package_file.parent / f".{package_file.name}" + assert not package_like(hidden_file) + + +def test_package_like_sig(package_ahriman: Package) -> None: + """ + package_like must return false for signature files + """ + package_file = package_ahriman.packages[package_ahriman.base].filepath + sig_file = package_file.parent / f"{package_file.name}.sig" + assert not package_like(sig_file) + + +def test_parse_version() -> None: + """ + must correctly parse version into components + """ + assert parse_version("1.2.3-4") == (None, "1.2.3", "4") + assert parse_version("5:1.2.3-4") == ("5", "1.2.3", "4") + assert parse_version("1.2.3-4.2") == (None, "1.2.3", "4.2") + assert parse_version("0:1.2.3-4.2") == ("0", "1.2.3", "4.2") + assert parse_version("0:1.2.3-4") == ("0", "1.2.3", "4") + + +def test_partition() -> None: + """ + must partition list based on predicate + """ + even, odd = partition([1, 4, 2, 1, 3, 4], lambda i: i % 2 == 0) + assert even == [4, 2, 4] + assert odd == [1, 1, 3] + + +def test_pretty_datetime() -> None: + """ + must generate string from timestamp value + """ + assert pretty_datetime(0) == "1970-01-01 00:00:00" + + +def test_pretty_datetime_datetime() -> None: + """ + must generate string from datetime object + """ + assert pretty_datetime(datetime.datetime(1970, 1, 1, 0, 0, 0)) == "1970-01-01 00:00:00" + + +def test_pretty_datetime_empty() -> None: + """ + must generate empty string from None timestamp + """ + assert pretty_datetime(None) == "" + + +def test_pretty_size_bytes() -> None: + """ + must generate bytes string for bytes value + """ + value, abbrev = pretty_size(42).split() + assert value == "42.0" + assert abbrev == "B" + + +def test_pretty_size_kbytes() -> None: + """ + must generate kibibytes string for kibibytes value + """ + value, abbrev = pretty_size(42 * 1024).split() + assert value == "42.0" + assert abbrev == "KiB" + + +def test_pretty_size_mbytes() -> None: + """ + must generate mebibytes string for mebibytes value + """ + value, abbrev = pretty_size(42 * 1024 * 1024).split() + assert value == "42.0" + assert abbrev == "MiB" + + +def test_pretty_size_gbytes() -> None: + """ + must generate gibibytes string for gibibytes value + """ + value, abbrev = pretty_size(42 * 1024 * 1024 * 1024).split() + assert value == "42.0" + assert abbrev == "GiB" + + +def test_pretty_size_pbytes() -> None: + """ + must generate pebibytes string for pebibytes value + """ + value, abbrev = pretty_size(42 * 1024 * 1024 * 1024 * 1024).split() + assert value == "43008.0" + assert abbrev == "GiB" + + +def test_pretty_size_pbytes_failure() -> None: + """ + must raise exception if level >= 4 supplied + """ + with pytest.raises(OptionError): + pretty_size(42 * 1024 * 1024 * 1024 * 1024, 4).split() + + +def test_pretty_size_empty() -> None: + """ + must generate empty string for None value + """ + assert pretty_size(None) == "" + + +def test_safe_filename() -> None: + """ + must replace unsafe characters by dashes + """ + # so far I found only plus sign + assert safe_filename( + "gconf-3.2.6+11+g07808097-10-x86_64.pkg.tar.zst") == "gconf-3.2.6-11-g07808097-10-x86_64.pkg.tar.zst" + assert safe_filename( + "netkit-telnet-ssl-0.17.41+0.2-6-x86_64.pkg.tar.zst") == "netkit-telnet-ssl-0.17.41-0.2-6-x86_64.pkg.tar.zst" + assert safe_filename("spotify-1:1.1.84.716-2-x86_64.pkg.tar.zst") == "spotify-1:1.1.84.716-2-x86_64.pkg.tar.zst" + assert safe_filename("tolua++-1.0.93-4-x86_64.pkg.tar.zst") == "tolua---1.0.93-4-x86_64.pkg.tar.zst" + + +def test_srcinfo_property() -> None: + """ + must correctly extract properties + """ + assert srcinfo_property("key", {"key": "root"}, {"key": "overrides"}, default="default") == "overrides" + assert srcinfo_property("key", {"key": "root"}, {}, default="default") == "root" + assert srcinfo_property("key", {}, {"key": "overrides"}, default="default") == "overrides" + assert srcinfo_property("key", {}, {}, default="default") == "default" + assert srcinfo_property("key", {}, {}) is None + + +def test_srcinfo_property_list() -> None: + """ + must correctly extract property list + """ + assert srcinfo_property_list("key", {"key": ["root"]}, {"key": ["overrides"]}) == ["overrides"] + assert srcinfo_property_list("key", {"key": ["root"]}, {"key_x86_64": ["overrides"]}, architecture="x86_64") == [ + "root", "overrides" + ] + assert srcinfo_property_list("key", {"key": ["root"], "key_x86_64": ["overrides"]}, {}, architecture="x86_64") == [ + "root", "overrides" + ] + assert srcinfo_property_list("key", {"key_x86_64": ["overrides"]}, {}, architecture="x86_64") == ["overrides"] + + +def test_trim_package() -> None: + """ + must trim package version + """ + assert trim_package("package=1") == "package" + assert trim_package("package>=1") == "package" + assert trim_package("package>1") == "package" + assert trim_package("package<1") == "package" + assert trim_package("package<=1") == "package" + assert trim_package("package: a description") == "package" + + +def test_utcnow() -> None: + """ + must generate correct timestamp + """ + ts1 = utcnow() + ts2 = utcnow() + assert 1 > (ts2 - ts1).total_seconds() > 0 + + +def test_walk(resource_path_root: Path) -> None: + """ + must traverse directory recursively + """ + expected = sorted([ + resource_path_root / "core" / "ahriman.ini", + resource_path_root / "core" / "arcanisrepo.files.tar.gz", + resource_path_root / "core" / "logging.ini", + resource_path_root / "models" / "aur_error", + resource_path_root / "models" / "big_file_checksum", + resource_path_root / "models" / "empty_file_checksum", + resource_path_root / "models" / "official_error", + resource_path_root / "models" / "package_ahriman_aur", + resource_path_root / "models" / "package_akonadi_aur", + resource_path_root / "models" / "package_ahriman_files", + resource_path_root / "models" / "package_ahriman_srcinfo", + resource_path_root / "models" / "package_gcc10_srcinfo", + resource_path_root / "models" / "package_jellyfin-ffmpeg5-bin_srcinfo", + resource_path_root / "models" / "package_tpacpi-bat-git_srcinfo", + resource_path_root / "models" / "package_yay_srcinfo", + resource_path_root / "web" / "templates" / "build-status" / "alerts.jinja2", + resource_path_root / "web" / "templates" / "build-status" / "key-import-modal.jinja2", + resource_path_root / "web" / "templates" / "build-status" / "login-modal.jinja2", + resource_path_root / "web" / "templates" / "build-status" / "package-add-modal.jinja2", + resource_path_root / "web" / "templates" / "build-status" / "package-info-modal.jinja2", + resource_path_root / "web" / "templates" / "build-status" / "package-rebuild-modal.jinja2", + resource_path_root / "web" / "templates" / "build-status" / "table.jinja2", + resource_path_root / "web" / "templates" / "static" / "favicon.ico", + resource_path_root / "web" / "templates" / "static" / "logo.svg", + resource_path_root / "web" / "templates" / "utils" / "bootstrap-scripts.jinja2", + resource_path_root / "web" / "templates" / "utils" / "style.jinja2", + resource_path_root / "web" / "templates" / "api.jinja2", + resource_path_root / "web" / "templates" / "build-status.jinja2", + resource_path_root / "web" / "templates" / "email-index.jinja2", + resource_path_root / "web" / "templates" / "error.jinja2", + resource_path_root / "web" / "templates" / "repo-index.jinja2", + resource_path_root / "web" / "templates" / "shell", + resource_path_root / "web" / "templates" / "telegram-index.jinja2", + ]) + local_files = list(sorted(walk(resource_path_root))) + assert local_files == expected diff --git a/tests/ahriman/core/triggers/test_trigger_loader.py b/tests/ahriman/core/triggers/test_trigger_loader.py index f3d5efe8..82a5d361 100644 --- a/tests/ahriman/core/triggers/test_trigger_loader.py +++ b/tests/ahriman/core/triggers/test_trigger_loader.py @@ -77,7 +77,7 @@ def test_load_trigger_class_package_not_trigger(trigger_loader: TriggerLoader) - must raise InvalidExtension if imported module is not a type """ with pytest.raises(ExtensionError): - trigger_loader.load_trigger_class("ahriman.core.util.check_output") + trigger_loader.load_trigger_class("ahriman.core.utils.check_output") def test_load_trigger_class_package_is_not_trigger(trigger_loader: TriggerLoader) -> None: diff --git a/tests/ahriman/core/upload/test_github.py b/tests/ahriman/core/upload/test_github.py index 06afcfe1..e39b1914 100644 --- a/tests/ahriman/core/upload/test_github.py +++ b/tests/ahriman/core/upload/test_github.py @@ -132,7 +132,7 @@ def test_get_local_files(github: GitHub, resource_path_root: Path, mocker: Mocke """ must get all local files recursively """ - walk_mock = mocker.patch("ahriman.core.util.walk") + walk_mock = mocker.patch("ahriman.core.utils.walk") github.get_local_files(resource_path_root) walk_mock.assert_called() diff --git a/tests/ahriman/core/upload/test_s3.py b/tests/ahriman/core/upload/test_s3.py index 6bd6d4af..8ca3ecf2 100644 --- a/tests/ahriman/core/upload/test_s3.py +++ b/tests/ahriman/core/upload/test_s3.py @@ -103,7 +103,7 @@ def test_get_local_files(s3: S3, resource_path_root: Path, mocker: MockerFixture """ must get all local files recursively """ - walk_mock = mocker.patch("ahriman.core.util.walk") + walk_mock = mocker.patch("ahriman.core.utils.walk") s3.get_local_files(resource_path_root) walk_mock.assert_called() diff --git a/tests/ahriman/models/test_package.py b/tests/ahriman/models/test_package.py index 7596787d..31027eaa 100644 --- a/tests/ahriman/models/test_package.py +++ b/tests/ahriman/models/test_package.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock from ahriman.core.alpm.pacman import Pacman from ahriman.core.exceptions import PackageInfoError -from ahriman.core.util import utcnow +from ahriman.core.utils import utcnow from ahriman.models.aur_package import AURPackage from ahriman.models.package import Package from ahriman.models.package_description import PackageDescription @@ -526,6 +526,8 @@ def test_next_pkgrel(package_ahriman: Package) -> None: package_ahriman.version = "1.0.0-2" assert package_ahriman.next_pkgrel("1.0.0-1.1") is None + assert package_ahriman.next_pkgrel(None) is None + def test_build_status_pretty_print(package_ahriman: Package) -> None: """ diff --git a/tests/ahriman/test_tests.py b/tests/ahriman/test_tests.py index 897f4ce7..b48d28fc 100644 --- a/tests/ahriman/test_tests.py +++ b/tests/ahriman/test_tests.py @@ -1,6 +1,6 @@ from pathlib import Path -from ahriman.core.util import walk +from ahriman.core.utils import walk def test_test_coverage() -> None: diff --git a/tests/ahriman/web/test_routes.py b/tests/ahriman/web/test_routes.py index 10568591..7d8ce98c 100644 --- a/tests/ahriman/web/test_routes.py +++ b/tests/ahriman/web/test_routes.py @@ -7,7 +7,7 @@ from pytest_mock import MockerFixture from types import ModuleType from ahriman.core.configuration import Configuration -from ahriman.core.util import walk +from ahriman.core.utils import walk from ahriman.web.routes import _dynamic_routes, _module, _modules, setup_routes