From 12f6bb0aafa3b7c4d1639e0747a443f1357270d2 Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Sun, 20 Nov 2022 03:51:42 +0200 Subject: [PATCH] mplemet log storage at backend --- CONTRIBUTING.md | 2 +- docs/ahriman.core.database.migrations.rst | 8 ++ docs/ahriman.core.database.operations.rst | 8 ++ docs/ahriman.core.log.rst | 37 +++++++ docs/ahriman.core.rst | 9 +- docs/ahriman.models.rst | 8 ++ docs/ahriman.web.views.status.rst | 8 ++ .../application/application_properties.py | 5 +- src/ahriman/application/handlers/handler.py | 4 +- src/ahriman/application/lock.py | 4 +- src/ahriman/core/alpm/pacman.py | 2 +- src/ahriman/core/alpm/remote/remote.py | 2 +- src/ahriman/core/alpm/repo.py | 2 +- src/ahriman/core/auth/auth.py | 2 +- src/ahriman/core/build_tools/sources.py | 14 +-- src/ahriman/core/build_tools/task.py | 12 ++- src/ahriman/core/configuration.py | 32 +----- .../core/database/migrations/__init__.py | 2 +- .../core/database/migrations/m004_logs.py | 34 +++++++ .../core/database/operations/__init__.py | 1 + .../database/operations/build_operations.py | 2 +- .../database/operations/logs_operations.py | 94 ++++++++++++++++++ .../core/database/operations/operations.py | 3 +- src/ahriman/core/database/sqlite.py | 5 +- src/ahriman/core/gitremote/remote_pull.py | 2 +- src/ahriman/core/gitremote/remote_push.py | 2 +- src/ahriman/core/log/__init__.py | 21 ++++ src/ahriman/core/log/http_log_handler.py | 80 +++++++++++++++ src/ahriman/core/{ => log}/lazy_logging.py | 24 +++++ src/ahriman/core/log/log.py | 61 ++++++++++++ src/ahriman/core/report/report.py | 2 +- src/ahriman/core/repository/executor.py | 10 +- .../core/repository/repository_properties.py | 7 +- src/ahriman/core/repository/update_handler.py | 8 +- src/ahriman/core/sign/gpg.py | 8 +- src/ahriman/core/spawn.py | 2 +- src/ahriman/core/status/client.py | 49 ++++++---- src/ahriman/core/status/watcher.py | 52 +++++++++- src/ahriman/core/status/web_client.py | 72 ++++++++++---- src/ahriman/core/triggers/trigger.py | 2 +- src/ahriman/core/triggers/trigger_loader.py | 2 +- src/ahriman/core/upload/upload.py | 2 +- src/ahriman/core/util.py | 3 +- src/ahriman/models/log_record_id.py | 34 +++++++ src/ahriman/models/package.py | 7 +- src/ahriman/web/routes.py | 9 ++ src/ahriman/web/views/status/logs.py | 97 +++++++++++++++++++ src/ahriman/web/views/status/package.py | 34 +++---- .../application/handlers/test_handler.py | 8 +- .../handlers/test_handler_status.py | 2 +- .../handlers/test_handler_status_update.py | 2 +- tests/ahriman/conftest.py | 2 +- tests/ahriman/core/conftest.py | 12 +++ .../migrations/test_m002_user_access.py | 2 +- .../migrations/test_m003_patch_variables.py | 2 +- .../database/migrations/test_m004_logs.py | 8 ++ .../operations/test_build_operations.py | 2 +- .../operations/test_logs_operations.py | 23 +++++ .../ahriman/core/log/test_http_log_handler.py | 56 +++++++++++ .../core/{ => log}/test_lazy_logging.py | 17 ++++ tests/ahriman/core/log/test_log.py | 35 +++++++ .../repository/test_repository_properties.py | 24 ----- tests/ahriman/core/status/test_client.py | 24 ++++- tests/ahriman/core/status/test_watcher.py | 54 ++++++++++- tests/ahriman/core/status/test_web_client.py | 56 ++++++++++- tests/ahriman/core/test_configuration.py | 25 +---- tests/ahriman/models/test_log_record_id.py | 0 .../views/status/test_views_status_logs.py | 75 ++++++++++++++ .../views/status/test_views_status_package.py | 50 +++++----- 69 files changed, 1134 insertions(+), 235 deletions(-) create mode 100644 docs/ahriman.core.log.rst create mode 100644 src/ahriman/core/database/migrations/m004_logs.py create mode 100644 src/ahriman/core/database/operations/logs_operations.py create mode 100644 src/ahriman/core/log/__init__.py create mode 100644 src/ahriman/core/log/http_log_handler.py rename src/ahriman/core/{ => log}/lazy_logging.py (70%) create mode 100644 src/ahriman/core/log/log.py create mode 100644 src/ahriman/models/log_record_id.py create mode 100644 src/ahriman/web/views/status/logs.py create mode 100644 tests/ahriman/core/database/migrations/test_m004_logs.py create mode 100644 tests/ahriman/core/database/operations/test_logs_operations.py create mode 100644 tests/ahriman/core/log/test_http_log_handler.py rename tests/ahriman/core/{ => log}/test_lazy_logging.py (62%) create mode 100644 tests/ahriman/core/log/test_log.py create mode 100644 tests/ahriman/models/test_log_record_id.py create mode 100644 tests/ahriman/web/views/status/test_views_status_logs.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c5e36ad1..8abab5b6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,7 +61,7 @@ Again, the most checks can be performed by `make check` command, though some add * The file size mentioned above must be applicable in general. In case of big classes consider splitting them into traits. Note, however, that `pylint` includes comments and docstrings into counter, thus you need to check file size by other tools. * No global variable is allowed outside of `ahriman.version` module. * Single quotes are not allowed. The reason behind this restriction is the fact that docstrings must be written by using double quotes only, and we would like to make style consistent. -* If your class writes anything to log, the `ahriman.core.lazy_logging.LazyLogging` trait must be used. +* If your class writes anything to log, the `ahriman.core.log.LazyLogging` trait must be used. ### Other checks diff --git a/docs/ahriman.core.database.migrations.rst b/docs/ahriman.core.database.migrations.rst index 9fd425af..4a13a96e 100644 --- a/docs/ahriman.core.database.migrations.rst +++ b/docs/ahriman.core.database.migrations.rst @@ -36,6 +36,14 @@ ahriman.core.database.migrations.m003\_patch\_variables module :no-undoc-members: :show-inheritance: +ahriman.core.database.migrations.m004\_logs module +-------------------------------------------------- + +.. automodule:: ahriman.core.database.migrations.m004_logs + :members: + :no-undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/ahriman.core.database.operations.rst b/docs/ahriman.core.database.operations.rst index bcc55124..4a8887f0 100644 --- a/docs/ahriman.core.database.operations.rst +++ b/docs/ahriman.core.database.operations.rst @@ -20,6 +20,14 @@ ahriman.core.database.operations.build\_operations module :no-undoc-members: :show-inheritance: +ahriman.core.database.operations.logs\_operations module +-------------------------------------------------------- + +.. automodule:: ahriman.core.database.operations.logs_operations + :members: + :no-undoc-members: + :show-inheritance: + ahriman.core.database.operations.operations module -------------------------------------------------- diff --git a/docs/ahriman.core.log.rst b/docs/ahriman.core.log.rst new file mode 100644 index 00000000..ded2b4d8 --- /dev/null +++ b/docs/ahriman.core.log.rst @@ -0,0 +1,37 @@ +ahriman.core.log package +======================== + +Submodules +---------- + +ahriman.core.log.http\_log\_handler module +------------------------------------------ + +.. automodule:: ahriman.core.log.http_log_handler + :members: + :no-undoc-members: + :show-inheritance: + +ahriman.core.log.lazy\_logging module +------------------------------------- + +.. automodule:: ahriman.core.log.lazy_logging + :members: + :no-undoc-members: + :show-inheritance: + +ahriman.core.log.log module +--------------------------- + +.. automodule:: ahriman.core.log.log + :members: + :no-undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: ahriman.core.log + :members: + :no-undoc-members: + :show-inheritance: diff --git a/docs/ahriman.core.rst b/docs/ahriman.core.rst index 1462484e..67795328 100644 --- a/docs/ahriman.core.rst +++ b/docs/ahriman.core.rst @@ -13,6 +13,7 @@ Subpackages ahriman.core.database ahriman.core.formatters ahriman.core.gitremote + ahriman.core.log ahriman.core.report ahriman.core.repository ahriman.core.sign @@ -39,14 +40,6 @@ ahriman.core.exceptions module :no-undoc-members: :show-inheritance: -ahriman.core.lazy\_logging module ---------------------------------- - -.. automodule:: ahriman.core.lazy_logging - :members: - :no-undoc-members: - :show-inheritance: - ahriman.core.spawn module ------------------------- diff --git a/docs/ahriman.models.rst b/docs/ahriman.models.rst index be4468cd..87c62602 100644 --- a/docs/ahriman.models.rst +++ b/docs/ahriman.models.rst @@ -52,6 +52,14 @@ ahriman.models.internal\_status module :no-undoc-members: :show-inheritance: +ahriman.models.log\_record\_id module +------------------------------------- + +.. automodule:: ahriman.models.log_record_id + :members: + :no-undoc-members: + :show-inheritance: + ahriman.models.migration module ------------------------------- diff --git a/docs/ahriman.web.views.status.rst b/docs/ahriman.web.views.status.rst index a5dcd410..ba44c277 100644 --- a/docs/ahriman.web.views.status.rst +++ b/docs/ahriman.web.views.status.rst @@ -4,6 +4,14 @@ ahriman.web.views.status package Submodules ---------- +ahriman.web.views.status.logs module +------------------------------------ + +.. automodule:: ahriman.web.views.status.logs + :members: + :no-undoc-members: + :show-inheritance: + ahriman.web.views.status.package module --------------------------------------- diff --git a/src/ahriman/application/application/application_properties.py b/src/ahriman/application/application/application_properties.py index 7b185639..30a43357 100644 --- a/src/ahriman/application/application/application_properties.py +++ b/src/ahriman/application/application/application_properties.py @@ -19,7 +19,7 @@ # from ahriman.core.configuration import Configuration from ahriman.core.database import SQLite -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.core.repository import Repository @@ -44,7 +44,8 @@ class ApplicationProperties(LazyLogging): configuration(Configuration): configuration instance report(bool): force enable or disable reporting unsafe(bool): if set no user check will be performed before path creation - refresh_pacman_database(int): pacman database syncronization level, ``0`` is disabled + refresh_pacman_database(int, optional): pacman database syncronization level, ``0`` is disabled + (Default value = 0) """ self.configuration = configuration self.architecture = architecture diff --git a/src/ahriman/application/handlers/handler.py b/src/ahriman/application/handlers/handler.py index abafd9f6..48b66a0f 100644 --- a/src/ahriman/application/handlers/handler.py +++ b/src/ahriman/application/handlers/handler.py @@ -28,6 +28,7 @@ from typing import List, Type from ahriman.application.lock import Lock from ahriman.core.configuration import Configuration from ahriman.core.exceptions import ExitCode, MissingArchitectureError, MultipleArchitecturesError +from ahriman.core.log import Log from ahriman.models.repository_paths import RepositoryPaths @@ -94,7 +95,8 @@ class Handler: bool: True on success, False otherwise """ try: - configuration = Configuration.from_path(args.configuration, architecture, args.quiet) + configuration = Configuration.from_path(args.configuration, architecture) + Log.load(configuration, quiet=args.quiet, report=args.report) with Lock(args, architecture, configuration): cls.run(args, architecture, configuration, report=args.report, unsafe=args.unsafe) return True diff --git a/src/ahriman/application/lock.py b/src/ahriman/application/lock.py index b3c70969..315e3ca4 100644 --- a/src/ahriman/application/lock.py +++ b/src/ahriman/application/lock.py @@ -28,7 +28,7 @@ from typing import Literal, Optional, Type from ahriman import version from ahriman.core.configuration import Configuration from ahriman.core.exceptions import DuplicateRunError -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.core.status.client import Client from ahriman.core.util import check_user from ahriman.models.build_status import BuildStatusEnum @@ -73,7 +73,7 @@ class Lock(LazyLogging): self.unsafe = args.unsafe self.paths = configuration.repository_paths - self.reporter = Client.load(configuration) if args.report else Client() + self.reporter = Client.load(configuration, report=args.report) def __enter__(self) -> Lock: """ diff --git a/src/ahriman/core/alpm/pacman.py b/src/ahriman/core/alpm/pacman.py index 4b8db9c6..4bf4ea17 100644 --- a/src/ahriman/core/alpm/pacman.py +++ b/src/ahriman/core/alpm/pacman.py @@ -24,7 +24,7 @@ from pyalpm import DB, Handle, Package, SIG_PACKAGE, error as PyalpmError # typ from typing import Generator, Set from ahriman.core.configuration import Configuration -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.models.repository_paths import RepositoryPaths diff --git a/src/ahriman/core/alpm/remote/remote.py b/src/ahriman/core/alpm/remote/remote.py index 8398755b..5dfe8080 100644 --- a/src/ahriman/core/alpm/remote/remote.py +++ b/src/ahriman/core/alpm/remote/remote.py @@ -22,7 +22,7 @@ from __future__ import annotations from typing import Dict, List, Type from ahriman.core.alpm.pacman import Pacman -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.models.aur_package import AURPackage diff --git a/src/ahriman/core/alpm/repo.py b/src/ahriman/core/alpm/repo.py index 9667d8ab..e4eef297 100644 --- a/src/ahriman/core/alpm/repo.py +++ b/src/ahriman/core/alpm/repo.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import List from ahriman.core.exceptions import BuildError -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.core.util import check_output from ahriman.models.repository_paths import RepositoryPaths diff --git a/src/ahriman/core/auth/auth.py b/src/ahriman/core/auth/auth.py index e179a449..9d52a2ad 100644 --- a/src/ahriman/core/auth/auth.py +++ b/src/ahriman/core/auth/auth.py @@ -23,7 +23,7 @@ from typing import Optional, Type from ahriman.core.configuration import Configuration from ahriman.core.database import SQLite -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.models.auth_settings import AuthSettings from ahriman.models.user_access import UserAccess diff --git a/src/ahriman/core/build_tools/sources.py b/src/ahriman/core/build_tools/sources.py index 784e79d5..863eea0f 100644 --- a/src/ahriman/core/build_tools/sources.py +++ b/src/ahriman/core/build_tools/sources.py @@ -23,7 +23,7 @@ import shutil from pathlib import Path from typing import List, Optional -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.core.util import check_output, walk from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch @@ -174,7 +174,8 @@ class Sources(LazyLogging): sources_dir(Path): local path to git repository remote(RemoteSource): remote target, branch and url *pattern(str): glob patterns - commit_author(Optional[str]): commit author in form of git config (i.e. ``user ``) + commit_author(Optional[str], optional): commit author in form of git config (i.e. ``user ``) + (Default value = None) """ instance = Sources() instance.add(sources_dir, *pattern) @@ -188,7 +189,8 @@ class Sources(LazyLogging): Args: sources_dir(Path): local path to git repository *pattern(str): glob patterns - intent_to_add(bool): record only the fact that it will be added later, acts as --intent-to-add git flag + intent_to_add(bool, optional): record only the fact that it will be added later, acts as + --intent-to-add git flag (Default value = False) """ # glob directory to find files which match the specified patterns found_files: List[Path] = [] @@ -208,9 +210,9 @@ class Sources(LazyLogging): Args: sources_dir(Path): local path to git repository - message(Optional[str]): optional commit message if any. If none set, message will be generated according to - the current timestamp - author(Optional[str]): optional commit author if any + message(Optional[str], optional): optional commit message if any. If none set, message will be generated + according to the current timestamp (Default value = None) + author(Optional[str], optional): optional commit author if any (Default value = None) """ if message is None: message = f"Autogenerated commit at {datetime.datetime.utcnow()}" diff --git a/src/ahriman/core/build_tools/task.py b/src/ahriman/core/build_tools/task.py index 275222af..06674d90 100644 --- a/src/ahriman/core/build_tools/task.py +++ b/src/ahriman/core/build_tools/task.py @@ -24,7 +24,7 @@ from ahriman.core.build_tools.sources import Sources from ahriman.core.configuration import Configuration from ahriman.core.database import SQLite from ahriman.core.exceptions import BuildError -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.core.util import check_output from ahriman.models.package import Package from ahriman.models.repository_paths import RepositoryPaths @@ -84,10 +84,12 @@ class Task(LazyLogging): user=self.uid) # well it is not actually correct, but we can deal with it - packages = Task._check_output("makepkg", "--packagelist", - exception=BuildError(self.package.base), - cwd=sources_dir, - logger=self.logger).splitlines() + packages = Task._check_output( + "makepkg", "--packagelist", + exception=BuildError(self.package.base), + cwd=sources_dir, + logger=self.logger + ).splitlines() return [Path(package) for package in packages] def init(self, sources_dir: Path, database: SQLite) -> None: diff --git a/src/ahriman/core/configuration.py b/src/ahriman/core/configuration.py index d7dab1a3..499537ed 100644 --- a/src/ahriman/core/configuration.py +++ b/src/ahriman/core/configuration.py @@ -20,10 +20,8 @@ from __future__ import annotations import configparser -import logging import sys -from logging.config import fileConfig from pathlib import Path from typing import Any, Dict, Generator, List, Optional, Tuple, Type @@ -38,8 +36,6 @@ class Configuration(configparser.RawConfigParser): Attributes: ARCHITECTURE_SPECIFIC_SECTIONS(List[str]): (class attribute) known sections which can be architecture specific. Required by dump and merging functions - DEFAULT_LOG_FORMAT(str): (class attribute) default log format (in case of fallback) - DEFAULT_LOG_LEVEL(int): (class attribute) default log level (in case of fallback) SYSTEM_CONFIGURATION_PATH(Path): (class attribute) default system configuration path distributed by package architecture(Optional[str]): repository architecture path(Optional[Path]): path to root configuration file @@ -64,9 +60,6 @@ class Configuration(configparser.RawConfigParser): >>> path, architecture = configuration.check_loaded() """ - DEFAULT_LOG_FORMAT = "[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d %(funcName)s]: %(message)s" - DEFAULT_LOG_LEVEL = logging.DEBUG - ARCHITECTURE_SPECIFIC_SECTIONS = ["build", "sign", "web"] SYSTEM_CONFIGURATION_PATH = Path(sys.prefix) / "share" / "ahriman" / "settings" / "ahriman.ini" @@ -75,8 +68,8 @@ class Configuration(configparser.RawConfigParser): default constructor. In the most cases must not be called directly Args: - allow_no_value(bool): copies ``configparser.RawConfigParser`` behaviour. In case if it is set to ``True``, - the keys without values will be allowed + allow_no_value(bool, optional): copies ``configparser.RawConfigParser`` behaviour. In case if it is set + to ``True``, the keys without values will be allowed (Default value = False) """ configparser.RawConfigParser.__init__(self, allow_no_value=allow_no_value, converters={ "list": self.__convert_list, @@ -117,14 +110,13 @@ class Configuration(configparser.RawConfigParser): return RepositoryPaths(self.getpath("repository", "root"), architecture) @classmethod - def from_path(cls: Type[Configuration], path: Path, architecture: str, quiet: bool) -> Configuration: + def from_path(cls: Type[Configuration], path: Path, architecture: str) -> Configuration: """ constructor with full object initialization Args: path(Path): path to root configuration file architecture(str): repository architecture - quiet(bool): force disable any log messages Returns: Configuration: configuration instance @@ -132,7 +124,6 @@ class Configuration(configparser.RawConfigParser): configuration = cls() configuration.load(path) configuration.merge_sections(architecture) - configuration.load_logging(quiet) return configuration @staticmethod @@ -281,23 +272,6 @@ class Configuration(configparser.RawConfigParser): except (FileNotFoundError, configparser.NoOptionError, configparser.NoSectionError): pass - def load_logging(self, quiet: bool) -> None: - """ - setup logging settings from configuration - - Args: - quiet(bool): force disable any log messages - """ - try: - path = self.logging_path - fileConfig(path) - except Exception: - logging.basicConfig(filename=None, format=self.DEFAULT_LOG_FORMAT, - level=self.DEFAULT_LOG_LEVEL) - logging.exception("could not load logging from configuration, fallback to stderr") - if quiet: - logging.disable(logging.WARNING) # only print errors here - def merge_sections(self, architecture: str) -> None: """ merge architecture specific sections into main configuration diff --git a/src/ahriman/core/database/migrations/__init__.py b/src/ahriman/core/database/migrations/__init__.py index 8542f701..ac887233 100644 --- a/src/ahriman/core/database/migrations/__init__.py +++ b/src/ahriman/core/database/migrations/__init__.py @@ -27,7 +27,7 @@ from typing import List, Type from ahriman.core.configuration import Configuration from ahriman.core.database.data import migrate_data -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.models.migration import Migration from ahriman.models.migration_result import MigrationResult diff --git a/src/ahriman/core/database/migrations/m004_logs.py b/src/ahriman/core/database/migrations/m004_logs.py new file mode 100644 index 00000000..bc09c434 --- /dev/null +++ b/src/ahriman/core/database/migrations/m004_logs.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2021-2022 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +__all__ = ["steps"] + + +steps = [ + """ + create table logs ( + package_base text not null, + created real not null, + record text + ) + """, + """ + create index logs_package_base on logs (package_base) + """, +] diff --git a/src/ahriman/core/database/operations/__init__.py b/src/ahriman/core/database/operations/__init__.py index 88d69f53..204f8923 100644 --- a/src/ahriman/core/database/operations/__init__.py +++ b/src/ahriman/core/database/operations/__init__.py @@ -21,5 +21,6 @@ from ahriman.core.database.operations.operations import Operations from ahriman.core.database.operations.auth_operations import AuthOperations from ahriman.core.database.operations.build_operations import BuildOperations +from ahriman.core.database.operations.logs_operations import LogsOperations from ahriman.core.database.operations.package_operations import PackageOperations from ahriman.core.database.operations.patch_operations import PatchOperations diff --git a/src/ahriman/core/database/operations/build_operations.py b/src/ahriman/core/database/operations/build_operations.py index 17e6904e..7b37618e 100644 --- a/src/ahriman/core/database/operations/build_operations.py +++ b/src/ahriman/core/database/operations/build_operations.py @@ -26,7 +26,7 @@ from ahriman.models.package import Package class BuildOperations(Operations): """ - operations for main functions + operations for build queue functions """ def build_queue_clear(self, package_base: Optional[str]) -> None: diff --git a/src/ahriman/core/database/operations/logs_operations.py b/src/ahriman/core/database/operations/logs_operations.py new file mode 100644 index 00000000..cd14d943 --- /dev/null +++ b/src/ahriman/core/database/operations/logs_operations.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2021-2022 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from sqlite3 import Connection +from typing import List + +from ahriman.core.database.operations import Operations + + +class LogsOperations(Operations): + """ + logs operations + """ + + def logs_delete(self, package_base: str) -> None: + """ + delete log records for the specified package + + Args: + package_base(str): package base to remove logs + """ + def run(connection: Connection) -> None: + connection.execute( + """delete from logs where package_base = :package_base""", + {"package_base": package_base} + ) + + return self.with_connection(run, commit=True) + + def logs_get(self, package_base: str) -> str: + """ + extract logs for specified package base + + Args: + package_base(str): package base to extract logs + + Return: + str: full package log + """ + def run(connection: Connection) -> List[str]: + return [ + row["record"] + for row in connection.execute( + """ + select record from logs where package_base = :package_base + order by created asc + """, + {"package_base": package_base}) + ] + + records = self.with_connection(run) + return "\n".join(records) + + def logs_insert(self, package_base: str, created: float, record: str) -> None: + """ + write new log record to database + + Args: + package_base(str): package base + created(float): log created timestamp from log record attribute + record(str): log record + """ + def run(connection: Connection) -> None: + connection.execute( + """ + insert into logs + (package_base, created, record) + values + (:package_base, :created, :record) + """, + dict( + package_base=package_base, + created=created, + record=record + ) + ) + + return self.with_connection(run, commit=True) diff --git a/src/ahriman/core/database/operations/operations.py b/src/ahriman/core/database/operations/operations.py index cab7f5ec..2d7503a7 100644 --- a/src/ahriman/core/database/operations/operations.py +++ b/src/ahriman/core/database/operations/operations.py @@ -22,7 +22,8 @@ import sqlite3 from pathlib import Path from typing import Any, Dict, Tuple, TypeVar, Callable -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging + T = TypeVar("T") diff --git a/src/ahriman/core/database/sqlite.py b/src/ahriman/core/database/sqlite.py index 842941cb..74fd8418 100644 --- a/src/ahriman/core/database/sqlite.py +++ b/src/ahriman/core/database/sqlite.py @@ -27,10 +27,11 @@ from typing import Type from ahriman.core.configuration import Configuration from ahriman.core.database.migrations import Migrations -from ahriman.core.database.operations import AuthOperations, BuildOperations, PackageOperations, PatchOperations +from ahriman.core.database.operations import AuthOperations, BuildOperations, LogsOperations, PackageOperations, \ + PatchOperations -class SQLite(AuthOperations, BuildOperations, PackageOperations, PatchOperations): +class SQLite(AuthOperations, BuildOperations, LogsOperations, PackageOperations, PatchOperations): """ wrapper for sqlite3 database diff --git a/src/ahriman/core/gitremote/remote_pull.py b/src/ahriman/core/gitremote/remote_pull.py index f5146a48..afa94131 100644 --- a/src/ahriman/core/gitremote/remote_pull.py +++ b/src/ahriman/core/gitremote/remote_pull.py @@ -25,7 +25,7 @@ from tempfile import TemporaryDirectory from ahriman.core.build_tools.sources import Sources from ahriman.core.configuration import Configuration from ahriman.core.exceptions import GitRemoteError -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.core.util import walk from ahriman.models.package_source import PackageSource from ahriman.models.remote_source import RemoteSource diff --git a/src/ahriman/core/gitremote/remote_push.py b/src/ahriman/core/gitremote/remote_push.py index 9f0b03ff..32d96f10 100644 --- a/src/ahriman/core/gitremote/remote_push.py +++ b/src/ahriman/core/gitremote/remote_push.py @@ -26,7 +26,7 @@ from typing import Generator from ahriman.core.build_tools.sources import Sources from ahriman.core.configuration import Configuration from ahriman.core.exceptions import GitRemoteError -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.remote_source import RemoteSource diff --git a/src/ahriman/core/log/__init__.py b/src/ahriman/core/log/__init__.py new file mode 100644 index 00000000..cd6a7014 --- /dev/null +++ b/src/ahriman/core/log/__init__.py @@ -0,0 +1,21 @@ +# +# Copyright (c) 2021-2022 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from ahriman.core.log.lazy_logging import LazyLogging +from ahriman.core.log.log import Log diff --git a/src/ahriman/core/log/http_log_handler.py b/src/ahriman/core/log/http_log_handler.py new file mode 100644 index 00000000..f7a90707 --- /dev/null +++ b/src/ahriman/core/log/http_log_handler.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2021-2022 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from __future__ import annotations + +import logging + +from ahriman.core.configuration import Configuration + + +class HttpLogHandler(logging.Handler): + """ + handler for the http logging. Because default ``logging.handlers.HTTPHandler`` does not support cookies + authorization, we have to implement own handler which overrides the ``logging.handlers.HTTPHandler.emit`` method + + Attributes: + reporter(Client): build status reporter instance + """ + + def __init__(self, configuration: Configuration, *, report: bool) -> None: + """ + default constructor + + Args: + configuration(Configuration): configuration instance + report(bool): force enable or disable reporting + """ + # we don't really care about those parameters because they will be handled by the reporter + logging.Handler.__init__(self) + + # client has to be importer here because of circular imports + from ahriman.core.status.client import Client + self.reporter = Client.load(configuration, report=report) + + @classmethod + def load(cls, configuration: Configuration, *, report: bool) -> HttpLogHandler: + """ + install logger. This function creates handler instance and adds it to the handler list in case if no other + http handler found + + Args: + configuration(Configuration): configuration instance + report(bool): force enable or disable reporting + """ + root = logging.getLogger() + if (handler := next((handler for handler in root.handlers if isinstance(handler, cls)), None)) is not None: + return handler # there is already registered instance + + handler = cls(configuration, report=report) + root.addHandler(handler) + + return handler + + def emit(self, record: logging.LogRecord) -> None: + """ + emit log records using reporter client + + Args: + record(logging.LogRecord): log record to log + """ + try: + self.reporter.logs(record) + except Exception: + self.handleError(record) diff --git a/src/ahriman/core/lazy_logging.py b/src/ahriman/core/log/lazy_logging.py similarity index 70% rename from src/ahriman/core/lazy_logging.py rename to src/ahriman/core/log/lazy_logging.py index e1c51b42..dc1beecd 100644 --- a/src/ahriman/core/lazy_logging.py +++ b/src/ahriman/core/log/lazy_logging.py @@ -62,3 +62,27 @@ class LazyLogging: clazz = self.__class__ prefix = "" if clazz.__module__ is None else f"{clazz.__module__}." return f"{prefix}{clazz.__qualname__}" + + def package_logger_reset(self) -> None: + """ + reset package logger to empty one + """ + self.logger.debug("reset package logging") + logging.setLogRecordFactory(logging.LogRecord) + + def package_logger_set(self, package_base: str) -> None: + """ + set package base as extra info to the logger + + Args: + package_base(str): package base + """ + current_factory = logging.getLogRecordFactory() + + def package_record_factory(*args: Any, **kwargs: Any) -> logging.LogRecord: + record = current_factory(*args, **kwargs) + record.package_base = package_base + return record + + logging.setLogRecordFactory(package_record_factory) + self.logger.debug("start package %s logging", package_base) diff --git a/src/ahriman/core/log/log.py b/src/ahriman/core/log/log.py new file mode 100644 index 00000000..d41e7f53 --- /dev/null +++ b/src/ahriman/core/log/log.py @@ -0,0 +1,61 @@ +# +# Copyright (c) 2021-2022 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import logging + +from logging.config import fileConfig + +from ahriman.core.configuration import Configuration +from ahriman.core.log.http_log_handler import HttpLogHandler + + +class Log: + """ + simple static method class which setups application loggers + + Attributes: + DEFAULT_LOG_FORMAT(str): (class attribute) default log format (in case of fallback) + DEFAULT_LOG_LEVEL(int): (class attribute) default log level (in case of fallback) + """ + + DEFAULT_LOG_FORMAT = "[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d %(funcName)s]: %(message)s" + DEFAULT_LOG_LEVEL = logging.DEBUG + + @staticmethod + def load(configuration: Configuration, *, quiet: bool, report: bool) -> None: + """ + setup logging settings from configuration + + Args: + configuration(Configuration): configuration instance + quiet(bool): force disable any log messages + report(bool): force enable or disable reporting + """ + try: + path = configuration.logging_path + fileConfig(path) + except Exception: + logging.basicConfig(filename=None, format=Log.DEFAULT_LOG_FORMAT, + level=Log.DEFAULT_LOG_LEVEL) + logging.exception("could not load logging from configuration, fallback to stderr") + + HttpLogHandler.load(configuration, report=report) + + if quiet: + logging.disable(logging.WARNING) # only print errors here diff --git a/src/ahriman/core/report/report.py b/src/ahriman/core/report/report.py index d05c185e..a41f2491 100644 --- a/src/ahriman/core/report/report.py +++ b/src/ahriman/core/report/report.py @@ -23,7 +23,7 @@ from typing import Iterable, Type from ahriman.core.configuration import Configuration from ahriman.core.exceptions import ReportError -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.models.package import Package from ahriman.models.report_settings import ReportSettings from ahriman.models.result import Result diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py index 86c3aa72..daa28a3a 100644 --- a/src/ahriman/core/repository/executor.py +++ b/src/ahriman/core/repository/executor.py @@ -153,21 +153,21 @@ class Executor(Cleaner): Returns: Result: path to repository database """ - def rename(archive: PackageDescription, base: str) -> None: + def rename(archive: PackageDescription, package_base: str) -> None: if archive.filename is None: - self.logger.warning("received empty package name for base %s", base) + self.logger.warning("received empty package name for base %s", package_base) return # suppress type checking, it never can be none actually if (safe := safe_filename(archive.filename)) != archive.filename: shutil.move(self.paths.packages / archive.filename, self.paths.packages / safe) archive.filename = safe - def update_single(name: Optional[str], base: str) -> None: + def update_single(name: Optional[str], package_base: str) -> None: if name is None: - self.logger.warning("received empty package name for base %s", base) + self.logger.warning("received empty package name for base %s", package_base) return # suppress type checking, it never can be none actually # in theory, it might be NOT packages directory, but we suppose it is full_path = self.paths.packages / name - files = self.sign.process_sign_package(full_path, base) + files = self.sign.process_sign_package(full_path, package_base) for src in files: dst = self.paths.repository / safe_filename(src.name) shutil.move(src, dst) diff --git a/src/ahriman/core/repository/repository_properties.py b/src/ahriman/core/repository/repository_properties.py index 09c15a75..5e2ebd55 100644 --- a/src/ahriman/core/repository/repository_properties.py +++ b/src/ahriman/core/repository/repository_properties.py @@ -22,7 +22,7 @@ from ahriman.core.alpm.repo import Repo from ahriman.core.configuration import Configuration from ahriman.core.database import SQLite from ahriman.core.exceptions import UnsafeRunError -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.core.sign.gpg import GPG from ahriman.core.status.client import Client from ahriman.core.triggers import TriggerLoader @@ -58,7 +58,8 @@ class RepositoryProperties(LazyLogging): database(SQLite): database instance report(bool): force enable or disable reporting unsafe(bool): if set no user check will be performed before path creation - refresh_pacman_database(int): pacman database syncronization level, ``0`` is disabled + refresh_pacman_database(int, optional): pacman database syncronization level, ``0`` is disabled + (Default value = 0) """ self.architecture = architecture self.configuration = configuration @@ -77,5 +78,5 @@ class RepositoryProperties(LazyLogging): self.pacman = Pacman(architecture, configuration, refresh_database=refresh_pacman_database) self.sign = GPG(architecture, configuration) self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args) - self.reporter = Client.load(configuration) if report else Client() + self.reporter = Client.load(configuration, report=report) self.triggers = TriggerLoader(architecture, configuration) diff --git a/src/ahriman/core/repository/update_handler.py b/src/ahriman/core/repository/update_handler.py index 71996b58..d12dcd45 100644 --- a/src/ahriman/core/repository/update_handler.py +++ b/src/ahriman/core/repository/update_handler.py @@ -89,10 +89,10 @@ class UpdateHandler(Cleaner): result: List[Package] = [] packages = {local.base: local for local in self.packages()} - for dirname in self.paths.cache.iterdir(): + for cache_dir in self.paths.cache.iterdir(): try: - Sources.fetch(dirname, remote=None) - remote = Package.from_build(dirname) + Sources.fetch(cache_dir, remote=None) + remote = Package.from_build(cache_dir) local = packages.get(remote.base) if local is None: @@ -102,7 +102,7 @@ class UpdateHandler(Cleaner): self.reporter.set_pending(local.base) result.append(remote) except Exception: - self.logger.exception("could not process package at %s", dirname) + self.logger.exception("could not process package at %s", cache_dir) return result diff --git a/src/ahriman/core/sign/gpg.py b/src/ahriman/core/sign/gpg.py index ced87819..6ed9fe18 100644 --- a/src/ahriman/core/sign/gpg.py +++ b/src/ahriman/core/sign/gpg.py @@ -24,7 +24,7 @@ from typing import List, Optional, Set, Tuple from ahriman.core.configuration import Configuration from ahriman.core.exceptions import BuildError -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.core.util import check_output, exception_response_text from ahriman.models.sign_settings import SignSettings @@ -157,20 +157,20 @@ class GPG(LazyLogging): logger=self.logger) return [path, path.parent / f"{path.name}.sig"] - def process_sign_package(self, path: Path, base: str) -> List[Path]: + def process_sign_package(self, path: Path, package_base: str) -> List[Path]: """ sign package if required by configuration Args: path(Path): path to file to sign - base(str): package base required to check for key overrides + package_base(str): package base required to check for key overrides Returns: List[Path]: list of generated files including original file """ if SignSettings.Packages not in self.targets: return [path] - key = self.configuration.get("sign", f"key_{base}", fallback=self.default_key) + key = self.configuration.get("sign", f"key_{package_base}", fallback=self.default_key) if key is None: self.logger.error("no default key set, skip package %s sign", path) return [path] diff --git a/src/ahriman/core/spawn.py b/src/ahriman/core/spawn.py index 81bcacdb..84d8773c 100644 --- a/src/ahriman/core/spawn.py +++ b/src/ahriman/core/spawn.py @@ -27,7 +27,7 @@ from threading import Lock, Thread from typing import Callable, Dict, Iterable, Tuple from ahriman.core.configuration import Configuration -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.models.package_source import PackageSource diff --git a/src/ahriman/core/status/client.py b/src/ahriman/core/status/client.py index bf84e1cb..fcea1451 100644 --- a/src/ahriman/core/status/client.py +++ b/src/ahriman/core/status/client.py @@ -19,6 +19,8 @@ # from __future__ import annotations +import logging + from typing import List, Optional, Tuple, Type from ahriman.core.configuration import Configuration @@ -33,19 +35,24 @@ class Client: """ @classmethod - def load(cls: Type[Client], configuration: Configuration) -> Client: + def load(cls: Type[Client], configuration: Configuration, *, report: bool) -> Client: """ load client from settings Args: configuration(Configuration): configuration instance + report(bool): force enable or disable reporting Returns: Client: client according to current settings """ + if not report: + return cls() + address = configuration.get("web", "address", fallback=None) host = configuration.get("web", "host", fallback=None) port = configuration.getint("web", "port", fallback=None) + if address or (host and port): from ahriman.core.status.web_client import WebClient return WebClient(configuration) @@ -60,17 +67,17 @@ class Client: status(BuildStatusEnum): current package build status """ - def get(self, base: Optional[str]) -> List[Tuple[Package, BuildStatus]]: + def get(self, package_base: Optional[str]) -> List[Tuple[Package, BuildStatus]]: """ get package status Args: - base(Optional[str]): package base to get + package_base(Optional[str]): package base to get Returns: List[Tuple[Package, BuildStatus]]: list of current package description and status if it has been found """ - del base + del package_base return [] def get_internal(self) -> InternalStatus: @@ -82,20 +89,28 @@ class Client: """ return InternalStatus(status=BuildStatus()) - def remove(self, base: str) -> None: + def logs(self, record: logging.LogRecord) -> None: + """ + post log record + + Args: + record(logging.LogRecord): log record to post to api + """ + + def remove(self, package_base: str) -> None: """ remove packages from watcher Args: - base(str): package base to remove + package_base(str): package base to remove """ - def update(self, base: str, status: BuildStatusEnum) -> None: + def update(self, package_base: str, status: BuildStatusEnum) -> None: """ update package build status. Unlike ``add`` it does not update package properties Args: - base(str): package base to update + package_base(str): package base to update status(BuildStatusEnum): current package build status """ @@ -107,32 +122,32 @@ class Client: status(BuildStatusEnum): current ahriman status """ - def set_building(self, base: str) -> None: + def set_building(self, package_base: str) -> None: """ set package status to building Args: - base(str): package base to update + package_base(str): package base to update """ - return self.update(base, BuildStatusEnum.Building) + return self.update(package_base, BuildStatusEnum.Building) - def set_failed(self, base: str) -> None: + def set_failed(self, package_base: str) -> None: """ set package status to failed Args: - base(str): package base to update + package_base(str): package base to update """ - return self.update(base, BuildStatusEnum.Failed) + return self.update(package_base, BuildStatusEnum.Failed) - def set_pending(self, base: str) -> None: + def set_pending(self, package_base: str) -> None: """ set package status to pending Args: - base(str): package base to update + package_base(str): package base to update """ - return self.update(base, BuildStatusEnum.Pending) + return self.update(package_base, BuildStatusEnum.Pending) def set_success(self, package: Package) -> None: """ diff --git a/src/ahriman/core/status/watcher.py b/src/ahriman/core/status/watcher.py index 470225ad..8e059f59 100644 --- a/src/ahriman/core/status/watcher.py +++ b/src/ahriman/core/status/watcher.py @@ -17,14 +17,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +import os + from typing import Dict, List, Optional, Tuple from ahriman.core.configuration import Configuration from ahriman.core.database import SQLite from ahriman.core.exceptions import UnknownPackageError -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.core.repository import Repository from ahriman.models.build_status import BuildStatus, BuildStatusEnum +from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package @@ -57,6 +60,9 @@ class Watcher(LazyLogging): self.known: Dict[str, Tuple[Package, BuildStatus]] = {} self.status = BuildStatus() + # special variables for updating logs + self._last_log_record_id = LogRecordId("", os.getpid()) + @property def packages(self) -> List[Tuple[Package, BuildStatus]]: """ @@ -67,12 +73,12 @@ class Watcher(LazyLogging): """ return list(self.known.values()) - def get(self, base: str) -> Tuple[Package, BuildStatus]: + def get(self, package_base: str) -> Tuple[Package, BuildStatus]: """ get current package base build status Args: - base(str): package base + package_base(str): package base Returns: Tuple[Package, BuildStatus]: package and its status @@ -81,9 +87,21 @@ class Watcher(LazyLogging): UnknownPackage: if no package found """ try: - return self.known[base] + return self.known[package_base] except KeyError: - raise UnknownPackageError(base) + raise UnknownPackageError(package_base) + + def get_logs(self, package_base: str) -> str: + """ + extract logs for the package base + + Args: + package_base(str): package base + + Returns: + str: package logs + """ + return self.database.logs_get(package_base) def load(self) -> None: """ @@ -111,6 +129,15 @@ class Watcher(LazyLogging): self.known.pop(package_base, None) self.database.package_remove(package_base) + def remove_logs(self, package_base: str) -> None: + """ + remove package related logs + + Args: + package_base(str): package base + """ + self.database.logs_delete(package_base) + def update(self, package_base: str, status: BuildStatusEnum, package: Optional[Package]) -> None: """ update package status and description @@ -132,6 +159,21 @@ class Watcher(LazyLogging): self.known[package_base] = (package, full_status) self.database.package_update(package, full_status) + def update_logs(self, log_record_id: LogRecordId, created: float, record: str) -> None: + """ + make new log record into database + + Args: + log_record_id(LogRecordId): log record id + created(float): log created record + record(str): log record + """ + if self._last_log_record_id != log_record_id: + # there is new log record, so we remove old ones + self.database.logs_delete(log_record_id.package_base) + self._last_log_record_id = log_record_id + self.database.logs_insert(log_record_id.package_base, created, record) + def update_self(self, status: BuildStatusEnum) -> None: """ update service status diff --git a/src/ahriman/core/status/web_client.py b/src/ahriman/core/status/web_client.py index ad1657d5..59309358 100644 --- a/src/ahriman/core/status/web_client.py +++ b/src/ahriman/core/status/web_client.py @@ -17,12 +17,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +import logging import requests from typing import List, Optional, Tuple from ahriman.core.configuration import Configuration -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.core.status.client import Client from ahriman.core.util import exception_response_text from ahriman.models.build_status import BuildStatusEnum, BuildStatus @@ -114,17 +115,29 @@ class WebClient(Client, LazyLogging): except Exception: self.logger.exception("could not login as %s", self.user) - def _package_url(self, base: str = "") -> str: + def _logs_url(self, package_base: str) -> str: + """ + get url for the logs api + + Args: + package_base(str): package base + + Returns: + str: full url for web service for logs + """ + return f"{self.address}/api/v1/packages/{package_base}/logs" + + def _package_url(self, package_base: str = "") -> str: """ url generator Args: - base(str, optional): package base to generate url (Default value = "") + package_base(str, optional): package base to generate url (Default value = "") Returns: str: full url of web service for specific package base """ - return f"{self.address}/api/v1/packages/{base}" + return f"{self.address}/api/v1/packages/{package_base}" def add(self, package: Package, status: BuildStatusEnum) -> None: """ @@ -147,18 +160,18 @@ class WebClient(Client, LazyLogging): except Exception: self.logger.exception("could not add %s", package.base) - def get(self, base: Optional[str]) -> List[Tuple[Package, BuildStatus]]: + def get(self, package_base: Optional[str]) -> List[Tuple[Package, BuildStatus]]: """ get package status Args: - base(Optional[str]): package base to get + package_base(Optional[str]): package base to get Returns: List[Tuple[Package, BuildStatus]]: list of current package description and status if it has been found """ try: - response = self.__session.get(self._package_url(base or "")) + response = self.__session.get(self._package_url(package_base or "")) response.raise_for_status() status_json = response.json() @@ -167,9 +180,9 @@ class WebClient(Client, LazyLogging): for package in status_json ] except requests.HTTPError as e: - self.logger.exception("could not get %s: %s", base, exception_response_text(e)) + self.logger.exception("could not get %s: %s", package_base, exception_response_text(e)) except Exception: - self.logger.exception("could not get %s", base) + self.logger.exception("could not get %s", package_base) return [] def get_internal(self) -> InternalStatus: @@ -191,38 +204,59 @@ class WebClient(Client, LazyLogging): self.logger.exception("could not get web service status") return InternalStatus(status=BuildStatus()) - def remove(self, base: str) -> None: + def logs(self, record: logging.LogRecord) -> None: + """ + post log record + + Args: + record(logging.LogRecord): log record to post to api + """ + package_base = getattr(record, "package_base", None) + if package_base is None: + return # in case if no package base supplised we need just skip log message + + payload = { + "created": record.created, + "message": record.getMessage(), + "process_id": record.process, + } + + # in this method exception has to be handled outside in logger handler + response = self.__session.post(self._logs_url(package_base), json=payload) + response.raise_for_status() + + def remove(self, package_base: str) -> None: """ remove packages from watcher Args: - base(str): basename to remove + package_base(str): basename to remove """ try: - response = self.__session.delete(self._package_url(base)) + response = self.__session.delete(self._package_url(package_base)) response.raise_for_status() except requests.HTTPError as e: - self.logger.exception("could not delete %s: %s", base, exception_response_text(e)) + self.logger.exception("could not delete %s: %s", package_base, exception_response_text(e)) except Exception: - self.logger.exception("could not delete %s", base) + self.logger.exception("could not delete %s", package_base) - def update(self, base: str, status: BuildStatusEnum) -> None: + def update(self, package_base: str, status: BuildStatusEnum) -> None: """ update package build status. Unlike ``add`` it does not update package properties Args: - base(str): package base to update + package_base(str): package base to update status(BuildStatusEnum): current package build status """ payload = {"status": status.value} try: - response = self.__session.post(self._package_url(base), json=payload) + response = self.__session.post(self._package_url(package_base), json=payload) response.raise_for_status() except requests.HTTPError as e: - self.logger.exception("could not update %s: %s", base, exception_response_text(e)) + self.logger.exception("could not update %s: %s", package_base, exception_response_text(e)) except Exception: - self.logger.exception("could not update %s", base) + self.logger.exception("could not update %s", package_base) def update_self(self, status: BuildStatusEnum) -> None: """ diff --git a/src/ahriman/core/triggers/trigger.py b/src/ahriman/core/triggers/trigger.py index 86262d45..d0020c76 100644 --- a/src/ahriman/core/triggers/trigger.py +++ b/src/ahriman/core/triggers/trigger.py @@ -20,7 +20,7 @@ from typing import Iterable from ahriman.core.configuration import Configuration -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.models.package import Package from ahriman.models.result import Result diff --git a/src/ahriman/core/triggers/trigger_loader.py b/src/ahriman/core/triggers/trigger_loader.py index 43db2c15..8e64b7a7 100644 --- a/src/ahriman/core/triggers/trigger_loader.py +++ b/src/ahriman/core/triggers/trigger_loader.py @@ -27,7 +27,7 @@ from typing import Generator, Iterable from ahriman.core.configuration import Configuration from ahriman.core.exceptions import ExtensionError -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.core.triggers import Trigger from ahriman.models.package import Package from ahriman.models.result import Result diff --git a/src/ahriman/core/upload/upload.py b/src/ahriman/core/upload/upload.py index 730e9b73..34480647 100644 --- a/src/ahriman/core/upload/upload.py +++ b/src/ahriman/core/upload/upload.py @@ -24,7 +24,7 @@ from typing import Iterable, Type from ahriman.core.configuration import Configuration from ahriman.core.exceptions import SynchronizationError -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.models.package import Package from ahriman.models.upload_settings import UploadSettings diff --git a/src/ahriman/core/util.py b/src/ahriman/core/util.py index ae28d8e0..7aa064cc 100644 --- a/src/ahriman/core/util.py +++ b/src/ahriman/core/util.py @@ -44,7 +44,8 @@ def check_output(*args: str, exception: Optional[Exception] = None, cwd: Optiona Args: *args(str): command line arguments - exception(Optional[Exception]): exception which has to be reraised instead of default subprocess exception + exception(Optional[Exception], optional): exception which has to be reraised instead of default subprocess + exception (Default value = None) cwd(Optional[Path], optional): current working directory (Default value = None) input_data(Optional[str], optional): data which will be written to command stdin (Default value = None) logger(Optional[Logger], optional): logger to log command result if required (Default value = None) diff --git a/src/ahriman/models/log_record_id.py b/src/ahriman/models/log_record_id.py new file mode 100644 index 00000000..7bb0439e --- /dev/null +++ b/src/ahriman/models/log_record_id.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2021-2022 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from dataclasses import dataclass + + +@dataclass(frozen=True) +class LogRecordId: + """ + log record process identifier + + Attributes: + package_base(str): package base for which log record belongs + process_id(int): process id from which log record was emitted + """ + + package_base: str + process_id: int diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index e0402d99..0e95aaa1 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -30,7 +30,7 @@ from typing import Any, Dict, Iterable, List, Optional, Set, Type from ahriman.core.alpm.pacman import Pacman from ahriman.core.alpm.remote import AUR, Official, OfficialSyncdb from ahriman.core.exceptions import PackageInfoError -from ahriman.core.lazy_logging import LazyLogging +from ahriman.core.log import LazyLogging from ahriman.core.util import check_output, full_version from ahriman.models.package_description import PackageDescription from ahriman.models.package_source import PackageSource @@ -218,7 +218,7 @@ class Package(LazyLogging): Args: name(str): package name (either base or normal name) pacman(Pacman): alpm wrapper instance - use_syncdb(bool): use pacman databases instead of official repositories RPC (Default value = True) + use_syncdb(bool, optional): use pacman databases instead of official repositories RPC (Default value = True) Returns: Package: package properties @@ -365,7 +365,8 @@ class Package(LazyLogging): Args: remote(Package): package properties from remote source paths(RepositoryPaths): repository paths instance. Required for VCS packages cache - calculate_version(bool, optional): expand version to actual value (by calculating git versions) (Default value = True) + calculate_version(bool, optional): expand version to actual value (by calculating git versions) + (Default value = True) Returns: bool: True if the package is out-of-dated and False otherwise diff --git a/src/ahriman/web/routes.py b/src/ahriman/web/routes.py index 12f401cc..419036b4 100644 --- a/src/ahriman/web/routes.py +++ b/src/ahriman/web/routes.py @@ -25,6 +25,7 @@ from ahriman.web.views.service.add import AddView from ahriman.web.views.service.remove import RemoveView from ahriman.web.views.service.request import RequestView from ahriman.web.views.service.search import SearchView +from ahriman.web.views.status.logs import LogsView from ahriman.web.views.status.package import PackageView from ahriman.web.views.status.packages import PackagesView from ahriman.web.views.status.status import StatusView @@ -61,6 +62,10 @@ def setup_routes(application: Application, static_path: Path) -> None: * ``GET /api/v1/package/:base`` get package base status * ``POST /api/v1/package/:base`` update package base status + * ``DELETE /api/v1/packages/{package}/logs`` delete package related logs + * ``GET /api/v1/packages/{package}/logs`` create log record for the package + * ``POST /api/v1/packages/{package}/logs`` get last package logs + * ``GET /api/v1/status`` get service status itself * ``POST /api/v1/status`` update service status itself @@ -94,6 +99,10 @@ def setup_routes(application: Application, static_path: Path) -> None: application.router.add_get("/api/v1/packages/{package}", PackageView, allow_head=True) application.router.add_post("/api/v1/packages/{package}", PackageView) + application.router.add_delete("/api/v1/packages/{package}/logs", LogsView) + application.router.add_get("/api/v1/packages/{package}/logs", LogsView, allow_head=True) + application.router.add_post("/api/v1/packages/{package}/logs", LogsView) + application.router.add_get("/api/v1/status", StatusView, allow_head=True) application.router.add_post("/api/v1/status", StatusView) diff --git a/src/ahriman/web/views/status/logs.py b/src/ahriman/web/views/status/logs.py new file mode 100644 index 00000000..8614305b --- /dev/null +++ b/src/ahriman/web/views/status/logs.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2021-2022 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from aiohttp.web import HTTPBadRequest, HTTPNoContent, Response, json_response + +from ahriman.models.log_record_id import LogRecordId +from ahriman.models.user_access import UserAccess +from ahriman.web.views.base import BaseView + + +class LogsView(BaseView): + """ + package logs web view + + Attributes: + DELETE_PERMISSION(UserAccess): (class attribute) delete permissions of self + GET_PERMISSION(UserAccess): (class attribute) get permissions of self + HEAD_PERMISSION(UserAccess): (class attribute) head permissions of self + POST_PERMISSION(UserAccess): (class attribute) post permissions of self + """ + + DELETE_PERMISSION = POST_PERMISSION = UserAccess.Full + GET_PERMISSION = HEAD_PERMISSION = UserAccess.Read + + async def delete(self) -> None: + """ + delete package logs + + Raises: + HTTPNoContent: on success response + """ + package_base = self.request.match_info["package"] + self.service.remove_logs(package_base) + + raise HTTPNoContent() + + async def get(self) -> Response: + """ + get last package logs + + Returns: + Response: 200 with package logs on success + """ + package_base = self.request.match_info["package"] + logs = self.service.get_logs(package_base) + + response = { + "package_base": package_base, + "logs": logs + } + return json_response(response) + + async def post(self) -> None: + """ + create new package log record + + JSON body must be supplied, the following model is used:: + + { + "created": 42.001, # log record created timestamp + "message": "log message", # log record + "process_id": 42 # process id from which log record was emitted + } + + Raises: + HTTPBadRequest: if bad data is supplied + HTTPNoContent: in case of success response + """ + package_base = self.request.match_info["package"] + data = await self.extract_data() + + try: + created = data["created"] + record = data["message"] + process_id = data["process_id"] + except Exception as e: + raise HTTPBadRequest(reason=str(e)) + + self.service.update_logs(LogRecordId(package_base, process_id), created, record) + + raise HTTPNoContent() diff --git a/src/ahriman/web/views/status/package.py b/src/ahriman/web/views/status/package.py index abf4b743..f6258563 100644 --- a/src/ahriman/web/views/status/package.py +++ b/src/ahriman/web/views/status/package.py @@ -40,6 +40,18 @@ class PackageView(BaseView): DELETE_PERMISSION = POST_PERMISSION = UserAccess.Full GET_PERMISSION = HEAD_PERMISSION = UserAccess.Read + async def delete(self) -> None: + """ + delete package base from status page + + Raises: + HTTPNoContent: on success response + """ + package_base = self.request.match_info["package"] + self.service.remove(package_base) + + raise HTTPNoContent() + async def get(self) -> Response: """ get current package base status @@ -50,10 +62,10 @@ class PackageView(BaseView): Raises: HTTPNotFound: if no package was found """ - base = self.request.match_info["package"] + package_base = self.request.match_info["package"] try: - package, status = self.service.get(base) + package, status = self.service.get(package_base) except UnknownPackageError: raise HTTPNotFound() @@ -65,18 +77,6 @@ class PackageView(BaseView): ] return json_response(response) - async def delete(self) -> None: - """ - delete package base from status page - - Raises: - HTTPNoContent: on success response - """ - base = self.request.match_info["package"] - self.service.remove(base) - - raise HTTPNoContent() - async def post(self) -> None: """ update package build status @@ -93,7 +93,7 @@ class PackageView(BaseView): HTTPBadRequest: if bad data is supplied HTTPNoContent: in case of success response """ - base = self.request.match_info["package"] + package_base = self.request.match_info["package"] data = await self.extract_data() try: @@ -103,8 +103,8 @@ class PackageView(BaseView): raise HTTPBadRequest(reason=str(e)) try: - self.service.update(base, status, package) + self.service.update(package_base, status, package) except UnknownPackageError: - raise HTTPBadRequest(reason=f"Package {base} is unknown, but no package body set") + raise HTTPBadRequest(reason=f"Package {package_base} is unknown, but no package body set") raise HTTPNoContent() diff --git a/tests/ahriman/application/handlers/test_handler.py b/tests/ahriman/application/handlers/test_handler.py index 7d1485b6..703b8aea 100644 --- a/tests/ahriman/application/handlers/test_handler.py +++ b/tests/ahriman/application/handlers/test_handler.py @@ -51,18 +51,22 @@ def test_architectures_extract_specified(args: argparse.Namespace) -> None: assert Handler.architectures_extract(args) == sorted(set(architectures)) -def test_call(args: argparse.Namespace, mocker: MockerFixture) -> None: +def test_call(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None: """ must call inside lock """ args.configuration = Path("") args.quiet = False + args.report = False mocker.patch("ahriman.application.handlers.Handler.run") - mocker.patch("ahriman.core.configuration.Configuration.from_path") + configuration_mock = mocker.patch("ahriman.core.configuration.Configuration.from_path", return_value=configuration) + log_load_mock = mocker.patch("ahriman.core.log.Log.load") enter_mock = mocker.patch("ahriman.application.lock.Lock.__enter__") exit_mock = mocker.patch("ahriman.application.lock.Lock.__exit__") assert Handler.call(args, "x86_64") + configuration_mock.assert_called_once_with(args.configuration, "x86_64") + log_load_mock.assert_called_once_with(configuration, quiet=args.quiet, report=args.report) enter_mock.assert_called_once_with() exit_mock.assert_called_once_with(None, None, None) diff --git a/tests/ahriman/application/handlers/test_handler_status.py b/tests/ahriman/application/handlers/test_handler_status.py index 145e8862..ead701fa 100644 --- a/tests/ahriman/application/handlers/test_handler_status.py +++ b/tests/ahriman/application/handlers/test_handler_status.py @@ -120,7 +120,7 @@ def test_imply_with_report(args: argparse.Namespace, configuration: Configuratio load_mock = mocker.patch("ahriman.core.status.client.Client.load") Status.run(args, "x86_64", configuration, report=False, unsafe=False) - load_mock.assert_called_once_with(configuration) + load_mock.assert_called_once_with(configuration, report=True) def test_disallow_auto_architecture_run() -> None: diff --git a/tests/ahriman/application/handlers/test_handler_status_update.py b/tests/ahriman/application/handlers/test_handler_status_update.py index 565c394c..1d9072b8 100644 --- a/tests/ahriman/application/handlers/test_handler_status_update.py +++ b/tests/ahriman/application/handlers/test_handler_status_update.py @@ -75,7 +75,7 @@ def test_imply_with_report(args: argparse.Namespace, configuration: Configuratio load_mock = mocker.patch("ahriman.core.status.client.Client.load") StatusUpdate.run(args, "x86_64", configuration, report=False, unsafe=False) - load_mock.assert_called_once_with(configuration) + load_mock.assert_called_once_with(configuration, report=True) def test_disallow_auto_architecture_run() -> None: diff --git a/tests/ahriman/conftest.py b/tests/ahriman/conftest.py index d0169a94..721652b6 100644 --- a/tests/ahriman/conftest.py +++ b/tests/ahriman/conftest.py @@ -215,7 +215,7 @@ def configuration(resource_path_root: Path) -> Configuration: Configuration: configuration test instance """ path = resource_path_root / "core" / "ahriman.ini" - return Configuration.from_path(path=path, architecture="x86_64", quiet=False) + return Configuration.from_path(path=path, architecture="x86_64") @pytest.fixture diff --git a/tests/ahriman/core/conftest.py b/tests/ahriman/core/conftest.py index 22c9b4f0..dff526c3 100644 --- a/tests/ahriman/core/conftest.py +++ b/tests/ahriman/core/conftest.py @@ -1,3 +1,4 @@ +import logging import pytest from ahriman.core.alpm.repo import Repo @@ -36,6 +37,17 @@ def leaf_python_schedule(package_python_schedule: Package) -> Leaf: return Leaf(package_python_schedule, set()) +@pytest.fixture +def log_record() -> logging.LogRecord: + """ + fixture for log record object + + Returns: + logging.LogRecord: log record test instance + """ + return logging.LogRecord("record", logging.INFO, "path", 42, "log message", args=(), exc_info=None) + + @pytest.fixture def repo(configuration: Configuration, repository_paths: RepositoryPaths) -> Repo: """ diff --git a/tests/ahriman/core/database/migrations/test_m002_user_access.py b/tests/ahriman/core/database/migrations/test_m002_user_access.py index d6d33053..b4e2a281 100644 --- a/tests/ahriman/core/database/migrations/test_m002_user_access.py +++ b/tests/ahriman/core/database/migrations/test_m002_user_access.py @@ -1,7 +1,7 @@ from ahriman.core.database.migrations.m002_user_access import steps -def test_migration_package_source() -> None: +def test_migration_user_access() -> None: """ migration must not be empty """ diff --git a/tests/ahriman/core/database/migrations/test_m003_patch_variables.py b/tests/ahriman/core/database/migrations/test_m003_patch_variables.py index f03b786c..19e6e83a 100644 --- a/tests/ahriman/core/database/migrations/test_m003_patch_variables.py +++ b/tests/ahriman/core/database/migrations/test_m003_patch_variables.py @@ -1,7 +1,7 @@ from ahriman.core.database.migrations.m003_patch_variables import steps -def test_migration_package_source() -> None: +def test_migration_patches() -> None: """ migration must not be empty """ diff --git a/tests/ahriman/core/database/migrations/test_m004_logs.py b/tests/ahriman/core/database/migrations/test_m004_logs.py new file mode 100644 index 00000000..b83bc88b --- /dev/null +++ b/tests/ahriman/core/database/migrations/test_m004_logs.py @@ -0,0 +1,8 @@ +from ahriman.core.database.migrations.m004_logs import steps + + +def test_migration_logs() -> None: + """ + migration must not be empty + """ + assert steps diff --git a/tests/ahriman/core/database/operations/test_build_operations.py b/tests/ahriman/core/database/operations/test_build_operations.py index dfe9dada..3e43e93d 100644 --- a/tests/ahriman/core/database/operations/test_build_operations.py +++ b/tests/ahriman/core/database/operations/test_build_operations.py @@ -35,7 +35,7 @@ def test_build_queue_insert_get(database: SQLite, package_ahriman: Package) -> N def test_build_queue_insert(database: SQLite, package_ahriman: Package) -> None: """ - must update user in the database + must update build queue in the database """ database.build_queue_insert(package_ahriman) assert database.build_queue_get() == [package_ahriman] diff --git a/tests/ahriman/core/database/operations/test_logs_operations.py b/tests/ahriman/core/database/operations/test_logs_operations.py new file mode 100644 index 00000000..456176c5 --- /dev/null +++ b/tests/ahriman/core/database/operations/test_logs_operations.py @@ -0,0 +1,23 @@ +from ahriman.core.database import SQLite +from ahriman.models.package import Package + + +def test_logs_insert_delete(database: SQLite, package_ahriman: Package, package_python_schedule: Package) -> None: + """ + must clear all packages + """ + database.logs_insert(package_ahriman.base, 0.001, "message 1") + database.logs_insert(package_python_schedule.base, 0.002, "message 2") + + database.logs_delete(package_ahriman.base) + assert not database.logs_get(package_ahriman.base) + assert database.logs_get(package_python_schedule.base) + + +def test_logs_insert_get(database: SQLite, package_ahriman: Package) -> None: + """ + must insert and get package logs + """ + database.logs_insert(package_ahriman.base, 0.002, "message 2") + database.logs_insert(package_ahriman.base, 0.001, "message 1") + assert database.logs_get(package_ahriman.base) == "message 1\nmessage 2" diff --git a/tests/ahriman/core/log/test_http_log_handler.py b/tests/ahriman/core/log/test_http_log_handler.py new file mode 100644 index 00000000..0f82d67f --- /dev/null +++ b/tests/ahriman/core/log/test_http_log_handler.py @@ -0,0 +1,56 @@ +import logging + +from pytest_mock import MockerFixture + +from ahriman.core.configuration import Configuration +from ahriman.core.log.http_log_handler import HttpLogHandler + + +def test_load(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must load handler + """ + # because of test cases we need to reset handler list + root = logging.getLogger() + current_handler = next((handler for handler in root.handlers if isinstance(handler, HttpLogHandler)), None) + root.removeHandler(current_handler) + + add_mock = mocker.patch("logging.Logger.addHandler") + load_mock = mocker.patch("ahriman.core.status.client.Client.load") + + handler = HttpLogHandler.load(configuration, report=False) + assert handler + add_mock.assert_called_once_with(handler) + load_mock.assert_called_once_with(configuration, report=False) + + +def test_load_exist(configuration: Configuration) -> None: + """ + must not load handler if already set + """ + handler = HttpLogHandler.load(configuration, report=False) + new_handler = HttpLogHandler.load(configuration, report=False) + assert handler is new_handler + + +def test_emit(configuration: Configuration, log_record: logging.LogRecord, mocker: MockerFixture) -> None: + """ + must emit log record to reporter + """ + log_mock = mocker.patch("ahriman.core.status.client.Client.logs") + handler = HttpLogHandler(configuration, report=False) + + handler.emit(log_record) + log_mock.assert_called_once_with(log_record) + + +def test_emit_failed(configuration: Configuration, log_record: logging.LogRecord, mocker: MockerFixture) -> None: + """ + must call handle error on exception + """ + mocker.patch("ahriman.core.status.client.Client.logs", side_effect=Exception()) + handle_error_mock = mocker.patch("logging.Handler.handleError") + handler = HttpLogHandler(configuration, report=False) + + handler.emit(log_record) + handle_error_mock.assert_called_once_with(log_record) diff --git a/tests/ahriman/core/test_lazy_logging.py b/tests/ahriman/core/log/test_lazy_logging.py similarity index 62% rename from tests/ahriman/core/test_lazy_logging.py rename to tests/ahriman/core/log/test_lazy_logging.py index b4712e17..4722d962 100644 --- a/tests/ahriman/core/test_lazy_logging.py +++ b/tests/ahriman/core/log/test_lazy_logging.py @@ -1,3 +1,4 @@ +import logging import pytest from ahriman.core.alpm.repo import Repo @@ -26,3 +27,19 @@ def test_logger_name(database: SQLite, repo: Repo) -> None: """ assert database.logger_name == "ahriman.core.database.sqlite.SQLite" assert repo.logger_name == "ahriman.core.alpm.repo.Repo" + + +def test_package_logger_set_reset(database: SQLite) -> None: + """ + must set and reset package base attribute + """ + package_base = "package base" + + database.package_logger_set(package_base) + record = logging.makeLogRecord({}) + assert record.package_base == package_base + + database.package_logger_reset() + record = logging.makeLogRecord({}) + with pytest.raises(AttributeError): + record.package_base diff --git a/tests/ahriman/core/log/test_log.py b/tests/ahriman/core/log/test_log.py new file mode 100644 index 00000000..13dfb05f --- /dev/null +++ b/tests/ahriman/core/log/test_log.py @@ -0,0 +1,35 @@ +import logging + +from pytest_mock import MockerFixture + +from ahriman.core.configuration import Configuration +from ahriman.core.log import Log + + +def test_load(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must load logging + """ + logging_mock = mocker.patch("ahriman.core.log.log.fileConfig") + http_log_mock = mocker.patch("ahriman.core.log.http_log_handler.HttpLogHandler.load") + + Log.load(configuration, quiet=False, report=False) + logging_mock.assert_called_once_with(configuration.logging_path) + http_log_mock.assert_called_once_with(configuration, report=False) + + +def test_load_fallback(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must fallback to stderr without errors + """ + mocker.patch("ahriman.core.log.log.fileConfig", side_effect=PermissionError()) + Log.load(configuration, quiet=False, report=False) + + +def test_load_quiet(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must disable logging in case if quiet flag set + """ + disable_mock = mocker.patch("logging.disable") + Log.load(configuration, quiet=True, report=False) + disable_mock.assert_called_once_with(logging.WARNING) diff --git a/tests/ahriman/core/repository/test_repository_properties.py b/tests/ahriman/core/repository/test_repository_properties.py index 739ab99c..61c745a9 100644 --- a/tests/ahriman/core/repository/test_repository_properties.py +++ b/tests/ahriman/core/repository/test_repository_properties.py @@ -4,7 +4,6 @@ from ahriman.core.configuration import Configuration from ahriman.core.database import SQLite from ahriman.core.exceptions import UnsafeRunError from ahriman.core.repository.repository_properties import RepositoryProperties -from ahriman.core.status.web_client import WebClient def test_create_tree_on_load(configuration: Configuration, database: SQLite, mocker: MockerFixture) -> None: @@ -27,26 +26,3 @@ def test_create_tree_on_load_unsafe(configuration: Configuration, database: SQLi RepositoryProperties("x86_64", configuration, database, report=False, unsafe=False) tree_create_mock.assert_not_called() - - -def test_create_dummy_report_client(configuration: Configuration, database: SQLite, mocker: MockerFixture) -> None: - """ - must create dummy report client if report is disabled - """ - mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create") - load_mock = mocker.patch("ahriman.core.status.client.Client.load") - properties = RepositoryProperties("x86_64", configuration, database, report=False, unsafe=False) - - load_mock.assert_not_called() - assert not isinstance(properties.reporter, WebClient) - - -def test_create_full_report_client(configuration: Configuration, database: SQLite, mocker: MockerFixture) -> None: - """ - must create load report client if report is enabled - """ - mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create") - load_mock = mocker.patch("ahriman.core.status.client.Client.load") - RepositoryProperties("x86_64", configuration, database, report=True, unsafe=True) - - load_mock.assert_called_once_with(configuration) diff --git a/tests/ahriman/core/status/test_client.py b/tests/ahriman/core/status/test_client.py index f8a4241e..87651426 100644 --- a/tests/ahriman/core/status/test_client.py +++ b/tests/ahriman/core/status/test_client.py @@ -1,3 +1,5 @@ +import logging + from pytest_mock import MockerFixture from ahriman.core.configuration import Configuration @@ -12,7 +14,16 @@ def test_load_dummy_client(configuration: Configuration) -> None: """ must load dummy client if no settings set """ - assert isinstance(Client.load(configuration), Client) + assert not isinstance(Client.load(configuration, report=True), WebClient) + + +def test_load_dummy_client_disabled(configuration: Configuration) -> None: + """ + must load dummy client if report is set to False + """ + configuration.set_option("web", "host", "localhost") + configuration.set_option("web", "port", "8080") + assert not isinstance(Client.load(configuration, report=False), WebClient) def test_load_full_client(configuration: Configuration) -> None: @@ -21,7 +32,7 @@ def test_load_full_client(configuration: Configuration) -> None: """ configuration.set_option("web", "host", "localhost") configuration.set_option("web", "port", "8080") - assert isinstance(Client.load(configuration), WebClient) + assert isinstance(Client.load(configuration, report=True), WebClient) def test_load_full_client_from_address(configuration: Configuration) -> None: @@ -29,7 +40,7 @@ def test_load_full_client_from_address(configuration: Configuration) -> None: must load full client by using address """ configuration.set_option("web", "address", "http://localhost:8080") - assert isinstance(Client.load(configuration), WebClient) + assert isinstance(Client.load(configuration, report=True), WebClient) def test_add(client: Client, package_ahriman: Package) -> None: @@ -57,6 +68,13 @@ def test_get_internal(client: Client) -> None: assert actual == expected +def test_log(client: Client, log_record: logging.LogRecord) -> None: + """ + must process log record without errors + """ + client.logs(log_record) + + def test_remove(client: Client, package_ahriman: Package) -> None: """ must process remove without errors diff --git a/tests/ahriman/core/status/test_watcher.py b/tests/ahriman/core/status/test_watcher.py index cc772755..1eada003 100644 --- a/tests/ahriman/core/status/test_watcher.py +++ b/tests/ahriman/core/status/test_watcher.py @@ -8,6 +8,7 @@ from ahriman.core.exceptions import UnknownPackageError from ahriman.core.status.watcher import Watcher from ahriman.core.status.web_client import WebClient from ahriman.models.build_status import BuildStatus, BuildStatusEnum +from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package @@ -18,10 +19,7 @@ def test_force_no_report(configuration: Configuration, database: SQLite, mocker: configuration.set_option("web", "port", "8080") mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create") - load_mock = mocker.patch("ahriman.core.status.client.Client.load") watcher = Watcher("x86_64", configuration, database) - - load_mock.assert_not_called() assert not isinstance(watcher.repository.reporter, WebClient) @@ -43,6 +41,15 @@ def test_get_failed(watcher: Watcher, package_ahriman: Package) -> None: watcher.get(package_ahriman.base) +def test_get_logs(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must return package logs + """ + logs_mock = mocker.patch("ahriman.core.database.SQLite.logs_get") + watcher.get_logs(package_ahriman.base) + logs_mock.assert_called_once_with(package_ahriman.base) + + def test_load(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: """ must correctly load packages @@ -83,6 +90,15 @@ def test_remove(watcher: Watcher, package_ahriman: Package, mocker: MockerFixtur cache_mock.assert_called_once_with(package_ahriman.base) +def test_remove_logs(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must remove package logs + """ + logs_mock = mocker.patch("ahriman.core.database.SQLite.logs_delete") + watcher.remove_logs(package_ahriman.base) + logs_mock.assert_called_once_with(package_ahriman.base) + + def test_remove_unknown(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: """ must not fail on unknown base removal @@ -128,6 +144,38 @@ def test_update_unknown(watcher: Watcher, package_ahriman: Package) -> None: watcher.update(package_ahriman.base, BuildStatusEnum.Unknown, None) +def test_update_logs_new(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must create package logs record for new package + """ + delete_mock = mocker.patch("ahriman.core.database.SQLite.logs_delete") + insert_mock = mocker.patch("ahriman.core.database.SQLite.logs_insert") + + log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.process_id) + assert watcher._last_log_record_id != log_record_id + + watcher.update_logs(log_record_id, 42.01, "log record") + delete_mock.assert_called_once_with(package_ahriman.base) + insert_mock.assert_called_once_with(package_ahriman.base, 42.01, "log record") + + assert watcher._last_log_record_id == log_record_id + + +def test_update_logs_update(watcher: Watcher, package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must create package logs record for current package + """ + delete_mock = mocker.patch("ahriman.core.database.SQLite.logs_delete") + insert_mock = mocker.patch("ahriman.core.database.SQLite.logs_insert") + + log_record_id = LogRecordId(package_ahriman.base, watcher._last_log_record_id.process_id) + watcher._last_log_record_id = log_record_id + + watcher.update_logs(log_record_id, 42.01, "log record") + delete_mock.assert_not_called() + insert_mock.assert_called_once_with(package_ahriman.base, 42.01, "log record") + + def test_update_self(watcher: Watcher) -> None: """ must update service status diff --git a/tests/ahriman/core/status/test_web_client.py b/tests/ahriman/core/status/test_web_client.py index 525bc397..49b6fa11 100644 --- a/tests/ahriman/core/status/test_web_client.py +++ b/tests/ahriman/core/status/test_web_client.py @@ -1,4 +1,5 @@ import json +import logging import pytest import requests @@ -13,6 +14,14 @@ from ahriman.models.package import Package from ahriman.models.user import User +def test_status_url(web_client: WebClient) -> None: + """ + must generate login url correctly + """ + assert web_client._login_url.startswith(web_client.address) + assert web_client._login_url.endswith("/api/v1/login") + + def test_status_url(web_client: WebClient) -> None: """ must generate package status url correctly @@ -75,9 +84,17 @@ def test_login_skip(web_client: WebClient, mocker: MockerFixture) -> None: requests_mock.assert_not_called() +def test_logs_url(web_client: WebClient, package_ahriman: Package) -> None: + """ + must generate logs url correctly + """ + assert web_client._logs_url(package_ahriman.base).startswith(web_client.address) + assert web_client._logs_url(package_ahriman.base).endswith(f"/api/v1/packages/{package_ahriman.base}/logs") + + def test_package_url(web_client: WebClient, package_ahriman: Package) -> None: """ - must generate package status correctly + must generate package status url correctly """ assert web_client._package_url(package_ahriman.base).startswith(web_client.address) assert web_client._package_url(package_ahriman.base).endswith(f"/api/v1/packages/{package_ahriman.base}") @@ -192,6 +209,43 @@ def test_get_internal_failed_http_error(web_client: WebClient, mocker: MockerFix assert web_client.get_internal().architecture is None +def test_logs(web_client: WebClient, log_record: logging.LogRecord, package_ahriman: Package, + mocker: MockerFixture) -> None: + """ + must process log record + """ + requests_mock = mocker.patch("requests.Session.post") + log_record.package_base = package_ahriman.base + payload = { + "created": log_record.created, + "message": log_record.getMessage(), + "process_id": log_record.process, + } + + web_client.logs(log_record) + requests_mock.assert_called_once_with(pytest.helpers.anyvar(str, True), json=payload) + + +def test_log_failed(web_client: WebClient, log_record: logging.LogRecord, package_ahriman: Package, + mocker: MockerFixture) -> None: + """ + must pass exception during log post + """ + mocker.patch("requests.Session.post", side_effect=Exception()) + log_record.package_base = package_ahriman.base + with pytest.raises(Exception): + web_client.logs(log_record) + + +def test_log_skip(web_client: WebClient, log_record: logging.LogRecord, mocker: MockerFixture) -> None: + """ + must skip log record posting if no package base set + """ + requests_mock = mocker.patch("requests.Session.post") + web_client.logs(log_record) + requests_mock.assert_not_called() + + def test_remove(web_client: WebClient, package_ahriman: Package, mocker: MockerFixture) -> None: """ must process package removal diff --git a/tests/ahriman/core/test_configuration.py b/tests/ahriman/core/test_configuration.py index f70dcce7..1f31e304 100644 --- a/tests/ahriman/core/test_configuration.py +++ b/tests/ahriman/core/test_configuration.py @@ -1,5 +1,4 @@ import configparser -import logging import pytest from pathlib import Path @@ -25,14 +24,12 @@ def test_from_path(mocker: MockerFixture) -> None: mocker.patch("pathlib.Path.is_file", return_value=True) read_mock = mocker.patch("ahriman.core.configuration.Configuration.read") load_includes_mock = mocker.patch("ahriman.core.configuration.Configuration.load_includes") - load_logging_mock = mocker.patch("ahriman.core.configuration.Configuration.load_logging") path = Path("path") - configuration = Configuration.from_path(path, "x86_64", True) + configuration = Configuration.from_path(path, "x86_64") assert configuration.path == path read_mock.assert_called_once_with(path) load_includes_mock.assert_called_once_with() - load_logging_mock.assert_called_once_with(True) def test_from_path_file_missing(mocker: MockerFixture) -> None: @@ -41,10 +38,9 @@ def test_from_path_file_missing(mocker: MockerFixture) -> None: """ mocker.patch("pathlib.Path.is_file", return_value=False) mocker.patch("ahriman.core.configuration.Configuration.load_includes") - mocker.patch("ahriman.core.configuration.Configuration.load_logging") read_mock = mocker.patch("ahriman.core.configuration.Configuration.read") - configuration = Configuration.from_path(Path("path"), "x86_64", True) + configuration = Configuration.from_path(Path("path"), "x86_64") read_mock.assert_called_once_with(configuration.SYSTEM_CONFIGURATION_PATH) @@ -263,23 +259,6 @@ def test_load_includes_no_section(configuration: Configuration) -> None: configuration.load_includes() -def test_load_logging_fallback(configuration: Configuration, mocker: MockerFixture) -> None: - """ - must fallback to stderr without errors - """ - mocker.patch("ahriman.core.configuration.fileConfig", side_effect=PermissionError()) - configuration.load_logging(quiet=False) - - -def test_load_logging_quiet(configuration: Configuration, mocker: MockerFixture) -> None: - """ - must disable logging in case if quiet flag set - """ - disable_mock = mocker.patch("logging.disable") - configuration.load_logging(quiet=True) - disable_mock.assert_called_once_with(logging.WARNING) - - def test_merge_sections_missing(configuration: Configuration) -> None: """ must merge create section if not exists diff --git a/tests/ahriman/models/test_log_record_id.py b/tests/ahriman/models/test_log_record_id.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/ahriman/web/views/status/test_views_status_logs.py b/tests/ahriman/web/views/status/test_views_status_logs.py new file mode 100644 index 00000000..4f059368 --- /dev/null +++ b/tests/ahriman/web/views/status/test_views_status_logs.py @@ -0,0 +1,75 @@ +import pytest + +from aiohttp.test_utils import TestClient + +from ahriman.models.package import Package +from ahriman.models.user_access import UserAccess +from ahriman.web.views.status.logs import LogsView + + +async def test_get_permission() -> None: + """ + must return correct permission for the request + """ + for method in ("GET", "HEAD"): + request = pytest.helpers.request("", "", method) + assert await LogsView.get_permission(request) == UserAccess.Read + for method in ("DELETE", "POST"): + request = pytest.helpers.request("", "", method) + assert await LogsView.get_permission(request) == UserAccess.Full + + +async def test_delete(client: TestClient, package_ahriman: Package, package_python_schedule: Package) -> None: + """ + must delete logs for package + """ + await client.post(f"/api/v1/packages/{package_ahriman.base}/logs", + json={"created": 0.001, "message": "message", "process_id": 42}) + await client.post(f"/api/v1/packages/{package_python_schedule.base}/logs", + json={"created": 0.001, "message": "message", "process_id": 42}) + + response = await client.delete(f"/api/v1/packages/{package_ahriman.base}/logs") + assert response.status == 204 + + response = await client.get(f"/api/v1/packages/{package_ahriman.base}/logs") + logs = await response.json() + assert not logs["logs"] + + response = await client.get(f"/api/v1/packages/{package_python_schedule.base}/logs") + logs = await response.json() + assert logs["logs"] + + +async def test_get(client: TestClient, package_ahriman: Package) -> None: + """ + must get logs for package + """ + await client.post(f"/api/v1/packages/{package_ahriman.base}/logs", + json={"created": 0.001, "message": "message", "process_id": 42}) + + response = await client.get(f"/api/v1/packages/{package_ahriman.base}/logs") + assert response.status == 200 + + logs = await response.json() + assert logs == {"package_base": package_ahriman.base, "logs": "message"} + + +async def test_post(client: TestClient, package_ahriman: Package) -> None: + """ + must create logs record + """ + post_response = await client.post(f"/api/v1/packages/{package_ahriman.base}/logs", + json={"created": 0.001, "message": "message", "process_id": 42}) + assert post_response.status == 204 + + response = await client.get(f"/api/v1/packages/{package_ahriman.base}/logs") + logs = await response.json() + assert logs == {"package_base": package_ahriman.base, "logs": "message"} + + +async def test_post_exception(client: TestClient, package_ahriman: Package) -> None: + """ + must raise exception on invalid payload + """ + post_response = await client.post(f"/api/v1/packages/{package_ahriman.base}/logs", json={}) + assert post_response.status == 400 diff --git a/tests/ahriman/web/views/status/test_views_status_package.py b/tests/ahriman/web/views/status/test_views_status_package.py index 733add44..8becb4b4 100644 --- a/tests/ahriman/web/views/status/test_views_status_package.py +++ b/tests/ahriman/web/views/status/test_views_status_package.py @@ -20,31 +20,6 @@ async def test_get_permission() -> None: assert await PackageView.get_permission(request) == UserAccess.Full -async def test_get(client: TestClient, package_ahriman: Package, package_python_schedule: Package) -> None: - """ - must return status for specific package - """ - await client.post(f"/api/v1/packages/{package_ahriman.base}", - json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()}) - await client.post(f"/api/v1/packages/{package_python_schedule.base}", - json={"status": BuildStatusEnum.Success.value, "package": package_python_schedule.view()}) - - response = await client.get(f"/api/v1/packages/{package_ahriman.base}") - assert response.ok - - packages = [Package.from_json(item["package"]) for item in await response.json()] - assert packages - assert {package.base for package in packages} == {package_ahriman.base} - - -async def test_get_not_found(client: TestClient, package_ahriman: Package) -> None: - """ - must return Not Found for unknown package - """ - response = await client.get(f"/api/v1/packages/{package_ahriman.base}") - assert response.status == 404 - - async def test_delete(client: TestClient, package_ahriman: Package, package_python_schedule: Package) -> None: """ must delete single base @@ -81,6 +56,31 @@ async def test_delete_unknown(client: TestClient, package_ahriman: Package, pack assert response.ok +async def test_get(client: TestClient, package_ahriman: Package, package_python_schedule: Package) -> None: + """ + must return status for specific package + """ + await client.post(f"/api/v1/packages/{package_ahriman.base}", + json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()}) + await client.post(f"/api/v1/packages/{package_python_schedule.base}", + json={"status": BuildStatusEnum.Success.value, "package": package_python_schedule.view()}) + + response = await client.get(f"/api/v1/packages/{package_ahriman.base}") + assert response.ok + + packages = [Package.from_json(item["package"]) for item in await response.json()] + assert packages + assert {package.base for package in packages} == {package_ahriman.base} + + +async def test_get_not_found(client: TestClient, package_ahriman: Package) -> None: + """ + must return Not Found for unknown package + """ + response = await client.get(f"/api/v1/packages/{package_ahriman.base}") + assert response.status == 404 + + async def test_post(client: TestClient, package_ahriman: Package) -> None: """ must update package status