From 137d62e2f8386ceaf5eb47770da9b418ee7fc9eb Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Tue, 22 Nov 2022 02:58:22 +0300 Subject: [PATCH] Extended package status page (#76) * implement log storage at backend * handle process id during removal. During one process we can write logs from different packages in different times (e.g. check and update later) and we would like to store all logs belong to the same process * set package context in main functions * implement logs support in interface * filter out logs posting http logs * add timestamp to log records * hide getting logs under reporter permission List of breaking changes: * `ahriman.core.lazy_logging.LazyLogging` has been renamed to `ahriman.core.log.LazyLogging` * `ahriman.core.configuration.Configuration.from_path` does not have `quiet` attribute now * `ahriman.core.configuration.Configuration` class does not have `load_logging` method now * `ahriman.core.status.client.Client.load` requires `report` argument now --- CONTRIBUTING.md | 2 +- docs/ahriman.core.database.migrations.rst | 8 ++ docs/ahriman.core.database.operations.rst | 8 ++ docs/ahriman.core.log.rst | 37 ++++++ docs/ahriman.core.rst | 9 +- docs/ahriman.models.rst | 8 ++ docs/ahriman.web.views.status.rst | 8 ++ .../ahriman/templates/build-status.jinja2 | 9 +- .../build-status/failed-modal.jinja2 | 14 ++- .../templates/build-status/login-modal.jinja2 | 4 +- .../build-status/package-add-modal.jinja2 | 8 +- .../build-status/package-info-modal.jinja2 | 71 ++++++++++++ .../build-status/success-modal.jinja2 | 14 ++- .../templates/build-status/table.jinja2 | 15 ++- .../share/ahriman/templates/repo-index.jinja2 | 26 ++++- .../templates/utils/bootstrap-scripts.jinja2 | 9 +- .../ahriman/templates/utils/style.jinja2 | 20 +++- setup.py | 1 + .../application/application_properties.py | 5 +- src/ahriman/application/handlers/handler.py | 4 +- src/ahriman/application/lock.py | 4 +- src/ahriman/core/alpm/pacman.py | 2 +- src/ahriman/core/alpm/remote/remote.py | 2 +- src/ahriman/core/alpm/repo.py | 2 +- src/ahriman/core/auth/auth.py | 4 +- src/ahriman/core/auth/oauth.py | 2 +- src/ahriman/core/build_tools/sources.py | 14 ++- src/ahriman/core/build_tools/task.py | 12 +- src/ahriman/core/configuration.py | 32 +----- .../core/database/migrations/__init__.py | 2 +- .../core/database/migrations/m004_logs.py | 35 ++++++ .../core/database/operations/__init__.py | 1 + .../database/operations/build_operations.py | 2 +- .../database/operations/logs_operations.py | 102 +++++++++++++++++ .../core/database/operations/operations.py | 3 +- src/ahriman/core/database/sqlite.py | 5 +- src/ahriman/core/gitremote/remote_pull.py | 2 +- src/ahriman/core/gitremote/remote_push.py | 2 +- src/ahriman/core/log/__init__.py | 21 ++++ .../core/log/filtered_access_logger.py | 61 ++++++++++ src/ahriman/core/log/http_log_handler.py | 80 +++++++++++++ src/ahriman/core/{ => log}/lazy_logging.py | 47 +++++++- src/ahriman/core/log/log.py | 61 ++++++++++ src/ahriman/core/report/report.py | 2 +- src/ahriman/core/repository/executor.py | 49 ++++---- .../core/repository/repository_properties.py | 7 +- src/ahriman/core/repository/update_handler.py | 65 +++++------ src/ahriman/core/sign/gpg.py | 8 +- src/ahriman/core/spawn.py | 2 +- src/ahriman/core/status/client.py | 49 +++++--- src/ahriman/core/status/watcher.py | 54 ++++++++- src/ahriman/core/status/web_client.py | 72 ++++++++---- src/ahriman/core/triggers/trigger.py | 2 +- src/ahriman/core/triggers/trigger_loader.py | 2 +- src/ahriman/core/upload/upload.py | 2 +- src/ahriman/core/util.py | 3 +- src/ahriman/models/log_record_id.py | 34 ++++++ src/ahriman/models/package.py | 7 +- src/ahriman/web/routes.py | 9 ++ src/ahriman/web/views/status/logs.py | 105 ++++++++++++++++++ src/ahriman/web/views/status/package.py | 34 +++--- src/ahriman/web/web.py | 3 +- .../application/handlers/test_handler.py | 8 +- .../handlers/test_handler_status.py | 2 +- .../handlers/test_handler_status_update.py | 2 +- tests/ahriman/conftest.py | 2 +- tests/ahriman/core/conftest.py | 12 ++ .../migrations/test_m002_user_access.py | 2 +- .../migrations/test_m003_patch_variables.py | 2 +- .../database/migrations/test_m004_logs.py | 8 ++ .../operations/test_build_operations.py | 2 +- .../operations/test_logs_operations.py | 39 +++++++ tests/ahriman/core/log/conftest.py | 16 +++ .../core/log/test_filtered_access_logger.py | 71 ++++++++++++ .../ahriman/core/log/test_http_log_handler.py | 56 ++++++++++ tests/ahriman/core/log/test_lazy_logging.py | 76 +++++++++++++ tests/ahriman/core/log/test_log.py | 35 ++++++ .../ahriman/core/repository/test_executor.py | 2 + .../repository/test_repository_properties.py | 24 ---- .../core/repository/test_update_handler.py | 11 +- tests/ahriman/core/status/test_client.py | 24 +++- tests/ahriman/core/status/test_watcher.py | 56 +++++++++- tests/ahriman/core/status/test_web_client.py | 56 +++++++++- tests/ahriman/core/test_configuration.py | 25 +---- tests/ahriman/core/test_lazy_logging.py | 28 ----- tests/ahriman/core/test_util.py | 1 + tests/ahriman/models/test_log_record_id.py | 0 tests/ahriman/web/test_web.py | 19 +++- .../views/status/test_views_status_logs.py | 94 ++++++++++++++++ .../views/status/test_views_status_package.py | 50 ++++----- 90 files changed, 1650 insertions(+), 360 deletions(-) create mode 100644 docs/ahriman.core.log.rst create mode 100644 package/share/ahriman/templates/build-status/package-info-modal.jinja2 create mode 100644 src/ahriman/core/database/migrations/m004_logs.py create mode 100644 src/ahriman/core/database/operations/logs_operations.py create mode 100644 src/ahriman/core/log/__init__.py create mode 100644 src/ahriman/core/log/filtered_access_logger.py create mode 100644 src/ahriman/core/log/http_log_handler.py rename src/ahriman/core/{ => log}/lazy_logging.py (56%) create mode 100644 src/ahriman/core/log/log.py create mode 100644 src/ahriman/models/log_record_id.py create mode 100644 src/ahriman/web/views/status/logs.py create mode 100644 tests/ahriman/core/database/migrations/test_m004_logs.py create mode 100644 tests/ahriman/core/database/operations/test_logs_operations.py create mode 100644 tests/ahriman/core/log/conftest.py create mode 100644 tests/ahriman/core/log/test_filtered_access_logger.py create mode 100644 tests/ahriman/core/log/test_http_log_handler.py create mode 100644 tests/ahriman/core/log/test_lazy_logging.py create mode 100644 tests/ahriman/core/log/test_log.py delete mode 100644 tests/ahriman/core/test_lazy_logging.py create mode 100644 tests/ahriman/models/test_log_record_id.py create mode 100644 tests/ahriman/web/views/status/test_views_status_logs.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c5e36ad1..8abab5b6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,7 +61,7 @@ Again, the most checks can be performed by `make check` command, though some add * The file size mentioned above must be applicable in general. In case of big classes consider splitting them into traits. Note, however, that `pylint` includes comments and docstrings into counter, thus you need to check file size by other tools. * No global variable is allowed outside of `ahriman.version` module. * Single quotes are not allowed. The reason behind this restriction is the fact that docstrings must be written by using double quotes only, and we would like to make style consistent. -* If your class writes anything to log, the `ahriman.core.lazy_logging.LazyLogging` trait must be used. +* If your class writes anything to log, the `ahriman.core.log.LazyLogging` trait must be used. ### Other checks diff --git a/docs/ahriman.core.database.migrations.rst b/docs/ahriman.core.database.migrations.rst index 9fd425af..4a13a96e 100644 --- a/docs/ahriman.core.database.migrations.rst +++ b/docs/ahriman.core.database.migrations.rst @@ -36,6 +36,14 @@ ahriman.core.database.migrations.m003\_patch\_variables module :no-undoc-members: :show-inheritance: +ahriman.core.database.migrations.m004\_logs module +-------------------------------------------------- + +.. automodule:: ahriman.core.database.migrations.m004_logs + :members: + :no-undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/ahriman.core.database.operations.rst b/docs/ahriman.core.database.operations.rst index bcc55124..4a8887f0 100644 --- a/docs/ahriman.core.database.operations.rst +++ b/docs/ahriman.core.database.operations.rst @@ -20,6 +20,14 @@ ahriman.core.database.operations.build\_operations module :no-undoc-members: :show-inheritance: +ahriman.core.database.operations.logs\_operations module +-------------------------------------------------------- + +.. automodule:: ahriman.core.database.operations.logs_operations + :members: + :no-undoc-members: + :show-inheritance: + ahriman.core.database.operations.operations module -------------------------------------------------- diff --git a/docs/ahriman.core.log.rst b/docs/ahriman.core.log.rst new file mode 100644 index 00000000..ded2b4d8 --- /dev/null +++ b/docs/ahriman.core.log.rst @@ -0,0 +1,37 @@ +ahriman.core.log package +======================== + +Submodules +---------- + +ahriman.core.log.http\_log\_handler module +------------------------------------------ + +.. automodule:: ahriman.core.log.http_log_handler + :members: + :no-undoc-members: + :show-inheritance: + +ahriman.core.log.lazy\_logging module +------------------------------------- + +.. automodule:: ahriman.core.log.lazy_logging + :members: + :no-undoc-members: + :show-inheritance: + +ahriman.core.log.log module +--------------------------- + +.. automodule:: ahriman.core.log.log + :members: + :no-undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: ahriman.core.log + :members: + :no-undoc-members: + :show-inheritance: diff --git a/docs/ahriman.core.rst b/docs/ahriman.core.rst index 1462484e..67795328 100644 --- a/docs/ahriman.core.rst +++ b/docs/ahriman.core.rst @@ -13,6 +13,7 @@ Subpackages ahriman.core.database ahriman.core.formatters ahriman.core.gitremote + ahriman.core.log ahriman.core.report ahriman.core.repository ahriman.core.sign @@ -39,14 +40,6 @@ ahriman.core.exceptions module :no-undoc-members: :show-inheritance: -ahriman.core.lazy\_logging module ---------------------------------- - -.. automodule:: ahriman.core.lazy_logging - :members: - :no-undoc-members: - :show-inheritance: - ahriman.core.spawn module ------------------------- diff --git a/docs/ahriman.models.rst b/docs/ahriman.models.rst index be4468cd..87c62602 100644 --- a/docs/ahriman.models.rst +++ b/docs/ahriman.models.rst @@ -52,6 +52,14 @@ ahriman.models.internal\_status module :no-undoc-members: :show-inheritance: +ahriman.models.log\_record\_id module +------------------------------------- + +.. automodule:: ahriman.models.log_record_id + :members: + :no-undoc-members: + :show-inheritance: + ahriman.models.migration module ------------------------------- diff --git a/docs/ahriman.web.views.status.rst b/docs/ahriman.web.views.status.rst index a5dcd410..ba44c277 100644 --- a/docs/ahriman.web.views.status.rst +++ b/docs/ahriman.web.views.status.rst @@ -4,6 +4,14 @@ ahriman.web.views.status package Submodules ---------- +ahriman.web.views.status.logs module +------------------------------------ + +.. automodule:: ahriman.web.views.status.logs + :members: + :no-undoc-members: + :show-inheritance: + ahriman.web.views.status.package module --------------------------------------- diff --git a/package/share/ahriman/templates/build-status.jinja2 b/package/share/ahriman/templates/build-status.jinja2 index 33d07b8f..7306953b 100644 --- a/package/share/ahriman/templates/build-status.jinja2 +++ b/package/share/ahriman/templates/build-status.jinja2 @@ -40,7 +40,6 @@