mirror of
https://github.com/arcan1s/ahriman.git
synced 2025-05-05 20:53:50 +00:00
Compare commits
20 Commits
1d85a61cc4
...
013ba3d3ab
Author | SHA1 | Date | |
---|---|---|---|
013ba3d3ab | |||
6099a5957d | |||
41343fd9e1 | |||
a576a0b612 | |||
05562d2ee5 | |||
3098132de2 | |||
4e246d3a67 | |||
6577ca9db1 | |||
6e37a60cf0 | |||
a23a1bc613 | |||
fc508e19b8 | |||
09c8fd945d | |||
b90d93f3c0 | |||
cd98b7f6e6 | |||
08c1b08902 | |||
a9003993fa | |||
54a331cc96 | |||
5f79cbc34b | |||
ea4193eef4 | |||
40fa94afbb |
8
.github/workflows/setup.sh
vendored
8
.github/workflows/setup.sh
vendored
@ -10,7 +10,7 @@ echo -e '[arcanisrepo]\nServer = https://repo.arcanis.me/$arch\nSigLevel = Never
|
||||
# refresh the image
|
||||
pacman -Syu --noconfirm
|
||||
# main dependencies
|
||||
pacman -Sy --noconfirm devtools git pyalpm python-cerberus python-inflection python-passlib python-pyelftools python-requests python-srcinfo python-systemd sudo
|
||||
pacman -Sy --noconfirm devtools git pyalpm python-inflection python-passlib python-pyelftools python-requests python-srcinfo python-systemd sudo
|
||||
# make dependencies
|
||||
pacman -Sy --noconfirm --asdeps base-devel python-build python-flit python-installer python-tox python-wheel
|
||||
# optional dependencies
|
||||
@ -20,7 +20,7 @@ if [[ -z $MINIMAL_INSTALL ]]; then
|
||||
# web server
|
||||
pacman -Sy --noconfirm python-aioauth-client python-aiohttp python-aiohttp-apispec-git python-aiohttp-cors python-aiohttp-jinja2 python-aiohttp-security python-aiohttp-session python-cryptography python-jinja
|
||||
# additional features
|
||||
pacman -Sy --noconfirm gnupg python-boto3 rsync
|
||||
pacman -Sy --noconfirm gnupg python-boto3 python-cerberus python-matplotlib rsync
|
||||
fi
|
||||
# FIXME since 1.0.4 devtools requires dbus to be run, which doesn't work now in container
|
||||
cp "docker/systemd-nspawn.sh" "/usr/local/bin/systemd-nspawn"
|
||||
@ -42,12 +42,12 @@ pacman -Qdtq | pacman -Rscn --noconfirm -
|
||||
# initial setup command as root
|
||||
[[ -z $MINIMAL_INSTALL ]] && WEB_ARGS=("--web-port" "8080")
|
||||
ahriman -a x86_64 -r "github" service-setup --packager "ahriman bot <ahriman@example.com>" "${WEB_ARGS[@]}"
|
||||
# validate configuration
|
||||
ahriman service-config-validate --exit-code
|
||||
# enable services
|
||||
systemctl enable ahriman-web
|
||||
systemctl enable ahriman@x86_64-github.timer
|
||||
if [[ -z $MINIMAL_INSTALL ]]; then
|
||||
# validate configuration
|
||||
ahriman service-config-validate --exit-code
|
||||
# run web service (detached)
|
||||
sudo -u ahriman -- ahriman web &
|
||||
WEB_PID=$!
|
||||
|
@ -12,6 +12,7 @@ python:
|
||||
extra_requirements:
|
||||
- docs
|
||||
- s3
|
||||
- validator
|
||||
- web
|
||||
|
||||
formats:
|
||||
|
@ -36,6 +36,7 @@ Again, the most checks can be performed by `tox` command, though some additional
|
||||
|
||||
Notes:
|
||||
Very important note about this function
|
||||
Probably multi-line
|
||||
|
||||
Args:
|
||||
argument(str): an argument. This argument has
|
||||
@ -70,6 +71,7 @@ Again, the most checks can be performed by `tox` command, though some additional
|
||||
Attributes:
|
||||
CLAZZ_ATTRIBUTE(int): (class attribute) a brand-new class attribute
|
||||
instance_attribute(str): an instance attribute
|
||||
with the long description
|
||||
|
||||
Examples:
|
||||
Very informative class usage example, e.g.::
|
||||
@ -92,7 +94,7 @@ Again, the most checks can be performed by `tox` command, though some additional
|
||||
```
|
||||
|
||||
* Type annotations are the must, even for local functions. For the function argument `self` (for instance methods) and `cls` (for class methods) should not be annotated.
|
||||
* For collection types built-in classes must be used if possible (e.g. `dict` instead of `typing.Dict`, `tuple` instead of `typing.Tuple`). In case if built-in type is not available, but `collections.abc` provides interface, it must be used (e.g. `collections.abc.Awaitable` instead of `typing.Awaitable`, `collections.abc.Iterable` instead of `typing.Iterable`). For union classes, the bar operator (`|`) must be used (e.g. `float | int` instead of `typing.Union[float, int]`), which also includes `typinng.Optional` (e.g. `str | None` instead of `Optional[str]`).
|
||||
* For collection types built-in classes must be used if possible (e.g. `dict` instead of `typing.Dict`, `tuple` instead of `typing.Tuple`). In case if built-in type is not available, but `collections.abc` provides interface, it must be used (e.g. `collections.abc.Awaitable` instead of `typing.Awaitable`, `collections.abc.Iterable` instead of `typing.Iterable`). For union classes, the bar operator (`|`) must be used (e.g. `float | int` instead of `typing.Union[float, int]`), which also includes `typing.Optional` (e.g. `str | None` instead of `Optional[str]`).
|
||||
* `classmethod` should (almost) always return `Self`. In case of mypy warning (e.g. if there is a branch in which function doesn't return the instance of `cls`) consider using `staticmethod` instead.
|
||||
* Recommended order of function definitions in class:
|
||||
|
||||
@ -132,7 +134,7 @@ Again, the most checks can be performed by `tox` command, though some additional
|
||||
* For any path interactions `pathlib.Path` must be used.
|
||||
* Configuration interactions must go through `ahriman.core.configuration.Configuration` class instance.
|
||||
* In case if class load requires some actions, it is recommended to create class method which can be used for class instantiating.
|
||||
* The code must follow the exception safety, unless it is explicitly asked by end user. It means that most exceptions must be handled and printed to log, no other actions must be done (e.g. raising another exception).
|
||||
* The most (expected) exceptions must be handled and printed to log, allowing service to continue work. However, fatal and (in some cases) unexpected exceptions may lead to the application termination.
|
||||
* Exceptions without parameters should be raised without parentheses, e.g.:
|
||||
|
||||
```python
|
||||
|
45
Dockerfile
45
Dockerfile
@ -32,11 +32,46 @@ RUN useradd -m -d "/home/build" -s "/usr/bin/nologin" build && \
|
||||
COPY "docker/install-aur-package.sh" "/usr/local/bin/install-aur-package"
|
||||
## install package dependencies
|
||||
## darcs is not installed by reasons, because it requires a lot haskell packages which dramatically increase image size
|
||||
RUN pacman -Sy --noconfirm --asdeps devtools git pyalpm python-cerberus python-inflection python-passlib python-pyelftools python-requests python-srcinfo && \
|
||||
pacman -Sy --noconfirm --asdeps base-devel python-build python-flit python-installer python-wheel && \
|
||||
pacman -Sy --noconfirm --asdeps breezy git mercurial python-aiohttp python-boto3 python-cryptography python-jinja python-systemd rsync subversion && \
|
||||
runuser -u build -- install-aur-package python-aioauth-client python-webargs python-aiohttp-apispec-git python-aiohttp-cors \
|
||||
python-aiohttp-jinja2 python-aiohttp-session python-aiohttp-security python-requests-unixsocket2
|
||||
RUN pacman -Sy --noconfirm --asdeps \
|
||||
devtools \
|
||||
git \
|
||||
pyalpm \
|
||||
python-inflection \
|
||||
python-passlib \
|
||||
python-pyelftools \
|
||||
python-requests \
|
||||
python-srcinfo \
|
||||
&& \
|
||||
pacman -Sy --noconfirm --asdeps \
|
||||
base-devel \
|
||||
python-build \
|
||||
python-flit \
|
||||
python-installer \
|
||||
python-wheel \
|
||||
&& \
|
||||
pacman -Sy --noconfirm --asdeps \
|
||||
breezy \
|
||||
git \
|
||||
mercurial \
|
||||
python-aiohttp \
|
||||
python-boto3 \
|
||||
python-cerberus \
|
||||
python-cryptography \
|
||||
python-jinja \
|
||||
python-matplotlib \
|
||||
python-systemd \
|
||||
rsync \
|
||||
subversion \
|
||||
&& \
|
||||
runuser -u build -- install-aur-package \
|
||||
python-aioauth-client \
|
||||
python-webargs \
|
||||
python-aiohttp-apispec-git \
|
||||
python-aiohttp-cors \
|
||||
python-aiohttp-jinja2 \
|
||||
python-aiohttp-session \
|
||||
python-aiohttp-security \
|
||||
python-requests-unixsocket2
|
||||
|
||||
## FIXME since 1.0.4 devtools requires dbus to be run, which doesn't work now in container
|
||||
COPY "docker/systemd-nspawn.sh" "/usr/local/bin/systemd-nspawn"
|
||||
|
@ -2,7 +2,7 @@
|
||||
|
||||
[](https://github.com/arcan1s/ahriman/actions/workflows/tests.yml)
|
||||
[](https://github.com/arcan1s/ahriman/actions/workflows/setup.yml)
|
||||
[](https://hub.docker.com/r/arcan1s/ahriman)
|
||||
[](https://hub.docker.com/r/arcan1s/ahriman)
|
||||
[](https://www.codefactor.io/repository/github/arcan1s/ahriman)
|
||||
[](https://ahriman.readthedocs.io)
|
||||
|
||||
@ -40,3 +40,5 @@ The application provides reasonable defaults which allow to use it out-of-box; h
|
||||
* [Build status page](https://ahriman-demo.arcanis.me). You can log in as `demo` user by using `demo` password. However, you will not be able to run tasks. [HTTP API documentation](https://ahriman-demo.arcanis.me/api-docs) is also available.
|
||||
* [Repository index](https://repo.arcanis.me/arcanisrepo/x86_64/).
|
||||
* [Telegram feed](https://t.me/arcanisrepo).
|
||||
|
||||
Do you have any success story? You can [share it](https://github.com/arcan1s/ahriman/issues/new?template=04-discussion.md)!
|
||||
|
@ -8,9 +8,6 @@ cat <<EOF > "/etc/ahriman.ini.d/00-docker.ini"
|
||||
[repository]
|
||||
root = $AHRIMAN_REPOSITORY_ROOT
|
||||
|
||||
[settings]
|
||||
database = $AHRIMAN_REPOSITORY_ROOT/ahriman.db
|
||||
|
||||
[web]
|
||||
host = $AHRIMAN_HOST
|
||||
|
||||
|
@ -172,6 +172,14 @@ ahriman.application.handlers.sign module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.application.handlers.statistics module
|
||||
----------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.application.handlers.statistics
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.application.handlers.status module
|
||||
------------------------------------------
|
||||
|
||||
|
@ -4,6 +4,14 @@ ahriman.core.build\_tools package
|
||||
Submodules
|
||||
----------
|
||||
|
||||
ahriman.core.build\_tools.package\_archive module
|
||||
-------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.build_tools.package_archive
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.build\_tools.sources module
|
||||
----------------------------------------
|
||||
|
||||
|
@ -116,6 +116,14 @@ ahriman.core.database.migrations.m013\_dependencies module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.database.migrations.m014\_auditlog module
|
||||
------------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.database.migrations.m014_auditlog
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
Module contents
|
||||
---------------
|
||||
|
||||
|
@ -36,6 +36,14 @@ ahriman.core.database.operations.dependencies\_operations module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.database.operations.event\_operations module
|
||||
---------------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.database.operations.event_operations
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.database.operations.logs\_operations module
|
||||
--------------------------------------------------------
|
||||
|
||||
|
@ -44,6 +44,14 @@ ahriman.core.formatters.configuration\_printer module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.formatters.event\_stats\_printer module
|
||||
----------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.formatters.event_stats_printer
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.formatters.package\_printer module
|
||||
-----------------------------------------------
|
||||
|
||||
@ -52,6 +60,14 @@ ahriman.core.formatters.package\_printer module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.formatters.package\_stats\_printer module
|
||||
------------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.formatters.package_stats_printer
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.formatters.patch\_printer module
|
||||
---------------------------------------------
|
||||
|
||||
|
@ -60,6 +60,14 @@ ahriman.core.report.report\_trigger module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.report.rss module
|
||||
------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.report.rss
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.report.telegram module
|
||||
-----------------------------------
|
||||
|
||||
|
@ -12,6 +12,14 @@ ahriman.core.repository.cleaner module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.repository.event\_logger module
|
||||
--------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.core.repository.event_logger
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.core.repository.executor module
|
||||
---------------------------------------
|
||||
|
||||
|
@ -68,6 +68,14 @@ ahriman.models.dependencies module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.models.event module
|
||||
---------------------------
|
||||
|
||||
.. automodule:: ahriman.models.event
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.models.filesystem\_package module
|
||||
-----------------------------------------
|
||||
|
||||
@ -100,6 +108,14 @@ ahriman.models.log\_record\_id module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.models.metrics\_timer module
|
||||
------------------------------------
|
||||
|
||||
.. automodule:: ahriman.models.metrics_timer
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.models.migration module
|
||||
-------------------------------
|
||||
|
||||
@ -124,14 +140,6 @@ ahriman.models.package module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.models.package\_archive module
|
||||
--------------------------------------
|
||||
|
||||
.. automodule:: ahriman.models.package_archive
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.models.package\_description module
|
||||
------------------------------------------
|
||||
|
||||
|
@ -60,6 +60,22 @@ ahriman.web.schemas.error\_schema module
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.web.schemas.event\_schema module
|
||||
----------------------------------------
|
||||
|
||||
.. automodule:: ahriman.web.schemas.event_schema
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.web.schemas.event\_search\_schema module
|
||||
------------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.web.schemas.event_search_schema
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
ahriman.web.schemas.file\_schema module
|
||||
---------------------------------------
|
||||
|
||||
|
21
docs/ahriman.web.views.v1.auditlog.rst
Normal file
21
docs/ahriman.web.views.v1.auditlog.rst
Normal file
@ -0,0 +1,21 @@
|
||||
ahriman.web.views.v1.auditlog package
|
||||
=====================================
|
||||
|
||||
Submodules
|
||||
----------
|
||||
|
||||
ahriman.web.views.v1.auditlog.events module
|
||||
-------------------------------------------
|
||||
|
||||
.. automodule:: ahriman.web.views.v1.auditlog.events
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
||||
|
||||
Module contents
|
||||
---------------
|
||||
|
||||
.. automodule:: ahriman.web.views.v1.auditlog
|
||||
:members:
|
||||
:no-undoc-members:
|
||||
:show-inheritance:
|
@ -7,6 +7,7 @@ Subpackages
|
||||
.. toctree::
|
||||
:maxdepth: 4
|
||||
|
||||
ahriman.web.views.v1.auditlog
|
||||
ahriman.web.views.v1.distributed
|
||||
ahriman.web.views.v1.packages
|
||||
ahriman.web.views.v1.service
|
||||
|
@ -17,14 +17,33 @@ There are two variable types which have been added to default ones, they are pat
|
||||
|
||||
Path values, except for casting to ``pathlib.Path`` type, will be also expanded to absolute paths relative to the configuration path. E.g. if path is set to ``ahriman.ini.d/logging.ini`` and root configuration path is ``/etc/ahriman.ini``, the value will be expanded to ``/etc/ahriman.ini.d/logging.ini``. In order to disable path expand, use the full path, e.g. ``/etc/ahriman.ini.d/logging.ini``.
|
||||
|
||||
Configuration allows string interpolation from environment variables, e.g.:
|
||||
Configuration allows string interpolation from the same configuration file, e.g.:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[section]
|
||||
key = ${anoher_key}
|
||||
another_key = value
|
||||
|
||||
will read value for the ``section.key`` option from ``section.another_key``. In case if the cross-section reference is required, the ``${section:another_key}`` notation must be used. It also allows string interpolation from environment variables, e.g.:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[section]
|
||||
key = $SECRET
|
||||
|
||||
will try to read value from ``SECRET`` environment variable. In case if the required environment variable wasn't found, it will keep original value (i.e. ``$SECRET`` in the example). Dollar sign can be set as ``$$``.
|
||||
will try to read value from ``SECRET`` environment variable. In case if the required environment variable wasn't found, it will keep original value (i.e. ``$SECRET`` in the example). Dollar sign can be set as ``$$``. All those interpolations will be applied in succession and - expected to be - recursively, e.g. the following configuration:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[section1]
|
||||
key = ${section2:key}
|
||||
|
||||
[section2]
|
||||
key = ${home}
|
||||
home = $HOME
|
||||
|
||||
will eventually lead ``section1.key`` option to be set to the value of ``HOME`` environment variable (if available).
|
||||
|
||||
There is also additional subcommand which will allow to validate configuration and print found errors. In order to do so, run ``service-config-validate`` subcommand, e.g.:
|
||||
|
||||
@ -81,14 +100,13 @@ Authorized users are stored inside internal database, if any of external provide
|
||||
|
||||
Build related configuration. Group name can refer to architecture, e.g. ``build:x86_64`` can be used for x86_64 architecture specific settings.
|
||||
|
||||
* ``allowed_scan_paths`` - paths to be used for implicit dependencies scan, scape separated list of paths, optional.
|
||||
* ``archbuild_flags`` - additional flags passed to ``archbuild`` command, space separated list of strings, optional.
|
||||
* ``blacklisted_scan_paths`` - paths to be excluded for implicit dependencies scan, scape separated list of paths, optional. Normally all elements of this option must be child paths of any of ``allowed_scan_paths`` element.
|
||||
* ``build_command`` - default build command, string, required.
|
||||
* ``ignore_packages`` - list packages to ignore during a regular update (manual update will still work), space separated list of strings, optional.
|
||||
* ``include_debug_packages`` - distribute debug packages, boolean, optional, default ``yes``.
|
||||
* ``makepkg_flags`` - additional flags passed to ``makepkg`` command, space separated list of strings, optional.
|
||||
* ``makechrootpkg_flags`` - additional flags passed to ``makechrootpkg`` command, space separated list of strings, optional.
|
||||
* ``scan_paths`` - paths to be used for implicit dependencies scan, space separated list of strings, optional. If any of those paths is matched against the path, it will be added to the allowed list.
|
||||
* ``triggers`` - list of ``ahriman.core.triggers.Trigger`` class implementation (e.g. ``ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger``) which will be loaded and run at the end of processing, space separated list of strings, optional. You can also specify triggers by their paths, e.g. ``/usr/lib/python3.10/site-packages/ahriman/core/report/report.py.ReportTrigger``. Triggers are run in the order of definition.
|
||||
* ``triggers_known`` - optional list of ``ahriman.core.triggers.Trigger`` class implementations which are not run automatically and used only for trigger discovery and configuration validation.
|
||||
* ``vcs_allowed_age`` - maximal age in seconds of the VCS packages before their version will be updated with its remote source, integer, optional, default is 7 days.
|
||||
@ -252,6 +270,7 @@ Section name must be either ``email`` (plus optional architecture name, e.g. ``e
|
||||
* ``password`` - SMTP password to authenticate, string, optional.
|
||||
* ``port`` - SMTP port for sending emails, integer, required.
|
||||
* ``receivers`` - SMTP receiver addresses, space separated list of strings, required.
|
||||
* ``rss_url`` - link to RSS feed, string, optional.
|
||||
* ``sender`` - SMTP sender address, string, required.
|
||||
* ``ssl`` - SSL mode for SMTP connection, one of ``ssl``, ``starttls``, ``disabled``, optional, default ``disabled``.
|
||||
* ``template`` - Jinja2 template name, string, required.
|
||||
@ -267,7 +286,8 @@ Section name must be either ``html`` (plus optional architecture name, e.g. ``ht
|
||||
* ``type`` - type of the report, string, optional, must be set to ``html`` if exists.
|
||||
* ``homepage`` - link to homepage, string, optional.
|
||||
* ``link_path`` - prefix for HTML links, string, required.
|
||||
* ``path`` - path to html report file, string, required.
|
||||
* ``path`` - path to HTML report file, string, required.
|
||||
* ``rss_url`` - link to RSS feed, string, optional.
|
||||
* ``template`` - Jinja2 template name, string, required.
|
||||
* ``templates`` - path to templates directories, space separated list of paths, required.
|
||||
|
||||
@ -282,6 +302,20 @@ Section name must be either ``remote-call`` (plus optional architecture name, e.
|
||||
* ``manual`` - update manually built packages, boolean, optional, default ``no``.
|
||||
* ``wait_timeout`` - maximum amount of time in seconds to be waited before remote process will be terminated, integer, optional, default ``-1``.
|
||||
|
||||
``rss`` type
|
||||
^^^^^^^^^^^^
|
||||
|
||||
Section name must be either ``rss`` (plus optional architecture name, e.g. ``rss:x86_64``) or random name with ``type`` set.
|
||||
|
||||
* ``type`` - type of the report, string, optional, must be set to ``rss`` if exists.
|
||||
* ``homepage`` - link to homepage, string, optional.
|
||||
* ``link_path`` - prefix for HTML links, string, required.
|
||||
* ``max_entries`` - maximal amount of entries to be included to the report, negative means no limit, integer, optional, default ``-1``.
|
||||
* ``path`` - path to generated RSS file, string, required.
|
||||
* ``rss_url`` - link to RSS feed, string, optional.
|
||||
* ``template`` - Jinja2 template name, string, required.
|
||||
* ``templates`` - path to templates directories, space separated list of paths, required.
|
||||
|
||||
``telegram`` type
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
||||
@ -292,6 +326,7 @@ Section name must be either ``telegram`` (plus optional architecture name, e.g.
|
||||
* ``chat_id`` - telegram chat id, either string with ``@`` or integer value, required.
|
||||
* ``homepage`` - link to homepage, string, optional.
|
||||
* ``link_path`` - prefix for HTML links, string, required.
|
||||
* ``rss_url`` - link to RSS feed, string, optional.
|
||||
* ``template`` - Jinja2 template name, string, required.
|
||||
* ``template_type`` - ``parse_mode`` to be passed to telegram API, one of ``MarkdownV2``, ``HTML``, ``Markdown``, string, optional, default ``HTML``.
|
||||
* ``templates`` - path to templates directories, space separated list of paths, required.
|
||||
|
@ -292,7 +292,7 @@ Worker nodes (applicable for all workers) config (``worker.ini``) as:
|
||||
|
||||
Command to run worker nodes (considering there will be two workers, one is on ``8081`` port and other is on ``8082``):
|
||||
|
||||
.. code-block:: ini
|
||||
.. code-block:: shell
|
||||
|
||||
docker run --privileged -p 8081:8081 -e AHRIMAN_PORT=8081 -v worker.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web
|
||||
docker run --privileged -p 8082:8082 -e AHRIMAN_PORT=8082 -v worker.ini:/etc/ahriman.ini.d/overrides.ini arcan1s/ahriman:latest web
|
||||
|
@ -379,7 +379,7 @@ After the success build the application extracts all linked libraries and used d
|
||||
|
||||
In order to disable this check completely, the ``--no-check-files`` flag can be used.
|
||||
|
||||
In addition, there is possibility to control paths which will be used for checking, by using options ``build.allowed_scan_paths`` and ``build.blacklisted_scan_paths``. Leaving ``build.allowed_scan_paths`` blank will effectively disable any check too.
|
||||
In addition, there is possibility to control paths which will be used for checking, by using option ``build.scan_paths``, which supports regular expressions. Leaving ``build.scan_paths`` blank will effectively disable any check too.
|
||||
|
||||
How to install built packages
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
@ -28,8 +28,8 @@ How to report by email
|
||||
sender = me@example.com
|
||||
user = me@example.com
|
||||
|
||||
How to generate index page for S3
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
How to generate index page
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
#.
|
||||
Install dependencies:
|
||||
@ -47,10 +47,30 @@ How to generate index page for S3
|
||||
target = html
|
||||
|
||||
[html]
|
||||
path = /var/lib/ahriman/repository/aur-clone/x86_64/index.html
|
||||
path = ${repository:root}/repository/aur-clone/x86_64/index.html
|
||||
link_path = http://example.com/aur-clone/x86_64
|
||||
|
||||
After these steps ``index.html`` file will be automatically synced to S3.
|
||||
Having this configuration, the generated ``index.html`` will be also automatically synced to remote services (e.g. S3).
|
||||
|
||||
How to generate RSS feed for index page
|
||||
"""""""""""""""""""""""""""""""""""""""
|
||||
|
||||
In addition to previous steps, the following configuration is required:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[report]
|
||||
target = html rss
|
||||
|
||||
[html]
|
||||
rss_url = ${html:link_path}/rss.xml
|
||||
|
||||
[rss]
|
||||
link_path = ${html:link_path}
|
||||
path = ${repository:root}/repository/ahriman-demo/x86_64/rss.xml
|
||||
rss_url = ${html:link_path}/rss.xml
|
||||
|
||||
With the appended configuration, the service fill also generate ``rss.xml``, link it to generated ``index.html`` and put it together.
|
||||
|
||||
How to post build report to telegram
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
@ -7,7 +7,7 @@ pkgdesc="ArcH linux ReposItory MANager"
|
||||
arch=('any')
|
||||
url="https://github.com/arcan1s/ahriman"
|
||||
license=('GPL3')
|
||||
depends=('devtools>=1:1.0.0' 'git' 'pyalpm' 'python-cerberus' 'python-inflection' 'python-passlib' 'python-pyelftools' 'python-requests' 'python-srcinfo')
|
||||
depends=('devtools>=1:1.0.0' 'git' 'pyalpm' 'python-inflection' 'python-passlib' 'python-pyelftools' 'python-requests' 'python-srcinfo')
|
||||
makedepends=('python-build' 'python-flit' 'python-installer' 'python-wheel')
|
||||
optdepends=('breezy: -bzr packages support'
|
||||
'darcs: -darcs packages support'
|
||||
@ -20,7 +20,9 @@ optdepends=('breezy: -bzr packages support'
|
||||
'python-aiohttp-security: web server with authorization'
|
||||
'python-aiohttp-session: web server with authorization'
|
||||
'python-boto3: sync to s3'
|
||||
'python-cerberus: configuration validator'
|
||||
'python-cryptography: web server with authorization'
|
||||
'python-matplotlib: usage statistics chart'
|
||||
'python-requests-unixsocket2: client report to web server by unix socket'
|
||||
'python-jinja: html report generation'
|
||||
'python-systemd: journal support'
|
||||
|
@ -6,7 +6,7 @@ logging = ahriman.ini.d/logging.ini
|
||||
; Perform database migrations on the application start. Do not touch this option unless you know what are you doing.
|
||||
;apply_migrations = yes
|
||||
; Path to the application SQLite database.
|
||||
database = /var/lib/ahriman/ahriman.db
|
||||
database = ${repository:root}/ahriman.db
|
||||
|
||||
[alpm]
|
||||
; Path to pacman system database cache.
|
||||
@ -17,7 +17,7 @@ mirror = https://geo.mirror.pkgbuild.com/$repo/os/$arch
|
||||
repositories = core extra multilib
|
||||
; Pacman's root directory. In the most cases it must point to the system root.
|
||||
root = /
|
||||
; Sync files databases too, which is required by deep dependencies check
|
||||
; Sync files databases too, which is required by deep dependencies check.
|
||||
sync_files_database = yes
|
||||
; Use local packages cache. If this option is enabled, the service will be able to synchronize databases (available
|
||||
; as additional option for some subcommands). If set to no, databases must be synchronized manually.
|
||||
@ -50,22 +50,20 @@ allow_read_only = yes
|
||||
;salt =
|
||||
|
||||
[build]
|
||||
; List of paths to be used for implicit dependency scan
|
||||
allowed_scan_paths = /usr/lib
|
||||
; List of additional flags passed to archbuild command.
|
||||
;archbuild_flags =
|
||||
; List of paths to be excluded for implicit dependency scan. Usually they should be subpaths of allowed_scan_paths
|
||||
blacklisted_scan_paths = /usr/lib/cmake
|
||||
; Path to build command
|
||||
; Path to build command.
|
||||
;build_command =
|
||||
; List of packages to be ignored during automatic updates.
|
||||
;ignore_packages =
|
||||
; Include debug packages
|
||||
; Include debug packages.
|
||||
;include_debug_packages = yes
|
||||
; List of additional flags passed to makechrootpkg command.
|
||||
;makechrootpkg_flags =
|
||||
; List of additional flags passed to makepkg command.
|
||||
makepkg_flags = --nocolor --ignorearch
|
||||
; List of paths to be used for implicit dependency scan. Regular expressions are supported.
|
||||
scan_paths = ^usr/lib(?!/cmake).*$
|
||||
; List of enabled triggers in the order of calls.
|
||||
triggers = ahriman.core.gitremote.RemotePullTrigger ahriman.core.report.ReportTrigger ahriman.core.upload.UploadTrigger ahriman.core.gitremote.RemotePushTrigger
|
||||
; List of well-known triggers. Used only for configuration purposes.
|
||||
@ -121,9 +119,9 @@ host = 127.0.0.1
|
||||
; Disable status (e.g. package status, logs, etc) endpoints. Useful for build only modes.
|
||||
;service_only = no
|
||||
; Path to directory with static files.
|
||||
static_path = /usr/share/ahriman/templates/static
|
||||
static_path = ${templates}/static
|
||||
; List of directories with templates.
|
||||
templates = /usr/share/ahriman/templates
|
||||
templates = ${prefix}/share/ahriman/templates
|
||||
; Path to unix socket. If none set, unix socket will be disabled.
|
||||
;unix_socket =
|
||||
; Allow unix socket to be world readable.
|
||||
@ -214,14 +212,14 @@ target = console
|
||||
|
||||
; Console reporting trigger configuration sample.
|
||||
[console]
|
||||
; Trigger type name
|
||||
; Trigger type name.
|
||||
;type = console
|
||||
; Use utf8 symbols in output.
|
||||
use_utf = yes
|
||||
|
||||
; Email reporting trigger configuration sample.
|
||||
[email]
|
||||
; Trigger type name
|
||||
; Trigger type name.
|
||||
;type = email
|
||||
; Optional URL to the repository homepage.
|
||||
;homepage=
|
||||
@ -237,6 +235,8 @@ use_utf = yes
|
||||
;port =
|
||||
; List of emails to receive the reports.
|
||||
;receivers =
|
||||
; Optional link to the RSS feed.
|
||||
;rss_url =
|
||||
; Sender email.
|
||||
;sender =
|
||||
; SMTP server SSL mode, one of ssl, starttls, disabled.
|
||||
@ -246,13 +246,13 @@ template = email-index.jinja2
|
||||
; Template name to be used for full packages list generation (same as HTML report).
|
||||
;template_full =
|
||||
; List of directories with templates.
|
||||
templates = /usr/share/ahriman/templates
|
||||
templates = ${prefix}/share/ahriman/templates
|
||||
; SMTP user.
|
||||
;user =
|
||||
|
||||
; HTML reporting trigger configuration sample.
|
||||
[html]
|
||||
; Trigger type name
|
||||
; Trigger type name.
|
||||
;type = html
|
||||
; Optional URL to the repository homepage.
|
||||
;homepage=
|
||||
@ -260,14 +260,16 @@ templates = /usr/share/ahriman/templates
|
||||
;link_path =
|
||||
; Output path for the HTML report.
|
||||
;path =
|
||||
; Optional link to the RSS feed.
|
||||
;rss_url =
|
||||
; Template name to be used.
|
||||
template = repo-index.jinja2
|
||||
; List of directories with templates.
|
||||
templates = /usr/share/ahriman/templates
|
||||
templates = ${prefix}/share/ahriman/templates
|
||||
|
||||
; Remote service callback trigger configuration sample.
|
||||
[remote-call]
|
||||
; Trigger type name
|
||||
; Trigger type name.
|
||||
;type = remote-call
|
||||
; Call for AUR packages update.
|
||||
;aur = no
|
||||
@ -278,9 +280,26 @@ templates = /usr/share/ahriman/templates
|
||||
; Wait until remote process will be terminated in seconds.
|
||||
;wait_timeout = -1
|
||||
|
||||
; RSS reporting trigger configuration sample.
|
||||
[rss]
|
||||
; Trigger type name.
|
||||
;type = rss
|
||||
; Optional URL to the repository homepage.
|
||||
;homepage=
|
||||
; Prefix for packages links. Link to a package will be formed as link_path / filename.
|
||||
;link_path =
|
||||
; Output path for the RSS report.
|
||||
;path =
|
||||
; Optional link to the RSS feed.
|
||||
;rss_url =
|
||||
; Template name to be used.
|
||||
template = rss.jinja2
|
||||
; List of directories with templates.
|
||||
templates = ${prefix}/share/ahriman/templates
|
||||
|
||||
; Telegram reporting trigger configuration sample.
|
||||
[telegram]
|
||||
; Trigger type name
|
||||
; Trigger type name.
|
||||
;type = telegram
|
||||
; Telegram bot API key.
|
||||
;api_key =
|
||||
@ -290,12 +309,14 @@ templates = /usr/share/ahriman/templates
|
||||
;homepage=
|
||||
; Prefix for packages links. Link to a package will be formed as link_path / filename.
|
||||
;link_path =
|
||||
; Optional link to the RSS feed.
|
||||
;rss_url =
|
||||
; Template name to be used.
|
||||
template = telegram-index.jinja2
|
||||
; Telegram specific template mode, one of MarkdownV2, HTML or Markdown.
|
||||
;template_type = HTML
|
||||
; List of directories with templates.
|
||||
templates = /usr/share/ahriman/templates
|
||||
templates = ${prefix}/share/ahriman/templates
|
||||
; HTTP request timeout in seconds.
|
||||
;timeout = 30
|
||||
|
||||
@ -306,7 +327,7 @@ target =
|
||||
|
||||
; GitHub upload trigger configuration sample.
|
||||
[github]
|
||||
; Trigger type name
|
||||
; Trigger type name.
|
||||
;type = github
|
||||
; GitHub repository owner username.
|
||||
;owner =
|
||||
@ -323,14 +344,14 @@ target =
|
||||
|
||||
; Remote instance upload trigger configuration sample.
|
||||
[remote-service]
|
||||
; Trigger type name
|
||||
; Trigger type name.
|
||||
;type = remote-service
|
||||
; HTTP request timeout in seconds.
|
||||
;timeout = 30
|
||||
|
||||
; rsync upload trigger configuration sample.
|
||||
[rsync]
|
||||
; Trigger type name
|
||||
; Trigger type name.
|
||||
;type = rsync
|
||||
; rsync command to run.
|
||||
command = rsync --archive --compress --partial --delete
|
||||
@ -340,7 +361,7 @@ command = rsync --archive --compress --partial --delete
|
||||
|
||||
; S3 upload trigger configuration sample.
|
||||
[s3]
|
||||
; Trigger type name
|
||||
; Trigger type name.
|
||||
;type = s3
|
||||
; AWS services access key.
|
||||
;access_key =
|
||||
|
@ -1,5 +1,5 @@
|
||||
[loggers]
|
||||
keys = root,http,stderr,boto3,botocore,nose,s3transfer
|
||||
keys = root,http,stderr,boto3,botocore,nose,s3transfer,sql
|
||||
|
||||
[handlers]
|
||||
keys = console_handler,journald_handler,syslog_handler
|
||||
@ -64,3 +64,8 @@ propagate = 0
|
||||
level = INFO
|
||||
qualname = s3transfer
|
||||
propagate = 0
|
||||
|
||||
[logger_sql]
|
||||
level = INFO
|
||||
qualname = sql
|
||||
propagate = 0
|
||||
|
@ -1,8 +1,12 @@
|
||||
<script>
|
||||
const alertPlaceholder = $("#alert-placeholder");
|
||||
|
||||
function createAlert(title, message, clz, action) {
|
||||
function createAlert(title, message, clz, action, id) {
|
||||
if (!id) id = $.md5(title + message); // MD5 id from the content
|
||||
if (alertPlaceholder.find(`#${id}`).length > 0) return; // check if there are duplicates
|
||||
|
||||
const wrapper = document.createElement("div");
|
||||
wrapper.id = id;
|
||||
wrapper.classList.add("toast", clz);
|
||||
wrapper.role = "alert";
|
||||
wrapper.ariaLive = "assertive";
|
||||
|
@ -45,8 +45,9 @@
|
||||
|
||||
<nav>
|
||||
<div class="nav nav-tabs" role="tablist">
|
||||
<button id="package-info-logs-button" class="nav-link active" data-bs-toggle="tab" data-bs-target="#package-info-logs" type="button" role="tab" aria-controls="package-info-logs" aria-selected="true"><h3>Build logs</h3></button>
|
||||
<button id="package-info-changes-button" class="nav-link" data-bs-toggle="tab" data-bs-target="#package-info-changes" type="button" role="tab" aria-controls="package-info-changes" aria-selected="false"><h3>Changes</h3></button>
|
||||
<button id="package-info-logs-button" class="nav-link active" data-bs-toggle="tab" data-bs-target="#package-info-logs" type="button" role="tab" aria-controls="package-info-logs" aria-selected="true">Build logs</button>
|
||||
<button id="package-info-changes-button" class="nav-link" data-bs-toggle="tab" data-bs-target="#package-info-changes" type="button" role="tab" aria-controls="package-info-changes" aria-selected="false">Changes</button>
|
||||
<button id="package-info-events-button" class="nav-link" data-bs-toggle="tab" data-bs-target="#package-info-events" type="button" role="tab" aria-controls="package-info-events" aria-selected="false">Events</button>
|
||||
</div>
|
||||
</nav>
|
||||
<div class="tab-content" id="nav-tabContent">
|
||||
@ -56,6 +57,23 @@
|
||||
<div id="package-info-changes" class="tab-pane fade" role="tabpanel" aria-labelledby="package-info-changes-button" tabindex="0">
|
||||
<pre class="language-diff"><code id="package-info-changes-input" class="pre-scrollable language-diff"></code><button id="package-info-changes-copy-button" type="button" class="btn language-diff" onclick="copyChanges()"><i class="bi bi-clipboard"></i> copy</button></pre>
|
||||
</div>
|
||||
<div id="package-info-events" class="tab-pane fade" role="tabpanel" aria-labelledby="package-info-events-button" tabindex="0">
|
||||
<canvas id="package-info-events-update-chart"></canvas>
|
||||
<table id="package-info-events-table"
|
||||
data-classes="table table-hover"
|
||||
data-sortable="true"
|
||||
data-sort-name="timestamp"
|
||||
data-sort-order="desc"
|
||||
data-toggle="table">
|
||||
<thead class="table-primary">
|
||||
<tr>
|
||||
<th data-align="right" data-field="timestamp">date</th>
|
||||
<th data-field="event">event</th>
|
||||
<th data-field="message">description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
@ -79,6 +97,10 @@
|
||||
const packageInfoChangesInput = $("#package-info-changes-input");
|
||||
const packageInfoChangesCopyButton = $("#package-info-changes-copy-button");
|
||||
|
||||
const packageInfoEventsTable = $("#package-info-events-table");
|
||||
const packageInfoEventsUpdateChartCanvas = document.getElementById("package-info-events-update-chart");
|
||||
let packageInfoEventsUpdateChart = null;
|
||||
|
||||
const packageInfoAurUrl = $("#package-info-aur-url");
|
||||
const packageInfoDepends = $("#package-info-depends");
|
||||
const packageInfoGroups = $("#package-info-groups");
|
||||
@ -91,6 +113,13 @@
|
||||
const packageInfoVariablesBlock = $("#package-info-variables-block");
|
||||
const packageInfoVariablesDiv = $("#package-info-variables-div");
|
||||
|
||||
function clearChart() {
|
||||
if (packageInfoEventsUpdateChart) {
|
||||
packageInfoEventsUpdateChart.data = {};
|
||||
packageInfoEventsUpdateChart.update();
|
||||
}
|
||||
}
|
||||
|
||||
async function copyChanges() {
|
||||
const changes = packageInfoChangesInput.text();
|
||||
await copyToClipboard(changes, packageInfoChangesCopyButton);
|
||||
@ -106,6 +135,11 @@
|
||||
packageInfoUpdateButton.attr("hidden", hidden);
|
||||
}
|
||||
|
||||
function highlight(element) {
|
||||
delete element.dataset.highlighted;
|
||||
hljs.highlightElement(element);
|
||||
}
|
||||
|
||||
function insertVariable(packageBase, variable) {
|
||||
const variableInput = document.createElement("div");
|
||||
variableInput.classList.add("input-group");
|
||||
@ -159,7 +193,51 @@
|
||||
success: response => {
|
||||
const changes = response.changes;
|
||||
packageInfoChangesInput.text(changes || "");
|
||||
packageInfoChangesInput.map((_, el) => hljs.highlightElement(el));
|
||||
packageInfoChangesInput.map((_, el) => highlight(el));
|
||||
},
|
||||
error: onFailure,
|
||||
});
|
||||
}
|
||||
|
||||
function loadEvents(packageBase, onFailure) {
|
||||
packageInfoEventsTable.bootstrapTable("showLoading");
|
||||
clearChart();
|
||||
|
||||
$.ajax({
|
||||
url: `/api/v1/events`,
|
||||
data: {
|
||||
architecture: repository.architecture,
|
||||
repository: repository.repository,
|
||||
object_id: packageBase,
|
||||
limit: 30,
|
||||
},
|
||||
type: "GET",
|
||||
dataType: "json",
|
||||
success: response => {
|
||||
const events = response.map(event => {
|
||||
return {
|
||||
timestamp: new Date(1000 * event.created).toISOStringShort(),
|
||||
event: event.event,
|
||||
message: event.message || "",
|
||||
};
|
||||
});
|
||||
|
||||
if (packageInfoEventsUpdateChart) {
|
||||
const chart = response.filter(event => event.event === "package-updated");
|
||||
packageInfoEventsUpdateChart.config.data = {
|
||||
labels: chart.map(event => new Date(1000 * event.created).toISOStringShort()),
|
||||
datasets: [{
|
||||
label: "update duration, s",
|
||||
data: chart.map(event => event.data.took),
|
||||
cubicInterpolationMode: "monotone",
|
||||
tension: 0.4,
|
||||
}],
|
||||
};
|
||||
packageInfoEventsUpdateChart.update();
|
||||
}
|
||||
|
||||
packageInfoEventsTable.bootstrapTable("load", events);
|
||||
packageInfoEventsTable.bootstrapTable("hideLoading");
|
||||
},
|
||||
error: onFailure,
|
||||
});
|
||||
@ -179,7 +257,7 @@
|
||||
return `[${new Date(1000 * log_record.created).toISOString()}] ${log_record.message}`;
|
||||
});
|
||||
packageInfoLogsInput.text(logs.join("\n"));
|
||||
packageInfoLogsInput.map((_, el) => hljs.highlightElement(el));
|
||||
packageInfoLogsInput.map((_, el) => highlight(el));
|
||||
},
|
||||
error: onFailure,
|
||||
});
|
||||
@ -285,12 +363,21 @@
|
||||
loadPackage(packageBase, onFailure);
|
||||
loadPatches(packageBase, onFailure);
|
||||
loadLogs(packageBase, onFailure);
|
||||
loadChanges(packageBase, onFailure)
|
||||
loadChanges(packageBase, onFailure);
|
||||
loadEvents(packageBase, onFailure);
|
||||
|
||||
if (isPackageBaseSet) packageInfoModal.modal("show");
|
||||
}
|
||||
|
||||
$(_ => {
|
||||
packageInfoEventsUpdateChart = new Chart(packageInfoEventsUpdateChartCanvas, {
|
||||
type: "line",
|
||||
data: {},
|
||||
options: {
|
||||
responsive: true,
|
||||
},
|
||||
});
|
||||
|
||||
packageInfoModal.on("hidden.bs.modal", _ => {
|
||||
packageInfoAurUrl.empty();
|
||||
packageInfoDepends.empty();
|
||||
@ -306,6 +393,8 @@
|
||||
|
||||
packageInfoLogsInput.empty();
|
||||
packageInfoChangesInput.empty();
|
||||
packageInfoEventsTable.bootstrapTable("load", []);
|
||||
clearChart();
|
||||
|
||||
packageInfoModal.trigger("reset");
|
||||
|
||||
|
@ -7,6 +7,10 @@
|
||||
|
||||
{% include "utils/style.jinja2" %}
|
||||
{% include "user-style.jinja2" ignore missing %}
|
||||
|
||||
{% if rss_url is not none %}
|
||||
<link rel="alternate" href="{{ rss_url }}" type="application/rss+xml">
|
||||
{% endif %}
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
27
package/share/ahriman/templates/rss.jinja2
Normal file
27
package/share/ahriman/templates/rss.jinja2
Normal file
@ -0,0 +1,27 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
|
||||
<channel>
|
||||
<title>{{ repository }}: Recent package updates</title>
|
||||
{% if homepage is not none %}
|
||||
<link>{{ homepage }}</link>
|
||||
{% endif %}
|
||||
<description>Recently updated packages in the {{ repository }}.</description>
|
||||
{% if rss_url is not none %}
|
||||
<atom:link href="{{ rss_url }}" rel="self"/>
|
||||
{% endif %}
|
||||
<language>en-us</language>
|
||||
<lastBuildDate>{{ last_update }}</lastBuildDate>
|
||||
|
||||
{% for package in packages %}
|
||||
<item>
|
||||
<title>{{ package.name }} {{ package.version }} {{ package.architecture }}</title>
|
||||
<link>{{ link_path }}/{{ package.filename }}</link>
|
||||
<description>{{ package.description }}</description>
|
||||
<pubDate>{{ package.build_date }}</pubDate>
|
||||
<guid isPermaLink="false">{{ package.tag }}</guid>
|
||||
<category>{{ repository }}</category>
|
||||
<category>{{ package.architecture }}</category>
|
||||
</item>
|
||||
{% endfor %}
|
||||
</channel>
|
||||
</rss>
|
@ -1,4 +1,5 @@
|
||||
<script src="https://cdn.jsdelivr.net/npm/jquery@3.7.1/dist/jquery.min.js" crossorigin="anonymous" type="application/javascript"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/jquery.md5@1.0.2/index.min.js" crossorigin="anonymous" type="application/javascript"></script>
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/npm/moment@2.29.4/moment.min.js" crossorigin="anonymous" type="application/javascript"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/daterangepicker@3.1.0/daterangepicker.min.js" crossorigin="anonymous" type="application/javascript"></script>
|
||||
@ -17,6 +18,8 @@
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@11.9.0/build/highlight.min.js" crossorigin="anonymous" type="application/javascript"></script>
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.4/dist/chart.umd.min.js" crossorigin="anonymous" type="application/javascript"></script>
|
||||
|
||||
<script>
|
||||
async function copyToClipboard(text, button) {
|
||||
if (navigator.clipboard === undefined) {
|
||||
|
@ -17,7 +17,6 @@ authors = [
|
||||
]
|
||||
|
||||
dependencies = [
|
||||
"cerberus",
|
||||
"inflection",
|
||||
"passlib",
|
||||
"pyelftools",
|
||||
@ -62,6 +61,9 @@ pacman = [
|
||||
s3 = [
|
||||
"boto3",
|
||||
]
|
||||
stats = [
|
||||
"matplotlib",
|
||||
]
|
||||
tests = [
|
||||
"pytest",
|
||||
"pytest-aiohttp",
|
||||
@ -71,6 +73,9 @@ tests = [
|
||||
"pytest-resource-path",
|
||||
"pytest-spec",
|
||||
]
|
||||
validator = [
|
||||
"cerberus",
|
||||
]
|
||||
web = [
|
||||
"Jinja2",
|
||||
"aioauth-client",
|
||||
|
@ -1,6 +1,7 @@
|
||||
# Index
|
||||
|
||||
1. Setup repository named `ahriman-demo` with architecture `x86_64`.
|
||||
2. Generate index page.
|
||||
2. Generate index page and RSS feed.
|
||||
3. Repository is available at `http://localhost:8080/repo`.
|
||||
4. Index page is available at `http://localhost:8080/repo/ahriman-demo/x86_64/index.html`
|
||||
5. Index page is available at `http://localhost:8080/repo/ahriman-demo/x86_64/rss.xml`
|
||||
|
@ -1,6 +1,12 @@
|
||||
[report]
|
||||
target = html
|
||||
target = html rss
|
||||
|
||||
[html]
|
||||
path = /var/lib/ahriman/ahriman/repository/ahriman-demo/x86_64/index.html
|
||||
path = ${repository:root}/repository/ahriman-demo/x86_64/index.html
|
||||
link_path = http://localhost:8080/repo/ahriman-demo/x86_64
|
||||
rss_url = ${html:link_path}/rss.xml
|
||||
|
||||
[rss]
|
||||
link_path = ${html:link_path}
|
||||
path = ${repository:root}/repository/ahriman-demo/x86_64/rss.xml
|
||||
rss_url = ${html:link_path}/rss.xml
|
||||
|
@ -28,6 +28,7 @@ from ahriman.application import handlers
|
||||
from ahriman.core.utils import enum_values, extract_user
|
||||
from ahriman.models.action import Action
|
||||
from ahriman.models.build_status import BuildStatusEnum
|
||||
from ahriman.models.event import EventType
|
||||
from ahriman.models.log_handler import LogHandler
|
||||
from ahriman.models.package_source import PackageSource
|
||||
from ahriman.models.sign_settings import SignSettings
|
||||
@ -119,6 +120,7 @@ def _parser() -> argparse.ArgumentParser:
|
||||
_set_repo_report_parser(subparsers)
|
||||
_set_repo_restore_parser(subparsers)
|
||||
_set_repo_sign_parser(subparsers)
|
||||
_set_repo_statistics_parser(subparsers)
|
||||
_set_repo_status_update_parser(subparsers)
|
||||
_set_repo_sync_parser(subparsers)
|
||||
_set_repo_tree_parser(subparsers)
|
||||
@ -735,6 +737,30 @@ def _set_repo_sign_parser(root: SubParserAction) -> argparse.ArgumentParser:
|
||||
return parser
|
||||
|
||||
|
||||
def _set_repo_statistics_parser(root: SubParserAction) -> argparse.ArgumentParser:
|
||||
"""
|
||||
add parser for repository statistics subcommand
|
||||
|
||||
Args:
|
||||
root(SubParserAction): subparsers for the commands
|
||||
|
||||
Returns:
|
||||
argparse.ArgumentParser: created argument parser
|
||||
"""
|
||||
parser = root.add_parser("repo-statistics", help="repository statistics",
|
||||
description="fetch repository statistics", formatter_class=_formatter)
|
||||
parser.add_argument("package", help="fetch only events for the specified package", nargs="?")
|
||||
parser.add_argument("--chart", help="create updates chart and save it to the specified path", type=Path)
|
||||
parser.add_argument("-e", "--event", help="event type filter",
|
||||
type=EventType, choices=enum_values(EventType), default=EventType.PackageUpdated)
|
||||
parser.add_argument("--from-date", help="only fetch events which are newer than the date")
|
||||
parser.add_argument("--limit", help="limit response by specified amount of events", type=int, default=-1)
|
||||
parser.add_argument("--offset", help="skip specified amount of events", type=int, default=0)
|
||||
parser.add_argument("--to-date", help="only fetch events which are older than the date")
|
||||
parser.set_defaults(handler=handlers.Statistics, lock=None, quiet=True, report=False, unsafe=True)
|
||||
return parser
|
||||
|
||||
|
||||
def _set_repo_status_update_parser(root: SubParserAction) -> argparse.ArgumentParser:
|
||||
"""
|
||||
add parser for repository status update subcommand
|
||||
|
@ -120,8 +120,7 @@ class Application(ApplicationPackages, ApplicationRepository):
|
||||
process_dependencies(bool): if no set, dependencies will not be processed
|
||||
|
||||
Returns:
|
||||
list[Package]: updated packages list. Packager for dependencies will be copied from
|
||||
original package
|
||||
list[Package]: updated packages list. Packager for dependencies will be copied from the original package
|
||||
|
||||
Examples:
|
||||
In the most cases, in order to avoid build failure, it is required to add missing packages, which can be
|
||||
|
@ -38,6 +38,7 @@ from ahriman.application.handlers.service_updates import ServiceUpdates
|
||||
from ahriman.application.handlers.setup import Setup
|
||||
from ahriman.application.handlers.shell import Shell
|
||||
from ahriman.application.handlers.sign import Sign
|
||||
from ahriman.application.handlers.statistics import Statistics
|
||||
from ahriman.application.handlers.status import Status
|
||||
from ahriman.application.handlers.status_update import StatusUpdate
|
||||
from ahriman.application.handlers.structure import Structure
|
||||
|
@ -59,7 +59,7 @@ class Handler:
|
||||
repository_id(RepositoryId): repository unique identifier
|
||||
|
||||
Returns:
|
||||
bool: True on success, False otherwise
|
||||
bool: ``True`` on success, ``False`` otherwise
|
||||
"""
|
||||
try:
|
||||
configuration = Configuration.from_path(args.configuration, repository_id)
|
||||
@ -129,7 +129,7 @@ class Handler:
|
||||
check condition and flag and raise ExitCode exception in case if it is enabled and condition match
|
||||
|
||||
Args:
|
||||
enabled(bool): if False no check will be performed
|
||||
enabled(bool): if ``False`` no check will be performed
|
||||
predicate(bool): indicates condition on which exception should be thrown
|
||||
|
||||
Raises:
|
||||
|
170
src/ahriman/application/handlers/statistics.py
Normal file
170
src/ahriman/application/handlers/statistics.py
Normal file
@ -0,0 +1,170 @@
|
||||
#
|
||||
# Copyright (c) 2021-2024 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import argparse
|
||||
import datetime
|
||||
import itertools
|
||||
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from ahriman.application.application import Application
|
||||
from ahriman.application.handlers.handler import Handler
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.formatters import EventStatsPrinter, PackageStatsPrinter
|
||||
from ahriman.core.utils import pretty_datetime
|
||||
from ahriman.models.event import Event
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
|
||||
|
||||
class Statistics(Handler):
|
||||
"""
|
||||
repository statistics handler
|
||||
"""
|
||||
|
||||
ALLOW_MULTI_ARCHITECTURE_RUN = False # conflicting io
|
||||
|
||||
@classmethod
|
||||
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
|
||||
report: bool) -> None:
|
||||
"""
|
||||
callback for command line
|
||||
|
||||
Args:
|
||||
args(argparse.Namespace): command line args
|
||||
repository_id(RepositoryId): repository unique identifier
|
||||
configuration(Configuration): configuration instance
|
||||
report(bool): force enable or disable reporting
|
||||
"""
|
||||
application = Application(repository_id, configuration, report=True)
|
||||
|
||||
from_date = to_date = None
|
||||
if (value := args.from_date) is not None:
|
||||
from_date = datetime.datetime.fromisoformat(value).timestamp()
|
||||
if (value := args.to_date) is not None:
|
||||
to_date = datetime.datetime.fromisoformat(value).timestamp()
|
||||
|
||||
events = application.reporter.event_get(args.event, args.package, from_date, to_date, args.limit, args.offset)
|
||||
|
||||
match args.package:
|
||||
case None:
|
||||
Statistics.stats_per_package(args.event, events, args.chart)
|
||||
case _:
|
||||
Statistics.stats_for_package(args.event, events, args.chart)
|
||||
|
||||
@staticmethod
|
||||
def event_stats(event_type: str, events: list[Event]) -> None:
|
||||
"""
|
||||
calculate event stats
|
||||
|
||||
Args:
|
||||
event_type(str): event type
|
||||
events(list[Event]): list of events
|
||||
"""
|
||||
times = [event.get("took") for event in events if event.get("took") is not None]
|
||||
EventStatsPrinter(f"{event_type} duration, s", times)(verbose=True)
|
||||
|
||||
@staticmethod
|
||||
def plot_packages(event_type: str, events: dict[str, int], path: Path) -> None:
|
||||
"""
|
||||
plot packages frequency
|
||||
|
||||
Args:
|
||||
event_type(str): event type
|
||||
events(dict[str, int]): list of events
|
||||
path(Path): path to save plot
|
||||
"""
|
||||
from matplotlib import pyplot as plt
|
||||
|
||||
x, y = list(events.keys()), list(events.values())
|
||||
plt.bar(x, y)
|
||||
|
||||
plt.xlabel("Package base")
|
||||
plt.ylabel("Frequency")
|
||||
plt.title(f"Frequency of the {event_type} event per package")
|
||||
|
||||
plt.savefig(path)
|
||||
|
||||
@staticmethod
|
||||
def plot_times(event_type: str, events: list[Event], path: Path) -> None:
|
||||
"""
|
||||
plot events timeline
|
||||
|
||||
Args:
|
||||
event_type(str): event type
|
||||
events(list[Event]): list of events
|
||||
path(Path): path to save plot
|
||||
"""
|
||||
from matplotlib import pyplot as plt
|
||||
|
||||
figure = plt.figure()
|
||||
|
||||
x, y = zip(*[(pretty_datetime(event.created), event.get("took")) for event in events])
|
||||
plt.plot(x, y)
|
||||
|
||||
plt.xlabel("Event timestamp")
|
||||
plt.ylabel("Duration, s")
|
||||
plt.title(f"Duration of the {event_type} event")
|
||||
figure.autofmt_xdate()
|
||||
|
||||
plt.savefig(path)
|
||||
|
||||
@staticmethod
|
||||
def stats_for_package(event_type: str, events: list[Event], chart_path: Path | None) -> None:
|
||||
"""
|
||||
calculate statistics for a package
|
||||
|
||||
Args:
|
||||
event_type(str): event type
|
||||
events(list[Event]): list of events
|
||||
chart_path(Path): path to save plot if any
|
||||
"""
|
||||
# event statistics
|
||||
Statistics.event_stats(event_type, events)
|
||||
|
||||
# chart if enabled
|
||||
if chart_path is not None:
|
||||
Statistics.plot_times(event_type, events, chart_path)
|
||||
|
||||
@staticmethod
|
||||
def stats_per_package(event_type: str, events: list[Event], chart_path: Path | None) -> None:
|
||||
"""
|
||||
calculate overall statistics
|
||||
|
||||
Args:
|
||||
event_type(str): event type
|
||||
events(list[Event]): list of events
|
||||
chart_path(Path): path to save plot if any
|
||||
"""
|
||||
key: Callable[[Event], str] = lambda event: event.object_id
|
||||
by_object_id = {
|
||||
object_id: len(list(related))
|
||||
for object_id, related in itertools.groupby(sorted(events, key=key), key=key)
|
||||
}
|
||||
|
||||
# distribution per package
|
||||
PackageStatsPrinter(by_object_id)(verbose=True)
|
||||
EventStatsPrinter(f"{event_type} frequency", list(by_object_id.values()))(verbose=True)
|
||||
|
||||
# event statistics
|
||||
Statistics.event_stats(event_type, events)
|
||||
|
||||
# chart if enabled
|
||||
if chart_path is not None:
|
||||
Statistics.plot_packages(event_type, by_object_id, chart_path)
|
@ -25,7 +25,6 @@ from typing import Any
|
||||
from ahriman.application.handlers.handler import Handler
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.configuration.schema import CONFIGURATION_SCHEMA, ConfigurationSchema
|
||||
from ahriman.core.configuration.validator import Validator
|
||||
from ahriman.core.exceptions import ExtensionError
|
||||
from ahriman.core.formatters import ValidationPrinter
|
||||
from ahriman.core.triggers import TriggerLoader
|
||||
@ -51,6 +50,8 @@ class Validate(Handler):
|
||||
configuration(Configuration): configuration instance
|
||||
report(bool): force enable or disable reporting
|
||||
"""
|
||||
from ahriman.core.configuration.validator import Validator
|
||||
|
||||
schema = Validate.schema(repository_id, configuration)
|
||||
validator = Validator(configuration=configuration, schema=schema)
|
||||
|
||||
|
@ -59,7 +59,7 @@ class Lock(LazyLogging):
|
||||
>>> configuration = Configuration()
|
||||
>>> try:
|
||||
>>> with Lock(args, RepositoryId("x86_64", "aur-clone"), configuration):
|
||||
>>> perform_actions()
|
||||
>>> do_something()
|
||||
>>> except Exception as exception:
|
||||
>>> handle_exceptions(exception)
|
||||
"""
|
||||
@ -97,7 +97,7 @@ class Lock(LazyLogging):
|
||||
fd(int): file descriptor:
|
||||
|
||||
Returns:
|
||||
bool: True in case if file is locked and False otherwise
|
||||
bool: ``True`` in case if file is locked and ``False`` otherwise
|
||||
"""
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
@ -119,7 +119,7 @@ class Lock(LazyLogging):
|
||||
watch until lock disappear
|
||||
|
||||
Returns:
|
||||
bool: True in case if file is locked and False otherwise
|
||||
bool: ``True`` in case if file is locked and ``False`` otherwise
|
||||
"""
|
||||
# there are reasons why we are not using inotify here. First of all, if we would use it, it would bring to
|
||||
# race conditions because multiple processes will be notified at the same time. Secondly, it is good library,
|
||||
@ -223,7 +223,7 @@ class Lock(LazyLogging):
|
||||
exc_tb(TracebackType): exception traceback if any
|
||||
|
||||
Returns:
|
||||
Literal[False]: always False (do not suppress any exception)
|
||||
Literal[False]: always ``False`` (do not suppress any exception)
|
||||
"""
|
||||
self.clear()
|
||||
status = BuildStatusEnum.Success if exc_val is None else BuildStatusEnum.Failed
|
||||
|
@ -43,7 +43,7 @@ class Pacman(LazyLogging):
|
||||
configuration(Configuration): configuration instance
|
||||
refresh_database(PacmanSynchronization): synchronize local cache to remote
|
||||
repository_id(RepositoryId): repository unique identifier
|
||||
repository_path(RepositoryPaths): repository paths instance
|
||||
repository_paths(RepositoryPaths): repository paths instance
|
||||
"""
|
||||
|
||||
def __init__(self, repository_id: RepositoryId, configuration: Configuration, *,
|
||||
@ -188,8 +188,8 @@ class Pacman(LazyLogging):
|
||||
Returns:
|
||||
dict[str, set[str]]: map of package name to its list of files
|
||||
"""
|
||||
def extract(tar: tarfile.TarFile, package_names: dict[str, str]) -> Generator[tuple[str, set[str]], None, None]:
|
||||
for package_name, version in package_names.items():
|
||||
def extract(tar: tarfile.TarFile, versions: dict[str, str]) -> Generator[tuple[str, set[str]], None, None]:
|
||||
for package_name, version in versions.items():
|
||||
path = Path(f"{package_name}-{version}") / "files"
|
||||
try:
|
||||
content = tar.extractfile(str(path))
|
||||
|
@ -59,7 +59,8 @@ class PacmanDatabase(SyncHttpClient):
|
||||
|
||||
self.sync_files_database = configuration.getboolean("alpm", "sync_files_database")
|
||||
|
||||
def copy(self, remote_path: Path, local_path: Path) -> None:
|
||||
@staticmethod
|
||||
def copy(remote_path: Path, local_path: Path) -> None:
|
||||
"""
|
||||
copy local database file
|
||||
|
||||
@ -101,7 +102,7 @@ class PacmanDatabase(SyncHttpClient):
|
||||
local_path(Path): path to locally stored file
|
||||
|
||||
Returns:
|
||||
bool: True in case if remote file is newer than local file
|
||||
bool: ``True`` in case if remote file is newer than local file
|
||||
|
||||
Raises:
|
||||
PacmanError: in case if no last-modified header was found
|
||||
|
@ -96,7 +96,7 @@ class Auth(LazyLogging):
|
||||
password(str | None): entered password
|
||||
|
||||
Returns:
|
||||
bool: True in case if password matches, False otherwise
|
||||
bool: ``True`` in case if password matches, ``False`` otherwise
|
||||
"""
|
||||
del username, password
|
||||
return True
|
||||
@ -109,7 +109,7 @@ class Auth(LazyLogging):
|
||||
username(str): username
|
||||
|
||||
Returns:
|
||||
bool: True in case if user is known and can be authorized and False otherwise
|
||||
bool: ``True`` in case if user is known and can be authorized and ``False`` otherwise
|
||||
"""
|
||||
del username
|
||||
return True
|
||||
@ -124,7 +124,7 @@ class Auth(LazyLogging):
|
||||
context(str | None): URI request path
|
||||
|
||||
Returns:
|
||||
bool: True in case if user is allowed to do this request and False otherwise
|
||||
bool: ``True`` in case if user is allowed to do this request and ``False`` otherwise
|
||||
"""
|
||||
del username, required, context
|
||||
return True
|
||||
|
@ -38,7 +38,7 @@ async def authorized_userid(*args: Any, **kwargs: Any) -> Any:
|
||||
**kwargs(Any): named argument list as provided by authorized_userid function
|
||||
|
||||
Returns:
|
||||
Any: None in case if no aiohttp_security module found and function call otherwise
|
||||
Any: ``None`` in case if no aiohttp_security module found and function call otherwise
|
||||
"""
|
||||
if _has_aiohttp_security:
|
||||
return await aiohttp_security.authorized_userid(*args, **kwargs) # pylint: disable=no-value-for-parameter
|
||||
@ -54,7 +54,7 @@ async def check_authorized(*args: Any, **kwargs: Any) -> Any:
|
||||
**kwargs(Any): named argument list as provided by authorized_userid function
|
||||
|
||||
Returns:
|
||||
Any: None in case if no aiohttp_security module found and function call otherwise
|
||||
Any: ``None`` in case if no aiohttp_security module found and function call otherwise
|
||||
"""
|
||||
if _has_aiohttp_security:
|
||||
return await aiohttp_security.check_authorized(*args, **kwargs) # pylint: disable=no-value-for-parameter
|
||||
@ -70,7 +70,7 @@ async def forget(*args: Any, **kwargs: Any) -> Any:
|
||||
**kwargs(Any): named argument list as provided by authorized_userid function
|
||||
|
||||
Returns:
|
||||
Any: None in case if no aiohttp_security module found and function call otherwise
|
||||
Any: ``None`` in case if no aiohttp_security module found and function call otherwise
|
||||
"""
|
||||
if _has_aiohttp_security:
|
||||
return await aiohttp_security.forget(*args, **kwargs) # pylint: disable=no-value-for-parameter
|
||||
@ -86,7 +86,7 @@ async def remember(*args: Any, **kwargs: Any) -> Any:
|
||||
**kwargs(Any): named argument list as provided by authorized_userid function
|
||||
|
||||
Returns:
|
||||
Any: None in case if no aiohttp_security module found and function call otherwise
|
||||
Any: ``None`` in case if no aiohttp_security module found and function call otherwise
|
||||
"""
|
||||
if _has_aiohttp_security:
|
||||
return await aiohttp_security.remember(*args, **kwargs) # pylint: disable=no-value-for-parameter
|
||||
|
@ -57,7 +57,7 @@ class Mapping(Auth):
|
||||
password(str | None): entered password
|
||||
|
||||
Returns:
|
||||
bool: True in case if password matches, False otherwise
|
||||
bool: ``True`` in case if password matches, ``False`` otherwise
|
||||
"""
|
||||
if password is None:
|
||||
return False # invalid data supplied
|
||||
@ -72,7 +72,7 @@ class Mapping(Auth):
|
||||
username(str): username
|
||||
|
||||
Returns:
|
||||
User | None: user descriptor if username is known and None otherwise
|
||||
User | None: user descriptor if username is known and ``None`` otherwise
|
||||
"""
|
||||
return self.database.user_get(username)
|
||||
|
||||
@ -84,7 +84,7 @@ class Mapping(Auth):
|
||||
username(str): username
|
||||
|
||||
Returns:
|
||||
bool: True in case if user is known and can be authorized and False otherwise
|
||||
bool: ``True`` in case if user is known and can be authorized and ``False`` otherwise
|
||||
"""
|
||||
return username is not None and self.get_user(username) is not None
|
||||
|
||||
@ -98,7 +98,7 @@ class Mapping(Auth):
|
||||
context(str | None): URI request path
|
||||
|
||||
Returns:
|
||||
bool: True in case if user is allowed to do this request and False otherwise
|
||||
bool: ``True`` in case if user is allowed to do this request and ``False`` otherwise
|
||||
"""
|
||||
user = self.get_user(username)
|
||||
return user is not None and user.verify_access(required)
|
||||
|
@ -79,7 +79,7 @@ class PAM(Mapping):
|
||||
password(str | None): entered password
|
||||
|
||||
Returns:
|
||||
bool: True in case if password matches, False otherwise
|
||||
bool: ``True`` in case if password matches, ``False`` otherwise
|
||||
"""
|
||||
if password is None:
|
||||
return False # invalid data supplied
|
||||
@ -101,7 +101,7 @@ class PAM(Mapping):
|
||||
username(str): username
|
||||
|
||||
Returns:
|
||||
bool: True in case if user is known and can be authorized and False otherwise
|
||||
bool: ``True`` in case if user is known and can be authorized and ``False`` otherwise
|
||||
"""
|
||||
try:
|
||||
_ = getpwnam(username)
|
||||
@ -119,7 +119,7 @@ class PAM(Mapping):
|
||||
context(str | None): URI request path
|
||||
|
||||
Returns:
|
||||
bool: True in case if user is allowed to do this request and False otherwise
|
||||
bool: ``True`` in case if user is allowed to do this request and ``False`` otherwise
|
||||
"""
|
||||
# this method is basically inverted, first we check overrides in database and then fallback to the PAM logic
|
||||
if (user := self.get_user(username)) is not None:
|
||||
|
@ -17,7 +17,6 @@
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from dataclasses import dataclass
|
||||
from elftools.elf.dynamic import DynamicSection
|
||||
from elftools.elf.elffile import ELFFile
|
||||
from pathlib import Path
|
||||
@ -33,7 +32,6 @@ from ahriman.models.package import Package
|
||||
from ahriman.models.scan_paths import ScanPaths
|
||||
|
||||
|
||||
@dataclass
|
||||
class PackageArchive:
|
||||
"""
|
||||
helper for package archives
|
||||
@ -45,10 +43,20 @@ class PackageArchive:
|
||||
scan_paths(ScanPaths): scan paths holder
|
||||
"""
|
||||
|
||||
root: Path
|
||||
package: Package
|
||||
pacman: Pacman
|
||||
scan_paths: ScanPaths
|
||||
def __init__(self, root: Path, package: Package, pacman: Pacman, scan_paths: ScanPaths) -> None:
|
||||
"""
|
||||
default constructor
|
||||
|
||||
Args:
|
||||
root(Path): path to root filesystem
|
||||
package(Package): package descriptor
|
||||
pacman(Pacman): alpm wrapper instance
|
||||
scan_paths(ScanPaths): scan paths holder
|
||||
"""
|
||||
self.root = root
|
||||
self.package = package
|
||||
self.pacman = pacman
|
||||
self.scan_paths = scan_paths
|
||||
|
||||
@staticmethod
|
||||
def dynamic_needed(binary_path: Path) -> list[str]:
|
||||
@ -163,7 +171,7 @@ class PackageArchive:
|
||||
|
||||
result: dict[Path, list[FilesystemPackage]] = {}
|
||||
# sort items from children directories to root
|
||||
for path, packages in reversed(sorted(source.items())):
|
||||
for path, packages in sorted(source.items(), reverse=True):
|
||||
# skip if this path belongs to the one of the base packages
|
||||
if any(package.package_name in base_packages for package in packages):
|
||||
continue
|
||||
@ -228,7 +236,7 @@ class PackageArchive:
|
||||
extract list of the installed packages and their content
|
||||
|
||||
Returns:
|
||||
dict[str, FilesystemPackage]; map of package name to list of directories and files contained
|
||||
dict[str, FilesystemPackage]: map of package name to list of directories and files contained
|
||||
by this package
|
||||
"""
|
||||
result = {}
|
@ -138,7 +138,7 @@ class Sources(LazyLogging):
|
||||
sources_dir(Path): local path to git repository
|
||||
|
||||
Returns:
|
||||
bool: True in case if there is any remote and false otherwise
|
||||
bool: ``True`` in case if there is any remote and false otherwise
|
||||
"""
|
||||
instance = Sources()
|
||||
remotes = check_output(*instance.git(), "remote", cwd=sources_dir, logger=instance.logger)
|
||||
@ -261,7 +261,7 @@ class Sources(LazyLogging):
|
||||
commit_author(tuple[str, str] | None, optional): optional commit author if any (Default value = None)
|
||||
|
||||
Returns:
|
||||
bool: True in case if changes have been committed and False otherwise
|
||||
bool: ``True`` in case if changes have been committed and ``False`` otherwise
|
||||
"""
|
||||
if not self.has_changes(sources_dir):
|
||||
return False # nothing to commit
|
||||
@ -351,7 +351,7 @@ class Sources(LazyLogging):
|
||||
sources_dir(Path): local path to git repository
|
||||
|
||||
Returns:
|
||||
bool: True if there are uncommitted changes and False otherwise
|
||||
bool: ``True`` if there are uncommitted changes and ``False`` otherwise
|
||||
"""
|
||||
# there is --exit-code argument to diff, however, there might be other process errors
|
||||
changes = check_output(*self.git(), "diff", "--cached", "--name-only", cwd=sources_dir, logger=self.logger)
|
||||
|
@ -46,7 +46,8 @@ class Configuration(configparser.RawConfigParser):
|
||||
Examples:
|
||||
Configuration class provides additional method in order to handle application configuration. Since this class is
|
||||
derived from built-in :class:`configparser.RawConfigParser` class, the same flow is applicable here.
|
||||
Nevertheless, it is recommended to use :func:`from_path` class method which also calls initialization methods::
|
||||
Nevertheless, it is recommended to use :func:`from_path()` class method which also calls initialization
|
||||
methods::
|
||||
|
||||
>>> from pathlib import Path
|
||||
>>>
|
||||
@ -57,7 +58,7 @@ class Configuration(configparser.RawConfigParser):
|
||||
The configuration instance loaded in this way will contain only sections which are defined for the specified
|
||||
architecture according to the merge rules. Moreover, the architecture names will be removed from section names.
|
||||
|
||||
In order to get current settings, the :func:`check_loaded` method can be used. This method will raise an
|
||||
In order to get current settings, the :func:`check_loaded()` method can be used. This method will raise an
|
||||
:exc:`ahriman.core.exceptions.InitializeError` in case if configuration was not yet loaded::
|
||||
|
||||
>>> path, repository_id = configuration.check_loaded()
|
||||
@ -344,7 +345,8 @@ class Configuration(configparser.RawConfigParser):
|
||||
|
||||
def set_option(self, section: str, option: str, value: str) -> None:
|
||||
"""
|
||||
set option. Unlike default :func:`configparser.RawConfigParser.set` it also creates section if it does not exist
|
||||
set option. Unlike default :func:`configparser.RawConfigParser.set()` it also creates section if
|
||||
it does not exist
|
||||
|
||||
Args:
|
||||
section(str): section name
|
||||
|
@ -169,14 +169,6 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
|
||||
"build": {
|
||||
"type": "dict",
|
||||
"schema": {
|
||||
"allowed_scan_paths": {
|
||||
"type": "list",
|
||||
"coerce": "list",
|
||||
"schema": {
|
||||
"type": "path",
|
||||
"coerce": "absolute_path",
|
||||
},
|
||||
},
|
||||
"archbuild_flags": {
|
||||
"type": "list",
|
||||
"coerce": "list",
|
||||
@ -185,14 +177,6 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
|
||||
"empty": False,
|
||||
},
|
||||
},
|
||||
"blacklisted_scan_paths": {
|
||||
"type": "list",
|
||||
"coerce": "list",
|
||||
"schema": {
|
||||
"type": "path",
|
||||
"coerce": "absolute_path",
|
||||
},
|
||||
},
|
||||
"build_command": {
|
||||
"type": "string",
|
||||
"required": True,
|
||||
@ -226,6 +210,14 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
|
||||
"empty": False,
|
||||
},
|
||||
},
|
||||
"scan_paths": {
|
||||
"type": "list",
|
||||
"coerce": "list",
|
||||
"schema": {
|
||||
"type": "string",
|
||||
"empty": False,
|
||||
},
|
||||
},
|
||||
"triggers": {
|
||||
"type": "list",
|
||||
"coerce": "list",
|
||||
|
@ -19,21 +19,93 @@
|
||||
#
|
||||
import configparser
|
||||
import os
|
||||
import sys
|
||||
|
||||
from collections.abc import Mapping, MutableMapping
|
||||
from collections.abc import Generator, Mapping, MutableMapping
|
||||
from string import Template
|
||||
|
||||
|
||||
class ExtendedTemplate(Template):
|
||||
"""
|
||||
extension to the default :class:`Template` class, which also enabled braces regex to lookup in sections
|
||||
|
||||
Attributes:
|
||||
braceidpattern(str): regular expression to match a colon inside braces
|
||||
"""
|
||||
|
||||
braceidpattern = r"(?a:[_a-z0-9][_a-z0-9:]*)"
|
||||
|
||||
|
||||
class ShellInterpolator(configparser.Interpolation):
|
||||
"""
|
||||
custom string interpolator, because we cannot use defaults argument due to config validation
|
||||
"""
|
||||
|
||||
DATA_LINK_ESCAPE = "\x10"
|
||||
|
||||
@staticmethod
|
||||
def _extract_variables(parser: MutableMapping[str, Mapping[str, str]], value: str,
|
||||
defaults: Mapping[str, str]) -> Generator[tuple[str, str], None, None]:
|
||||
"""
|
||||
extract keys and values (if available) from the configuration. In case if a key is not available, it will be
|
||||
silently skipped from the result
|
||||
|
||||
Args:
|
||||
parser(MutableMapping[str, Mapping[str, str]]): option parser
|
||||
value(str): source (not-converted) value
|
||||
defaults(Mapping[str, str]): default values
|
||||
|
||||
Yields:
|
||||
tuple[str, str]: variable name used for substitution and its value
|
||||
"""
|
||||
def identifiers() -> Generator[tuple[str | None, str], None, None]:
|
||||
# extract all found identifiers and parse them
|
||||
for identifier in ExtendedTemplate(value).get_identifiers():
|
||||
match identifier.split(":"):
|
||||
case [lookup_option]: # single option from the same section
|
||||
yield None, lookup_option
|
||||
case [lookup_section, lookup_option]: # reference to another section
|
||||
yield lookup_section, lookup_option
|
||||
|
||||
for section, option in identifiers():
|
||||
# key to be substituted
|
||||
key = f"{section}:{option}" if section else option
|
||||
|
||||
if section is not None: # foreign section case
|
||||
if section not in parser:
|
||||
continue # section was not found, silently skip it
|
||||
values = parser[section]
|
||||
else: # same section
|
||||
values = defaults
|
||||
|
||||
if (raw := values.get(option)) is not None:
|
||||
yield key, raw
|
||||
|
||||
@staticmethod
|
||||
def environment() -> dict[str, str]:
|
||||
"""
|
||||
extract environment variables
|
||||
|
||||
Returns:
|
||||
dict[str, str]: environment variables and some custom variables
|
||||
"""
|
||||
return os.environ | {
|
||||
"prefix": sys.prefix,
|
||||
}
|
||||
|
||||
def before_get(self, parser: MutableMapping[str, Mapping[str, str]], section: str, option: str, value: str,
|
||||
defaults: Mapping[str, str]) -> str:
|
||||
"""
|
||||
interpolate option value
|
||||
|
||||
Notes:
|
||||
This method is using :class:`string.Template` class in order to render both cross-references and
|
||||
environment variables, because it seems that it is the most legit way to handle it. In addition,
|
||||
we are using shell-like variables in some cases (see :attr:`alpm.mirror` option), thus we would like
|
||||
to keep them alive.
|
||||
|
||||
First this method resolves substitution from the configuration and then renders environment variables
|
||||
|
||||
Args:
|
||||
parser(MutableMapping[str, Mapping[str, str]]): option parser
|
||||
section(str): section name
|
||||
@ -44,8 +116,15 @@ class ShellInterpolator(configparser.Interpolation):
|
||||
Returns:
|
||||
str: substituted value
|
||||
"""
|
||||
# At the moment it seems that it is the most legit way to handle environment variables
|
||||
# Template behaviour is literally the same as shell
|
||||
# In addition, we are using shell-like variables in some cases (see :attr:`alpm.mirror` option),
|
||||
# thus we would like to keep them alive
|
||||
return Template(value).safe_substitute(os.environ)
|
||||
# because any substitution effectively replace escaped $ ($$) in result, we have to escape it manually
|
||||
escaped = value.replace("$$", self.DATA_LINK_ESCAPE)
|
||||
|
||||
# resolve internal references
|
||||
variables = dict(self._extract_variables(parser, value, defaults))
|
||||
internal = ExtendedTemplate(escaped).safe_substitute(variables)
|
||||
|
||||
# resolve enriched environment variables by using default Template class
|
||||
environment = Template(internal).safe_substitute(self.environment())
|
||||
|
||||
# replace escaped values back
|
||||
return environment.replace(self.DATA_LINK_ESCAPE, "$")
|
||||
|
@ -35,8 +35,9 @@ class Validator(RootValidator):
|
||||
configuration(Configuration): configuration instance
|
||||
"""
|
||||
|
||||
types_mapping = RootValidator.types_mapping.copy()
|
||||
types_mapping["path"] = TypeDefinition("path", (Path,), ())
|
||||
types_mapping = RootValidator.types_mapping | {
|
||||
"path": TypeDefinition("path", (Path,), ()),
|
||||
}
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
"""
|
||||
@ -149,7 +150,7 @@ class Validator(RootValidator):
|
||||
check if paths exists
|
||||
|
||||
Args:
|
||||
constraint(bool): True in case if path must exist and False otherwise
|
||||
constraint(bool): ``True`` in case if path must exist and ``False`` otherwise
|
||||
field(str): field name to be checked
|
||||
value(Path): value to be checked
|
||||
|
||||
|
38
src/ahriman/core/database/migrations/m014_auditlog.py
Normal file
38
src/ahriman/core/database/migrations/m014_auditlog.py
Normal file
@ -0,0 +1,38 @@
|
||||
#
|
||||
# Copyright (c) 2021-2024 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
__all__ = ["steps"]
|
||||
|
||||
|
||||
steps = [
|
||||
"""
|
||||
create table auditlog (
|
||||
created integer not null,
|
||||
repository text not null,
|
||||
event text not null,
|
||||
object_id text not null,
|
||||
message text,
|
||||
data json
|
||||
)
|
||||
""",
|
||||
"""
|
||||
create index auditlog_created_repository_event_object_id
|
||||
on auditlog (created, repository, event, object_id)
|
||||
""",
|
||||
]
|
@ -21,6 +21,7 @@ from ahriman.core.database.operations.auth_operations import AuthOperations
|
||||
from ahriman.core.database.operations.build_operations import BuildOperations
|
||||
from ahriman.core.database.operations.changes_operations import ChangesOperations
|
||||
from ahriman.core.database.operations.dependencies_operations import DependenciesOperations
|
||||
from ahriman.core.database.operations.event_operations import EventOperations
|
||||
from ahriman.core.database.operations.logs_operations import LogsOperations
|
||||
from ahriman.core.database.operations.package_operations import PackageOperations
|
||||
from ahriman.core.database.operations.patch_operations import PatchOperations
|
||||
|
@ -39,7 +39,7 @@ class DependenciesOperations(Operations):
|
||||
repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
|
||||
|
||||
Returns:
|
||||
Dependencies: changes for the package base if available
|
||||
Dependencies: dependencies for the package base if available
|
||||
"""
|
||||
repository_id = repository_id or self._repository_id
|
||||
|
||||
|
108
src/ahriman/core/database/operations/event_operations.py
Normal file
108
src/ahriman/core/database/operations/event_operations.py
Normal file
@ -0,0 +1,108 @@
|
||||
#
|
||||
# Copyright (c) 2021-2024 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from sqlite3 import Connection
|
||||
|
||||
from ahriman.core.database.operations.operations import Operations
|
||||
from ahriman.models.event import Event, EventType
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
|
||||
|
||||
class EventOperations(Operations):
|
||||
"""
|
||||
operations for audit log table
|
||||
"""
|
||||
|
||||
def event_get(self, event: str | EventType | None = None, object_id: str | None = None,
|
||||
from_date: int | float | None = None, to_date: int | float | None = None,
|
||||
limit: int = -1, offset: int = 0, repository_id: RepositoryId | None = None) -> list[Event]:
|
||||
"""
|
||||
get list of events with filters applied
|
||||
|
||||
Args:
|
||||
event(str | EventType | None, optional): filter by event type (Default value = None)
|
||||
object_id(str | None, optional): filter by event object (Default value = None)
|
||||
from_date(int | float | None, optional): minimal creation date, inclusive (Default value = None)
|
||||
to_date(int | float | None, optional): maximal creation date, exclusive (Default value = None)
|
||||
limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1)
|
||||
offset(int, optional): records offset (Default value = 0)
|
||||
repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
|
||||
|
||||
Returns:
|
||||
list[Event]: list of audit log events
|
||||
"""
|
||||
repository_id = repository_id or self._repository_id
|
||||
|
||||
def run(connection: Connection) -> list[Event]:
|
||||
return [
|
||||
Event.from_json(row)
|
||||
for row in connection.execute(
|
||||
"""
|
||||
select created, event, object_id, message, data from (
|
||||
select * from auditlog
|
||||
where (:event is null or event = :event)
|
||||
and (:object_id is null or object_id = :object_id)
|
||||
and (:from_date is null or created >= :from_date)
|
||||
and (:to_date is null or created < :to_date)
|
||||
and repository = :repository
|
||||
order by created desc limit :limit offset :offset
|
||||
) order by created asc
|
||||
""",
|
||||
{
|
||||
"event": event,
|
||||
"object_id": object_id,
|
||||
"repository": repository_id.id,
|
||||
"from_date": from_date,
|
||||
"to_date": to_date,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
}
|
||||
)
|
||||
]
|
||||
|
||||
return self.with_connection(run)
|
||||
|
||||
def event_insert(self, event: Event, repository_id: RepositoryId | None = None) -> None:
|
||||
"""
|
||||
insert audit log event
|
||||
|
||||
Args:
|
||||
event(Event): event to insert
|
||||
repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
|
||||
"""
|
||||
repository_id = repository_id or self._repository_id
|
||||
|
||||
def run(connection: Connection) -> None:
|
||||
connection.execute(
|
||||
"""
|
||||
insert into auditlog
|
||||
(created, repository, event, object_id, message, data)
|
||||
values
|
||||
(:created, :repository, :event, :object_id, :message, :data)
|
||||
""",
|
||||
{
|
||||
"created": event.created,
|
||||
"repository": repository_id.id,
|
||||
"event": event.event,
|
||||
"object_id": event.object_id,
|
||||
"message": event.message,
|
||||
"data": event.data,
|
||||
})
|
||||
|
||||
return self.with_connection(run, commit=True)
|
@ -50,9 +50,11 @@ class LogsOperations(Operations):
|
||||
(row["created"], row["record"])
|
||||
for row in connection.execute(
|
||||
"""
|
||||
select created, record from logs
|
||||
select created, record from (
|
||||
select * from logs
|
||||
where package_base = :package_base and repository = :repository
|
||||
order by created limit :limit offset :offset
|
||||
order by created desc limit :limit offset :offset
|
||||
) order by created asc
|
||||
""",
|
||||
{
|
||||
"package_base": package_base,
|
||||
|
@ -46,11 +46,22 @@ class Operations(LazyLogging):
|
||||
Args:
|
||||
path(Path): path to the database file
|
||||
repository_id(RepositoryId): repository unique identifier
|
||||
repository_paths(RepositoryPaths): repository paths
|
||||
"""
|
||||
self.path = path
|
||||
self._repository_id = repository_id
|
||||
self._repository_paths = repository_paths
|
||||
|
||||
@property
|
||||
def logger_name(self) -> str:
|
||||
"""
|
||||
extract logger name for the class
|
||||
|
||||
Returns:
|
||||
str: logger name override
|
||||
"""
|
||||
return "sql"
|
||||
|
||||
@staticmethod
|
||||
def factory(cursor: sqlite3.Cursor, row: tuple[Any, ...]) -> dict[str, Any]:
|
||||
"""
|
||||
@ -74,12 +85,13 @@ class Operations(LazyLogging):
|
||||
|
||||
Args:
|
||||
query(Callable[[Connection], T]): function to be called with connection
|
||||
commit(bool, optional): if True commit() will be called on success (Default value = False)
|
||||
commit(bool, optional): if ``True`` commit() will be called on success (Default value = False)
|
||||
|
||||
Returns:
|
||||
T: result of the ``query`` call
|
||||
"""
|
||||
with sqlite3.connect(self.path, detect_types=sqlite3.PARSE_DECLTYPES) as connection:
|
||||
connection.set_trace_callback(self.logger.debug)
|
||||
connection.row_factory = self.factory
|
||||
result = query(connection)
|
||||
if commit:
|
||||
|
@ -26,7 +26,7 @@ from typing import Self
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.database.migrations import Migrations
|
||||
from ahriman.core.database.operations import AuthOperations, BuildOperations, ChangesOperations, \
|
||||
DependenciesOperations, LogsOperations, PackageOperations, PatchOperations
|
||||
DependenciesOperations, EventOperations, LogsOperations, PackageOperations, PatchOperations
|
||||
|
||||
|
||||
# pylint: disable=too-many-ancestors
|
||||
@ -35,6 +35,7 @@ class SQLite(
|
||||
BuildOperations,
|
||||
ChangesOperations,
|
||||
DependenciesOperations,
|
||||
EventOperations,
|
||||
LogsOperations,
|
||||
PackageOperations,
|
||||
PatchOperations):
|
||||
|
@ -22,7 +22,9 @@ from ahriman.core.formatters.build_printer import BuildPrinter
|
||||
from ahriman.core.formatters.changes_printer import ChangesPrinter
|
||||
from ahriman.core.formatters.configuration_paths_printer import ConfigurationPathsPrinter
|
||||
from ahriman.core.formatters.configuration_printer import ConfigurationPrinter
|
||||
from ahriman.core.formatters.event_stats_printer import EventStatsPrinter
|
||||
from ahriman.core.formatters.package_printer import PackagePrinter
|
||||
from ahriman.core.formatters.package_stats_printer import PackageStatsPrinter
|
||||
from ahriman.core.formatters.patch_printer import PatchPrinter
|
||||
from ahriman.core.formatters.printer import Printer
|
||||
from ahriman.core.formatters.repository_printer import RepositoryPrinter
|
||||
|
@ -32,7 +32,7 @@ class BuildPrinter(StringPrinter):
|
||||
|
||||
Args:
|
||||
package(Package): built package
|
||||
is_success(bool): True in case if build has success status and False otherwise
|
||||
is_success(bool): ``True`` in case if build has success status and ``False`` otherwise
|
||||
use_utf(bool): use utf instead of normal symbols
|
||||
"""
|
||||
StringPrinter.__init__(self, f"{self.sign(is_success, use_utf)} {package.base}")
|
||||
@ -43,7 +43,7 @@ class BuildPrinter(StringPrinter):
|
||||
generate sign according to settings
|
||||
|
||||
Args:
|
||||
is_success(bool): True in case if build has success status and False otherwise
|
||||
is_success(bool): ``True`` in case if build has success status and ``False`` otherwise
|
||||
use_utf(bool): use utf instead of normal symbols
|
||||
|
||||
Returns:
|
||||
|
@ -57,7 +57,7 @@ class ChangesPrinter(Printer):
|
||||
generate entry title from content
|
||||
|
||||
Returns:
|
||||
str | None: content title if it can be generated and None otherwise
|
||||
str | None: content title if it can be generated and ``None`` otherwise
|
||||
"""
|
||||
if self.changes.is_empty:
|
||||
return None
|
||||
|
74
src/ahriman/core/formatters/event_stats_printer.py
Normal file
74
src/ahriman/core/formatters/event_stats_printer.py
Normal file
@ -0,0 +1,74 @@
|
||||
#
|
||||
# Copyright (c) 2021-2024 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import statistics
|
||||
|
||||
from ahriman.core.formatters.string_printer import StringPrinter
|
||||
from ahriman.core.utils import minmax
|
||||
from ahriman.models.property import Property
|
||||
|
||||
|
||||
class EventStatsPrinter(StringPrinter):
|
||||
"""
|
||||
print event statistics
|
||||
|
||||
Attributes:
|
||||
events(list[float | int]): event values to build statistics
|
||||
"""
|
||||
|
||||
def __init__(self, event_type: str, events: list[float | int]) -> None:
|
||||
"""
|
||||
default constructor
|
||||
|
||||
Args:
|
||||
event_type(str): event type used for this statistics
|
||||
events(list[float | int]): event values to build statistics
|
||||
"""
|
||||
StringPrinter.__init__(self, event_type)
|
||||
self.events = events
|
||||
|
||||
def properties(self) -> list[Property]:
|
||||
"""
|
||||
convert content into printable data
|
||||
|
||||
Returns:
|
||||
list[Property]: list of content properties
|
||||
"""
|
||||
properties = [
|
||||
Property("total", len(self.events)),
|
||||
]
|
||||
|
||||
# time statistics
|
||||
if self.events:
|
||||
min_time, max_time = minmax(self.events)
|
||||
mean = statistics.mean(self.events)
|
||||
|
||||
if len(self.events) > 1:
|
||||
stdev = statistics.stdev(self.events)
|
||||
average = f"{mean:.3f} ± {stdev:.3f}"
|
||||
else:
|
||||
average = f"{mean:.3f}"
|
||||
|
||||
properties.extend([
|
||||
Property("min", min_time),
|
||||
Property("average", average),
|
||||
Property("max", max_time),
|
||||
])
|
||||
|
||||
return properties
|
58
src/ahriman/core/formatters/package_stats_printer.py
Normal file
58
src/ahriman/core/formatters/package_stats_printer.py
Normal file
@ -0,0 +1,58 @@
|
||||
#
|
||||
# Copyright (c) 2021-2024 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from ahriman.core.formatters.string_printer import StringPrinter
|
||||
from ahriman.models.property import Property
|
||||
|
||||
|
||||
class PackageStatsPrinter(StringPrinter):
|
||||
"""
|
||||
print packages statistics
|
||||
|
||||
Attributes:
|
||||
events(dict[str, int]): map of package to its event frequency
|
||||
"""
|
||||
|
||||
MAX_COUNT = 10
|
||||
|
||||
def __init__(self, events: dict[str, int]) -> None:
|
||||
"""
|
||||
default constructor
|
||||
|
||||
Args:
|
||||
events(dict[str, int]): map of package to its event frequency
|
||||
"""
|
||||
StringPrinter.__init__(self, "The most frequent packages")
|
||||
self.events = events
|
||||
|
||||
def properties(self) -> list[Property]:
|
||||
"""
|
||||
convert content into printable data
|
||||
|
||||
Returns:
|
||||
list[Property]: list of content properties
|
||||
"""
|
||||
if not self.events:
|
||||
return [] # no events found, discard any stats
|
||||
|
||||
properties = []
|
||||
for object_id, count in sorted(self.events.items(), key=lambda pair: pair[1], reverse=True)[:self.MAX_COUNT]:
|
||||
properties.append(Property(object_id, count))
|
||||
|
||||
return properties
|
@ -63,7 +63,7 @@ class Printer:
|
||||
generate entry title from content
|
||||
|
||||
Returns:
|
||||
str | None: content title if it can be generated and None otherwise
|
||||
str | None: content title if it can be generated and ``None`` otherwise
|
||||
"""
|
||||
return None
|
||||
|
||||
|
@ -42,6 +42,6 @@ class StringPrinter(Printer):
|
||||
generate entry title from content
|
||||
|
||||
Returns:
|
||||
str | None: content title if it can be generated and None otherwise
|
||||
str | None: content title if it can be generated and ``None`` otherwise
|
||||
"""
|
||||
return self.content
|
||||
|
@ -85,7 +85,7 @@ class SyncHttpClient(LazyLogging):
|
||||
exception(requests.RequestException): exception raised
|
||||
|
||||
Returns:
|
||||
str: text of the response if it is not None and empty string otherwise
|
||||
str: text of the response if it is not ``None`` and empty string otherwise
|
||||
"""
|
||||
result: str = exception.response.text if exception.response is not None else ""
|
||||
return result
|
||||
|
@ -29,7 +29,8 @@ from ahriman.models.repository_id import RepositoryId
|
||||
class HttpLogHandler(logging.Handler):
|
||||
"""
|
||||
handler for the http logging. Because default :class:`logging.handlers.HTTPHandler` does not support cookies
|
||||
authorization, we have to implement own handler which overrides the :func:`logging.handlers.HTTPHandler.emit` method
|
||||
authorization, we have to implement own handler which overrides the :func:`logging.handlers.HTTPHandler.emit()`
|
||||
method
|
||||
|
||||
Attributes:
|
||||
reporter(Client): build status reporter instance
|
||||
|
@ -17,14 +17,16 @@
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import datetime
|
||||
import jinja2
|
||||
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.sign.gpg import GPG
|
||||
from ahriman.core.utils import pretty_datetime, pretty_size
|
||||
from ahriman.core.utils import pretty_datetime, pretty_size, utcnow
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
from ahriman.models.result import Result
|
||||
from ahriman.models.sign_settings import SignSettings
|
||||
@ -37,30 +39,34 @@ class JinjaTemplate:
|
||||
It uses jinja2 templates for report generation, the following variables are allowed:
|
||||
|
||||
* homepage - link to homepage, string, optional
|
||||
* last_update - report generation time, pretty printed datetime, required
|
||||
* link_path - prefix fo packages to download, string, required
|
||||
* has_package_signed - True in case if package sign enabled, False otherwise, required
|
||||
* has_repo_signed - True in case if repository database sign enabled, False otherwise, required
|
||||
* has_package_signed - ``True`` in case if package sign enabled, ``False`` otherwise, required
|
||||
* has_repo_signed - ``True`` in case if repository database sign enabled, ``False`` otherwise, required
|
||||
* packages - sorted list of packages properties, required
|
||||
* architecture, string
|
||||
* archive_size, pretty printed size, string
|
||||
* build_date, pretty printed datetime, string
|
||||
* depends, sorted list of strings
|
||||
* description, string
|
||||
* filename, string,
|
||||
* filename, string
|
||||
* groups, sorted list of strings
|
||||
* installed_size, pretty printed datetime, string
|
||||
* installed_size, pretty printed size, string
|
||||
* licenses, sorted list of strings
|
||||
* name, string
|
||||
* tag, string
|
||||
* url, string
|
||||
* version, string
|
||||
* pgp_key - default PGP key ID, string, optional
|
||||
* repository - repository name, string, required
|
||||
* rss_url - optional link to the RSS feed, string, optional
|
||||
|
||||
Attributes:
|
||||
default_pgp_key(str | None): default PGP key
|
||||
homepage(str | None): homepage link if any (for footer)
|
||||
link_path(str): prefix fo packages to download
|
||||
name(str): repository name
|
||||
rss_url(str | None): link to the RSS feed
|
||||
sign_targets(set[SignSettings]): targets to sign enabled in configuration
|
||||
templates(list[Path]): list of directories with templates
|
||||
"""
|
||||
@ -80,8 +86,36 @@ class JinjaTemplate:
|
||||
self.homepage = configuration.get(section, "homepage", fallback=None)
|
||||
self.link_path = configuration.get(section, "link_path")
|
||||
self.name = repository_id.name
|
||||
self.rss_url = configuration.get(section, "rss_url", fallback=None)
|
||||
self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration)
|
||||
|
||||
@staticmethod
|
||||
def format_datetime(timestamp: datetime.datetime | float | int | None) -> str:
|
||||
"""
|
||||
convert datetime object to string
|
||||
|
||||
Args:
|
||||
timestamp(datetime.datetime | float | int | None): datetime to convert
|
||||
|
||||
Returns:
|
||||
str: datetime as string representation
|
||||
"""
|
||||
return pretty_datetime(timestamp)
|
||||
|
||||
@staticmethod
|
||||
def sort_content(content: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""
|
||||
sort content before rendering
|
||||
|
||||
Args:
|
||||
content(list[dict[str, str]]): content of the template
|
||||
|
||||
Returns:
|
||||
list[dict[str, str]]: sorted content according to comparator defined
|
||||
"""
|
||||
comparator: Callable[[dict[str, str]], str] = lambda item: item["filename"]
|
||||
return sorted(content, key=comparator)
|
||||
|
||||
def make_html(self, result: Result, template_name: Path | str) -> str:
|
||||
"""
|
||||
generate report for the specified packages
|
||||
@ -104,7 +138,7 @@ class JinjaTemplate:
|
||||
{
|
||||
"architecture": properties.architecture or "",
|
||||
"archive_size": pretty_size(properties.archive_size),
|
||||
"build_date": pretty_datetime(properties.build_date),
|
||||
"build_date": self.format_datetime(properties.build_date),
|
||||
"depends": properties.depends,
|
||||
"description": properties.description or "",
|
||||
"filename": properties.filename,
|
||||
@ -112,17 +146,20 @@ class JinjaTemplate:
|
||||
"installed_size": pretty_size(properties.installed_size),
|
||||
"licenses": properties.licenses,
|
||||
"name": package,
|
||||
"tag": f"tag:{self.name}:{properties.architecture}:{package}:{base.version}:{properties.build_date}",
|
||||
"url": properties.url or "",
|
||||
"version": base.version
|
||||
"version": base.version,
|
||||
} for base in result.success for package, properties in base.packages.items()
|
||||
]
|
||||
comparator: Callable[[dict[str, str]], str] = lambda item: item["filename"]
|
||||
|
||||
return template.render(
|
||||
homepage=self.homepage,
|
||||
last_update=self.format_datetime(utcnow()),
|
||||
link_path=self.link_path,
|
||||
has_package_signed=SignSettings.Packages in self.sign_targets,
|
||||
has_repo_signed=SignSettings.Repository in self.sign_targets,
|
||||
packages=sorted(content, key=comparator),
|
||||
packages=self.sort_content(content),
|
||||
pgp_key=self.default_pgp_key,
|
||||
repository=self.name)
|
||||
repository=self.name,
|
||||
rss_url=self.rss_url,
|
||||
)
|
||||
|
@ -78,7 +78,7 @@ class RemoteCall(Report):
|
||||
process_id(str): remote process id
|
||||
|
||||
Returns:
|
||||
bool: True in case if remote process is alive and False otherwise
|
||||
bool: ``True`` in case if remote process is alive and ``False`` otherwise
|
||||
"""
|
||||
try:
|
||||
response = self.client.make_request("GET", f"{self.client.address}/api/v1/service/process/{process_id}")
|
||||
|
@ -66,7 +66,7 @@ class Report(LazyLogging):
|
||||
self.configuration = configuration
|
||||
|
||||
@staticmethod
|
||||
def load(repository_id: RepositoryId, configuration: Configuration, target: str) -> Report:
|
||||
def load(repository_id: RepositoryId, configuration: Configuration, target: str) -> Report: # pylint: disable=too-many-return-statements
|
||||
"""
|
||||
load client from settings
|
||||
|
||||
@ -92,6 +92,9 @@ class Report(LazyLogging):
|
||||
case ReportSettings.Telegram:
|
||||
from ahriman.core.report.telegram import Telegram
|
||||
return Telegram(repository_id, configuration, section)
|
||||
case ReportSettings.RSS:
|
||||
from ahriman.core.report.rss import RSS
|
||||
return RSS(repository_id, configuration, section)
|
||||
case ReportSettings.RemoteCall:
|
||||
from ahriman.core.report.remote_call import RemoteCall
|
||||
return RemoteCall(repository_id, configuration, section)
|
||||
|
@ -116,6 +116,11 @@ class ReportTrigger(Trigger):
|
||||
"required": True,
|
||||
"empty": False,
|
||||
},
|
||||
"rss_url": {
|
||||
"type": "string",
|
||||
"empty": False,
|
||||
"is_url": ["http", "https"],
|
||||
},
|
||||
"sender": {
|
||||
"type": "string",
|
||||
"required": True,
|
||||
@ -187,6 +192,11 @@ class ReportTrigger(Trigger):
|
||||
"coerce": "absolute_path",
|
||||
"required": True,
|
||||
},
|
||||
"rss_url": {
|
||||
"type": "string",
|
||||
"empty": False,
|
||||
"is_url": ["http", "https"],
|
||||
},
|
||||
"template": {
|
||||
"type": "string",
|
||||
"excludes": ["template_path"],
|
||||
@ -243,6 +253,11 @@ class ReportTrigger(Trigger):
|
||||
"empty": False,
|
||||
"is_url": [],
|
||||
},
|
||||
"rss_url": {
|
||||
"type": "string",
|
||||
"empty": False,
|
||||
"is_url": ["http", "https"],
|
||||
},
|
||||
"template": {
|
||||
"type": "string",
|
||||
"excludes": ["template_path"],
|
||||
@ -304,7 +319,67 @@ class ReportTrigger(Trigger):
|
||||
"coerce": "integer",
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
"rss": {
|
||||
"type": "dict",
|
||||
"schema": {
|
||||
"type": {
|
||||
"type": "string",
|
||||
"allowed": ["rss"],
|
||||
},
|
||||
"homepage": {
|
||||
"type": "string",
|
||||
"empty": False,
|
||||
"is_url": ["http", "https"],
|
||||
},
|
||||
"link_path": {
|
||||
"type": "string",
|
||||
"required": True,
|
||||
"empty": False,
|
||||
"is_url": [],
|
||||
},
|
||||
"max_entries": {
|
||||
"type": "integer",
|
||||
"coerce": "integer",
|
||||
},
|
||||
"path": {
|
||||
"type": "path",
|
||||
"coerce": "absolute_path",
|
||||
"required": True,
|
||||
},
|
||||
"rss_url": {
|
||||
"type": "string",
|
||||
"empty": False,
|
||||
"is_url": ["http", "https"],
|
||||
},
|
||||
"template": {
|
||||
"type": "string",
|
||||
"excludes": ["template_path"],
|
||||
"dependencies": ["templates"],
|
||||
"required": True,
|
||||
"empty": False,
|
||||
},
|
||||
"template_path": {
|
||||
"type": "path",
|
||||
"coerce": "absolute_path",
|
||||
"excludes": ["template"],
|
||||
"required": True,
|
||||
"path_exists": True,
|
||||
"path_type": "file",
|
||||
},
|
||||
"templates": {
|
||||
"type": "list",
|
||||
"coerce": "list",
|
||||
"schema": {
|
||||
"type": "path",
|
||||
"coerce": "absolute_path",
|
||||
"path_exists": True,
|
||||
"path_type": "dir",
|
||||
},
|
||||
"empty": False,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
|
||||
|
130
src/ahriman/core/report/rss.py
Normal file
130
src/ahriman/core/report/rss.py
Normal file
@ -0,0 +1,130 @@
|
||||
#
|
||||
# Copyright (c) 2021-2024 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import datetime
|
||||
|
||||
from collections.abc import Callable
|
||||
from email.utils import format_datetime, parsedate_to_datetime
|
||||
from typing import Any
|
||||
|
||||
from ahriman.core import context
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.report.jinja_template import JinjaTemplate
|
||||
from ahriman.core.report.report import Report
|
||||
from ahriman.core.status import Client
|
||||
from ahriman.models.event import EventType
|
||||
from ahriman.models.package import Package
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
from ahriman.models.result import Result
|
||||
|
||||
|
||||
class RSS(Report, JinjaTemplate):
|
||||
"""
|
||||
RSS report generator
|
||||
|
||||
Attributes:
|
||||
max_entries(int): the maximal amount of entries in RSS
|
||||
report_path(Path): output path to RSS report
|
||||
template(str): name of the template
|
||||
"""
|
||||
|
||||
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
|
||||
"""
|
||||
default constructor
|
||||
|
||||
Args:
|
||||
repository_id(RepositoryId): repository unique identifier
|
||||
configuration(Configuration): configuration instance
|
||||
section(str): settings section name
|
||||
"""
|
||||
Report.__init__(self, repository_id, configuration)
|
||||
JinjaTemplate.__init__(self, repository_id, configuration, section)
|
||||
|
||||
self.max_entries = configuration.getint(section, "max_entries", fallback=-1)
|
||||
self.report_path = configuration.getpath(section, "path")
|
||||
self.template = configuration.get(section, "template")
|
||||
|
||||
@staticmethod
|
||||
def format_datetime(timestamp: datetime.datetime | float | int | None) -> str:
|
||||
"""
|
||||
convert datetime object to string
|
||||
|
||||
Args:
|
||||
timestamp(datetime.datetime | float | int | None): datetime to convert
|
||||
|
||||
Returns:
|
||||
str: datetime as string representation
|
||||
"""
|
||||
if timestamp is None:
|
||||
return ""
|
||||
if isinstance(timestamp, (int, float)):
|
||||
timestamp = datetime.datetime.fromtimestamp(timestamp, datetime.UTC)
|
||||
return format_datetime(timestamp)
|
||||
|
||||
@staticmethod
|
||||
def sort_content(content: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""
|
||||
sort content before rendering
|
||||
|
||||
Args:
|
||||
content(list[dict[str, str]]): content of the template
|
||||
|
||||
Returns:
|
||||
list[dict[str, str]]: sorted content according to comparator defined
|
||||
"""
|
||||
comparator: Callable[[dict[str, str]], datetime.datetime] = \
|
||||
lambda item: parsedate_to_datetime(item["build_date"])
|
||||
return sorted(content, key=comparator, reverse=True)
|
||||
|
||||
def content(self, packages: list[Package]) -> Result:
|
||||
"""
|
||||
extract result to be written to template
|
||||
|
||||
Args:
|
||||
packages(list[Package]): list of packages to generate report
|
||||
|
||||
Returns:
|
||||
Result: result descriptor
|
||||
"""
|
||||
ctx = context.get()
|
||||
reporter = ctx.get(Client)
|
||||
events = reporter.event_get(EventType.PackageUpdated, None, limit=self.max_entries)
|
||||
|
||||
known_packages = {package.base: package for package in packages}
|
||||
|
||||
result = Result()
|
||||
for event in events:
|
||||
package = known_packages.get(event.object_id)
|
||||
if package is None:
|
||||
continue # package not found
|
||||
result.add_updated(package)
|
||||
|
||||
return result
|
||||
|
||||
def generate(self, packages: list[Package], result: Result) -> None:
|
||||
"""
|
||||
generate report for the specified packages
|
||||
|
||||
Args:
|
||||
packages(list[Package]): list of packages to generate report
|
||||
result(Result): build result
|
||||
"""
|
||||
result = self.content(packages)
|
||||
rss = self.make_html(result, self.template)
|
||||
self.report_path.write_text(rss, encoding="utf8")
|
84
src/ahriman/core/repository/event_logger.py
Normal file
84
src/ahriman/core/repository/event_logger.py
Normal file
@ -0,0 +1,84 @@
|
||||
#
|
||||
# Copyright (c) 2021-2024 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import contextlib
|
||||
|
||||
from typing import Generator
|
||||
|
||||
from ahriman.core.status import Client
|
||||
from ahriman.models.event import Event, EventType
|
||||
from ahriman.models.metrics_timer import MetricsTimer
|
||||
|
||||
|
||||
class EventLogger:
|
||||
"""
|
||||
wrapper for logging events
|
||||
|
||||
Attributes:
|
||||
reporter(Client): build status reporter instance
|
||||
"""
|
||||
|
||||
reporter: Client
|
||||
|
||||
def event(self, package_base: str, event: EventType, message: str | None = None) -> None:
|
||||
"""
|
||||
log single event. For timed events use context manager :func:`in_event()` instead
|
||||
|
||||
Args:
|
||||
package_base(str): package base name
|
||||
event(EventType): event type to be logged on success action
|
||||
message(str | None, optional): optional message describing the action (Default value = None)
|
||||
|
||||
Examples:
|
||||
This method must be used as simple wrapper for :class:`ahriman.core.status.Client` methods, e.g.::
|
||||
|
||||
>>> do_something()
|
||||
>>> self.event(package_base, EventType.PackageUpdated)
|
||||
"""
|
||||
self.reporter.event_add(Event(event, package_base, message))
|
||||
|
||||
@contextlib.contextmanager
|
||||
def in_event(self, package_base: str, event: EventType, message: str | None = None,
|
||||
failure: EventType | None = None) -> Generator[None, None, None]:
|
||||
"""
|
||||
perform action in package context and log event with time elapsed
|
||||
|
||||
Args:
|
||||
package_base(str): package base name
|
||||
event(EventType): event type to be logged on success action
|
||||
message(str | None, optional): optional message describing the action (Default value = None)
|
||||
failure(EventType | None, optional): event type to be logged on exception (Default value = None)
|
||||
|
||||
Examples:
|
||||
This method must be used to perform action in context with time measurement::
|
||||
|
||||
>>> with self.in_event(package_base, EventType.PackageUpdated):
|
||||
>>> do_something()
|
||||
|
||||
Additional parameter ``failure`` can be set in order to emit an event on exception occured. If none set
|
||||
(default), then no event will be recorded on exception
|
||||
"""
|
||||
with MetricsTimer() as timer:
|
||||
try:
|
||||
yield
|
||||
self.reporter.event_add(Event(event, package_base, message, took=timer.elapsed))
|
||||
except Exception:
|
||||
if failure is not None:
|
||||
self.reporter.event_add(Event(failure, package_base, took=timer.elapsed))
|
||||
raise
|
@ -23,13 +23,14 @@ from collections.abc import Iterable
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from ahriman.core.build_tools.package_archive import PackageArchive
|
||||
from ahriman.core.build_tools.task import Task
|
||||
from ahriman.core.repository.cleaner import Cleaner
|
||||
from ahriman.core.repository.package_info import PackageInfo
|
||||
from ahriman.core.utils import safe_filename
|
||||
from ahriman.models.changes import Changes
|
||||
from ahriman.models.event import EventType
|
||||
from ahriman.models.package import Package
|
||||
from ahriman.models.package_archive import PackageArchive
|
||||
from ahriman.models.package_description import PackageDescription
|
||||
from ahriman.models.packagers import Packagers
|
||||
from ahriman.models.result import Result
|
||||
@ -75,12 +76,13 @@ class Executor(PackageInfo, Cleaner):
|
||||
with self.in_package_context(single.base, local_versions.get(single.base)), \
|
||||
TemporaryDirectory(ignore_cleanup_errors=True) as dir_name:
|
||||
try:
|
||||
with self.in_event(single.base, EventType.PackageUpdated, failure=EventType.PackageUpdateFailed):
|
||||
packager = self.packager(packagers, single.base)
|
||||
last_commit_sha = build_single(single, Path(dir_name), packager.packager_id)
|
||||
# clear changes and update commit hash
|
||||
self.reporter.package_changes_update(single.base, Changes(last_commit_sha))
|
||||
# update dependencies list
|
||||
package_archive = PackageArchive(self.paths.build_directory, single, self.pacman, self.scan_paths)
|
||||
package_archive = PackageArchive(self.paths.build_root, single, self.pacman, self.scan_paths)
|
||||
dependencies = package_archive.depends_on()
|
||||
self.reporter.package_dependencies_update(single.base, dependencies)
|
||||
# update result set
|
||||
@ -104,6 +106,7 @@ class Executor(PackageInfo, Cleaner):
|
||||
"""
|
||||
def remove_base(package_base: str) -> None:
|
||||
try:
|
||||
with self.in_event(package_base, EventType.PackageRemoved):
|
||||
self.reporter.package_remove(package_base)
|
||||
except Exception:
|
||||
self.logger.exception("could not remove base %s", package_base)
|
||||
|
@ -22,6 +22,7 @@ from ahriman.core.alpm.repo import Repo
|
||||
from ahriman.core.configuration import Configuration
|
||||
from ahriman.core.database import SQLite
|
||||
from ahriman.core.log import LazyLogging
|
||||
from ahriman.core.repository.event_logger import EventLogger
|
||||
from ahriman.core.sign.gpg import GPG
|
||||
from ahriman.core.status import Client
|
||||
from ahriman.core.triggers import TriggerLoader
|
||||
@ -34,7 +35,7 @@ from ahriman.models.user import User
|
||||
from ahriman.models.user_access import UserAccess
|
||||
|
||||
|
||||
class RepositoryProperties(LazyLogging):
|
||||
class RepositoryProperties(EventLogger, LazyLogging):
|
||||
"""
|
||||
repository internal objects holder
|
||||
|
||||
@ -80,10 +81,7 @@ class RepositoryProperties(LazyLogging):
|
||||
self.reporter = Client.load(repository_id, configuration, database, report=report)
|
||||
self.triggers = TriggerLoader.load(repository_id, configuration)
|
||||
|
||||
self.scan_paths = ScanPaths(
|
||||
allowed_paths=configuration.getpathlist("build", "allowed_scan_paths", fallback=[]),
|
||||
blacklisted_paths=configuration.getpathlist("build", "blacklisted_scan_paths", fallback=[]),
|
||||
)
|
||||
self.scan_paths = ScanPaths(configuration.getlist("build", "scan_paths", fallback=[]))
|
||||
|
||||
@property
|
||||
def architecture(self) -> str:
|
||||
|
@ -23,6 +23,7 @@ from ahriman.core.build_tools.sources import Sources
|
||||
from ahriman.core.exceptions import UnknownPackageError
|
||||
from ahriman.core.repository.cleaner import Cleaner
|
||||
from ahriman.core.repository.package_info import PackageInfo
|
||||
from ahriman.models.event import EventType
|
||||
from ahriman.models.package import Package
|
||||
from ahriman.models.package_source import PackageSource
|
||||
from ahriman.models.remote_source import RemoteSource
|
||||
@ -71,6 +72,7 @@ class UpdateHandler(PackageInfo, Cleaner):
|
||||
vcs_allowed_age=self.vcs_allowed_age,
|
||||
calculate_version=vcs):
|
||||
self.reporter.set_pending(local.base)
|
||||
self.event(local.base, EventType.PackageOutdated, "Remote version is newer than local")
|
||||
result.append(remote)
|
||||
except Exception:
|
||||
self.reporter.set_failed(local.base)
|
||||
@ -98,8 +100,8 @@ class UpdateHandler(PackageInfo, Cleaner):
|
||||
return files
|
||||
|
||||
result: list[Package] = []
|
||||
for package in self.packages(filter_packages):
|
||||
dependencies = self.reporter.package_dependencies_get(package.base)
|
||||
for local in self.packages(filter_packages):
|
||||
dependencies = self.reporter.package_dependencies_get(local.base)
|
||||
if not dependencies.paths:
|
||||
continue # skip check if no package dependencies found
|
||||
|
||||
@ -112,7 +114,10 @@ class UpdateHandler(PackageInfo, Cleaner):
|
||||
continue
|
||||
|
||||
# there are no packages found in filesystem with the same paths
|
||||
result.append(package)
|
||||
self.reporter.set_pending(local.base)
|
||||
self.event(local.base, EventType.PackageOutdated, "Implicit dependencies are broken")
|
||||
result.append(local)
|
||||
|
||||
break
|
||||
|
||||
return result
|
||||
@ -153,6 +158,7 @@ class UpdateHandler(PackageInfo, Cleaner):
|
||||
vcs_allowed_age=self.vcs_allowed_age,
|
||||
calculate_version=vcs):
|
||||
self.reporter.set_pending(local.base)
|
||||
self.event(local.base, EventType.PackageOutdated, "Locally pulled sources are outdated")
|
||||
result.append(remote)
|
||||
except Exception:
|
||||
self.logger.exception("could not process package at %s", cache_dir)
|
||||
@ -176,6 +182,7 @@ class UpdateHandler(PackageInfo, Cleaner):
|
||||
self.reporter.set_unknown(local)
|
||||
else:
|
||||
self.reporter.set_pending(local.base)
|
||||
self.event(local.base, EventType.PackageOutdated, "Manual update is requested")
|
||||
except Exception:
|
||||
self.logger.exception("could not load packages from database")
|
||||
self.clear_queue()
|
||||
|
@ -20,7 +20,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from collections.abc import Callable, Iterable
|
||||
@ -28,6 +27,7 @@ from multiprocessing import Process, Queue
|
||||
from threading import Lock, Thread
|
||||
|
||||
from ahriman.core.log import LazyLogging
|
||||
from ahriman.models.metrics_timer import MetricsTimer
|
||||
from ahriman.models.pkgbuild_patch import PkgbuildPatch
|
||||
from ahriman.models.process_status import ProcessStatus
|
||||
from ahriman.models.repository_id import RepositoryId
|
||||
@ -72,7 +72,7 @@ class Spawn(Thread, LazyLogging):
|
||||
value(bool): command line argument value
|
||||
|
||||
Returns:
|
||||
str: if ``value`` is True, then returns positive flag and negative otherwise
|
||||
str: if ``value`` is ``True``, then returns positive flag and negative otherwise
|
||||
"""
|
||||
return name if value else f"no-{name}"
|
||||
|
||||
@ -90,11 +90,9 @@ class Spawn(Thread, LazyLogging):
|
||||
process_id(str): process unique identifier
|
||||
queue(Queue[ProcessStatus | None]): output queue
|
||||
"""
|
||||
start_time = time.monotonic()
|
||||
with MetricsTimer() as timer:
|
||||
result = callback(args, repository_id)
|
||||
stop_time = time.monotonic()
|
||||
|
||||
consumed_time = int(1000 * (stop_time - start_time))
|
||||
consumed_time = timer.elapsed
|
||||
|
||||
queue.put(ProcessStatus(process_id, result, consumed_time))
|
||||
|
||||
@ -153,7 +151,7 @@ class Spawn(Thread, LazyLogging):
|
||||
process_id(str): process id to be checked as returned by :func:`_spawn_process()`
|
||||
|
||||
Returns:
|
||||
bool: True in case if process still counts as active and False otherwise
|
||||
bool: ``True`` in case if process still counts as active and ``False`` otherwise
|
||||
"""
|
||||
with self._lock:
|
||||
return process_id in self.active
|
||||
@ -271,7 +269,7 @@ class Spawn(Thread, LazyLogging):
|
||||
"""
|
||||
for terminated in iter(self.queue.get, None):
|
||||
self.logger.info("process %s has been terminated with status %s, consumed time %ss",
|
||||
terminated.process_id, terminated.status, terminated.consumed_time / 1000)
|
||||
terminated.process_id, terminated.status, terminated.consumed_time)
|
||||
|
||||
with self._lock:
|
||||
process = self.active.pop(terminated.process_id, None)
|
||||
|
@ -25,6 +25,7 @@ from ahriman.core.database import SQLite
|
||||
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
|
||||
from ahriman.models.changes import Changes
|
||||
from ahriman.models.dependencies import Dependencies
|
||||
from ahriman.models.event import Event, EventType
|
||||
from ahriman.models.internal_status import InternalStatus
|
||||
from ahriman.models.log_record_id import LogRecordId
|
||||
from ahriman.models.package import Package
|
||||
@ -79,6 +80,40 @@ class Client:
|
||||
|
||||
return make_local_client()
|
||||
|
||||
def event_add(self, event: Event) -> None:
|
||||
"""
|
||||
create new event
|
||||
|
||||
Args:
|
||||
event(Event): audit log event
|
||||
|
||||
Raises:
|
||||
NotImplementedError: not implemented method
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def event_get(self, event: str | EventType | None, object_id: str | None,
|
||||
from_date: int | float | None = None, to_date: int | float | None = None,
|
||||
limit: int = -1, offset: int = 0) -> list[Event]:
|
||||
"""
|
||||
retrieve list of events
|
||||
|
||||
Args:
|
||||
event(str | EventType | None): filter by event type
|
||||
object_id(str | None): filter by event object
|
||||
from_date(int | float | None, optional): minimal creation date, inclusive (Default value = None)
|
||||
to_date(int | float | None, optional): maximal creation date, exclusive (Default value = None)
|
||||
limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1)
|
||||
offset(int, optional): records offset (Default value = 0)
|
||||
|
||||
Returns:
|
||||
list[Event]: list of audit log events
|
||||
|
||||
Raises:
|
||||
NotImplementedError: not implemented method
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def package_changes_get(self, package_base: str) -> Changes:
|
||||
"""
|
||||
get package changes
|
||||
@ -184,7 +219,7 @@ class Client:
|
||||
|
||||
Args:
|
||||
package_base(str): package base
|
||||
version(str | None): package version to remove logs. If None set, all logs will be removed
|
||||
version(str | None): package version to remove logs. If ``None`` is set, all logs will be removed
|
||||
|
||||
Raises:
|
||||
NotImplementedError: not implemented method
|
||||
@ -213,7 +248,7 @@ class Client:
|
||||
|
||||
Args:
|
||||
package_base(str): package base to update
|
||||
variable(str | None): patch name. If None set, all patches will be removed
|
||||
variable(str | None): patch name. If ``None`` is set, all patches will be removed
|
||||
|
||||
Raises:
|
||||
NotImplementedError: not implemented method
|
||||
@ -310,7 +345,7 @@ class Client:
|
||||
def set_unknown(self, package: Package) -> None:
|
||||
"""
|
||||
set package status to unknown. Unlike other methods, this method also checks if package is known,
|
||||
and - in case if it is - it silently skips updatd
|
||||
and - in case if it is - it silently skips update
|
||||
|
||||
Args:
|
||||
package(Package): current package properties
|
||||
|
@ -22,6 +22,7 @@ from ahriman.core.status import Client
|
||||
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
|
||||
from ahriman.models.changes import Changes
|
||||
from ahriman.models.dependencies import Dependencies
|
||||
from ahriman.models.event import Event, EventType
|
||||
from ahriman.models.log_record_id import LogRecordId
|
||||
from ahriman.models.package import Package
|
||||
from ahriman.models.pkgbuild_patch import PkgbuildPatch
|
||||
@ -48,6 +49,34 @@ class LocalClient(Client):
|
||||
self.database = database
|
||||
self.repository_id = repository_id
|
||||
|
||||
def event_add(self, event: Event) -> None:
|
||||
"""
|
||||
create new event
|
||||
|
||||
Args:
|
||||
event(Event): audit log event
|
||||
"""
|
||||
self.database.event_insert(event, self.repository_id)
|
||||
|
||||
def event_get(self, event: str | EventType | None, object_id: str | None,
|
||||
from_date: int | float | None = None, to_date: int | float | None = None,
|
||||
limit: int = -1, offset: int = 0) -> list[Event]:
|
||||
"""
|
||||
retrieve list of events
|
||||
|
||||
Args:
|
||||
event(str | EventType | None): filter by event type
|
||||
object_id(str | None): filter by event object
|
||||
from_date(int | float | None, optional): minimal creation date, inclusive (Default value = None)
|
||||
to_date(int | float | None, optional): maximal creation date, exclusive (Default value = None)
|
||||
limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1)
|
||||
offset(int, optional): records offset (Default value = 0)
|
||||
|
||||
Returns:
|
||||
list[Event]: list of audit log events
|
||||
"""
|
||||
return self.database.event_get(event, object_id, from_date, to_date, limit, offset, self.repository_id)
|
||||
|
||||
def package_changes_get(self, package_base: str) -> Changes:
|
||||
"""
|
||||
get package changes
|
||||
@ -138,7 +167,7 @@ class LocalClient(Client):
|
||||
|
||||
Args:
|
||||
package_base(str): package base
|
||||
version(str | None): package version to remove logs. If None set, all logs will be removed
|
||||
version(str | None): package version to remove logs. If ``None`` is set, all logs will be removed
|
||||
"""
|
||||
self.database.logs_remove(package_base, version, self.repository_id)
|
||||
|
||||
@ -162,7 +191,7 @@ class LocalClient(Client):
|
||||
|
||||
Args:
|
||||
package_base(str): package base to update
|
||||
variable(str | None): patch name. If None set, all patches will be removed
|
||||
variable(str | None): patch name. If ``None`` is set, all patches will be removed
|
||||
"""
|
||||
variables = [variable] if variable is not None else None
|
||||
self.database.patches_remove(package_base, variables)
|
||||
|
@ -27,6 +27,7 @@ from ahriman.core.status import Client
|
||||
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
|
||||
from ahriman.models.changes import Changes
|
||||
from ahriman.models.dependencies import Dependencies
|
||||
from ahriman.models.event import Event, EventType
|
||||
from ahriman.models.log_record_id import LogRecordId
|
||||
from ahriman.models.package import Package
|
||||
from ahriman.models.pkgbuild_patch import PkgbuildPatch
|
||||
@ -68,6 +69,10 @@ class Watcher(LazyLogging):
|
||||
with self._lock:
|
||||
return list(self._known.values())
|
||||
|
||||
event_add: Callable[[Event], None]
|
||||
|
||||
event_get: Callable[[str | EventType | None, str | None, int | None, int | None, int, int], list[Event]]
|
||||
|
||||
def load(self) -> None:
|
||||
"""
|
||||
load packages from local database
|
||||
|
@ -27,6 +27,7 @@ from ahriman.core.status import Client
|
||||
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
|
||||
from ahriman.models.changes import Changes
|
||||
from ahriman.models.dependencies import Dependencies
|
||||
from ahriman.models.event import Event, EventType
|
||||
from ahriman.models.internal_status import InternalStatus
|
||||
from ahriman.models.log_record_id import LogRecordId
|
||||
from ahriman.models.package import Package
|
||||
@ -109,6 +110,15 @@ class WebClient(Client, SyncAhrimanClient):
|
||||
"""
|
||||
return f"{self.address}/api/v1/packages/{urlencode(package_base)}/dependencies"
|
||||
|
||||
def _events_url(self) -> str:
|
||||
"""
|
||||
get url for the events api
|
||||
|
||||
Returns:
|
||||
str: full url for web service for events
|
||||
"""
|
||||
return f"{self.address}/api/v1/events"
|
||||
|
||||
def _logs_url(self, package_base: str) -> str:
|
||||
"""
|
||||
get url for the logs api
|
||||
@ -157,6 +167,51 @@ class WebClient(Client, SyncAhrimanClient):
|
||||
"""
|
||||
return f"{self.address}/api/v1/status"
|
||||
|
||||
def event_add(self, event: Event) -> None:
|
||||
"""
|
||||
create new event
|
||||
|
||||
Args:
|
||||
event(Event): audit log event
|
||||
"""
|
||||
with contextlib.suppress(Exception):
|
||||
self.make_request("POST", self._events_url(), params=self.repository_id.query(), json=event.view())
|
||||
|
||||
def event_get(self, event: str | EventType | None, object_id: str | None,
|
||||
from_date: int | float | None = None, to_date: int | float | None = None,
|
||||
limit: int = -1, offset: int = 0) -> list[Event]:
|
||||
"""
|
||||
retrieve list of events
|
||||
|
||||
Args:
|
||||
event(str | EventType | None): filter by event type
|
||||
object_id(str | None): filter by event object
|
||||
from_date(int | float | None, optional): minimal creation date, inclusive (Default value = None)
|
||||
to_date(int | float | None, optional): maximal creation date, exclusive (Default value = None)
|
||||
limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1)
|
||||
offset(int, optional): records offset (Default value = 0)
|
||||
|
||||
Returns:
|
||||
list[Event]: list of audit log events
|
||||
"""
|
||||
query = self.repository_id.query() + [("limit", str(limit)), ("offset", str(offset))]
|
||||
if event is not None:
|
||||
query.append(("event", str(event)))
|
||||
if object_id is not None:
|
||||
query.append(("object_id", object_id))
|
||||
if from_date is not None:
|
||||
query.append(("from_date", str(from_date)))
|
||||
if to_date is not None:
|
||||
query.append(("to_date", str(to_date)))
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
response = self.make_request("GET", self._events_url(), params=query)
|
||||
response_json = response.json()
|
||||
|
||||
return [Event.from_json(event) for event in response_json]
|
||||
|
||||
return []
|
||||
|
||||
def package_changes_get(self, package_base: str) -> Changes:
|
||||
"""
|
||||
get package changes
|
||||
@ -274,8 +329,9 @@ class WebClient(Client, SyncAhrimanClient):
|
||||
Returns:
|
||||
list[tuple[float, str]]: package logs
|
||||
"""
|
||||
with contextlib.suppress(Exception):
|
||||
query = self.repository_id.query() + [("limit", str(limit)), ("offset", str(offset))]
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
response = self.make_request("GET", self._logs_url(package_base), params=query)
|
||||
response_json = response.json()
|
||||
|
||||
@ -289,12 +345,13 @@ class WebClient(Client, SyncAhrimanClient):
|
||||
|
||||
Args:
|
||||
package_base(str): package base
|
||||
version(str | None): package version to remove logs. If None set, all logs will be removed
|
||||
version(str | None): package version to remove logs. If ``None`` is set, all logs will be removed
|
||||
"""
|
||||
with contextlib.suppress(Exception):
|
||||
query = self.repository_id.query()
|
||||
if version is not None:
|
||||
query += [("version", version)]
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
self.make_request("DELETE", self._logs_url(package_base), params=query)
|
||||
|
||||
def package_patches_get(self, package_base: str, variable: str | None) -> list[PkgbuildPatch]:
|
||||
@ -323,7 +380,7 @@ class WebClient(Client, SyncAhrimanClient):
|
||||
|
||||
Args:
|
||||
package_base(str): package base to update
|
||||
variable(str | None): patch name. If None set, all patches will be removed
|
||||
variable(str | None): patch name. If ``None`` is set, all patches will be removed
|
||||
"""
|
||||
with contextlib.suppress(Exception):
|
||||
self.make_request("DELETE", self._patches_url(package_base, variable or ""))
|
||||
@ -361,6 +418,7 @@ class WebClient(Client, SyncAhrimanClient):
|
||||
NotImplementedError: not implemented method
|
||||
"""
|
||||
payload = {"status": status.value}
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
self.make_request("POST", self._package_url(package_base),
|
||||
params=self.repository_id.query(), json=payload)
|
||||
@ -380,6 +438,7 @@ class WebClient(Client, SyncAhrimanClient):
|
||||
"status": status.value,
|
||||
"package": package.view(),
|
||||
}
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
self.make_request("POST", self._package_url(package.base),
|
||||
params=self.repository_id.query(), json=payload)
|
||||
@ -407,5 +466,6 @@ class WebClient(Client, SyncAhrimanClient):
|
||||
status(BuildStatusEnum): current ahriman status
|
||||
"""
|
||||
payload = {"status": status.value}
|
||||
|
||||
with contextlib.suppress(Exception):
|
||||
self.make_request("POST", self._status_url(), params=self.repository_id.query(), json=payload)
|
||||
|
@ -64,7 +64,7 @@ class Leaf:
|
||||
packages(Iterable[Leaf]): list of known leaves
|
||||
|
||||
Returns:
|
||||
bool: True in case if package is dependency of others and False otherwise
|
||||
bool: ``True`` in case if package is dependency of others and ``False`` otherwise
|
||||
"""
|
||||
for leaf in packages:
|
||||
if leaf.dependencies.intersection(self.items):
|
||||
@ -79,7 +79,7 @@ class Leaf:
|
||||
packages(Iterable[Leaf]): list of known leaves
|
||||
|
||||
Returns:
|
||||
bool: True if any of packages is dependency of the leaf, False otherwise
|
||||
bool: ``True`` if any of packages is dependency of the leaf, ``False`` otherwise
|
||||
"""
|
||||
for leaf in packages:
|
||||
if self.dependencies.intersection(leaf.items):
|
||||
|
@ -160,7 +160,7 @@ class GitHub(Upload, HttpUpload):
|
||||
get release object if any
|
||||
|
||||
Returns:
|
||||
dict[str, Any] | None: GitHub API release object if release found and None otherwise
|
||||
dict[str, Any] | None: GitHub API release object if release found and ``None`` otherwise
|
||||
"""
|
||||
url = f"https://api.github.com/repos/{self.github_owner}/{
|
||||
self.github_repository}/releases/tags/{self.github_release_tag}"
|
||||
|
@ -39,14 +39,15 @@ class Upload(LazyLogging):
|
||||
|
||||
Examples:
|
||||
These classes provide the way to upload packages to remote sources as it is described in their implementations.
|
||||
Basic flow includes class instantiating by using the :func:`load` method and then calling the :func:`run` method
|
||||
which wraps any internal exceptions into the :exc:`ahriman.core.exceptions.SynchronizationError` exception::
|
||||
Basic flow includes class instantiating by using the :func:`load()` method and then calling the :func:`run()`
|
||||
method which wraps any internal exceptions into the :exc:`ahriman.core.exceptions.SynchronizationError`
|
||||
exception::
|
||||
|
||||
>>> configuration = Configuration()
|
||||
>>> upload = Upload.load(RepositoryId("x86_64", "aur-clone"), configuration, "s3")
|
||||
>>> upload.run(configuration.repository_paths.repository, [])
|
||||
|
||||
Or in case if direct access to exception is required, the :func:`sync` method can be used::
|
||||
Or in case if direct access to exception is required, the :func:`sync()` method can be used::
|
||||
|
||||
>>> try:
|
||||
>>> upload.sync(configuration.repository_paths.repository, [])
|
||||
|
@ -225,7 +225,7 @@ def extract_user() -> str | None:
|
||||
extract user from system environment
|
||||
|
||||
Returns:
|
||||
str | None: SUDO_USER in case if set and USER otherwise. It can return None in case if environment has been
|
||||
str | None: SUDO_USER in case if set and USER otherwise. It can return ``None`` in case if environment has been
|
||||
cleared before application start
|
||||
"""
|
||||
return os.getenv("SUDO_USER") or os.getenv("DOAS_USER") or os.getenv("USER")
|
||||
@ -295,7 +295,7 @@ def package_like(filename: Path) -> bool:
|
||||
filename(Path): name of file to check
|
||||
|
||||
Returns:
|
||||
bool: True in case if name contains ``.pkg.`` and not signature, False otherwise
|
||||
bool: ``True`` in case if name contains ``.pkg.`` and not signature, ``False`` otherwise
|
||||
"""
|
||||
name = filename.name
|
||||
return not name.startswith(".") and ".pkg." in name and not name.endswith(".sig")
|
||||
|
@ -44,7 +44,7 @@ class AuthSettings(StrEnum):
|
||||
get enabled flag
|
||||
|
||||
Returns:
|
||||
bool: False in case if authorization is disabled and True otherwise
|
||||
bool: ``False`` in case if authorization is disabled and ``True`` otherwise
|
||||
"""
|
||||
return self != AuthSettings.Disabled
|
||||
|
||||
|
140
src/ahriman/models/event.py
Normal file
140
src/ahriman/models/event.py
Normal file
@ -0,0 +1,140 @@
|
||||
#
|
||||
# Copyright (c) 2021-2024 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
from enum import StrEnum
|
||||
from typing import Any, Self
|
||||
|
||||
from ahriman.core.utils import utcnow
|
||||
|
||||
|
||||
class EventType(StrEnum):
|
||||
"""
|
||||
predefined event types
|
||||
|
||||
Attributes:
|
||||
PackageOutdated(EventType): (class attribute) package has been marked as out-of-date
|
||||
PackageRemoved(EventType): (class attribute) package has been removed
|
||||
PackageUpdateFailed(EventType): (class attribute) package update has been failed
|
||||
PackageUpdated(EventType): (class attribute) package has been updated
|
||||
"""
|
||||
|
||||
PackageOutdated = "package-outdated"
|
||||
PackageRemoved = "package-removed"
|
||||
PackageUpdateFailed = "package-update-failed"
|
||||
PackageUpdated = "package-updated"
|
||||
|
||||
|
||||
class Event:
|
||||
"""
|
||||
audit log event
|
||||
|
||||
Attributes:
|
||||
created(int): event timestamp
|
||||
data(dict[str, Any]): event metadata
|
||||
event(str | EventType): event type
|
||||
message(str | None): event message if available
|
||||
object_id(str): object identifier
|
||||
"""
|
||||
|
||||
def __init__(self, event: str | EventType, object_id: str, message: str | None = None, created: int | None = None,
|
||||
**kwargs: Any):
|
||||
"""
|
||||
default constructor
|
||||
|
||||
Args:
|
||||
event(str | EventType): event type
|
||||
object_id(str): object identifier
|
||||
message(str | None): event message if available
|
||||
created(int | None, optional): event timestamp (Default value = None)
|
||||
**kwargs(Any): event metadata
|
||||
"""
|
||||
self.event = EventType(event) if event in EventType else event
|
||||
self.object_id = object_id
|
||||
self.created = created or int(utcnow().timestamp())
|
||||
|
||||
self.message = message
|
||||
self.data = kwargs
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, dump: dict[str, Any]) -> Self:
|
||||
"""
|
||||
construct event from the json dump
|
||||
|
||||
Args:
|
||||
dump(dict[str, Any]): json dump body
|
||||
|
||||
Returns:
|
||||
Self: dependencies object
|
||||
"""
|
||||
return cls(
|
||||
event=dump["event"],
|
||||
object_id=dump["object_id"],
|
||||
message=dump.get("message"),
|
||||
created=dump.get("created"),
|
||||
**dump.get("data", {}),
|
||||
)
|
||||
|
||||
def get(self, key: str) -> Any:
|
||||
"""
|
||||
get a property
|
||||
|
||||
Args:
|
||||
key(str): key to lookup in data
|
||||
|
||||
Returns:
|
||||
Any: metadata property if available or ``None`` otherwise
|
||||
"""
|
||||
return self.data.get(key)
|
||||
|
||||
def view(self) -> dict[str, Any]:
|
||||
"""
|
||||
generate json event view
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: json-friendly dictionary
|
||||
"""
|
||||
dump = {
|
||||
"event": self.event,
|
||||
"object_id": self.object_id,
|
||||
"created": self.created,
|
||||
}
|
||||
if self.message is not None:
|
||||
dump["message"] = self.message
|
||||
if self.data:
|
||||
dump["data"] = self.data
|
||||
|
||||
return dump
|
||||
|
||||
def __eq__(self, other: Any) -> bool:
|
||||
"""
|
||||
check if other is the same object
|
||||
|
||||
Args:
|
||||
other(Any): other object instance
|
||||
|
||||
Returns:
|
||||
bool: ``True`` if the other object is the same and ``False`` otherwise
|
||||
"""
|
||||
if not isinstance(other, Event):
|
||||
return False
|
||||
return self.event == other.event \
|
||||
and self.object_id == other.object_id \
|
||||
and self.message == other.message \
|
||||
and self.created == other.created \
|
||||
and self.data == other.data
|
93
src/ahriman/models/metrics_timer.py
Normal file
93
src/ahriman/models/metrics_timer.py
Normal file
@ -0,0 +1,93 @@
|
||||
#
|
||||
# Copyright (c) 2021-2024 ahriman team.
|
||||
#
|
||||
# This file is part of ahriman
|
||||
# (see https://github.com/arcan1s/ahriman).
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import time
|
||||
|
||||
from types import TracebackType
|
||||
from typing import Literal, Self
|
||||
|
||||
from ahriman.core.exceptions import InitializeError
|
||||
|
||||
|
||||
class MetricsTimer:
|
||||
"""
|
||||
metrics implementation
|
||||
|
||||
Attributes:
|
||||
start_time(float | None): timer start time in monotonic time
|
||||
|
||||
Examples:
|
||||
This class implements simple timer which allows to measure the time elapsed of the function. Usually it should
|
||||
be used like::
|
||||
|
||||
>>> with MetricsTimer() as timer:
|
||||
>>> do_something()
|
||||
>>> print("Time elapsed for first function: %f", timer.elapsed)
|
||||
>>> do_something_different()
|
||||
>>> print("Time elapsed for all functions: %f", timer.elapsed)
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""
|
||||
default constructor
|
||||
"""
|
||||
self.start_time: float | None = None
|
||||
|
||||
@property
|
||||
def elapsed(self) -> float:
|
||||
"""
|
||||
get elapsed time since the start of the timer
|
||||
|
||||
Returns:
|
||||
float: time elapsed in seconds
|
||||
|
||||
Raises:
|
||||
InitializeError: in case if timer was not initialized correctly
|
||||
"""
|
||||
if self.start_time is None:
|
||||
raise InitializeError("Timer must be started in the context manager")
|
||||
|
||||
stop_time = time.monotonic()
|
||||
consumed_time_ms = int(1000 * (stop_time - self.start_time))
|
||||
return consumed_time_ms / 1000
|
||||
|
||||
def __enter__(self) -> Self:
|
||||
"""
|
||||
start timer context
|
||||
|
||||
Returns:
|
||||
Self: always instance of self
|
||||
"""
|
||||
self.start_time = time.monotonic()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type: type[Exception] | None, exc_val: Exception | None,
|
||||
exc_tb: TracebackType) -> Literal[False]:
|
||||
"""
|
||||
finish timer context
|
||||
|
||||
Args:
|
||||
exc_type(type[Exception] | None): exception type name if any
|
||||
exc_val(Exception | None): exception raised if any
|
||||
exc_tb(TracebackType): exception traceback if any
|
||||
|
||||
Returns:
|
||||
Literal[False]: always ``False`` (do not suppress any exception)
|
||||
"""
|
||||
return False
|
@ -41,7 +41,7 @@ class MigrationResult:
|
||||
check migration and check if there are pending migrations
|
||||
|
||||
Returns:
|
||||
bool: True in case if it requires migrations and False otherwise
|
||||
bool: ``True`` in case if it requires migrations and ``False`` otherwise
|
||||
"""
|
||||
self.validate()
|
||||
return self.new_version > self.old_version
|
||||
|
@ -158,7 +158,7 @@ class Package(LazyLogging):
|
||||
get VCS flag based on the package base
|
||||
|
||||
Returns:
|
||||
bool: True in case if package base looks like VCS package and False otherwise
|
||||
bool: ``True`` in case if package base looks like VCS package and ``False`` otherwise
|
||||
"""
|
||||
return self.base.endswith("-bzr") \
|
||||
or self.base.endswith("-csv") \
|
||||
@ -504,8 +504,8 @@ class Package(LazyLogging):
|
||||
timestamp(float | int): timestamp to check build date against
|
||||
|
||||
Returns:
|
||||
bool: True in case if package was built after the specified date and False otherwise. In case if build date
|
||||
is not set by any of packages, it returns False
|
||||
bool: ``True`` in case if package was built after the specified date and ``False`` otherwise.
|
||||
In case if build date is not set by any of packages, it returns False
|
||||
"""
|
||||
return any(
|
||||
package.build_date > timestamp
|
||||
@ -528,7 +528,7 @@ class Package(LazyLogging):
|
||||
(Default value = True)
|
||||
|
||||
Returns:
|
||||
bool: True if the package is out-of-dated and False otherwise
|
||||
bool: ``True`` if the package is out-of-dated and ``False`` otherwise
|
||||
"""
|
||||
min_vcs_build_date = utcnow().timestamp() - vcs_allowed_age
|
||||
if calculate_version and not self.is_newer_than(min_vcs_build_date):
|
||||
|
@ -55,7 +55,7 @@ class PkgbuildPatch:
|
||||
parse key and define whether it function or not
|
||||
|
||||
Returns:
|
||||
bool: True in case if key ends with parentheses and False otherwise
|
||||
bool: ``True`` in case if key ends with parentheses and ``False`` otherwise
|
||||
"""
|
||||
return self.key is not None and self.key.endswith("()")
|
||||
|
||||
@ -65,7 +65,7 @@ class PkgbuildPatch:
|
||||
check if patch is full diff one or just single-variable patch
|
||||
|
||||
Returns:
|
||||
bool: True in case key set and False otherwise
|
||||
bool: ``True`` in case key set and ``False`` otherwise
|
||||
"""
|
||||
return self.key is None
|
||||
|
||||
|
@ -28,9 +28,9 @@ class ProcessStatus:
|
||||
Attributes:
|
||||
process_id(str): unique process identifier
|
||||
status(bool): process exit code status
|
||||
consumed_time(int): consumed time in ms
|
||||
consumed_time(float): consumed time in seconds
|
||||
"""
|
||||
|
||||
process_id: str
|
||||
status: bool
|
||||
consumed_time: int
|
||||
consumed_time: float
|
||||
|
@ -29,7 +29,7 @@ class Property:
|
||||
Attributes:
|
||||
name(str): name of the property
|
||||
value(Any): property value
|
||||
is_required(bool): if set to True then this property is required
|
||||
is_required(bool): if set to ``True`` then this property is required
|
||||
indent(int): property indentation level
|
||||
"""
|
||||
|
||||
|
@ -57,7 +57,7 @@ class RemoteSource:
|
||||
check if source is remote
|
||||
|
||||
Returns:
|
||||
bool: True in case if package is well-known remote source (e.g. AUR) and False otherwise
|
||||
bool: ``True`` in case if package is well-known remote source (e.g. AUR) and ``False`` otherwise
|
||||
"""
|
||||
return self.source in (PackageSource.AUR, PackageSource.Repository)
|
||||
|
||||
|
@ -32,6 +32,7 @@ class ReportSettings(StrEnum):
|
||||
Email(ReportSettings): (class attribute) email report generation
|
||||
Console(ReportSettings): (class attribute) print result to console
|
||||
Telegram(ReportSettings): (class attribute) markdown report to telegram channel
|
||||
RSS(ReportSettings): (class attribute) RSS report generation
|
||||
RemoteCall(ReportSettings): (class attribute) remote ahriman server call
|
||||
"""
|
||||
|
||||
@ -40,10 +41,11 @@ class ReportSettings(StrEnum):
|
||||
Email = "email"
|
||||
Console = "console"
|
||||
Telegram = "telegram"
|
||||
RSS = "rss"
|
||||
RemoteCall = "remote-call"
|
||||
|
||||
@staticmethod
|
||||
def from_option(value: str) -> ReportSettings:
|
||||
def from_option(value: str) -> ReportSettings: # pylint: disable=too-many-return-statements
|
||||
"""
|
||||
construct value from configuration
|
||||
|
||||
@ -62,6 +64,8 @@ class ReportSettings(StrEnum):
|
||||
return ReportSettings.Console
|
||||
case "telegram":
|
||||
return ReportSettings.Telegram
|
||||
case "rss":
|
||||
return ReportSettings.RSS
|
||||
case "ahriman" | "remote-call":
|
||||
return ReportSettings.RemoteCall
|
||||
case _:
|
||||
|
@ -52,7 +52,7 @@ class RepositoryId:
|
||||
check if all data is supplied for the loading
|
||||
|
||||
Returns:
|
||||
bool: True in case if architecture or name are not set and False otherwise
|
||||
bool: ``True`` in case if architecture or name are not set and ``False`` otherwise
|
||||
"""
|
||||
return not self.architecture or not self.name
|
||||
|
||||
@ -85,7 +85,7 @@ class RepositoryId:
|
||||
other(Any): other object to compare
|
||||
|
||||
Returns:
|
||||
bool: True in case if this is less than other and False otherwise
|
||||
bool: ``True`` in case if this is less than other and ``False`` otherwise
|
||||
|
||||
Raises:
|
||||
TypeError: if other is different from RepositoryId type
|
||||
|
@ -85,7 +85,7 @@ class RepositoryPaths(LazyLogging):
|
||||
return Path(self.repository_id.name) / self.repository_id.architecture
|
||||
|
||||
@property
|
||||
def build_directory(self) -> Path:
|
||||
def build_root(self) -> Path:
|
||||
"""
|
||||
same as :attr:`chroot`, but exactly build chroot
|
||||
|
||||
@ -113,7 +113,7 @@ class RepositoryPaths(LazyLogging):
|
||||
Returns:
|
||||
Path: full patch to devtools chroot directory
|
||||
"""
|
||||
# for the chroot directory devtools will create own tree, and we don"t have to specify architecture here
|
||||
# for the chroot directory devtools will create own tree, and we don't have to specify architecture here
|
||||
return self.root / "chroot" / self.repository_id.name
|
||||
|
||||
@property
|
||||
|
@ -82,7 +82,7 @@ class Result:
|
||||
get if build result is empty or not
|
||||
|
||||
Returns:
|
||||
bool: True in case if success list is empty and False otherwise
|
||||
bool: ``True`` in case if success list is empty and ``False`` otherwise
|
||||
"""
|
||||
return not self._added and not self._updated
|
||||
|
||||
@ -191,7 +191,7 @@ class Result:
|
||||
other(Any): other object instance
|
||||
|
||||
Returns:
|
||||
bool: True if the other object is the same and False otherwise
|
||||
bool: ``True`` if the other object is the same and ``False`` otherwise
|
||||
"""
|
||||
if not isinstance(other, Result):
|
||||
return False
|
||||
|
@ -17,29 +17,33 @@
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
import re
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
@dataclass(frozen=True)
|
||||
class ScanPaths:
|
||||
"""
|
||||
paths used for scan filesystem
|
||||
|
||||
Attributes:
|
||||
allowed_paths(list[Path]): list of whitelisted paths
|
||||
blacklisted_paths(list[Path]): list of paths to be skipped from scan
|
||||
paths(list[str]): list of regular expressions to be used to match paths
|
||||
"""
|
||||
|
||||
allowed_paths: list[Path]
|
||||
blacklisted_paths: list[Path]
|
||||
paths: list[str]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
@cached_property
|
||||
def patterns(self) -> list[re.Pattern[str]]:
|
||||
"""
|
||||
compute relative to / paths
|
||||
compiled regular expressions
|
||||
|
||||
Returns:
|
||||
list[re.Pattern]: a list of compiled regular expressions
|
||||
"""
|
||||
object.__setattr__(self, "allowed_paths", [path.relative_to("/") for path in self.allowed_paths])
|
||||
object.__setattr__(self, "blacklisted_paths", [path.relative_to("/") for path in self.blacklisted_paths])
|
||||
return [re.compile(path) for path in self.paths]
|
||||
|
||||
def is_allowed(self, path: Path) -> bool:
|
||||
"""
|
||||
@ -49,10 +53,7 @@ class ScanPaths:
|
||||
path(Path): path to be checked
|
||||
|
||||
Returns:
|
||||
bool: ``True`` in case if :attr:`allowed_paths` contains element which is parent for the path and
|
||||
:attr:`blacklisted_paths` doesn't and ``False`` otherwise
|
||||
bool: ``True`` in case if :attr:`paths` contains at least one element to which the path is matched
|
||||
and ``False`` otherwise
|
||||
"""
|
||||
if any(path.is_relative_to(blacklisted) for blacklisted in self.blacklisted_paths):
|
||||
return False # path is blacklisted
|
||||
# check if we actually have to check this path
|
||||
return any(path.is_relative_to(allowed) for allowed in self.allowed_paths)
|
||||
return any(pattern.match(str(path)) for pattern in self.patterns)
|
||||
|
@ -98,7 +98,7 @@ class User:
|
||||
salt(str): salt for hashed password
|
||||
|
||||
Returns:
|
||||
bool: True in case if password matches, False otherwise
|
||||
bool: ``True`` in case if password matches, ``False`` otherwise
|
||||
"""
|
||||
try:
|
||||
verified: bool = self._HASHER.verify(password + salt, self.password)
|
||||
@ -131,7 +131,7 @@ class User:
|
||||
required(UserAccess): required access level
|
||||
|
||||
Returns:
|
||||
bool: True in case if user is allowed to do this request and False otherwise
|
||||
bool: ``True`` in case if user is allowed to do this request and ``False`` otherwise
|
||||
"""
|
||||
return self.access.permits(required)
|
||||
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user