From 1770793e69b47c2e9501321beae6526051298789 Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Thu, 11 Mar 2021 03:55:17 +0300 Subject: [PATCH] improvements * multi-sign and multi-web configuration * change default configuration to do not use architecture * change units to be templated * some refactoring --- CONFIGURING.md | 26 ++++---- README.md | 11 ++-- package/archlinux/PKGBUILD | 5 +- package/archlinux/ahriman.sudoers | 4 -- package/etc/ahriman.ini | 8 +-- .../lib/systemd/system/ahriman-web.service | 11 ---- .../lib/systemd/system/ahriman-web@.service | 15 +++++ package/lib/systemd/system/ahriman.service | 7 --- package/lib/systemd/system/ahriman@.service | 7 +++ .../system/{ahriman.timer => ahriman@.timer} | 2 +- .../{index.jinja2 => build-status.jinja2} | 0 setup.py | 8 +-- src/ahriman/application/ahriman.py | 63 +++---------------- src/ahriman/application/application.py | 10 ++- src/ahriman/application/lock.py | 59 +++++++++++++++++ src/ahriman/core/build_tools/task.py | 6 +- src/ahriman/core/configuration.py | 16 +++-- src/ahriman/core/exceptions.py | 18 +++--- src/ahriman/core/report/html.py | 4 +- src/ahriman/core/report/report.py | 7 ++- src/ahriman/core/repository.py | 31 +++++---- src/ahriman/core/sign/gpg_wrapper.py | 8 +-- src/ahriman/core/upload/rsync.py | 4 +- src/ahriman/core/upload/s3.py | 4 +- src/ahriman/core/upload/uploader.py | 7 ++- src/ahriman/core/watcher/client.py | 22 ++++++- src/ahriman/models/package.py | 3 +- src/ahriman/models/repository_paths.py | 15 +++++ .../web/middlewares/exception_handler.py | 3 +- src/ahriman/web/views/base.py | 1 + src/ahriman/web/views/index.py | 7 +-- src/ahriman/web/web.py | 16 ++--- 32 files changed, 235 insertions(+), 173 deletions(-) delete mode 100644 package/archlinux/ahriman.sudoers delete mode 100644 package/lib/systemd/system/ahriman-web.service create mode 100644 package/lib/systemd/system/ahriman-web@.service delete mode 100644 package/lib/systemd/system/ahriman.service create mode 100644 package/lib/systemd/system/ahriman@.service rename package/lib/systemd/system/{ahriman.timer => ahriman@.timer} (58%) rename package/share/ahriman/{index.jinja2 => build-status.jinja2} (100%) create mode 100644 src/ahriman/application/lock.py diff --git a/CONFIGURING.md b/CONFIGURING.md index ba9a0dd5..c9449231 100644 --- a/CONFIGURING.md +++ b/CONFIGURING.md @@ -1,17 +1,17 @@ # ahriman configuration -Some groups can be specified for each architecture separately with default values. E.g. if there are `build` and `build_x86_64` groups it will use the `build_x86_64` for the `x86_64` architecture and `build` for any other. +Some groups can be specified for each architecture separately. E.g. if there are `build` and `build_x86_64` groups it will use the `build_x86_64` for the `x86_64` architecture and `build` for any other (architecture specific group has higher priority). ## `settings` group -Base configuration settings: +Base configuration settings. * `include` - path to directory with configuration files overrides, string, required. * `logging` - path to logging configuration, string, required. Check `logging.ini` for reference. ## `aur` group -AUR related configuration: +AUR related configuration. * `url` - base url for AUR, string, required. @@ -27,25 +27,25 @@ Build related configuration. Group name must refer to architecture, e.g. it shou ## `repository` group -Base repository settings: +Base repository settings. * `name` - repository name, string, required. * `root` - root path for application, string, required. -## `sign` group +## `sign_*` groups -Settings for signing packages or repository: +Settings for signing packages or repository. Group name must refer to architecture, e.g. it should be `sign_x86_64` for x86_64 architecture. * `target` - configuration flag to enable signing, space separated list of strings, required. Allowed values are `package` (sign each package separately), `repository` (sign repository database file). * `key` - PGP key, string, required. ## `report` group -Report generation settings: +Report generation settings. * `target` - list of reports to be generated, space separated list of strings, optional. Allowed values are `html`. -### `html_*` group +### `html_*` groups Group name must refer to architecture, e.g. it should be `html_x86_64` for x86_64 architecture. @@ -56,25 +56,25 @@ Group name must refer to architecture, e.g. it should be `html_x86_64` for x86_6 ## `upload` group -Remote synchronization settings: +Remote synchronization settings. * `target` - list of synchronizations to be used, space separated list of strings, optional. Allowed values are `rsync`, `s3`. -### `rsync_*` group +### `rsync_*` groups Group name must refer to architecture, e.g. it should be `rsync_x86_64` for x86_64 architecture. Requires `rsync` package to be installed. Do not forget to configure ssh for user `ahriman`. * `remote` - remote server to rsync (e.g. `1.2.3.4:5678:path/to/sync`), string, required. -### `s3_*` group +### `s3_*` groups Group name must refer to architecture, e.g. it should be `s3_x86_64` for x86_64 architecture. Requires `aws-cli` package to be installed. Do not forget to configure it for user `ahriman`. * `bucket` - bucket name (e.g. `s3://bucket/path`), string, required. -## `web` group +## `web_*` groups -Web server settings. If any of `host`/`port` is not set, web intergration will be disabled. +Web server settings. If any of `host`/`port` is not set, web integration will be disabled. Group name must refer to architecture, e.g. it should be `web_x86_64` for x86_64 architecture. * `host` - host to bind, string, optional. * `port` - port to bind, int, optional. diff --git a/README.md b/README.md index 1b2be2dc..2c322911 100644 --- a/README.md +++ b/README.md @@ -5,9 +5,10 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github ## Features * Install-configure-forget manager for own repository +* Multi-architecture support * VCS packages support * Sign support with gpg -* Synchronization to remote services and report generation +* Synchronization to remote services (rsync, s3) and report generation (html) * Repository status interface ## Installation and run @@ -16,11 +17,11 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github * Change settings if required, see `CONFIGURING.md` for more details. * Create `/var/lib/ahriman/.makepkg.conf` with `makepkg.conf` overrides if required (at least you might want to set `PACKAGER`). * Configure build tools (it might be required if your package will use any custom repositories): - * create build command if required, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/custom-x86_64-build` (you can choose any name for command); + * create build command, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/custom-x86_64-build` (you can choose any name for command); * create configuration file, e.g. `cp /usr/share/devtools/pacman-{extra,custom}.conf`; - * change configuration file: add your own repository, add multilib repository; - * set `build.build_command` to point to your command; - * configure `/etc/sudoers.d/ahriman` to allow to run command without password. + * change configuration file, add your own repository, add multilib repository etc; + * set `build.build_command` setting to point to your command; + * configure `/etc/sudoers.d/ahriman` to allow running command without password. * Start and enable `ahriman.timer` via `systemctl`. * Add packages by using `ahriman add {package}` command. diff --git a/package/archlinux/PKGBUILD b/package/archlinux/PKGBUILD index f5557fe4..c61b30da 100644 --- a/package/archlinux/PKGBUILD +++ b/package/archlinux/PKGBUILD @@ -21,11 +21,9 @@ optdepends=('aws-cli: sync to s3' 'rsync: sync by using rsync' 'subversion: -svn packages support') source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz" - 'ahriman.sudoers' 'ahriman.sysusers' 'ahriman.tmpfiles') -sha512sums=('2c811060106aea6f8826cc6beac9f5733370386a43448b359051ea377e233218aae0c5ec3ef7b1ec399fa6a53c02059015b7398b0d88b5a2e7129f167d025539' - '8c9b5b63ac3f7b4d9debaf801a1e9c060877c33d3ecafe18010fcca778e5fa2f2e46909d3d0ff1b229ff8aa978445d8243fd36e1fc104117ed678d5e21901167' +sha512sums=('941821639fe4410152a21251d9b0fe5f96ee3a60b88e2067ea4a83ef04b5d1393828152ef4843575449bdef8d44ad6a69f9e41e82516d4d1850bd14f17822785' '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075' '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4') backup=('etc/ahriman.ini' @@ -42,7 +40,6 @@ package() { python setup.py install --root="$pkgdir" - install -Dm400 "$srcdir/$pkgname.sudoers" "$pkgdir/etc/sudoers.d/$pkgname" install -Dm644 "$srcdir/$pkgname.sysusers" "$pkgdir/usr/lib/sysusers.d/$pkgname.conf" install -Dm644 "$srcdir/$pkgname.tmpfiles" "$pkgdir/usr/lib/tmpfiles.d/$pkgname.conf" } diff --git a/package/archlinux/ahriman.sudoers b/package/archlinux/ahriman.sudoers deleted file mode 100644 index d633ca28..00000000 --- a/package/archlinux/ahriman.sudoers +++ /dev/null @@ -1,4 +0,0 @@ -# Used by ArcHlinux ReposItory MANager with default settings - -Cmnd_Alias ARCHBUILD_CMD = /usr/bin/extra-x86_64-build *, /usr/bin/multilib-build * -ahriman ALL=(ALL) NOPASSWD: ARCHBUILD_CMD \ No newline at end of file diff --git a/package/etc/ahriman.ini b/package/etc/ahriman.ini index 0ff9e0fc..723986ce 100644 --- a/package/etc/ahriman.ini +++ b/package/etc/ahriman.ini @@ -5,7 +5,7 @@ logging = /etc/ahriman.ini.d/logging.ini [aur] url = https://aur.archlinux.org -[build_x86_64] +[build] archbuild_flags = build_command = extra-x86_64-build ignore_packages = @@ -23,7 +23,7 @@ key = [report] target = -[html_x86_64] +[html] path = homepage = link_path = @@ -32,10 +32,10 @@ template_path = /usr/share/ahriman/repo-index.jinja2 [upload] target = -[rsync_x86_64] +[rsync] remote = -[s3_x86_64] +[s3] bucket = [web] diff --git a/package/lib/systemd/system/ahriman-web.service b/package/lib/systemd/system/ahriman-web.service deleted file mode 100644 index 67b5b0e1..00000000 --- a/package/lib/systemd/system/ahriman-web.service +++ /dev/null @@ -1,11 +0,0 @@ -[Unit] -Description=ArcHlinux ReposItory MANager web server - -[Service] -Type=simple -ExecStart=/usr/bin/ahriman --architecture x86_64 web -User=ahriman -Group=ahriman - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/package/lib/systemd/system/ahriman-web@.service b/package/lib/systemd/system/ahriman-web@.service new file mode 100644 index 00000000..b399b162 --- /dev/null +++ b/package/lib/systemd/system/ahriman-web@.service @@ -0,0 +1,15 @@ +[Unit] +Description=ArcHlinux ReposItory MANager web server (%I architecture) +After=network.target + +[Service] +Type=simple +ExecStart=/usr/bin/ahriman --architecture %i web +User=ahriman +Group=ahriman + +KillSignal=SIGQUIT +SuccessExitStatus=SIGQUIT + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/package/lib/systemd/system/ahriman.service b/package/lib/systemd/system/ahriman.service deleted file mode 100644 index 9a030c01..00000000 --- a/package/lib/systemd/system/ahriman.service +++ /dev/null @@ -1,7 +0,0 @@ -[Unit] -Description=ArcHlinux ReposItory MANager - -[Service] -ExecStart=/usr/bin/ahriman --architecture x86_64 update -User=ahriman -Group=ahriman \ No newline at end of file diff --git a/package/lib/systemd/system/ahriman@.service b/package/lib/systemd/system/ahriman@.service new file mode 100644 index 00000000..236ebccc --- /dev/null +++ b/package/lib/systemd/system/ahriman@.service @@ -0,0 +1,7 @@ +[Unit] +Description=ArcHlinux ReposItory MANager (%I architecture) + +[Service] +ExecStart=/usr/bin/ahriman --architecture %i update +User=ahriman +Group=ahriman \ No newline at end of file diff --git a/package/lib/systemd/system/ahriman.timer b/package/lib/systemd/system/ahriman@.timer similarity index 58% rename from package/lib/systemd/system/ahriman.timer rename to package/lib/systemd/system/ahriman@.timer index bcee1f2c..ba3b9ece 100644 --- a/package/lib/systemd/system/ahriman.timer +++ b/package/lib/systemd/system/ahriman@.timer @@ -1,5 +1,5 @@ [Unit] -Description=ArcHlinux ReposItory MANager timer +Description=ArcHlinux ReposItory MANager timer (%I architecture) [Timer] OnCalendar=daily diff --git a/package/share/ahriman/index.jinja2 b/package/share/ahriman/build-status.jinja2 similarity index 100% rename from package/share/ahriman/index.jinja2 rename to package/share/ahriman/build-status.jinja2 diff --git a/setup.py b/setup.py index 257ddc0f..10700061 100644 --- a/setup.py +++ b/setup.py @@ -49,12 +49,12 @@ setup( 'package/etc/ahriman.ini.d/logging.ini', ]), ('lib/systemd/system', [ - 'package/lib/systemd/system/ahriman.service', - 'package/lib/systemd/system/ahriman.timer', - 'package/lib/systemd/system/ahriman-web.service', + 'package/lib/systemd/system/ahriman@.service', + 'package/lib/systemd/system/ahriman@.timer', + 'package/lib/systemd/system/ahriman-web@.service', ]), ('share/ahriman', [ - 'package/share/ahriman/index.jinja2', + 'package/share/ahriman/build-status.jinja2', 'package/share/ahriman/repo-index.jinja2', ]), ], diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index b16f1b7e..638eace2 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -18,72 +18,38 @@ # along with this program. If not, see . # import argparse -import os - -from typing import Optional import ahriman.version as version from ahriman.application.application import Application +from ahriman.application.lock import Lock from ahriman.core.configuration import Configuration -def _get_app(args: argparse.Namespace) -> Application: - config = _get_config(args.config) - return Application(args.architecture, config) - - -def _get_config(config_path: str) -> Configuration: - config = Configuration() - config.load(config_path) - config.load_logging() - return config - - -def _lock_check(path: Optional[str]) -> None: - if path is None: - return - if os.path.exists(args.lock): - raise RuntimeError('Another application instance is run') - - -def _lock_create(path: Optional[str]) -> None: - if path is None: - return - open(path, 'w').close() - - -def _lock_remove(path: Optional[str]) -> None: - if path is None: - return - if os.path.exists(path): - os.remove(path) - - def add(args: argparse.Namespace) -> None: - _get_app(args).add(args.package) + Application.from_args(args).add(args.package) def rebuild(args: argparse.Namespace) -> None: - app = _get_app(args) + app = Application.from_args(args) packages = app.repository.packages() app.update(packages) def remove(args: argparse.Namespace) -> None: - _get_app(args).remove(args.package) + Application.from_args(args).remove(args.package) def report(args: argparse.Namespace) -> None: - _get_app(args).report(args.target) + Application.from_args(args).report(args.target) def sync(args: argparse.Namespace) -> None: - _get_app(args).sync(args.target) + Application.from_args(args).sync(args.target) def update(args: argparse.Namespace) -> None: - app = _get_app(args) + app = Application.from_args(args) log_fn = lambda line: print(line) if args.dry_run else app.logger.info(line) packages = app.get_updates(args.no_aur, args.no_manual, args.no_vcs, log_fn) if args.dry_run: @@ -93,9 +59,9 @@ def update(args: argparse.Namespace) -> None: def web(args: argparse.Namespace) -> None: from ahriman.web.web import run_server, setup_service - config = _get_config(args.config) + config = Configuration.from_path(args.config) app = setup_service(args.architecture, config) - run_server(app) + run_server(app, args.architecture) if __name__ == '__main__': @@ -140,18 +106,9 @@ if __name__ == '__main__': web_parser.set_defaults(fn=web, lock=None) args = parser.parse_args() - - if args.force: - _lock_remove(args.lock) - _lock_check(args.lock) - if 'fn' not in args: parser.print_help() exit(1) - try: - _lock_create(args.lock) + with Lock(args.lock, args.force): args.fn(args) - finally: - _lock_remove(args.lock) - diff --git a/src/ahriman/application/application.py b/src/ahriman/application/application.py index e58932cf..0b44c896 100644 --- a/src/ahriman/application/application.py +++ b/src/ahriman/application/application.py @@ -17,11 +17,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +from __future__ import annotations + +import argparse import logging import os import shutil -from typing import Callable, List, Optional +from typing import Callable, List, Optional, Type from ahriman.core.build_tools.task import Task from ahriman.core.configuration import Configuration @@ -37,6 +40,11 @@ class Application: self.architecture = architecture self.repository = Repository(architecture, config) + @classmethod + def from_args(cls: Type[Application], args: argparse.Namespace) -> Application: + config = Configuration.from_path(args.config) + return cls(args.architecture, config) + def _finalize(self) -> None: self.report() self.sync() diff --git a/src/ahriman/application/lock.py b/src/ahriman/application/lock.py new file mode 100644 index 00000000..d0009781 --- /dev/null +++ b/src/ahriman/application/lock.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import os + +from typing import Optional + +from ahriman.core.exceptions import DuplicateRun + + +class Lock: + + def __init__(self, path: Optional[str], force: bool) -> None: + self.path = path + self.force = force + + def __enter__(self): + if self.force: + self.remove() + self.check() + self.create() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.remove() + return True + + def check(self) -> None: + if self.path is None: + return + if os.path.exists(self.path): + raise DuplicateRun() + + def create(self) -> None: + if self.path is None: + return + open(self.path, 'w').close() + + def remove(self) -> None: + if self.path is None: + return + if os.path.exists(self.path): + os.remove(self.path) \ No newline at end of file diff --git a/src/ahriman/core/build_tools/task.py b/src/ahriman/core/build_tools/task.py index d3f1a8af..4c9296c2 100644 --- a/src/ahriman/core/build_tools/task.py +++ b/src/ahriman/core/build_tools/task.py @@ -39,10 +39,10 @@ class Task: self.paths = paths section = config.get_section_name('build', architecture) - self.archbuild_flags = config.get_list(section, 'archbuild_flags') + self.archbuild_flags = config.getlist(section, 'archbuild_flags') self.build_command = config.get(section, 'build_command') - self.makepkg_flags = config.get_list(section, 'makepkg_flags') - self.makechrootpkg_flags = config.get_list(section, 'makechrootpkg_flags') + self.makepkg_flags = config.getlist(section, 'makepkg_flags') + self.makechrootpkg_flags = config.getlist(section, 'makechrootpkg_flags') @property def git_path(self) -> str: diff --git a/src/ahriman/core/configuration.py b/src/ahriman/core/configuration.py index cd4ad95b..f76282cf 100644 --- a/src/ahriman/core/configuration.py +++ b/src/ahriman/core/configuration.py @@ -17,11 +17,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +from __future__ import annotations + import configparser import os from logging.config import fileConfig -from typing import List, Optional, Set +from typing import List, Optional, Type # built-in configparser extension @@ -35,7 +37,14 @@ class Configuration(configparser.RawConfigParser): def include(self) -> str: return self.get('settings', 'include') - def get_list(self, section: str, key: str) -> List[str]: + @classmethod + def from_path(cls: Type[Configuration], path: str) -> Configuration: + config = cls() + config.load(path) + config.load_logging() + return config + + def getlist(self, section: str, key: str) -> List[str]: raw = self.get(section, key, fallback=None) if not raw: # empty string or none return [] @@ -52,8 +61,7 @@ class Configuration(configparser.RawConfigParser): def load_includes(self) -> None: try: - include_dir = self.include - for conf in filter(lambda p: p.endswith('.ini'), sorted(os.listdir(include_dir))): + for conf in filter(lambda p: p.endswith('.ini'), sorted(os.listdir(self.include))): self.read(os.path.join(self.include, conf)) except (FileNotFoundError, configparser.NoOptionError): pass diff --git a/src/ahriman/core/exceptions.py b/src/ahriman/core/exceptions.py index f5d6ea38..3f1398bf 100644 --- a/src/ahriman/core/exceptions.py +++ b/src/ahriman/core/exceptions.py @@ -25,6 +25,11 @@ class BuildFailed(Exception): Exception.__init__(self, f'Package {package} build failed, check logs for details') +class DuplicateRun(Exception): + def __init__(self) -> None: + Exception.__init__(self, 'Another application instance is run') + + class InitializeException(Exception): def __init__(self) -> None: Exception.__init__(self, 'Could not load service') @@ -40,16 +45,11 @@ class InvalidPackageInfo(Exception): Exception.__init__(self, f'There are errors during reading package information: `{details}`') -class MissingConfiguration(Exception): - def __init__(self, name: str) -> None: - Exception.__init__(self, f'No section `{name}` found') - - class ReportFailed(Exception): - def __init__(self, cause: Exception) -> None: - Exception.__init__(self, f'Report failed with reason {cause}') + def __init__(self) -> None: + Exception.__init__(self, 'Report failed') class SyncFailed(Exception): - def __init__(self, cause: Exception) -> None: - Exception.__init__(self, f'Sync failed with reason {cause}') \ No newline at end of file + def __init__(self) -> None: + Exception.__init__(self, 'Sync failed') \ No newline at end of file diff --git a/src/ahriman/core/report/html.py b/src/ahriman/core/report/html.py index 054d55dd..3cc6bc88 100644 --- a/src/ahriman/core/report/html.py +++ b/src/ahriman/core/report/html.py @@ -32,14 +32,14 @@ class HTML(Report): def __init__(self, architecture: str, config: Configuration) -> None: Report.__init__(self, architecture, config) - section = self.config.get_section_name('html', self.architecture) + section = config.get_section_name('html', architecture) self.report_path = config.get(section, 'path') self.link_path = config.get(section, 'link_path') self.template_path = config.get(section, 'template_path') # base template vars - self.sign_targets = [SignSettings.from_option(opt) for opt in config.get_list('sign', 'target')] + self.sign_targets = [SignSettings.from_option(opt) for opt in config.getlist('sign', 'target')] self.pgp_key = config.get('sign', 'key', fallback=None) self.homepage = config.get(section, 'homepage', fallback=None) self.repository = config.get('repository', 'name') diff --git a/src/ahriman/core/report/report.py b/src/ahriman/core/report/report.py index bceab960..47e3f92a 100644 --- a/src/ahriman/core/report/report.py +++ b/src/ahriman/core/report/report.py @@ -27,9 +27,9 @@ from ahriman.models.report_settings import ReportSettings class Report: def __init__(self, architecture: str, config: Configuration) -> None: + self.logger = logging.getLogger('builder') self.architecture = architecture self.config = config - self.logger = logging.getLogger('builder') @staticmethod def run(architecture: str, config: Configuration, target: str, path: str) -> None: @@ -42,8 +42,9 @@ class Report: try: report.generate(path) - except Exception as e: - raise ReportFailed(e) from e + except Exception: + report.logger.exception('report generation failed', exc_info=True) + raise ReportFailed() def generate(self, path: str) -> None: pass \ No newline at end of file diff --git a/src/ahriman/core/repository.py b/src/ahriman/core/repository.py index 36804c66..55eec6b3 100644 --- a/src/ahriman/core/repository.py +++ b/src/ahriman/core/repository.py @@ -31,7 +31,6 @@ from ahriman.core.sign.gpg_wrapper import GPGWrapper from ahriman.core.upload.uploader import Uploader from ahriman.core.util import package_like from ahriman.core.watcher.client import Client -from ahriman.models.build_status import BuildStatusEnum from ahriman.models.package import Package from ahriman.models.repository_paths import RepositoryPaths @@ -46,13 +45,12 @@ class Repository: self.aur_url = config.get('aur', 'url') self.name = config.get('repository', 'name') - self.paths = RepositoryPaths(config.get('repository', 'root'), self.architecture) + self.paths = RepositoryPaths(config.get('repository', 'root'), architecture) self.paths.create_tree() - self.sign = GPGWrapper(config) + self.sign = GPGWrapper(architecture, config) self.wrapper = RepoWrapper(self.name, self.paths, self.sign.repository_sign_args) - - self.web_report = Client.load(config) + self.web_report = Client.load(architecture, config) def _clear_build(self) -> None: for package in os.listdir(self.paths.sources): @@ -82,7 +80,7 @@ class Repository: def process_build(self, updates: List[Package]) -> List[str]: def build_single(package: Package) -> None: - self.web_report.update(package.base, BuildStatusEnum.Building) + self.web_report.set_building(package.base) task = Task(package, self.architecture, self.config, self.paths) task.clone() built = task.build() @@ -94,7 +92,7 @@ class Repository: try: build_single(package) except Exception: - self.web_report.update(package.base, BuildStatusEnum.Failed) + self.web_report.set_failed(package.base) self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True) continue self._clear_build() @@ -126,13 +124,13 @@ class Repository: def process_report(self, targets: Optional[List[str]]) -> None: if targets is None: - targets = self.config.get_list('report', 'target') + targets = self.config.getlist('report', 'target') for target in targets: Report.run(self.architecture, self.config, target, self.paths.repository) def process_sync(self, targets: Optional[List[str]]) -> None: if targets is None: - targets = self.config.get_list('upload', 'target') + targets = self.config.getlist('upload', 'target') for target in targets: Uploader.run(self.architecture, self.config, target, self.paths.repository) @@ -146,18 +144,19 @@ class Repository: shutil.move(src, dst) package_fn = os.path.join(self.paths.repository, os.path.basename(package)) self.wrapper.add(package_fn) - self.web_report.add(local, BuildStatusEnum.Success) + self.web_report.set_success(local) except Exception: self.logger.exception(f'could not process {package}', exc_info=True) - self.web_report.update(local.base, BuildStatusEnum.Failed) + self.web_report.set_failed(local.base) self._clear_packages() return self.wrapper.repo_path def updates_aur(self, no_vcs: bool) -> List[Package]: result: List[Package] = [] - ignore_list = self.config.get_list( - self.config.get_section_name('build', self.architecture), 'ignore_packages') + + build_section = self.config.get_section_name('build', self.architecture) + ignore_list = self.config.getlist(build_section, 'ignore_packages') for local in self.packages(): if local.base in ignore_list: @@ -169,9 +168,9 @@ class Repository: remote = Package.load(local.base, self.aur_url) if local.is_outdated(remote): result.append(remote) - self.web_report.update(local.base, BuildStatusEnum.Pending) + self.web_report.set_pending(local.base) except Exception: - self.web_report.update(local.base, BuildStatusEnum.Failed) + self.web_report.set_failed(local.base) self.logger.exception(f'could not load remote package {local.base}', exc_info=True) continue @@ -184,7 +183,7 @@ class Repository: try: local = Package.load(os.path.join(self.paths.manual, fn), self.aur_url) result.append(local) - self.web_report.add(local, BuildStatusEnum.Unknown) + self.web_report.set_unknown(local) except Exception: self.logger.exception(f'could not add package from {fn}', exc_info=True) self._clear_manual() diff --git a/src/ahriman/core/sign/gpg_wrapper.py b/src/ahriman/core/sign/gpg_wrapper.py index e0b0d8c1..09c3eee4 100644 --- a/src/ahriman/core/sign/gpg_wrapper.py +++ b/src/ahriman/core/sign/gpg_wrapper.py @@ -30,11 +30,11 @@ from ahriman.models.sign_settings import SignSettings class GPGWrapper: - def __init__(self, config: Configuration) -> None: + def __init__(self, architecture: str, config: Configuration) -> None: self.logger = logging.getLogger('build_details') - - self.target = [SignSettings.from_option(opt) for opt in config.get_list('sign', 'target')] - self.key = config.get('sign', 'key') if self.target else None + section = config.get_section_name('sign', architecture) + self.target = [SignSettings.from_option(opt) for opt in config.getlist(section, 'target')] + self.key = config.get(section, 'key') if self.target else None @property def repository_sign_args(self) -> List[str]: diff --git a/src/ahriman/core/upload/rsync.py b/src/ahriman/core/upload/rsync.py index 88e83d6b..d8808f21 100644 --- a/src/ahriman/core/upload/rsync.py +++ b/src/ahriman/core/upload/rsync.py @@ -26,8 +26,8 @@ class Rsync(Uploader): def __init__(self, architecture: str, config: Configuration) -> None: Uploader.__init__(self, architecture, config) - section = self.config.get_section_name('rsync', self.architecture) - self.remote = self.config.get(section, 'remote') + section = config.get_section_name('rsync', architecture) + self.remote = config.get(section, 'remote') def sync(self, path: str) -> None: check_output('rsync', '--archive', '--verbose', '--compress', '--partial', '--progress', '--delete', path, self.remote, diff --git a/src/ahriman/core/upload/s3.py b/src/ahriman/core/upload/s3.py index 3171f00b..00c6e60c 100644 --- a/src/ahriman/core/upload/s3.py +++ b/src/ahriman/core/upload/s3.py @@ -26,8 +26,8 @@ class S3(Uploader): def __init__(self, architecture: str, config: Configuration) -> None: Uploader.__init__(self, architecture, config) - section = self.config.get_section_name('s3', self.architecture) - self.bucket = self.config.get(section, 'bucket') + section = config.get_section_name('s3', architecture) + self.bucket = config.get(section, 'bucket') def sync(self, path: str) -> None: # TODO rewrite to boto, but it is bullshit diff --git a/src/ahriman/core/upload/uploader.py b/src/ahriman/core/upload/uploader.py index caeeda4e..f4ae6281 100644 --- a/src/ahriman/core/upload/uploader.py +++ b/src/ahriman/core/upload/uploader.py @@ -27,9 +27,9 @@ from ahriman.models.upload_settings import UploadSettings class Uploader: def __init__(self, architecture: str, config: Configuration) -> None: + self.logger = logging.getLogger('builder') self.architecture = architecture self.config = config - self.logger = logging.getLogger('builder') @staticmethod def run(architecture: str, config: Configuration, target: str, path: str) -> None: @@ -45,8 +45,9 @@ class Uploader: try: uploader.sync(path) - except Exception as e: - raise SyncFailed(e) from e + except Exception: + uploader.logger.exception('remote sync failed', exc_info=True) + raise SyncFailed() def sync(self, path: str) -> None: pass diff --git a/src/ahriman/core/watcher/client.py b/src/ahriman/core/watcher/client.py index ff335eb4..cdb11ee6 100644 --- a/src/ahriman/core/watcher/client.py +++ b/src/ahriman/core/watcher/client.py @@ -39,10 +39,26 @@ class Client: def update(self, base: str, status: BuildStatusEnum) -> None: pass + def set_building(self, base: str) -> None: + return self.update(base, BuildStatusEnum.Building) + + def set_failed(self, base: str) -> None: + return self.update(base, BuildStatusEnum.Failed) + + def set_pending(self, base: str) -> None: + return self.update(base, BuildStatusEnum.Pending) + + def set_success(self, package: Package) -> None: + return self.add(package, BuildStatusEnum.Success) + + def set_unknown(self, package: Package) -> None: + return self.add(package, BuildStatusEnum.Unknown) + @staticmethod - def load(config: Configuration) -> Client: - host = config.get('web', 'host', fallback=None) - port = config.getint('web', 'port', fallback=None) + def load(architecture: str, config: Configuration) -> Client: + section = config.get_section_name('web', architecture) + host = config.get(section, 'host', fallback=None) + port = config.getint(section, 'port', fallback=None) if host is None or port is None: return Client() return WebClient(host, port) diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 45fbabd3..1b70e0ca 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -19,10 +19,9 @@ # from __future__ import annotations -import shutil - import aur import os +import shutil import tempfile from dataclasses import dataclass, field diff --git a/src/ahriman/models/repository_paths.py b/src/ahriman/models/repository_paths.py index c5d722a9..02e4e04e 100644 --- a/src/ahriman/models/repository_paths.py +++ b/src/ahriman/models/repository_paths.py @@ -29,22 +29,37 @@ class RepositoryPaths: @property def chroot(self) -> str: + ''' + :return: directory for devtools chroot + ''' return os.path.join(self.root, 'chroot') @property def manual(self) -> str: + ''' + :return: directory for manual updates (i.e. from add command) + ''' return os.path.join(self.root, 'manual') @property def packages(self) -> str: + ''' + :return: directory for built packages + ''' return os.path.join(self.root, 'packages') @property def repository(self) -> str: + ''' + :return: repository directory + ''' return os.path.join(self.root, 'repository', self.architecture) @property def sources(self) -> str: + ''' + :return: directory for downloaded PKGBUILDs for current build + ''' return os.path.join(self.root, 'sources') def create_tree(self) -> None: diff --git a/src/ahriman/web/middlewares/exception_handler.py b/src/ahriman/web/middlewares/exception_handler.py index 7cfb0032..9fa714e1 100644 --- a/src/ahriman/web/middlewares/exception_handler.py +++ b/src/ahriman/web/middlewares/exception_handler.py @@ -18,11 +18,10 @@ # along with this program. If not, see . # from aiohttp.web import middleware, Request, Response +from aiohttp.web_exceptions import HTTPClientError from logging import Logger from typing import Callable -from aiohttp.web_exceptions import HTTPClientError - def exception_handler(logger: Logger) -> Callable: @middleware diff --git a/src/ahriman/web/views/base.py b/src/ahriman/web/views/base.py index dc3fc331..dac671f2 100644 --- a/src/ahriman/web/views/base.py +++ b/src/ahriman/web/views/base.py @@ -22,6 +22,7 @@ from aiohttp.web import View from ahriman.core.watcher.watcher import Watcher +# special class to make it typed class BaseView(View): @property diff --git a/src/ahriman/web/views/index.py b/src/ahriman/web/views/index.py index 0cf903c1..c7f83756 100644 --- a/src/ahriman/web/views/index.py +++ b/src/ahriman/web/views/index.py @@ -17,18 +17,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from typing import Any, Dict - from aiohttp_jinja2 import template +from typing import Any, Dict from ahriman.web.views.base import BaseView class IndexView(BaseView): - @template("index.jinja2") + @template("build-status.jinja2") async def get(self) -> Dict[str, Any]: - # some magic to make it jinja-readable + # some magic to make it jinja-friendly packages = [ { 'base': package.base, diff --git a/src/ahriman/web/web.py b/src/ahriman/web/web.py index 30c3605f..01b23fa8 100644 --- a/src/ahriman/web/web.py +++ b/src/ahriman/web/web.py @@ -38,17 +38,19 @@ async def on_startup(app: web.Application) -> None: app.logger.info('server started') try: app['watcher'].load() - except Exception as e: + except Exception: app.logger.exception('could not load packages', exc_info=True) - raise InitializeException() from e + raise InitializeException() -def run_server(app: web.Application) -> None: +def run_server(app: web.Application, architecture: str) -> None: app.logger.info('start server') - web.run_app(app, - host=app['config'].get('web', 'host'), - port=app['config'].getint('web', 'port'), - handle_signals=False) + + section = app['config'].get_section_name('web', architecture) + host = app['config'].get(section, 'host') + port = app['config'].getint(section, 'port') + + web.run_app(app, host=host, port=port, handle_signals=False) def setup_service(architecture: str, config: Configuration) -> web.Application: