From 77db49a37921e743cfc3e7505d48aed78c6016ee Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Sun, 7 Mar 2021 15:20:41 +0300 Subject: [PATCH] clear code, allow to set overrides for each architecture --- CONFIGURING.md | 14 +++++--- package/archlinux/PKGBUILD | 2 +- package/etc/ahriman.ini | 6 ++-- src/ahriman/application/application.py | 45 ++++++++++++++------------ src/ahriman/core/build_tools/task.py | 10 +++--- src/ahriman/core/configuration.py | 14 ++++++-- src/ahriman/core/report/html.py | 11 ++++--- src/ahriman/core/report/report.py | 9 +++--- src/ahriman/core/repository.py | 9 +++--- src/ahriman/core/upload/rsync.py | 7 ++-- src/ahriman/core/upload/s3.py | 9 +++--- src/ahriman/core/upload/uploader.py | 11 ++++--- src/ahriman/core/util.py | 11 +------ 13 files changed, 85 insertions(+), 73 deletions(-) diff --git a/CONFIGURING.md b/CONFIGURING.md index bb11524b..3588d362 100644 --- a/CONFIGURING.md +++ b/CONFIGURING.md @@ -1,5 +1,7 @@ # ahriman configuration +Some groups can be specified for each architecture separately with default values. E.g. if there are `build` and `build_x86_64` groups it will use the `build_x86_64` for the `x86_64` architecture and `build` for any other. + ## `settings` group Base configuration settings: @@ -42,7 +44,9 @@ Report generation settings: * `target` - list of reports to be generated, space separated list of strings, optional. Allowed values are `html`. -### `html` group +### `html_*` group + +Group name must refer to architecture, e.g. it should be `html_x86_64` for x86_64 architecture. * `path` - path to html report file, string, required. * `css_path` - path to CSS to include in HTML, string, optional. @@ -54,14 +58,14 @@ Remote synchronization settings: * `target` - list of synchronizations to be used, space separated list of strings, optional. Allowed values are `rsync`, `s3`. -### `s3` +### `s3_*` group -Requires `aws-cli` package to be installed. Do not forget to configure it for user `ahriman`. +Group name must refer to architecture, e.g. it should be `s3_x86_64` for x86_64 architecture. Requires `aws-cli` package to be installed. Do not forget to configure it for user `ahriman`. * `bucket` - bucket name (e.g. `s3://bucket/path`), string, required. -### `rsync` +### `rsync_*` group -Requires `rsync` package to be installed. Do not forget to configure ssh for user `ahriman`. +Group name must refer to architecture, e.g. it should be `rsync_x86_64` for x86_64 architecture. Requires `rsync` package to be installed. Do not forget to configure ssh for user `ahriman`. * `remote` - remote server to rsync (e.g. `1.2.3.4:5678:path/to/sync`), string, required. diff --git a/package/archlinux/PKGBUILD b/package/archlinux/PKGBUILD index 589740e3..1b53e124 100644 --- a/package/archlinux/PKGBUILD +++ b/package/archlinux/PKGBUILD @@ -16,7 +16,7 @@ source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$ 'ahriman.sudoers' 'ahriman.sysusers' 'ahriman.tmpfiles') -sha512sums=('392e6f5f0ed9b333896f20ff4fa4f1a2ee1a43efe74eff8b63672419bc31ec2b9e2df100586a1dfd4eca89e53d131187532191c163d3420695e0361c335f3fe3' +sha512sums=('db4409942e3189da01fa41f5432e634c8a3ce4013cd19ef15d63cb1123010702c2f5636fb25ba43e64cdf198b082f8d54dfd448f30288b2fc96e7b30ea213258' '8c9b5b63ac3f7b4d9debaf801a1e9c060877c33d3ecafe18010fcca778e5fa2f2e46909d3d0ff1b229ff8aa978445d8243fd36e1fc104117ed678d5e21901167' '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075' '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4') diff --git a/package/etc/ahriman.ini b/package/etc/ahriman.ini index a1f6e457..284f1942 100644 --- a/package/etc/ahriman.ini +++ b/package/etc/ahriman.ini @@ -22,7 +22,7 @@ key = [report] target = -[html] +[html_x86_64] path = css_path = link_path = @@ -30,8 +30,8 @@ link_path = [upload] target = -[s3] +[s3_x86_64] bucket = -[rsync] +[rsync_x86_64] remote = \ No newline at end of file diff --git a/src/ahriman/application/application.py b/src/ahriman/application/application.py index c1ee82a7..737c02f5 100644 --- a/src/ahriman/application/application.py +++ b/src/ahriman/application/application.py @@ -36,26 +36,7 @@ class Application: self.architecture = architecture self.repository = Repository(architecture, config) - def add(self, names: List[str]) -> None: - for name in names: - package = Package.load(name, self.config.get('aur', 'url')) - task = Task(package, self.architecture, self.config, self.repository.paths) - task.fetch(os.path.join(self.repository.paths.manual, package.name)) - - def remove(self, names: List[str]) -> None: - self.repository.process_remove(names) - - def report(self, target: Optional[List[str]] = None) -> None: - targets = target or None - self.repository.process_report(targets) - - def sync(self, target: Optional[List[str]] = None) -> None: - targets = target or None - self.repository.process_sync(targets) - - def update(self, updates: List[Package]) -> None: - packages = self.repository.process_build(updates) - self.repository.process_update(packages) + def _finalize(self) -> None: self.report() self.sync() @@ -72,3 +53,27 @@ class Application: log_fn(f'{package.name} = {package.version}') return updates + + def add(self, names: List[str]) -> None: + for name in names: + package = Package.load(name, self.config.get('aur', 'url')) + task = Task(package, self.architecture, self.config, self.repository.paths) + task.fetch(os.path.join(self.repository.paths.manual, package.name)) + + def remove(self, names: List[str]) -> None: + self.repository.process_remove(names) + self._finalize() + + def report(self, target: Optional[List[str]] = None) -> None: + targets = target or None + self.repository.process_report(targets) + + def sync(self, target: Optional[List[str]] = None) -> None: + targets = target or None + self.repository.process_sync(targets) + + def update(self, updates: List[Package]) -> None: + packages = self.repository.process_build(updates) + self.repository.process_update(packages) + self._finalize() + diff --git a/src/ahriman/core/build_tools/task.py b/src/ahriman/core/build_tools/task.py index d24909e2..ade6bfa7 100644 --- a/src/ahriman/core/build_tools/task.py +++ b/src/ahriman/core/build_tools/task.py @@ -25,7 +25,7 @@ from typing import List, Optional from ahriman.core.configuration import Configuration from ahriman.core.exceptions import BuildFailed -from ahriman.core.util import check_output, options_list +from ahriman.core.util import check_output from ahriman.models.package import Package from ahriman.models.repository_paths import RepositoryPaths @@ -38,11 +38,11 @@ class Task: self.package = package self.paths = paths - section = f'build_{architecture}' - self.archbuild_flags = options_list(config, section, 'archbuild_flags') + section = config.get_section_name('build', architecture) + self.archbuild_flags = config.get_list(section, 'archbuild_flags') self.build_command = config.get(section, 'build_command') - self.makepkg_flags = options_list(config, section, 'makepkg_flags') - self.makechrootpkg_flags = options_list(config, section, 'makechrootpkg_flags') + self.makepkg_flags = config.get_list(section, 'makepkg_flags') + self.makechrootpkg_flags = config.get_list(section, 'makechrootpkg_flags') @property def git_path(self) -> str: diff --git a/src/ahriman/core/configuration.py b/src/ahriman/core/configuration.py index ccd62831..b23016ae 100644 --- a/src/ahriman/core/configuration.py +++ b/src/ahriman/core/configuration.py @@ -21,9 +21,7 @@ import configparser import os from logging.config import fileConfig -from typing import Dict, Optional - -from ahriman.core.exceptions import MissingConfiguration +from typing import List, Optional # built-in configparser extension @@ -37,6 +35,16 @@ class Configuration(configparser.RawConfigParser): def include(self) -> str: return self.get('settings', 'include') + def get_list(self, section: str, key: str) -> List[str]: + raw = self.get(section, key, fallback=None) + if not raw: # empty string or none + return [] + return raw.split() + + def get_section_name(self, prefix: str, suffix: str) -> str: + probe = f'{prefix}_{suffix}' + return probe if self.has_section(probe) else prefix + def load(self, path: str) -> None: self.path = path self.read(self.path) diff --git a/src/ahriman/core/report/html.py b/src/ahriman/core/report/html.py index c8424e0b..8171f6a4 100644 --- a/src/ahriman/core/report/html.py +++ b/src/ahriman/core/report/html.py @@ -25,11 +25,12 @@ from ahriman.core.report.report import Report class HTML(Report): - def __init__(self, config: Configuration) -> None: - Report.__init__(self, config) - self.report_path = config.get('html', 'path') - self.css_path = config.get('html', 'css_path') - self.link_path = config.get('html', 'link_path') + def __init__(self, architecture: str, config: Configuration) -> None: + Report.__init__(self, architecture, config) + section = self.config.get_section_name('html', self.architecture) + self.report_path = config.get(section, 'path') + self.css_path = config.get(section, 'css_path') + self.link_path = config.get(section, 'link_path') self.title = config.get('repository', 'name') def generate(self, path: str) -> None: diff --git a/src/ahriman/core/report/report.py b/src/ahriman/core/report/report.py index 35b4123a..07ffdace 100644 --- a/src/ahriman/core/report/report.py +++ b/src/ahriman/core/report/report.py @@ -26,19 +26,20 @@ from ahriman.models.report_settings import ReportSettings class Report: - def __init__(self, config: Configuration) -> None: + def __init__(self, architecture: str, config: Configuration) -> None: + self.architecture = architecture self.config = config self.logger = logging.getLogger('builder') @staticmethod - def run(config: Configuration, target: str, path: str) -> None: + def run(architecture: str, config: Configuration, target: str, path: str) -> None: provider = ReportSettings.from_option(target) if provider == ReportSettings.HTML: from ahriman.core.report.html import HTML - report: Report = HTML(config) + report: Report = HTML(architecture, config) else: from ahriman.core.report.dummy import Dummy - report = Dummy(config) + report = Dummy(architecture, config) try: report.generate(path) diff --git a/src/ahriman/core/repository.py b/src/ahriman/core/repository.py index 319967cd..aa92d101 100644 --- a/src/ahriman/core/repository.py +++ b/src/ahriman/core/repository.py @@ -29,7 +29,6 @@ from ahriman.core.repo.repo_wrapper import RepoWrapper from ahriman.core.report.report import Report from ahriman.core.sign.gpg_wrapper import GPGWrapper from ahriman.core.upload.uploader import Uploader -from ahriman.core.util import options_list from ahriman.models.package import Package from ahriman.models.repository_paths import RepositoryPaths @@ -120,15 +119,15 @@ class Repository: def process_report(self, targets: Optional[List[str]]) -> None: if targets is None: - targets = options_list(self.config, 'report', 'target') + targets = self.config.get_list('report', 'target') for target in targets: - Report.run(self.config, target, self.paths.repository) + Report.run(self.architecture, self.config, target, self.paths.repository) def process_sync(self, targets: Optional[List[str]]) -> None: if targets is None: - targets = options_list(self.config, 'upload', 'target') + targets = self.config.get_list('upload', 'target') for target in targets: - Uploader.run(self.config, target, self.paths.repository) + Uploader.run(self.architecture, self.config, target, self.paths.repository) def process_update(self, packages: List[str]) -> str: for package in packages: diff --git a/src/ahriman/core/upload/rsync.py b/src/ahriman/core/upload/rsync.py index 95c068cd..7cb18703 100644 --- a/src/ahriman/core/upload/rsync.py +++ b/src/ahriman/core/upload/rsync.py @@ -24,9 +24,10 @@ from ahriman.core.util import check_output class Rsync(Uploader): - def __init__(self, config: Configuration) -> None: - Uploader.__init__(self, config) - self.remote = self.config.get('rsync', 'remote') + def __init__(self, architecture: str, config: Configuration) -> None: + Uploader.__init__(self, architecture, config) + section = self.config.get_section_name('rsync', self.architecture) + self.remote = self.config.get(section, 'remote') def sync(self, path: str) -> None: check_output('rsync', '--archive', '--verbose', '--compress', '--partial', '--progress', '--delete', path, self.remote, diff --git a/src/ahriman/core/upload/s3.py b/src/ahriman/core/upload/s3.py index 2c3abf2e..4c449453 100644 --- a/src/ahriman/core/upload/s3.py +++ b/src/ahriman/core/upload/s3.py @@ -24,12 +24,13 @@ from ahriman.core.util import check_output class S3(Uploader): - def __init__(self, config: Configuration) -> None: - Uploader.__init__(self, config) - self.bucket = self.config.get('s3', 'bucket') + def __init__(self, architecture: str, config: Configuration) -> None: + Uploader.__init__(self, architecture, config) + section = self.config.get_section_name('s3', self.architecture) + self.bucket = self.config.get(section, 'bucket') def sync(self, path: str) -> None: # TODO rewrite to boto, but it is bullshit - check_output('aws', 's3', 'sync', path, self.bucket, + check_output('aws', 's3', 'sync', '--delete', path, self.bucket, exception=None, logger=self.logger) diff --git a/src/ahriman/core/upload/uploader.py b/src/ahriman/core/upload/uploader.py index e29d4c6f..bb4e50ab 100644 --- a/src/ahriman/core/upload/uploader.py +++ b/src/ahriman/core/upload/uploader.py @@ -26,22 +26,23 @@ from ahriman.models.upload_settings import UploadSettings class Uploader: - def __init__(self, config: Configuration) -> None: + def __init__(self, architecture: str, config: Configuration) -> None: + self.architecture = architecture self.config = config self.logger = logging.getLogger('builder') @staticmethod - def run(config: Configuration, target: str, path: str) -> None: + def run(architecture: str, config: Configuration, target: str, path: str) -> None: provider = UploadSettings.from_option(target) if provider == UploadSettings.Rsync: from ahriman.core.upload.rsync import Rsync - uploader: Uploader = Rsync(config) + uploader: Uploader = Rsync(architecture, config) elif provider == UploadSettings.S3: from ahriman.core.upload.s3 import S3 - uploader = S3(config) + uploader = S3(architecture, config) else: from ahriman.core.upload.dummy import Dummy - uploader = Dummy(config) + uploader = Dummy(architecture, config) try: uploader.sync(path) diff --git a/src/ahriman/core/util.py b/src/ahriman/core/util.py index efe21c2e..c85e593d 100644 --- a/src/ahriman/core/util.py +++ b/src/ahriman/core/util.py @@ -20,9 +20,7 @@ import subprocess from logging import Logger -from typing import List, Optional - -from ahriman.core.configuration import Configuration +from typing import Optional def check_output(*args: str, exception: Optional[Exception], @@ -39,10 +37,3 @@ def check_output(*args: str, exception: Optional[Exception], logger.debug(line) raise exception or e return result - - -def options_list(config: Configuration, section: str, key: str) -> List[str]: - raw = config.get(section, key, fallback=None) - if not raw: # empty string or none - return [] - return raw.split() \ No newline at end of file