From 088698d8e172e87d07834b342201b7370b6c4a32 Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Sun, 7 Mar 2021 00:04:25 +0300 Subject: [PATCH] report support, readme --- CONFIGURING.md | 67 +++++++++++++++++++ README.md | 20 ++++++ make_release.sh | 2 +- package/etc/ahriman.ini | 18 +++-- src/ahriman/application/ahriman.py | 20 ++++-- src/ahriman/application/application.py | 27 ++++++-- src/ahriman/core/build_tools/task.py | 18 +++-- src/ahriman/core/exceptions.py | 5 ++ src/ahriman/core/report/dummy.py | 26 +++++++ src/ahriman/core/report/html.py | 52 ++++++++++++++ src/ahriman/core/report/report.py | 49 ++++++++++++++ src/ahriman/core/repository.py | 21 ++++-- .../core/sign/{sign.py => gpg_wrapper.py} | 2 +- src/ahriman/core/upload/s3.py | 2 +- src/ahriman/core/upload/uploader.py | 6 +- src/ahriman/core/util.py | 13 +++- src/ahriman/models/package.py | 4 -- src/ahriman/models/report_settings.py | 35 ++++++++++ src/ahriman/models/upload_settings.py | 5 +- 19 files changed, 346 insertions(+), 46 deletions(-) create mode 100644 CONFIGURING.md create mode 100644 src/ahriman/core/report/dummy.py create mode 100644 src/ahriman/core/report/html.py create mode 100644 src/ahriman/core/report/report.py rename src/ahriman/core/sign/{sign.py => gpg_wrapper.py} (99%) create mode 100644 src/ahriman/models/report_settings.py diff --git a/CONFIGURING.md b/CONFIGURING.md new file mode 100644 index 00000000..23feb4dc --- /dev/null +++ b/CONFIGURING.md @@ -0,0 +1,67 @@ +# ahriman configuration + +## `settings` group + +Base configuration settings: + +* `include` - path to directory with configuration files overrides, string, required. +* `logging` - path to logging configuration, string, required. Check `logging.ini` for reference. + +## `aur` group + +AUR related configuration: + +* `url` - base url for AUR, string, required. + +## `build` group + +Build related configuration: + +* `archbuild_flags` - additional flags passed to `archbuild` command, space separated list of strings, optional. +* `build_command` - default build command, string, required. +* `makepkg_flags` - additional flags passed to `makepkg` command, space separated list of strings, optional. +* `makechrootpkg_flags` - additional flags passed to `makechrootpkg` command, space separated list of strings, optional. + +## `repository` group + +Base repository settings: + +* `name` - repository name, string, required. +* `root` - root path for application, string, required. + +## `sign` group + +Settings for signing packages or repository: + +* `enabled` - configuration flag to enable signing, string, required. Allowed values are `disabled`, `package` (sign each package separately), `repository` (sign repository database file). +* `key` - PGP key, string, optional. + +## `report` group + +Report generation settings: + +* `target` - list of reports to be generated, space separated list of strings, optional. Allowed values are `html`. + +### `html` group + +* `path` - path to html report file, string, required. +* `css_path` - path to CSS to include in HTML, string, optional. +* `link_path` - prefix for HTML links, string, required. + +## `upload` group + +Remote synchronization settings: + +* `target` - list of synchronizations to be used, space separated list of strings, optional. Allowed values are `rsync`, `s3`. + +### `s3` + +Requires `aws-cli` package to be installed. Do not forget to configure it for user `ahriman`. + +* `bucket` - bucket name (e.g. `s3://bucket/path`), string, required. + +### `rsync` + +Requires `rsync` package to be installed. Do not forget to configure ssh for user `ahriman`. + +* `remote` - remote server to rsync (e.g. `1.2.3.4:5678:path/to/sync`), string, required. diff --git a/README.md b/README.md index e69de29b..3eb5b6be 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,20 @@ +# ArcHlinux ReposItory MANager + +Wrapper for managing custom repository inspired by [repo-scripts](https://github.com/arcan1s/repo-scripts). + +## Installation and run + +* Install package as usual. +* Change settings if required, see `CONFIGURING.md` for more details. +* Create `/var/lib/ahriman/.makepkg.conf` with `makepkg.conf` overrides if required (at least you might want to set `PACKAGER`). +* Configure build tools (it might be required if your package will use any custom repositories): + * create build command if required, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/custom-x86_64-build` (you can choose any name for command); + * create configuration file, e.g. `cp /usr/share/devtools/pacman-{extra,custom}.conf`; + * change configuration file: add your own repository, add multilib repository; + * set `build.build_command` to point to your command. +* Start and enable `ahriman.timer` via `systemctl`. +* Add packages by using `ahriman add {package}` command. + +## Limitations + +* It does not manage dependencies, so you have to add them before main package. \ No newline at end of file diff --git a/make_release.sh b/make_release.sh index ed60554b..efe4833c 100755 --- a/make_release.sh +++ b/make_release.sh @@ -4,7 +4,7 @@ set -e VERSION="$1" ARCHIVE="ahriman" -FILES="COPYING README.md package src setup.py" +FILES="COPYING CONFIGURING.md README.md package src setup.py" IGNORELIST="build .idea package/archlinux package/*src.tar.xz" # set version diff --git a/package/etc/ahriman.ini b/package/etc/ahriman.ini index 5750fb0d..9583bbc8 100644 --- a/package/etc/ahriman.ini +++ b/package/etc/ahriman.ini @@ -6,10 +6,10 @@ logging = /etc/ahriman.ini.d/logging.ini url = https://aur.archlinux.org [build] -archbuild_flags = -c -extra_build = extra-x86_64-build +archbuild_flags = +build_command = extra-x86_64-build +makechrootpkg_flags = makepkg_flags = --skippgpcheck -multilib_build = multilib-build [repository] name = aur-clone @@ -19,11 +19,19 @@ root = /var/lib/ahriman enabled = disabled key = +[report] +target = + +[html] +path = +css_path = +link_path = + [upload] -enabled = disabled +target = [s3] bucket = [rsync] -remote = +remote = \ No newline at end of file diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index d81486b8..754022bc 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -51,12 +51,17 @@ def remove(args: argparse.Namespace) -> None: _get_app(args).remove(args.package) +def report(args: argparse.Namespace) -> None: + _get_app(args).report(args.target) + + def sync(args: argparse.Namespace) -> None: - _get_app(args).sync() + _get_app(args).sync(args.target) def update(args: argparse.Namespace) -> None: - _get_app(args).update(args.sync) + check_only = (args.command == 'check') + _get_app(args).update(check_only) if __name__ == '__main__': @@ -65,21 +70,28 @@ if __name__ == '__main__': parser.add_argument('--force', help='force run, remove file lock', action='store_true') parser.add_argument('--lock', help='lock file', default='/tmp/ahriman.lock') parser.add_argument('-v', '--version', action='version', version=version.__version__) - subparsers = parser.add_subparsers(title='commands') + subparsers = parser.add_subparsers(title='command') add_parser = subparsers.add_parser('add', description='add package') add_parser.add_argument('package', help='package name', nargs='+') add_parser.set_defaults(fn=add) + check_parser = subparsers.add_parser('check', description='check for updates') + check_parser.set_defaults(fn=update) + remove_parser = subparsers.add_parser('remove', description='remove package') remove_parser.add_argument('package', help='package name', nargs='+') remove_parser.set_defaults(fn=remove) + report_parser = subparsers.add_parser('report', description='generate report') + report_parser.add_argument('target', help='target to generate report', nargs='*') + report_parser.set_defaults(fn=report) + sync_parser = subparsers.add_parser('sync', description='sync packages to remote server') + sync_parser.add_argument('target', help='target to sync', nargs='*') sync_parser.set_defaults(fn=sync) update_parser = subparsers.add_parser('update', description='run updates') - update_parser.add_argument('-s', '--sync', help='sync packages to remote server', action='store_true') update_parser.set_defaults(fn=update) args = parser.parse_args() diff --git a/src/ahriman/application/application.py b/src/ahriman/application/application.py index 7323d8b1..f3920e4f 100644 --- a/src/ahriman/application/application.py +++ b/src/ahriman/application/application.py @@ -17,9 +17,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +import logging import os -from typing import List +from typing import List, Optional from ahriman.core.build_tools.task import Task from ahriman.core.configuration import Configuration @@ -30,6 +31,7 @@ from ahriman.models.package import Package class Application: def __init__(self, config: Configuration) -> None: + self.logger = logging.getLogger('root') self.config = config self.repository = Repository(config) @@ -42,12 +44,25 @@ class Application: def remove(self, names: List[str]) -> None: self.repository.process_remove(names) - def sync(self) -> None: - self.repository.process_sync() + def report(self, target: Optional[List[str]] = None) -> None: + targets = target or None + self.repository.process_report(targets) - def update(self, sync: bool) -> None: + def sync(self, target: Optional[List[str]] = None) -> None: + targets = target or None + self.repository.process_sync(targets) + + def update(self, dry_run: bool) -> None: updates = self.repository.updates() + log_fn = print if dry_run else self.logger.info + for package in updates: + log_fn(f'{package.name} = {package.version}') # type: ignore + + if dry_run: + return + packages = self.repository.process_build(updates) self.repository.process_update(packages) - if sync: - self.sync() \ No newline at end of file + + self.report() + self.sync() diff --git a/src/ahriman/core/build_tools/task.py b/src/ahriman/core/build_tools/task.py index fd4ffc27..4abdb6ec 100644 --- a/src/ahriman/core/build_tools/task.py +++ b/src/ahriman/core/build_tools/task.py @@ -25,7 +25,7 @@ from typing import List, Optional from ahriman.core.configuration import Configuration from ahriman.core.exceptions import BuildFailed -from ahriman.core.util import check_output +from ahriman.core.util import check_output, options_list from ahriman.models.package import Package from ahriman.models.repository_paths import RepositoryPaths @@ -38,22 +38,20 @@ class Task: self.package = package self.paths = paths - self.archbuild_flags = config.get('build_tools', 'archbuild_flags').split() - self.extra_build = config.get('build_tools', 'extra_build') - self.makepkg_flags = config.get('build_tools', 'makepkg_flags').split() - self.multilib_build = config.get('build_tools', 'multilib_build') + self.archbuild_flags = options_list(config, 'build', 'archbuild_flags') + self.build_command = config.get('build', 'build_command') + self.makepkg_flags = options_list(config, 'build', 'makepkg_flags') + self.makechrootpkg_flags = options_list(config, 'build', 'makechrootpkg_flags') @property def git_path(self) -> str: return os.path.join(self.paths.sources, self.package.name) def build(self) -> List[str]: - build_tool = self.multilib_build if self.package.is_multilib else self.extra_build - - cmd = [build_tool, '-r', self.paths.chroot] + cmd = [self.build_command, '-r', self.paths.chroot] cmd.extend(self.archbuild_flags) - if self.makepkg_flags: - cmd.extend(['--', '--'] + self.makepkg_flags) + cmd.extend(['--'] + self.makechrootpkg_flags) + cmd.extend(['--'] + self.makepkg_flags) self.logger.info(f'using {cmd} for {self.package.name}') check_output( diff --git a/src/ahriman/core/exceptions.py b/src/ahriman/core/exceptions.py index 4333a9f9..6c50f92b 100644 --- a/src/ahriman/core/exceptions.py +++ b/src/ahriman/core/exceptions.py @@ -40,6 +40,11 @@ class MissingConfiguration(Exception): Exception.__init__(self, f'No section `{name}` found') +class ReportFailed(Exception): + def __init__(self, cause: Exception) -> None: + Exception.__init__(self, f'Report failed with reason {cause}') + + class SyncFailed(Exception): def __init__(self, cause: Exception) -> None: Exception.__init__(self, f'Sync failed with reason {cause}') \ No newline at end of file diff --git a/src/ahriman/core/report/dummy.py b/src/ahriman/core/report/dummy.py new file mode 100644 index 00000000..1b7c4733 --- /dev/null +++ b/src/ahriman/core/report/dummy.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from ahriman.core.report.report import Report + + +class Dummy(Report): + + def generate(self, path: str) -> None: + pass diff --git a/src/ahriman/core/report/html.py b/src/ahriman/core/report/html.py new file mode 100644 index 00000000..c8424e0b --- /dev/null +++ b/src/ahriman/core/report/html.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import os + +from ahriman.core.configuration import Configuration +from ahriman.core.report.report import Report + + +class HTML(Report): + + def __init__(self, config: Configuration) -> None: + Report.__init__(self, config) + self.report_path = config.get('html', 'path') + self.css_path = config.get('html', 'css_path') + self.link_path = config.get('html', 'link_path') + self.title = config.get('repository', 'name') + + def generate(self, path: str) -> None: + # lets not use libraries here + html = f'''{self.title}''' + if self.css_path: + html += f'''''' + html += '''''' + + html += '''
    ''' + for package in sorted(os.listdir(path)): + if '.pkg.' not in package: + continue + html += f'''
  • {package}
    • ''' + html += '''
''' + + html += '''''' + + with open(self.report_path, 'w') as out: + out.write(html) diff --git a/src/ahriman/core/report/report.py b/src/ahriman/core/report/report.py new file mode 100644 index 00000000..35b4123a --- /dev/null +++ b/src/ahriman/core/report/report.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import logging + +from ahriman.core.configuration import Configuration +from ahriman.core.exceptions import ReportFailed +from ahriman.models.report_settings import ReportSettings + + +class Report: + + def __init__(self, config: Configuration) -> None: + self.config = config + self.logger = logging.getLogger('builder') + + @staticmethod + def run(config: Configuration, target: str, path: str) -> None: + provider = ReportSettings.from_option(target) + if provider == ReportSettings.HTML: + from ahriman.core.report.html import HTML + report: Report = HTML(config) + else: + from ahriman.core.report.dummy import Dummy + report = Dummy(config) + + try: + report.generate(path) + except Exception as e: + raise ReportFailed(e) from e + + def generate(self, path: str) -> None: + raise NotImplementedError \ No newline at end of file diff --git a/src/ahriman/core/repository.py b/src/ahriman/core/repository.py index e949db6b..19129a5e 100644 --- a/src/ahriman/core/repository.py +++ b/src/ahriman/core/repository.py @@ -21,13 +21,15 @@ import logging import os import shutil -from typing import List +from typing import List, Optional from ahriman.core.build_tools.task import Task from ahriman.core.configuration import Configuration from ahriman.core.repo.repo_wrapper import RepoWrapper -from ahriman.core.sign.sign import Sign +from ahriman.core.report.report import Report +from ahriman.core.sign.gpg_wrapper import GPGWrapper from ahriman.core.upload.uploader import Uploader +from ahriman.core.util import options_list from ahriman.models.package import Package from ahriman.models.repository_paths import RepositoryPaths @@ -44,7 +46,7 @@ class Repository: self.paths = RepositoryPaths(config.get('repository', 'root')) self.paths.create_tree() - self.sign = Sign(config) + self.sign = GPGWrapper(config) self.wrapper = RepoWrapper(self.name, self.paths) def _clear_build(self) -> None: @@ -96,8 +98,17 @@ class Repository: self.sign.sign_repository(self.wrapper.repo_path) return self.wrapper.repo_path - def process_sync(self) -> None: - return Uploader.run(self.config, self.paths.repository) + def process_report(self, targets: Optional[List[str]]) -> None: + if targets is None: + targets = options_list(self.config, 'report', 'target') + for target in targets: + Report.run(self.config, target, self.paths.repository) + + def process_sync(self, targets: Optional[List[str]]) -> None: + if targets is None: + targets = options_list(self.config, 'upload', 'target') + for target in targets: + Uploader.run(self.config, target, self.paths.repository) def process_update(self, packages: List[str]) -> str: for package in packages: diff --git a/src/ahriman/core/sign/sign.py b/src/ahriman/core/sign/gpg_wrapper.py similarity index 99% rename from src/ahriman/core/sign/sign.py rename to src/ahriman/core/sign/gpg_wrapper.py index e37240b2..552c4e3a 100644 --- a/src/ahriman/core/sign/sign.py +++ b/src/ahriman/core/sign/gpg_wrapper.py @@ -28,7 +28,7 @@ from ahriman.core.util import check_output from ahriman.models.sign_settings import SignSettings -class Sign: +class GPGWrapper: def __init__(self, config: Configuration) -> None: self.logger = logging.getLogger('build_details') diff --git a/src/ahriman/core/upload/s3.py b/src/ahriman/core/upload/s3.py index f975cf32..2c3abf2e 100644 --- a/src/ahriman/core/upload/s3.py +++ b/src/ahriman/core/upload/s3.py @@ -29,7 +29,7 @@ class S3(Uploader): self.bucket = self.config.get('s3', 'bucket') def sync(self, path: str) -> None: - # TODO rewrite to boto, but it is bs + # TODO rewrite to boto, but it is bullshit check_output('aws', 's3', 'sync', path, self.bucket, exception=None, logger=self.logger) diff --git a/src/ahriman/core/upload/uploader.py b/src/ahriman/core/upload/uploader.py index f270c56c..e29d4c6f 100644 --- a/src/ahriman/core/upload/uploader.py +++ b/src/ahriman/core/upload/uploader.py @@ -31,8 +31,8 @@ class Uploader: self.logger = logging.getLogger('builder') @staticmethod - def run(config: Configuration, path: str) -> None: - provider = UploadSettings.from_option(config.get('upload', 'enabled')) + def run(config: Configuration, target: str, path: str) -> None: + provider = UploadSettings.from_option(target) if provider == UploadSettings.Rsync: from ahriman.core.upload.rsync import Rsync uploader: Uploader = Rsync(config) @@ -46,7 +46,7 @@ class Uploader: try: uploader.sync(path) except Exception as e: - raise SyncFailed from e + raise SyncFailed(e) from e def sync(self, path: str) -> None: raise NotImplementedError diff --git a/src/ahriman/core/util.py b/src/ahriman/core/util.py index 80c2ec29..efe21c2e 100644 --- a/src/ahriman/core/util.py +++ b/src/ahriman/core/util.py @@ -20,7 +20,9 @@ import subprocess from logging import Logger -from typing import Optional +from typing import List, Optional + +from ahriman.core.configuration import Configuration def check_output(*args: str, exception: Optional[Exception], @@ -36,4 +38,11 @@ def check_output(*args: str, exception: Optional[Exception], for line in e.output.decode('utf8').splitlines(): logger.debug(line) raise exception or e - return result \ No newline at end of file + return result + + +def options_list(config: Configuration, section: str, key: str) -> List[str]: + raw = config.get(section, key, fallback=None) + if not raw: # empty string or none + return [] + return raw.split() \ No newline at end of file diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 1dfdcbae..e62d0bf8 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -37,10 +37,6 @@ class Package: version: str url: str - @property - def is_multilib(self) -> bool: - return self.name.startswith('lib32-') - @classmethod def from_archive(cls: Type[Package], path: str, aur_url: str) -> Package: name, version = check_output('expac', '-p', '%n %v', path, exception=None).split() diff --git a/src/ahriman/models/report_settings.py b/src/ahriman/models/report_settings.py new file mode 100644 index 00000000..9af1090e --- /dev/null +++ b/src/ahriman/models/report_settings.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from __future__ import annotations + +from enum import Enum, auto +from typing import Type + +from ahriman.core.exceptions import InvalidOptionException + + +class ReportSettings(Enum): + HTML = auto() + + @classmethod + def from_option(cls: Type[ReportSettings], value: str) -> ReportSettings: + if value.lower() in ('html',): + return cls.HTML + raise InvalidOptionException(value) diff --git a/src/ahriman/models/upload_settings.py b/src/ahriman/models/upload_settings.py index 605deb5a..cff0b4cc 100644 --- a/src/ahriman/models/upload_settings.py +++ b/src/ahriman/models/upload_settings.py @@ -26,15 +26,12 @@ from ahriman.core.exceptions import InvalidOptionException class UploadSettings(Enum): - Disabled = auto() Rsync = auto() S3 = auto() @classmethod def from_option(cls: Type[UploadSettings], value: str) -> UploadSettings: - if value.lower() in ('no', 'disabled'): - return cls.Disabled - elif value.lower() in ('rsync',): + if value.lower() in ('rsync',): return cls.Rsync elif value.lower() in ('s3',): return cls.S3