diff --git a/README.md b/README.md index a4a2b562..e5c6e5c9 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,54 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github ## Installation and run * Install package as usual. -* Change settings if required, see `CONFIGURING.md` for more details. -* Create `/var/lib/ahriman/.makepkg.conf` with `makepkg.conf` overrides if required (at least you might want to set `PACKAGER`). -* Configure build tools (it might be required if your package will use any custom repositories): - * create build command, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/custom-x86_64-build` (you can choose any name for command); - * create configuration file, e.g. `cp /usr/share/devtools/pacman-{extra,custom}.conf`; - * change configuration file, add your own repository, add multilib repository etc; - * set `build.build_command` setting to point to your command; - * configure `/etc/sudoers.d/ahriman` to allow running command without password. -* Start and enable `ahriman.timer` via `systemctl`. -* Add packages by using `ahriman add {package}` command. +* Change settings if required, see [CONFIGURING](CONFIGURING.md) for more details. +* Create `/var/lib/ahriman/.makepkg.conf` with `makepkg.conf` overrides if required (at least you might want to set `PACKAGER`): + + ```shell + echo 'PACKAGES="John Doe "' | sudo -u ahriman tee -a /var/lib/ahriman/.makepkg.conf + ``` + +* Configure build tools (it is required for correct dependency management system): + + * create build command, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build` (you can choose any name for command, basically it should be `{name}-{arch}-build`); + * create configuration file, e.g. `cp /usr/share/devtools/pacman-{extra,ahriman}.conf` (same as previous `pacman-{name}.conf`); + * change configuration file, add your own repository, add multilib repository etc. Hint: you can use `Include` option as well; + * set `build_command` option to point to your command; + * configure `/etc/sudoers.d/ahriman` to allow running command without a password. + + ```shell + ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build + cp /usr/share/devtools/pacman-{extra,ahriman}.conf + + echo '[multilib]' | tee -a /usr/share/devtools/pacman-ahriman.conf + echo 'Include = /etc/pacman.d/mirrorlist' | tee -a /usr/share/devtools/pacman-ahriman.conf + + echo '[aur-clone]' | tee -a /usr/share/devtools/pacman-ahriman.conf + echo 'SigLevel = Optional TrustAll' | tee -a /usr/share/devtools/pacman-ahriman.conf + echo 'Server = file:///var/lib/ahriman/repository/$arch' | tee -a /usr/share/devtools/pacman-ahriman.conf + + echo '[build]' | tee -a /etc/ahriman.ini.d/build.ini + echo 'build_command = ahriman-x86_64-build' | tee -a /etc/ahriman.ini.d/build.ini + + echo 'Cmnd_Alias CARCHBUILD_CMD = /usr/local/bin/ahriman-x86_64-build *' | tee -a /etc/sudoers.d/ahriman + echo 'ahriman ALL=(ALL) NOPASSWD: CARCHBUILD_CMD' | tee -a /etc/sudoers.d/ahriman + chmod 400 /etc/sudoers.d/ahriman + ``` + +* Start and enable `ahriman@.timer` via `systemctl`: + + ```shell + systemctl enable --now ahriman@x86_64.timer + ``` + +* Start and enable status page: + + ```shell + systemctl enable --now ahriman-web@x86_64 + ``` + +* Add packages by using `ahriman add {package}` command: + + ```shell + sudo -u ahriman ahriman -a x86_64 add yay + ``` diff --git a/package/archlinux/PKGBUILD b/package/archlinux/PKGBUILD index 93995ebb..3894827b 100644 --- a/package/archlinux/PKGBUILD +++ b/package/archlinux/PKGBUILD @@ -23,7 +23,7 @@ optdepends=('aws-cli: sync to s3' source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz" 'ahriman.sysusers' 'ahriman.tmpfiles') -sha512sums=('b835d745fb77e400ca31ba4d93547b7db8e9dfe5d6c04b60e3953efeeaa7f561a1c60b2ade2684d3c7ba9a87e470c65610f33340315f192661c1676746b91298' +sha512sums=('d7c4c0808eef7a1ebd7a137777e4855eebd4666304e18ea6ece9499f1ed8cf459299561d9ec4917f6c454e5fff7eee0b97e1d6efcc80be7308aac75141584cd5' '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075' '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4') backup=('etc/ahriman.ini' diff --git a/package/etc/ahriman.ini.d/logging.ini b/package/etc/ahriman.ini.d/logging.ini index dd998588..476a39b3 100644 --- a/package/etc/ahriman.ini.d/logging.ini +++ b/package/etc/ahriman.ini.d/logging.ini @@ -2,17 +2,11 @@ keys = root,builder,build_details,http [handlers] -keys = console_handler,build_file_handler,file_handler,http_handler +keys = build_file_handler,file_handler,http_handler [formatters] keys = generic_format -[handler_console_handler] -class = StreamHandler -level = DEBUG -formatter = generic_format -args = (sys.stdout,) - [handler_file_handler] class = logging.handlers.RotatingFileHandler level = DEBUG diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 943d56c9..c440722c 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -18,6 +18,8 @@ # along with this program. If not, see . # import argparse +import logging +import sys from multiprocessing import Pool @@ -28,15 +30,21 @@ from ahriman.application.lock import Lock from ahriman.core.configuration import Configuration -def _call(args: argparse.Namespace, architecture: str, config: Configuration) -> None: +def _call(args: argparse.Namespace, architecture: str, config: Configuration) -> bool: ''' additional function to wrap all calls for multiprocessing library :param args: command line args :param architecture: repository architecture :param config: configuration instance + :return: True on success, False otherwise ''' - with Lock(args.lock, architecture, args.force, config): - args.fn(args, architecture, config) + try: + with Lock(args.lock, architecture, args.force, args.unsafe, config): + args.fn(args, architecture, config) + return True + except Exception: + logging.getLogger('root').exception('process exception', exc_info=True) + return False def add(args: argparse.Namespace, architecture: str, config: Configuration) -> None: @@ -56,7 +64,8 @@ def clean(args: argparse.Namespace, architecture: str, config: Configuration) -> :param architecture: repository architecture :param config: configuration instance ''' - Application(architecture, config).clean() + Application(architecture, config).clean(args.no_build, args.no_cache, args.no_chroot, + args.no_manual, args.no_packages) def rebuild(args: argparse.Namespace, architecture: str, config: Configuration) -> None: @@ -134,10 +143,15 @@ def web(args: argparse.Namespace, architecture: str, config: Configuration) -> N if __name__ == '__main__': parser = argparse.ArgumentParser(prog='ahriman', description='ArcHlinux ReposItory MANager') - parser.add_argument('-a', '--architecture', help='target architectures', action='append') + parser.add_argument( + '-a', + '--architecture', + help='target architectures (can be used multiple times)', + action='append') parser.add_argument('-c', '--config', help='configuration path', default='/etc/ahriman.ini') parser.add_argument('--force', help='force run, remove file lock', action='store_true') parser.add_argument('--lock', help='lock file', default='/tmp/ahriman.lock') + parser.add_argument('--unsafe', help='allow to run ahriman as non-ahriman user', action='store_true') parser.add_argument('-v', '--version', action='version', version=version.__version__) subparsers = parser.add_subparsers(title='command') @@ -152,6 +166,14 @@ if __name__ == '__main__': check_parser.set_defaults(fn=update, no_aur=False, no_manual=True, dry_run=True) clean_parser = subparsers.add_parser('clean', description='clear all local caches') + clean_parser.add_argument('--no-build', help='do not clear directory with package sources', action='store_true') + clean_parser.add_argument('--no-cache', help='do not clear directory with package caches', action='store_true') + clean_parser.add_argument('--no-chroot', help='do not clear build chroot', action='store_true') + clean_parser.add_argument( + '--no-manual', + help='do not clear directory with manually added packages', + action='store_true') + clean_parser.add_argument('--no-packages', help='do not clear directory with built packages', action='store_true') clean_parser.set_defaults(fn=clean) rebuild_parser = subparsers.add_parser('rebuild', description='rebuild whole repository') @@ -188,4 +210,6 @@ if __name__ == '__main__': config = Configuration.from_path(args.config) with Pool(len(args.architecture)) as pool: - pool.starmap(_call, [(args, architecture, config) for architecture in args.architecture]) + result = pool.starmap(_call, [(args, architecture, config) for architecture in args.architecture]) + + sys.exit(0 if all(result) else 1) diff --git a/src/ahriman/application/application.py b/src/ahriman/application/application.py index edfdada9..d5b8b0c9 100644 --- a/src/ahriman/application/application.py +++ b/src/ahriman/application/application.py @@ -126,15 +126,25 @@ class Application: for name in names: process_single(name) - def clean(self) -> None: + def clean(self, no_build: bool, no_cache: bool, no_chroot: bool, no_manual: bool, no_packages: bool) -> None: ''' - run all clean methods + run all clean methods. Warning: some functions might not be available under non-root + :param no_build: do not clear directory with package sources + :param no_cache: do not clear directory with package caches + :param no_chroot: do not clear build chroot + :param no_manual: do not clear directory with manually added packages + :param no_packages: do not clear directory with built packages ''' - self.repository._clear_build() - self.repository._clear_cache() - self.repository._clear_chroot() - self.repository._clear_manual() - self.repository._clear_packages() + if not no_build: + self.repository._clear_build() + if not no_cache: + self.repository._clear_cache() + if not no_chroot: + self.repository._clear_chroot() + if not no_manual: + self.repository._clear_manual() + if not no_packages: + self.repository._clear_packages() def remove(self, names: Iterable[str]) -> None: ''' diff --git a/src/ahriman/application/lock.py b/src/ahriman/application/lock.py index 1ba580b8..604f4ffc 100644 --- a/src/ahriman/application/lock.py +++ b/src/ahriman/application/lock.py @@ -25,7 +25,7 @@ from types import TracebackType from typing import Literal, Optional, Type from ahriman.core.configuration import Configuration -from ahriman.core.exceptions import DuplicateRun +from ahriman.core.exceptions import DuplicateRun, UnsafeRun from ahriman.core.watcher.client import Client from ahriman.models.build_status import BuildStatusEnum @@ -36,30 +36,38 @@ class Lock: :ivar force: remove lock file on start if any :ivar path: path to lock file if any :ivar reporter: build status reporter instance + :ivar root: repository root (i.e. ahriman home) + :ivar unsafe: skip user check ''' - def __init__(self, path: Optional[str], architecture: str, force: bool, config: Configuration) -> None: + def __init__(self, path: Optional[str], architecture: str, force: bool, unsafe: bool, + config: Configuration) -> None: ''' default constructor :param path: optional path to lock file, if empty no file lock will be used :param architecture: repository architecture :param force: remove lock file on start if any + :param unsafe: skip user check :param config: configuration instance ''' self.path = f'{path}_{architecture}' if path is not None else None self.force = force + self.unsafe = unsafe + self.root = config.get('repository', 'root') self.reporter = Client.load(architecture, config) def __enter__(self) -> Lock: ''' default workflow is the following: + check user UID remove lock file if force flag is set check if there is lock file create lock file report to web if enabled ''' + self.check_user() if self.force: self.remove() self.check() @@ -90,6 +98,17 @@ class Lock: if os.path.exists(self.path): raise DuplicateRun() + def check_user(self) -> None: + ''' + check if current user is actually owner of ahriman root + ''' + if self.unsafe: + return + current_uid = os.getuid() + root_uid = os.stat(self.root).st_uid + if current_uid != root_uid: + raise UnsafeRun(current_uid, root_uid) + def create(self) -> None: ''' create lock file diff --git a/src/ahriman/core/build_tools/task.py b/src/ahriman/core/build_tools/task.py index e6df1cc7..6ea5aef8 100644 --- a/src/ahriman/core/build_tools/task.py +++ b/src/ahriman/core/build_tools/task.py @@ -80,12 +80,13 @@ class Task: :param remote: remote target (from where to fetch) :param branch: branch name to checkout, master by default ''' + logger = logging.getLogger('build_details') if os.path.isdir(local): - check_output('git', 'fetch', 'origin', branch, cwd=local, exception=None) + check_output('git', 'fetch', 'origin', branch, exception=None, cwd=local, logger=logger) else: - check_output('git', 'clone', remote, local, exception=None) + check_output('git', 'clone', remote, local, exception=None, logger=logger) # and now force reset to our branch - check_output('git', 'reset', '--hard', f'origin/{branch}', cwd=local, exception=None) + check_output('git', 'reset', '--hard', f'origin/{branch}', exception=None, cwd=local, logger=logger) def build(self) -> List[str]: ''' @@ -107,7 +108,8 @@ class Task: # well it is not actually correct, but we can deal with it return check_output('makepkg', '--packagelist', exception=BuildFailed(self.package.base), - cwd=self.git_path).splitlines() + cwd=self.git_path, + logger=self.build_logger).splitlines() def init(self, path: Optional[str] = None) -> None: ''' diff --git a/src/ahriman/core/configuration.py b/src/ahriman/core/configuration.py index be40eed3..285b7692 100644 --- a/src/ahriman/core/configuration.py +++ b/src/ahriman/core/configuration.py @@ -20,6 +20,7 @@ from __future__ import annotations import configparser +import logging import os from logging.config import fileConfig @@ -30,8 +31,13 @@ class Configuration(configparser.RawConfigParser): ''' extension for built-in configuration parser :ivar path: path to root configuration file + :cvar DEFAULT_LOG_FORMAT: default log format (in case of fallback) + :cvar DEFAULT_LOG_LEVEL: default log level (in case of fallback) ''' + DEFAULT_LOG_FORMAT = '%(asctime)s : %(levelname)s : %(funcName)s : %(message)s' + DEFAULT_LOG_LEVEL = logging.DEBUG + def __init__(self) -> None: ''' default constructor @@ -97,10 +103,15 @@ class Configuration(configparser.RawConfigParser): for conf in filter(lambda p: p.endswith('.ini'), sorted(os.listdir(self.include))): self.read(os.path.join(self.include, conf)) except (FileNotFoundError, configparser.NoOptionError): - pass + passDEFAULT_LOG_LEVEL def load_logging(self) -> None: ''' setup logging settings from configuration ''' - fileConfig(self.get('settings', 'logging')) + try: + fileConfig(self.get('settings', 'logging')) + except PermissionError: + logging.basicConfig(filename=None, format=Configuration.DEFAULT_LOG_FORMAT, + level=Configuration.DEFAULT_LOG_LEVEL) + logging.error('could not create logfile, fallback to stderr', exc_info=True) diff --git a/src/ahriman/core/exceptions.py b/src/ahriman/core/exceptions.py index 44848676..c20c4d60 100644 --- a/src/ahriman/core/exceptions.py +++ b/src/ahriman/core/exceptions.py @@ -105,3 +105,19 @@ class SyncFailed(Exception): default constructor ''' Exception.__init__(self, 'Sync failed') + + +class UnsafeRun(Exception): + ''' + exception which will be raised in case if user is not owner of repository + ''' + + def __init__(self, current_uid: int, root_uid: int) -> None: + ''' + default constructor + ''' + Exception.__init__( + self, + f'''Current UID {current_uid} differs from root owner {root_uid}. +Note that for the most actions it is unsafe to run application as different user. +If you are 100% sure that it must be there try --unsafe option''') diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index bdfbe98d..404fbadd 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -19,6 +19,8 @@ # from __future__ import annotations +import logging + import aur # type: ignore import datetime import os @@ -86,13 +88,15 @@ class Package: return self.version from ahriman.core.build_tools.task import Task - clone_dir = os.path.join(paths.cache, self.base) + clone_dir = os.path.join(paths.cache, self.base) + logger = logging.getLogger('build_details') Task.fetch(clone_dir, self.git_url) + # update pkgver first - check_output('makepkg', '--nodeps', '--nobuild', exception=None, cwd=clone_dir) + check_output('makepkg', '--nodeps', '--nobuild', exception=None, cwd=clone_dir, logger=logger) # generate new .SRCINFO and put it to parser - src_info_source = check_output('makepkg', '--printsrcinfo', exception=None, cwd=clone_dir) + src_info_source = check_output('makepkg', '--printsrcinfo', exception=None, cwd=clone_dir, logger=logger) src_info, errors = parse_srcinfo(src_info_source) if errors: raise InvalidPackageInfo(errors) diff --git a/src/ahriman/web/web.py b/src/ahriman/web/web.py index 0f5dae9f..66c34f61 100644 --- a/src/ahriman/web/web.py +++ b/src/ahriman/web/web.py @@ -63,7 +63,8 @@ def run_server(application: web.Application, architecture: str) -> None: host = application['config'].get(section, 'host') port = application['config'].getint(section, 'port') - web.run_app(application, host=host, port=port, handle_signals=False) + web.run_app(application, host=host, port=port, handle_signals=False, + access_log=logging.getLogger('http')) def setup_service(architecture: str, config: Configuration) -> web.Application: