diff --git a/CONFIGURING.md b/CONFIGURING.md index 23feb4dc..bb11524b 100644 --- a/CONFIGURING.md +++ b/CONFIGURING.md @@ -13,9 +13,9 @@ AUR related configuration: * `url` - base url for AUR, string, required. -## `build` group +## `build_*` groups -Build related configuration: +Build related configuration. Group name must refer to architecture, e.g. it should be `build_x86_64` for x86_64 architecture. * `archbuild_flags` - additional flags passed to `archbuild` command, space separated list of strings, optional. * `build_command` - default build command, string, required. diff --git a/README.md b/README.md index 3eb5b6be..99dbda44 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,8 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github * create build command if required, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/custom-x86_64-build` (you can choose any name for command); * create configuration file, e.g. `cp /usr/share/devtools/pacman-{extra,custom}.conf`; * change configuration file: add your own repository, add multilib repository; - * set `build.build_command` to point to your command. + * set `build.build_command` to point to your command; + * configure `/etc/sudoers.d/ahriman` to allow to run command without password. * Start and enable `ahriman.timer` via `systemctl`. * Add packages by using `ahriman add {package}` command. diff --git a/package/archlinux/PKGBUILD b/package/archlinux/PKGBUILD index a8314521..589740e3 100644 --- a/package/archlinux/PKGBUILD +++ b/package/archlinux/PKGBUILD @@ -7,7 +7,7 @@ pkgdesc="ArcHlinux ReposItory MANager" arch=('any') url="https://github.com/arcan1s/ahriman" license=('GPL3') -depends=('devtools' 'python-aur' 'python-srcinfo') +depends=('devtools' 'expac' 'git' 'python-aur' 'python-srcinfo') makedepends=('python-pip') optdepends=('aws-cli: sync to s3' 'gnupg: package and repository sign' @@ -16,7 +16,7 @@ source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$ 'ahriman.sudoers' 'ahriman.sysusers' 'ahriman.tmpfiles') -sha512sums=('d42d779279493c0de86f8e6880cd644a2d549d61cf6c03c27706a155ca4350158d9a309ac77377de13002071727f2e8532144fb3aa1f2ff95811bd9f3cffd9f3' +sha512sums=('392e6f5f0ed9b333896f20ff4fa4f1a2ee1a43efe74eff8b63672419bc31ec2b9e2df100586a1dfd4eca89e53d131187532191c163d3420695e0361c335f3fe3' '8c9b5b63ac3f7b4d9debaf801a1e9c060877c33d3ecafe18010fcca778e5fa2f2e46909d3d0ff1b229ff8aa978445d8243fd36e1fc104117ed678d5e21901167' '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075' '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4') diff --git a/package/etc/ahriman.ini b/package/etc/ahriman.ini index 9583bbc8..a1f6e457 100644 --- a/package/etc/ahriman.ini +++ b/package/etc/ahriman.ini @@ -5,7 +5,7 @@ logging = /etc/ahriman.ini.d/logging.ini [aur] url = https://aur.archlinux.org -[build] +[build_x86_64] archbuild_flags = build_command = extra-x86_64-build makechrootpkg_flags = diff --git a/package/lib/systemd/system/ahriman.service b/package/lib/systemd/system/ahriman.service index 3a2c8238..9a030c01 100644 --- a/package/lib/systemd/system/ahriman.service +++ b/package/lib/systemd/system/ahriman.service @@ -2,6 +2,6 @@ Description=ArcHlinux ReposItory MANager [Service] -ExecStart=/usr/bin/ahriman update +ExecStart=/usr/bin/ahriman --architecture x86_64 update User=ahriman Group=ahriman \ No newline at end of file diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 754022bc..6780660e 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -19,7 +19,6 @@ # import argparse import os -import shutil import ahriman.version as version @@ -29,7 +28,7 @@ from ahriman.core.configuration import Configuration def _get_app(args: argparse.Namespace) -> Application: config = _get_config(args.config) - return Application(config) + return Application(args.architecture, config) def _get_config(config_path: str) -> Configuration: @@ -40,13 +39,20 @@ def _get_config(config_path: str) -> Configuration: def _remove_lock(path: str) -> None: - shutil.rmtree(path, ignore_errors=True) + if os.path.exists(path): + os.remove(path) def add(args: argparse.Namespace) -> None: _get_app(args).add(args.package) +def rebuild(args: argparse.Namespace) -> None: + app = _get_app(args) + packages = app.repository.packages() + app.update(packages) + + def remove(args: argparse.Namespace) -> None: _get_app(args).remove(args.package) @@ -60,12 +66,17 @@ def sync(args: argparse.Namespace) -> None: def update(args: argparse.Namespace) -> None: - check_only = (args.command == 'check') - _get_app(args).update(check_only) + app = _get_app(args) + log_fn = lambda line: print(line) if args.dry_run else app.logger.info(line) + packages = app.get_updates(args.no_aur, args.no_manual, log_fn) + if args.dry_run: + return + app.update(packages) # type: ignore if __name__ == '__main__': - parser = argparse.ArgumentParser(description='ArcHlinux ReposItory MANager') + parser = argparse.ArgumentParser(prog='ahriman', description='ArcHlinux ReposItory MANager') + parser.add_argument('-a', '--architecture', help='target architecture', required=True) parser.add_argument('-c', '--config', help='configuration path', default='/etc/ahriman.ini') parser.add_argument('--force', help='force run, remove file lock', action='store_true') parser.add_argument('--lock', help='lock file', default='/tmp/ahriman.lock') @@ -77,7 +88,10 @@ if __name__ == '__main__': add_parser.set_defaults(fn=add) check_parser = subparsers.add_parser('check', description='check for updates') - check_parser.set_defaults(fn=update) + check_parser.set_defaults(fn=update, no_aur=False, no_manual=True, dry_run=False) + + rebuild_parser = subparsers.add_parser('rebuild', description='rebuild whole repository') + rebuild_parser.set_defaults(fn=rebuild) remove_parser = subparsers.add_parser('remove', description='remove package') remove_parser.add_argument('package', help='package name', nargs='+') @@ -92,6 +106,9 @@ if __name__ == '__main__': sync_parser.set_defaults(fn=sync) update_parser = subparsers.add_parser('update', description='run updates') + update_parser.add_argument('--dry-run', help='just perform check for updates, same as check command', action='store_true') + update_parser.add_argument('--no-aur', help='do not check for AUR updates', action='store_true') + update_parser.add_argument('--no-manual', help='do not include manual updates', action='store_true') update_parser.set_defaults(fn=update) args = parser.parse_args() diff --git a/src/ahriman/application/application.py b/src/ahriman/application/application.py index f3920e4f..c1ee82a7 100644 --- a/src/ahriman/application/application.py +++ b/src/ahriman/application/application.py @@ -20,7 +20,7 @@ import logging import os -from typing import List, Optional +from typing import Callable, List, Optional from ahriman.core.build_tools.task import Task from ahriman.core.configuration import Configuration @@ -30,15 +30,16 @@ from ahriman.models.package import Package class Application: - def __init__(self, config: Configuration) -> None: + def __init__(self, architecture: str, config: Configuration) -> None: self.logger = logging.getLogger('root') self.config = config - self.repository = Repository(config) + self.architecture = architecture + self.repository = Repository(architecture, config) def add(self, names: List[str]) -> None: for name in names: package = Package.load(name, self.config.get('aur', 'url')) - task = Task(package, self.config, self.repository.paths) + task = Task(package, self.architecture, self.config, self.repository.paths) task.fetch(os.path.join(self.repository.paths.manual, package.name)) def remove(self, names: List[str]) -> None: @@ -52,17 +53,22 @@ class Application: targets = target or None self.repository.process_sync(targets) - def update(self, dry_run: bool) -> None: - updates = self.repository.updates() - log_fn = print if dry_run else self.logger.info - for package in updates: - log_fn(f'{package.name} = {package.version}') # type: ignore - - if dry_run: - return - + def update(self, updates: List[Package]) -> None: packages = self.repository.process_build(updates) self.repository.process_update(packages) - self.report() self.sync() + + def get_updates(self, no_aur: bool, no_manual: bool, log_fn: Callable[[str], None]) -> List[Package]: + updates = [] + checked: List[str] = [] + + if not no_aur: + updates.extend(self.repository.updates_aur(checked)) + if not no_manual: + updates.extend(self.repository.updates_aur(checked)) + + for package in updates: + log_fn(f'{package.name} = {package.version}') + + return updates diff --git a/src/ahriman/core/build_tools/task.py b/src/ahriman/core/build_tools/task.py index 4abdb6ec..d24909e2 100644 --- a/src/ahriman/core/build_tools/task.py +++ b/src/ahriman/core/build_tools/task.py @@ -32,16 +32,17 @@ from ahriman.models.repository_paths import RepositoryPaths class Task: - def __init__(self, package: Package, config: Configuration, paths: RepositoryPaths) -> None: + def __init__(self, package: Package, architecture: str, config: Configuration, paths: RepositoryPaths) -> None: self.logger = logging.getLogger('builder') self.build_logger = logging.getLogger('build_details') self.package = package self.paths = paths - self.archbuild_flags = options_list(config, 'build', 'archbuild_flags') - self.build_command = config.get('build', 'build_command') - self.makepkg_flags = options_list(config, 'build', 'makepkg_flags') - self.makechrootpkg_flags = options_list(config, 'build', 'makechrootpkg_flags') + section = f'build_{architecture}' + self.archbuild_flags = options_list(config, section, 'archbuild_flags') + self.build_command = config.get(section, 'build_command') + self.makepkg_flags = options_list(config, section, 'makepkg_flags') + self.makechrootpkg_flags = options_list(config, section, 'makechrootpkg_flags') @property def git_path(self) -> str: @@ -67,5 +68,5 @@ class Task: def fetch(self, path: Optional[str] = None) -> None: git_path = path or self.git_path - shutil.rmtree(git_path, ignore_errors=True) + shutil.rmtree(git_path, ignore_errors=True) # remote in case if file exists check_output('git', 'clone', self.package.url, git_path, exception=None) diff --git a/src/ahriman/core/repo/repo_wrapper.py b/src/ahriman/core/repo/repo_wrapper.py index 5d7d5ed5..fda11dd1 100644 --- a/src/ahriman/core/repo/repo_wrapper.py +++ b/src/ahriman/core/repo/repo_wrapper.py @@ -19,7 +19,6 @@ # import logging import os -import shutil from ahriman.core.exceptions import BuildFailed from ahriman.core.util import check_output @@ -45,8 +44,10 @@ class RepoWrapper: logger=self.logger) def remove(self, path: str, package: str) -> None: - shutil.rmtree(path, ignore_errors=True) - shutil.rmtree(f'{path}.sig', ignore_errors=True) # sign if any + os.remove(path) + sign_path = f'{path}.sig' + if os.path.exists(sign_path): + os.remove(sign_path) check_output( 'repo-remove', self.repo_path, package, exception=BuildFailed(path), diff --git a/src/ahriman/core/repository.py b/src/ahriman/core/repository.py index 19129a5e..319967cd 100644 --- a/src/ahriman/core/repository.py +++ b/src/ahriman/core/repository.py @@ -21,7 +21,7 @@ import logging import os import shutil -from typing import List, Optional +from typing import Dict, List, Optional from ahriman.core.build_tools.task import Task from ahriman.core.configuration import Configuration @@ -36,14 +36,15 @@ from ahriman.models.repository_paths import RepositoryPaths class Repository: - def __init__(self, config: Configuration) -> None: + def __init__(self, architecture: str, config: Configuration) -> None: self.logger = logging.getLogger('builder') + self.architecture = architecture self.config = config self.aur_url = config.get('aur', 'url') self.name = config.get('repository', 'name') - self.paths = RepositoryPaths(config.get('repository', 'root')) + self.paths = RepositoryPaths(config.get('repository', 'root'), self.architecture) self.paths.create_tree() self.sign = GPGWrapper(config) @@ -51,27 +52,46 @@ class Repository: def _clear_build(self) -> None: for package in os.listdir(self.paths.sources): - shutil.rmtree(os.path.join(self.paths.sources, package), ignore_errors=True) + shutil.rmtree(os.path.join(self.paths.sources, package)) def _clear_manual(self) -> None: for package in os.listdir(self.paths.manual): - shutil.rmtree(os.path.join(self.paths.manual, package), ignore_errors=True) + shutil.rmtree(os.path.join(self.paths.manual, package)) def _clear_packages(self) -> None: for package in os.listdir(self.paths.packages): - shutil.rmtree(os.path.join(self.paths.packages, package), ignore_errors=True) + os.remove(os.path.join(self.paths.packages, package)) + + def packages(self) -> List[Package]: + result: Dict[str, Package] = {} + for fn in os.listdir(self.paths.repository): + if '.pkg.' not in fn: + continue + full_path = os.path.join(self.paths.repository, fn) + try: + local = Package.load(full_path, self.aur_url) + if local.name in result: + continue + result[local.name] = local + except Exception: + self.logger.exception(f'could not load package from {fn}', exc_info=True) + continue + return list(result.values()) def process_build(self, updates: List[Package]) -> List[str]: + def build_single(package: Package) -> None: + task = Task(package, self.architecture, self.config, self.paths) + task.fetch() + built = task.build() + for src in built: + dst = os.path.join(self.paths.packages, os.path.basename(src)) + shutil.move(src, dst) + for package in updates: try: - task = Task(package, self.config, self.paths) - task.fetch() - built = task.build() - for src in built: - dst = os.path.join(self.paths.packages, os.path.basename(src)) - shutil.move(src, dst) + build_single(package) except Exception: - self.logger.exception(f'{package.name} build exception', exc_info=True) + self.logger.exception(f'{package.name} ({self.architecture}) build exception', exc_info=True) continue self._clear_build() @@ -123,11 +143,9 @@ class Repository: self.sign.sign_repository(self.wrapper.repo_path) return self.wrapper.repo_path - def updates(self) -> List[Package]: + def updates_aur(self, checked: List[str]) -> List[Package]: result: List[Package] = [] - checked_base: List[str] = [] - # repository updates for fn in os.listdir(self.paths.repository): if '.pkg.' not in fn: continue @@ -138,20 +156,24 @@ class Repository: except Exception: self.logger.exception(f'could not load package from {fn}', exc_info=True) continue - if local.name in checked_base: + if local.name in checked: continue if local.is_outdated(remote): result.append(remote) - checked_base.append(local.name) + checked.append(local.name) + + return result + + def updates_manual(self, checked: List[str]) -> List[Package]: + result: List[Package] = [] - # manual updates for fn in os.listdir(self.paths.manual): local = Package.load(os.path.join(self.paths.manual, fn), self.aur_url) - if local.name in checked_base: + if local.name in checked: continue result.append(local) - checked_base.append(local.name) + checked.append(local.name) self._clear_manual() return result \ No newline at end of file diff --git a/src/ahriman/models/repository_paths.py b/src/ahriman/models/repository_paths.py index 38db35ab..c5d722a9 100644 --- a/src/ahriman/models/repository_paths.py +++ b/src/ahriman/models/repository_paths.py @@ -25,6 +25,7 @@ from dataclasses import dataclass @dataclass class RepositoryPaths: root: str + architecture: str @property def chroot(self) -> str: @@ -40,7 +41,7 @@ class RepositoryPaths: @property def repository(self) -> str: - return os.path.join(self.root, 'repository') + return os.path.join(self.root, 'repository', self.architecture) @property def sources(self) -> str: