arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-04-24 07:17:17 +00:00
Code Issues Projects Releases Wiki Activity

improve ahriman

Browse Source 
* fix dependency list for packaging
* add ability to rebuild everything or skip specific things during an
  update
* add architecture configuration
* fix file and directory removals
This commit is contained in:
Evgenii Alekseev 2021-03-07 06:55:00 +03:00
parent 088698d8e1
commit b69076eb18
11 changed files with 108 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CONFIGURING.md
View File

@ -13,9 +13,9 @@ AUR related configuration:
* `url` - base url for AUR, string, required.
## `build` group
## `build_*` groups
Build related configuration:
Build related configuration. Group name must refer to architecture, e.g. it should be `build_x86_64` for x86_64 architecture.
* `archbuild_flags` - additional flags passed to `archbuild` command, space separated list of strings, optional.
* `build_command` - default build command, string, required.

3
README.md
View File

@ -11,7 +11,8 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github
* create build command if required, e.g. `ln -s /usr/bin/archbuild /usr/local/bin/custom-x86_64-build` (you can choose any name for command);
* create configuration file, e.g. `cp /usr/share/devtools/pacman-{extra,custom}.conf`;
* change configuration file: add your own repository, add multilib repository;
* set `build.build_command` to point to your command.
* set `build.build_command` to point to your command;
* configure `/etc/sudoers.d/ahriman` to allow to run command without password.
* Start and enable `ahriman.timer` via `systemctl`.
* Add packages by using `ahriman add {package}` command.

4
package/archlinux/PKGBUILD
View File

@ -7,7 +7,7 @@ pkgdesc="ArcHlinux ReposItory MANager"
arch=('any')
url="https://github.com/arcan1s/ahriman"
license=('GPL3')
depends=('devtools' 'python-aur' 'python-srcinfo')
depends=('devtools' 'expac' 'git' 'python-aur' 'python-srcinfo')
makedepends=('python-pip')
optdepends=('aws-cli: sync to s3'
'gnupg: package and repository sign'
@ -16,7 +16,7 @@ source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$
'ahriman.sudoers'
'ahriman.sysusers'
'ahriman.tmpfiles')
sha512sums=('d42d779279493c0de86f8e6880cd644a2d549d61cf6c03c27706a155ca4350158d9a309ac77377de13002071727f2e8532144fb3aa1f2ff95811bd9f3cffd9f3'
sha512sums=('392e6f5f0ed9b333896f20ff4fa4f1a2ee1a43efe74eff8b63672419bc31ec2b9e2df100586a1dfd4eca89e53d131187532191c163d3420695e0361c335f3fe3'
'8c9b5b63ac3f7b4d9debaf801a1e9c060877c33d3ecafe18010fcca778e5fa2f2e46909d3d0ff1b229ff8aa978445d8243fd36e1fc104117ed678d5e21901167'
'13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
'55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')

2
package/etc/ahriman.ini
View File

@ -5,7 +5,7 @@ logging = /etc/ahriman.ini.d/logging.ini
[aur]
url = https://aur.archlinux.org
[build]
[build_x86_64]
archbuild_flags =
build_command = extra-x86_64-build
makechrootpkg_flags =

2
package/lib/systemd/system/ahriman.service
View File

@ -2,6 +2,6 @@
Description=ArcHlinux ReposItory MANager
[Service]
ExecStart=/usr/bin/ahriman update
ExecStart=/usr/bin/ahriman --architecture x86_64 update
User=ahriman
Group=ahriman

31
src/ahriman/application/ahriman.py
View File

@ -19,7 +19,6 @@
#
import argparse
import os
import shutil
import ahriman.version as version
@ -29,7 +28,7 @@ from ahriman.core.configuration import Configuration
def _get_app(args: argparse.Namespace) -> Application:
config = _get_config(args.config)
return Application(config)
return Application(args.architecture, config)
def _get_config(config_path: str) -> Configuration:
@ -40,13 +39,20 @@ def _get_config(config_path: str) -> Configuration:
def _remove_lock(path: str) -> None:
shutil.rmtree(path, ignore_errors=True)
if os.path.exists(path):
os.remove(path)
def add(args: argparse.Namespace) -> None:
_get_app(args).add(args.package)
def rebuild(args: argparse.Namespace) -> None:
app = _get_app(args)
packages = app.repository.packages()
app.update(packages)
def remove(args: argparse.Namespace) -> None:
_get_app(args).remove(args.package)
@ -60,12 +66,17 @@ def sync(args: argparse.Namespace) -> None:
def update(args: argparse.Namespace) -> None:
check_only = (args.command == 'check')
_get_app(args).update(check_only)
app = _get_app(args)
log_fn = lambda line: print(line) if args.dry_run else app.logger.info(line)
packages = app.get_updates(args.no_aur, args.no_manual, log_fn)
if args.dry_run:
return
app.update(packages) # type: ignore
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='ArcHlinux ReposItory MANager')
parser = argparse.ArgumentParser(prog='ahriman', description='ArcHlinux ReposItory MANager')
parser.add_argument('-a', '--architecture', help='target architecture', required=True)
parser.add_argument('-c', '--config', help='configuration path', default='/etc/ahriman.ini')
parser.add_argument('--force', help='force run, remove file lock', action='store_true')
parser.add_argument('--lock', help='lock file', default='/tmp/ahriman.lock')
@ -77,7 +88,10 @@ if __name__ == '__main__':
add_parser.set_defaults(fn=add)
check_parser = subparsers.add_parser('check', description='check for updates')
check_parser.set_defaults(fn=update)
check_parser.set_defaults(fn=update, no_aur=False, no_manual=True, dry_run=False)
rebuild_parser = subparsers.add_parser('rebuild', description='rebuild whole repository')
rebuild_parser.set_defaults(fn=rebuild)
remove_parser = subparsers.add_parser('remove', description='remove package')
remove_parser.add_argument('package', help='package name', nargs='+')
@ -92,6 +106,9 @@ if __name__ == '__main__':
sync_parser.set_defaults(fn=sync)
update_parser = subparsers.add_parser('update', description='run updates')
update_parser.add_argument('--dry-run', help='just perform check for updates, same as check command', action='store_true')
update_parser.add_argument('--no-aur', help='do not check for AUR updates', action='store_true')
update_parser.add_argument('--no-manual', help='do not include manual updates', action='store_true')
update_parser.set_defaults(fn=update)
args = parser.parse_args()

34
src/ahriman/application/application.py
View File

@ -20,7 +20,7 @@
import logging
import os
from typing import List, Optional
from typing import Callable, List, Optional
from ahriman.core.build_tools.task import Task
from ahriman.core.configuration import Configuration
@ -30,15 +30,16 @@ from ahriman.models.package import Package
class Application:
def __init__(self, config: Configuration) -> None:
def __init__(self, architecture: str, config: Configuration) -> None:
self.logger = logging.getLogger('root')
self.config = config
self.repository = Repository(config)
self.architecture = architecture
self.repository = Repository(architecture, config)
def add(self, names: List[str]) -> None:
for name in names:
package = Package.load(name, self.config.get('aur', 'url'))
task = Task(package, self.config, self.repository.paths)
task = Task(package, self.architecture, self.config, self.repository.paths)
task.fetch(os.path.join(self.repository.paths.manual, package.name))
def remove(self, names: List[str]) -> None:
@ -52,17 +53,22 @@ class Application:
targets = target or None
self.repository.process_sync(targets)
def update(self, dry_run: bool) -> None:
updates = self.repository.updates()
log_fn = print if dry_run else self.logger.info
for package in updates:
log_fn(f'{package.name} = {package.version}') # type: ignore
if dry_run:
return
def update(self, updates: List[Package]) -> None:
packages = self.repository.process_build(updates)
self.repository.process_update(packages)
self.report()
self.sync()
def get_updates(self, no_aur: bool, no_manual: bool, log_fn: Callable[[str], None]) -> List[Package]:
updates = []
checked: List[str] = []
if not no_aur:
updates.extend(self.repository.updates_aur(checked))
if not no_manual:
updates.extend(self.repository.updates_aur(checked))
for package in updates:
log_fn(f'{package.name} = {package.version}')
return updates

13
src/ahriman/core/build_tools/task.py
View File

@ -32,16 +32,17 @@ from ahriman.models.repository_paths import RepositoryPaths
class Task:
def __init__(self, package: Package, config: Configuration, paths: RepositoryPaths) -> None:
def __init__(self, package: Package, architecture: str, config: Configuration, paths: RepositoryPaths) -> None:
self.logger = logging.getLogger('builder')
self.build_logger = logging.getLogger('build_details')
self.package = package
self.paths = paths
self.archbuild_flags = options_list(config, 'build', 'archbuild_flags')
self.build_command = config.get('build', 'build_command')
self.makepkg_flags = options_list(config, 'build', 'makepkg_flags')
self.makechrootpkg_flags = options_list(config, 'build', 'makechrootpkg_flags')
section = f'build_{architecture}'
self.archbuild_flags = options_list(config, section, 'archbuild_flags')
self.build_command = config.get(section, 'build_command')
self.makepkg_flags = options_list(config, section, 'makepkg_flags')
self.makechrootpkg_flags = options_list(config, section, 'makechrootpkg_flags')
@property
def git_path(self) -> str:
@ -67,5 +68,5 @@ class Task:
def fetch(self, path: Optional[str] = None) -> None:
git_path = path or self.git_path
shutil.rmtree(git_path, ignore_errors=True)
shutil.rmtree(git_path, ignore_errors=True) # remote in case if file exists
check_output('git', 'clone', self.package.url, git_path, exception=None)

7
src/ahriman/core/repo/repo_wrapper.py
View File

@ -19,7 +19,6 @@
#
import logging
import os
import shutil
from ahriman.core.exceptions import BuildFailed
from ahriman.core.util import check_output
@ -45,8 +44,10 @@ class RepoWrapper:
logger=self.logger)
def remove(self, path: str, package: str) -> None:
shutil.rmtree(path, ignore_errors=True)
shutil.rmtree(f'{path}.sig', ignore_errors=True) # sign if any
os.remove(path)
sign_path = f'{path}.sig'
if os.path.exists(sign_path):
os.remove(sign_path)
check_output(
'repo-remove', self.repo_path, package,
exception=BuildFailed(path),

64
src/ahriman/core/repository.py
View File

@ -21,7 +21,7 @@ import logging
import os
import shutil
from typing import List, Optional
from typing import Dict, List, Optional
from ahriman.core.build_tools.task import Task
from ahriman.core.configuration import Configuration
@ -36,14 +36,15 @@ from ahriman.models.repository_paths import RepositoryPaths
class Repository:
def __init__(self, config: Configuration) -> None:
def __init__(self, architecture: str, config: Configuration) -> None:
self.logger = logging.getLogger('builder')
self.architecture = architecture
self.config = config
self.aur_url = config.get('aur', 'url')
self.name = config.get('repository', 'name')
self.paths = RepositoryPaths(config.get('repository', 'root'))
self.paths = RepositoryPaths(config.get('repository', 'root'), self.architecture)
self.paths.create_tree()
self.sign = GPGWrapper(config)
@ -51,27 +52,46 @@ class Repository:
def _clear_build(self) -> None:
for package in os.listdir(self.paths.sources):
shutil.rmtree(os.path.join(self.paths.sources, package), ignore_errors=True)
shutil.rmtree(os.path.join(self.paths.sources, package))
def _clear_manual(self) -> None:
for package in os.listdir(self.paths.manual):
shutil.rmtree(os.path.join(self.paths.manual, package), ignore_errors=True)
shutil.rmtree(os.path.join(self.paths.manual, package))
def _clear_packages(self) -> None:
for package in os.listdir(self.paths.packages):
shutil.rmtree(os.path.join(self.paths.packages, package), ignore_errors=True)
os.remove(os.path.join(self.paths.packages, package))
def packages(self) -> List[Package]:
result: Dict[str, Package] = {}
for fn in os.listdir(self.paths.repository):
if '.pkg.' not in fn:
continue
full_path = os.path.join(self.paths.repository, fn)
try:
local = Package.load(full_path, self.aur_url)
if local.name in result:
continue
result[local.name] = local
except Exception:
self.logger.exception(f'could not load package from {fn}', exc_info=True)
continue
return list(result.values())
def process_build(self, updates: List[Package]) -> List[str]:
def build_single(package: Package) -> None:
task = Task(package, self.architecture, self.config, self.paths)
task.fetch()
built = task.build()
for src in built:
dst = os.path.join(self.paths.packages, os.path.basename(src))
shutil.move(src, dst)
for package in updates:
try:
task = Task(package, self.config, self.paths)
task.fetch()
built = task.build()
for src in built:
dst = os.path.join(self.paths.packages, os.path.basename(src))
shutil.move(src, dst)
build_single(package)
except Exception:
self.logger.exception(f'{package.name} build exception', exc_info=True)
self.logger.exception(f'{package.name} ({self.architecture}) build exception', exc_info=True)
continue
self._clear_build()
@ -123,11 +143,9 @@ class Repository:
self.sign.sign_repository(self.wrapper.repo_path)
return self.wrapper.repo_path
def updates(self) -> List[Package]:
def updates_aur(self, checked: List[str]) -> List[Package]:
result: List[Package] = []
checked_base: List[str] = []
# repository updates
for fn in os.listdir(self.paths.repository):
if '.pkg.' not in fn:
continue
@ -138,20 +156,24 @@ class Repository:
except Exception:
self.logger.exception(f'could not load package from {fn}', exc_info=True)
continue
if local.name in checked_base:
if local.name in checked:
continue
if local.is_outdated(remote):
result.append(remote)
checked_base.append(local.name)
checked.append(local.name)
return result
def updates_manual(self, checked: List[str]) -> List[Package]:
result: List[Package] = []
# manual updates
for fn in os.listdir(self.paths.manual):
local = Package.load(os.path.join(self.paths.manual, fn), self.aur_url)
if local.name in checked_base:
if local.name in checked:
continue
result.append(local)
checked_base.append(local.name)
checked.append(local.name)
self._clear_manual()
return result

3
src/ahriman/models/repository_paths.py
View File

@ -25,6 +25,7 @@ from dataclasses import dataclass
@dataclass
class RepositoryPaths:
root: str
architecture: str
@property
def chroot(self) -> str:
@ -40,7 +41,7 @@ class RepositoryPaths:
@property
def repository(self) -> str:
return os.path.join(self.root, 'repository')
return os.path.join(self.root, 'repository', self.architecture)
@property
def sources(self) -> str: