arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-04-24 07:17:17 +00:00
Code Issues Projects Releases Wiki Activity

add depdendency manager and switch to pyalpm instead of expac

Browse Source
This commit is contained in:
Evgenii Alekseev 2021-03-12 00:04:37 +03:00
parent 2d351fa94f
commit 371019f899
19 changed files with 241 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
CONFIGURING.md
View File

@ -9,11 +9,14 @@ Base configuration settings.
* `include` - path to directory with configuration files overrides, string, required.
* `logging` - path to logging configuration, string, required. Check `logging.ini` for reference.
## `aur` group
## `alpm` group
AUR related configuration.
libalpm and AUR related configuration.
* `url` - base url for AUR, string, required.
* `aur_url` - base url for AUR, string, required.
* `database` - path to pacman local database cache, string, required.
* `repositories` - list of pacman repositories, space separated list of strings, required.
* `root` - root for alpm library, string, required.
## `build_*` groups

7
README.md
View File

@ -8,7 +8,8 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github
* Multi-architecture support
* VCS packages support
* Sign support with gpg (repository, package, per package settings)
* Synchronization to remote services (rsync, s3) and report generation (html)
* Synchronization to remote services (rsync, s3) and report generation (html)
* Dependency manager
* Repository status interface
## Installation and run
@ -24,7 +25,3 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github
* configure `/etc/sudoers.d/ahriman` to allow running command without password.
* Start and enable `ahriman.timer` via `systemctl`.
* Add packages by using `ahriman add {package}` command.
## Limitations
* It does not manage dependencies, so you have to add them before main package.

4
package/archlinux/PKGBUILD
View File

@ -7,7 +7,7 @@ pkgdesc="ArcHlinux ReposItory MANager"
arch=('any')
url="https://github.com/arcan1s/ahriman"
license=('GPL3')
depends=('devtools' 'expac' 'git' 'python-aur' 'python-srcinfo')
depends=('devtools' 'git' 'pyalpm' 'python-aur' 'python-srcinfo')
makedepends=('python-pip')
optdepends=('aws-cli: sync to s3'
'breezy: -bzr packages support'
@ -23,7 +23,7 @@ optdepends=('aws-cli: sync to s3'
source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz"
'ahriman.sysusers'
'ahriman.tmpfiles')
sha512sums=('941821639fe4410152a21251d9b0fe5f96ee3a60b88e2067ea4a83ef04b5d1393828152ef4843575449bdef8d44ad6a69f9e41e82516d4d1850bd14f17822785'
sha512sums=('ddbae1368359c93e4a00a196b06e003895fe5bd36f763f2313c2cdc1cf37db96f258807be087a8aceebb8d608c332cd88695600fd28a728af17187f1568711b4'
'13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
'55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')
backup=('etc/ahriman.ini'

7
package/etc/ahriman.ini
View File

@ -2,8 +2,11 @@
include = /etc/ahriman.ini.d
logging = /etc/ahriman.ini.d/logging.ini
[aur]
url = https://aur.archlinux.org
[alpm]
aur_url = https://aur.archlinux.org
database = /var/lib/pacman
repositories = core extra community multilib
root = /
[build]
archbuild_flags =

1
setup.py
View File

@ -28,6 +28,7 @@ setup(
],
install_requires=[
'aur',
'pyalpm',
'srcinfo',
],
setup_requires=[

3
src/ahriman/application/ahriman.py
View File

@ -27,7 +27,7 @@ from ahriman.core.configuration import Configuration
def add(args: argparse.Namespace) -> None:
Application.from_args(args).add(args.package)
Application.from_args(args).add(args.package, args.without_dependencies)
def rebuild(args: argparse.Namespace) -> None:
@ -75,6 +75,7 @@ if __name__ == '__main__':
add_parser = subparsers.add_parser('add', description='add package')
add_parser.add_argument('package', help='package name or archive path', nargs='+')
add_parser.add_argument('--without-dependencies', help='do not add dependencies', action='store_true')
add_parser.set_defaults(fn=add)
check_parser = subparsers.add_parser('check', description='check for updates')

63
src/ahriman/application/application.py
View File

@ -24,11 +24,12 @@ import logging
import os
import shutil
from typing import Callable, List, Optional, Type
from typing import Callable, Iterable, List, Optional, Set, Type
from ahriman.core.build_tools.task import Task
from ahriman.core.configuration import Configuration
from ahriman.core.repository import Repository
from ahriman.core.tree import Tree
from ahriman.models.package import Package
@ -45,6 +46,14 @@ class Application:
config = Configuration.from_path(args.config)
return cls(args.architecture, config)
def _known_packages(self) -> Set[str]:
known_packages = set()
# local set
for package in self.repository.packages():
known_packages.update(package.packages)
known_packages.update(self.repository.pacman.all_packages())
return known_packages
def _finalize(self) -> None:
self.report()
self.sync()
@ -63,35 +72,55 @@ class Application:
return updates
def add(self, names: List[str]) -> None:
def add_manual(name: str) -> None:
package = Package.load(name, self.config.get('aur', 'url'))
Task.fetch(os.path.join(self.repository.paths.manual, package.base), package.git_url)
def add(self, names: Iterable[str], without_dependencies: bool) -> None:
known_packages = self._known_packages()
def add_manual(name: str) -> str:
package = Package.load(name, self.repository.pacman, self.config.get('alpm', 'aur_url'))
path = os.path.join(self.repository.paths.manual, package.base)
Task.fetch(path, package.git_url)
return path
def add_archive(src: str) -> None:
dst = os.path.join(self.repository.paths.packages, os.path.basename(src))
shutil.move(src, dst)
for name in names:
if os.path.isfile(name):
add_archive(name)
else:
add_manual(name)
def process_dependencies(path: str) -> None:
if without_dependencies:
return
dependencies = Package.dependencies(path)
self.add(dependencies.difference(known_packages), without_dependencies)
def remove(self, names: List[str]) -> None:
def process_single(name: str) -> None:
if not os.path.isfile(name):
path = add_manual(name)
process_dependencies(path)
else:
add_archive(name)
for name in names:
process_single(name)
def remove(self, names: Iterable[str]) -> None:
self.repository.process_remove(names)
self._finalize()
def report(self, target: Optional[List[str]] = None) -> None:
def report(self, target: Optional[Iterable[str]] = None) -> None:
targets = target or None
self.repository.process_report(targets)
def sync(self, target: Optional[List[str]] = None) -> None:
def sync(self, target: Optional[Iterable[str]] = None) -> None:
targets = target or None
self.repository.process_sync(targets)
def update(self, updates: List[Package]) -> None:
packages = self.repository.process_build(updates)
self.repository.process_update(packages)
self._finalize()
def update(self, updates: Iterable[Package]) -> None:
def process_single(portion: Iterable[Package]):
packages = self.repository.process_build(portion)
self.repository.process_update(packages)
self._finalize()
tree = Tree()
tree.load(updates)
for num, level in enumerate(tree.levels()):
self.logger.info(f'processing level #{num} {[package.base for package in level]}')
process_single(level)

1
src/ahriman/application/lock.py
View File

@ -39,7 +39,6 @@ class Lock:
def __exit__(self, exc_type, exc_val, exc_tb):
self.remove()
return True
def check(self) -> None:
if self.path is None:

0
src/ahriman/core/repo/__init__.py → src/ahriman/core/alpm/__init__.py
View File

40
src/ahriman/core/alpm/pacman.py Normal file
View File

@ -0,0 +1,40 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from pyalpm import Handle
from typing import List, Set
from ahriman.core.configuration import Configuration
class Pacman:
def __init__(self, config: Configuration) -> None:
root = config.get('alpm', 'root')
pacman_root = config.get('alpm', 'database')
self.handle = Handle(root, pacman_root)
for repository in config.getlist('alpm', 'repositories'):
self.handle.register_syncdb(repository, 0) # 0 is pgp_level
def all_packages(self) -> List[str]:
result: Set[str] = set()
for database in self.handle.get_syncdbs():
result.update({package.name for package in database.pkgcache})
return list(result)

2
src/ahriman/core/repo/repo_wrapper.py → src/ahriman/core/alpm/repo.py
View File

@ -27,7 +27,7 @@ from ahriman.core.util import check_output
from ahriman.models.repository_paths import RepositoryPaths
class RepoWrapper:
class Repo:
def __init__(self, name: str, paths: RepositoryPaths, sign_args: List[str]) -> None:
self.logger = logging.getLogger('build_details')

2
src/ahriman/core/exceptions.py
View File

@ -35,7 +35,7 @@ class InitializeException(Exception):
Exception.__init__(self, 'Could not load service')
class InvalidOptionException(Exception):
class InvalidOption(Exception):
def __init__(self, value: Any) -> None:
Exception.__init__(self, f'Invalid or unknown option value `{value}`')

58
src/ahriman/core/repository.py
View File

@ -21,13 +21,14 @@ import logging
import os
import shutil
from typing import Dict, List, Optional
from typing import Dict, Iterable, List, Optional
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.repo import Repo
from ahriman.core.build_tools.task import Task
from ahriman.core.configuration import Configuration
from ahriman.core.repo.repo_wrapper import RepoWrapper
from ahriman.core.report.report import Report
from ahriman.core.sign.gpg_wrapper import GPGWrapper
from ahriman.core.sign.gpg import GPG
from ahriman.core.upload.uploader import Uploader
from ahriman.core.util import package_like
from ahriman.core.watcher.client import Client
@ -42,15 +43,16 @@ class Repository:
self.architecture = architecture
self.config = config
self.aur_url = config.get('aur', 'url')
self.aur_url = config.get('alpm', 'aur_url')
self.name = config.get('repository', 'name')
self.paths = RepositoryPaths(config.get('repository', 'root'), architecture)
self.paths.create_tree()
self.sign = GPGWrapper(architecture, config)
self.wrapper = RepoWrapper(self.name, self.paths, self.sign.repository_sign_args)
self.web_report = Client.load(architecture, config)
self.pacman = Pacman(config)
self.sign = GPG(architecture, config)
self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
self.web = Client.load(architecture, config)
def _clear_build(self) -> None:
for package in os.listdir(self.paths.sources):
@ -71,16 +73,16 @@ class Repository:
continue
full_path = os.path.join(self.paths.repository, fn)
try:
local = Package.load(full_path, self.aur_url)
local = Package.load(full_path, self.pacman, self.aur_url)
result.setdefault(local.base, local).packages.update(local.packages)
except Exception:
self.logger.exception(f'could not load package from {fn}', exc_info=True)
continue
return list(result.values())
def process_build(self, updates: List[Package]) -> List[str]:
def process_build(self, updates: Iterable[Package]) -> List[str]:
def build_single(package: Package) -> None:
self.web_report.set_building(package.base)
self.web.set_building(package.base)
task = Task(package, self.architecture, self.config, self.paths)
task.clone()
built = task.build()
@ -92,7 +94,7 @@ class Repository:
try:
build_single(package)
except Exception:
self.web_report.set_failed(package.base)
self.web.set_failed(package.base)
self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True)
continue
self._clear_build()
@ -102,10 +104,10 @@ class Repository:
for fn in os.listdir(self.paths.packages)
]
def process_remove(self, packages: List[str]) -> str:
def process_remove(self, packages: Iterable[str]) -> str:
def remove_single(package: str) -> None:
try:
self.wrapper.remove(package, package)
self.repo.remove(package, package)
except Exception:
self.logger.exception(f'could not remove {package}', exc_info=True)
@ -116,41 +118,41 @@ class Repository:
to_remove = local.packages.intersection(packages)
else:
to_remove = set()
self.web_report.remove(local.base, to_remove)
self.web.remove(local.base, to_remove)
for package in to_remove:
remove_single(package)
return self.wrapper.repo_path
return self.repo.repo_path
def process_report(self, targets: Optional[List[str]]) -> None:
def process_report(self, targets: Optional[Iterable[str]]) -> None:
if targets is None:
targets = self.config.getlist('report', 'target')
for target in targets:
Report.run(self.architecture, self.config, target, self.paths.repository)
def process_sync(self, targets: Optional[List[str]]) -> None:
def process_sync(self, targets: Optional[Iterable[str]]) -> None:
if targets is None:
targets = self.config.getlist('upload', 'target')
for target in targets:
Uploader.run(self.architecture, self.config, target, self.paths.repository)
def process_update(self, packages: List[str]) -> str:
def process_update(self, packages: Iterable[str]) -> str:
for package in packages:
local = Package.load(package, self.aur_url) # we will use it for status reports
local = Package.load(package, self.pacman, self.aur_url) # we will use it for status reports
try:
files = self.sign.sign_package(package, local.base)
for src in files:
dst = os.path.join(self.paths.repository, os.path.basename(src))
shutil.move(src, dst)
package_fn = os.path.join(self.paths.repository, os.path.basename(package))
self.wrapper.add(package_fn)
self.web_report.set_success(local)
self.repo.add(package_fn)
self.web.set_success(local)
except Exception:
self.logger.exception(f'could not process {package}', exc_info=True)
self.web_report.set_failed(local.base)
self.web.set_failed(local.base)
self._clear_packages()
return self.wrapper.repo_path
return self.repo.repo_path
def updates_aur(self, no_vcs: bool) -> List[Package]:
result: List[Package] = []
@ -165,12 +167,12 @@ class Repository:
continue
try:
remote = Package.load(local.base, self.aur_url)
remote = Package.load(local.base, self.pacman, self.aur_url)
if local.is_outdated(remote):
result.append(remote)
self.web_report.set_pending(local.base)
self.web.set_pending(local.base)
except Exception:
self.web_report.set_failed(local.base)
self.web.set_failed(local.base)
self.logger.exception(f'could not load remote package {local.base}', exc_info=True)
continue
@ -181,9 +183,9 @@ class Repository:
for fn in os.listdir(self.paths.manual):
try:
local = Package.load(os.path.join(self.paths.manual, fn), self.aur_url)
local = Package.load(os.path.join(self.paths.manual, fn), self.pacman, self.aur_url)
result.append(local)
self.web_report.set_unknown(local)
self.web.set_unknown(local)
except Exception:
self.logger.exception(f'could not add package from {fn}', exc_info=True)
self._clear_manual()

2
src/ahriman/core/sign/gpg_wrapper.py → src/ahriman/core/sign/gpg.py
View File

@ -28,7 +28,7 @@ from ahriman.core.util import check_output
from ahriman.models.sign_settings import SignSettings
class GPGWrapper:
class GPG:
def __init__(self, architecture: str, config: Configuration) -> None:
self.logger = logging.getLogger('build_details')

75
src/ahriman/core/tree.py Normal file
View File

@ -0,0 +1,75 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import annotations
import shutil
import tempfile
from typing import Iterable, List, Set
from ahriman.core.build_tools.task import Task
from ahriman.models.package import Package
class Tree:
def __init__(self) -> None:
self.packages: List[Leaf] = []
def levels(self) -> List[List[Package]]:
result: List[List[Package]] = []
unprocessed = [leaf for leaf in self.packages]
while unprocessed:
result.append([leaf.package for leaf in unprocessed if leaf.is_root(unprocessed)])
unprocessed = [leaf for leaf in unprocessed if not leaf.is_root(unprocessed)]
return result
def load(self, packages: Iterable[Package]) -> None:
for package in packages:
leaf = Leaf(package)
leaf.load_dependencies()
self.packages.append(leaf)
class Leaf:
def __init__(self, package: Package) -> None:
self.package = package
self.dependencies: Set[str] = set()
def is_root(self, packages: Iterable[Leaf]) -> bool:
'''
:param packages:
:return: true if any of packages is dependency of the leaf, false otherwise
'''
for package in packages:
if package.package.packages.intersection(self.dependencies):
return False
return True
def load_dependencies(self) -> None:
clone_dir = tempfile.mkdtemp()
try:
Task.fetch(clone_dir, self.package.git_url)
self.dependencies = Package.dependencies(clone_dir)
finally:
shutil.rmtree(clone_dir, ignore_errors=True)

29
src/ahriman/models/package.py
View File

@ -25,9 +25,11 @@ import shutil
import tempfile
from dataclasses import dataclass, field
from pyalpm import Handle
from srcinfo.parse import parse_srcinfo
from typing import Set, Type
from typing import List, Set, Type
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.exceptions import InvalidPackageInfo
from ahriman.core.util import check_output
@ -79,9 +81,9 @@ class Package:
shutil.rmtree(clone_dir, ignore_errors=True)
@classmethod
def from_archive(cls: Type[Package], path: str, aur_url: str) -> Package:
package, base, version = check_output('expac', '-p', '%n %e %v', path, exception=None).split()
return cls(base, version, aur_url, {package})
def from_archive(cls: Type[Package], path: str, pacman: Pacman, aur_url: str) -> Package:
package = pacman.handle.load_pkg(path)
return cls(package.base, package.version, aur_url, {package.name})
@classmethod
def from_aur(cls: Type[Package], name: str, aur_url: str)-> Package:
@ -99,12 +101,27 @@ class Package:
return cls(src_info['pkgbase'], f'{src_info["pkgver"]}-{src_info["pkgrel"]}', aur_url, packages)
@staticmethod
def load(path: str, aur_url: str) -> Package:
def dependencies(path: str) -> Set[str]:
with open(os.path.join(path, '.SRCINFO')) as fn:
src_info, errors = parse_srcinfo(fn.read())
if errors:
raise InvalidPackageInfo(errors)
makedepends = src_info['makedepends']
# sum over each package
depends: List[str] = src_info.get('depends', [])
for package in src_info['packages'].values():
depends.extend(package.get('depends', []))
# we are not interested in dependencies inside pkgbase
packages = set(src_info['packages'].keys())
return set(depends + makedepends) - packages
@staticmethod
def load(path: str, pacman: Pacman, aur_url: str) -> Package:
try:
if os.path.isdir(path):
package: Package = Package.from_build(path, aur_url)
elif os.path.exists(path):
package = Package.from_archive(path, aur_url)
package = Package.from_archive(path, pacman, aur_url)
else:
package = Package.from_aur(path, aur_url)
return package

4
src/ahriman/models/report_settings.py
View File

@ -21,7 +21,7 @@ from __future__ import annotations
from enum import Enum, auto
from ahriman.core.exceptions import InvalidOptionException
from ahriman.core.exceptions import InvalidOption
class ReportSettings(Enum):
@ -31,4 +31,4 @@ class ReportSettings(Enum):
def from_option(value: str) -> ReportSettings:
if value.lower() in ('html',):
return ReportSettings.HTML
raise InvalidOptionException(value)
raise InvalidOption(value)

4
src/ahriman/models/sign_settings.py
View File

@ -21,7 +21,7 @@ from __future__ import annotations
from enum import Enum, auto
from ahriman.core.exceptions import InvalidOptionException
from ahriman.core.exceptions import InvalidOption
class SignSettings(Enum):
@ -34,4 +34,4 @@ class SignSettings(Enum):
return SignSettings.SignPackages
elif value.lower() in ('repository', 'sign-repository'):
return SignSettings.SignRepository
raise InvalidOptionException(value)
raise InvalidOption(value)

4
src/ahriman/models/upload_settings.py
View File

@ -21,7 +21,7 @@ from __future__ import annotations
from enum import Enum, auto
from ahriman.core.exceptions import InvalidOptionException
from ahriman.core.exceptions import InvalidOption
class UploadSettings(Enum):
@ -34,4 +34,4 @@ class UploadSettings(Enum):
return UploadSettings.Rsync
elif value.lower() in ('s3',):
return UploadSettings.S3
raise InvalidOptionException(value)
raise InvalidOption(value)