Compare commits

...

7 Commits

Author SHA1 Message Date
30ededb2cd Release 0.10.0 2021-03-11 01:59:05 +03:00
2fca108fa4 process null lock file 2021-03-11 01:58:33 +03:00
262d8d8647 multisign option 2021-03-11 01:39:45 +03:00
fd2049b334 web server support 2021-03-11 01:14:09 +03:00
422445da85 Release 0.9.1 2021-03-10 02:26:44 +03:00
aad893fe69 smart remove function and use built-in packages() function everywhere 2021-03-10 01:25:22 +03:00
8e72ee05ba Release 0.9.0 2021-03-08 16:19:12 +03:00
42 changed files with 880 additions and 189 deletions

View File

@ -36,8 +36,8 @@ Base repository settings:
Settings for signing packages or repository:
* `enabled` - configuration flag to enable signing, string, required. Allowed values are `disabled`, `package` (sign each package separately), `repository` (sign repository database file).
* `key` - PGP key, string, optional.
* `target` - configuration flag to enable signing, space separated list of strings, required. Allowed values are `package` (sign each package separately), `repository` (sign repository database file).
* `key` - PGP key, string, required.
## `report` group
@ -60,14 +60,22 @@ Remote synchronization settings:
* `target` - list of synchronizations to be used, space separated list of strings, optional. Allowed values are `rsync`, `s3`.
### `rsync_*` group
Group name must refer to architecture, e.g. it should be `rsync_x86_64` for x86_64 architecture. Requires `rsync` package to be installed. Do not forget to configure ssh for user `ahriman`.
* `remote` - remote server to rsync (e.g. `1.2.3.4:5678:path/to/sync`), string, required.
### `s3_*` group
Group name must refer to architecture, e.g. it should be `s3_x86_64` for x86_64 architecture. Requires `aws-cli` package to be installed. Do not forget to configure it for user `ahriman`.
* `bucket` - bucket name (e.g. `s3://bucket/path`), string, required.
### `rsync_*` group
## `web` group
Group name must refer to architecture, e.g. it should be `rsync_x86_64` for x86_64 architecture. Requires `rsync` package to be installed. Do not forget to configure ssh for user `ahriman`.
Web server settings. If any of `host`/`port` is not set, web intergration will be disabled.
* `remote` - remote server to rsync (e.g. `1.2.3.4:5678:path/to/sync`), string, required.
* `host` - host to bind, string, optional.
* `port` - port to bind, int, optional.
* `templates` - path to templates directory, string, required.

View File

@ -2,6 +2,14 @@
Wrapper for managing custom repository inspired by [repo-scripts](https://github.com/arcan1s/repo-scripts).
## Features
* Install-configure-forget manager for own repository
* VCS packages support
* Sign support with gpg
* Synchronization to remote services and report generation
* Repository status interface
## Installation and run
* Install package as usual.

View File

@ -29,7 +29,7 @@ find . -type f -name '*src.tar.xz' -not -name "*$VERSION-src.tar.xz" -exec rm -f
read -p "Publish release? [Ny] " -n 1 -r
if [[ $REPLY =~ ^[Yy]$ ]]; then
git add package/archlinux/PKGBUILD
git add package/archlinux/PKGBUILD src/ahriman/version.py
git commit -m "Release $VERSION" && git push
git tag "$VERSION" && git push --tags
fi

View File

@ -1,7 +1,7 @@
# Maintainer: Evgeniy Alekseev
pkgname='ahriman'
pkgver=0.1.0
pkgver=0.10.0
pkgrel=1
pkgdesc="ArcHlinux ReposItory MANager"
arch=('any')
@ -14,14 +14,17 @@ optdepends=('aws-cli: sync to s3'
'darcs: -darcs packages support'
'gnupg: package and repository sign'
'mercurial: -hg packages support'
'python-aiohttp: web server'
'python-aiohttp-jinja2: web server'
'python-jinja: html report generation'
'python-requests: web server'
'rsync: sync by using rsync'
'subversion: -svn packages support')
source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz"
'ahriman.sudoers'
'ahriman.sysusers'
'ahriman.tmpfiles')
sha512sums=('f77b434f5174b2e7a8817bdcc1883e9ded3bd57c30be09b438ec55b65aa83bcd1326f1241ad9ca86552b5fb71f9a15637af7997b37636db9d9ee61c24d235c5c'
sha512sums=('2c811060106aea6f8826cc6beac9f5733370386a43448b359051ea377e233218aae0c5ec3ef7b1ec399fa6a53c02059015b7398b0d88b5a2e7129f167d025539'
'8c9b5b63ac3f7b4d9debaf801a1e9c060877c33d3ecafe18010fcca778e5fa2f2e46909d3d0ff1b229ff8aa978445d8243fd36e1fc104117ed678d5e21901167'
'13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
'55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')

View File

@ -17,7 +17,7 @@ name = aur-clone
root = /var/lib/ahriman
[sign]
enabled = disabled
target =
key =
[report]
@ -27,13 +27,18 @@ target =
path =
homepage =
link_path =
template_path = /usr/share/ahriman/index.jinja2
template_path = /usr/share/ahriman/repo-index.jinja2
[upload]
target =
[rsync_x86_64]
remote =
[s3_x86_64]
bucket =
[rsync_x86_64]
remote =
[web]
host =
port =
templates = /usr/share/ahriman

View File

@ -1,8 +1,8 @@
[loggers]
keys = root,builder,build_details
keys = root,builder,build_details,http
[handlers]
keys = console_handler,build_file_handler,file_handler
keys = console_handler,build_file_handler,file_handler,http_handler
[formatters]
keys = generic_format
@ -25,6 +25,12 @@ level = DEBUG
formatter = generic_format
args = ('/var/log/ahriman/build.log', 'a', 20971520, 20)
[handler_http_handler]
class = logging.handlers.RotatingFileHandler
level = DEBUG
formatter = generic_format
args = ('/var/log/ahriman/http.log', 'a', 20971520, 20)
[formatter_generic_format]
format = %(asctime)s : %(levelname)s : %(funcName)s : %(message)s
datefmt =
@ -45,3 +51,9 @@ level = DEBUG
handlers = build_file_handler
qualname = build_details
propagate = 0
[logger_http]
level = DEBUG
handlers = http_handler
qualname = http
propagate = 0

View File

@ -0,0 +1,11 @@
[Unit]
Description=ArcHlinux ReposItory MANager web server
[Service]
Type=simple
ExecStart=/usr/bin/ahriman --architecture x86_64 web
User=ahriman
Group=ahriman
[Install]
WantedBy=multi-user.target

View File

@ -2,30 +2,83 @@
<html lang="en">
<head>
<title>{{ repository|e }}</title>
<style>
:root {
--color-building: 250, 255, 146;
--color-failed: 255, 94, 94;
--color-pending: 250, 255, 146;
--color-success: 121, 255, 94;
--color-unknown: 197, 197, 197;
}
@keyframes blink-building {
0% { background-color: rgba(var(--color-building), 1.0); }
10% { background-color: rgba(var(--color-building), 0.9); }
20% { background-color: rgba(var(--color-building), 0.8); }
30% { background-color: rgba(var(--color-building), 0.7); }
40% { background-color: rgba(var(--color-building), 0.6); }
50% { background-color: rgba(var(--color-building), 0.5); }
60% { background-color: rgba(var(--color-building), 0.4); }
70% { background-color: rgba(var(--color-building), 0.3); }
80% { background-color: rgba(var(--color-building), 0.2); }
90% { background-color: rgba(var(--color-building), 0.1); }
100% { background-color: rgba(var(--color-building), 0.0); }
}
table, th, td {
padding: 5px;
}
td.package {
font-weight: bolder;
}
td.package-unknown {
background-color: rgba(var(--color-unknown), 1.0);
}
td.package-pending {
background-color: rgba(var(--color-pending), 1.0);
}
td.package-building {
background-color: rgba(var(--color-building), 1.0);
animation-name: blink-building;
animation-duration: 1s;
animation-timing-function: linear;
animation-iteration-count: infinite;
animation-direction: alternate;
}
td.package-failed {
background-color: rgba(var(--color-failed), 1.0);
}
td.package-success {
background-color: rgba(var(--color-success), 1.0);
}
</style>
<script src="https://www.kryogenix.org/code/browser/sorttable/sorttable.js"></script>
</head>
<body>
<h1>{{ repository|e }} ArchLinux custom repository</h1>
<table class="sortable" id="builds">
<tr>
<th>package base</th>
<th>packages</th>
<th>version</th>
<th>architecture</th>
<th>timestamp</th>
<th>status</th>
</tr>
{% if pgp_key is not none %}
<p>All packages are signed with <a href="http://keys.gnupg.net/pks/lookup?search=0x{{ pgp_key|e }}" title="key search">{{ pgp_key|e }}</a>.</p>
{% endif %}
<code>
$ cat /etc/pacman.conf<br>
[{{ repository|e }}]<br>
Server = {{ link_path|e }}
</code>
<p>Packages:</p>
<ul>
{% for package, package_url in packages.items() %}
<li><a href="{{ package_url|e }}" title="{{ package|e }}">{{ package|e }}</a></li>
{% for package in packages %}
<tr>
<td class="package"><a href="{{ package.web_url|e }}" title="{{ package.base|e }}">{{ package.base|e }}</a></td>
<td>{{ package.packages|join("<br>"|safe) }}</td>
<td>{{ package.version|e }}</td>
<td>{{ architecture|e }}</td>
<td>{{ package.timestamp|e }}</td>
<td class="package-{{ package.status|e }}">{{ package.status|e }}</td>
</tr>
{% endfor %}
</ul>
{% if homepage is not none %}
<footer><a href="{{ homepage|e }}" title="homepage">Homepage</a></footer>
{% endif %}
</table>
</body>
</html>

View File

@ -0,0 +1,32 @@
<!doctype html>
<html lang="en">
<head>
<title>{{ repository|e }}</title>
</head>
<body>
<h1>Archlinux custom repository</h1>
{% if pgp_key is not none %}
<p>This repository is signed with <a href="http://keys.gnupg.net/pks/lookup?search=0x{{ pgp_key|e }}" title="key search">{{ pgp_key|e }}</a>.</p>
{% endif %}
<code>
$ cat /etc/pacman.conf<br>
[{{ repository|e }}]<br>
Server = {{ link_path|e }}<br>
SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Package{% if has_package_signed %}Required{% else %}Never{% endif %} TrustedOnly
</code>
<p>Packages:</p>
<ul>
{% for package, package_url in packages.items() %}
<li><a href="{{ package_url|e }}" title="{{ package|e }}">{{ package|e }}</a></li>
{% endfor %}
</ul>
{% if homepage is not none %}
<footer><a href="{{ homepage|e }}" title="homepage">Homepage</a></footer>
{% endif %}
</body>
</html>

View File

@ -42,17 +42,26 @@ setup(
'package/bin/ahriman',
],
data_files=[
('/etc', ['package/etc/ahriman.ini']),
('/etc/ahriman.ini.d', ['package/etc/ahriman.ini.d/logging.ini']),
('/etc', [
'package/etc/ahriman.ini',
]),
('/etc/ahriman.ini.d', [
'package/etc/ahriman.ini.d/logging.ini',
]),
('lib/systemd/system', [
'package/lib/systemd/system/ahriman.service',
'package/lib/systemd/system/ahriman.timer'
'package/lib/systemd/system/ahriman.timer',
'package/lib/systemd/system/ahriman-web.service',
]),
('share/ahriman', [
'package/share/ahriman/index.jinja2',
'package/share/ahriman/repo-index.jinja2',
]),
('share/ahriman', ['package/share/ahriman/index.jinja2']),
],
extras_require={
'html-templates': ['Jinja2'],
'test': ['coverage', 'pytest'],
'web': ['Jinja2', 'aiohttp', 'aiohttp_jinja2', 'requests'],
},
)

View File

@ -20,6 +20,8 @@
import argparse
import os
from typing import Optional
import ahriman.version as version
from ahriman.application.application import Application
@ -38,7 +40,22 @@ def _get_config(config_path: str) -> Configuration:
return config
def _remove_lock(path: str) -> None:
def _lock_check(path: Optional[str]) -> None:
if path is None:
return
if os.path.exists(args.lock):
raise RuntimeError('Another application instance is run')
def _lock_create(path: Optional[str]) -> None:
if path is None:
return
open(path, 'w').close()
def _lock_remove(path: Optional[str]) -> None:
if path is None:
return
if os.path.exists(path):
os.remove(path)
@ -74,6 +91,13 @@ def update(args: argparse.Namespace) -> None:
app.update(packages)
def web(args: argparse.Namespace) -> None:
from ahriman.web.web import run_server, setup_service
config = _get_config(args.config)
app = setup_service(args.architecture, config)
run_server(app)
if __name__ == '__main__':
parser = argparse.ArgumentParser(prog='ahriman', description='ArcHlinux ReposItory MANager')
parser.add_argument('-a', '--architecture', help='target architecture', required=True)
@ -112,20 +136,22 @@ if __name__ == '__main__':
update_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true')
update_parser.set_defaults(fn=update)
web_parser = subparsers.add_parser('web', description='start web server')
web_parser.set_defaults(fn=web, lock=None)
args = parser.parse_args()
if args.force:
_remove_lock(args.lock)
if os.path.exists(args.lock):
raise RuntimeError('Another application instance is run')
_lock_remove(args.lock)
_lock_check(args.lock)
if 'fn' not in args:
parser.print_help()
exit(1)
try:
open(args.lock, 'w').close()
_lock_create(args.lock)
args.fn(args)
finally:
_remove_lock(args.lock)
_lock_remove(args.lock)

View File

@ -44,22 +44,21 @@ class Application:
def get_updates(self, no_aur: bool, no_manual: bool, no_vcs: bool,
log_fn: Callable[[str], None]) -> List[Package]:
updates = []
checked: List[str] = []
if not no_aur:
updates.extend(self.repository.updates_aur(no_vcs, checked))
updates.extend(self.repository.updates_aur(no_vcs))
if not no_manual:
updates.extend(self.repository.updates_manual(checked))
updates.extend(self.repository.updates_manual())
for package in updates:
log_fn(f'{package.name} = {package.version}')
log_fn(f'{package.base} = {package.version}')
return updates
def add(self, names: List[str]) -> None:
def add_manual(name: str) -> None:
package = Package.load(name, self.config.get('aur', 'url'))
Task.fetch(os.path.join(self.repository.paths.manual, package.name), package.url)
Task.fetch(os.path.join(self.repository.paths.manual, package.base), package.git_url)
def add_archive(src: str) -> None:
dst = os.path.join(self.repository.paths.packages, os.path.basename(src))

View File

@ -46,7 +46,7 @@ class Task:
@property
def git_path(self) -> str:
return os.path.join(self.paths.sources, self.package.name)
return os.path.join(self.paths.sources, self.package.base)
@staticmethod
def fetch(local: str, remote: str) -> None:
@ -58,19 +58,19 @@ class Task:
cmd.extend(self.archbuild_flags)
cmd.extend(['--'] + self.makechrootpkg_flags)
cmd.extend(['--'] + self.makepkg_flags)
self.logger.info(f'using {cmd} for {self.package.name}')
self.logger.info(f'using {cmd} for {self.package.base}')
check_output(
*cmd,
exception=BuildFailed(self.package.name),
exception=BuildFailed(self.package.base),
cwd=self.git_path,
logger=self.build_logger)
# well it is not actually correct, but we can deal with it
return check_output('makepkg', '--packagelist',
exception=BuildFailed(self.package.name),
exception=BuildFailed(self.package.base),
cwd=self.git_path).splitlines()
def clone(self, path: Optional[str] = None) -> None:
git_path = path or self.git_path
return Task.fetch(git_path, self.package.url)
return Task.fetch(git_path, self.package.git_url)

View File

@ -21,7 +21,7 @@ import configparser
import os
from logging.config import fileConfig
from typing import List, Optional
from typing import List, Optional, Set
# built-in configparser extension

View File

@ -25,6 +25,11 @@ class BuildFailed(Exception):
Exception.__init__(self, f'Package {package} build failed, check logs for details')
class InitializeException(Exception):
def __init__(self) -> None:
Exception.__init__(self, 'Could not load service')
class InvalidOptionException(Exception):
def __init__(self, value: Any) -> None:
Exception.__init__(self, f'Invalid or unknown option value `{value}`')

View File

@ -46,13 +46,12 @@ class RepoWrapper:
cwd=self.paths.repository,
logger=self.logger)
def remove(self, path: str, package: str) -> None:
os.remove(path)
sign_path = f'{path}.sig'
if os.path.exists(sign_path):
os.remove(sign_path)
def remove(self, prefix: str, package: str) -> None:
for fn in filter(lambda f: f.startswith(prefix), os.listdir(self.paths.repository)):
full_path = os.path.join(self.paths.repository, fn)
os.remove(full_path)
check_output(
'repo-remove', *self.sign_args, self.repo_path, package,
exception=BuildFailed(path),
exception=BuildFailed(package),
cwd=self.paths.repository,
logger=self.logger)

View File

@ -39,10 +39,8 @@ class HTML(Report):
self.template_path = config.get(section, 'template_path')
# base template vars
if SignSettings.from_option(config.get('sign', 'enabled')) != SignSettings.Disabled:
self.pgp_key = config.get('sign', 'key')
else:
self.pgp_key = None
self.sign_targets = [SignSettings.from_option(opt) for opt in config.get_list('sign', 'target')]
self.pgp_key = config.get('sign', 'key', fallback=None)
self.homepage = config.get(section, 'homepage', fallback=None)
self.repository = config.get('repository', 'name')
@ -62,6 +60,8 @@ class HTML(Report):
html = template.render(
homepage=self.homepage,
link_path=self.link_path,
has_package_signed=SignSettings.SignPackages in self.sign_targets,
has_repo_signed=SignSettings.SignRepository in self.sign_targets,
packages=packages,
pgp_key=self.pgp_key,
repository=self.repository)

View File

@ -38,8 +38,7 @@ class Report:
from ahriman.core.report.html import HTML
report: Report = HTML(architecture, config)
else:
from ahriman.core.report.dummy import Dummy
report = Dummy(architecture, config)
report = Report(architecture, config)
try:
report.generate(path)
@ -47,4 +46,4 @@ class Report:
raise ReportFailed(e) from e
def generate(self, path: str) -> None:
raise NotImplementedError
pass

View File

@ -30,6 +30,8 @@ from ahriman.core.report.report import Report
from ahriman.core.sign.gpg_wrapper import GPGWrapper
from ahriman.core.upload.uploader import Uploader
from ahriman.core.util import package_like
from ahriman.core.watcher.client import Client
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths
@ -50,6 +52,8 @@ class Repository:
self.sign = GPGWrapper(config)
self.wrapper = RepoWrapper(self.name, self.paths, self.sign.repository_sign_args)
self.web_report = Client.load(config)
def _clear_build(self) -> None:
for package in os.listdir(self.paths.sources):
shutil.rmtree(os.path.join(self.paths.sources, package))
@ -70,9 +74,7 @@ class Repository:
full_path = os.path.join(self.paths.repository, fn)
try:
local = Package.load(full_path, self.aur_url)
if local.name in result:
continue
result[local.name] = local
result.setdefault(local.base, local).packages.update(local.packages)
except Exception:
self.logger.exception(f'could not load package from {fn}', exc_info=True)
continue
@ -80,6 +82,7 @@ class Repository:
def process_build(self, updates: List[Package]) -> List[str]:
def build_single(package: Package) -> None:
self.web_report.update(package.base, BuildStatusEnum.Building)
task = Task(package, self.architecture, self.config, self.paths)
task.clone()
built = task.build()
@ -91,7 +94,8 @@ class Repository:
try:
build_single(package)
except Exception:
self.logger.exception(f'{package.name} ({self.architecture}) build exception', exc_info=True)
self.web_report.update(package.base, BuildStatusEnum.Failed)
self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True)
continue
self._clear_build()
@ -101,19 +105,22 @@ class Repository:
]
def process_remove(self, packages: List[str]) -> str:
for fn in os.listdir(self.paths.repository):
if not package_like(fn):
continue
full_path = os.path.join(self.paths.repository, fn)
def remove_single(package: str) -> None:
try:
local = Package.load(full_path, self.aur_url)
if local.name not in packages:
continue
self.wrapper.remove(full_path, local.name)
self.wrapper.remove(package, package)
except Exception:
self.logger.exception(f'could not load package from {fn}', exc_info=True)
continue
self.logger.exception(f'could not remove {package}', exc_info=True)
for local in self.packages():
if local.base in packages:
to_remove = local.packages
elif local.packages.intersection(packages):
to_remove = local.packages.intersection(packages)
else:
to_remove = set()
self.web_report.remove(local.base, to_remove)
for package in to_remove:
remove_single(package)
return self.wrapper.repo_path
@ -131,53 +138,55 @@ class Repository:
def process_update(self, packages: List[str]) -> str:
for package in packages:
local = Package.load(package, self.aur_url) # we will use it for status reports
try:
files = self.sign.sign_package(package)
for src in files:
dst = os.path.join(self.paths.repository, os.path.basename(src))
shutil.move(src, dst)
package_fn = os.path.join(self.paths.repository, os.path.basename(package))
self.wrapper.add(package_fn)
self.web_report.add(local, BuildStatusEnum.Success)
except Exception:
self.logger.exception(f'could not process {package}', exc_info=True)
self.web_report.update(local.base, BuildStatusEnum.Failed)
self._clear_packages()
return self.wrapper.repo_path
def updates_aur(self, no_vcs: bool, checked: List[str]) -> List[Package]:
def updates_aur(self, no_vcs: bool) -> List[Package]:
result: List[Package] = []
ignore_list = self.config.get_list(
self.config.get_section_name('build', self.architecture), 'ignore_packages')
for fn in os.listdir(self.paths.repository):
if not package_like(fn):
continue
try:
local = Package.load(os.path.join(self.paths.repository, fn), self.aur_url)
remote = Package.load(local.name, self.aur_url)
except Exception:
self.logger.exception(f'could not load package from {fn}', exc_info=True)
continue
if local.name in checked:
continue
if local.name in ignore_list:
for local in self.packages():
if local.base in ignore_list:
continue
if local.is_vcs and no_vcs:
continue
try:
remote = Package.load(local.base, self.aur_url)
if local.is_outdated(remote):
result.append(remote)
checked.append(local.name)
self.web_report.update(local.base, BuildStatusEnum.Pending)
except Exception:
self.web_report.update(local.base, BuildStatusEnum.Failed)
self.logger.exception(f'could not load remote package {local.base}', exc_info=True)
continue
return result
def updates_manual(self, checked: List[str]) -> List[Package]:
def updates_manual(self) -> List[Package]:
result: List[Package] = []
for fn in os.listdir(self.paths.manual):
try:
local = Package.load(os.path.join(self.paths.manual, fn), self.aur_url)
if local.name in checked:
continue
result.append(local)
checked.append(local.name)
self.web_report.add(local, BuildStatusEnum.Unknown)
except Exception:
self.logger.exception(f'could not add package from {fn}', exc_info=True)
self._clear_manual()
return result

View File

@ -33,12 +33,12 @@ class GPGWrapper:
def __init__(self, config: Configuration) -> None:
self.logger = logging.getLogger('build_details')
self.key = config.get('sign', 'key', fallback=None)
self.sign = SignSettings.from_option(config.get('sign', 'enabled'))
self.target = [SignSettings.from_option(opt) for opt in config.get_list('sign', 'target')]
self.key = config.get('sign', 'key') if self.target else None
@property
def repository_sign_args(self) -> List[str]:
if self.sign != SignSettings.SignRepository:
if SignSettings.SignRepository not in self.target:
return []
return ['--sign', '--key', self.key] if self.key else ['--sign']
@ -58,11 +58,11 @@ class GPGWrapper:
return cmd
def sign_package(self, path: str) -> List[str]:
if self.sign != SignSettings.SignPackages:
if SignSettings.SignPackages not in self.target:
return [path]
return self.process(path)
def sign_repository(self, path: str) -> List[str]:
if self.sign != SignSettings.SignRepository:
if SignSettings.SignRepository not in self.target:
return [path]
return self.process(path)

View File

@ -41,8 +41,7 @@ class Uploader:
from ahriman.core.upload.s3 import S3
uploader = S3(architecture, config)
else:
from ahriman.core.upload.dummy import Dummy
uploader = Dummy(architecture, config)
uploader = Uploader(architecture, config)
try:
uploader.sync(path)
@ -50,4 +49,4 @@ class Uploader:
raise SyncFailed(e) from e
def sync(self, path: str) -> None:
raise NotImplementedError
pass

View File

@ -17,10 +17,3 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.report.report import Report
class Dummy(Report):
def generate(self, path: str) -> None:
pass

View File

@ -0,0 +1,98 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import annotations
import logging
from typing import Any, Dict, Set
from ahriman.core.configuration import Configuration
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package
class Client:
def add(self, package: Package, status: BuildStatusEnum) -> None:
pass
def remove(self, base: str, packages: Set[str]) -> None:
pass
def update(self, base: str, status: BuildStatusEnum) -> None:
pass
@staticmethod
def load(config: Configuration) -> Client:
host = config.get('web', 'host', fallback=None)
port = config.getint('web', 'port', fallback=None)
if host is None or port is None:
return Client()
return WebClient(host, port)
class WebClient(Client):
def __init__(self, host: str, port: int) -> None:
self.logger = logging.getLogger('http')
self.host = host
self.port = port
def _url(self, base: str) -> str:
return f'http://{self.host}:{self.port}/api/v1/packages/{base}'
def add(self, package: Package, status: BuildStatusEnum) -> None:
import requests
payload: Dict[str, Any] = {
'status': status.value,
'base': package.base,
'packages': [p for p in package.packages],
'version': package.version,
'url': package.web_url
}
try:
response = requests.post(self._url(package.base), json=payload)
response.raise_for_status()
except:
self.logger.exception(f'could not add {package.base}', exc_info=True)
def remove(self, base: str, packages: Set[str]) -> None:
if not packages:
return
import requests
try:
response = requests.delete(self._url(base))
response.raise_for_status()
except:
self.logger.exception(f'could not delete {base}', exc_info=True)
def update(self, base: str, status: BuildStatusEnum) -> None:
import requests
payload: Dict[str, Any] = {'status': status.value}
try:
response = requests.post(self._url(base), json=payload)
response.raise_for_status()
except:
self.logger.exception(f'could not update {base}', exc_info=True)

View File

@ -0,0 +1,57 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from typing import Dict, List, Optional, Tuple
from ahriman.core.configuration import Configuration
from ahriman.core.repository import Repository
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.package import Package
class Watcher:
def __init__(self, architecture: str, config: Configuration) -> None:
self.architecture = architecture
self.repository = Repository(architecture, config)
self.known: Dict[str, Tuple[Package, BuildStatus]] = {}
@property
def packages(self) -> List[Tuple[Package, BuildStatus]]:
return [pair for pair in self.known.values()]
def load(self) -> None:
for package in self.repository.packages():
# get status of build or assign unknown
current = self.known.get(package.base)
if current is None:
status = BuildStatus()
else:
_, status = current
self.known[package.base] = (package, status)
def remove(self, base: str) -> None:
self.known.pop(base, None)
def update(self, base: str, status: BuildStatusEnum, package: Optional[Package]) -> None:
if package is None:
package, _ = self.known[base]
full_status = BuildStatus(status)
self.known[base] = (package, full_status)

View File

@ -0,0 +1,43 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import datetime
from enum import Enum
from typing import Optional, Union
class BuildStatusEnum(Enum):
Unknown = 'unknown'
Pending = 'pending'
Building = 'building'
Failed = 'failed'
Success = 'success'
class BuildStatus:
def __init__(self, status: Union[BuildStatusEnum, str, None] = None,
timestamp: Optional[datetime.datetime] = None) -> None:
self.status = BuildStatusEnum(status) if status else BuildStatusEnum.Unknown
self._timestamp = timestamp or datetime.datetime.utcnow()
@property
def timestamp(self) -> str:
return self._timestamp.strftime('%Y-%m-%d %H:%M:%S')

View File

@ -25,10 +25,9 @@ import aur
import os
import tempfile
from configparser import RawConfigParser
from dataclasses import dataclass
from dataclasses import dataclass, field
from srcinfo.parse import parse_srcinfo
from typing import Type
from typing import Set, Type
from ahriman.core.exceptions import InvalidPackageInfo
from ahriman.core.util import check_output
@ -36,19 +35,27 @@ from ahriman.core.util import check_output
@dataclass
class Package:
name: str
base: str
version: str
url: str
remote: bool
aur_url: str
packages: Set[str] = field(default_factory=set)
@property
def git_url(self) -> str:
return f'{self.aur_url}/{self.base}.git'
@property
def is_vcs(self) -> bool:
return self.name.endswith('-bzr') \
or self.name.endswith('-csv')\
or self.name.endswith('-darcs')\
or self.name.endswith('-git')\
or self.name.endswith('-hg')\
or self.name.endswith('-svn')
return self.base.endswith('-bzr') \
or self.base.endswith('-csv')\
or self.base.endswith('-darcs')\
or self.base.endswith('-git')\
or self.base.endswith('-hg')\
or self.base.endswith('-svn')
@property
def web_url(self) -> str:
return f'{self.aur_url}/packages/{self.base}'
# additional method to handle vcs versions
def actual_version(self) -> str:
@ -58,7 +65,7 @@ class Package:
from ahriman.core.build_tools.task import Task
clone_dir = tempfile.mkdtemp()
try:
Task.fetch(clone_dir, self.url)
Task.fetch(clone_dir, self.git_url)
# update pkgver first
check_output('makepkg', '--nodeps', '--noprepare', '--nobuild',
exception=None, cwd=clone_dir)
@ -74,36 +81,33 @@ class Package:
@classmethod
def from_archive(cls: Type[Package], path: str, aur_url: str) -> Package:
name, version = check_output('expac', '-p', '%e %v', path, exception=None).split()
return cls(name, version, f'{aur_url}/{name}.git', False)
package, base, version = check_output('expac', '-p', '%n %e %v', path, exception=None).split()
return cls(base, version, aur_url, {package})
@classmethod
def from_aur(cls: Type[Package], name: str, aur_url: str)-> Package:
package = aur.info(name)
return cls(package.package_base, package.version, f'{aur_url}/{package.package_base}.git', True)
return cls(package.package_base, package.version, aur_url, {package.name})
@classmethod
def from_build(cls: Type[Package], path: str) -> Package:
git_config = RawConfigParser()
git_config.read(os.path.join(path, '.git', 'config'))
def from_build(cls: Type[Package], path: str, aur_url: str) -> Package:
with open(os.path.join(path, '.SRCINFO')) as fn:
src_info, errors = parse_srcinfo(fn.read())
if errors:
raise InvalidPackageInfo(errors)
packages = set(src_info['packages'].keys())
return cls(src_info['pkgbase'], f'{src_info["pkgver"]}-{src_info["pkgrel"]}',
git_config.get('remote "origin"', 'url'), False)
return cls(src_info['pkgbase'], f'{src_info["pkgver"]}-{src_info["pkgrel"]}', aur_url, packages)
@classmethod
def load(cls: Type[Package], path: str, aur_url: str) -> Package:
@staticmethod
def load(path: str, aur_url: str) -> Package:
try:
if os.path.isdir(path):
package: Package = cls.from_build(path)
package: Package = Package.from_build(path, aur_url)
elif os.path.exists(path):
package = cls.from_archive(path, aur_url)
package = Package.from_archive(path, aur_url)
else:
package = cls.from_aur(path, aur_url)
package = Package.from_aur(path, aur_url)
return package
except InvalidPackageInfo:
raise

View File

@ -20,7 +20,6 @@
from __future__ import annotations
from enum import Enum, auto
from typing import Type
from ahriman.core.exceptions import InvalidOptionException
@ -28,8 +27,8 @@ from ahriman.core.exceptions import InvalidOptionException
class ReportSettings(Enum):
HTML = auto()
@classmethod
def from_option(cls: Type[ReportSettings], value: str) -> ReportSettings:
@staticmethod
def from_option(value: str) -> ReportSettings:
if value.lower() in ('html',):
return cls.HTML
return ReportSettings.HTML
raise InvalidOptionException(value)

View File

@ -20,22 +20,18 @@
from __future__ import annotations
from enum import Enum, auto
from typing import Type
from ahriman.core.exceptions import InvalidOptionException
class SignSettings(Enum):
Disabled = auto()
SignPackages = auto()
SignRepository = auto()
@classmethod
def from_option(cls: Type[SignSettings], value: str) -> SignSettings:
if value.lower() in ('no', 'disabled'):
return cls.Disabled
elif value.lower() in ('package', 'packages', 'sign-package'):
return cls.SignPackages
@staticmethod
def from_option(value: str) -> SignSettings:
if value.lower() in ('package', 'packages', 'sign-package'):
return SignSettings.SignPackages
elif value.lower() in ('repository', 'sign-repository'):
return cls.SignRepository
return SignSettings.SignRepository
raise InvalidOptionException(value)

View File

@ -20,7 +20,6 @@
from __future__ import annotations
from enum import Enum, auto
from typing import Type
from ahriman.core.exceptions import InvalidOptionException
@ -29,10 +28,10 @@ class UploadSettings(Enum):
Rsync = auto()
S3 = auto()
@classmethod
def from_option(cls: Type[UploadSettings], value: str) -> UploadSettings:
@staticmethod
def from_option(value: str) -> UploadSettings:
if value.lower() in ('rsync',):
return cls.Rsync
return UploadSettings.Rsync
elif value.lower() in ('s3',):
return cls.S3
return UploadSettings.S3
raise InvalidOptionException(value)

View File

@ -17,4 +17,4 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__version__ = '0.1.0'
__version__ = '0.10.0'

View File

@ -17,10 +17,3 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.upload.uploader import Uploader
class Dummy(Uploader):
def sync(self, path: str) -> None:
pass

View File

@ -0,0 +1,19 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#

View File

@ -0,0 +1,38 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from aiohttp.web import middleware, Request, Response
from logging import Logger
from typing import Callable
from aiohttp.web_exceptions import HTTPClientError
def exception_handler(logger: Logger) -> Callable:
@middleware
async def handle(request: Request, handler: Callable) -> Response:
try:
return await handler(request)
except HTTPClientError:
raise
except Exception:
logger.exception(f'exception during performing request to {request.path}', exc_info=True)
raise
return handle

34
src/ahriman/web/routes.py Normal file
View File

@ -0,0 +1,34 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from aiohttp.web import Application
from ahriman.web.views.index import IndexView
from ahriman.web.views.package import PackageView
from ahriman.web.views.packages import PackagesView
def setup_routes(app: Application) -> None:
app.router.add_get('/', IndexView)
app.router.add_get('/index.html', IndexView)
app.router.add_post('/api/v1/packages', PackagesView)
app.router.add_delete('/api/v1/packages/{package}', PackageView)
app.router.add_post('/api/v1/packages/{package}', PackageView)

View File

@ -0,0 +1,19 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#

View File

@ -0,0 +1,29 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from aiohttp.web import View
from ahriman.core.watcher.watcher import Watcher
class BaseView(View):
@property
def service(self) -> Watcher:
return self.request.app['watcher']

View File

@ -0,0 +1,47 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from typing import Any, Dict
from aiohttp_jinja2 import template
from ahriman.web.views.base import BaseView
class IndexView(BaseView):
@template("index.jinja2")
async def get(self) -> Dict[str, Any]:
# some magic to make it jinja-readable
packages = [
{
'base': package.base,
'packages': [p for p in sorted(package.packages)],
'status': status.status.value,
'timestamp': status.timestamp,
'version': package.version,
'web_url': package.web_url
} for package, status in sorted(self.service.packages, key=lambda item: item[0].base)
]
return {
'architecture': self.service.architecture,
'packages': packages,
'repository': self.service.repository.name,
}

View File

@ -0,0 +1,43 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from aiohttp.web import HTTPOk, Response
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package
from ahriman.web.views.base import BaseView
class PackageView(BaseView):
async def delete(self) -> Response:
base = self.request.match_info['package']
self.service.remove(base)
return HTTPOk()
async def post(self) -> Response:
base = self.request.match_info['package']
data = await self.request.json()
package = Package(**data['package']) if 'package' in data else None
status = BuildStatusEnum(data.get('status', 'unknown'))
self.service.update(base, status, package)
return HTTPOk()

View File

@ -0,0 +1,30 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from aiohttp.web import HTTPOk, Response
from ahriman.web.views.base import BaseView
class PackagesView(BaseView):
async def post(self) -> Response:
self.service.load()
return HTTPOk()

73
src/ahriman/web/web.py Normal file
View File

@ -0,0 +1,73 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import aiohttp_jinja2
import jinja2
import logging
from aiohttp import web
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import InitializeException
from ahriman.core.watcher.watcher import Watcher
from ahriman.web.middlewares.exception_handler import exception_handler
from ahriman.web.routes import setup_routes
async def on_shutdown(app: web.Application) -> None:
app.logger.warning('server terminated')
async def on_startup(app: web.Application) -> None:
app.logger.info('server started')
try:
app['watcher'].load()
except Exception as e:
app.logger.exception('could not load packages', exc_info=True)
raise InitializeException() from e
def run_server(app: web.Application) -> None:
app.logger.info('start server')
web.run_app(app,
host=app['config'].get('web', 'host'),
port=app['config'].getint('web', 'port'),
handle_signals=False)
def setup_service(architecture: str, config: Configuration) -> web.Application:
app = web.Application(logger=logging.getLogger('http'))
app.on_shutdown.append(on_shutdown)
app.on_startup.append(on_startup)
app.middlewares.append(web.normalize_path_middleware(append_slash=False, remove_slash=True))
app.middlewares.append(exception_handler(app.logger))
app.logger.info('setup routes')
setup_routes(app)
app.logger.info('setup templates')
aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader(config.get('web', 'templates')))
app.logger.info('setup configuration')
app['config'] = config
app.logger.info('setup watcher')
app['watcher'] = Watcher(architecture, config)
return app