Compare commits

...

19 Commits

57 changed files with 1520 additions and 351 deletions

View File

@ -1,7 +1,7 @@
# Maintainer: Evgeniy Alekseev # Maintainer: Evgeniy Alekseev
pkgname='ahriman' pkgname='ahriman'
pkgver=0.11.3 pkgver=0.12.1
pkgrel=1 pkgrel=1
pkgdesc="ArcHlinux ReposItory MANager" pkgdesc="ArcHlinux ReposItory MANager"
arch=('any') arch=('any')
@ -23,7 +23,7 @@ optdepends=('aws-cli: sync to s3'
source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz" source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz"
'ahriman.sysusers' 'ahriman.sysusers'
'ahriman.tmpfiles') 'ahriman.tmpfiles')
sha512sums=('ef3c5bf73f95a87f226bc7ca90cee990abf8e7d7dacf56ce5ab346e689019a414a1ffe186a38c75bbf2c9fcc7e04822455e704c67fe2e125c5883dee8066f4bc' sha512sums=('21a1a216af25aeb580f5f2f574e9eed6eb79b74a1038c40a2e1594f18ca0afcd8a64358d11494b87a9a9ea25d5f6be27c39591b124a6ae2d7d4e78fbe44fdd1d'
'13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075' '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
'55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4') '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')
backup=('etc/ahriman.ini' backup=('etc/ahriman.ini'

View File

@ -3,84 +3,40 @@
<head> <head>
<title>{{ repository|e }}</title> <title>{{ repository|e }}</title>
<style> {% include "style.jinja2" %}
:root {
--color-building: 250, 255, 146;
--color-failed: 255, 94, 94;
--color-pending: 250, 255, 146;
--color-success: 121, 255, 94;
--color-unknown: 197, 197, 197;
}
@keyframes blink-building { {% include "sorttable.jinja2" %}
0% { background-color: rgba(var(--color-building), 1.0); } {% include "search.jinja2" %}
10% { background-color: rgba(var(--color-building), 0.9); }
20% { background-color: rgba(var(--color-building), 0.8); }
30% { background-color: rgba(var(--color-building), 0.7); }
40% { background-color: rgba(var(--color-building), 0.6); }
50% { background-color: rgba(var(--color-building), 0.5); }
60% { background-color: rgba(var(--color-building), 0.4); }
70% { background-color: rgba(var(--color-building), 0.3); }
80% { background-color: rgba(var(--color-building), 0.2); }
90% { background-color: rgba(var(--color-building), 0.1); }
100% { background-color: rgba(var(--color-building), 0.0); }
}
table, th, td {
padding: 5px;
}
td.package {
font-weight: bolder;
}
td.package-unknown {
background-color: rgba(var(--color-unknown), 1.0);
}
td.package-pending {
background-color: rgba(var(--color-pending), 1.0);
}
td.package-building {
background-color: rgba(var(--color-building), 1.0);
animation-name: blink-building;
animation-duration: 1s;
animation-timing-function: linear;
animation-iteration-count: infinite;
animation-direction: alternate;
}
td.package-failed {
background-color: rgba(var(--color-failed), 1.0);
}
td.package-success {
background-color: rgba(var(--color-success), 1.0);
}
</style>
<script src="https://www.kryogenix.org/code/browser/sorttable/sorttable.js"></script>
</head> </head>
<body> <body>
<h1>ahriman {{ version|e }}</h1> <div class="root">
<h1>ahriman {{ version|e }} ({{ architecture|e }})<sup class="service-{{ service.status|e }}" title="{{ service.timestamp }}">{{ service.status|e }}</sup></h1>
<table class="sortable" id="builds"> {% include "search-line.jinja2" %}
<tr>
<th>package base</th>
<th>packages</th>
<th>version</th>
<th>architecture</th>
<th>timestamp</th>
<th>status</th>
</tr>
{% for package in packages %} <section class="element">
<tr> <table class="sortable search-table">
<td class="package"><a href="{{ package.web_url|e }}" title="{{ package.base|e }}">{{ package.base|e }}</a></td> <tr class="header">
<td>{{ package.packages|join("<br>"|safe) }}</td> <th>package base</th>
<td>{{ package.version|e }}</td> <th>packages</th>
<td>{{ architecture|e }}</td> <th>version</th>
<td>{{ package.timestamp|e }}</td> <th>last update</th>
<td class="package-{{ package.status|e }}">{{ package.status|e }}</td> <th>status</th>
</tr> </tr>
{% endfor %}
</table> {% for package in packages %}
<tr class="package">
<td class="include-search"><a href="{{ package.web_url|e }}" title="{{ package.base|e }}">{{ package.base|e }}</a></td>
<td class="include-search">{{ package.packages|join("<br>"|safe) }}</td>
<td>{{ package.version|e }}</td>
<td>{{ package.timestamp|e }}</td>
<td class="package-{{ package.status|e }}">{{ package.status|e }}</td>
</tr>
{% endfor %}
</table>
</section>
</div>
</body> </body>
</html> </html>

View File

@ -2,31 +2,53 @@
<html lang="en"> <html lang="en">
<head> <head>
<title>{{ repository|e }}</title> <title>{{ repository|e }}</title>
{% include "style.jinja2" %}
{% include "sorttable.jinja2" %}
{% include "search.jinja2" %}
</head> </head>
<body> <body>
<h1>Archlinux custom repository</h1> <div class="root">
<h1>Archlinux user repository</h1>
{% if pgp_key is not none %} <section class="element">
<p>This repository is signed with <a href="http://keys.gnupg.net/pks/lookup?search=0x{{ pgp_key|e }}" title="key search">{{ pgp_key|e }}</a>.</p> {% if pgp_key is not none %}
{% endif %} <p>This repository is signed with <a href="http://keys.gnupg.net/pks/lookup?search=0x{{ pgp_key|e }}&fingerprint=on&op=index" title="key search">{{ pgp_key|e }}</a> by default.</p>
{% endif %}
<code> <code>
$ cat /etc/pacman.conf<br> $ cat /etc/pacman.conf<br>
[{{ repository|e }}]<br> [{{ repository|e }}]<br>
Server = {{ link_path|e }}<br> Server = {{ link_path|e }}<br>
SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Package{% if has_package_signed %}Required{% else %}Never{% endif %} TrustedOnly SigLevel = Database{% if has_repo_signed %}Required{% else %}Never{% endif %} Package{% if has_package_signed %}Required{% else %}Never{% endif %} TrustedOnly
</code> </code>
</section>
<p>Packages:</p> {% include "search-line.jinja2" %}
<ul>
{% for package, package_url in packages.items() %}
<li><a href="{{ package_url|e }}" title="{{ package|e }}">{{ package|e }}</a></li>
{% endfor %}
</ul>
{% if homepage is not none %} <section class="element">
<footer><a href="{{ homepage|e }}" title="homepage">Homepage</a></footer> <table class="sortable search-table">
{% endif %} <tr class="header">
<th>package</th>
<th>version</th>
<th>installed size</th>
</tr>
{% for package in packages %}
<tr class="package">
<td class="include-search"><a href="{{ link_path|e }}/{{ package.filename|e }}" title="{{ package.name|e }}">{{ package.name|e }}</a></td>
<td>{{ package.version|e }}</td>
<td>{{ package.installed_size|e }}</td>
</tr>
{% endfor %}
</table>
</section>
{% if homepage is not none %}
<footer><a href="{{ homepage|e }}" title="homepage">Homepage</a></footer>
{% endif %}
</div>
</body> </body>
</html> </html>

View File

@ -0,0 +1,3 @@
<section class="element">
<input type="search" id="search" onkeyup="searchInTable()" placeholder="search for package" title="search for package"/>
</section>

View File

@ -0,0 +1,25 @@
<script type="text/javascript">
function searchInTable() {
const input = document.getElementById("search");
const filter = input.value.toLowerCase();
const tables = document.getElementsByClassName("search-table");
for (let i = 0; i < tables.length; i++) {
const tr = tables[i].getElementsByTagName("tr");
// from 1 coz of header
for (let i = 1; i < tr.length; i++) {
let td = tr[i].getElementsByClassName("include-search");
let display = "none";
for (let j = 0; j < td.length; j++) {
if (td[j].tagName.toLowerCase() === "td") {
if (td[j].innerHTML.toLowerCase().indexOf(filter) > -1) {
display = "";
break;
}
}
}
tr[i].style.display = display;
}
}
}
</script>

View File

@ -0,0 +1 @@
<script src="https://www.kryogenix.org/code/browser/sorttable/sorttable.js"></script>

View File

@ -0,0 +1,100 @@
<style>
:root {
--color-building: 255, 255, 146;
--color-failed: 255, 94, 94;
--color-pending: 255, 255, 146;
--color-success: 94, 255, 94;
--color-unknown: 225, 225, 225;
}
@keyframes blink-building {
0% { background-color: rgba(var(--color-building), 1.0); }
10% { background-color: rgba(var(--color-building), 0.9); }
20% { background-color: rgba(var(--color-building), 0.8); }
30% { background-color: rgba(var(--color-building), 0.7); }
40% { background-color: rgba(var(--color-building), 0.6); }
50% { background-color: rgba(var(--color-building), 0.5); }
60% { background-color: rgba(var(--color-building), 0.4); }
70% { background-color: rgba(var(--color-building), 0.3); }
80% { background-color: rgba(var(--color-building), 0.2); }
90% { background-color: rgba(var(--color-building), 0.1); }
100% { background-color: rgba(var(--color-building), 0.0); }
}
div.root {
width: 70%;
padding: 15px 15% 0;
}
section.element {
width: 100%;
padding: 10px 0;
}
code, input, table {
width: inherit;
}
th, td {
padding: 5px;
}
tr.package:nth-child(odd) {
background-color: rgba(255, 255, 255, 1);
}
tr.package:nth-child(even) {
background-color: rgba(235, 235, 255, 1);
}
tr.package:hover {
background-color: rgba(255, 255, 225, 1);
}
tr.header{
background-color: rgba(200, 200, 255, 1);
}
td.package-unknown {
background-color: rgba(var(--color-unknown), 1.0);
}
td.package-pending {
background-color: rgba(var(--color-pending), 1.0);
}
td.package-building {
background-color: rgba(var(--color-building), 1.0);
animation-name: blink-building;
animation-duration: 1s;
animation-timing-function: linear;
animation-iteration-count: infinite;
animation-direction: alternate;
}
td.package-failed {
background-color: rgba(var(--color-failed), 1.0);
}
td.package-success {
background-color: rgba(var(--color-success), 1.0);
}
sup.service-unknown {
font-weight: lighter;
background-color: rgba(var(--color-unknown), 1.0);
}
sup.service-building {
font-weight: lighter;
background-color: rgba(var(--color-building), 1.0);
animation-name: blink-building;
animation-duration: 1s;
animation-timing-function: linear;
animation-iteration-count: infinite;
animation-direction: alternate;
}
sup.service-failed {
font-weight: lighter;
background-color: rgba(var(--color-failed), 1.0);
}
sup.service-success {
font-weight: lighter;
background-color: rgba(var(--color-success), 1.0);
}
</style>

View File

@ -57,6 +57,10 @@ setup(
('share/ahriman', [ ('share/ahriman', [
'package/share/ahriman/build-status.jinja2', 'package/share/ahriman/build-status.jinja2',
'package/share/ahriman/repo-index.jinja2', 'package/share/ahriman/repo-index.jinja2',
'package/share/ahriman/search.jinja2',
'package/share/ahriman/search-line.jinja2',
'package/share/ahriman/sorttable.jinja2',
'package/share/ahriman/style.jinja2',
]), ]),
], ],

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -26,40 +26,78 @@ from ahriman.application.lock import Lock
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
def add(args: argparse.Namespace) -> None: def add(args: argparse.Namespace, config: Configuration) -> None:
Application.from_args(args).add(args.package, args.without_dependencies) '''
add packages callback
:param args: command line args
:param config: configuration instance
'''
Application.from_args(args, config).add(args.package, args.without_dependencies)
def rebuild(args: argparse.Namespace) -> None: def rebuild(args: argparse.Namespace, config: Configuration) -> None:
app = Application.from_args(args) '''
world rebuild callback
:param args: command line args
:param config: configuration instance
'''
app = Application.from_args(args, config)
packages = app.repository.packages() packages = app.repository.packages()
app.update(packages) app.update(packages)
def remove(args: argparse.Namespace) -> None: def remove(args: argparse.Namespace, config: Configuration) -> None:
Application.from_args(args).remove(args.package) '''
remove packages callback
:param args: command line args
:param config: configuration instance
'''
Application.from_args(args, config).remove(args.package)
def report(args: argparse.Namespace) -> None: def report(args: argparse.Namespace, config: Configuration) -> None:
Application.from_args(args).report(args.target) '''
generate report callback
:param args: command line args
:param config: configuration instance
'''
Application.from_args(args, config).report(args.target)
def sync(args: argparse.Namespace) -> None: def sync(args: argparse.Namespace, config: Configuration) -> None:
Application.from_args(args).sync(args.target) '''
sync to remote server callback
:param args: command line args
:param config: configuration instance
'''
Application.from_args(args, config).sync(args.target)
def update(args: argparse.Namespace) -> None: def update(args: argparse.Namespace, config: Configuration) -> None:
app = Application.from_args(args) '''
log_fn = lambda line: print(line) if args.dry_run else app.logger.info(line) update packages callback
:param args: command line args
:param config: configuration instance
'''
# typing workaround
def log_fn(line: str) -> None:
return print(line) if args.dry_run else app.logger.info(line)
app = Application.from_args(args, config)
packages = app.get_updates(args.package, args.no_aur, args.no_manual, args.no_vcs, log_fn) packages = app.get_updates(args.package, args.no_aur, args.no_manual, args.no_vcs, log_fn)
if args.dry_run: if args.dry_run:
return return
app.update(packages) app.update(packages)
def web(args: argparse.Namespace) -> None: def web(args: argparse.Namespace, config: Configuration) -> None:
'''
web server callback
:param args: command line args
:param config: configuration instance
'''
from ahriman.web.web import run_server, setup_service from ahriman.web.web import run_server, setup_service
config = Configuration.from_path(args.config)
app = setup_service(args.architecture, config) app = setup_service(args.architecture, config)
run_server(app, args.architecture) run_server(app, args.architecture)
@ -99,7 +137,8 @@ if __name__ == '__main__':
update_parser = subparsers.add_parser('update', description='run updates') update_parser = subparsers.add_parser('update', description='run updates')
update_parser.add_argument('package', help='filter check by packages', nargs='*') update_parser.add_argument('package', help='filter check by packages', nargs='*')
update_parser.add_argument('--dry-run', help='just perform check for updates, same as check command', action='store_true') update_parser.add_argument(
'--dry-run', help='just perform check for updates, same as check command', action='store_true')
update_parser.add_argument('--no-aur', help='do not check for AUR updates', action='store_true') update_parser.add_argument('--no-aur', help='do not check for AUR updates', action='store_true')
update_parser.add_argument('--no-manual', help='do not include manual updates', action='store_true') update_parser.add_argument('--no-manual', help='do not include manual updates', action='store_true')
update_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true') update_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true')
@ -113,5 +152,6 @@ if __name__ == '__main__':
parser.print_help() parser.print_help()
exit(1) exit(1)
with Lock(args.lock, args.force): config = Configuration.from_path(args.config)
args.fn(args) with Lock(args.lock, args.architecture, args.force, config):
args.fn(args, config)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -34,32 +34,65 @@ from ahriman.models.package import Package
class Application: class Application:
'''
base application class
:ivar architecture: repository architecture
:ivar config: configuration instance
:ivar logger: application logger
:ivar repository: repository instance
'''
def __init__(self, architecture: str, config: Configuration) -> None: def __init__(self, architecture: str, config: Configuration) -> None:
'''
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
self.logger = logging.getLogger('root') self.logger = logging.getLogger('root')
self.config = config self.config = config
self.architecture = architecture self.architecture = architecture
self.repository = Repository(architecture, config) self.repository = Repository(architecture, config)
@classmethod @classmethod
def from_args(cls: Type[Application], args: argparse.Namespace) -> Application: def from_args(cls: Type[Application], args: argparse.Namespace, config: Configuration) -> Application:
config = Configuration.from_path(args.config) '''
constructor which has to be used to build instance from command line args
:param args: command line args
:param config: configuration instance
:return: application instance
'''
return cls(args.architecture, config) return cls(args.architecture, config)
def _known_packages(self) -> Set[str]: def _known_packages(self) -> Set[str]:
known_packages = set() '''
load packages from repository and pacman repositories
:return: list of known packages
'''
known_packages: Set[str] = set()
# local set # local set
for package in self.repository.packages(): for package in self.repository.packages():
known_packages.update(package.packages) known_packages.update(package.packages.keys())
known_packages.update(self.repository.pacman.all_packages()) known_packages.update(self.repository.pacman.all_packages())
return known_packages return known_packages
def _finalize(self) -> None: def _finalize(self) -> None:
'''
generate report and sync to remote server
'''
self.report() self.report()
self.sync() self.sync()
def get_updates(self, filter_packages: List[str], no_aur: bool, no_manual: bool, no_vcs: bool, def get_updates(self, filter_packages: List[str], no_aur: bool, no_manual: bool, no_vcs: bool,
log_fn: Callable[[str], None]) -> List[Package]: log_fn: Callable[[str], None]) -> List[Package]:
'''
get list of packages to run update process
:param filter_packages: do not check every package just specified in the list
:param no_aur: do not check for aur updates
:param no_manual: do not check for manual updates
:param no_vcs: do not check VCS packages
:param log_fn: logger function to log updates
:return: list of out-of-dated packages
'''
updates = [] updates = []
if not no_aur: if not no_aur:
@ -73,6 +106,11 @@ class Application:
return updates return updates
def add(self, names: Iterable[str], without_dependencies: bool) -> None: def add(self, names: Iterable[str], without_dependencies: bool) -> None:
'''
add packages for the next build
:param names: list of package bases to add
:param without_dependencies: if set, dependency check will be disabled
'''
known_packages = self._known_packages() known_packages = self._known_packages()
def add_manual(name: str) -> str: def add_manual(name: str) -> str:
@ -102,18 +140,34 @@ class Application:
process_single(name) process_single(name)
def remove(self, names: Iterable[str]) -> None: def remove(self, names: Iterable[str]) -> None:
'''
remove packages from repository
:param names: list of packages (either base or name) to remove
'''
self.repository.process_remove(names) self.repository.process_remove(names)
self._finalize() self._finalize()
def report(self, target: Optional[Iterable[str]] = None) -> None: def report(self, target: Optional[Iterable[str]] = None) -> None:
'''
generate report
:param target: list of targets to run (e.g. html)
'''
targets = target or None targets = target or None
self.repository.process_report(targets) self.repository.process_report(targets)
def sync(self, target: Optional[Iterable[str]] = None) -> None: def sync(self, target: Optional[Iterable[str]] = None) -> None:
'''
sync to remote server
:param target: list of targets to run (e.g. s3)
'''
targets = target or None targets = target or None
self.repository.process_sync(targets) self.repository.process_sync(targets)
def update(self, updates: Iterable[Package]) -> None: def update(self, updates: Iterable[Package]) -> None:
'''
run package updates
:param updates: list of packages to update
'''
def process_update(paths: Iterable[str]) -> None: def process_update(paths: Iterable[str]) -> None:
self.repository.process_update(paths) self.repository.process_update(paths)
self._finalize() self._finalize()
@ -128,4 +182,4 @@ class Application:
for num, level in enumerate(tree.levels()): for num, level in enumerate(tree.levels()):
self.logger.info(f'processing level #{num} {[package.base for package in level]}') self.logger.info(f'processing level #{num} {[package.base for package in level]}')
packages = self.repository.process_build(level) packages = self.repository.process_build(level)
process_update(packages) process_update(packages)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -17,42 +17,92 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
from __future__ import annotations
import os import os
from typing import Optional from types import TracebackType
from typing import Literal, Optional, Type
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import DuplicateRun from ahriman.core.exceptions import DuplicateRun
from ahriman.core.watcher.client import Client
from ahriman.models.build_status import BuildStatusEnum
class Lock: class Lock:
'''
wrapper for application lock file
:ivar force: remove lock file on start if any
:ivar path: path to lock file if any
:ivar reporter: build status reporter instance
'''
def __init__(self, path: Optional[str], force: bool) -> None: def __init__(self, path: Optional[str], architecture: str, force: bool, config: Configuration) -> None:
self.path = path '''
default constructor
:param path: optional path to lock file, if empty no file lock will be used
:param architecture: repository architecture
:param force: remove lock file on start if any
:param config: configuration instance
'''
self.path = f'{path}_{architecture}' if path is not None else None
self.force = force self.force = force
def __enter__(self): self.reporter = Client.load(architecture, config)
def __enter__(self) -> Lock:
'''
default workflow is the following:
remove lock file if force flag is set
check if there is lock file
create lock file
report to web if enabled
'''
if self.force: if self.force:
self.remove() self.remove()
self.check() self.check()
self.create() self.create()
self.reporter.update_self(BuildStatusEnum.Building)
return self return self
def __exit__(self, exc_type, exc_val, exc_tb): def __exit__(self, exc_type: Optional[Type[Exception]], exc_val: Optional[Exception],
exc_tb: TracebackType) -> Literal[False]:
'''
remove lock file when done
:param exc_type: exception type name if any
:param exc_val: exception raised if any
:param exc_tb: exception traceback if any
:return: always False (do not suppress any exception)
'''
self.remove() self.remove()
status = BuildStatusEnum.Success if exc_val is None else BuildStatusEnum.Failed
self.reporter.update_self(status)
return False
def check(self) -> None: def check(self) -> None:
'''
check if lock file exists, raise exception if it does
'''
if self.path is None: if self.path is None:
return return
if os.path.exists(self.path): if os.path.exists(self.path):
raise DuplicateRun() raise DuplicateRun()
def create(self) -> None: def create(self) -> None:
'''
create lock file
'''
if self.path is None: if self.path is None:
return return
open(self.path, 'w').close() open(self.path, 'w').close()
def remove(self) -> None: def remove(self) -> None:
'''
remove lock file
'''
if self.path is None: if self.path is None:
return return
if os.path.exists(self.path): if os.path.exists(self.path):
os.remove(self.path) os.remove(self.path)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -17,15 +17,23 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
from pyalpm import Handle from pyalpm import Handle # type: ignore
from typing import List, Set from typing import List, Set
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
class Pacman: class Pacman:
'''
alpm wrapper
:ivar handle: pyalpm root `Handle`
'''
def __init__(self, config: Configuration) -> None: def __init__(self, config: Configuration) -> None:
'''
default constructor
:param config: configuration instance
'''
root = config.get('alpm', 'root') root = config.get('alpm', 'root')
pacman_root = config.get('alpm', 'database') pacman_root = config.get('alpm', 'database')
self.handle = Handle(root, pacman_root) self.handle = Handle(root, pacman_root)
@ -33,8 +41,12 @@ class Pacman:
self.handle.register_syncdb(repository, 0) # 0 is pgp_level self.handle.register_syncdb(repository, 0) # 0 is pgp_level
def all_packages(self) -> List[str]: def all_packages(self) -> List[str]:
'''
get list of packages known for alpm
:return: list of package names
'''
result: Set[str] = set() result: Set[str] = set()
for database in self.handle.get_syncdbs(): for database in self.handle.get_syncdbs():
result.update({package.name for package in database.pkgcache}) result.update({package.name for package in database.pkgcache})
return list(result) return list(result)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -28,8 +28,21 @@ from ahriman.models.repository_paths import RepositoryPaths
class Repo: class Repo:
'''
repo-add and repo-remove wrapper
:ivar logger: class logger
:ivar name: repository name
:ivar paths: repository paths instance
:ivar sign_args: additional args which have to be used to sign repository archive
'''
def __init__(self, name: str, paths: RepositoryPaths, sign_args: List[str]) -> None: def __init__(self, name: str, paths: RepositoryPaths, sign_args: List[str]) -> None:
'''
default constructor
:param name: repository name
:param paths: repository paths instance
:param sign_args: additional args which have to be used to sign repository archive
'''
self.logger = logging.getLogger('build_details') self.logger = logging.getLogger('build_details')
self.name = name self.name = name
self.paths = paths self.paths = paths
@ -37,19 +50,32 @@ class Repo:
@property @property
def repo_path(self) -> str: def repo_path(self) -> str:
'''
:return: path to repository database
'''
return os.path.join(self.paths.repository, f'{self.name}.db.tar.gz') return os.path.join(self.paths.repository, f'{self.name}.db.tar.gz')
def add(self, path: str) -> None: def add(self, path: str) -> None:
'''
add new package to repository
:param path: path to archive to add
'''
check_output( check_output(
'repo-add', *self.sign_args, '-R', self.repo_path, path, 'repo-add', *self.sign_args, '-R', self.repo_path, path,
exception=BuildFailed(path), exception=BuildFailed(path),
cwd=self.paths.repository, cwd=self.paths.repository,
logger=self.logger) logger=self.logger)
def remove(self, prefix: str, package: str) -> None: def remove(self, package: str) -> None:
for fn in filter(lambda f: f.startswith(prefix), os.listdir(self.paths.repository)): '''
remove package from repository
:param package: package name to remove
'''
# remove package and signature (if any) from filesystem
for fn in filter(lambda f: f.startswith(package), os.listdir(self.paths.repository)):
full_path = os.path.join(self.paths.repository, fn) full_path = os.path.join(self.paths.repository, fn)
os.remove(full_path) os.remove(full_path)
# remove package from registry
check_output( check_output(
'repo-remove', *self.sign_args, self.repo_path, package, 'repo-remove', *self.sign_args, self.repo_path, package,
exception=BuildFailed(package), exception=BuildFailed(package),

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -31,8 +31,22 @@ from ahriman.models.repository_paths import RepositoryPaths
class Task: class Task:
'''
base package build task
:ivar build_logger: logger for build process
:ivar logger: class logger
:ivar package: package definitions
:ivar paths: repository paths instance
'''
def __init__(self, package: Package, architecture: str, config: Configuration, paths: RepositoryPaths) -> None: def __init__(self, package: Package, architecture: str, config: Configuration, paths: RepositoryPaths) -> None:
'''
default constructor
:param package: package definitions
:param architecture: repository architecture
:param config: configuration instance
:param paths: repository paths instance
'''
self.logger = logging.getLogger('builder') self.logger = logging.getLogger('builder')
self.build_logger = logging.getLogger('build_details') self.build_logger = logging.getLogger('build_details')
self.package = package self.package = package
@ -46,14 +60,26 @@ class Task:
@property @property
def git_path(self) -> str: def git_path(self) -> str:
'''
:return: path to clone package from git
'''
return os.path.join(self.paths.sources, self.package.base) return os.path.join(self.paths.sources, self.package.base)
@staticmethod @staticmethod
def fetch(local: str, remote: str) -> None: def fetch(local: str, remote: str) -> None:
'''
fetch package from git
:param local: local path to fetch
:param remote: remote target (from where to fetch)
'''
shutil.rmtree(local, ignore_errors=True) # remove in case if file exists shutil.rmtree(local, ignore_errors=True) # remove in case if file exists
check_output('git', 'clone', remote, local, exception=None) check_output('git', 'clone', remote, local, exception=None)
def build(self) -> List[str]: def build(self) -> List[str]:
'''
run package build
:return: paths of produced packages
'''
cmd = [self.build_command, '-r', self.paths.chroot] cmd = [self.build_command, '-r', self.paths.chroot]
cmd.extend(self.archbuild_flags) cmd.extend(self.archbuild_flags)
cmd.extend(['--'] + self.makechrootpkg_flags) cmd.extend(['--'] + self.makechrootpkg_flags)
@ -72,5 +98,9 @@ class Task:
cwd=self.git_path).splitlines() cwd=self.git_path).splitlines()
def clone(self, path: Optional[str] = None) -> None: def clone(self, path: Optional[str] = None) -> None:
'''
fetch package from git
:param path: optional local path to fetch. If not set default path will be used
'''
git_path = path or self.git_path git_path = path or self.git_path
return Task.fetch(git_path, self.package.git_url) return Task.fetch(git_path, self.package.git_url)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -26,40 +26,73 @@ from logging.config import fileConfig
from typing import List, Optional, Type from typing import List, Optional, Type
# built-in configparser extension
class Configuration(configparser.RawConfigParser): class Configuration(configparser.RawConfigParser):
'''
extension for built-in configuration parser
:ivar path: path to root configuration file
'''
def __init__(self) -> None: def __init__(self) -> None:
'''
default constructor
'''
configparser.RawConfigParser.__init__(self, allow_no_value=True) configparser.RawConfigParser.__init__(self, allow_no_value=True)
self.path: Optional[str] = None self.path: Optional[str] = None
@property @property
def include(self) -> str: def include(self) -> str:
'''
:return: path to directory with configuration includes
'''
return self.get('settings', 'include') return self.get('settings', 'include')
@classmethod @classmethod
def from_path(cls: Type[Configuration], path: str) -> Configuration: def from_path(cls: Type[Configuration], path: str) -> Configuration:
'''
constructor with full object initialization
:param path: path to root configuration file
:return: configuration instance
'''
config = cls() config = cls()
config.load(path) config.load(path)
config.load_logging() config.load_logging()
return config return config
def getlist(self, section: str, key: str) -> List[str]: def getlist(self, section: str, key: str) -> List[str]:
'''
get space separated string list option
:param section: section name
:param key: key name
:return: list of string if option is set, empty list otherwise
'''
raw = self.get(section, key, fallback=None) raw = self.get(section, key, fallback=None)
if not raw: # empty string or none if not raw: # empty string or none
return [] return []
return raw.split() return raw.split()
def get_section_name(self, prefix: str, suffix: str) -> str: def get_section_name(self, prefix: str, suffix: str) -> str:
'''
check if there is `prefix`_`suffix` section and return it on success. Return `prefix` otherwise
:param prefix: section name prefix
:param suffix: section name suffix (e.g. architecture name)
:return: found section name
'''
probe = f'{prefix}_{suffix}' probe = f'{prefix}_{suffix}'
return probe if self.has_section(probe) else prefix return probe if self.has_section(probe) else prefix
def load(self, path: str) -> None: def load(self, path: str) -> None:
'''
fully load configuration
:param path: path to root configuration file
'''
self.path = path self.path = path
self.read(self.path) self.read(self.path)
self.load_includes() self.load_includes()
def load_includes(self) -> None: def load_includes(self) -> None:
'''
load configuration includes
'''
try: try:
for conf in filter(lambda p: p.endswith('.ini'), sorted(os.listdir(self.include))): for conf in filter(lambda p: p.endswith('.ini'), sorted(os.listdir(self.include))):
self.read(os.path.join(self.include, conf)) self.read(os.path.join(self.include, conf))
@ -67,4 +100,7 @@ class Configuration(configparser.RawConfigParser):
pass pass
def load_logging(self) -> None: def load_logging(self) -> None:
'''
setup logging settings from configuration
'''
fileConfig(self.get('settings', 'logging')) fileConfig(self.get('settings', 'logging'))

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -21,35 +21,87 @@ from typing import Any
class BuildFailed(Exception): class BuildFailed(Exception):
'''
base exception for failed builds
'''
def __init__(self, package: str) -> None: def __init__(self, package: str) -> None:
'''
default constructor
:param package: package base raised exception
'''
Exception.__init__(self, f'Package {package} build failed, check logs for details') Exception.__init__(self, f'Package {package} build failed, check logs for details')
class DuplicateRun(Exception): class DuplicateRun(Exception):
'''
exception which will be raised if there is another application instance
'''
def __init__(self) -> None: def __init__(self) -> None:
'''
default constructor
'''
Exception.__init__(self, 'Another application instance is run') Exception.__init__(self, 'Another application instance is run')
class InitializeException(Exception): class InitializeException(Exception):
'''
base service initialization exception
'''
def __init__(self) -> None: def __init__(self) -> None:
'''
default constructor
'''
Exception.__init__(self, 'Could not load service') Exception.__init__(self, 'Could not load service')
class InvalidOption(Exception): class InvalidOption(Exception):
'''
exception which will be raised on configuration errors
'''
def __init__(self, value: Any) -> None: def __init__(self, value: Any) -> None:
'''
default constructor
:param value: option value
'''
Exception.__init__(self, f'Invalid or unknown option value `{value}`') Exception.__init__(self, f'Invalid or unknown option value `{value}`')
class InvalidPackageInfo(Exception): class InvalidPackageInfo(Exception):
'''
exception which will be raised on package load errors
'''
def __init__(self, details: Any) -> None: def __init__(self, details: Any) -> None:
'''
default constructor
:param details: error details
'''
Exception.__init__(self, f'There are errors during reading package information: `{details}`') Exception.__init__(self, f'There are errors during reading package information: `{details}`')
class ReportFailed(Exception): class ReportFailed(Exception):
'''
report generation exception
'''
def __init__(self) -> None: def __init__(self) -> None:
'''
default constructor
'''
Exception.__init__(self, 'Report failed') Exception.__init__(self, 'Report failed')
class SyncFailed(Exception): class SyncFailed(Exception):
'''
remote synchronization exception
'''
def __init__(self) -> None: def __init__(self) -> None:
Exception.__init__(self, 'Sync failed') '''
default constructor
'''
Exception.__init__(self, 'Sync failed')

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -20,51 +20,87 @@
import jinja2 import jinja2
import os import os
from typing import Dict from typing import Callable, Dict, Iterable
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.report.report import Report from ahriman.core.report.report import Report
from ahriman.core.util import package_like from ahriman.models.package import Package
from ahriman.models.package_desciption import PackageDescription
from ahriman.models.sign_settings import SignSettings from ahriman.models.sign_settings import SignSettings
class HTML(Report): class HTML(Report):
'''
html report generator
It uses jinja2 templates for report generation, the following variables are allowed:
homepage - link to homepage, string, optional
link_path - prefix fo packages to download, string, required
has_package_signed - True in case if package sign enabled, False otherwise, required
has_repo_signed - True in case if repository database sign enabled, False otherwise, required
packages - sorted list of packages properties: filename, installed_size, name, version. Required
pgp_key - default PGP key ID, string, optional
repository - repository name, string, required
:ivar homepage: homepage link if any (for footer)
:ivar link_path: prefix fo packages to download
:ivar name: repository name
:ivar pgp_key: default PGP key
:ivar report_path: output path to html report
:ivar sign_targets: targets to sign enabled in configuration
:ivar tempate_path: path to directory with jinja templates
'''
def __init__(self, architecture: str, config: Configuration) -> None: def __init__(self, architecture: str, config: Configuration) -> None:
'''
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
Report.__init__(self, architecture, config) Report.__init__(self, architecture, config)
section = config.get_section_name('html', architecture) section = config.get_section_name('html', architecture)
self.report_path = config.get(section, 'path') self.report_path = config.get(section, 'path')
self.link_path = config.get(section, 'link_path') self.link_path = config.get(section, 'link_path')
self.template_path = config.get(section, 'template_path') self.template_path = config.get(section, 'template_path')
# base template vars # base template vars
self.sign_targets = [SignSettings.from_option(opt) for opt in config.getlist('sign', 'target')]
self.pgp_key = config.get('sign', 'key', fallback=None)
self.homepage = config.get(section, 'homepage', fallback=None) self.homepage = config.get(section, 'homepage', fallback=None)
self.repository = config.get('repository', 'name') self.name = config.get('repository', 'name')
def generate(self, path: str) -> None: sign_section = config.get_section_name('sign', architecture)
self.sign_targets = [SignSettings.from_option(opt) for opt in config.getlist(sign_section, 'target')]
self.pgp_key = config.get(sign_section, 'key') if self.sign_targets else None
def generate(self, packages: Iterable[Package]) -> None:
'''
generate report for the specified packages
:param packages: list of packages to generate report
'''
# idea comes from https://stackoverflow.com/a/38642558 # idea comes from https://stackoverflow.com/a/38642558
templates_dir, template_name = os.path.split(self.template_path) templates_dir, template_name = os.path.split(self.template_path)
loader = jinja2.FileSystemLoader(searchpath=templates_dir) loader = jinja2.FileSystemLoader(searchpath=templates_dir)
environment = jinja2.Environment(loader=loader) environment = jinja2.Environment(loader=loader)
template = environment.get_template(template_name) template = environment.get_template(template_name)
packages: Dict[str, str] = {} content = [
for fn in sorted(os.listdir(path)): {
if not package_like(fn): 'filename': properties.filename,
continue 'installed_size': PackageDescription.size_to_str(properties.installed_size),
packages[fn] = f'{self.link_path}/{fn}' 'name': package,
'version': base.version
} for base in packages for package, properties in base.packages.items()
]
comparator: Callable[[Dict[str, str]], str] = lambda item: item['filename']
html = template.render( html = template.render(
homepage=self.homepage, homepage=self.homepage,
link_path=self.link_path, link_path=self.link_path,
has_package_signed=SignSettings.SignPackages in self.sign_targets, has_package_signed=SignSettings.SignPackages in self.sign_targets,
has_repo_signed=SignSettings.SignRepository in self.sign_targets, has_repo_signed=SignSettings.SignRepository in self.sign_targets,
packages=packages, packages=sorted(content, key=comparator),
pgp_key=self.pgp_key, pgp_key=self.pgp_key,
repository=self.repository) repository=self.name)
with open(self.report_path, 'w') as out: with open(self.report_path, 'w') as out:
out.write(html) out.write(html)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -19,20 +19,41 @@
# #
import logging import logging
from typing import Iterable
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import ReportFailed from ahriman.core.exceptions import ReportFailed
from ahriman.models.package import Package
from ahriman.models.report_settings import ReportSettings from ahriman.models.report_settings import ReportSettings
class Report: class Report:
'''
base report generator
:ivar architecture: repository architecture
:ivar config: configuration instance
:ivar logger: class logger
'''
def __init__(self, architecture: str, config: Configuration) -> None: def __init__(self, architecture: str, config: Configuration) -> None:
'''
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
self.logger = logging.getLogger('builder') self.logger = logging.getLogger('builder')
self.architecture = architecture self.architecture = architecture
self.config = config self.config = config
@staticmethod @staticmethod
def run(architecture: str, config: Configuration, target: str, path: str) -> None: def run(architecture: str, config: Configuration, target: str, packages: Iterable[Package]) -> None:
'''
run report generation
:param architecture: repository architecture
:param config: configuration instance
:param target: target to generate report (e.g. html)
:param packages: list of packages to generate report
'''
provider = ReportSettings.from_option(target) provider = ReportSettings.from_option(target)
if provider == ReportSettings.HTML: if provider == ReportSettings.HTML:
from ahriman.core.report.html import HTML from ahriman.core.report.html import HTML
@ -41,10 +62,14 @@ class Report:
report = Report(architecture, config) report = Report(architecture, config)
try: try:
report.generate(path) report.generate(packages)
except Exception: except Exception:
report.logger.exception('report generation failed', exc_info=True) report.logger.exception('report generation failed', exc_info=True)
raise ReportFailed() raise ReportFailed()
def generate(self, path: str) -> None: def generate(self, packages: Iterable[Package]) -> None:
pass '''
generate report for the specified packages
:param packages: list of packages to generate report
'''
pass

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -37,8 +37,26 @@ from ahriman.models.repository_paths import RepositoryPaths
class Repository: class Repository:
'''
base repository control class
:ivar architecture: repository architecture
:ivar aur_url: base AUR url
:ivar config: configuration instance
:ivar logger: class logger
:ivar name: repository name
:ivar pacman: alpm wrapper instance
:ivar paths: repository paths instance
:ivar repo: repo commands wrapper instance
:ivar reporter: build status reporter instance
:ivar sign: GPG wrapper instance
'''
def __init__(self, architecture: str, config: Configuration) -> None: def __init__(self, architecture: str, config: Configuration) -> None:
'''
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
self.logger = logging.getLogger('builder') self.logger = logging.getLogger('builder')
self.architecture = architecture self.architecture = architecture
self.config = config self.config = config
@ -52,21 +70,34 @@ class Repository:
self.pacman = Pacman(config) self.pacman = Pacman(config)
self.sign = GPG(architecture, config) self.sign = GPG(architecture, config)
self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args) self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
self.web = Client.load(architecture, config) self.reporter = Client.load(architecture, config)
def _clear_build(self) -> None: def _clear_build(self) -> None:
'''
clear sources directory
'''
for package in os.listdir(self.paths.sources): for package in os.listdir(self.paths.sources):
shutil.rmtree(os.path.join(self.paths.sources, package)) shutil.rmtree(os.path.join(self.paths.sources, package))
def _clear_manual(self) -> None: def _clear_manual(self) -> None:
'''
clear directory with manual package updates
'''
for package in os.listdir(self.paths.manual): for package in os.listdir(self.paths.manual):
shutil.rmtree(os.path.join(self.paths.manual, package)) shutil.rmtree(os.path.join(self.paths.manual, package))
def _clear_packages(self) -> None: def _clear_packages(self) -> None:
'''
clear directory with built packages (NOT repository itself)
'''
for package in self.packages_built(): for package in self.packages_built():
os.remove(package) os.remove(package)
def packages(self) -> List[Package]: def packages(self) -> List[Package]:
'''
generate list of repository packages
:return: list of packages properties
'''
result: Dict[str, Package] = {} result: Dict[str, Package] = {}
for fn in os.listdir(self.paths.repository): for fn in os.listdir(self.paths.repository):
if not package_like(fn): if not package_like(fn):
@ -81,14 +112,23 @@ class Repository:
return list(result.values()) return list(result.values())
def packages_built(self) -> List[str]: def packages_built(self) -> List[str]:
'''
get list of files in built packages directory
:return: list of filenames from the directory
'''
return [ return [
os.path.join(self.paths.packages, fn) os.path.join(self.paths.packages, fn)
for fn in os.listdir(self.paths.packages) for fn in os.listdir(self.paths.packages)
] ]
def process_build(self, updates: Iterable[Package]) -> List[str]: def process_build(self, updates: Iterable[Package]) -> List[str]:
'''
build packages
:param updates: list of packages properties to build
:return: `packages_built`
'''
def build_single(package: Package) -> None: def build_single(package: Package) -> None:
self.web.set_building(package.base) self.reporter.set_building(package.base)
task = Task(package, self.architecture, self.config, self.paths) task = Task(package, self.architecture, self.config, self.paths)
task.clone() task.clone()
built = task.build() built = task.build()
@ -100,7 +140,7 @@ class Repository:
try: try:
build_single(package) build_single(package)
except Exception: except Exception:
self.web.set_failed(package.base) self.reporter.set_failed(package.base)
self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True) self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True)
continue continue
self._clear_build() self._clear_build()
@ -108,38 +148,57 @@ class Repository:
return self.packages_built() return self.packages_built()
def process_remove(self, packages: Iterable[str]) -> str: def process_remove(self, packages: Iterable[str]) -> str:
'''
remove packages from list
:param packages: list of package names or bases to rmeove
:return: path to repository database
'''
def remove_single(package: str) -> None: def remove_single(package: str) -> None:
try: try:
self.repo.remove(package, package) self.repo.remove(package)
except Exception: except Exception:
self.logger.exception(f'could not remove {package}', exc_info=True) self.logger.exception(f'could not remove {package}', exc_info=True)
requested = set(packages)
for local in self.packages(): for local in self.packages():
if local.base in packages: if local.base in packages:
to_remove = local.packages to_remove = set(local.packages.keys())
elif local.packages.intersection(packages): self.reporter.remove(local.base) # we only update status page in case of base removal
to_remove = local.packages.intersection(packages) elif requested.intersection(local.packages.keys()):
to_remove = requested.intersection(local.packages.keys())
else: else:
to_remove = set() to_remove = set()
self.web.remove(local.base, to_remove)
for package in to_remove: for package in to_remove:
remove_single(package) remove_single(package)
return self.repo.repo_path return self.repo.repo_path
def process_report(self, targets: Optional[Iterable[str]]) -> None: def process_report(self, targets: Optional[Iterable[str]]) -> None:
'''
generate reports
:param targets: list of targets to generate reports. Configuration option will be used if it is not set
'''
if targets is None: if targets is None:
targets = self.config.getlist('report', 'target') targets = self.config.getlist('report', 'target')
for target in targets: for target in targets:
Report.run(self.architecture, self.config, target, self.paths.repository) Report.run(self.architecture, self.config, target, self.packages())
def process_sync(self, targets: Optional[Iterable[str]]) -> None: def process_sync(self, targets: Optional[Iterable[str]]) -> None:
'''
process synchronization to remote servers
:param targets: list of targets to sync. Configuration option will be used if it is not set
'''
if targets is None: if targets is None:
targets = self.config.getlist('upload', 'target') targets = self.config.getlist('upload', 'target')
for target in targets: for target in targets:
Uploader.run(self.architecture, self.config, target, self.paths.repository) Uploader.run(self.architecture, self.config, target, self.paths.repository)
def process_update(self, packages: Iterable[str]) -> str: def process_update(self, packages: Iterable[str]) -> str:
'''
sign packages, add them to repository and update repository database
:param packages: list of filenames to run
:return: path to repository database
'''
for package in packages: for package in packages:
local = Package.load(package, self.pacman, self.aur_url) # we will use it for status reports local = Package.load(package, self.pacman, self.aur_url) # we will use it for status reports
try: try:
@ -149,15 +208,21 @@ class Repository:
shutil.move(src, dst) shutil.move(src, dst)
package_fn = os.path.join(self.paths.repository, os.path.basename(package)) package_fn = os.path.join(self.paths.repository, os.path.basename(package))
self.repo.add(package_fn) self.repo.add(package_fn)
self.web.set_success(local) self.reporter.set_success(local)
except Exception: except Exception:
self.logger.exception(f'could not process {package}', exc_info=True) self.logger.exception(f'could not process {package}', exc_info=True)
self.web.set_failed(local.base) self.reporter.set_failed(local.base)
self._clear_packages() self._clear_packages()
return self.repo.repo_path return self.repo.repo_path
def updates_aur(self, filter_packages: Iterable[str], no_vcs: bool) -> List[Package]: def updates_aur(self, filter_packages: Iterable[str], no_vcs: bool) -> List[Package]:
'''
check AUR for updates
:param filter_packages: do not check every package just specified in the list
:param no_vcs: do not check VCS packages
:return: list of packages which are out-of-dated
'''
result: List[Package] = [] result: List[Package] = []
build_section = self.config.get_section_name('build', self.architecture) build_section = self.config.get_section_name('build', self.architecture)
@ -175,24 +240,28 @@ class Repository:
remote = Package.load(local.base, self.pacman, self.aur_url) remote = Package.load(local.base, self.pacman, self.aur_url)
if local.is_outdated(remote): if local.is_outdated(remote):
result.append(remote) result.append(remote)
self.web.set_pending(local.base) self.reporter.set_pending(local.base)
except Exception: except Exception:
self.web.set_failed(local.base) self.reporter.set_failed(local.base)
self.logger.exception(f'could not load remote package {local.base}', exc_info=True) self.logger.exception(f'could not load remote package {local.base}', exc_info=True)
continue continue
return result return result
def updates_manual(self) -> List[Package]: def updates_manual(self) -> List[Package]:
'''
check for packages for which manual update has been requested
:return: list of packages which are out-of-dated
'''
result: List[Package] = [] result: List[Package] = []
for fn in os.listdir(self.paths.manual): for fn in os.listdir(self.paths.manual):
try: try:
local = Package.load(os.path.join(self.paths.manual, fn), self.pacman, self.aur_url) local = Package.load(os.path.join(self.paths.manual, fn), self.pacman, self.aur_url)
result.append(local) result.append(local)
self.web.set_unknown(local) self.reporter.set_unknown(local)
except Exception: except Exception:
self.logger.exception(f'could not add package from {fn}', exc_info=True) self.logger.exception(f'could not add package from {fn}', exc_info=True)
self._clear_manual() self._clear_manual()
return result return result

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -29,8 +29,21 @@ from ahriman.models.sign_settings import SignSettings
class GPG: class GPG:
'''
gnupg wrapper
:ivar architecture: repository architecture
:ivar config: configuration instance
:ivar default_key: default PGP key ID to use
:ivar logger: class logger
:ivar target: list of targets to sign (repository, package etc)
'''
def __init__(self, architecture: str, config: Configuration) -> None: def __init__(self, architecture: str, config: Configuration) -> None:
'''
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
self.logger = logging.getLogger('build_details') self.logger = logging.getLogger('build_details')
self.config = config self.config = config
self.section = config.get_section_name('sign', architecture) self.section = config.get_section_name('sign', architecture)
@ -39,11 +52,20 @@ class GPG:
@property @property
def repository_sign_args(self) -> List[str]: def repository_sign_args(self) -> List[str]:
'''
:return: command line arguments for repo-add command to sign database
'''
if SignSettings.SignRepository not in self.target: if SignSettings.SignRepository not in self.target:
return [] return []
return ['--sign', '--key', self.default_key] return ['--sign', '--key', self.default_key]
def process(self, path: str, key: str) -> List[str]: def process(self, path: str, key: str) -> List[str]:
'''
gpg command wrapper
:param path: path to file to sign
:param key: PGP key ID
:return: list of generated files including original file
'''
check_output( check_output(
*self.sign_cmd(path, key), *self.sign_cmd(path, key),
exception=BuildFailed(path), exception=BuildFailed(path),
@ -52,18 +74,33 @@ class GPG:
return [path, f'{path}.sig'] return [path, f'{path}.sig']
def sign_cmd(self, path: str, key: str) -> List[str]: def sign_cmd(self, path: str, key: str) -> List[str]:
cmd = ['gpg'] '''
cmd.extend(['-u', key]) gpg command to run
cmd.extend(['-b', path]) :param path: path to file to sign
return cmd :param key: PGP key ID
:return: gpg command with all required arguments
'''
return ['gpg', '-u', key, '-b', path]
def sign_package(self, path: str, base: str) -> List[str]: def sign_package(self, path: str, base: str) -> List[str]:
'''
sign package if required by configuration
:param path: path to file to sign
:param base: package base required to check for key overrides
:return: list of generated files including original file
'''
if SignSettings.SignPackages not in self.target: if SignSettings.SignPackages not in self.target:
return [path] return [path]
key = self.config.get(self.section, f'key_{base}', fallback=self.default_key) key = self.config.get(self.section, f'key_{base}', fallback=self.default_key)
return self.process(path, key) return self.process(path, key)
def sign_repository(self, path: str) -> List[str]: def sign_repository(self, path: str) -> List[str]:
'''
sign repository if required by configuration
:note: more likely you just want to pass `repository_sign_args` to repo wrapper
:param path: path to repository database
:return: list of generated files including original file
'''
if SignSettings.SignRepository not in self.target: if SignSettings.SignRepository not in self.target:
return [path] return [path]
return self.process(path, self.default_key) return self.process(path, self.default_key)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -28,15 +28,71 @@ from ahriman.core.build_tools.task import Task
from ahriman.models.package import Package from ahriman.models.package import Package
class Leaf:
'''
tree leaf implementation
:ivar dependencies: list of package dependencies
:ivar package: leaf package properties
'''
def __init__(self, package: Package) -> None:
'''
default constructor
:param package: package properties
'''
self.package = package
self.dependencies: Set[str] = set()
@property
def items(self) -> Iterable[str]:
'''
:return: packages containing in this leaf
'''
return self.package.packages.keys()
def is_root(self, packages: Iterable[Leaf]) -> bool:
'''
check if package depends on any other package from list of not
:param packages: list of known leaves
:return: True if any of packages is dependency of the leaf, False otherwise
'''
for leaf in packages:
if self.dependencies.intersection(leaf.items):
return False
return True
def load_dependencies(self) -> None:
'''
load dependencies for the leaf
'''
clone_dir = tempfile.mkdtemp()
try:
Task.fetch(clone_dir, self.package.git_url)
self.dependencies = Package.dependencies(clone_dir)
finally:
shutil.rmtree(clone_dir, ignore_errors=True)
class Tree: class Tree:
'''
dependency tree implementation
:ivar leaves: list of tree leaves
'''
def __init__(self) -> None: def __init__(self) -> None:
self.packages: List[Leaf] = [] '''
default constructor
'''
self.leaves: List[Leaf] = []
def levels(self) -> List[List[Package]]: def levels(self) -> List[List[Package]]:
'''
get build levels starting from the packages which do not require any other package to build
:return: list of packages lists
'''
result: List[List[Package]] = [] result: List[List[Package]] = []
unprocessed = [leaf for leaf in self.packages] unprocessed = [leaf for leaf in self.leaves]
while unprocessed: while unprocessed:
result.append([leaf.package for leaf in unprocessed if leaf.is_root(unprocessed)]) result.append([leaf.package for leaf in unprocessed if leaf.is_root(unprocessed)])
unprocessed = [leaf for leaf in unprocessed if not leaf.is_root(unprocessed)] unprocessed = [leaf for leaf in unprocessed if not leaf.is_root(unprocessed)]
@ -44,32 +100,11 @@ class Tree:
return result return result
def load(self, packages: Iterable[Package]) -> None: def load(self, packages: Iterable[Package]) -> None:
'''
load tree from packages
:param packages: packages list
'''
for package in packages: for package in packages:
leaf = Leaf(package) leaf = Leaf(package)
leaf.load_dependencies() leaf.load_dependencies()
self.packages.append(leaf) self.leaves.append(leaf)
class Leaf:
def __init__(self, package: Package) -> None:
self.package = package
self.dependencies: Set[str] = set()
def is_root(self, packages: Iterable[Leaf]) -> bool:
'''
:param packages:
:return: true if any of packages is dependency of the leaf, false otherwise
'''
for package in packages:
if package.package.packages.intersection(self.dependencies):
return False
return True
def load_dependencies(self) -> None:
clone_dir = tempfile.mkdtemp()
try:
Task.fetch(clone_dir, self.package.git_url)
self.dependencies = Package.dependencies(clone_dir)
finally:
shutil.rmtree(clone_dir, ignore_errors=True)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -23,13 +23,26 @@ from ahriman.core.util import check_output
class Rsync(Uploader): class Rsync(Uploader):
'''
rsync wrapper
:ivar remote: remote address to sync
'''
def __init__(self, architecture: str, config: Configuration) -> None: def __init__(self, architecture: str, config: Configuration) -> None:
'''
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
Uploader.__init__(self, architecture, config) Uploader.__init__(self, architecture, config)
section = config.get_section_name('rsync', architecture) section = config.get_section_name('rsync', architecture)
self.remote = config.get(section, 'remote') self.remote = config.get(section, 'remote')
def sync(self, path: str) -> None: def sync(self, path: str) -> None:
'''
sync data to remote server
:param path: local path to sync
'''
check_output('rsync', '--archive', '--verbose', '--compress', '--partial', '--progress', '--delete', path, self.remote, check_output('rsync', '--archive', '--verbose', '--compress', '--partial', '--progress', '--delete', path, self.remote,
exception=None, exception=None,
logger=self.logger) logger=self.logger)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -23,13 +23,26 @@ from ahriman.core.util import check_output
class S3(Uploader): class S3(Uploader):
'''
aws-cli wrapper
:ivar bucket: full bucket name
'''
def __init__(self, architecture: str, config: Configuration) -> None: def __init__(self, architecture: str, config: Configuration) -> None:
'''
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
Uploader.__init__(self, architecture, config) Uploader.__init__(self, architecture, config)
section = config.get_section_name('s3', architecture) section = config.get_section_name('s3', architecture)
self.bucket = config.get(section, 'bucket') self.bucket = config.get(section, 'bucket')
def sync(self, path: str) -> None: def sync(self, path: str) -> None:
'''
sync data to remote server
:param path: local path to sync
'''
# TODO rewrite to boto, but it is bullshit # TODO rewrite to boto, but it is bullshit
check_output('aws', 's3', 'sync', '--delete', path, self.bucket, check_output('aws', 's3', 'sync', '--delete', path, self.bucket,
exception=None, exception=None,

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -25,14 +25,32 @@ from ahriman.models.upload_settings import UploadSettings
class Uploader: class Uploader:
'''
base remote sync class
:ivar architecture: repository architecture
:ivar config: configuration instance
:ivar logger: application logger
'''
def __init__(self, architecture: str, config: Configuration) -> None: def __init__(self, architecture: str, config: Configuration) -> None:
'''
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
self.logger = logging.getLogger('builder') self.logger = logging.getLogger('builder')
self.architecture = architecture self.architecture = architecture
self.config = config self.config = config
@staticmethod @staticmethod
def run(architecture: str, config: Configuration, target: str, path: str) -> None: def run(architecture: str, config: Configuration, target: str, path: str) -> None:
'''
run remote sync
:param architecture: repository architecture
:param config: configuration instance
:param target: target to run sync (e.g. s3)
:param path: local path to sync
'''
provider = UploadSettings.from_option(target) provider = UploadSettings.from_option(target)
if provider == UploadSettings.Rsync: if provider == UploadSettings.Rsync:
from ahriman.core.upload.rsync import Rsync from ahriman.core.upload.rsync import Rsync
@ -50,4 +68,8 @@ class Uploader:
raise SyncFailed() raise SyncFailed()
def sync(self, path: str) -> None: def sync(self, path: str) -> None:
'''
sync data to remote server
:param path: local path to sync
'''
pass pass

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -24,8 +24,17 @@ from typing import Optional
def check_output(*args: str, exception: Optional[Exception], def check_output(*args: str, exception: Optional[Exception],
cwd = None, stderr: int = subprocess.STDOUT, cwd: Optional[str] = None, stderr: int = subprocess.STDOUT,
logger: Optional[Logger] = None) -> str: logger: Optional[Logger] = None) -> str:
'''
subprocess wrapper
:param args: command line arguments
:param exception: exception which has to be reraised instead of default subprocess exception
:param cwd: current working directory
:param stderr: standard error output mode
:param logger: logger to log command result if required
:return: command output
'''
try: try:
result = subprocess.check_output(args, cwd=cwd, stderr=stderr).decode('utf8').strip() result = subprocess.check_output(args, cwd=cwd, stderr=stderr).decode('utf8').strip()
if logger is not None: if logger is not None:
@ -40,4 +49,9 @@ def check_output(*args: str, exception: Optional[Exception],
def package_like(filename: str) -> bool: def package_like(filename: str) -> bool:
'''
check if file looks like package
:param filename: name of file to check
:return: True in case if name contains `.pkg.` and not signature, False otherwise
'''
return '.pkg.' in filename and not filename.endswith('.sig') return '.pkg.' in filename and not filename.endswith('.sig')

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -19,98 +19,94 @@
# #
from __future__ import annotations from __future__ import annotations
import logging
from typing import Any, Dict, Set
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.models.build_status import BuildStatusEnum from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package from ahriman.models.package import Package
class Client: class Client:
'''
base build status reporter client
'''
def add(self, package: Package, status: BuildStatusEnum) -> None: def add(self, package: Package, status: BuildStatusEnum) -> None:
'''
add new package with status
:param package: package properties
:param status: current package build status
'''
pass pass
def remove(self, base: str, packages: Set[str]) -> None: def remove(self, base: str) -> None:
'''
remove packages from watcher
:param base: basename to remove
'''
pass pass
def update(self, base: str, status: BuildStatusEnum) -> None: def update(self, base: str, status: BuildStatusEnum) -> None:
'''
update package build status. Unlike `add` it does not update package properties
:param base: package base to update
:param status: current package build status
'''
pass
def update_self(self, status: BuildStatusEnum) -> None:
'''
update ahriman status itself
:param status: current ahriman status
'''
pass pass
def set_building(self, base: str) -> None: def set_building(self, base: str) -> None:
'''
set package status to building
:param base: package base to update
'''
return self.update(base, BuildStatusEnum.Building) return self.update(base, BuildStatusEnum.Building)
def set_failed(self, base: str) -> None: def set_failed(self, base: str) -> None:
'''
set package status to failed
:param base: package base to update
'''
return self.update(base, BuildStatusEnum.Failed) return self.update(base, BuildStatusEnum.Failed)
def set_pending(self, base: str) -> None: def set_pending(self, base: str) -> None:
'''
set package status to pending
:param base: package base to update
'''
return self.update(base, BuildStatusEnum.Pending) return self.update(base, BuildStatusEnum.Pending)
def set_success(self, package: Package) -> None: def set_success(self, package: Package) -> None:
'''
set package status to success
:param package: current package properties
'''
return self.add(package, BuildStatusEnum.Success) return self.add(package, BuildStatusEnum.Success)
def set_unknown(self, package: Package) -> None: def set_unknown(self, package: Package) -> None:
'''
set package status to unknown
:param package: current package properties
'''
return self.add(package, BuildStatusEnum.Unknown) return self.add(package, BuildStatusEnum.Unknown)
@staticmethod @staticmethod
def load(architecture: str, config: Configuration) -> Client: def load(architecture: str, config: Configuration) -> Client:
'''
load client from settings
:param architecture: repository architecture
:param config: configuration instance
:return: client according to current settings
'''
section = config.get_section_name('web', architecture) section = config.get_section_name('web', architecture)
host = config.get(section, 'host', fallback=None) host = config.get(section, 'host', fallback=None)
port = config.getint(section, 'port', fallback=None) port = config.getint(section, 'port', fallback=None)
if host is None or port is None: if host is None or port is None:
return Client() return Client()
from ahriman.core.watcher.web_client import WebClient
return WebClient(host, port) return WebClient(host, port)
class WebClient(Client):
def __init__(self, host: str, port: int) -> None:
self.logger = logging.getLogger('http')
self.host = host
self.port = port
def _url(self, base: str) -> str:
return f'http://{self.host}:{self.port}/api/v1/packages/{base}'
def add(self, package: Package, status: BuildStatusEnum) -> None:
import requests
payload: Dict[str, Any] = {
'status': status.value,
'package': {
'base': package.base,
'packages': [p for p in package.packages],
'version': package.version,
'aur_url': package.aur_url
}
}
try:
response = requests.post(self._url(package.base), json=payload)
response.raise_for_status()
except:
self.logger.exception(f'could not add {package.base}', exc_info=True)
def remove(self, base: str, packages: Set[str]) -> None:
if not packages:
return
import requests
try:
response = requests.delete(self._url(base))
response.raise_for_status()
except:
self.logger.exception(f'could not delete {base}', exc_info=True)
def update(self, base: str, status: BuildStatusEnum) -> None:
import requests
payload: Dict[str, Any] = {'status': status.value}
try:
response = requests.post(self._url(base), json=payload)
response.raise_for_status()
except:
self.logger.exception(f'could not update {base}', exc_info=True)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -26,18 +26,36 @@ from ahriman.models.package import Package
class Watcher: class Watcher:
'''
package status watcher
:ivar architecture: repository architecture
:ivar known: list of known packages. For the most cases `packages` should be used instead
:ivar repository: repository object
'''
def __init__(self, architecture: str, config: Configuration) -> None: def __init__(self, architecture: str, config: Configuration) -> None:
'''
default constructor
:param architecture: repository architecture
:param config: configuration instance
'''
self.architecture = architecture self.architecture = architecture
self.repository = Repository(architecture, config) self.repository = Repository(architecture, config)
self.known: Dict[str, Tuple[Package, BuildStatus]] = {} self.known: Dict[str, Tuple[Package, BuildStatus]] = {}
self.status = BuildStatus()
@property @property
def packages(self) -> List[Tuple[Package, BuildStatus]]: def packages(self) -> List[Tuple[Package, BuildStatus]]:
'''
:return: list of packages together with their statuses
'''
return [pair for pair in self.known.values()] return [pair for pair in self.known.values()]
def load(self) -> None: def load(self) -> None:
'''
load packages from local repository. In case if last status is known, it will use it
'''
for package in self.repository.packages(): for package in self.repository.packages():
# get status of build or assign unknown # get status of build or assign unknown
current = self.known.get(package.base) current = self.known.get(package.base)
@ -48,10 +66,27 @@ class Watcher:
self.known[package.base] = (package, status) self.known[package.base] = (package, status)
def remove(self, base: str) -> None: def remove(self, base: str) -> None:
'''
remove package base from known list if any
:param base: package base
'''
self.known.pop(base, None) self.known.pop(base, None)
def update(self, base: str, status: BuildStatusEnum, package: Optional[Package]) -> None: def update(self, base: str, status: BuildStatusEnum, package: Optional[Package]) -> None:
'''
update package status and description
:param base: package base to update
:param status: new build status
:param package: optional new package description. In case if not set current properties will be used
'''
if package is None: if package is None:
package, _ = self.known[base] package, _ = self.known[base]
full_status = BuildStatus(status) full_status = BuildStatus(status)
self.known[base] = (package, full_status) self.known[base] = (package, full_status)
def update_self(self, status: BuildStatusEnum) -> None:
'''
update service status
:param status: new service status
'''
self.status = BuildStatus(status)

View File

@ -0,0 +1,117 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import requests
from dataclasses import asdict
from typing import Any, Dict
from ahriman.core.watcher.client import Client
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package
class WebClient(Client):
'''
build status reporter web client
:ivar host: host of web service
:ivar logger: class logger
:ivar port: port of web service
'''
def __init__(self, host: str, port: int) -> None:
'''
default constructor
:param host: host of web service
:param port: port of web service
'''
self.logger = logging.getLogger('http')
self.host = host
self.port = port
def _ahriman_url(self) -> str:
'''
url generator
:return: full url for web service for ahriman service itself
'''
return f'http://{self.host}:{self.port}/api/v1/ahriman'
def _package_url(self, base: str) -> str:
'''
url generator
:param base: package base to generate url
:return: full url of web service for specific package base
'''
return f'http://{self.host}:{self.port}/api/v1/packages/{base}'
def add(self, package: Package, status: BuildStatusEnum) -> None:
'''
add new package with status
:param package: package properties
:param status: current package build status
'''
payload: Dict[str, Any] = {
'status': status.value,
'package': asdict(package)
}
try:
response = requests.post(self._package_url(package.base), json=payload)
response.raise_for_status()
except Exception:
self.logger.exception(f'could not add {package.base}', exc_info=True)
def remove(self, base: str) -> None:
'''
remove packages from watcher
:param base: basename to remove
'''
try:
response = requests.delete(self._package_url(base))
response.raise_for_status()
except Exception:
self.logger.exception(f'could not delete {base}', exc_info=True)
def update(self, base: str, status: BuildStatusEnum) -> None:
'''
update package build status. Unlike `add` it does not update package properties
:param base: package base to update
:param status: current package build status
'''
payload: Dict[str, Any] = {'status': status.value}
try:
response = requests.post(self._package_url(base), json=payload)
response.raise_for_status()
except Exception:
self.logger.exception(f'could not update {base}', exc_info=True)
def update_self(self, status: BuildStatusEnum) -> None:
'''
update ahriman status itself
:param status: current ahriman status
'''
payload: Dict[str, Any] = {'status': status.value}
try:
response = requests.post(self._ahriman_url(), json=payload)
response.raise_for_status()
except Exception:
self.logger.exception(f'could not update service status', exc_info=True)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -24,6 +24,15 @@ from typing import Optional, Union
class BuildStatusEnum(Enum): class BuildStatusEnum(Enum):
'''
build status enumeration
:cvar Unknown: build status is unknown
:cvar Pending: package is out-of-dated and will be built soon
:cvar Building: package is building right now
:cvar Failed: package build failed
:cvar Success: package has been built without errors
'''
Unknown = 'unknown' Unknown = 'unknown'
Pending = 'pending' Pending = 'pending'
Building = 'building' Building = 'building'
@ -32,12 +41,25 @@ class BuildStatusEnum(Enum):
class BuildStatus: class BuildStatus:
'''
build status holder
:ivar status: build status
:ivar _timestamp: build status update time
'''
def __init__(self, status: Union[BuildStatusEnum, str, None] = None, def __init__(self, status: Union[BuildStatusEnum, str, None] = None,
timestamp: Optional[datetime.datetime] = None) -> None: timestamp: Optional[datetime.datetime] = None) -> None:
'''
default constructor
:param status: current build status if known. `BuildStatusEnum.Unknown` will be used if not set
:param timestamp: build status timestamp. Current timestamp will be used if not set
'''
self.status = BuildStatusEnum(status) if status else BuildStatusEnum.Unknown self.status = BuildStatusEnum(status) if status else BuildStatusEnum.Unknown
self._timestamp = timestamp or datetime.datetime.utcnow() self._timestamp = timestamp or datetime.datetime.utcnow()
@property @property
def timestamp(self) -> str: def timestamp(self) -> str:
'''
:return: string representation of build status timestamp
'''
return self._timestamp.strftime('%Y-%m-%d %H:%M:%S') return self._timestamp.strftime('%Y-%m-%d %H:%M:%S')

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -19,46 +19,68 @@
# #
from __future__ import annotations from __future__ import annotations
import aur import aur # type: ignore
import os import os
import shutil import shutil
import tempfile import tempfile
from dataclasses import dataclass, field from dataclasses import dataclass
from srcinfo.parse import parse_srcinfo from pyalpm import vercmp # type: ignore
from typing import List, Optional, Set, Type from srcinfo.parse import parse_srcinfo # type: ignore
from typing import Dict, List, Optional, Set, Type
from ahriman.core.alpm.pacman import Pacman from ahriman.core.alpm.pacman import Pacman
from ahriman.core.exceptions import InvalidPackageInfo from ahriman.core.exceptions import InvalidPackageInfo
from ahriman.core.util import check_output from ahriman.core.util import check_output
from ahriman.models.package_desciption import PackageDescription
@dataclass @dataclass
class Package: class Package:
'''
package properties representation
:ivar aurl_url: AUR root url
:ivar base: package base name
:ivar packages: map of package names to their properties. Filled only on load from archive
:ivar version: package full version
'''
base: str base: str
version: str version: str
aur_url: str aur_url: str
packages: Set[str] = field(default_factory=set) packages: Dict[str, PackageDescription]
@property @property
def git_url(self) -> str: def git_url(self) -> str:
'''
:return: package git url to clone
'''
return f'{self.aur_url}/{self.base}.git' return f'{self.aur_url}/{self.base}.git'
@property @property
def is_vcs(self) -> bool: def is_vcs(self) -> bool:
'''
:return: True in case if package base looks like VCS package and false otherwise
'''
return self.base.endswith('-bzr') \ return self.base.endswith('-bzr') \
or self.base.endswith('-csv')\ or self.base.endswith('-csv')\
or self.base.endswith('-darcs')\ or self.base.endswith('-darcs')\
or self.base.endswith('-git')\ or self.base.endswith('-git')\
or self.base.endswith('-hg')\ or self.base.endswith('-hg')\
or self.base.endswith('-svn') or self.base.endswith('-svn')
@property @property
def web_url(self) -> str: def web_url(self) -> str:
'''
:return: package AUR url
'''
return f'{self.aur_url}/packages/{self.base}' return f'{self.aur_url}/packages/{self.base}'
# additional method to handle vcs versions
def actual_version(self) -> str: def actual_version(self) -> str:
'''
additional method to handle VCS package versions
:return: package version if package is not VCS and current version according to VCS otherwise
'''
if not self.is_vcs: if not self.is_vcs:
return self.version return self.version
@ -81,32 +103,57 @@ class Package:
@classmethod @classmethod
def from_archive(cls: Type[Package], path: str, pacman: Pacman, aur_url: str) -> Package: def from_archive(cls: Type[Package], path: str, pacman: Pacman, aur_url: str) -> Package:
'''
construct package properties from package archive
:param path: path to package archive
:param pacman: alpm wrapper instance
:param aur_url: AUR root url
:return: package properties
'''
package = pacman.handle.load_pkg(path) package = pacman.handle.load_pkg(path)
return cls(package.base, package.version, aur_url, {package.name}) properties = PackageDescription(os.path.basename(path), package.isize)
return cls(package.base, package.version, aur_url, {package.name: properties})
@classmethod @classmethod
def from_aur(cls: Type[Package], name: str, aur_url: str)-> Package: def from_aur(cls: Type[Package], name: str, aur_url: str) -> Package:
'''
construct package properties from AUR page
:param name: package name (either base or normal name)
:param aur_url: AUR root url
:return: package properties
'''
package = aur.info(name) package = aur.info(name)
return cls(package.package_base, package.version, aur_url, {package.name}) return cls(package.package_base, package.version, aur_url, {package.name: PackageDescription()})
@classmethod @classmethod
def from_build(cls: Type[Package], path: str, aur_url: str) -> Package: def from_build(cls: Type[Package], path: str, aur_url: str) -> Package:
'''
construct package properties from sources directory
:param path: path to package sources directory
:param aur_url: AUR root url
:return: package properties
'''
with open(os.path.join(path, '.SRCINFO')) as fn: with open(os.path.join(path, '.SRCINFO')) as fn:
src_info, errors = parse_srcinfo(fn.read()) src_info, errors = parse_srcinfo(fn.read())
if errors: if errors:
raise InvalidPackageInfo(errors) raise InvalidPackageInfo(errors)
packages = set(src_info['packages'].keys()) packages = {key: PackageDescription() for key in src_info['packages'].keys()}
version = cls.full_version(src_info.get('epoch'), src_info['pkgver'], src_info['pkgrel']) version = cls.full_version(src_info.get('epoch'), src_info['pkgver'], src_info['pkgrel'])
return cls(src_info['pkgbase'], version, aur_url, packages) return cls(src_info['pkgbase'], version, aur_url, packages)
@staticmethod @staticmethod
def dependencies(path: str) -> Set[str]: def dependencies(path: str) -> Set[str]:
'''
load dependencies from package sources
:param path: path to package sources directory
:return: list of package dependencies including makedepends array, but excluding packages from this base
'''
with open(os.path.join(path, '.SRCINFO')) as fn: with open(os.path.join(path, '.SRCINFO')) as fn:
src_info, errors = parse_srcinfo(fn.read()) src_info, errors = parse_srcinfo(fn.read())
if errors: if errors:
raise InvalidPackageInfo(errors) raise InvalidPackageInfo(errors)
makedepends = src_info['makedepends'] makedepends = src_info.get('makedepends', [])
# sum over each package # sum over each package
depends: List[str] = src_info.get('depends', []) depends: List[str] = src_info.get('depends', [])
for package in src_info['packages'].values(): for package in src_info['packages'].values():
@ -117,11 +164,25 @@ class Package:
@staticmethod @staticmethod
def full_version(epoch: Optional[str], pkgver: str, pkgrel: str) -> str: def full_version(epoch: Optional[str], pkgver: str, pkgrel: str) -> str:
'''
generate full version from components
:param epoch: package epoch if any
:param pkgver: package version
:param pkgrel: package release version (archlinux specific)
:return: generated version
'''
prefix = f'{epoch}:' if epoch else '' prefix = f'{epoch}:' if epoch else ''
return f'{prefix}{pkgver}-{pkgrel}' return f'{prefix}{pkgver}-{pkgrel}'
@staticmethod @staticmethod
def load(path: str, pacman: Pacman, aur_url: str) -> Package: def load(path: str, pacman: Pacman, aur_url: str) -> Package:
'''
package constructor from available sources
:param path: one of path to sources directory, path to archive or package name/base
:param pacman: alpm wrapper instance (required to load from archive)
:param aur_url: AUR root url
:return: package properties
'''
try: try:
if os.path.isdir(path): if os.path.isdir(path):
package: Package = Package.from_build(path, aur_url) package: Package = Package.from_build(path, aur_url)
@ -136,6 +197,11 @@ class Package:
raise InvalidPackageInfo(str(e)) raise InvalidPackageInfo(str(e))
def is_outdated(self, remote: Package) -> bool: def is_outdated(self, remote: Package) -> bool:
'''
check if package is out-of-dated
:param remote: package properties from remote source
:return: True if the package is out-of-dated and False otherwise
'''
remote_version = remote.actual_version() # either normal version or updated VCS remote_version = remote.actual_version() # either normal version or updated VCS
result = check_output('vercmp', self.version, remote_version, exception=None) result: int = vercmp(self.version, remote_version)
return True if int(result) < 0 else False return result < 0

View File

@ -0,0 +1,57 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass
from typing import Optional
from ahriman.core.exceptions import InvalidOption
@dataclass
class PackageDescription:
'''
package specific properties
'''
filename: Optional[str] = None
installed_size: Optional[int] = None
@staticmethod
def size_to_str(size: Optional[float], level: int = 0) -> str:
'''
convert size to string
:param size: size to convert
:param level: represents current units, 0 is B, 1 is KiB etc
:return: pretty printable size as string
'''
def str_level() -> str:
if level == 0:
return 'B'
elif level == 1:
return 'KiB'
elif level == 2:
return 'MiB'
elif level == 3:
return 'GiB'
raise InvalidOption(level)
if size is None:
return ''
elif size < 1024:
return f'{round(size, 2)} {str_level()}'
return PackageDescription.size_to_str(size / 1024, level + 1)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -25,10 +25,20 @@ from ahriman.core.exceptions import InvalidOption
class ReportSettings(Enum): class ReportSettings(Enum):
'''
report targets enumeration
:ivar HTML: html report generation
'''
HTML = auto() HTML = auto()
@staticmethod @staticmethod
def from_option(value: str) -> ReportSettings: def from_option(value: str) -> ReportSettings:
'''
construct value from configuration
:param value: configuration value
:return: parsed value
'''
if value.lower() in ('html',): if value.lower() in ('html',):
return ReportSettings.HTML return ReportSettings.HTML
raise InvalidOption(value) raise InvalidOption(value)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -24,6 +24,12 @@ from dataclasses import dataclass
@dataclass @dataclass
class RepositoryPaths: class RepositoryPaths:
'''
repository paths holder
:ivar root: repository root (i.e. ahriman home)
:ivar architecture: repository architecture
'''
root: str root: str
architecture: str architecture: str
@ -63,8 +69,11 @@ class RepositoryPaths:
return os.path.join(self.root, 'sources') return os.path.join(self.root, 'sources')
def create_tree(self) -> None: def create_tree(self) -> None:
'''
create ahriman working tree
'''
os.makedirs(self.chroot, mode=0o755, exist_ok=True) os.makedirs(self.chroot, mode=0o755, exist_ok=True)
os.makedirs(self.manual, mode=0o755, exist_ok=True) os.makedirs(self.manual, mode=0o755, exist_ok=True)
os.makedirs(self.packages, mode=0o755, exist_ok=True) os.makedirs(self.packages, mode=0o755, exist_ok=True)
os.makedirs(self.repository, mode=0o755, exist_ok=True) os.makedirs(self.repository, mode=0o755, exist_ok=True)
os.makedirs(self.sources, mode=0o755, exist_ok=True) os.makedirs(self.sources, mode=0o755, exist_ok=True)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -25,13 +25,24 @@ from ahriman.core.exceptions import InvalidOption
class SignSettings(Enum): class SignSettings(Enum):
'''
sign targets enumeration
:ivar SignPackages: sign each package
:ivar SignRepository: sign repository database file
'''
SignPackages = auto() SignPackages = auto()
SignRepository = auto() SignRepository = auto()
@staticmethod @staticmethod
def from_option(value: str) -> SignSettings: def from_option(value: str) -> SignSettings:
'''
construct value from configuration
:param value: configuration value
:return: parsed value
'''
if value.lower() in ('package', 'packages', 'sign-package'): if value.lower() in ('package', 'packages', 'sign-package'):
return SignSettings.SignPackages return SignSettings.SignPackages
elif value.lower() in ('repository', 'sign-repository'): elif value.lower() in ('repository', 'sign-repository'):
return SignSettings.SignRepository return SignSettings.SignRepository
raise InvalidOption(value) raise InvalidOption(value)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -25,11 +25,22 @@ from ahriman.core.exceptions import InvalidOption
class UploadSettings(Enum): class UploadSettings(Enum):
'''
remote synchronization targets enumeration
:ivar Rsync: sync via rsync
:ivar S3: sync to Amazon S3
'''
Rsync = auto() Rsync = auto()
S3 = auto() S3 = auto()
@staticmethod @staticmethod
def from_option(value: str) -> UploadSettings: def from_option(value: str) -> UploadSettings:
'''
construct value from configuration
:param value: configuration value
:return: parsed value
'''
if value.lower() in ('rsync',): if value.lower() in ('rsync',):
return UploadSettings.Rsync return UploadSettings.Rsync
elif value.lower() in ('s3',): elif value.lower() in ('s3',):

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -17,4 +17,4 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
__version__ = '0.11.3' __version__ = '0.12.1'

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -17,15 +17,24 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
from aiohttp.web import middleware, Request, Response from aiohttp.web import middleware, Request
from aiohttp.web_exceptions import HTTPClientError from aiohttp.web_exceptions import HTTPClientError
from aiohttp.web_response import StreamResponse
from logging import Logger from logging import Logger
from typing import Callable from typing import Awaitable, Callable
def exception_handler(logger: Logger) -> Callable: HandlerType = Callable[[Request], Awaitable[StreamResponse]]
def exception_handler(logger: Logger) -> Callable[[Request, HandlerType], Awaitable[StreamResponse]]:
'''
exception handler middleware. Just log any exception (except for client ones)
:param logger: class logger
:return: built middleware
'''
@middleware @middleware
async def handle(request: Request, handler: Callable) -> Response: async def handle(request: Request, handler: HandlerType) -> StreamResponse:
try: try:
return await handler(request) return await handler(request)
except HTTPClientError: except HTTPClientError:

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -19,16 +19,36 @@
# #
from aiohttp.web import Application from aiohttp.web import Application
from ahriman.web.views.ahriman import AhrimanView
from ahriman.web.views.index import IndexView from ahriman.web.views.index import IndexView
from ahriman.web.views.package import PackageView from ahriman.web.views.package import PackageView
from ahriman.web.views.packages import PackagesView from ahriman.web.views.packages import PackagesView
def setup_routes(app: Application) -> None: def setup_routes(application: Application) -> None:
app.router.add_get('/', IndexView) '''
app.router.add_get('/index.html', IndexView) setup all defined routes
app.router.add_post('/api/v1/packages', PackagesView) Available routes are:
app.router.add_delete('/api/v1/packages/{package}', PackageView) GET / get build status page
app.router.add_post('/api/v1/packages/{package}', PackageView) GET /index.html same as above
POST /api/v1/ahriman update service status
POST /api/v1/packages force update every package from repository
POST /api/v1/package/:base update package base status
DELETE /api/v1/package/:base delete package base from status page
:param application: web application instance
'''
application.router.add_get('/', IndexView)
application.router.add_get('/index.html', IndexView)
application.router.add_post('/api/v1/ahriman', AhrimanView)
application.router.add_post('/api/v1/packages', PackagesView)
application.router.add_delete('/api/v1/packages/{package}', PackageView)
application.router.add_post('/api/v1/packages/{package}', PackageView)

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify

View File

@ -0,0 +1,51 @@
#
# Copyright (c) 2021 Evgenii Alekseev.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from aiohttp.web import HTTPBadRequest, HTTPOk, Response
from ahriman.models.build_status import BuildStatusEnum
from ahriman.web.views.base import BaseView
class AhrimanView(BaseView):
'''
service status web view
'''
async def post(self) -> Response:
'''
update service status
JSON body must be supplied, the following model is used:
{
"status": "unknown", # service status string, must be valid `BuildStatusEnum`
}
:return: 200 on success
'''
data = await self.request.json()
try:
status = BuildStatusEnum(data['status'])
except Exception as e:
raise HTTPBadRequest(text=str(e))
self.service.update_self(status)
return HTTPOk()

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -22,9 +22,15 @@ from aiohttp.web import View
from ahriman.core.watcher.watcher import Watcher from ahriman.core.watcher.watcher import Watcher
# special class to make it typed
class BaseView(View): class BaseView(View):
'''
base web view to make things typed
'''
@property @property
def service(self) -> Watcher: def service(self) -> Watcher:
return self.request.app['watcher'] '''
:return: build status watcher instance
'''
watcher: Watcher = self.request.app['watcher']
return watcher

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -17,7 +17,8 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
from aiohttp_jinja2 import template import aiohttp_jinja2 # type: ignore
from typing import Any, Dict from typing import Any, Dict
import ahriman.version as version import ahriman.version as version
@ -26,9 +27,25 @@ from ahriman.web.views.base import BaseView
class IndexView(BaseView): class IndexView(BaseView):
'''
root view
@template("build-status.jinja2") It uses jinja2 templates for report generation, the following variables are allowed:
architecture - repository architecture, string, required
packages - sorted list of packages properties: base, packages (sorted list), status,
timestamp, version, web_url. Required
repository - repository name, string, required
service - service status properties: status, timestamp. Required
version - ahriman version, string, required
'''
@aiohttp_jinja2.template("build-status.jinja2") # type: ignore
async def get(self) -> Dict[str, Any]: async def get(self) -> Dict[str, Any]:
'''
process get request. No parameters supported here
:return: parameters for jinja template
'''
# some magic to make it jinja-friendly # some magic to make it jinja-friendly
packages = [ packages = [
{ {
@ -40,10 +57,15 @@ class IndexView(BaseView):
'web_url': package.web_url 'web_url': package.web_url
} for package, status in sorted(self.service.packages, key=lambda item: item[0].base) } for package, status in sorted(self.service.packages, key=lambda item: item[0].base)
] ]
service = {
'status': self.service.status.status.value,
'timestamp': self.service.status.timestamp
}
return { return {
'architecture': self.service.architecture, 'architecture': self.service.architecture,
'packages': packages, 'packages': packages,
'repository': self.service.repository.name, 'repository': self.service.repository.name,
'service': service,
'version': version.__version__, 'version': version.__version__,
} }

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -17,7 +17,7 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
from aiohttp.web import HTTPOk, Response from aiohttp.web import HTTPBadRequest, HTTPOk, Response
from ahriman.models.build_status import BuildStatusEnum from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package from ahriman.models.package import Package
@ -25,19 +25,45 @@ from ahriman.web.views.base import BaseView
class PackageView(BaseView): class PackageView(BaseView):
'''
package base specific web view
'''
async def delete(self) -> Response: async def delete(self) -> Response:
'''
delete package base from status page
:return: 200 on success
'''
base = self.request.match_info['package'] base = self.request.match_info['package']
self.service.remove(base) self.service.remove(base)
return HTTPOk() return HTTPOk()
async def post(self) -> Response: async def post(self) -> Response:
'''
update package build status
JSON body must be supplied, the following model is used:
{
"status": "unknown", # package build status string, must be valid `BuildStatusEnum`
"package": {} # package body (use `dataclasses.asdict` to generate one), optional.
# Must be supplied in case if package base is unknown
}
:return: 200 on success
'''
base = self.request.match_info['package'] base = self.request.match_info['package']
data = await self.request.json() data = await self.request.json()
package = Package(**data['package']) if 'package' in data else None try:
status = BuildStatusEnum(data['status']) package = Package(**data['package']) if 'package' in data else None
self.service.update(base, status, package) status = BuildStatusEnum(data['status'])
except Exception as e:
raise HTTPBadRequest(text=str(e))
try:
self.service.update(base, status, package)
except KeyError:
raise HTTPBadRequest(text=f'Package {base} is unknown, but no package body set')
return HTTPOk() return HTTPOk()

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -23,8 +23,15 @@ from ahriman.web.views.base import BaseView
class PackagesView(BaseView): class PackagesView(BaseView):
'''
global watcher view
'''
async def post(self) -> Response: async def post(self) -> Response:
'''
reload all packages from repository. No parameters supported here
:return: 200 on success
'''
self.service.load() self.service.load()
return HTTPOk() return HTTPOk()

View File

@ -1,7 +1,7 @@
# #
# Copyright (c) 2021 Evgenii Alekseev. # Copyright (c) 2021 Evgenii Alekseev.
# #
# This file is part of ahriman # This file is part of ahriman
# (see https://github.com/arcan1s/ahriman). # (see https://github.com/arcan1s/ahriman).
# #
# This program is free software: you can redistribute it and/or modify # This program is free software: you can redistribute it and/or modify
@ -17,7 +17,7 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
import aiohttp_jinja2 import aiohttp_jinja2 # type: ignore
import jinja2 import jinja2
import logging import logging
@ -30,30 +30,49 @@ from ahriman.web.middlewares.exception_handler import exception_handler
from ahriman.web.routes import setup_routes from ahriman.web.routes import setup_routes
async def on_shutdown(app: web.Application) -> None: async def on_shutdown(application: web.Application) -> None:
app.logger.warning('server terminated') '''
web application shutdown handler
:param application: web application instance
'''
application.logger.warning('server terminated')
async def on_startup(app: web.Application) -> None: async def on_startup(application: web.Application) -> None:
app.logger.info('server started') '''
web application start handler
:param application: web application instance
'''
application.logger.info('server started')
try: try:
app['watcher'].load() application['watcher'].load()
except Exception: except Exception:
app.logger.exception('could not load packages', exc_info=True) application.logger.exception('could not load packages', exc_info=True)
raise InitializeException() raise InitializeException()
def run_server(app: web.Application, architecture: str) -> None: def run_server(application: web.Application, architecture: str) -> None:
app.logger.info('start server') '''
run web application
:param application: web application instance
:param architecture: repository architecture
'''
application.logger.info('start server')
section = app['config'].get_section_name('web', architecture) section = application['config'].get_section_name('web', architecture)
host = app['config'].get(section, 'host') host = application['config'].get(section, 'host')
port = app['config'].getint(section, 'port') port = application['config'].getint(section, 'port')
web.run_app(app, host=host, port=port, handle_signals=False) web.run_app(application, host=host, port=port, handle_signals=False)
def setup_service(architecture: str, config: Configuration) -> web.Application: def setup_service(architecture: str, config: Configuration) -> web.Application:
'''
create web application
:param architecture: repository architecture
:param config: configuration instance
:return: web application instance
'''
app = web.Application(logger=logging.getLogger('http')) app = web.Application(logger=logging.getLogger('http'))
app.on_shutdown.append(on_shutdown) app.on_shutdown.append(on_shutdown)
app.on_startup.append(on_startup) app.on_startup.append(on_startup)