diff --git a/CONFIGURING.md b/CONFIGURING.md index bf19ca0c..df103949 100644 --- a/CONFIGURING.md +++ b/CONFIGURING.md @@ -60,14 +60,22 @@ Remote synchronization settings: * `target` - list of synchronizations to be used, space separated list of strings, optional. Allowed values are `rsync`, `s3`. +### `rsync_*` group + +Group name must refer to architecture, e.g. it should be `rsync_x86_64` for x86_64 architecture. Requires `rsync` package to be installed. Do not forget to configure ssh for user `ahriman`. + +* `remote` - remote server to rsync (e.g. `1.2.3.4:5678:path/to/sync`), string, required. + ### `s3_*` group Group name must refer to architecture, e.g. it should be `s3_x86_64` for x86_64 architecture. Requires `aws-cli` package to be installed. Do not forget to configure it for user `ahriman`. * `bucket` - bucket name (e.g. `s3://bucket/path`), string, required. -### `rsync_*` group +## `web` group -Group name must refer to architecture, e.g. it should be `rsync_x86_64` for x86_64 architecture. Requires `rsync` package to be installed. Do not forget to configure ssh for user `ahriman`. +Web server settings. If any of `host`/`port` is not set, web intergration will be disabled. -* `remote` - remote server to rsync (e.g. `1.2.3.4:5678:path/to/sync`), string, required. +* `host` - host to bind, string, optional. +* `port` - port to bind, int, optional. +* `templates` - path to templates directory, string, required. diff --git a/package/archlinux/PKGBUILD b/package/archlinux/PKGBUILD index 062c197b..94a55d5a 100644 --- a/package/archlinux/PKGBUILD +++ b/package/archlinux/PKGBUILD @@ -14,14 +14,17 @@ optdepends=('aws-cli: sync to s3' 'darcs: -darcs packages support' 'gnupg: package and repository sign' 'mercurial: -hg packages support' + 'python-aiohttp: web server' + 'python-aiohttp-jinja2: web server' 'python-jinja: html report generation' + 'python-requests: web server' 'rsync: sync by using rsync' 'subversion: -svn packages support') source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$pkgver-src.tar.xz" 'ahriman.sudoers' 'ahriman.sysusers' 'ahriman.tmpfiles') -sha512sums=('82a8208554956f009db0334b0cb20891889d96617b4c9b9d2af7a007d668f6cbc6d46d0be8b5ee11ffb2b69d124b5d5d6db1a6f7d5a0a2c719c0d8e07dca24d8' +sha512sums=('bc4880fc2f4196dc959f14a199135bbf09c75fbaad722709c1ca7c1fdae0475b3cfcdff5bf33bc9bcdf4f17a0e29b42bd26de7b3d551356dd63a705ec496e111' '8c9b5b63ac3f7b4d9debaf801a1e9c060877c33d3ecafe18010fcca778e5fa2f2e46909d3d0ff1b229ff8aa978445d8243fd36e1fc104117ed678d5e21901167' '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075' '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4') diff --git a/package/etc/ahriman.ini b/package/etc/ahriman.ini index b438bb39..008f5d83 100644 --- a/package/etc/ahriman.ini +++ b/package/etc/ahriman.ini @@ -27,13 +27,18 @@ target = path = homepage = link_path = -template_path = /usr/share/ahriman/index.jinja2 +template_path = /usr/share/ahriman/repo-index.jinja2 [upload] target = +[rsync_x86_64] +remote = + [s3_x86_64] bucket = -[rsync_x86_64] -remote = \ No newline at end of file +[web] +host = +port = +templates = /usr/share/ahriman \ No newline at end of file diff --git a/package/etc/ahriman.ini.d/logging.ini b/package/etc/ahriman.ini.d/logging.ini index 09c2a770..dd998588 100644 --- a/package/etc/ahriman.ini.d/logging.ini +++ b/package/etc/ahriman.ini.d/logging.ini @@ -1,8 +1,8 @@ [loggers] -keys = root,builder,build_details +keys = root,builder,build_details,http [handlers] -keys = console_handler,build_file_handler,file_handler +keys = console_handler,build_file_handler,file_handler,http_handler [formatters] keys = generic_format @@ -25,6 +25,12 @@ level = DEBUG formatter = generic_format args = ('/var/log/ahriman/build.log', 'a', 20971520, 20) +[handler_http_handler] +class = logging.handlers.RotatingFileHandler +level = DEBUG +formatter = generic_format +args = ('/var/log/ahriman/http.log', 'a', 20971520, 20) + [formatter_generic_format] format = %(asctime)s : %(levelname)s : %(funcName)s : %(message)s datefmt = @@ -45,3 +51,9 @@ level = DEBUG handlers = build_file_handler qualname = build_details propagate = 0 + +[logger_http] +level = DEBUG +handlers = http_handler +qualname = http +propagate = 0 diff --git a/package/lib/systemd/system/ahriman-web.service b/package/lib/systemd/system/ahriman-web.service new file mode 100644 index 00000000..67b5b0e1 --- /dev/null +++ b/package/lib/systemd/system/ahriman-web.service @@ -0,0 +1,11 @@ +[Unit] +Description=ArcHlinux ReposItory MANager web server + +[Service] +Type=simple +ExecStart=/usr/bin/ahriman --architecture x86_64 web +User=ahriman +Group=ahriman + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/package/share/ahriman/index.jinja2 b/package/share/ahriman/index.jinja2 index 7758fb8d..1b5756f6 100644 --- a/package/share/ahriman/index.jinja2 +++ b/package/share/ahriman/index.jinja2 @@ -2,30 +2,83 @@ {{ repository|e }} + + + + -

{{ repository|e }} ArchLinux custom repository

+ + + + + + + + + - {% if pgp_key is not none %} -

All packages are signed with {{ pgp_key|e }}.

- {% endif %} - - - $ cat /etc/pacman.conf
- [{{ repository|e }}]
- Server = {{ link_path|e }} - - -

Packages:

- + + + + + + + {% endfor %} - - - {% if homepage is not none %} - - {% endif %} +
package basepackagesversionarchitecturetimestampstatus
{{ package.base|e }}{{ package.packages|join("
"|safe) }}		{{ package.version|e }}{{ architecture|e }}{{ package.timestamp|e }}{{ package.status|e }}
- \ No newline at end of file + + diff --git a/package/share/ahriman/repo-index.jinja2 b/package/share/ahriman/repo-index.jinja2 new file mode 100644 index 00000000..7758fb8d --- /dev/null +++ b/package/share/ahriman/repo-index.jinja2 @@ -0,0 +1,31 @@ + + + + {{ repository|e }} + + + +

{{ repository|e }} ArchLinux custom repository

+ + {% if pgp_key is not none %} +

All packages are signed with {{ pgp_key|e }}.

+ {% endif %} + + + $ cat /etc/pacman.conf
+ [{{ repository|e }}]
+ Server = {{ link_path|e }} + + +

Packages:

+ + + {% if homepage is not none %} + + {% endif %} + + \ No newline at end of file diff --git a/setup.py b/setup.py index da758be6..257ddc0f 100644 --- a/setup.py +++ b/setup.py @@ -42,17 +42,26 @@ setup( 'package/bin/ahriman', ], data_files=[ - ('/etc', ['package/etc/ahriman.ini']), - ('/etc/ahriman.ini.d', ['package/etc/ahriman.ini.d/logging.ini']), + ('/etc', [ + 'package/etc/ahriman.ini', + ]), + ('/etc/ahriman.ini.d', [ + 'package/etc/ahriman.ini.d/logging.ini', + ]), ('lib/systemd/system', [ 'package/lib/systemd/system/ahriman.service', - 'package/lib/systemd/system/ahriman.timer' + 'package/lib/systemd/system/ahriman.timer', + 'package/lib/systemd/system/ahriman-web.service', + ]), + ('share/ahriman', [ + 'package/share/ahriman/index.jinja2', + 'package/share/ahriman/repo-index.jinja2', ]), - ('share/ahriman', ['package/share/ahriman/index.jinja2']), ], extras_require={ 'html-templates': ['Jinja2'], 'test': ['coverage', 'pytest'], + 'web': ['Jinja2', 'aiohttp', 'aiohttp_jinja2', 'requests'], }, ) diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 280334ad..2fc90450 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -74,6 +74,13 @@ def update(args: argparse.Namespace) -> None: app.update(packages) +def web(args: argparse.Namespace) -> None: + from ahriman.web.web import run_server, setup_service + config = _get_config(args.config) + app = setup_service(args.architecture, config) + run_server(app) + + if __name__ == '__main__': parser = argparse.ArgumentParser(prog='ahriman', description='ArcHlinux ReposItory MANager') parser.add_argument('-a', '--architecture', help='target architecture', required=True) @@ -112,6 +119,9 @@ if __name__ == '__main__': update_parser.add_argument('--no-vcs', help='do not check VCS packages', action='store_true') update_parser.set_defaults(fn=update) + web_parser = subparsers.add_parser('web', description='start web server') + web_parser.set_defaults(fn=web, lock='/tmp/ahriman-web.lock') + args = parser.parse_args() if args.force: diff --git a/src/ahriman/application/application.py b/src/ahriman/application/application.py index e1b4362b..e58932cf 100644 --- a/src/ahriman/application/application.py +++ b/src/ahriman/application/application.py @@ -58,7 +58,7 @@ class Application: def add(self, names: List[str]) -> None: def add_manual(name: str) -> None: package = Package.load(name, self.config.get('aur', 'url')) - Task.fetch(os.path.join(self.repository.paths.manual, package.base), package.url) + Task.fetch(os.path.join(self.repository.paths.manual, package.base), package.git_url) def add_archive(src: str) -> None: dst = os.path.join(self.repository.paths.packages, os.path.basename(src)) diff --git a/src/ahriman/core/build_tools/task.py b/src/ahriman/core/build_tools/task.py index 401687da..d3f1a8af 100644 --- a/src/ahriman/core/build_tools/task.py +++ b/src/ahriman/core/build_tools/task.py @@ -73,4 +73,4 @@ class Task: def clone(self, path: Optional[str] = None) -> None: git_path = path or self.git_path - return Task.fetch(git_path, self.package.url) + return Task.fetch(git_path, self.package.git_url) diff --git a/src/ahriman/core/exceptions.py b/src/ahriman/core/exceptions.py index 6c50f92b..f5d6ea38 100644 --- a/src/ahriman/core/exceptions.py +++ b/src/ahriman/core/exceptions.py @@ -25,6 +25,11 @@ class BuildFailed(Exception): Exception.__init__(self, f'Package {package} build failed, check logs for details') +class InitializeException(Exception): + def __init__(self) -> None: + Exception.__init__(self, 'Could not load service') + + class InvalidOptionException(Exception): def __init__(self, value: Any) -> None: Exception.__init__(self, f'Invalid or unknown option value `{value}`') diff --git a/src/ahriman/core/report/dummy.py b/src/ahriman/core/report/dummy.py index 1b7c4733..9caec0ce 100644 --- a/src/ahriman/core/report/dummy.py +++ b/src/ahriman/core/report/dummy.py @@ -1,7 +1,7 @@ # # Copyright (c) 2021 Evgenii Alekseev. # -# This file is part of ahriman +# This file is part of ahriman # (see https://github.com/arcan1s/ahriman). # # This program is free software: you can redistribute it and/or modify diff --git a/src/ahriman/core/report/html.py b/src/ahriman/core/report/html.py index d9da00bb..b66bf3ee 100644 --- a/src/ahriman/core/report/html.py +++ b/src/ahriman/core/report/html.py @@ -1,7 +1,7 @@ # # Copyright (c) 2021 Evgenii Alekseev. # -# This file is part of ahriman +# This file is part of ahriman # (see https://github.com/arcan1s/ahriman). # # This program is free software: you can redistribute it and/or modify diff --git a/src/ahriman/core/report/report.py b/src/ahriman/core/report/report.py index 07ffdace..e2828329 100644 --- a/src/ahriman/core/report/report.py +++ b/src/ahriman/core/report/report.py @@ -1,7 +1,7 @@ # # Copyright (c) 2021 Evgenii Alekseev. # -# This file is part of ahriman +# This file is part of ahriman # (see https://github.com/arcan1s/ahriman). # # This program is free software: you can redistribute it and/or modify diff --git a/src/ahriman/core/repository.py b/src/ahriman/core/repository.py index 2756a520..36804c66 100644 --- a/src/ahriman/core/repository.py +++ b/src/ahriman/core/repository.py @@ -30,6 +30,8 @@ from ahriman.core.report.report import Report from ahriman.core.sign.gpg_wrapper import GPGWrapper from ahriman.core.upload.uploader import Uploader from ahriman.core.util import package_like +from ahriman.core.watcher.client import Client +from ahriman.models.build_status import BuildStatusEnum from ahriman.models.package import Package from ahriman.models.repository_paths import RepositoryPaths @@ -50,6 +52,8 @@ class Repository: self.sign = GPGWrapper(config) self.wrapper = RepoWrapper(self.name, self.paths, self.sign.repository_sign_args) + self.web_report = Client.load(config) + def _clear_build(self) -> None: for package in os.listdir(self.paths.sources): shutil.rmtree(os.path.join(self.paths.sources, package)) @@ -78,6 +82,7 @@ class Repository: def process_build(self, updates: List[Package]) -> List[str]: def build_single(package: Package) -> None: + self.web_report.update(package.base, BuildStatusEnum.Building) task = Task(package, self.architecture, self.config, self.paths) task.clone() built = task.build() @@ -89,6 +94,7 @@ class Repository: try: build_single(package) except Exception: + self.web_report.update(package.base, BuildStatusEnum.Failed) self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True) continue self._clear_build() @@ -112,6 +118,7 @@ class Repository: to_remove = local.packages.intersection(packages) else: to_remove = set() + self.web_report.remove(local.base, to_remove) for package in to_remove: remove_single(package) @@ -131,12 +138,18 @@ class Repository: def process_update(self, packages: List[str]) -> str: for package in packages: - files = self.sign.sign_package(package) - for src in files: - dst = os.path.join(self.paths.repository, os.path.basename(src)) - shutil.move(src, dst) - package_fn = os.path.join(self.paths.repository, os.path.basename(package)) - self.wrapper.add(package_fn) + local = Package.load(package, self.aur_url) # we will use it for status reports + try: + files = self.sign.sign_package(package) + for src in files: + dst = os.path.join(self.paths.repository, os.path.basename(src)) + shutil.move(src, dst) + package_fn = os.path.join(self.paths.repository, os.path.basename(package)) + self.wrapper.add(package_fn) + self.web_report.add(local, BuildStatusEnum.Success) + except Exception: + self.logger.exception(f'could not process {package}', exc_info=True) + self.web_report.update(local.base, BuildStatusEnum.Failed) self._clear_packages() return self.wrapper.repo_path @@ -156,7 +169,9 @@ class Repository: remote = Package.load(local.base, self.aur_url) if local.is_outdated(remote): result.append(remote) + self.web_report.update(local.base, BuildStatusEnum.Pending) except Exception: + self.web_report.update(local.base, BuildStatusEnum.Failed) self.logger.exception(f'could not load remote package {local.base}', exc_info=True) continue @@ -166,8 +181,12 @@ class Repository: result: List[Package] = [] for fn in os.listdir(self.paths.manual): - local = Package.load(os.path.join(self.paths.manual, fn), self.aur_url) - result.append(local) + try: + local = Package.load(os.path.join(self.paths.manual, fn), self.aur_url) + result.append(local) + self.web_report.add(local, BuildStatusEnum.Unknown) + except Exception: + self.logger.exception(f'could not add package from {fn}', exc_info=True) self._clear_manual() return result \ No newline at end of file diff --git a/src/ahriman/core/upload/dummy.py b/src/ahriman/core/upload/dummy.py index b5d9cfd8..32471dc3 100644 --- a/src/ahriman/core/upload/dummy.py +++ b/src/ahriman/core/upload/dummy.py @@ -1,7 +1,7 @@ # # Copyright (c) 2021 Evgenii Alekseev. # -# This file is part of ahriman +# This file is part of ahriman # (see https://github.com/arcan1s/ahriman). # # This program is free software: you can redistribute it and/or modify diff --git a/src/ahriman/core/upload/rsync.py b/src/ahriman/core/upload/rsync.py index 7cb18703..88e83d6b 100644 --- a/src/ahriman/core/upload/rsync.py +++ b/src/ahriman/core/upload/rsync.py @@ -1,7 +1,7 @@ # # Copyright (c) 2021 Evgenii Alekseev. # -# This file is part of ahriman +# This file is part of ahriman # (see https://github.com/arcan1s/ahriman). # # This program is free software: you can redistribute it and/or modify diff --git a/src/ahriman/core/upload/s3.py b/src/ahriman/core/upload/s3.py index 4c449453..3171f00b 100644 --- a/src/ahriman/core/upload/s3.py +++ b/src/ahriman/core/upload/s3.py @@ -1,7 +1,7 @@ # # Copyright (c) 2021 Evgenii Alekseev. # -# This file is part of ahriman +# This file is part of ahriman # (see https://github.com/arcan1s/ahriman). # # This program is free software: you can redistribute it and/or modify diff --git a/src/ahriman/core/upload/uploader.py b/src/ahriman/core/upload/uploader.py index bb4e50ab..3f6d79a4 100644 --- a/src/ahriman/core/upload/uploader.py +++ b/src/ahriman/core/upload/uploader.py @@ -1,7 +1,7 @@ # # Copyright (c) 2021 Evgenii Alekseev. # -# This file is part of ahriman +# This file is part of ahriman # (see https://github.com/arcan1s/ahriman). # # This program is free software: you can redistribute it and/or modify diff --git a/src/ahriman/core/watcher/__init__.py b/src/ahriman/core/watcher/__init__.py new file mode 100644 index 00000000..a0f149a5 --- /dev/null +++ b/src/ahriman/core/watcher/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# diff --git a/src/ahriman/core/watcher/client.py b/src/ahriman/core/watcher/client.py new file mode 100644 index 00000000..ff335eb4 --- /dev/null +++ b/src/ahriman/core/watcher/client.py @@ -0,0 +1,98 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from __future__ import annotations + +import logging + +from typing import Any, Dict, Set + +from ahriman.core.configuration import Configuration +from ahriman.models.build_status import BuildStatusEnum +from ahriman.models.package import Package + + +class Client: + + def add(self, package: Package, status: BuildStatusEnum) -> None: + pass + + def remove(self, base: str, packages: Set[str]) -> None: + pass + + def update(self, base: str, status: BuildStatusEnum) -> None: + pass + + @staticmethod + def load(config: Configuration) -> Client: + host = config.get('web', 'host', fallback=None) + port = config.getint('web', 'port', fallback=None) + if host is None or port is None: + return Client() + return WebClient(host, port) + + +class WebClient(Client): + + def __init__(self, host: str, port: int) -> None: + self.logger = logging.getLogger('http') + self.host = host + self.port = port + + def _url(self, base: str) -> str: + return f'http://{self.host}:{self.port}/api/v1/packages/{base}' + + def add(self, package: Package, status: BuildStatusEnum) -> None: + import requests + + payload: Dict[str, Any] = { + 'status': status.value, + 'base': package.base, + 'packages': [p for p in package.packages], + 'version': package.version, + 'url': package.web_url + } + + try: + response = requests.post(self._url(package.base), json=payload) + response.raise_for_status() + except: + self.logger.exception(f'could not add {package.base}', exc_info=True) + + def remove(self, base: str, packages: Set[str]) -> None: + if not packages: + return + import requests + + try: + response = requests.delete(self._url(base)) + response.raise_for_status() + except: + self.logger.exception(f'could not delete {base}', exc_info=True) + + def update(self, base: str, status: BuildStatusEnum) -> None: + import requests + + payload: Dict[str, Any] = {'status': status.value} + + try: + response = requests.post(self._url(base), json=payload) + response.raise_for_status() + except: + self.logger.exception(f'could not update {base}', exc_info=True) \ No newline at end of file diff --git a/src/ahriman/core/watcher/watcher.py b/src/ahriman/core/watcher/watcher.py new file mode 100644 index 00000000..3c6c896f --- /dev/null +++ b/src/ahriman/core/watcher/watcher.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from typing import Dict, List, Optional, Tuple + +from ahriman.core.configuration import Configuration +from ahriman.core.repository import Repository +from ahriman.models.build_status import BuildStatus, BuildStatusEnum +from ahriman.models.package import Package + + +class Watcher: + + def __init__(self, architecture: str, config: Configuration) -> None: + self.architecture = architecture + self.repository = Repository(architecture, config) + + self.known: Dict[str, Tuple[Package, BuildStatus]] = {} + + @property + def packages(self) -> List[Tuple[Package, BuildStatus]]: + return [pair for pair in self.known.values()] + + def load(self) -> None: + for package in self.repository.packages(): + # get status of build or assign unknown + current = self.known.get(package.base) + if current is None: + status = BuildStatus() + else: + _, status = current + self.known[package.base] = (package, status) + + def remove(self, base: str) -> None: + self.known.pop(base, None) + + def update(self, base: str, status: BuildStatusEnum, package: Optional[Package]) -> None: + if package is None: + package, _ = self.known[base] + full_status = BuildStatus(status) + self.known[base] = (package, full_status) diff --git a/src/ahriman/models/build_status.py b/src/ahriman/models/build_status.py new file mode 100644 index 00000000..f8a490ff --- /dev/null +++ b/src/ahriman/models/build_status.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import datetime + +from enum import Enum +from typing import Optional, Union + + +class BuildStatusEnum(Enum): + Unknown = 'unknown' + Pending = 'pending' + Building = 'building' + Failed = 'failed' + Success = 'success' + + +class BuildStatus: + + def __init__(self, status: Union[BuildStatusEnum, str, None] = None, + timestamp: Optional[datetime.datetime] = None) -> None: + self.status = BuildStatusEnum(status) if status else BuildStatusEnum.Unknown + self._timestamp = timestamp or datetime.datetime.utcnow() + + @property + def timestamp(self) -> str: + return self._timestamp.strftime('%Y-%m-%d %H:%M:%S') diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 47aaf7d0..45fbabd3 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -25,7 +25,6 @@ import aur import os import tempfile -from configparser import RawConfigParser from dataclasses import dataclass, field from srcinfo.parse import parse_srcinfo from typing import Set, Type @@ -38,9 +37,13 @@ from ahriman.core.util import check_output class Package: base: str version: str - url: str + aur_url: str packages: Set[str] = field(default_factory=set) + @property + def git_url(self) -> str: + return f'{self.aur_url}/{self.base}.git' + @property def is_vcs(self) -> bool: return self.base.endswith('-bzr') \ @@ -50,6 +53,10 @@ class Package: or self.base.endswith('-hg')\ or self.base.endswith('-svn') + @property + def web_url(self) -> str: + return f'{self.aur_url}/packages/{self.base}' + # additional method to handle vcs versions def actual_version(self) -> str: if not self.is_vcs: @@ -58,7 +65,7 @@ class Package: from ahriman.core.build_tools.task import Task clone_dir = tempfile.mkdtemp() try: - Task.fetch(clone_dir, self.url) + Task.fetch(clone_dir, self.git_url) # update pkgver first check_output('makepkg', '--nodeps', '--noprepare', '--nobuild', exception=None, cwd=clone_dir) @@ -75,37 +82,32 @@ class Package: @classmethod def from_archive(cls: Type[Package], path: str, aur_url: str) -> Package: package, base, version = check_output('expac', '-p', '%n %e %v', path, exception=None).split() - return cls(base, version, f'{aur_url}/{base}.git', packages={package}) + return cls(base, version, aur_url, {package}) @classmethod def from_aur(cls: Type[Package], name: str, aur_url: str)-> Package: package = aur.info(name) - return cls(package.package_base, package.version, f'{aur_url}/{package.package_base}.git', - packages={package.name}) + return cls(package.package_base, package.version, aur_url, {package.name}) @classmethod - def from_build(cls: Type[Package], path: str) -> Package: - git_config = RawConfigParser() - git_config.read(os.path.join(path, '.git', 'config')) - + def from_build(cls: Type[Package], path: str, aur_url: str) -> Package: with open(os.path.join(path, '.SRCINFO')) as fn: src_info, errors = parse_srcinfo(fn.read()) if errors: raise InvalidPackageInfo(errors) packages = set(src_info['packages'].keys()) - return cls(src_info['pkgbase'], f'{src_info["pkgver"]}-{src_info["pkgrel"]}', - git_config.get('remote "origin"', 'url'), packages-packages) + return cls(src_info['pkgbase'], f'{src_info["pkgver"]}-{src_info["pkgrel"]}', aur_url, packages) - @classmethod - def load(cls: Type[Package], path: str, aur_url: str) -> Package: + @staticmethod + def load(path: str, aur_url: str) -> Package: try: if os.path.isdir(path): - package: Package = cls.from_build(path) + package: Package = Package.from_build(path, aur_url) elif os.path.exists(path): - package = cls.from_archive(path, aur_url) + package = Package.from_archive(path, aur_url) else: - package = cls.from_aur(path, aur_url) + package = Package.from_aur(path, aur_url) return package except InvalidPackageInfo: raise diff --git a/src/ahriman/models/report_settings.py b/src/ahriman/models/report_settings.py index 9af1090e..d9979247 100644 --- a/src/ahriman/models/report_settings.py +++ b/src/ahriman/models/report_settings.py @@ -20,7 +20,6 @@ from __future__ import annotations from enum import Enum, auto -from typing import Type from ahriman.core.exceptions import InvalidOptionException @@ -28,8 +27,8 @@ from ahriman.core.exceptions import InvalidOptionException class ReportSettings(Enum): HTML = auto() - @classmethod - def from_option(cls: Type[ReportSettings], value: str) -> ReportSettings: + @staticmethod + def from_option(value: str) -> ReportSettings: if value.lower() in ('html',): - return cls.HTML + return ReportSettings.HTML raise InvalidOptionException(value) diff --git a/src/ahriman/models/sign_settings.py b/src/ahriman/models/sign_settings.py index 88d8b7c7..44874b04 100644 --- a/src/ahriman/models/sign_settings.py +++ b/src/ahriman/models/sign_settings.py @@ -20,7 +20,6 @@ from __future__ import annotations from enum import Enum, auto -from typing import Type from ahriman.core.exceptions import InvalidOptionException @@ -30,12 +29,12 @@ class SignSettings(Enum): SignPackages = auto() SignRepository = auto() - @classmethod - def from_option(cls: Type[SignSettings], value: str) -> SignSettings: + @staticmethod + def from_option(value: str) -> SignSettings: if value.lower() in ('no', 'disabled'): - return cls.Disabled + return SignSettings.Disabled elif value.lower() in ('package', 'packages', 'sign-package'): - return cls.SignPackages + return SignSettings.SignPackages elif value.lower() in ('repository', 'sign-repository'): - return cls.SignRepository + return SignSettings.SignRepository raise InvalidOptionException(value) \ No newline at end of file diff --git a/src/ahriman/models/upload_settings.py b/src/ahriman/models/upload_settings.py index cff0b4cc..66419bc4 100644 --- a/src/ahriman/models/upload_settings.py +++ b/src/ahriman/models/upload_settings.py @@ -20,7 +20,6 @@ from __future__ import annotations from enum import Enum, auto -from typing import Type from ahriman.core.exceptions import InvalidOptionException @@ -29,10 +28,10 @@ class UploadSettings(Enum): Rsync = auto() S3 = auto() - @classmethod - def from_option(cls: Type[UploadSettings], value: str) -> UploadSettings: + @staticmethod + def from_option(value: str) -> UploadSettings: if value.lower() in ('rsync',): - return cls.Rsync + return UploadSettings.Rsync elif value.lower() in ('s3',): - return cls.S3 + return UploadSettings.S3 raise InvalidOptionException(value) diff --git a/src/ahriman/web/__init__.py b/src/ahriman/web/__init__.py new file mode 100644 index 00000000..a0f149a5 --- /dev/null +++ b/src/ahriman/web/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# diff --git a/src/ahriman/web/middlewares/__init__.py b/src/ahriman/web/middlewares/__init__.py new file mode 100644 index 00000000..a0f149a5 --- /dev/null +++ b/src/ahriman/web/middlewares/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# diff --git a/src/ahriman/web/middlewares/exception_handler.py b/src/ahriman/web/middlewares/exception_handler.py new file mode 100644 index 00000000..7cfb0032 --- /dev/null +++ b/src/ahriman/web/middlewares/exception_handler.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from aiohttp.web import middleware, Request, Response +from logging import Logger +from typing import Callable + +from aiohttp.web_exceptions import HTTPClientError + + +def exception_handler(logger: Logger) -> Callable: + @middleware + async def handle(request: Request, handler: Callable) -> Response: + try: + return await handler(request) + except HTTPClientError: + raise + except Exception: + logger.exception(f'exception during performing request to {request.path}', exc_info=True) + raise + + return handle diff --git a/src/ahriman/web/routes.py b/src/ahriman/web/routes.py new file mode 100644 index 00000000..ce893ca1 --- /dev/null +++ b/src/ahriman/web/routes.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from aiohttp.web import Application + +from ahriman.web.views.index import IndexView +from ahriman.web.views.package import PackageView +from ahriman.web.views.packages import PackagesView + + +def setup_routes(app: Application) -> None: + app.router.add_get('/', IndexView) + app.router.add_get('/index.html', IndexView) + + app.router.add_post('/api/v1/packages', PackagesView) + + app.router.add_delete('/api/v1/packages/{package}', PackageView) + app.router.add_post('/api/v1/packages/{package}', PackageView) \ No newline at end of file diff --git a/src/ahriman/web/views/__init__.py b/src/ahriman/web/views/__init__.py new file mode 100644 index 00000000..a0f149a5 --- /dev/null +++ b/src/ahriman/web/views/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# diff --git a/src/ahriman/web/views/base.py b/src/ahriman/web/views/base.py new file mode 100644 index 00000000..dc3fc331 --- /dev/null +++ b/src/ahriman/web/views/base.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from aiohttp.web import View + +from ahriman.core.watcher.watcher import Watcher + + +class BaseView(View): + + @property + def service(self) -> Watcher: + return self.request.app['watcher'] diff --git a/src/ahriman/web/views/index.py b/src/ahriman/web/views/index.py new file mode 100644 index 00000000..0cf903c1 --- /dev/null +++ b/src/ahriman/web/views/index.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from typing import Any, Dict + +from aiohttp_jinja2 import template + +from ahriman.web.views.base import BaseView + + +class IndexView(BaseView): + + @template("index.jinja2") + async def get(self) -> Dict[str, Any]: + # some magic to make it jinja-readable + packages = [ + { + 'base': package.base, + 'packages': [p for p in sorted(package.packages)], + 'status': status.status.value, + 'timestamp': status.timestamp, + 'version': package.version, + 'web_url': package.web_url + } for package, status in sorted(self.service.packages, key=lambda item: item[0].base) + ] + + return { + 'architecture': self.service.architecture, + 'packages': packages, + 'repository': self.service.repository.name, + } \ No newline at end of file diff --git a/src/ahriman/web/views/package.py b/src/ahriman/web/views/package.py new file mode 100644 index 00000000..0548a3a1 --- /dev/null +++ b/src/ahriman/web/views/package.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from aiohttp.web import HTTPOk, Response + +from ahriman.models.build_status import BuildStatusEnum +from ahriman.models.package import Package +from ahriman.web.views.base import BaseView + + +class PackageView(BaseView): + + async def delete(self) -> Response: + base = self.request.match_info['package'] + self.service.remove(base) + + return HTTPOk() + + async def post(self) -> Response: + base = self.request.match_info['package'] + data = await self.request.json() + + package = Package(**data['package']) if 'package' in data else None + status = BuildStatusEnum(data.get('status', 'unknown')) + self.service.update(base, status, package) + + return HTTPOk() diff --git a/src/ahriman/web/views/packages.py b/src/ahriman/web/views/packages.py new file mode 100644 index 00000000..977ea3f8 --- /dev/null +++ b/src/ahriman/web/views/packages.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from aiohttp.web import HTTPOk, Response + +from ahriman.web.views.base import BaseView + + +class PackagesView(BaseView): + + async def post(self) -> Response: + self.service.load() + + return HTTPOk() diff --git a/src/ahriman/web/web.py b/src/ahriman/web/web.py new file mode 100644 index 00000000..30c3605f --- /dev/null +++ b/src/ahriman/web/web.py @@ -0,0 +1,73 @@ +# +# Copyright (c) 2021 Evgenii Alekseev. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import aiohttp_jinja2 +import jinja2 +import logging + +from aiohttp import web + +from ahriman.core.configuration import Configuration +from ahriman.core.exceptions import InitializeException +from ahriman.core.watcher.watcher import Watcher +from ahriman.web.middlewares.exception_handler import exception_handler +from ahriman.web.routes import setup_routes + + +async def on_shutdown(app: web.Application) -> None: + app.logger.warning('server terminated') + + +async def on_startup(app: web.Application) -> None: + app.logger.info('server started') + try: + app['watcher'].load() + except Exception as e: + app.logger.exception('could not load packages', exc_info=True) + raise InitializeException() from e + + +def run_server(app: web.Application) -> None: + app.logger.info('start server') + web.run_app(app, + host=app['config'].get('web', 'host'), + port=app['config'].getint('web', 'port'), + handle_signals=False) + + +def setup_service(architecture: str, config: Configuration) -> web.Application: + app = web.Application(logger=logging.getLogger('http')) + app.on_shutdown.append(on_shutdown) + app.on_startup.append(on_startup) + + app.middlewares.append(web.normalize_path_middleware(append_slash=False, remove_slash=True)) + app.middlewares.append(exception_handler(app.logger)) + + app.logger.info('setup routes') + setup_routes(app) + app.logger.info('setup templates') + aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader(config.get('web', 'templates'))) + + app.logger.info('setup configuration') + app['config'] = config + + app.logger.info('setup watcher') + app['watcher'] = Watcher(architecture, config) + + return app