improvements

* multi-sign and multi-web configuration
* change default configuration to do not use architecture
* change units to be templated
* some refactoring

src/ahriman/application/ahriman.py

@@ -18,72 +18,38 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import argparse
-import os
-
-from typing import Optional
 
 import ahriman.version as version
 
 from ahriman.application.application import Application
+from ahriman.application.lock import Lock
 from ahriman.core.configuration import Configuration
 
 
-def _get_app(args: argparse.Namespace) -> Application:
-    config = _get_config(args.config)
-    return Application(args.architecture, config)
-
-
-def _get_config(config_path: str) -> Configuration:
-    config = Configuration()
-    config.load(config_path)
-    config.load_logging()
-    return config
-
-
-def _lock_check(path: Optional[str]) -> None:
-    if path is None:
-        return
-    if os.path.exists(args.lock):
-        raise RuntimeError('Another application instance is run')
-
-
-def _lock_create(path: Optional[str]) -> None:
-    if path is None:
-        return
-    open(path, 'w').close()
-
-
-def _lock_remove(path: Optional[str]) -> None:
-    if path is None:
-        return
-    if os.path.exists(path):
-        os.remove(path)
-
-
 def add(args: argparse.Namespace) -> None:
-    _get_app(args).add(args.package)
+    Application.from_args(args).add(args.package)
 
 
 def rebuild(args: argparse.Namespace) -> None:
-    app = _get_app(args)
+    app = Application.from_args(args)
     packages = app.repository.packages()
     app.update(packages)
 
 
 def remove(args: argparse.Namespace) -> None:
-    _get_app(args).remove(args.package)
+    Application.from_args(args).remove(args.package)
 
 
 def report(args: argparse.Namespace) -> None:
-    _get_app(args).report(args.target)
+    Application.from_args(args).report(args.target)
 
 
 def sync(args: argparse.Namespace) -> None:
-    _get_app(args).sync(args.target)
+    Application.from_args(args).sync(args.target)
 
 
 def update(args: argparse.Namespace) -> None:
-    app = _get_app(args)
+    app = Application.from_args(args)
     log_fn = lambda line: print(line) if args.dry_run else app.logger.info(line)
     packages = app.get_updates(args.no_aur, args.no_manual, args.no_vcs, log_fn)
     if args.dry_run:
@@ -93,9 +59,9 @@ def update(args: argparse.Namespace) -> None:
 
 def web(args: argparse.Namespace) -> None:
     from ahriman.web.web import run_server, setup_service
-    config = _get_config(args.config)
+    config = Configuration.from_path(args.config)
     app = setup_service(args.architecture, config)
-    run_server(app)
+    run_server(app, args.architecture)
 
 
 if __name__ == '__main__':
@@ -140,18 +106,9 @@ if __name__ == '__main__':
     web_parser.set_defaults(fn=web, lock=None)
 
     args = parser.parse_args()
-
-    if args.force:
-        _lock_remove(args.lock)
-    _lock_check(args.lock)
-
     if 'fn' not in args:
         parser.print_help()
         exit(1)
 
-    try:
-        _lock_create(args.lock)
+    with Lock(args.lock, args.force):
         args.fn(args)
-    finally:
-        _lock_remove(args.lock)
-

src/ahriman/application/application.py

@@ -17,11 +17,14 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+from __future__ import annotations
+
+import argparse
 import logging
 import os
 import shutil
 
-from typing import Callable, List, Optional
+from typing import Callable, List, Optional, Type
 
 from ahriman.core.build_tools.task import Task
 from ahriman.core.configuration import Configuration
@@ -37,6 +40,11 @@ class Application:
         self.architecture = architecture
         self.repository = Repository(architecture, config)
 
+    @classmethod
+    def from_args(cls: Type[Application], args: argparse.Namespace) -> Application:
+        config = Configuration.from_path(args.config)
+        return cls(args.architecture, config)
+
     def _finalize(self) -> None:
         self.report()
         self.sync()

src/ahriman/application/lock.py

@@ -0,0 +1,59 @@
+#
+# Copyright (c) 2021 Evgenii Alekseev.
+#
+# This file is part of ahriman 
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import os
+
+from typing import Optional
+
+from ahriman.core.exceptions import DuplicateRun
+
+
+class Lock:
+
+    def __init__(self, path: Optional[str], force: bool) -> None:
+        self.path = path
+        self.force = force
+
+    def __enter__(self):
+        if self.force:
+            self.remove()
+        self.check()
+        self.create()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.remove()
+        return True
+
+    def check(self) -> None:
+        if self.path is None:
+            return
+        if os.path.exists(self.path):
+            raise DuplicateRun()
+
+    def create(self) -> None:
+        if self.path is None:
+            return
+        open(self.path, 'w').close()
+
+    def remove(self) -> None:
+        if self.path is None:
+            return
+        if os.path.exists(self.path):
+            os.remove(self.path)

src/ahriman/core/build_tools/task.py

@@ -39,10 +39,10 @@ class Task:
         self.paths = paths
 
         section = config.get_section_name('build', architecture)
-        self.archbuild_flags = config.get_list(section, 'archbuild_flags')
+        self.archbuild_flags = config.getlist(section, 'archbuild_flags')
         self.build_command = config.get(section, 'build_command')
-        self.makepkg_flags = config.get_list(section, 'makepkg_flags')
-        self.makechrootpkg_flags = config.get_list(section, 'makechrootpkg_flags')
+        self.makepkg_flags = config.getlist(section, 'makepkg_flags')
+        self.makechrootpkg_flags = config.getlist(section, 'makechrootpkg_flags')
 
     @property
     def git_path(self) -> str:

src/ahriman/core/configuration.py

@@ -17,11 +17,13 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+from __future__ import annotations
+
 import configparser
 import os
 
 from logging.config import fileConfig
-from typing import List, Optional, Set
+from typing import List, Optional, Type
 
 
 # built-in configparser extension
@@ -35,7 +37,14 @@ class Configuration(configparser.RawConfigParser):
     def include(self) -> str:
         return self.get('settings', 'include')
 
-    def get_list(self, section: str, key: str) -> List[str]:
+    @classmethod
+    def from_path(cls: Type[Configuration], path: str) -> Configuration:
+        config = cls()
+        config.load(path)
+        config.load_logging()
+        return config
+
+    def getlist(self, section: str, key: str) -> List[str]:
         raw = self.get(section, key, fallback=None)
         if not raw:  # empty string or none
             return []
@@ -52,8 +61,7 @@ class Configuration(configparser.RawConfigParser):
 
     def load_includes(self) -> None:
         try:
-            include_dir = self.include
-            for conf in filter(lambda p: p.endswith('.ini'), sorted(os.listdir(include_dir))):
+            for conf in filter(lambda p: p.endswith('.ini'), sorted(os.listdir(self.include))):
                 self.read(os.path.join(self.include, conf))
         except (FileNotFoundError, configparser.NoOptionError):
             pass

src/ahriman/core/exceptions.py

@@ -25,6 +25,11 @@ class BuildFailed(Exception):
         Exception.__init__(self, f'Package {package} build failed, check logs for details')
 
 
+class DuplicateRun(Exception):
+    def __init__(self) -> None:
+        Exception.__init__(self, 'Another application instance is run')
+
+
 class InitializeException(Exception):
     def __init__(self) -> None:
         Exception.__init__(self, 'Could not load service')
@@ -40,16 +45,11 @@ class InvalidPackageInfo(Exception):
         Exception.__init__(self, f'There are errors during reading package information: `{details}`')
 
 
-class MissingConfiguration(Exception):
-    def __init__(self, name: str) -> None:
-        Exception.__init__(self, f'No section `{name}` found')
-
-
 class ReportFailed(Exception):
-    def __init__(self, cause: Exception) -> None:
-        Exception.__init__(self, f'Report failed with reason {cause}')
+    def __init__(self) -> None:
+        Exception.__init__(self, 'Report failed')
 
 
 class SyncFailed(Exception):
-    def __init__(self, cause: Exception) -> None:
-        Exception.__init__(self, f'Sync failed with reason {cause}')
+    def __init__(self) -> None:
+        Exception.__init__(self, 'Sync failed')

src/ahriman/core/report/html.py

@@ -32,14 +32,14 @@ class HTML(Report):
 
     def __init__(self, architecture: str, config: Configuration) -> None:
         Report.__init__(self, architecture, config)
-        section = self.config.get_section_name('html', self.architecture)
+        section = config.get_section_name('html', architecture)
         self.report_path = config.get(section, 'path')
 
         self.link_path = config.get(section, 'link_path')
         self.template_path = config.get(section, 'template_path')
 
         # base template vars
-        self.sign_targets = [SignSettings.from_option(opt) for opt in config.get_list('sign', 'target')]
+        self.sign_targets = [SignSettings.from_option(opt) for opt in config.getlist('sign', 'target')]
         self.pgp_key = config.get('sign', 'key', fallback=None)
         self.homepage = config.get(section, 'homepage', fallback=None)
         self.repository = config.get('repository', 'name')

src/ahriman/core/report/report.py

@@ -27,9 +27,9 @@ from ahriman.models.report_settings import ReportSettings
 class Report:
 
     def __init__(self, architecture: str, config: Configuration) -> None:
+        self.logger = logging.getLogger('builder')
         self.architecture = architecture
         self.config = config
-        self.logger = logging.getLogger('builder')
 
     @staticmethod
     def run(architecture: str, config: Configuration, target: str, path: str) -> None:
@@ -42,8 +42,9 @@ class Report:
 
         try:
             report.generate(path)
-        except Exception as e:
-            raise ReportFailed(e) from e
+        except Exception:
+            report.logger.exception('report generation failed', exc_info=True)
+            raise ReportFailed()
 
     def generate(self, path: str) -> None:
         pass

src/ahriman/core/repository.py

@@ -31,7 +31,6 @@ from ahriman.core.sign.gpg_wrapper import GPGWrapper
 from ahriman.core.upload.uploader import Uploader
 from ahriman.core.util import package_like
 from ahriman.core.watcher.client import Client
-from ahriman.models.build_status import BuildStatusEnum
 from ahriman.models.package import Package
 from ahriman.models.repository_paths import RepositoryPaths
 
@@ -46,13 +45,12 @@ class Repository:
         self.aur_url = config.get('aur', 'url')
         self.name = config.get('repository', 'name')
 
-        self.paths = RepositoryPaths(config.get('repository', 'root'), self.architecture)
+        self.paths = RepositoryPaths(config.get('repository', 'root'), architecture)
         self.paths.create_tree()
 
-        self.sign = GPGWrapper(config)
+        self.sign = GPGWrapper(architecture, config)
         self.wrapper = RepoWrapper(self.name, self.paths, self.sign.repository_sign_args)
-
-        self.web_report = Client.load(config)
+        self.web_report = Client.load(architecture, config)
 
     def _clear_build(self) -> None:
         for package in os.listdir(self.paths.sources):
@@ -82,7 +80,7 @@ class Repository:
 
     def process_build(self, updates: List[Package]) -> List[str]:
         def build_single(package: Package) -> None:
-            self.web_report.update(package.base, BuildStatusEnum.Building)
+            self.web_report.set_building(package.base)
             task = Task(package, self.architecture, self.config, self.paths)
             task.clone()
             built = task.build()
@@ -94,7 +92,7 @@ class Repository:
             try:
                 build_single(package)
             except Exception:
-                self.web_report.update(package.base, BuildStatusEnum.Failed)
+                self.web_report.set_failed(package.base)
                 self.logger.exception(f'{package.base} ({self.architecture}) build exception', exc_info=True)
                 continue
         self._clear_build()
@@ -126,13 +124,13 @@ class Repository:
 
     def process_report(self, targets: Optional[List[str]]) -> None:
         if targets is None:
-            targets = self.config.get_list('report', 'target')
+            targets = self.config.getlist('report', 'target')
         for target in targets:
             Report.run(self.architecture, self.config, target, self.paths.repository)
 
     def process_sync(self, targets: Optional[List[str]]) -> None:
         if targets is None:
-            targets = self.config.get_list('upload', 'target')
+            targets = self.config.getlist('upload', 'target')
         for target in targets:
             Uploader.run(self.architecture, self.config, target, self.paths.repository)
 
@@ -146,18 +144,19 @@ class Repository:
                     shutil.move(src, dst)
                 package_fn = os.path.join(self.paths.repository, os.path.basename(package))
                 self.wrapper.add(package_fn)
-                self.web_report.add(local, BuildStatusEnum.Success)
+                self.web_report.set_success(local)
             except Exception:
                 self.logger.exception(f'could not process {package}', exc_info=True)
-                self.web_report.update(local.base, BuildStatusEnum.Failed)
+                self.web_report.set_failed(local.base)
         self._clear_packages()
 
         return self.wrapper.repo_path
 
     def updates_aur(self, no_vcs: bool) -> List[Package]:
         result: List[Package] = []
-        ignore_list = self.config.get_list(
-            self.config.get_section_name('build', self.architecture), 'ignore_packages')
+
+        build_section = self.config.get_section_name('build', self.architecture)
+        ignore_list = self.config.getlist(build_section, 'ignore_packages')
 
         for local in self.packages():
             if local.base in ignore_list:
@@ -169,9 +168,9 @@ class Repository:
                 remote = Package.load(local.base, self.aur_url)
                 if local.is_outdated(remote):
                     result.append(remote)
-                    self.web_report.update(local.base, BuildStatusEnum.Pending)
+                    self.web_report.set_pending(local.base)
             except Exception:
-                self.web_report.update(local.base, BuildStatusEnum.Failed)
+                self.web_report.set_failed(local.base)
                 self.logger.exception(f'could not load remote package {local.base}', exc_info=True)
                 continue
 
@@ -184,7 +183,7 @@ class Repository:
             try:
                 local = Package.load(os.path.join(self.paths.manual, fn), self.aur_url)
                 result.append(local)
-                self.web_report.add(local, BuildStatusEnum.Unknown)
+                self.web_report.set_unknown(local)
             except Exception:
                 self.logger.exception(f'could not add package from {fn}', exc_info=True)
         self._clear_manual()

src/ahriman/core/sign/gpg_wrapper.py

@@ -30,11 +30,11 @@ from ahriman.models.sign_settings import SignSettings
 
 class GPGWrapper:
 
-    def __init__(self, config: Configuration) -> None:
+    def __init__(self, architecture: str, config: Configuration) -> None:
         self.logger = logging.getLogger('build_details')
-
-        self.target = [SignSettings.from_option(opt) for opt in config.get_list('sign', 'target')]
-        self.key = config.get('sign', 'key') if self.target else None
+        section = config.get_section_name('sign', architecture)
+        self.target = [SignSettings.from_option(opt) for opt in config.getlist(section, 'target')]
+        self.key = config.get(section, 'key') if self.target else None
 
     @property
     def repository_sign_args(self) -> List[str]:

src/ahriman/core/upload/rsync.py

@@ -26,8 +26,8 @@ class Rsync(Uploader):
 
     def __init__(self, architecture: str, config: Configuration) -> None:
         Uploader.__init__(self, architecture, config)
-        section = self.config.get_section_name('rsync', self.architecture)
-        self.remote = self.config.get(section, 'remote')
+        section = config.get_section_name('rsync', architecture)
+        self.remote = config.get(section, 'remote')
 
     def sync(self, path: str) -> None:
         check_output('rsync', '--archive', '--verbose', '--compress', '--partial', '--progress', '--delete', path, self.remote,

src/ahriman/core/upload/s3.py

@@ -26,8 +26,8 @@ class S3(Uploader):
 
     def __init__(self, architecture: str, config: Configuration) -> None:
         Uploader.__init__(self, architecture, config)
-        section = self.config.get_section_name('s3', self.architecture)
-        self.bucket = self.config.get(section, 'bucket')
+        section = config.get_section_name('s3', architecture)
+        self.bucket = config.get(section, 'bucket')
 
     def sync(self, path: str) -> None:
         # TODO rewrite to boto, but it is bullshit

src/ahriman/core/upload/uploader.py

@@ -27,9 +27,9 @@ from ahriman.models.upload_settings import UploadSettings
 class Uploader:
 
     def __init__(self, architecture: str, config: Configuration) -> None:
+        self.logger = logging.getLogger('builder')
         self.architecture = architecture
         self.config = config
-        self.logger = logging.getLogger('builder')
 
     @staticmethod
     def run(architecture: str, config: Configuration, target: str, path: str) -> None:
@@ -45,8 +45,9 @@ class Uploader:
 
         try:
             uploader.sync(path)
-        except Exception as e:
-            raise SyncFailed(e) from e
+        except Exception:
+            uploader.logger.exception('remote sync failed', exc_info=True)
+            raise SyncFailed()
 
     def sync(self, path: str) -> None:
         pass

src/ahriman/core/watcher/client.py

@@ -39,10 +39,26 @@ class Client:
     def update(self, base: str, status: BuildStatusEnum) -> None:
         pass
 
+    def set_building(self, base: str) -> None:
+        return self.update(base, BuildStatusEnum.Building)
+
+    def set_failed(self, base: str) -> None:
+        return self.update(base, BuildStatusEnum.Failed)
+
+    def set_pending(self, base: str) -> None:
+        return self.update(base, BuildStatusEnum.Pending)
+
+    def set_success(self, package: Package) -> None:
+        return self.add(package, BuildStatusEnum.Success)
+
+    def set_unknown(self, package: Package) -> None:
+        return self.add(package, BuildStatusEnum.Unknown)
+
     @staticmethod
-    def load(config: Configuration) -> Client:
-        host = config.get('web', 'host', fallback=None)
-        port = config.getint('web', 'port', fallback=None)
+    def load(architecture: str, config: Configuration) -> Client:
+        section = config.get_section_name('web', architecture)
+        host = config.get(section, 'host', fallback=None)
+        port = config.getint(section, 'port', fallback=None)
         if host is None or port is None:
             return Client()
         return WebClient(host, port)

src/ahriman/models/package.py

@@ -19,10 +19,9 @@
 #
 from __future__ import annotations
 
-import shutil
-
 import aur
 import os
+import shutil
 import tempfile
 
 from dataclasses import dataclass, field

src/ahriman/models/repository_paths.py

@@ -29,22 +29,37 @@ class RepositoryPaths:
 
     @property
     def chroot(self) -> str:
+        '''
+        :return: directory for devtools chroot
+        '''
         return os.path.join(self.root, 'chroot')
 
     @property
     def manual(self) -> str:
+        '''
+        :return: directory for manual updates (i.e. from add command)
+        '''
         return os.path.join(self.root, 'manual')
 
     @property
     def packages(self) -> str:
+        '''
+        :return: directory for built packages
+        '''
         return os.path.join(self.root, 'packages')
 
     @property
     def repository(self) -> str:
+        '''
+        :return: repository directory
+        '''
         return os.path.join(self.root, 'repository', self.architecture)
 
     @property
     def sources(self) -> str:
+        '''
+        :return: directory for downloaded PKGBUILDs for current build
+        '''
         return os.path.join(self.root, 'sources')
 
     def create_tree(self) -> None:

src/ahriman/web/middlewares/exception_handler.py

@@ -18,11 +18,10 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from aiohttp.web import middleware, Request, Response
+from aiohttp.web_exceptions import HTTPClientError
 from logging import Logger
 from typing import Callable
 
-from aiohttp.web_exceptions import HTTPClientError
-
 
 def exception_handler(logger: Logger) -> Callable:
     @middleware

src/ahriman/web/views/base.py

@@ -22,6 +22,7 @@ from aiohttp.web import View
 from ahriman.core.watcher.watcher import Watcher
 
 
+# special class to make it typed
 class BaseView(View):
 
     @property

src/ahriman/web/views/index.py

@@ -17,18 +17,17 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Any, Dict
-
 from aiohttp_jinja2 import template
+from typing import Any, Dict
 
 from ahriman.web.views.base import BaseView
 
 
 class IndexView(BaseView):
 
-    @template("index.jinja2")
+    @template("build-status.jinja2")
     async def get(self) -> Dict[str, Any]:
-        # some magic to make it jinja-readable
+        # some magic to make it jinja-friendly
         packages = [
             {
                 'base': package.base,

src/ahriman/web/web.py

@@ -38,17 +38,19 @@ async def on_startup(app: web.Application) -> None:
     app.logger.info('server started')
     try:
         app['watcher'].load()
-    except Exception as e:
+    except Exception:
         app.logger.exception('could not load packages', exc_info=True)
-        raise InitializeException() from e
+        raise InitializeException()
 
 
-def run_server(app: web.Application) -> None:
+def run_server(app: web.Application, architecture: str) -> None:
     app.logger.info('start server')
-    web.run_app(app,
-                host=app['config'].get('web', 'host'),
-                port=app['config'].getint('web', 'port'),
-                handle_signals=False)
+
+    section = app['config'].get_section_name('web', architecture)
+    host = app['config'].get(section, 'host')
+    port = app['config'].getint(section, 'port')
+
+    web.run_app(app, host=host, port=port, handle_signals=False)
 
 
 def setup_service(architecture: str, config: Configuration) -> web.Application: