clear code, allow to set overrides for each architecture

CONFIGURING.md

@@ -1,5 +1,7 @@
 # ahriman configuration
 
+Some groups can be specified for each architecture separately with default values. E.g. if there are `build` and `build_x86_64` groups it will use the `build_x86_64` for the `x86_64` architecture and `build` for any other.
+
 ## `settings` group
 
 Base configuration settings:
@@ -42,7 +44,9 @@ Report generation settings:
 
 * `target` - list of reports to be generated, space separated list of strings, optional. Allowed values are `html`.
 
-### `html` group
+### `html_*` group
+
+Group name must refer to architecture, e.g. it should be `html_x86_64` for x86_64 architecture.
 
 * `path` - path to html report file, string, required.
 * `css_path` - path to CSS to include in HTML, string, optional.
@@ -54,14 +58,14 @@ Remote synchronization settings:
 
 * `target` - list of synchronizations to be used, space separated list of strings, optional. Allowed values are `rsync`, `s3`.
 
-### `s3`
+### `s3_*` group
 
-Requires `aws-cli` package to be installed. Do not forget to configure it for user `ahriman`.
+Group name must refer to architecture, e.g. it should be `s3_x86_64` for x86_64 architecture. Requires `aws-cli` package to be installed. Do not forget to configure it for user `ahriman`.
 
 * `bucket` - bucket name (e.g. `s3://bucket/path`), string, required.
 
-### `rsync`
+### `rsync_*` group
 
-Requires `rsync` package to be installed. Do not forget to configure ssh for user `ahriman`.
+Group name must refer to architecture, e.g. it should be `rsync_x86_64` for x86_64 architecture. Requires `rsync` package to be installed. Do not forget to configure ssh for user `ahriman`.
 
 * `remote` - remote server to rsync (e.g. `1.2.3.4:5678:path/to/sync`), string, required.

package/archlinux/PKGBUILD

@@ -16,7 +16,7 @@ source=("https://github.com/arcan1s/ahriman/releases/download/$pkgver/$pkgname-$
         'ahriman.sudoers'
         'ahriman.sysusers'
         'ahriman.tmpfiles')
-sha512sums=('392e6f5f0ed9b333896f20ff4fa4f1a2ee1a43efe74eff8b63672419bc31ec2b9e2df100586a1dfd4eca89e53d131187532191c163d3420695e0361c335f3fe3'
+sha512sums=('db4409942e3189da01fa41f5432e634c8a3ce4013cd19ef15d63cb1123010702c2f5636fb25ba43e64cdf198b082f8d54dfd448f30288b2fc96e7b30ea213258'
             '8c9b5b63ac3f7b4d9debaf801a1e9c060877c33d3ecafe18010fcca778e5fa2f2e46909d3d0ff1b229ff8aa978445d8243fd36e1fc104117ed678d5e21901167'
             '13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075'
             '55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4')

package/etc/ahriman.ini

@@ -22,7 +22,7 @@ key =
 [report]
 target =
 
-[html]
+[html_x86_64]
 path =
 css_path =
 link_path =
@@ -30,8 +30,8 @@ link_path =
 [upload]
 target =
 
-[s3]
+[s3_x86_64]
 bucket =
 
-[rsync]
+[rsync_x86_64]
 remote =

src/ahriman/application/application.py

@@ -36,26 +36,7 @@ class Application:
         self.architecture = architecture
         self.repository = Repository(architecture, config)
 
-    def add(self, names: List[str]) -> None:
+    def _finalize(self) -> None:
-        for name in names:
-            package = Package.load(name, self.config.get('aur', 'url'))
-            task = Task(package, self.architecture, self.config, self.repository.paths)
-            task.fetch(os.path.join(self.repository.paths.manual, package.name))
-
-    def remove(self, names: List[str]) -> None:
-        self.repository.process_remove(names)
-
-    def report(self, target: Optional[List[str]] = None) -> None:
-        targets = target or None
-        self.repository.process_report(targets)
-
-    def sync(self, target: Optional[List[str]] = None) -> None:
-        targets = target or None
-        self.repository.process_sync(targets)
-
-    def update(self, updates: List[Package]) -> None:
-        packages = self.repository.process_build(updates)
-        self.repository.process_update(packages)
         self.report()
         self.sync()
 
@@ -72,3 +53,27 @@ class Application:
             log_fn(f'{package.name} = {package.version}')
 
         return updates
+
+    def add(self, names: List[str]) -> None:
+        for name in names:
+            package = Package.load(name, self.config.get('aur', 'url'))
+            task = Task(package, self.architecture, self.config, self.repository.paths)
+            task.fetch(os.path.join(self.repository.paths.manual, package.name))
+
+    def remove(self, names: List[str]) -> None:
+        self.repository.process_remove(names)
+        self._finalize()
+
+    def report(self, target: Optional[List[str]] = None) -> None:
+        targets = target or None
+        self.repository.process_report(targets)
+
+    def sync(self, target: Optional[List[str]] = None) -> None:
+        targets = target or None
+        self.repository.process_sync(targets)
+
+    def update(self, updates: List[Package]) -> None:
+        packages = self.repository.process_build(updates)
+        self.repository.process_update(packages)
+        self._finalize()
+

src/ahriman/core/build_tools/task.py

@@ -25,7 +25,7 @@ from typing import List, Optional
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.exceptions import BuildFailed
-from ahriman.core.util import check_output, options_list
+from ahriman.core.util import check_output
 from ahriman.models.package import Package
 from ahriman.models.repository_paths import RepositoryPaths
 
@@ -38,11 +38,11 @@ class Task:
         self.package = package
         self.paths = paths
 
-        section = f'build_{architecture}'
+        section = config.get_section_name('build', architecture)
-        self.archbuild_flags = options_list(config, section, 'archbuild_flags')
+        self.archbuild_flags = config.get_list(section, 'archbuild_flags')
         self.build_command = config.get(section, 'build_command')
-        self.makepkg_flags = options_list(config, section, 'makepkg_flags')
+        self.makepkg_flags = config.get_list(section, 'makepkg_flags')
-        self.makechrootpkg_flags = options_list(config, section, 'makechrootpkg_flags')
+        self.makechrootpkg_flags = config.get_list(section, 'makechrootpkg_flags')
 
     @property
     def git_path(self) -> str:

src/ahriman/core/configuration.py

@@ -21,9 +21,7 @@ import configparser
 import os
 
 from logging.config import fileConfig
-from typing import Dict, Optional
+from typing import List, Optional
-
-from ahriman.core.exceptions import MissingConfiguration
 
 
 # built-in configparser extension
@@ -37,6 +35,16 @@ class Configuration(configparser.RawConfigParser):
     def include(self) -> str:
         return self.get('settings', 'include')
 
+    def get_list(self, section: str, key: str) -> List[str]:
+        raw = self.get(section, key, fallback=None)
+        if not raw:  # empty string or none
+            return []
+        return raw.split()
+
+    def get_section_name(self, prefix: str, suffix: str) -> str:
+        probe = f'{prefix}_{suffix}'
+        return probe if self.has_section(probe) else prefix
+
     def load(self, path: str) -> None:
         self.path = path
         self.read(self.path)

src/ahriman/core/report/html.py

@@ -25,11 +25,12 @@ from ahriman.core.report.report import Report
 
 class HTML(Report):
 
-    def __init__(self, config: Configuration) -> None:
+    def __init__(self, architecture: str, config: Configuration) -> None:
-        Report.__init__(self, config)
+        Report.__init__(self, architecture, config)
-        self.report_path = config.get('html', 'path')
+        section = self.config.get_section_name('html', self.architecture)
-        self.css_path = config.get('html', 'css_path')
+        self.report_path = config.get(section, 'path')
-        self.link_path = config.get('html', 'link_path')
+        self.css_path = config.get(section, 'css_path')
+        self.link_path = config.get(section, 'link_path')
         self.title = config.get('repository', 'name')
 
     def generate(self, path: str) -> None:

src/ahriman/core/report/report.py

@@ -26,19 +26,20 @@ from ahriman.models.report_settings import ReportSettings
 
 class Report:
 
-    def __init__(self, config: Configuration) -> None:
+    def __init__(self, architecture: str, config: Configuration) -> None:
+        self.architecture = architecture
         self.config = config
         self.logger = logging.getLogger('builder')
 
     @staticmethod
-    def run(config: Configuration, target: str, path: str) -> None:
+    def run(architecture: str, config: Configuration, target: str, path: str) -> None:
         provider = ReportSettings.from_option(target)
         if provider == ReportSettings.HTML:
             from ahriman.core.report.html import HTML
-            report: Report = HTML(config)
+            report: Report = HTML(architecture, config)
         else:
             from ahriman.core.report.dummy import Dummy
-            report = Dummy(config)
+            report = Dummy(architecture, config)
 
         try:
             report.generate(path)

src/ahriman/core/repository.py

@@ -29,7 +29,6 @@ from ahriman.core.repo.repo_wrapper import RepoWrapper
 from ahriman.core.report.report import Report
 from ahriman.core.sign.gpg_wrapper import GPGWrapper
 from ahriman.core.upload.uploader import Uploader
-from ahriman.core.util import options_list
 from ahriman.models.package import Package
 from ahriman.models.repository_paths import RepositoryPaths
 
@@ -120,15 +119,15 @@ class Repository:
 
     def process_report(self, targets: Optional[List[str]]) -> None:
         if targets is None:
-            targets = options_list(self.config, 'report', 'target')
+            targets = self.config.get_list('report', 'target')
         for target in targets:
-            Report.run(self.config, target, self.paths.repository)
+            Report.run(self.architecture, self.config, target, self.paths.repository)
 
     def process_sync(self, targets: Optional[List[str]]) -> None:
         if targets is None:
-            targets = options_list(self.config, 'upload', 'target')
+            targets = self.config.get_list('upload', 'target')
         for target in targets:
-            Uploader.run(self.config, target, self.paths.repository)
+            Uploader.run(self.architecture, self.config, target, self.paths.repository)
 
     def process_update(self, packages: List[str]) -> str:
         for package in packages:

src/ahriman/core/upload/rsync.py

@@ -24,9 +24,10 @@ from ahriman.core.util import check_output
 
 class Rsync(Uploader):
 
-    def __init__(self, config: Configuration) -> None:
+    def __init__(self, architecture: str, config: Configuration) -> None:
-        Uploader.__init__(self, config)
+        Uploader.__init__(self, architecture, config)
-        self.remote = self.config.get('rsync', 'remote')
+        section = self.config.get_section_name('rsync', self.architecture)
+        self.remote = self.config.get(section, 'remote')
 
     def sync(self, path: str) -> None:
         check_output('rsync', '--archive', '--verbose', '--compress', '--partial', '--progress', '--delete', path, self.remote,

src/ahriman/core/upload/s3.py

@@ -24,12 +24,13 @@ from ahriman.core.util import check_output
 
 class S3(Uploader):
 
-    def __init__(self, config: Configuration) -> None:
+    def __init__(self, architecture: str, config: Configuration) -> None:
-        Uploader.__init__(self, config)
+        Uploader.__init__(self, architecture, config)
-        self.bucket = self.config.get('s3', 'bucket')
+        section = self.config.get_section_name('s3', self.architecture)
+        self.bucket = self.config.get(section, 'bucket')
 
     def sync(self, path: str) -> None:
         # TODO rewrite to boto, but it is bullshit
-        check_output('aws', 's3', 'sync', path, self.bucket,
+        check_output('aws', 's3', 'sync', '--delete', path, self.bucket,
                      exception=None,
                      logger=self.logger)

src/ahriman/core/upload/uploader.py

@@ -26,22 +26,23 @@ from ahriman.models.upload_settings import UploadSettings
 
 class Uploader:
 
-    def __init__(self, config: Configuration) -> None:
+    def __init__(self, architecture: str, config: Configuration) -> None:
+        self.architecture = architecture
         self.config = config
         self.logger = logging.getLogger('builder')
 
     @staticmethod
-    def run(config: Configuration, target: str, path: str) -> None:
+    def run(architecture: str, config: Configuration, target: str, path: str) -> None:
         provider = UploadSettings.from_option(target)
         if provider == UploadSettings.Rsync:
             from ahriman.core.upload.rsync import Rsync
-            uploader: Uploader = Rsync(config)
+            uploader: Uploader = Rsync(architecture, config)
         elif provider == UploadSettings.S3:
             from ahriman.core.upload.s3 import S3
-            uploader = S3(config)
+            uploader = S3(architecture, config)
         else:
             from ahriman.core.upload.dummy import Dummy
-            uploader = Dummy(config)
+            uploader = Dummy(architecture, config)
 
         try:
             uploader.sync(path)

src/ahriman/core/util.py

@@ -20,9 +20,7 @@
 import subprocess
 
 from logging import Logger
-from typing import List, Optional
+from typing import Optional
-
-from ahriman.core.configuration import Configuration
 
 
 def check_output(*args: str, exception: Optional[Exception],
@@ -39,10 +37,3 @@ def check_output(*args: str, exception: Optional[Exception],
                 logger.debug(line)
         raise exception or e
     return result
-
-
-def options_list(config: Configuration, section: str, key: str) -> List[str]:
-    raw = config.get(section, key, fallback=None)
-    if not raw:  # empty string or none
-        return []
-    return raw.split()