make configuration object arch-specific

Makefile

@@ -24,7 +24,7 @@ archlinux: archive
 	sed -i "/sha512sums=('[0-9A-Fa-f]*/s/[^'][^)]*/sha512sums=('$$(sha512sum $(PROJECT)-$(VERSION)-src.tar.xz | awk '{print $$1}')'/" package/archlinux/PKGBUILD
 	sed -i "s/pkgver=[0-9.]*/pkgver=$(VERSION)/" package/archlinux/PKGBUILD
 
-check:
+check: clean
 	cd src && mypy --implicit-reexport --strict -p "$(PROJECT)"
 	find "src/$(PROJECT)" tests -name "*.py" -execdir autopep8 --exit-code --max-line-length 120 -aa -i {} +
 	cd src && pylint --rcfile=../.pylintrc "$(PROJECT)"
@@ -43,7 +43,7 @@ push: archlinux
 	git tag "$(VERSION)"
 	git push --tags
 
-tests:
+tests: clean
 	python setup.py test
 
 version:

src/ahriman/application/handlers/dump.py

@@ -40,7 +40,7 @@ class Dump(Handler):
         :param architecture: repository architecture
         :param configuration: configuration instance
         """
-        dump = configuration.dump(architecture)
+        dump = configuration.dump()
         for section, values in sorted(dump.items()):
             Dump._print(f"[{section}]")
             for key, value in sorted(values.items()):

src/ahriman/application/handlers/handler.py

@@ -35,15 +35,15 @@ class Handler:
     """
 
     @classmethod
-    def _call(cls: Type[Handler], args: argparse.Namespace, architecture: str, configuration: Configuration) -> bool:
+    def _call(cls: Type[Handler], args: argparse.Namespace, architecture: str) -> bool:
         """
         additional function to wrap all calls for multiprocessing library
         :param args: command line args
         :param architecture: repository architecture
-        :param configuration: configuration instance
         :return: True on success, False otherwise
         """
         try:
+            configuration = Configuration.from_path(args.configuration, architecture, not args.no_log)
             with Lock(args, architecture, configuration):
                 cls.run(args, architecture, configuration)
             return True
@@ -58,10 +58,9 @@ class Handler:
         :param args: command line args
         :return: 0 on success, 1 otherwise
         """
-        configuration = Configuration.from_path(args.configuration, not args.no_log)
         with Pool(len(args.architecture)) as pool:
             result = pool.starmap(
-                cls._call, [(args, architecture, configuration) for architecture in args.architecture])
+                cls._call, [(args, architecture) for architecture in set(args.architecture)])
         return 0 if all(result) else 1
 
     @classmethod

src/ahriman/application/lock.py

@@ -54,7 +54,7 @@ class Lock:
         self.unsafe = args.unsafe
 
         self.root = Path(configuration.get("repository", "root"))
-        self.reporter = Client() if args.no_report else Client.load(architecture, configuration)
+        self.reporter = Client() if args.no_report else Client.load(configuration)
 
     def __enter__(self) -> Lock:
         """

src/ahriman/core/build_tools/task.py

@@ -41,12 +41,10 @@ class Task:
 
     _check_output = check_output
 
-    def __init__(self, package: Package, architecture: str, configuration: Configuration,
-                 paths: RepositoryPaths) -> None:
+    def __init__(self, package: Package, configuration: Configuration, paths: RepositoryPaths) -> None:
         """
         default constructor
         :param package: package definitions
-        :param architecture: repository architecture
         :param configuration: configuration instance
         :param paths: repository paths instance
         """
@@ -55,11 +53,10 @@ class Task:
         self.package = package
         self.paths = paths
 
-        self.archbuild_flags = configuration.wrap("build", architecture, "archbuild_flags", configuration.getlist)
-        self.build_command = configuration.wrap("build", architecture, "build_command", configuration.get)
-        self.makepkg_flags = configuration.wrap("build", architecture, "makepkg_flags", configuration.getlist)
-        self.makechrootpkg_flags = configuration.wrap("build", architecture, "makechrootpkg_flags",
-                                                      configuration.getlist)
+        self.archbuild_flags = configuration.getlist("build", "archbuild_flags")
+        self.build_command = configuration.get("build", "build_command")
+        self.makepkg_flags = configuration.getlist("build", "makepkg_flags")
+        self.makechrootpkg_flags = configuration.getlist("build", "makechrootpkg_flags")
 
     @property
     def cache_path(self) -> Path:

src/ahriman/core/configuration.py

@@ -24,10 +24,7 @@ import logging
 
 from logging.config import fileConfig
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Type, TypeVar
-
-
-T = TypeVar("T")
+from typing import Dict, List, Optional, Type
 
 
 class Configuration(configparser.RawConfigParser):
@@ -37,13 +34,11 @@ class Configuration(configparser.RawConfigParser):
     :cvar ARCHITECTURE_SPECIFIC_SECTIONS: known sections which can be architecture specific (required by dump)
     :cvar DEFAULT_LOG_FORMAT: default log format (in case of fallback)
     :cvar DEFAULT_LOG_LEVEL: default log level (in case of fallback)
-    :cvar STATIC_SECTIONS: known sections which are not architecture specific (required by dump)
     """
 
     DEFAULT_LOG_FORMAT = "[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s"
     DEFAULT_LOG_LEVEL = logging.DEBUG
 
-    STATIC_SECTIONS = ["alpm", "report", "repository", "settings", "upload"]
     ARCHITECTURE_SPECIFIC_SECTIONS = ["build", "html", "rsync", "s3", "sign", "web"]
 
     def __init__(self) -> None:
@@ -60,16 +55,24 @@ class Configuration(configparser.RawConfigParser):
         """
         return self.getpath("settings", "include")
 
+    @property
+    def logging_path(self) -> Path:
+        """
+        :return: path to logging configuration
+        """
+        return self.getpath("settings", "logging")
+
     @classmethod
-    def from_path(cls: Type[Configuration], path: Path, logfile: bool) -> Configuration:
+    def from_path(cls: Type[Configuration], path: Path, architecture: str, logfile: bool) -> Configuration:
         """
         constructor with full object initialization
         :param path: path to root configuration file
+        :param architecture: repository architecture
         :param logfile: use log file to output messages
         :return: configuration instance
         """
         config = cls()
-        config.load(path)
+        config.load(path, architecture)
         config.load_logging(logfile)
         return config
 
@@ -83,29 +86,15 @@ class Configuration(configparser.RawConfigParser):
         """
         return f"{section}_{architecture}"
 
-    def dump(self, architecture: str) -> Dict[str, Dict[str, str]]:
+    def dump(self) -> Dict[str, Dict[str, str]]:
         """
         dump configuration to dictionary
-        :param architecture: repository architecture
         :return: configuration dump for specific architecture
         """
-        result: Dict[str, Dict[str, str]] = {}
-        for section in Configuration.STATIC_SECTIONS:
-            if not self.has_section(section):
-                continue
-            result[section] = dict(self[section])
-        for section in Configuration.ARCHITECTURE_SPECIFIC_SECTIONS:
-            # get global settings
-            settings = dict(self[section]) if self.has_section(section) else {}
-            # get overrides
-            specific = self.section_name(section, architecture)
-            specific_settings = dict(self[specific]) if self.has_section(specific) else {}
-            # merge
-            settings.update(specific_settings)
-            if settings:  # append only in case if it is not empty
-                result[section] = settings
-
-        return result
+        return {
+            section: dict(self[section])
+            for section in self.sections()
+        }
 
     def getlist(self, section: str, key: str) -> List[str]:
         """
@@ -131,14 +120,16 @@ class Configuration(configparser.RawConfigParser):
             return value
         return self.path.parent / value
 
-    def load(self, path: Path) -> None:
+    def load(self, path: Path, architecture: str) -> None:
         """
         fully load configuration
         :param path: path to root configuration file
+        :param architecture: repository architecture
         """
         self.path = path
         self.read(self.path)
         self.load_includes()
+        self.merge_sections(architecture)
 
     def load_includes(self) -> None:
         """
@@ -146,6 +137,8 @@ class Configuration(configparser.RawConfigParser):
         """
         try:
             for path in sorted(self.include.glob("*.ini")):
+                if path == self.logging_path:
+                    continue  # we don't want to load logging explicitly
                 self.read(path)
         except (FileNotFoundError, configparser.NoOptionError):
             pass
@@ -157,32 +150,33 @@ class Configuration(configparser.RawConfigParser):
         """
         def file_logger() -> None:
             try:
-                path = self.getpath("settings", "logging")
+                path = self.logging_path
                 fileConfig(path)
             except (FileNotFoundError, PermissionError):
                 console_logger()
                 logging.exception("could not create logfile, fallback to stderr")
 
         def console_logger() -> None:
-            logging.basicConfig(filename=None, format=Configuration.DEFAULT_LOG_FORMAT,
-                                level=Configuration.DEFAULT_LOG_LEVEL)
+            logging.basicConfig(filename=None, format=self.DEFAULT_LOG_FORMAT,
+                                level=self.DEFAULT_LOG_LEVEL)
 
         if logfile:
             file_logger()
         else:
             console_logger()
 
-    def wrap(self, section: str, architecture: str, key: str, function: Callable[..., T], **kwargs: Any) -> T:
+    def merge_sections(self, architecture: str) -> None:
         """
-        wrapper to get option by either using architecture specific section or generic section
-        :param section: section name
+        merge architecture specific sections into main configuration
         :param architecture: repository architecture
-        :param key: key name
-        :param function: function to call, e.g. `Configuration.get`
-        :param kwargs: any other keywords which will be passed to function directly
-        :return: either value from architecture specific section or global value
         """
-        specific_section = self.section_name(section, architecture)
-        if self.has_option(specific_section, key):
-            return function(specific_section, key, **kwargs)
-        return function(section, key, **kwargs)
+        for section in self.ARCHITECTURE_SPECIFIC_SECTIONS:
+            # get overrides
+            specific = self.section_name(section, architecture)
+            if not self.has_section(specific):
+                continue  # no overrides
+            if not self.has_section(section):
+                self.add_section(section)  # add section if not exists
+            for key, value in self[specific].items():
+                self.set(section, key, value)
+            self.remove_section(specific)  # remove overrides

src/ahriman/core/report/html.py

@@ -70,15 +70,15 @@ class HTML(Report):
         :param configuration: configuration instance
         """
         Report.__init__(self, architecture, configuration)
-        self.report_path = configuration.wrap("html", architecture, "path", configuration.getpath)
-        self.link_path = configuration.wrap("html", architecture, "link_path", configuration.get)
-        self.template_path = configuration.wrap("html", architecture, "template_path", configuration.getpath)
+        self.report_path = configuration.getpath("html", "path")
+        self.link_path = configuration.get("html", "link_path")
+        self.template_path = configuration.getpath("html", "template_path")
 
         # base template vars
-        self.homepage = configuration.wrap("html", architecture, "homepage", configuration.get, fallback=None)
+        self.homepage = configuration.get("html", "homepage", fallback=None)
         self.name = configuration.get("repository", "name")
 
-        self.sign_targets, self.default_pgp_key = GPG.sign_options(architecture, configuration)
+        self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration)
 
     def generate(self, packages: Iterable[Package]) -> None:
         """

src/ahriman/core/repository/executor.py

@@ -49,7 +49,7 @@ class Executor(Cleaner):
         """
         def build_single(package: Package) -> None:
             self.reporter.set_building(package.base)
-            task = Task(package, self.architecture, self.configuration, self.paths)
+            task = Task(package, self.configuration, self.paths)
             task.init()
             built = task.build()
             for src in built:

src/ahriman/core/repository/properties.py

@@ -56,4 +56,4 @@ class Properties:
         self.pacman = Pacman(configuration)
         self.sign = GPG(architecture, configuration)
         self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
-        self.reporter = Client.load(architecture, configuration)
+        self.reporter = Client.load(configuration)

src/ahriman/core/repository/update_handler.py

@@ -44,7 +44,7 @@ class UpdateHandler(Cleaner):
         """
         result: List[Package] = []
 
-        ignore_list = self.configuration.wrap("build", self.architecture, "ignore_packages", self.configuration.getlist)
+        ignore_list = self.configuration.getlist("build", "ignore_packages")
 
         for local in self.packages():
             if local.base in ignore_list:

src/ahriman/core/sign/gpg.py

@@ -49,7 +49,7 @@ class GPG:
         self.logger = logging.getLogger("build_details")
         self.architecture = architecture
         self.configuration = configuration
-        self.targets, self.default_key = self.sign_options(architecture, configuration)
+        self.targets, self.default_key = self.sign_options(configuration)
 
     @property
     def repository_sign_args(self) -> List[str]:
@@ -74,18 +74,17 @@ class GPG:
         return ["gpg", "-u", key, "-b", str(path)]
 
     @staticmethod
-    def sign_options(architecture: str, configuration: Configuration) -> Tuple[Set[SignSettings], Optional[str]]:
+    def sign_options(configuration: Configuration) -> Tuple[Set[SignSettings], Optional[str]]:
         """
         extract default sign options from configuration
-        :param architecture: repository architecture
         :param configuration: configuration instance
         :return: tuple of sign targets and default PGP key
         """
         targets = {
             SignSettings.from_option(option)
-            for option in configuration.wrap("sign", architecture, "targets", configuration.getlist)
+            for option in configuration.getlist("sign", "targets")
         }
-        default_key = configuration.wrap("sign", architecture, "key", configuration.get) if targets else None
+        default_key = configuration.get("sign", "key") if targets else None
         return targets, default_key
 
     def process(self, path: Path, key: str) -> List[Path]:
@@ -110,8 +109,7 @@ class GPG:
         """
         if SignSettings.SignPackages not in self.targets:
             return [path]
-        key = self.configuration.wrap("sign", self.architecture, f"key_{base}",
-                                      self.configuration.get, fallback=self.default_key)
+        key = self.configuration.get("sign", f"key_{base}", fallback=self.default_key)
         if key is None:
             self.logger.error(f"no default key set, skip package {path} sign")
             return [path]

src/ahriman/core/status/client.py

@@ -111,15 +111,14 @@ class Client:
         return self.add(package, BuildStatusEnum.Unknown)
 
     @staticmethod
-    def load(architecture: str, configuration: Configuration) -> Client:
+    def load(configuration: Configuration) -> Client:
         """
         load client from settings
-        :param architecture: repository architecture
         :param configuration: configuration instance
         :return: client according to current settings
         """
-        host = configuration.wrap("web", architecture, "host", configuration.get, fallback=None)
-        port = configuration.wrap("web", architecture, "port", configuration.getint, fallback=None)
+        host = configuration.get("web", "host", fallback=None)
+        port = configuration.getint("web", "port", fallback=None)
         if host is None or port is None:
             return Client()
 

src/ahriman/core/upload/rsync.py

@@ -40,8 +40,8 @@ class Rsync(Upload):
         :param configuration: configuration instance
         """
         Upload.__init__(self, architecture, configuration)
-        self.command = configuration.wrap("rsync", architecture, "command", configuration.getlist)
-        self.remote = configuration.wrap("rsync", architecture, "remote", configuration.get)
+        self.command = configuration.getlist("rsync", "command")
+        self.remote = configuration.get("rsync", "remote")
 
     def sync(self, path: Path) -> None:
         """

src/ahriman/core/upload/s3.py

@@ -40,8 +40,8 @@ class S3(Upload):
         :param configuration: configuration instance
         """
         Upload.__init__(self, architecture, configuration)
-        self.bucket = configuration.wrap("s3", architecture, "bucket", configuration.get)
-        self.command = configuration.wrap("s3", architecture, "command", configuration.getlist)
+        self.bucket = configuration.get("s3", "bucket")
+        self.command = configuration.getlist("s3", "command")
 
     def sync(self, path: Path) -> None:
         """

src/ahriman/web/web.py

@@ -58,10 +58,9 @@ def run_server(application: web.Application) -> None:
     """
     application.logger.info("start server")
 
-    architecture: str = application["architecture"]
     configuration: Configuration = application["configuration"]
-    host = configuration.wrap("web", architecture, "host", configuration.get)
-    port = configuration.wrap("web", architecture, "port", configuration.getint)
+    host = configuration.get("web", "host")
+    port = configuration.getint("web", "port")
 
     web.run_app(application, host=host, port=port, handle_signals=False,
                 access_log=logging.getLogger("http"))
@@ -89,7 +88,6 @@ def setup_service(architecture: str, configuration: Configuration) -> web.Applic
 
     application.logger.info("setup configuration")
     application["configuration"] = configuration
-    application["architecture"] = architecture
 
     application.logger.info("setup watcher")
     application["watcher"] = Watcher(architecture, configuration)

tests/ahriman/application/handlers/test_handler.py

@@ -3,25 +3,27 @@ import argparse
 from pytest_mock import MockerFixture
 
 from ahriman.application.handlers import Handler
-from ahriman.core.configuration import Configuration
 
 
-def test_call(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
+def test_call(args: argparse.Namespace, mocker: MockerFixture) -> None:
     """
     must call inside lock
     """
+    args.configuration = ""
+    args.no_log = False
     mocker.patch("ahriman.application.handlers.Handler.run")
+    mocker.patch("ahriman.core.configuration.Configuration.from_path")
     enter_mock = mocker.patch("ahriman.application.lock.Lock.__enter__")
     exit_mock = mocker.patch("ahriman.application.lock.Lock.__exit__")
 
-    assert Handler._call(args, "x86_64", configuration)
+    assert Handler._call(args, "x86_64")
     enter_mock.assert_called_once()
     exit_mock.assert_called_once()
 
 
-def test_call_exception(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
+def test_call_exception(args: argparse.Namespace, mocker: MockerFixture) -> None:
     """
     must process exception
     """
     mocker.patch("ahriman.application.lock.Lock.__enter__", side_effect=Exception())
-    assert not Handler._call(args, "x86_64", configuration)
+    assert not Handler._call(args, "x86_64")

tests/ahriman/application/handlers/test_handler_dump.py

@@ -13,7 +13,7 @@ def test_run(args: argparse.Namespace, configuration: Configuration, mocker: Moc
     mocker.patch("pathlib.Path.mkdir")
     print_mock = mocker.patch("ahriman.application.handlers.dump.Dump._print")
     application_mock = mocker.patch("ahriman.core.configuration.Configuration.dump",
-                                    return_value=configuration.dump("x86_64"))
+                                    return_value=configuration.dump())
 
     Dump.run(args, "x86_64", configuration)
     application_mock.assert_called_once()

tests/ahriman/conftest.py

@@ -27,7 +27,7 @@ def anyvar(cls: Type[T], strict: bool = False) -> T:
 @pytest.fixture
 def configuration(resource_path_root: Path) -> Configuration:
     path = resource_path_root / "core" / "ahriman.ini"
-    return Configuration.from_path(path=path, logfile=False)
+    return Configuration.from_path(path=path, architecture="x86_64", logfile=False)
 
 
 @pytest.fixture

tests/ahriman/core/conftest.py

@@ -31,4 +31,4 @@ def repo(configuration: Configuration, repository_paths: RepositoryPaths) -> Rep
 
 @pytest.fixture
 def task_ahriman(package_ahriman: Package, configuration: Configuration, repository_paths: RepositoryPaths) -> Task:
-    return Task(package_ahriman, "x86_64", configuration, repository_paths)
+    return Task(package_ahriman, configuration, repository_paths)

tests/ahriman/core/status/test_client.py

@@ -104,7 +104,7 @@ def test_load_dummy_client(configuration: Configuration) -> None:
     """
     must load dummy client if no settings set
     """
-    assert isinstance(Client.load("x86_64", configuration), Client)
+    assert isinstance(Client.load(configuration), Client)
 
 
 def test_load_full_client(configuration: Configuration) -> None:
@@ -113,4 +113,4 @@ def test_load_full_client(configuration: Configuration) -> None:
     """
     configuration.set("web", "host", "localhost")
     configuration.set("web", "port", "8080")
-    assert isinstance(Client.load("x86_64", configuration), WebClient)
+    assert isinstance(Client.load(configuration), WebClient)

tests/ahriman/core/test_configuration.py

@@ -14,7 +14,7 @@ def test_from_path(mocker: MockerFixture) -> None:
     load_logging_mock = mocker.patch("ahriman.core.configuration.Configuration.load_logging")
     path = Path("path")
 
-    config = Configuration.from_path(path, True)
+    config = Configuration.from_path(path, "x86_64", True)
     assert config.path == path
     read_mock.assert_called_with(path)
     load_includes_mock.assert_called_once()
@@ -53,7 +53,7 @@ def test_dump(configuration: Configuration) -> None:
     """
     dump must not be empty
     """
-    assert configuration.dump("x86_64")
+    assert configuration.dump()
 
 
 def test_dump_architecture_specific(configuration: Configuration) -> None:
@@ -62,8 +62,9 @@ def test_dump_architecture_specific(configuration: Configuration) -> None:
     """
     configuration.add_section("build_x86_64")
     configuration.set("build_x86_64", "archbuild_flags", "hello flag")
+    configuration.merge_sections("x86_64")
 
-    dump = configuration.dump("x86_64")
+    dump = configuration.dump()
     assert dump
     assert "build" in dump
     assert "build_x86_64" not in dump
@@ -118,3 +119,15 @@ def test_load_logging_stderr(configuration: Configuration, mocker: MockerFixture
     logging_mock = mocker.patch("logging.config.fileConfig")
     configuration.load_logging(False)
     logging_mock.assert_not_called()
+
+
+def test_merge_sections_missing(configuration: Configuration) -> None:
+    """
+    must merge create section if not exists
+    """
+    configuration.remove_section("build")
+    configuration.add_section("build_x86_64")
+    configuration.set("build_x86_64", "key", "value")
+
+    configuration.merge_sections("x86_64")
+    assert configuration.get("build", "key") == "value"