diff --git a/Makefile b/Makefile index f12c835b..34fe7a2c 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ archlinux: archive sed -i "/sha512sums=('[0-9A-Fa-f]*/s/[^'][^)]*/sha512sums=('$$(sha512sum $(PROJECT)-$(VERSION)-src.tar.xz | awk '{print $$1}')'/" package/archlinux/PKGBUILD sed -i "s/pkgver=[0-9.]*/pkgver=$(VERSION)/" package/archlinux/PKGBUILD -check: +check: clean cd src && mypy --implicit-reexport --strict -p "$(PROJECT)" find "src/$(PROJECT)" tests -name "*.py" -execdir autopep8 --exit-code --max-line-length 120 -aa -i {} + cd src && pylint --rcfile=../.pylintrc "$(PROJECT)" @@ -43,7 +43,7 @@ push: archlinux git tag "$(VERSION)" git push --tags -tests: +tests: clean python setup.py test version: diff --git a/src/ahriman/application/handlers/dump.py b/src/ahriman/application/handlers/dump.py index 905d8300..8a9151c8 100644 --- a/src/ahriman/application/handlers/dump.py +++ b/src/ahriman/application/handlers/dump.py @@ -40,7 +40,7 @@ class Dump(Handler): :param architecture: repository architecture :param configuration: configuration instance """ - dump = configuration.dump(architecture) + dump = configuration.dump() for section, values in sorted(dump.items()): Dump._print(f"[{section}]") for key, value in sorted(values.items()): diff --git a/src/ahriman/application/handlers/handler.py b/src/ahriman/application/handlers/handler.py index 38bf1a8d..333ee13f 100644 --- a/src/ahriman/application/handlers/handler.py +++ b/src/ahriman/application/handlers/handler.py @@ -35,15 +35,15 @@ class Handler: """ @classmethod - def _call(cls: Type[Handler], args: argparse.Namespace, architecture: str, configuration: Configuration) -> bool: + def _call(cls: Type[Handler], args: argparse.Namespace, architecture: str) -> bool: """ additional function to wrap all calls for multiprocessing library :param args: command line args :param architecture: repository architecture - :param configuration: configuration instance :return: True on success, False otherwise """ try: + configuration = Configuration.from_path(args.configuration, architecture, not args.no_log) with Lock(args, architecture, configuration): cls.run(args, architecture, configuration) return True @@ -58,10 +58,9 @@ class Handler: :param args: command line args :return: 0 on success, 1 otherwise """ - configuration = Configuration.from_path(args.configuration, not args.no_log) with Pool(len(args.architecture)) as pool: result = pool.starmap( - cls._call, [(args, architecture, configuration) for architecture in args.architecture]) + cls._call, [(args, architecture) for architecture in set(args.architecture)]) return 0 if all(result) else 1 @classmethod diff --git a/src/ahriman/application/lock.py b/src/ahriman/application/lock.py index a17428c4..ea0dbe6d 100644 --- a/src/ahriman/application/lock.py +++ b/src/ahriman/application/lock.py @@ -54,7 +54,7 @@ class Lock: self.unsafe = args.unsafe self.root = Path(configuration.get("repository", "root")) - self.reporter = Client() if args.no_report else Client.load(architecture, configuration) + self.reporter = Client() if args.no_report else Client.load(configuration) def __enter__(self) -> Lock: """ diff --git a/src/ahriman/core/build_tools/task.py b/src/ahriman/core/build_tools/task.py index a1b35204..a64d74ff 100644 --- a/src/ahriman/core/build_tools/task.py +++ b/src/ahriman/core/build_tools/task.py @@ -41,12 +41,10 @@ class Task: _check_output = check_output - def __init__(self, package: Package, architecture: str, configuration: Configuration, - paths: RepositoryPaths) -> None: + def __init__(self, package: Package, configuration: Configuration, paths: RepositoryPaths) -> None: """ default constructor :param package: package definitions - :param architecture: repository architecture :param configuration: configuration instance :param paths: repository paths instance """ @@ -55,11 +53,10 @@ class Task: self.package = package self.paths = paths - self.archbuild_flags = configuration.wrap("build", architecture, "archbuild_flags", configuration.getlist) - self.build_command = configuration.wrap("build", architecture, "build_command", configuration.get) - self.makepkg_flags = configuration.wrap("build", architecture, "makepkg_flags", configuration.getlist) - self.makechrootpkg_flags = configuration.wrap("build", architecture, "makechrootpkg_flags", - configuration.getlist) + self.archbuild_flags = configuration.getlist("build", "archbuild_flags") + self.build_command = configuration.get("build", "build_command") + self.makepkg_flags = configuration.getlist("build", "makepkg_flags") + self.makechrootpkg_flags = configuration.getlist("build", "makechrootpkg_flags") @property def cache_path(self) -> Path: diff --git a/src/ahriman/core/configuration.py b/src/ahriman/core/configuration.py index 79036013..2fecf9b6 100644 --- a/src/ahriman/core/configuration.py +++ b/src/ahriman/core/configuration.py @@ -24,10 +24,7 @@ import logging from logging.config import fileConfig from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Type, TypeVar - - -T = TypeVar("T") +from typing import Dict, List, Optional, Type class Configuration(configparser.RawConfigParser): @@ -37,13 +34,11 @@ class Configuration(configparser.RawConfigParser): :cvar ARCHITECTURE_SPECIFIC_SECTIONS: known sections which can be architecture specific (required by dump) :cvar DEFAULT_LOG_FORMAT: default log format (in case of fallback) :cvar DEFAULT_LOG_LEVEL: default log level (in case of fallback) - :cvar STATIC_SECTIONS: known sections which are not architecture specific (required by dump) """ DEFAULT_LOG_FORMAT = "[%(levelname)s %(asctime)s] [%(filename)s:%(lineno)d] [%(funcName)s]: %(message)s" DEFAULT_LOG_LEVEL = logging.DEBUG - STATIC_SECTIONS = ["alpm", "report", "repository", "settings", "upload"] ARCHITECTURE_SPECIFIC_SECTIONS = ["build", "html", "rsync", "s3", "sign", "web"] def __init__(self) -> None: @@ -60,16 +55,24 @@ class Configuration(configparser.RawConfigParser): """ return self.getpath("settings", "include") + @property + def logging_path(self) -> Path: + """ + :return: path to logging configuration + """ + return self.getpath("settings", "logging") + @classmethod - def from_path(cls: Type[Configuration], path: Path, logfile: bool) -> Configuration: + def from_path(cls: Type[Configuration], path: Path, architecture: str, logfile: bool) -> Configuration: """ constructor with full object initialization :param path: path to root configuration file + :param architecture: repository architecture :param logfile: use log file to output messages :return: configuration instance """ config = cls() - config.load(path) + config.load(path, architecture) config.load_logging(logfile) return config @@ -83,29 +86,15 @@ class Configuration(configparser.RawConfigParser): """ return f"{section}_{architecture}" - def dump(self, architecture: str) -> Dict[str, Dict[str, str]]: + def dump(self) -> Dict[str, Dict[str, str]]: """ dump configuration to dictionary - :param architecture: repository architecture :return: configuration dump for specific architecture """ - result: Dict[str, Dict[str, str]] = {} - for section in Configuration.STATIC_SECTIONS: - if not self.has_section(section): - continue - result[section] = dict(self[section]) - for section in Configuration.ARCHITECTURE_SPECIFIC_SECTIONS: - # get global settings - settings = dict(self[section]) if self.has_section(section) else {} - # get overrides - specific = self.section_name(section, architecture) - specific_settings = dict(self[specific]) if self.has_section(specific) else {} - # merge - settings.update(specific_settings) - if settings: # append only in case if it is not empty - result[section] = settings - - return result + return { + section: dict(self[section]) + for section in self.sections() + } def getlist(self, section: str, key: str) -> List[str]: """ @@ -131,14 +120,16 @@ class Configuration(configparser.RawConfigParser): return value return self.path.parent / value - def load(self, path: Path) -> None: + def load(self, path: Path, architecture: str) -> None: """ fully load configuration :param path: path to root configuration file + :param architecture: repository architecture """ self.path = path self.read(self.path) self.load_includes() + self.merge_sections(architecture) def load_includes(self) -> None: """ @@ -146,6 +137,8 @@ class Configuration(configparser.RawConfigParser): """ try: for path in sorted(self.include.glob("*.ini")): + if path == self.logging_path: + continue # we don't want to load logging explicitly self.read(path) except (FileNotFoundError, configparser.NoOptionError): pass @@ -157,32 +150,33 @@ class Configuration(configparser.RawConfigParser): """ def file_logger() -> None: try: - path = self.getpath("settings", "logging") + path = self.logging_path fileConfig(path) except (FileNotFoundError, PermissionError): console_logger() logging.exception("could not create logfile, fallback to stderr") def console_logger() -> None: - logging.basicConfig(filename=None, format=Configuration.DEFAULT_LOG_FORMAT, - level=Configuration.DEFAULT_LOG_LEVEL) + logging.basicConfig(filename=None, format=self.DEFAULT_LOG_FORMAT, + level=self.DEFAULT_LOG_LEVEL) if logfile: file_logger() else: console_logger() - def wrap(self, section: str, architecture: str, key: str, function: Callable[..., T], **kwargs: Any) -> T: + def merge_sections(self, architecture: str) -> None: """ - wrapper to get option by either using architecture specific section or generic section - :param section: section name + merge architecture specific sections into main configuration :param architecture: repository architecture - :param key: key name - :param function: function to call, e.g. `Configuration.get` - :param kwargs: any other keywords which will be passed to function directly - :return: either value from architecture specific section or global value """ - specific_section = self.section_name(section, architecture) - if self.has_option(specific_section, key): - return function(specific_section, key, **kwargs) - return function(section, key, **kwargs) + for section in self.ARCHITECTURE_SPECIFIC_SECTIONS: + # get overrides + specific = self.section_name(section, architecture) + if not self.has_section(specific): + continue # no overrides + if not self.has_section(section): + self.add_section(section) # add section if not exists + for key, value in self[specific].items(): + self.set(section, key, value) + self.remove_section(specific) # remove overrides diff --git a/src/ahriman/core/report/html.py b/src/ahriman/core/report/html.py index cba34b77..c6d9992b 100644 --- a/src/ahriman/core/report/html.py +++ b/src/ahriman/core/report/html.py @@ -70,15 +70,15 @@ class HTML(Report): :param configuration: configuration instance """ Report.__init__(self, architecture, configuration) - self.report_path = configuration.wrap("html", architecture, "path", configuration.getpath) - self.link_path = configuration.wrap("html", architecture, "link_path", configuration.get) - self.template_path = configuration.wrap("html", architecture, "template_path", configuration.getpath) + self.report_path = configuration.getpath("html", "path") + self.link_path = configuration.get("html", "link_path") + self.template_path = configuration.getpath("html", "template_path") # base template vars - self.homepage = configuration.wrap("html", architecture, "homepage", configuration.get, fallback=None) + self.homepage = configuration.get("html", "homepage", fallback=None) self.name = configuration.get("repository", "name") - self.sign_targets, self.default_pgp_key = GPG.sign_options(architecture, configuration) + self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration) def generate(self, packages: Iterable[Package]) -> None: """ diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py index 813d8b43..5710ef55 100644 --- a/src/ahriman/core/repository/executor.py +++ b/src/ahriman/core/repository/executor.py @@ -49,7 +49,7 @@ class Executor(Cleaner): """ def build_single(package: Package) -> None: self.reporter.set_building(package.base) - task = Task(package, self.architecture, self.configuration, self.paths) + task = Task(package, self.configuration, self.paths) task.init() built = task.build() for src in built: diff --git a/src/ahriman/core/repository/properties.py b/src/ahriman/core/repository/properties.py index 8d6cecce..80c6cd55 100644 --- a/src/ahriman/core/repository/properties.py +++ b/src/ahriman/core/repository/properties.py @@ -56,4 +56,4 @@ class Properties: self.pacman = Pacman(configuration) self.sign = GPG(architecture, configuration) self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args) - self.reporter = Client.load(architecture, configuration) + self.reporter = Client.load(configuration) diff --git a/src/ahriman/core/repository/update_handler.py b/src/ahriman/core/repository/update_handler.py index 2411c8c4..1332c7df 100644 --- a/src/ahriman/core/repository/update_handler.py +++ b/src/ahriman/core/repository/update_handler.py @@ -44,7 +44,7 @@ class UpdateHandler(Cleaner): """ result: List[Package] = [] - ignore_list = self.configuration.wrap("build", self.architecture, "ignore_packages", self.configuration.getlist) + ignore_list = self.configuration.getlist("build", "ignore_packages") for local in self.packages(): if local.base in ignore_list: diff --git a/src/ahriman/core/sign/gpg.py b/src/ahriman/core/sign/gpg.py index a099f963..bbbf814a 100644 --- a/src/ahriman/core/sign/gpg.py +++ b/src/ahriman/core/sign/gpg.py @@ -49,7 +49,7 @@ class GPG: self.logger = logging.getLogger("build_details") self.architecture = architecture self.configuration = configuration - self.targets, self.default_key = self.sign_options(architecture, configuration) + self.targets, self.default_key = self.sign_options(configuration) @property def repository_sign_args(self) -> List[str]: @@ -74,18 +74,17 @@ class GPG: return ["gpg", "-u", key, "-b", str(path)] @staticmethod - def sign_options(architecture: str, configuration: Configuration) -> Tuple[Set[SignSettings], Optional[str]]: + def sign_options(configuration: Configuration) -> Tuple[Set[SignSettings], Optional[str]]: """ extract default sign options from configuration - :param architecture: repository architecture :param configuration: configuration instance :return: tuple of sign targets and default PGP key """ targets = { SignSettings.from_option(option) - for option in configuration.wrap("sign", architecture, "targets", configuration.getlist) + for option in configuration.getlist("sign", "targets") } - default_key = configuration.wrap("sign", architecture, "key", configuration.get) if targets else None + default_key = configuration.get("sign", "key") if targets else None return targets, default_key def process(self, path: Path, key: str) -> List[Path]: @@ -110,8 +109,7 @@ class GPG: """ if SignSettings.SignPackages not in self.targets: return [path] - key = self.configuration.wrap("sign", self.architecture, f"key_{base}", - self.configuration.get, fallback=self.default_key) + key = self.configuration.get("sign", f"key_{base}", fallback=self.default_key) if key is None: self.logger.error(f"no default key set, skip package {path} sign") return [path] diff --git a/src/ahriman/core/status/client.py b/src/ahriman/core/status/client.py index 781a8c8d..937d0a6f 100644 --- a/src/ahriman/core/status/client.py +++ b/src/ahriman/core/status/client.py @@ -111,15 +111,14 @@ class Client: return self.add(package, BuildStatusEnum.Unknown) @staticmethod - def load(architecture: str, configuration: Configuration) -> Client: + def load(configuration: Configuration) -> Client: """ load client from settings - :param architecture: repository architecture :param configuration: configuration instance :return: client according to current settings """ - host = configuration.wrap("web", architecture, "host", configuration.get, fallback=None) - port = configuration.wrap("web", architecture, "port", configuration.getint, fallback=None) + host = configuration.get("web", "host", fallback=None) + port = configuration.getint("web", "port", fallback=None) if host is None or port is None: return Client() diff --git a/src/ahriman/core/upload/rsync.py b/src/ahriman/core/upload/rsync.py index 428b735d..a154320d 100644 --- a/src/ahriman/core/upload/rsync.py +++ b/src/ahriman/core/upload/rsync.py @@ -40,8 +40,8 @@ class Rsync(Upload): :param configuration: configuration instance """ Upload.__init__(self, architecture, configuration) - self.command = configuration.wrap("rsync", architecture, "command", configuration.getlist) - self.remote = configuration.wrap("rsync", architecture, "remote", configuration.get) + self.command = configuration.getlist("rsync", "command") + self.remote = configuration.get("rsync", "remote") def sync(self, path: Path) -> None: """ diff --git a/src/ahriman/core/upload/s3.py b/src/ahriman/core/upload/s3.py index e6fb1c93..a48da67f 100644 --- a/src/ahriman/core/upload/s3.py +++ b/src/ahriman/core/upload/s3.py @@ -40,8 +40,8 @@ class S3(Upload): :param configuration: configuration instance """ Upload.__init__(self, architecture, configuration) - self.bucket = configuration.wrap("s3", architecture, "bucket", configuration.get) - self.command = configuration.wrap("s3", architecture, "command", configuration.getlist) + self.bucket = configuration.get("s3", "bucket") + self.command = configuration.getlist("s3", "command") def sync(self, path: Path) -> None: """ diff --git a/src/ahriman/web/web.py b/src/ahriman/web/web.py index 859de029..a18339de 100644 --- a/src/ahriman/web/web.py +++ b/src/ahriman/web/web.py @@ -58,10 +58,9 @@ def run_server(application: web.Application) -> None: """ application.logger.info("start server") - architecture: str = application["architecture"] configuration: Configuration = application["configuration"] - host = configuration.wrap("web", architecture, "host", configuration.get) - port = configuration.wrap("web", architecture, "port", configuration.getint) + host = configuration.get("web", "host") + port = configuration.getint("web", "port") web.run_app(application, host=host, port=port, handle_signals=False, access_log=logging.getLogger("http")) @@ -89,7 +88,6 @@ def setup_service(architecture: str, configuration: Configuration) -> web.Applic application.logger.info("setup configuration") application["configuration"] = configuration - application["architecture"] = architecture application.logger.info("setup watcher") application["watcher"] = Watcher(architecture, configuration) diff --git a/tests/ahriman/application/handlers/test_handler.py b/tests/ahriman/application/handlers/test_handler.py index 3a07dc38..0569490c 100644 --- a/tests/ahriman/application/handlers/test_handler.py +++ b/tests/ahriman/application/handlers/test_handler.py @@ -3,25 +3,27 @@ import argparse from pytest_mock import MockerFixture from ahriman.application.handlers import Handler -from ahriman.core.configuration import Configuration -def test_call(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None: +def test_call(args: argparse.Namespace, mocker: MockerFixture) -> None: """ must call inside lock """ + args.configuration = "" + args.no_log = False mocker.patch("ahriman.application.handlers.Handler.run") + mocker.patch("ahriman.core.configuration.Configuration.from_path") enter_mock = mocker.patch("ahriman.application.lock.Lock.__enter__") exit_mock = mocker.patch("ahriman.application.lock.Lock.__exit__") - assert Handler._call(args, "x86_64", configuration) + assert Handler._call(args, "x86_64") enter_mock.assert_called_once() exit_mock.assert_called_once() -def test_call_exception(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None: +def test_call_exception(args: argparse.Namespace, mocker: MockerFixture) -> None: """ must process exception """ mocker.patch("ahriman.application.lock.Lock.__enter__", side_effect=Exception()) - assert not Handler._call(args, "x86_64", configuration) + assert not Handler._call(args, "x86_64") diff --git a/tests/ahriman/application/handlers/test_handler_dump.py b/tests/ahriman/application/handlers/test_handler_dump.py index 6e113efc..1864ae8f 100644 --- a/tests/ahriman/application/handlers/test_handler_dump.py +++ b/tests/ahriman/application/handlers/test_handler_dump.py @@ -13,7 +13,7 @@ def test_run(args: argparse.Namespace, configuration: Configuration, mocker: Moc mocker.patch("pathlib.Path.mkdir") print_mock = mocker.patch("ahriman.application.handlers.dump.Dump._print") application_mock = mocker.patch("ahriman.core.configuration.Configuration.dump", - return_value=configuration.dump("x86_64")) + return_value=configuration.dump()) Dump.run(args, "x86_64", configuration) application_mock.assert_called_once() diff --git a/tests/ahriman/conftest.py b/tests/ahriman/conftest.py index 3a301906..d1a6da87 100644 --- a/tests/ahriman/conftest.py +++ b/tests/ahriman/conftest.py @@ -27,7 +27,7 @@ def anyvar(cls: Type[T], strict: bool = False) -> T: @pytest.fixture def configuration(resource_path_root: Path) -> Configuration: path = resource_path_root / "core" / "ahriman.ini" - return Configuration.from_path(path=path, logfile=False) + return Configuration.from_path(path=path, architecture="x86_64", logfile=False) @pytest.fixture diff --git a/tests/ahriman/core/conftest.py b/tests/ahriman/core/conftest.py index a6f6fc4b..6458232d 100644 --- a/tests/ahriman/core/conftest.py +++ b/tests/ahriman/core/conftest.py @@ -31,4 +31,4 @@ def repo(configuration: Configuration, repository_paths: RepositoryPaths) -> Rep @pytest.fixture def task_ahriman(package_ahriman: Package, configuration: Configuration, repository_paths: RepositoryPaths) -> Task: - return Task(package_ahriman, "x86_64", configuration, repository_paths) + return Task(package_ahriman, configuration, repository_paths) diff --git a/tests/ahriman/core/status/test_client.py b/tests/ahriman/core/status/test_client.py index c3c7d2e4..3a74b5c7 100644 --- a/tests/ahriman/core/status/test_client.py +++ b/tests/ahriman/core/status/test_client.py @@ -104,7 +104,7 @@ def test_load_dummy_client(configuration: Configuration) -> None: """ must load dummy client if no settings set """ - assert isinstance(Client.load("x86_64", configuration), Client) + assert isinstance(Client.load(configuration), Client) def test_load_full_client(configuration: Configuration) -> None: @@ -113,4 +113,4 @@ def test_load_full_client(configuration: Configuration) -> None: """ configuration.set("web", "host", "localhost") configuration.set("web", "port", "8080") - assert isinstance(Client.load("x86_64", configuration), WebClient) + assert isinstance(Client.load(configuration), WebClient) diff --git a/tests/ahriman/core/test_configuration.py b/tests/ahriman/core/test_configuration.py index 698701e2..8a977851 100644 --- a/tests/ahriman/core/test_configuration.py +++ b/tests/ahriman/core/test_configuration.py @@ -14,7 +14,7 @@ def test_from_path(mocker: MockerFixture) -> None: load_logging_mock = mocker.patch("ahriman.core.configuration.Configuration.load_logging") path = Path("path") - config = Configuration.from_path(path, True) + config = Configuration.from_path(path, "x86_64", True) assert config.path == path read_mock.assert_called_with(path) load_includes_mock.assert_called_once() @@ -53,7 +53,7 @@ def test_dump(configuration: Configuration) -> None: """ dump must not be empty """ - assert configuration.dump("x86_64") + assert configuration.dump() def test_dump_architecture_specific(configuration: Configuration) -> None: @@ -62,8 +62,9 @@ def test_dump_architecture_specific(configuration: Configuration) -> None: """ configuration.add_section("build_x86_64") configuration.set("build_x86_64", "archbuild_flags", "hello flag") + configuration.merge_sections("x86_64") - dump = configuration.dump("x86_64") + dump = configuration.dump() assert dump assert "build" in dump assert "build_x86_64" not in dump @@ -118,3 +119,15 @@ def test_load_logging_stderr(configuration: Configuration, mocker: MockerFixture logging_mock = mocker.patch("logging.config.fileConfig") configuration.load_logging(False) logging_mock.assert_not_called() + + +def test_merge_sections_missing(configuration: Configuration) -> None: + """ + must merge create section if not exists + """ + configuration.remove_section("build") + configuration.add_section("build_x86_64") + configuration.set("build_x86_64", "key", "value") + + configuration.merge_sections("x86_64") + assert configuration.get("build", "key") == "value"