From f163dd4be4ce59d36906b2020ce069b90baf2da7 Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Sun, 23 May 2021 16:13:02 +0300 Subject: [PATCH] add ability to generate list of architectures --- src/ahriman/application/ahriman.py | 24 ++--- src/ahriman/application/handlers/handler.py | 27 +++++- src/ahriman/core/configuration.py | 7 +- src/ahriman/core/exceptions.py | 13 +++ src/ahriman/models/repository_paths.py | 18 +++- .../application/handlers/test_handler.py | 34 ++++++- tests/ahriman/application/test_ahriman.py | 92 +++++++++++++++---- tests/ahriman/models/test_repository_paths.py | 11 ++- 8 files changed, 188 insertions(+), 38 deletions(-) diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 5edcd4ad..02e061e2 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -41,7 +41,7 @@ def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="ahriman", description="ArcHlinux ReposItory MANager", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-a", "--architecture", help="target architectures (can be used multiple times)", - action="append", required=True) + action="append") parser.add_argument("-c", "--configuration", help="configuration path", type=Path, default=Path("/etc/ahriman.ini")) parser.add_argument("--force", help="force run, remove file lock", action="store_true") parser.add_argument("-l", "--lock", help="lock file", type=Path, default=Path("/tmp/ahriman.lock")) @@ -84,7 +84,7 @@ def _set_add_parser(root: SubParserAction) -> argparse.ArgumentParser: parser.add_argument("package", help="package base/name or archive path", nargs="+") parser.add_argument("--now", help="run update function after", action="store_true") parser.add_argument("--without-dependencies", help="do not add dependencies", action="store_true") - parser.set_defaults(handler=handlers.Add) + parser.set_defaults(handler=handlers.Add, architecture=[]) return parser @@ -99,7 +99,7 @@ def _set_check_parser(root: SubParserAction) -> argparse.ArgumentParser: formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("package", help="filter check by package base", nargs="*") parser.add_argument("--no-vcs", help="do not check VCS packages", action="store_true") - parser.set_defaults(handler=handlers.Update, no_aur=False, no_manual=True, dry_run=True) + parser.set_defaults(handler=handlers.Update, architecture=[], no_aur=False, no_manual=True, dry_run=True) return parser @@ -116,7 +116,7 @@ def _set_clean_parser(root: SubParserAction) -> argparse.ArgumentParser: parser.add_argument("--no-chroot", help="do not clear build chroot", action="store_true") parser.add_argument("--no-manual", help="do not clear directory with manually added packages", action="store_true") parser.add_argument("--no-packages", help="do not clear directory with built packages", action="store_true") - parser.set_defaults(handler=handlers.Clean, no_log=True, unsafe=True) + parser.set_defaults(handler=handlers.Clean, architecture=[], no_log=True, unsafe=True) return parser @@ -157,7 +157,7 @@ def _set_key_import_parser(root: SubParserAction) -> argparse.ArgumentParser: formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--key-server", help="key server for key import", default="keys.gnupg.net") parser.add_argument("key", help="PGP key to import from public server") - parser.set_defaults(handler=handlers.KeyImport, lock=None, no_report=True) + parser.set_defaults(handler=handlers.KeyImport, architecture=[""], lock=None, no_report=True) return parser @@ -170,7 +170,7 @@ def _set_rebuild_parser(root: SubParserAction) -> argparse.ArgumentParser: parser = root.add_parser("rebuild", help="rebuild repository", description="rebuild whole repository", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--depends-on", help="only rebuild packages that depend on specified package", action="append") - parser.set_defaults(handler=handlers.Rebuild) + parser.set_defaults(handler=handlers.Rebuild, architecture=[]) return parser @@ -183,7 +183,7 @@ def _set_remove_parser(root: SubParserAction) -> argparse.ArgumentParser: parser = root.add_parser("remove", help="remove package", description="remove package", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("package", help="package name or base", nargs="+") - parser.set_defaults(handler=handlers.Remove) + parser.set_defaults(handler=handlers.Remove, architecture=[]) return parser @@ -196,7 +196,7 @@ def _set_report_parser(root: SubParserAction) -> argparse.ArgumentParser: parser = root.add_parser("report", help="generate report", description="generate report", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("target", help="target to generate report", nargs="*") - parser.set_defaults(handler=handlers.Report) + parser.set_defaults(handler=handlers.Report, architecture=[]) return parser @@ -208,7 +208,7 @@ def _set_search_parser(root: SubParserAction) -> argparse.ArgumentParser: """ parser = root.add_parser("search", help="search for package", description="search for package in AUR using API") parser.add_argument("search", help="search terms, can be specified multiple times", nargs="+") - parser.set_defaults(handler=handlers.Search, lock=None, no_log=True, no_report=True, unsafe=True) + parser.set_defaults(handler=handlers.Search, architecture=[""], lock=None, no_log=True, no_report=True, unsafe=True) return parser @@ -244,7 +244,7 @@ def _set_sign_parser(root: SubParserAction) -> argparse.ArgumentParser: parser = root.add_parser("sign", help="sign packages", description="(re-)sign packages and repository database", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("package", help="sign only specified packages", nargs="*") - parser.set_defaults(handler=handlers.Sign) + parser.set_defaults(handler=handlers.Sign, architecture=[]) return parser @@ -290,7 +290,7 @@ def _set_sync_parser(root: SubParserAction) -> argparse.ArgumentParser: parser = root.add_parser("sync", help="sync repository", description="sync packages to remote server", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("target", help="target to sync", nargs="*") - parser.set_defaults(handler=handlers.Sync) + parser.set_defaults(handler=handlers.Sync, architecture=[]) return parser @@ -307,7 +307,7 @@ def _set_update_parser(root: SubParserAction) -> argparse.ArgumentParser: parser.add_argument("--no-aur", help="do not check for AUR updates. Implies --no-vcs", action="store_true") parser.add_argument("--no-manual", help="do not include manual updates", action="store_true") parser.add_argument("--no-vcs", help="do not check VCS packages", action="store_true") - parser.set_defaults(handler=handlers.Update) + parser.set_defaults(handler=handlers.Update, architecture=[]) return parser diff --git a/src/ahriman/application/handlers/handler.py b/src/ahriman/application/handlers/handler.py index 333ee13f..6886b08f 100644 --- a/src/ahriman/application/handlers/handler.py +++ b/src/ahriman/application/handlers/handler.py @@ -23,10 +23,12 @@ import argparse import logging from multiprocessing import Pool -from typing import Type +from typing import List, Type from ahriman.application.lock import Lock from ahriman.core.configuration import Configuration +from ahriman.core.exceptions import MissingArchitecture +from ahriman.models.repository_paths import RepositoryPaths class Handler: @@ -58,11 +60,30 @@ class Handler: :param args: command line args :return: 0 on success, 1 otherwise """ - with Pool(len(args.architecture)) as pool: + architectures = cls.extract_architectures(args) + with Pool(len(architectures)) as pool: result = pool.starmap( - cls._call, [(args, architecture) for architecture in set(args.architecture)]) + cls._call, [(args, architecture) for architecture in set(architectures)]) return 0 if all(result) else 1 + @classmethod + def extract_architectures(cls: Type[Handler], args: argparse.Namespace) -> List[str]: + """ + get known architectures + :param args: command line args + :return: list of architectures for which tree is created + """ + if args.architecture is None: + raise MissingArchitecture(args.command) + if args.architecture: + architectures: List[str] = args.architecture # avoid mypy warning + return architectures + + config = Configuration() + config.load(args.configuration) + root = config.getpath("repository", "root") + return RepositoryPaths.known_architectures(root) + @classmethod def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, configuration: Configuration) -> None: """ diff --git a/src/ahriman/core/configuration.py b/src/ahriman/core/configuration.py index d619127a..b55d9e8d 100644 --- a/src/ahriman/core/configuration.py +++ b/src/ahriman/core/configuration.py @@ -72,7 +72,8 @@ class Configuration(configparser.RawConfigParser): :return: configuration instance """ config = cls() - config.load(path, architecture) + config.load(path) + config.merge_sections(architecture) config.load_logging(logfile) return config @@ -120,16 +121,14 @@ class Configuration(configparser.RawConfigParser): return value return self.path.parent / value - def load(self, path: Path, architecture: str) -> None: + def load(self, path: Path) -> None: """ fully load configuration :param path: path to root configuration file - :param architecture: repository architecture """ self.path = path self.read(self.path) self.load_includes() - self.merge_sections(architecture) def load_includes(self) -> None: """ diff --git a/src/ahriman/core/exceptions.py b/src/ahriman/core/exceptions.py index 2050b75d..0be1fadd 100644 --- a/src/ahriman/core/exceptions.py +++ b/src/ahriman/core/exceptions.py @@ -83,6 +83,19 @@ class InvalidPackageInfo(Exception): Exception.__init__(self, f"There are errors during reading package information: `{details}`") +class MissingArchitecture(Exception): + """ + exception which will be raised if architecture is required, but missing + """ + + def __init__(self, command: str) -> None: + """ + default constructor + :param command: command name which throws exception + """ + Exception.__init__(self, f"Architecture required for subcommand {command}, but missing") + + class ReportFailed(Exception): """ report generation exception diff --git a/src/ahriman/models/repository_paths.py b/src/ahriman/models/repository_paths.py index dcfe8cf0..9eaa398a 100644 --- a/src/ahriman/models/repository_paths.py +++ b/src/ahriman/models/repository_paths.py @@ -17,9 +17,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from pathlib import Path +from __future__ import annotations from dataclasses import dataclass +from pathlib import Path +from typing import List, Type @dataclass @@ -76,6 +78,20 @@ class RepositoryPaths: """ return self.root / "sources" / self.architecture + @classmethod + def known_architectures(cls: Type[RepositoryPaths], root: Path) -> List[str]: + """ + get known architectures + :param root: repository root + :return: list of architectures for which tree is created + """ + paths = cls(root, "") + return [ + path.name + for path in paths.repository.iterdir() + if path.is_dir() + ] + def create_tree(self) -> None: """ create ahriman working tree diff --git a/tests/ahriman/application/handlers/test_handler.py b/tests/ahriman/application/handlers/test_handler.py index c92a17f2..78d91d25 100644 --- a/tests/ahriman/application/handlers/test_handler.py +++ b/tests/ahriman/application/handlers/test_handler.py @@ -6,6 +6,7 @@ from pytest_mock import MockerFixture from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration +from ahriman.core.exceptions import MissingArchitecture def test_call(args: argparse.Namespace, mocker: MockerFixture) -> None: @@ -43,7 +44,38 @@ def test_execute(args: argparse.Namespace, mocker: MockerFixture) -> None: starmap_mock.assert_called_once() -def test_packages(args: argparse.Namespace, configuration: Configuration) -> None: +def test_extract_architectures(args: argparse.Namespace, mocker: MockerFixture) -> None: + """ + must generate list of available architectures + """ + args.architecture = [] + args.configuration = Path("") + mocker.patch("ahriman.core.configuration.Configuration.getpath") + known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures") + + Handler.extract_architectures(args) + known_architectures_mock.assert_called_once() + + +def test_extract_architectures_specified(args: argparse.Namespace) -> None: + """ + must return architecture list if it has been specified + """ + architectures = args.architecture = ["i686", "x86_64"] + assert Handler.extract_architectures(args) == architectures + + +def test_extract_architectures_exception(args: argparse.Namespace) -> None: + """ + must raise exception on missing architectures + """ + args.command = "config" + args.architecture = None + with pytest.raises(MissingArchitecture): + Handler.extract_architectures(args) + + +def test_run(args: argparse.Namespace, configuration: Configuration) -> None: """ must raise NotImplemented for missing method """ diff --git a/tests/ahriman/application/test_ahriman.py b/tests/ahriman/application/test_ahriman.py index a92ffe18..3e51622c 100644 --- a/tests/ahriman/application/test_ahriman.py +++ b/tests/ahriman/application/test_ahriman.py @@ -29,9 +29,9 @@ def test_parser_option_lock(parser: argparse.ArgumentParser) -> None: """ must convert lock option to Path instance """ - args = parser.parse_args(["-a", "x86_64", "update"]) + args = parser.parse_args(["update"]) assert isinstance(args.lock, Path) - args = parser.parse_args(["-a", "x86_64", "-l", "ahriman.lock", "update"]) + args = parser.parse_args(["-l", "ahriman.lock", "update"]) assert isinstance(args.lock, Path) @@ -43,11 +43,20 @@ def test_multiple_architectures(parser: argparse.ArgumentParser) -> None: assert len(args.architecture) == 2 +def test_subparsers_add(parser: argparse.ArgumentParser) -> None: + """ + add command must imply empty architectures list + """ + args = parser.parse_args(["add", "ahriman"]) + assert args.architecture == [] + + def test_subparsers_check(parser: argparse.ArgumentParser) -> None: """ - check command must imply no_aur, no_manual and dry_run + check command must imply empty architecture list, no-aur, no-manual and dry-run """ - args = parser.parse_args(["-a", "x86_64", "check"]) + args = parser.parse_args(["check"]) + assert args.architecture == [] assert not args.no_aur assert args.no_manual assert args.dry_run @@ -55,18 +64,19 @@ def test_subparsers_check(parser: argparse.ArgumentParser) -> None: def test_subparsers_clean(parser: argparse.ArgumentParser) -> None: """ - clean command must imply unsafe and no-log + clean command must imply empty architectures list, unsafe and no-log """ - args = parser.parse_args(["-a", "x86_64", "clean"]) + args = parser.parse_args(["clean"]) + assert args.architecture == [] assert args.no_log assert args.unsafe def test_subparsers_config(parser: argparse.ArgumentParser) -> None: """ - config command must imply lock, no_log, no_report and unsafe + config command must imply lock, no-log, no-report and unsafe """ - args = parser.parse_args(["-a", "x86_64", "config"]) + args = parser.parse_args(["config"]) assert args.lock is None assert args.no_log assert args.no_report @@ -77,24 +87,50 @@ def test_subparsers_init(parser: argparse.ArgumentParser) -> None: """ init command must imply no_report """ - args = parser.parse_args(["-a", "x86_64", "init"]) + args = parser.parse_args(["init"]) assert args.no_report def test_subparsers_key_import(parser: argparse.ArgumentParser) -> None: """ - key-import command must imply lock and no_report + key-import command must imply architecture list, lock and no-report """ - args = parser.parse_args(["-a", "x86_64", "key-import", "key"]) + args = parser.parse_args(["key-import", "key"]) + assert args.architecture == [""] assert args.lock is None assert args.no_report +def test_subparsers_rebuild(parser: argparse.ArgumentParser) -> None: + """ + rebuild command must imply empty architectures list + """ + args = parser.parse_args(["rebuild"]) + assert args.architecture == [] + + +def test_subparsers_remove(parser: argparse.ArgumentParser) -> None: + """ + remove command must imply empty architectures list + """ + args = parser.parse_args(["remove", "ahriman"]) + assert args.architecture == [] + + +def test_subparsers_report(parser: argparse.ArgumentParser) -> None: + """ + report command must imply empty architectures list + """ + args = parser.parse_args(["report"]) + assert args.architecture == [] + + def test_subparsers_search(parser: argparse.ArgumentParser) -> None: """ - search command must imply lock, no_log, no_report and unsafe + search command must imply architecture list, lock, no-log, no-report and unsafe """ - args = parser.parse_args(["-a", "x86_64", "search", "ahriman"]) + args = parser.parse_args(["search", "ahriman"]) + assert args.architecture == [""] assert args.lock is None assert args.no_log assert args.no_report @@ -103,7 +139,7 @@ def test_subparsers_search(parser: argparse.ArgumentParser) -> None: def test_subparsers_setup(parser: argparse.ArgumentParser) -> None: """ - setup command must imply lock, no_log, no_report and unsafe + setup command must imply lock, no-log, no-report and unsafe """ args = parser.parse_args(["-a", "x86_64", "setup", "--packager", "John Doe ", "--repository", "aur-clone"]) @@ -135,9 +171,17 @@ def test_subparsers_setup_option_sign_target(parser: argparse.ArgumentParser) -> assert all(isinstance(target, SignSettings) for target in args.sign_target) +def test_subparsers_sign(parser: argparse.ArgumentParser) -> None: + """ + sign command must imply empty architectures list + """ + args = parser.parse_args(["sign"]) + assert args.architecture == [] + + def test_subparsers_status(parser: argparse.ArgumentParser) -> None: """ - status command must imply lock, no_log, no_report and unsafe + status command must imply lock, no-log, no-report and unsafe """ args = parser.parse_args(["-a", "x86_64", "status"]) assert args.lock is None @@ -148,7 +192,7 @@ def test_subparsers_status(parser: argparse.ArgumentParser) -> None: def test_subparsers_status_update(parser: argparse.ArgumentParser) -> None: """ - status-update command must imply lock, no_log, no_report and unsafe + status-update command must imply lock, no-log, no-report and unsafe """ args = parser.parse_args(["-a", "x86_64", "status-update"]) assert args.lock is None @@ -167,6 +211,22 @@ def test_subparsers_status_update_option_status(parser: argparse.ArgumentParser) assert isinstance(args.status, BuildStatusEnum) +def test_subparsers_sync(parser: argparse.ArgumentParser) -> None: + """ + sync command must imply empty architectures list + """ + args = parser.parse_args(["sync"]) + assert args.architecture == [] + + +def test_subparsers_update(parser: argparse.ArgumentParser) -> None: + """ + update command must imply empty architectures list + """ + args = parser.parse_args(["update"]) + assert args.architecture == [] + + def test_subparsers_web(parser: argparse.ArgumentParser) -> None: """ web command must imply lock and no_report diff --git a/tests/ahriman/models/test_repository_paths.py b/tests/ahriman/models/test_repository_paths.py index 4f9226fd..cdac58c7 100644 --- a/tests/ahriman/models/test_repository_paths.py +++ b/tests/ahriman/models/test_repository_paths.py @@ -4,6 +4,15 @@ from unittest import mock from ahriman.models.repository_paths import RepositoryPaths +def test_known_architectures(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: + """ + must list available directory paths + """ + iterdir_mock = mocker.patch("pathlib.Path.iterdir") + repository_paths.known_architectures(repository_paths.root) + iterdir_mock.assert_called_once() + + def test_create_tree(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: """ must create whole tree @@ -11,7 +20,7 @@ def test_create_tree(repository_paths: RepositoryPaths, mocker: MockerFixture) - paths = { prop for prop in dir(repository_paths) - if not prop.startswith("_") and prop not in ("architecture", "create_tree", "root") + if not prop.startswith("_") and prop not in ("architecture", "create_tree", "known_architectures", "root") } mkdir_mock = mocker.patch("pathlib.Path.mkdir")