Feature/all archs (#21)

* add init subcommand

* add also init command to repository object

* add ability to generate list of architectures

* check if architecture list is not empty
This commit is contained in:
Evgenii Alekseev 2021-05-23 16:40:40 +03:00 committed by GitHub
parent 9d37e3ae23
commit 33cfa8fef8
14 changed files with 309 additions and 38 deletions

View File

@ -41,7 +41,7 @@ def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="ahriman", description="ArcHlinux ReposItory MANager",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-a", "--architecture", help="target architectures (can be used multiple times)",
action="append", required=True)
action="append")
parser.add_argument("-c", "--configuration", help="configuration path", type=Path, default=Path("/etc/ahriman.ini"))
parser.add_argument("--force", help="force run, remove file lock", action="store_true")
parser.add_argument("-l", "--lock", help="lock file", type=Path, default=Path("/tmp/ahriman.lock"))
@ -56,6 +56,7 @@ def _parser() -> argparse.ArgumentParser:
_set_check_parser(subparsers)
_set_clean_parser(subparsers)
_set_config_parser(subparsers)
_set_init_parser(subparsers)
_set_key_import_parser(subparsers)
_set_rebuild_parser(subparsers)
_set_remove_parser(subparsers)
@ -83,7 +84,7 @@ def _set_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser.add_argument("package", help="package base/name or archive path", nargs="+")
parser.add_argument("--now", help="run update function after", action="store_true")
parser.add_argument("--without-dependencies", help="do not add dependencies", action="store_true")
parser.set_defaults(handler=handlers.Add)
parser.set_defaults(handler=handlers.Add, architecture=[])
return parser
@ -98,7 +99,7 @@ def _set_check_parser(root: SubParserAction) -> argparse.ArgumentParser:
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("package", help="filter check by package base", nargs="*")
parser.add_argument("--no-vcs", help="do not check VCS packages", action="store_true")
parser.set_defaults(handler=handlers.Update, no_aur=False, no_manual=True, dry_run=True)
parser.set_defaults(handler=handlers.Update, architecture=[], no_aur=False, no_manual=True, dry_run=True)
return parser
@ -115,7 +116,7 @@ def _set_clean_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser.add_argument("--no-chroot", help="do not clear build chroot", action="store_true")
parser.add_argument("--no-manual", help="do not clear directory with manually added packages", action="store_true")
parser.add_argument("--no-packages", help="do not clear directory with built packages", action="store_true")
parser.set_defaults(handler=handlers.Clean, no_log=True, unsafe=True)
parser.set_defaults(handler=handlers.Clean, architecture=[], no_log=True, unsafe=True)
return parser
@ -132,6 +133,19 @@ def _set_config_parser(root: SubParserAction) -> argparse.ArgumentParser:
return parser
def _set_init_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for init subcommand
:param root: subparsers for the commands
:return: created argument parser
"""
parser = root.add_parser("init", help="create repository tree",
description="create empty repository tree. Optional command for auto architecture support",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.set_defaults(handler=handlers.Init, no_report=True)
return parser
def _set_key_import_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for key import subcommand
@ -143,7 +157,7 @@ def _set_key_import_parser(root: SubParserAction) -> argparse.ArgumentParser:
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--key-server", help="key server for key import", default="keys.gnupg.net")
parser.add_argument("key", help="PGP key to import from public server")
parser.set_defaults(handler=handlers.KeyImport, lock=None, no_report=True)
parser.set_defaults(handler=handlers.KeyImport, architecture=[""], lock=None, no_report=True)
return parser
@ -156,7 +170,7 @@ def _set_rebuild_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser = root.add_parser("rebuild", help="rebuild repository", description="rebuild whole repository",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--depends-on", help="only rebuild packages that depend on specified package", action="append")
parser.set_defaults(handler=handlers.Rebuild)
parser.set_defaults(handler=handlers.Rebuild, architecture=[])
return parser
@ -169,7 +183,7 @@ def _set_remove_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser = root.add_parser("remove", help="remove package", description="remove package",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("package", help="package name or base", nargs="+")
parser.set_defaults(handler=handlers.Remove)
parser.set_defaults(handler=handlers.Remove, architecture=[])
return parser
@ -182,7 +196,7 @@ def _set_report_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser = root.add_parser("report", help="generate report", description="generate report",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("target", help="target to generate report", nargs="*")
parser.set_defaults(handler=handlers.Report)
parser.set_defaults(handler=handlers.Report, architecture=[])
return parser
@ -194,7 +208,7 @@ def _set_search_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
parser = root.add_parser("search", help="search for package", description="search for package in AUR using API")
parser.add_argument("search", help="search terms, can be specified multiple times", nargs="+")
parser.set_defaults(handler=handlers.Search, lock=None, no_log=True, no_report=True, unsafe=True)
parser.set_defaults(handler=handlers.Search, architecture=[""], lock=None, no_log=True, no_report=True, unsafe=True)
return parser
@ -230,7 +244,7 @@ def _set_sign_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser = root.add_parser("sign", help="sign packages", description="(re-)sign packages and repository database",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("package", help="sign only specified packages", nargs="*")
parser.set_defaults(handler=handlers.Sign)
parser.set_defaults(handler=handlers.Sign, architecture=[])
return parser
@ -276,7 +290,7 @@ def _set_sync_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser = root.add_parser("sync", help="sync repository", description="sync packages to remote server",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("target", help="target to sync", nargs="*")
parser.set_defaults(handler=handlers.Sync)
parser.set_defaults(handler=handlers.Sync, architecture=[])
return parser
@ -293,7 +307,7 @@ def _set_update_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser.add_argument("--no-aur", help="do not check for AUR updates. Implies --no-vcs", action="store_true")
parser.add_argument("--no-manual", help="do not include manual updates", action="store_true")
parser.add_argument("--no-vcs", help="do not check VCS packages", action="store_true")
parser.set_defaults(handler=handlers.Update)
parser.set_defaults(handler=handlers.Update, architecture=[])
return parser

View File

@ -22,6 +22,7 @@ from ahriman.application.handlers.handler import Handler
from ahriman.application.handlers.add import Add
from ahriman.application.handlers.clean import Clean
from ahriman.application.handlers.dump import Dump
from ahriman.application.handlers.init import Init
from ahriman.application.handlers.key_import import KeyImport
from ahriman.application.handlers.rebuild import Rebuild
from ahriman.application.handlers.remove import Remove

View File

@ -23,10 +23,12 @@ import argparse
import logging
from multiprocessing import Pool
from typing import Type
from typing import Set, Type
from ahriman.application.lock import Lock
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import MissingArchitecture
from ahriman.models.repository_paths import RepositoryPaths
class Handler:
@ -58,11 +60,33 @@ class Handler:
:param args: command line args
:return: 0 on success, 1 otherwise
"""
with Pool(len(args.architecture)) as pool:
architectures = cls.extract_architectures(args)
with Pool(len(architectures)) as pool:
result = pool.starmap(
cls._call, [(args, architecture) for architecture in set(args.architecture)])
cls._call, [(args, architecture) for architecture in architectures])
return 0 if all(result) else 1
@classmethod
def extract_architectures(cls: Type[Handler], args: argparse.Namespace) -> Set[str]:
"""
get known architectures
:param args: command line args
:return: list of architectures for which tree is created
"""
if args.architecture is None:
raise MissingArchitecture(args.command)
if args.architecture:
return set(args.architecture)
config = Configuration()
config.load(args.configuration)
root = config.getpath("repository", "root")
architectures = RepositoryPaths.known_architectures(root)
if not architectures:
raise MissingArchitecture(args.command)
return architectures
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, configuration: Configuration) -> None:
"""

View File

@ -0,0 +1,42 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import argparse
from typing import Type
from ahriman.application.application import Application
from ahriman.application.handlers.handler import Handler
from ahriman.core.configuration import Configuration
class Init(Handler):
"""
repository init handler
"""
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, configuration: Configuration) -> None:
"""
callback for command line
:param args: command line args
:param architecture: repository architecture
:param configuration: configuration instance
"""
Application(architecture, configuration).repository.repo.init()

View File

@ -68,6 +68,16 @@ class Repo:
cwd=self.paths.repository,
logger=self.logger)
def init(self) -> None:
"""
create empty repository database
"""
Repo._check_output(
"repo-add", *self.sign_args, str(self.repo_path),
exception=None,
cwd=self.paths.repository,
logger=self.logger)
def remove(self, package: str, filename: Path) -> None:
"""
remove package from repository

View File

@ -72,7 +72,8 @@ class Configuration(configparser.RawConfigParser):
:return: configuration instance
"""
config = cls()
config.load(path, architecture)
config.load(path)
config.merge_sections(architecture)
config.load_logging(logfile)
return config
@ -120,16 +121,14 @@ class Configuration(configparser.RawConfigParser):
return value
return self.path.parent / value
def load(self, path: Path, architecture: str) -> None:
def load(self, path: Path) -> None:
"""
fully load configuration
:param path: path to root configuration file
:param architecture: repository architecture
"""
self.path = path
self.read(self.path)
self.load_includes()
self.merge_sections(architecture)
def load_includes(self) -> None:
"""

View File

@ -83,6 +83,19 @@ class InvalidPackageInfo(Exception):
Exception.__init__(self, f"There are errors during reading package information: `{details}`")
class MissingArchitecture(Exception):
"""
exception which will be raised if architecture is required, but missing
"""
def __init__(self, command: str) -> None:
"""
default constructor
:param command: command name which throws exception
"""
Exception.__init__(self, f"Architecture required for subcommand {command}, but missing")
class ReportFailed(Exception):
"""
report generation exception

View File

@ -17,9 +17,11 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from pathlib import Path
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Set, Type
@dataclass
@ -76,6 +78,20 @@ class RepositoryPaths:
"""
return self.root / "sources" / self.architecture
@classmethod
def known_architectures(cls: Type[RepositoryPaths], root: Path) -> Set[str]:
"""
get known architectures
:param root: repository root
:return: list of architectures for which tree is created
"""
paths = cls(root, "")
return {
path.name
for path in paths.repository.iterdir()
if path.is_dir()
}
def create_tree(self) -> None:
"""
create ahriman working tree

View File

@ -6,6 +6,7 @@ from pytest_mock import MockerFixture
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import MissingArchitecture
def test_call(args: argparse.Namespace, mocker: MockerFixture) -> None:
@ -43,7 +44,52 @@ def test_execute(args: argparse.Namespace, mocker: MockerFixture) -> None:
starmap_mock.assert_called_once()
def test_packages(args: argparse.Namespace, configuration: Configuration) -> None:
def test_extract_architectures(args: argparse.Namespace, mocker: MockerFixture) -> None:
"""
must generate list of available architectures
"""
args.architecture = []
args.configuration = Path("")
mocker.patch("ahriman.core.configuration.Configuration.getpath")
known_architectures_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures")
Handler.extract_architectures(args)
known_architectures_mock.assert_called_once()
def test_extract_architectures_empty(args: argparse.Namespace, mocker: MockerFixture) -> None:
"""
must raise exception if no available architectures found
"""
args.architecture = []
args.command = "config"
args.configuration = Path("")
mocker.patch("ahriman.core.configuration.Configuration.getpath")
mocker.patch("ahriman.models.repository_paths.RepositoryPaths.known_architectures", return_value=set())
with pytest.raises(MissingArchitecture):
Handler.extract_architectures(args)
def test_extract_architectures_exception(args: argparse.Namespace) -> None:
"""
must raise exception on missing architectures
"""
args.command = "config"
args.architecture = None
with pytest.raises(MissingArchitecture):
Handler.extract_architectures(args)
def test_extract_architectures_specified(args: argparse.Namespace) -> None:
"""
must return architecture list if it has been specified
"""
architectures = args.architecture = ["i686", "x86_64"]
assert Handler.extract_architectures(args) == set(architectures)
def test_run(args: argparse.Namespace, configuration: Configuration) -> None:
"""
must raise NotImplemented for missing method
"""

View File

@ -0,0 +1,18 @@
import argparse
from pytest_mock import MockerFixture
from ahriman.application.handlers import Init
from ahriman.core.configuration import Configuration
def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
"""
must run command
"""
create_tree_mock = mocker.patch("ahriman.models.repository_paths.RepositoryPaths.create_tree")
init_mock = mocker.patch("ahriman.core.alpm.repo.Repo.init")
Init.run(args, "x86_64", configuration)
create_tree_mock.assert_called_once()
init_mock.assert_called_once()

View File

@ -29,9 +29,9 @@ def test_parser_option_lock(parser: argparse.ArgumentParser) -> None:
"""
must convert lock option to Path instance
"""
args = parser.parse_args(["-a", "x86_64", "update"])
args = parser.parse_args(["update"])
assert isinstance(args.lock, Path)
args = parser.parse_args(["-a", "x86_64", "-l", "ahriman.lock", "update"])
args = parser.parse_args(["-l", "ahriman.lock", "update"])
assert isinstance(args.lock, Path)
@ -43,11 +43,20 @@ def test_multiple_architectures(parser: argparse.ArgumentParser) -> None:
assert len(args.architecture) == 2
def test_subparsers_add(parser: argparse.ArgumentParser) -> None:
"""
add command must imply empty architectures list
"""
args = parser.parse_args(["add", "ahriman"])
assert args.architecture == []
def test_subparsers_check(parser: argparse.ArgumentParser) -> None:
"""
check command must imply no_aur, no_manual and dry_run
check command must imply empty architecture list, no-aur, no-manual and dry-run
"""
args = parser.parse_args(["-a", "x86_64", "check"])
args = parser.parse_args(["check"])
assert args.architecture == []
assert not args.no_aur
assert args.no_manual
assert args.dry_run
@ -55,38 +64,73 @@ def test_subparsers_check(parser: argparse.ArgumentParser) -> None:
def test_subparsers_clean(parser: argparse.ArgumentParser) -> None:
"""
clean command must imply unsafe and no-log
clean command must imply empty architectures list, unsafe and no-log
"""
args = parser.parse_args(["-a", "x86_64", "clean"])
args = parser.parse_args(["clean"])
assert args.architecture == []
assert args.no_log
assert args.unsafe
def test_subparsers_config(parser: argparse.ArgumentParser) -> None:
"""
config command must imply lock, no_log, no_report and unsafe
config command must imply lock, no-log, no-report and unsafe
"""
args = parser.parse_args(["-a", "x86_64", "config"])
args = parser.parse_args(["config"])
assert args.lock is None
assert args.no_log
assert args.no_report
assert args.unsafe
def test_subparsers_init(parser: argparse.ArgumentParser) -> None:
"""
init command must imply no_report
"""
args = parser.parse_args(["init"])
assert args.no_report
def test_subparsers_key_import(parser: argparse.ArgumentParser) -> None:
"""
key-import command must imply lock and no_report
key-import command must imply architecture list, lock and no-report
"""
args = parser.parse_args(["-a", "x86_64", "key-import", "key"])
args = parser.parse_args(["key-import", "key"])
assert args.architecture == [""]
assert args.lock is None
assert args.no_report
def test_subparsers_rebuild(parser: argparse.ArgumentParser) -> None:
"""
rebuild command must imply empty architectures list
"""
args = parser.parse_args(["rebuild"])
assert args.architecture == []
def test_subparsers_remove(parser: argparse.ArgumentParser) -> None:
"""
remove command must imply empty architectures list
"""
args = parser.parse_args(["remove", "ahriman"])
assert args.architecture == []
def test_subparsers_report(parser: argparse.ArgumentParser) -> None:
"""
report command must imply empty architectures list
"""
args = parser.parse_args(["report"])
assert args.architecture == []
def test_subparsers_search(parser: argparse.ArgumentParser) -> None:
"""
search command must imply lock, no_log, no_report and unsafe
search command must imply architecture list, lock, no-log, no-report and unsafe
"""
args = parser.parse_args(["-a", "x86_64", "search", "ahriman"])
args = parser.parse_args(["search", "ahriman"])
assert args.architecture == [""]
assert args.lock is None
assert args.no_log
assert args.no_report
@ -95,7 +139,7 @@ def test_subparsers_search(parser: argparse.ArgumentParser) -> None:
def test_subparsers_setup(parser: argparse.ArgumentParser) -> None:
"""
setup command must imply lock, no_log, no_report and unsafe
setup command must imply lock, no-log, no-report and unsafe
"""
args = parser.parse_args(["-a", "x86_64", "setup", "--packager", "John Doe <john@doe.com>",
"--repository", "aur-clone"])
@ -127,9 +171,17 @@ def test_subparsers_setup_option_sign_target(parser: argparse.ArgumentParser) ->
assert all(isinstance(target, SignSettings) for target in args.sign_target)
def test_subparsers_sign(parser: argparse.ArgumentParser) -> None:
"""
sign command must imply empty architectures list
"""
args = parser.parse_args(["sign"])
assert args.architecture == []
def test_subparsers_status(parser: argparse.ArgumentParser) -> None:
"""
status command must imply lock, no_log, no_report and unsafe
status command must imply lock, no-log, no-report and unsafe
"""
args = parser.parse_args(["-a", "x86_64", "status"])
assert args.lock is None
@ -140,7 +192,7 @@ def test_subparsers_status(parser: argparse.ArgumentParser) -> None:
def test_subparsers_status_update(parser: argparse.ArgumentParser) -> None:
"""
status-update command must imply lock, no_log, no_report and unsafe
status-update command must imply lock, no-log, no-report and unsafe
"""
args = parser.parse_args(["-a", "x86_64", "status-update"])
assert args.lock is None
@ -159,6 +211,22 @@ def test_subparsers_status_update_option_status(parser: argparse.ArgumentParser)
assert isinstance(args.status, BuildStatusEnum)
def test_subparsers_sync(parser: argparse.ArgumentParser) -> None:
"""
sync command must imply empty architectures list
"""
args = parser.parse_args(["sync"])
assert args.architecture == []
def test_subparsers_update(parser: argparse.ArgumentParser) -> None:
"""
update command must imply empty architectures list
"""
args = parser.parse_args(["update"])
assert args.architecture == []
def test_subparsers_web(parser: argparse.ArgumentParser) -> None:
"""
web command must imply lock and no_report

View File

@ -24,6 +24,17 @@ def test_repo_add(repo: Repo, mocker: MockerFixture) -> None:
assert check_output_mock.call_args[0][0] == "repo-add"
def test_repo_init(repo: Repo, mocker: MockerFixture) -> None:
"""
must call repo-add with empty package list on repo initializing
"""
check_output_mock = mocker.patch("ahriman.core.alpm.repo.Repo._check_output")
repo.init()
check_output_mock.assert_called_once()
assert check_output_mock.call_args[0][0] == "repo-add"
def test_repo_remove(repo: Repo, mocker: MockerFixture) -> None:
"""
must call repo-remove on package addition

View File

@ -4,6 +4,15 @@ from unittest import mock
from ahriman.models.repository_paths import RepositoryPaths
def test_known_architectures(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
"""
must list available directory paths
"""
iterdir_mock = mocker.patch("pathlib.Path.iterdir")
repository_paths.known_architectures(repository_paths.root)
iterdir_mock.assert_called_once()
def test_create_tree(repository_paths: RepositoryPaths, mocker: MockerFixture) -> None:
"""
must create whole tree
@ -11,7 +20,7 @@ def test_create_tree(repository_paths: RepositoryPaths, mocker: MockerFixture) -
paths = {
prop
for prop in dir(repository_paths)
if not prop.startswith("_") and prop not in ("architecture", "create_tree", "root")
if not prop.startswith("_") and prop not in ("architecture", "create_tree", "known_architectures", "root")
}
mkdir_mock = mocker.patch("pathlib.Path.mkdir")

View File

@ -8,7 +8,7 @@ from ahriman.models.package import Package
async def test_get(client: TestClient, package_ahriman: Package) -> None:
"""
must generate web service status correctly)
must generate web service status correctly
"""
await client.post(f"/api/v1/packages/{package_ahriman.base}",
json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()})