add repo-structure subcommand

This commit also changes Tree class, replacing load method by resolve
This commit is contained in:
Evgenii Alekseev 2022-12-27 02:06:10 +02:00
parent 8c04dc4c2a
commit e0126bb811
17 changed files with 364 additions and 37 deletions

View File

@ -1,9 +1,9 @@
.TH AHRIMAN "1" "2022\-12\-11" "ahriman" "Generated Python Manual"
.TH AHRIMAN "1" "2022\-12\-27" "ahriman" "Generated Python Manual"
.SH NAME
ahriman
.SH SYNOPSIS
.B ahriman
[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--report | --no-report] [-q] [--unsafe] [-V] {aur-search,search,daemon,help,help-commands-unsafe,key-import,package-add,add,package-update,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,patch-set-add,repo-backup,repo-check,check,repo-clean,clean,repo-config,config,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-setup,init,repo-init,setup,repo-sign,sign,repo-status-update,repo-sync,sync,repo-triggers,repo-update,update,shell,user-add,user-list,user-remove,version,web} ...
[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--report | --no-report] [-q] [--unsafe] [-V] {aur-search,search,daemon,help,help-commands-unsafe,key-import,package-add,add,package-update,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,patch-set-add,repo-backup,repo-check,check,repo-clean,clean,repo-config,config,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-setup,init,repo-init,setup,repo-sign,sign,repo-status-update,repo-sync,sync,repo-tree,repo-triggers,repo-update,update,shell,user-add,user-list,user-remove,version,web} ...
.SH DESCRIPTION
ArcH linux ReposItory MANager
@ -121,6 +121,9 @@ update repository status
\fBahriman\fR \fI\,repo\-sync\/\fR
sync repository
.TP
\fBahriman\fR \fI\,repo\-tree\/\fR
dump repository tree
.TP
\fBahriman\fR \fI\,repo\-triggers\/\fR
run triggers
.TP
@ -583,6 +586,11 @@ usage: ahriman repo\-sync [\-h]
sync repository files to remote server according to current settings
.SH COMMAND \fI\,'ahriman repo\-tree'\/\fR
usage: ahriman repo\-tree [\-h]
dump repository tree based on packages dependencies
.SH COMMAND \fI\,'ahriman repo\-triggers'\/\fR
usage: ahriman repo\-triggers [\-h] [trigger ...]

View File

@ -156,6 +156,14 @@ ahriman.application.handlers.status\_update module
:no-undoc-members:
:show-inheritance:
ahriman.application.handlers.structure module
---------------------------------------------
.. automodule:: ahriman.application.handlers.structure
:members:
:no-undoc-members:
:show-inheritance:
ahriman.application.handlers.triggers module
--------------------------------------------

View File

@ -68,6 +68,14 @@ ahriman.core.formatters.string\_printer module
:no-undoc-members:
:show-inheritance:
ahriman.core.formatters.tree\_printer module
--------------------------------------------
.. automodule:: ahriman.core.formatters.tree_printer
:members:
:no-undoc-members:
:show-inheritance:
ahriman.core.formatters.update\_printer module
----------------------------------------------

View File

@ -109,6 +109,7 @@ def _parser() -> argparse.ArgumentParser:
_set_repo_sign_parser(subparsers)
_set_repo_status_update_parser(subparsers)
_set_repo_sync_parser(subparsers)
_set_repo_tree_parser(subparsers)
_set_repo_triggers_parser(subparsers)
_set_repo_update_parser(subparsers)
_set_shell_parser(subparsers)
@ -703,6 +704,23 @@ def _set_repo_sync_parser(root: SubParserAction) -> argparse.ArgumentParser:
return parser
def _set_repo_tree_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for repository tree subcommand
Args:
root(SubParserAction): subparsers for the commands
Returns:
argparse.ArgumentParser: created argument parser
"""
parser = root.add_parser("repo-tree", help="dump repository tree",
description="dump repository tree based on packages dependencies",
formatter_class=_formatter)
parser.set_defaults(handler=handlers.Structure, lock=None, report=False, quiet=True)
return parser
def _set_repo_triggers_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for repository triggers subcommand

View File

@ -145,8 +145,8 @@ class ApplicationRepository(ApplicationProperties):
process_update(packages, build_result)
# process manual packages
tree = Tree.load(updates, self.repository.paths, self.database)
for num, level in enumerate(tree.levels()):
tree = Tree.resolve(updates, self.repository.paths, self.database)
for num, level in enumerate(tree):
self.logger.info("processing level #%i %s", num, [package.base for package in level])
build_result = self.repository.process_build(level)
packages = self.repository.packages_built()
@ -181,8 +181,12 @@ class ApplicationRepository(ApplicationProperties):
local_versions = {package.base: package.version for package in self.repository.packages()}
updated_packages = [package for _, package in sorted(updates.items())]
for package in updated_packages:
UpdatePrinter(package, local_versions.get(package.base)).print(
verbose=True, log_fn=log_fn, separator=" -> ")
# reorder updates according to the dependency tree
tree = Tree.resolve(updated_packages, self.repository.paths, self.database)
for level in tree:
for package in level:
UpdatePrinter(package, local_versions.get(package.base)).print(
verbose=True, log_fn=log_fn, separator=" -> ")
return updated_packages

View File

@ -37,6 +37,7 @@ from ahriman.application.handlers.shell import Shell
from ahriman.application.handlers.sign import Sign
from ahriman.application.handlers.status import Status
from ahriman.application.handlers.status_update import StatusUpdate
from ahriman.application.handlers.structure import Structure
from ahriman.application.handlers.triggers import Triggers
from ahriman.application.handlers.unsafe_commands import UnsafeCommands
from ahriman.application.handlers.update import Update

View File

@ -0,0 +1,56 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import argparse
from typing import Type
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import TreePrinter
from ahriman.core.tree import Tree
class Structure(Handler):
"""
dump repository structure handler
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False
@classmethod
def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, configuration: Configuration, *,
report: bool, unsafe: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
unsafe(bool): if set no user check will be performed before path creation
"""
application = Application(architecture, configuration, report=report, unsafe=unsafe)
packages = application.repository.packages()
tree = Tree.resolve(packages, application.repository.paths, application.database)
for num, level in enumerate(tree):
TreePrinter(num, level).print(verbose=True, separator=" ")

View File

@ -26,6 +26,7 @@ from ahriman.core.formatters.configuration_printer import ConfigurationPrinter
from ahriman.core.formatters.package_printer import PackagePrinter
from ahriman.core.formatters.patch_printer import PatchPrinter
from ahriman.core.formatters.status_printer import StatusPrinter
from ahriman.core.formatters.tree_printer import TreePrinter
from ahriman.core.formatters.update_printer import UpdatePrinter
from ahriman.core.formatters.user_printer import UserPrinter
from ahriman.core.formatters.version_printer import VersionPrinter

View File

@ -0,0 +1,53 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from typing import Iterable, List
from ahriman.core.formatters import StringPrinter
from ahriman.models.package import Package
from ahriman.models.property import Property
class TreePrinter(StringPrinter):
"""
print content of the package tree level
Attributes:
packages(Iterable[Package]): packages which belong to this level
"""
def __init__(self, level: int, packages: Iterable[Package]) -> None:
"""
default constructor
Args:
level(int): dependencies tree level
packages(Iterable[Package]): packages which belong to this level
"""
StringPrinter.__init__(self, f"level {level}")
self.packages = packages
def properties(self) -> List[Property]:
"""
convert content into printable data
Returns:
List[Property]: list of content properties
"""
return [Property(package.base, package.version, is_required=True) for package in self.packages]

View File

@ -19,9 +19,11 @@
#
from __future__ import annotations
import itertools
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Iterable, List, Set, Type
from typing import Callable, Iterable, List, Set, Tuple, Type
from ahriman.core.build_tools.sources import Sources
from ahriman.core.database import SQLite
@ -77,6 +79,21 @@ class Leaf:
dependencies = Package.dependencies(clone_dir)
return cls(package, dependencies)
def is_dependency(self, packages: Iterable[Leaf]) -> bool:
"""
check if the package is dependency of any other package from list or not
Args:
packages(Iterable[Leaf]): list of known leaves
Returns:
bool: True in case if package is dependency of others and False otherwise
"""
for leaf in packages:
if leaf.dependencies.intersection(self.items):
return True
return False
def is_root(self, packages: Iterable[Leaf]) -> bool:
"""
check if package depends on any other package from list of not
@ -113,8 +130,8 @@ class Tree:
>>> repository = Repository.load("x86_64", configuration, database, report=True, unsafe=False)
>>> packages = repository.packages()
>>>
>>> tree = Tree.load(packages, configuration.repository_paths, database)
>>> for tree_level in tree.levels():
>>> tree = Tree.resolve(packages, configuration.repository_paths, database)
>>> for tree_level in tree:
>>> for package in tree_level:
>>> print(package.base)
>>> print()
@ -141,9 +158,10 @@ class Tree:
self.leaves = leaves
@classmethod
def load(cls: Type[Tree], packages: Iterable[Package], paths: RepositoryPaths, database: SQLite) -> Tree:
def resolve(cls: Type[Tree], packages: Iterable[Package], paths: RepositoryPaths,
database: SQLite) -> List[List[Package]]:
"""
load tree from packages
resolve dependency tree
Args:
packages(Iterable[Package]): packages list
@ -151,22 +169,45 @@ class Tree:
database(SQLite): database instance
Returns:
Tree: loaded class
List[List[Package]]: list of packages lists based on their dependencies
"""
return cls([Leaf.load(package, paths, database) for package in packages])
leaves = [Leaf.load(package, paths, database) for package in packages]
tree = cls(leaves)
return tree.levels()
def levels(self) -> List[List[Package]]:
"""
get build levels starting from the packages which do not require any other package to build
Returns:
List[List[Package]]: list of packages lists
List[List[Package]]: sorted list of packages lists based on their dependencies
"""
result: List[List[Package]] = []
# https://docs.python.org/dev/library/itertools.html#itertools-recipes
def partition(source: List[Leaf]) -> Tuple[List[Leaf], Iterable[Leaf]]:
first_iter, second_iter = itertools.tee(source)
filter_fn: Callable[[Leaf], bool] = lambda leaf: leaf.is_dependency(next_level)
# materialize first list and leave second as iterator
return list(filter(filter_fn, first_iter)), itertools.filterfalse(filter_fn, second_iter)
unsorted: List[List[Leaf]] = []
# build initial tree
unprocessed = self.leaves[:]
while unprocessed:
result.append([leaf.package for leaf in unprocessed if leaf.is_root(unprocessed)])
unsorted.append([leaf for leaf in unprocessed if leaf.is_root(unprocessed)])
unprocessed = [leaf for leaf in unprocessed if not leaf.is_root(unprocessed)]
return result
# move leaves to the end if they are not required at the next level
for current_num, current_level in enumerate(unsorted[:-1]):
next_num = current_num + 1
next_level = unsorted[next_num]
# change lists inside the collection
unsorted[current_num], to_be_moved = partition(current_level)
unsorted[next_num].extend(to_be_moved)
comparator: Callable[[Package], str] = lambda package: package.base
return [
sorted([leaf.package for leaf in level], key=comparator)
for level in unsorted if level
]

View File

@ -163,7 +163,7 @@ def test_update(application_repository: ApplicationRepository, package_ahriman:
paths = [package.filepath for package in package_ahriman.packages.values()]
tree = Tree([Leaf(package_ahriman, set())])
mocker.patch("ahriman.core.tree.Tree.load", return_value=tree)
mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels())
mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=paths)
build_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_build", return_value=result)
update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update", return_value=result)
@ -183,7 +183,7 @@ def test_update_empty(application_repository: ApplicationRepository, package_ahr
"""
tree = Tree([Leaf(package_ahriman, set())])
mocker.patch("ahriman.core.tree.Tree.load", return_value=tree)
mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels())
mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=[])
mocker.patch("ahriman.core.repository.executor.Executor.process_build")
update_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_update")
@ -197,6 +197,9 @@ def test_updates_all(application_repository: ApplicationRepository, package_ahri
"""
must get updates for all
"""
tree = Tree([Leaf(package_ahriman, set())])
mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels())
mocker.patch("ahriman.core.repository.repository.Repository.packages", return_value=[])
updates_aur_mock = mocker.patch("ahriman.core.repository.update_handler.UpdateHandler.updates_aur",
return_value=[package_ahriman])

View File

@ -0,0 +1,31 @@
import argparse
import pytest
from pytest_mock import MockerFixture
from ahriman.application.handlers import Structure
from ahriman.core.configuration import Configuration
from ahriman.core.repository import Repository
from ahriman.models.package import Package
def test_run(args: argparse.Namespace, configuration: Configuration, repository: Repository,
package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must run command
"""
mocker.patch("ahriman.core.repository.Repository.load", return_value=repository)
mocker.patch("ahriman.core.repository.Repository.packages", return_value=[package_ahriman])
application_mock = mocker.patch("ahriman.core.tree.Tree.resolve", return_value=[[package_ahriman]])
print_mock = mocker.patch("ahriman.core.formatters.Printer.print")
Structure.run(args, "x86_64", configuration, report=False, unsafe=False)
application_mock.assert_called_once_with([package_ahriman], repository.paths, pytest.helpers.anyvar(int))
print_mock.assert_called_once_with(verbose=True, separator=" ")
def test_disallow_auto_architecture_run() -> None:
"""
must not allow multi architecture run
"""
assert not Structure.ALLOW_AUTO_ARCHITECTURE_RUN

View File

@ -550,9 +550,29 @@ def test_subparsers_repo_sync_architecture(parser: argparse.ArgumentParser) -> N
"""
repo-sync command must correctly parse architecture list
"""
args = parser.parse_args(["repo-report"])
args = parser.parse_args(["repo-sync"])
assert args.architecture is None
args = parser.parse_args(["-a", "x86_64", "repo-report"])
args = parser.parse_args(["-a", "x86_64", "repo-sync"])
assert args.architecture == ["x86_64"]
def test_subparsers_repo_tree(parser: argparse.ArgumentParser) -> None:
"""
repo-tree command must imply lock, report and quiet
"""
args = parser.parse_args(["repo-tree"])
assert args.lock is None
assert not args.report
assert args.quiet
def test_subparsers_repo_tree_architecture(parser: argparse.ArgumentParser) -> None:
"""
repo-tree command must correctly parse architecture list
"""
args = parser.parse_args(["repo-tree"])
assert args.architecture is None
args = parser.parse_args(["-a", "x86_64", "repo-tree"])
assert args.architecture == ["x86_64"]

View File

@ -1,7 +1,7 @@
import pytest
from ahriman.core.formatters import AurPrinter, ConfigurationPrinter, PackagePrinter, PatchPrinter, StatusPrinter, \
StringPrinter, UpdatePrinter, UserPrinter, VersionPrinter
StringPrinter, TreePrinter, UpdatePrinter, UserPrinter, VersionPrinter
from ahriman.models.aur_package import AURPackage
from ahriman.models.build_status import BuildStatus
from ahriman.models.package import Package
@ -85,15 +85,29 @@ def string_printer() -> StringPrinter:
@pytest.fixture
def update_printer(package_ahriman: Package) -> UpdatePrinter:
def tree_printer(package_ahriman: Package) -> TreePrinter:
"""
fixture for build status printer
fixture for tree printer
Args:
package_ahriman(Package): package fixture
Returns:
UpdatePrinter: build status printer test instance
TreePrinter: tree printer test instance
"""
return TreePrinter(0, [package_ahriman])
@pytest.fixture
def update_printer(package_ahriman: Package) -> UpdatePrinter:
"""
fixture for update printer
Args:
package_ahriman(Package): package fixture
Returns:
UpdatePrinter: udpate printer test instance
"""
return UpdatePrinter(package_ahriman, None)

View File

@ -0,0 +1,15 @@
from ahriman.core.formatters import TreePrinter
def test_properties(tree_printer: TreePrinter) -> None:
"""
must return non-empty properties list
"""
assert tree_printer.properties()
def test_title(tree_printer: TreePrinter) -> None:
"""
must return non-empty title
"""
assert tree_printer.title() is not None

View File

@ -3,7 +3,7 @@ from ahriman.core.formatters import UpdatePrinter
def test_properties(update_printer: UpdatePrinter) -> None:
"""
must return empty properties list
must return non-empty properties list
"""
assert update_printer.properties()

View File

@ -5,6 +5,7 @@ from pytest_mock import MockerFixture
from ahriman.core.database import SQLite
from ahriman.core.tree import Leaf, Tree
from ahriman.models.package import Package
from ahriman.models.package_description import PackageDescription
from ahriman.models.repository_paths import RepositoryPaths
@ -53,6 +54,18 @@ def test_leaf_load(package_ahriman: Package, repository_paths: RepositoryPaths,
dependencies_mock.assert_called_once_with(pytest.helpers.anyvar(int))
def test_tree_resolve(package_ahriman: Package, package_python_schedule: Package, repository_paths: RepositoryPaths,
database: SQLite, mocker: MockerFixture) -> None:
"""
must resolve denendecnies tree
"""
mocker.patch("ahriman.core.tree.Leaf.load", side_effect=lambda package, p, d: Leaf(package, set(package.depends)))
tree = Tree.resolve([package_ahriman, package_python_schedule], repository_paths, database)
assert len(tree) == 1
assert len(tree[0]) == 2
def test_tree_levels(leaf_ahriman: Leaf, leaf_python_schedule: Leaf) -> None:
"""
must generate correct levels in the simples case
@ -60,21 +73,54 @@ def test_tree_levels(leaf_ahriman: Leaf, leaf_python_schedule: Leaf) -> None:
leaf_ahriman.dependencies = set(leaf_python_schedule.package.packages.keys())
tree = Tree([leaf_ahriman, leaf_python_schedule])
assert len(tree.levels()) == 2
first, second = tree.levels()
assert first == [leaf_python_schedule.package]
assert second == [leaf_ahriman.package]
def test_tree_load(package_ahriman: Package, package_python_schedule: Package, repository_paths: RepositoryPaths,
database: SQLite, mocker: MockerFixture) -> None:
def test_tree_levels_sorted() -> None:
"""
must package list
must reorder tree, moving packages which are not required for the next level further
"""
mocker.patch("tempfile.mkdtemp")
mocker.patch("ahriman.core.build_tools.sources.Sources.load")
mocker.patch("ahriman.models.package.Package.dependencies")
mocker.patch("shutil.rmtree")
leaf1 = Leaf(
Package(
base="package1",
version="1.0.0",
remote=None,
packages={"package1": PackageDescription()}
),
dependencies=set()
)
leaf2 = Leaf(
Package(
base="package2",
version="1.0.0",
remote=None,
packages={"package2": PackageDescription()}
),
dependencies={"package1"}
)
leaf3 = Leaf(
Package(
base="package3",
version="1.0.0",
remote=None,
packages={"package3": PackageDescription()}
),
dependencies={"package1"}
)
leaf4 = Leaf(
Package(
base="package4",
version="1.0.0",
remote=None,
packages={"package4": PackageDescription()}
),
dependencies={"package3"}
)
tree = Tree.load([package_ahriman, package_python_schedule], repository_paths, database)
assert len(tree.leaves) == 2
tree = Tree([leaf1, leaf2, leaf3, leaf4])
first, second, third = tree.levels()
assert first == [leaf1.package]
assert second == [leaf3.package]
assert third == [leaf2.package, leaf4.package]