diff --git a/src/ahriman/application/application/repository.py b/src/ahriman/application/application/repository.py index 5725b2df..1c023173 100644 --- a/src/ahriman/application/application/repository.py +++ b/src/ahriman/application/application/repository.py @@ -105,27 +105,37 @@ class Repository(Properties): targets = target or None self.repository.process_sync(targets, built_packages) - def unknown(self) -> List[Package]: + def unknown(self) -> List[str]: """ get packages which were not found in AUR - :return: unknown package list + :return: unknown package archive list """ - def has_aur(package_base: str, aur_url: str) -> bool: - try: - _ = Package.from_aur(package_base, aur_url) - except Exception: - return False - return True - - def has_local(package_base: str) -> bool: - cache_dir = self.repository.paths.cache_for(package_base) + def has_local(probe: Package) -> bool: + cache_dir = self.repository.paths.cache_for(probe.base) return cache_dir.is_dir() and not Sources.has_remotes(cache_dir) - return [ - package - for package in self.repository.packages() - if not has_aur(package.base, package.aur_url) and not has_local(package.base) - ] + def unknown_aur(probe: Package) -> List[str]: + packages: List[str] = [] + for single in probe.packages: + try: + _ = Package.from_aur(single, probe.aur_url) + except Exception: + packages.append(single) + return packages + + def unknown_local(probe: Package) -> List[str]: + cache_dir = self.repository.paths.cache_for(probe.base) + local = Package.from_build(cache_dir, probe.aur_url) + packages = set(probe.packages.keys()).difference(local.packages.keys()) + return list(packages) + + result = [] + for package in self.repository.packages(): + if has_local(package): + result.extend(unknown_local(package)) # there is local package + else: + result.extend(unknown_aur(package)) # local package not found + return result def update(self, updates: Iterable[Package]) -> None: """ diff --git a/src/ahriman/application/formatters/status_printer.py b/src/ahriman/application/formatters/status_printer.py index 9293b0ea..c3b6e6cb 100644 --- a/src/ahriman/application/formatters/status_printer.py +++ b/src/ahriman/application/formatters/status_printer.py @@ -17,11 +17,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from typing import List, Optional +from typing import Optional from ahriman.application.formatters.printer import Printer from ahriman.models.build_status import BuildStatus -from ahriman.models.property import Property class StatusPrinter(Printer): @@ -36,13 +35,6 @@ class StatusPrinter(Printer): """ self.content = status - def properties(self) -> List[Property]: - """ - convert content into printable data - :return: list of content properties - """ - return [] - def title(self) -> Optional[str]: """ generate entry title from content diff --git a/src/ahriman/application/formatters/string_printer.py b/src/ahriman/application/formatters/string_printer.py new file mode 100644 index 00000000..3d9e2706 --- /dev/null +++ b/src/ahriman/application/formatters/string_printer.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2021 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from typing import Optional + +from ahriman.application.formatters.printer import Printer + + +class StringPrinter(Printer): + """ + print content of the random string + """ + + def __init__(self, content: str) -> None: + """ + default constructor + :param content: any content string + """ + self.content = content + + def title(self) -> Optional[str]: + """ + generate entry title from content + :return: content title if it can be generated and None otherwise + """ + return self.content diff --git a/src/ahriman/application/handlers/remove_unknown.py b/src/ahriman/application/handlers/remove_unknown.py index b7ece2d9..6af7d682 100644 --- a/src/ahriman/application/handlers/remove_unknown.py +++ b/src/ahriman/application/handlers/remove_unknown.py @@ -22,10 +22,9 @@ import argparse from typing import Type from ahriman.application.application import Application -from ahriman.application.formatters.package_printer import PackagePrinter +from ahriman.application.formatters.string_printer import StringPrinter from ahriman.application.handlers.handler import Handler from ahriman.core.configuration import Configuration -from ahriman.models.build_status import BuildStatus class RemoveUnknown(Handler): @@ -46,8 +45,8 @@ class RemoveUnknown(Handler): application = Application(architecture, configuration, no_report) unknown_packages = application.unknown() if args.dry_run: - for package in unknown_packages: - PackagePrinter(package, BuildStatus()).print(args.info) + for package in sorted(unknown_packages): + StringPrinter(package).print(args.info) return - application.remove(package.base for package in unknown_packages) + application.remove(unknown_packages) diff --git a/src/ahriman/core/exceptions.py b/src/ahriman/core/exceptions.py index ac1a38cd..ba41574f 100644 --- a/src/ahriman/core/exceptions.py +++ b/src/ahriman/core/exceptions.py @@ -153,8 +153,12 @@ class UnknownPackage(ValueError): exception for status watcher which will be thrown on unknown package """ - def __init__(self, base: str) -> None: - ValueError.__init__(self, f"Package base {base} is unknown") + def __init__(self, package_base: str) -> None: + """ + default constructor + :param package_base: package base name + """ + ValueError.__init__(self, f"Package base {package_base} is unknown") class UnsafeRun(RuntimeError): @@ -165,9 +169,9 @@ class UnsafeRun(RuntimeError): def __init__(self, current_uid: int, root_uid: int) -> None: """ default constructor + :param current_uid: current user ID + :param root_uid: ID of the owner of root directory """ - RuntimeError.__init__( - self, - f"""Current UID {current_uid} differs from root owner {root_uid}. -Note that for the most actions it is unsafe to run application as different user. -If you are 100% sure that it must be there try --unsafe option""") + RuntimeError.__init__(self, f"Current UID {current_uid} differs from root owner {root_uid}. " + f"Note that for the most actions it is unsafe to run application as different user." + f" If you are 100% sure that it must be there try --unsafe option") diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py index 96b47749..c5a74f8f 100644 --- a/src/ahriman/core/repository/executor.py +++ b/src/ahriman/core/repository/executor.py @@ -20,14 +20,13 @@ import shutil from pathlib import Path -from typing import Dict, Iterable, List, Optional +from typing import Iterable, List, Optional, Set from ahriman.core.build_tools.task import Task from ahriman.core.report.report import Report from ahriman.core.repository.cleaner import Cleaner from ahriman.core.upload.upload import Upload from ahriman.models.package import Package -from ahriman.models.package_source import PackageSource class Executor(Cleaner): @@ -35,6 +34,14 @@ class Executor(Cleaner): trait for common repository update processes """ + def load_archives(self, packages: Iterable[Path]) -> List[Package]: + """ + load packages from list of archives + :param packages: paths to package archives + :return: list of read packages + """ + raise NotImplementedError + def packages(self) -> List[Package]: """ generate list of repository packages @@ -152,23 +159,24 @@ class Executor(Cleaner): package_path = self.paths.repository / name self.repo.add(package_path) - # we are iterating over bases, not single packages - updates: Dict[str, Package] = {} - for filename in packages: - try: - local = Package.load(str(filename), PackageSource.Archive, self.pacman, self.aur_url) - updates.setdefault(local.base, local).packages.update(local.packages) - except Exception: - self.logger.exception("could not load package from %s", filename) + current_packages = self.packages() + removed_packages: List[str] = [] # list of packages which have been removed from the base + updates = self.load_archives(packages) - for local in updates.values(): + for local in updates: try: for description in local.packages.values(): update_single(description.filename, local.base) self.reporter.set_success(local) + + current_package_archives: Set[str] = next( + (set(current.packages) for current in current_packages if current.base == local.base), set()) + removed_packages.extend(current_package_archives.difference(local.packages)) except Exception: self.reporter.set_failed(local.base) self.logger.exception("could not process %s", local.base) self.clear_packages() + self.process_remove(removed_packages) + return self.repo.repo_path diff --git a/src/ahriman/core/repository/repository.py b/src/ahriman/core/repository/repository.py index c6b0d133..33c76b86 100644 --- a/src/ahriman/core/repository/repository.py +++ b/src/ahriman/core/repository/repository.py @@ -18,7 +18,7 @@ # along with this program. If not, see . # from pathlib import Path -from typing import Dict, List +from typing import Dict, Iterable, List from ahriman.core.repository.executor import Executor from ahriman.core.repository.update_handler import UpdateHandler @@ -32,20 +32,35 @@ class Repository(Executor, UpdateHandler): base repository control class """ + def load_archives(self, packages: Iterable[Path]) -> List[Package]: + """ + load packages from list of archives + :param packages: paths to package archives + :return: list of read packages + """ + result: Dict[str, Package] = {} + # we are iterating over bases, not single packages + for full_path in packages: + try: + local = Package.load(str(full_path), PackageSource.Archive, self.pacman, self.aur_url) + current = result.setdefault(local.base, local) + if current.version != local.version: + # force version to max of them + self.logger.warning("version of %s differs, found %s and %s", + current.base, current.version, local.version) + if current.is_outdated(local, self.paths, calculate_version=False): + current.version = local.version + current.packages.update(local.packages) + except Exception: + self.logger.exception("could not load package from %s", full_path) + return list(result.values()) + def packages(self) -> List[Package]: """ generate list of repository packages :return: list of packages properties """ - result: Dict[str, Package] = {} - for full_path in filter(package_like, self.paths.repository.iterdir()): - try: - local = Package.load(str(full_path), PackageSource.Archive, self.pacman, self.aur_url) - result.setdefault(local.base, local).packages.update(local.packages) - except Exception: - self.logger.exception("could not load package from %s", full_path) - continue - return list(result.values()) + return self.load_archives(filter(package_like, self.paths.repository.iterdir())) def packages_built(self) -> List[Path]: """ diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 850b9f15..9c5145ca 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -257,14 +257,15 @@ class Package: return self.version - def is_outdated(self, remote: Package, paths: RepositoryPaths) -> bool: + def is_outdated(self, remote: Package, paths: RepositoryPaths, calculate_version: bool = True) -> bool: """ check if package is out-of-dated :param remote: package properties from remote source :param paths: repository paths instance. Required for VCS packages cache + :param calculate_version: expand version to actual value (by calculating git versions) :return: True if the package is out-of-dated and False otherwise """ - remote_version = remote.actual_version(paths) # either normal version or updated VCS + remote_version = remote.actual_version(paths) if calculate_version else remote.version result: int = vercmp(self.version, remote_version) return result < 0 diff --git a/tests/ahriman/application/application/test_application_repository.py b/tests/ahriman/application/application/test_application_repository.py index f5a9640e..9ac51b41 100644 --- a/tests/ahriman/application/application/test_application_repository.py +++ b/tests/ahriman/application/application/test_application_repository.py @@ -147,6 +147,7 @@ def test_unknown_no_aur(application_repository: Repository, package_ahriman: Pac """ mocker.patch("ahriman.core.repository.repository.Repository.packages", return_value=[package_ahriman]) mocker.patch("ahriman.models.package.Package.from_aur", side_effect=Exception()) + mocker.patch("ahriman.models.package.Package.from_build", return_value=package_ahriman) mocker.patch("pathlib.Path.is_dir", return_value=True) mocker.patch("ahriman.core.build_tools.sources.Sources.has_remotes", return_value=False) @@ -163,7 +164,7 @@ def test_unknown_no_aur_no_local(application_repository: Repository, package_ahr mocker.patch("pathlib.Path.is_dir", return_value=False) packages = application_repository.unknown() - assert packages == [package_ahriman] + assert packages == list(package_ahriman.packages.keys()) def test_unknown_no_local(application_repository: Repository, package_ahriman: Package, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/application/formatters/conftest.py b/tests/ahriman/application/formatters/conftest.py index 4bdb75d9..0394e05f 100644 --- a/tests/ahriman/application/formatters/conftest.py +++ b/tests/ahriman/application/formatters/conftest.py @@ -5,6 +5,7 @@ from ahriman.application.formatters.aur_printer import AurPrinter from ahriman.application.formatters.configuration_printer import ConfigurationPrinter from ahriman.application.formatters.package_printer import PackagePrinter from ahriman.application.formatters.status_printer import StatusPrinter +from ahriman.application.formatters.string_printer import StringPrinter from ahriman.application.formatters.update_printer import UpdatePrinter from ahriman.models.build_status import BuildStatus from ahriman.models.package import Package @@ -48,6 +49,15 @@ def status_printer() -> StatusPrinter: return StatusPrinter(BuildStatus()) +@pytest.fixture +def string_printer() -> StringPrinter: + """ + fixture for any string printer + :return: any string printer test instance + """ + return StringPrinter("hello, world") + + @pytest.fixture def update_printer(package_ahriman: Package) -> UpdatePrinter: """ diff --git a/tests/ahriman/application/formatters/test_string_printer.py b/tests/ahriman/application/formatters/test_string_printer.py new file mode 100644 index 00000000..c4e81906 --- /dev/null +++ b/tests/ahriman/application/formatters/test_string_printer.py @@ -0,0 +1,15 @@ +from ahriman.application.formatters.string_printer import StringPrinter + + +def test_properties(string_printer: StringPrinter) -> None: + """ + must return empty properties list + """ + assert not string_printer.properties() + + +def test_title(string_printer: StringPrinter) -> None: + """ + must return non empty title + """ + assert string_printer.title() is not None diff --git a/tests/ahriman/core/repository/test_executor.py b/tests/ahriman/core/repository/test_executor.py index 70df7403..d110edb6 100644 --- a/tests/ahriman/core/repository/test_executor.py +++ b/tests/ahriman/core/repository/test_executor.py @@ -11,6 +11,14 @@ from ahriman.core.upload.upload import Upload from ahriman.models.package import Package +def test_load_archives(executor: Executor) -> None: + """ + must raise NotImplemented for missing load_archives method + """ + with pytest.raises(NotImplementedError): + executor.load_archives([]) + + def test_packages(executor: Executor) -> None: """ must raise NotImplemented for missing method @@ -182,11 +190,13 @@ def test_process_update(executor: Executor, package_ahriman: Package, mocker: Mo """ must run update process """ - mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman) + mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_ahriman]) + mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) move_mock = mocker.patch("shutil.move") repo_add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add") sign_package_mock = mocker.patch("ahriman.core.sign.gpg.GPG.process_sign_package", side_effect=lambda fn, _: [fn]) status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_success") + remove_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_remove") # must return complete assert executor.process_update([package.filepath for package in package_ahriman.packages.values()]) @@ -201,6 +211,8 @@ def test_process_update(executor: Executor, package_ahriman: Package, mocker: Mo # must clear directory from ahriman.core.repository.cleaner import Cleaner Cleaner.clear_packages.assert_called_once() + # clear removed packages + remove_mock.assert_called_once_with([]) def test_process_update_group(executor: Executor, package_python_schedule: Package, @@ -209,9 +221,11 @@ def test_process_update_group(executor: Executor, package_python_schedule: Packa must group single packages under one base """ mocker.patch("shutil.move") - mocker.patch("ahriman.models.package.Package.load", return_value=package_python_schedule) + mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_python_schedule]) + mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_python_schedule]) repo_add_mock = mocker.patch("ahriman.core.alpm.repo.Repo.add") status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_success") + remove_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_remove") executor.process_update([package.filepath for package in package_python_schedule.packages.values()]) repo_add_mock.assert_has_calls([ @@ -219,6 +233,7 @@ def test_process_update_group(executor: Executor, package_python_schedule: Packa for package in package_python_schedule.packages.values() ], any_order=True) status_client_mock.assert_called_once_with(package_python_schedule) + remove_mock.assert_called_once_with([]) def test_process_empty_filename(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: @@ -226,7 +241,8 @@ def test_process_empty_filename(executor: Executor, package_ahriman: Package, mo must skip update for package which does not have path """ package_ahriman.packages[package_ahriman.base].filename = None - mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman) + mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_ahriman]) + mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) executor.process_update([package.filepath for package in package_ahriman.packages.values()]) @@ -235,18 +251,27 @@ def test_process_update_failed(executor: Executor, package_ahriman: Package, moc must process update for failed package """ mocker.patch("shutil.move", side_effect=Exception()) - mocker.patch("ahriman.models.package.Package.load", return_value=package_ahriman) + mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[package_ahriman]) + mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman]) status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_failed") executor.process_update([package.filepath for package in package_ahriman.packages.values()]) status_client_mock.assert_called_once() -def test_process_update_failed_on_load(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None: +def test_process_update_removed_package(executor: Executor, package_python_schedule: Package, + mocker: MockerFixture) -> None: """ - must process update even with failed package load + must remove packages which have been removed from the new base """ - mocker.patch("shutil.move") - mocker.patch("ahriman.models.package.Package.load", side_effect=Exception()) + without_python2 = Package.from_json(package_python_schedule.view()) + del without_python2.packages["python2-schedule"] - assert executor.process_update([package.filepath for package in package_ahriman.packages.values()]) + mocker.patch("shutil.move") + mocker.patch("ahriman.core.alpm.repo.Repo.add") + mocker.patch("ahriman.core.repository.executor.Executor.load_archives", return_value=[without_python2]) + mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_python_schedule]) + remove_mock = mocker.patch("ahriman.core.repository.executor.Executor.process_remove") + + executor.process_update([package.filepath for package in without_python2.packages.values()]) + remove_mock.assert_called_once_with(["python2-schedule"]) diff --git a/tests/ahriman/core/repository/test_repository.py b/tests/ahriman/core/repository/test_repository.py index 19120272..e43e668c 100644 --- a/tests/ahriman/core/repository/test_repository.py +++ b/tests/ahriman/core/repository/test_repository.py @@ -5,8 +5,8 @@ from ahriman.core.repository import Repository from ahriman.models.package import Package -def test_packages(package_ahriman: Package, package_python_schedule: Package, - repository: Repository, mocker: MockerFixture) -> None: +def test_load_archives(package_ahriman: Package, package_python_schedule: Package, + repository: Repository, mocker: MockerFixture) -> None: """ must return all packages grouped by package base """ @@ -17,12 +17,9 @@ def test_packages(package_ahriman: Package, package_python_schedule: Package, packages={package: props}) for package, props in package_python_schedule.packages.items() ] + [package_ahriman] - - mocker.patch("pathlib.Path.iterdir", - return_value=[Path("a.pkg.tar.xz"), Path("b.pkg.tar.xz"), Path("c.pkg.tar.xz")]) mocker.patch("ahriman.models.package.Package.load", side_effect=single_packages) - packages = repository.packages() + packages = repository.load_archives([Path("a.pkg.tar.xz"), Path("b.pkg.tar.xz"), Path("c.pkg.tar.xz")]) assert len(packages) == 2 assert {package.base for package in packages} == {package_ahriman.base, package_python_schedule.base} @@ -33,21 +30,48 @@ def test_packages(package_ahriman: Package, package_python_schedule: Package, assert set(archives) == expected -def test_packages_failed(repository: Repository, mocker: MockerFixture) -> None: +def test_load_archives_failed(repository: Repository, mocker: MockerFixture) -> None: """ must skip packages which cannot be loaded """ - mocker.patch("pathlib.Path.iterdir", return_value=[Path("a.pkg.tar.xz")]) mocker.patch("ahriman.models.package.Package.load", side_effect=Exception()) - assert not repository.packages() + assert not repository.load_archives([Path("a.pkg.tar.xz")]) -def test_packages_not_package(repository: Repository, mocker: MockerFixture) -> None: +def test_load_archives_not_package(repository: Repository) -> None: """ must skip not packages from iteration """ - mocker.patch("pathlib.Path.iterdir", return_value=[Path("a.tar.xz")]) - assert not repository.packages() + assert not repository.load_archives([Path("a.tar.xz")]) + + +def test_load_archives_different_version(repository: Repository, package_python_schedule: Package, + mocker: MockerFixture) -> None: + """ + must load packages with different versions choosing maximal + """ + single_packages = [ + Package(base=package_python_schedule.base, + version=package_python_schedule.version, + aur_url=package_python_schedule.aur_url, + packages={package: props}) + for package, props in package_python_schedule.packages.items() + ] + single_packages[0].version = "0.0.1-1" + mocker.patch("ahriman.models.package.Package.load", side_effect=single_packages) + + packages = repository.load_archives([Path("a.pkg.tar.xz"), Path("b.pkg.tar.xz")]) + assert len(packages) == 1 + assert packages[0].version == package_python_schedule.version + + +def test_packages(repository: Repository, mocker: MockerFixture) -> None: + """ + must return repository packages + """ + load_mock = mocker.patch("ahriman.core.repository.repository.Repository.load_archives") + repository.packages() + load_mock.assert_called_once() # it uses filter object so we cannot verity argument list =/ def test_packages_built(repository: Repository, mocker: MockerFixture) -> None: