From 683abca9e533ae9c76efdca0d250b1ac97ce4cf5 Mon Sep 17 00:00:00 2001 From: Evgenii Alekseev Date: Sun, 6 Feb 2022 03:44:57 +0300 Subject: [PATCH] use own aur wrapper (#49) --- package/archlinux/PKGBUILD | 2 +- setup.py | 2 +- .../application/formatters/aur_printer.py | 13 +- src/ahriman/application/handlers/search.py | 15 +- src/ahriman/core/alpm/aur.py | 152 +++++++++++++++ src/ahriman/core/util.py | 21 +-- src/ahriman/models/aur_package.py | 109 +++++++++++ src/ahriman/models/package.py | 4 +- src/ahriman/web/middlewares/auth_handler.py | 4 +- src/ahriman/web/views/service/search.py | 9 +- .../application/formatters/conftest.py | 4 +- .../handlers/test_handler_search.py | 24 +-- tests/ahriman/conftest.py | 78 +++++--- tests/ahriman/core/alpm/conftest.py | 12 ++ tests/ahriman/core/alpm/test_aur.py | 173 ++++++++++++++++++ tests/ahriman/core/test_util.py | 40 +--- tests/ahriman/core/upload/test_http_upload.py | 2 +- tests/ahriman/core/upload/test_s3.py | 2 +- tests/ahriman/models/test_aur_package.py | 47 +++++ tests/ahriman/models/test_package.py | 11 +- .../service/test_views_service_search.py | 10 +- .../web/views/user/test_views_user_login.py | 5 +- tests/testresources/models/aur_error | 7 + .../testresources/models/package_ahriman_aur | 54 ++++++ .../models/package_ahriman_srcinfo | 5 +- 25 files changed, 673 insertions(+), 132 deletions(-) create mode 100644 src/ahriman/core/alpm/aur.py create mode 100644 src/ahriman/models/aur_package.py create mode 100644 tests/ahriman/core/alpm/conftest.py create mode 100644 tests/ahriman/core/alpm/test_aur.py create mode 100644 tests/ahriman/models/test_aur_package.py create mode 100644 tests/testresources/models/aur_error create mode 100644 tests/testresources/models/package_ahriman_aur diff --git a/package/archlinux/PKGBUILD b/package/archlinux/PKGBUILD index fbebac2a..ac5ac857 100644 --- a/package/archlinux/PKGBUILD +++ b/package/archlinux/PKGBUILD @@ -7,7 +7,7 @@ pkgdesc="ArcH Linux ReposItory MANager" arch=('any') url="https://github.com/arcan1s/ahriman" license=('GPL3') -depends=('devtools' 'git' 'pyalpm' 'python-aur' 'python-passlib' 'python-srcinfo') +depends=('devtools' 'git' 'pyalpm' 'python-inflection' 'python-passlib' 'python-srcinfo') makedepends=('python-pip') optdepends=('breezy: -bzr packages support' 'darcs: -darcs packages support' diff --git a/setup.py b/setup.py index 3c81a41b..137cc605 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ setup( dependency_links=[ ], install_requires=[ - "aur", + "inflection", "passlib", "pyalpm", "requests", diff --git a/src/ahriman/application/formatters/aur_printer.py b/src/ahriman/application/formatters/aur_printer.py index dddc077c..0ec2ddc6 100644 --- a/src/ahriman/application/formatters/aur_printer.py +++ b/src/ahriman/application/formatters/aur_printer.py @@ -17,12 +17,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -import aur # type: ignore - from typing import List, Optional from ahriman.application.formatters.printer import Printer from ahriman.core.util import pretty_datetime +from ahriman.models.aur_package import AURPackage from ahriman.models.property import Property @@ -31,7 +30,7 @@ class AurPrinter(Printer): print content of the AUR package """ - def __init__(self, package: aur.Package) -> None: + def __init__(self, package: AURPackage) -> None: """ default constructor :param package: AUR package description @@ -46,12 +45,12 @@ class AurPrinter(Printer): return [ Property("Package base", self.content.package_base), Property("Description", self.content.description, is_required=True), - Property("Upstream URL", self.content.url), - Property("Licenses", self.content.license), # it should be actually a list - Property("Maintainer", self.content.maintainer or ""), # I think it is optional + Property("Upstream URL", self.content.url or ""), + Property("Licenses", ",".join(self.content.license)), + Property("Maintainer", self.content.maintainer or ""), Property("First submitted", pretty_datetime(self.content.first_submitted)), Property("Last updated", pretty_datetime(self.content.last_modified)), - # more fields coming https://github.com/cdown/aur/pull/29 + Property("Keywords", ",".join(self.content.keywords)), ] def title(self) -> Optional[str]: diff --git a/src/ahriman/application/handlers/search.py b/src/ahriman/application/handlers/search.py index 96b1c3f4..ee99d2d1 100644 --- a/src/ahriman/application/handlers/search.py +++ b/src/ahriman/application/handlers/search.py @@ -18,24 +18,27 @@ # along with this program. If not, see . # import argparse -import aur # type: ignore +from dataclasses import fields from typing import Callable, Iterable, List, Tuple, Type from ahriman.application.formatters.aur_printer import AurPrinter from ahriman.application.handlers.handler import Handler +from ahriman.core.alpm.aur import AUR from ahriman.core.configuration import Configuration from ahriman.core.exceptions import InvalidOption -from ahriman.core.util import aur_search +from ahriman.models.aur_package import AURPackage class Search(Handler): """ packages search handler + :cvar SORT_FIELDS: allowed fields to sort the package list """ ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture" - SORT_FIELDS = set(aur.Package._fields) # later we will have to remove some fields from here (lists) + # later we will have to remove some fields from here (lists) + SORT_FIELDS = {pair.name for pair in fields(AURPackage)} @classmethod def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, @@ -47,12 +50,12 @@ class Search(Handler): :param configuration: configuration instance :param no_report: force disable reporting """ - packages_list = aur_search(*args.search) + packages_list = AUR.multisearch(*args.search) for package in Search.sort(packages_list, args.sort_by): AurPrinter(package).print(args.info) @staticmethod - def sort(packages: Iterable[aur.Package], sort_by: str) -> List[aur.Package]: + def sort(packages: Iterable[AURPackage], sort_by: str) -> List[AURPackage]: """ sort package list by specified field :param packages: packages list to sort @@ -63,6 +66,6 @@ class Search(Handler): raise InvalidOption(sort_by) # always sort by package name at the last # well technically it is not a string, but we can deal with it - comparator: Callable[[aur.Package], Tuple[str, str]] =\ + comparator: Callable[[AURPackage], Tuple[str, str]] =\ lambda package: (getattr(package, sort_by), package.name) return sorted(packages, key=comparator) diff --git a/src/ahriman/core/alpm/aur.py b/src/ahriman/core/alpm/aur.py new file mode 100644 index 00000000..8d2eefec --- /dev/null +++ b/src/ahriman/core/alpm/aur.py @@ -0,0 +1,152 @@ +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from __future__ import annotations + +import logging +import requests + +from typing import Any, Dict, List, Optional, Type + +from ahriman.core.exceptions import InvalidPackageInfo +from ahriman.core.util import exception_response_text +from ahriman.models.aur_package import AURPackage + + +class AUR: + """ + AUR RPC wrapper + :cvar DEFAULT_RPC_URL: default AUR RPC url + :cvar DEFAULT_RPC_VERSION: default AUR RPC version + :ivar logger: class logger + :ivar rpc_url: AUR RPC url + :ivar rpc_version: AUR RPC version + """ + + DEFAULT_RPC_URL = "https://aur.archlinux.org/rpc" + DEFAULT_RPC_VERSION = "5" + + def __init__(self, rpc_url: Optional[str] = None, rpc_version: Optional[str] = None) -> None: + """ + default constructor + :param rpc_url: AUR RPC url + :param rpc_version: AUR RPC version + """ + self.rpc_url = rpc_url or self.DEFAULT_RPC_URL + self.rpc_version = rpc_version or self.DEFAULT_RPC_VERSION + self.logger = logging.getLogger("build_details") + + @classmethod + def info(cls: Type[AUR], package_name: str) -> AURPackage: + """ + get package info by its name + :param package_name: package name to search + :return: package which match the package name + """ + return cls().package_info(package_name) + + @classmethod + def multisearch(cls: Type[AUR], *keywords: str) -> List[AURPackage]: + """ + search in AUR by using API with multiple words. This method is required in order to handle + https://bugs.archlinux.org/task/49133. In addition short words will be dropped + :param keywords: search terms, e.g. "ahriman", "is", "cool" + :return: list of packages each of them matches all search terms + """ + instance = cls() + packages: Dict[str, AURPackage] = {} + for term in filter(lambda word: len(word) > 3, keywords): + portion = instance.search(term) + packages = { + package.package_base: package + for package in portion + if package.package_base in packages or not packages + } + return list(packages.values()) + + @classmethod + def search(cls: Type[AUR], *keywords: str) -> List[AURPackage]: + """ + search package in AUR web + :param keywords: keywords to search + :return: list of packages which match the criteria + """ + return cls().package_search(*keywords) + + @staticmethod + def parse_response(response: Dict[str, Any]) -> List[AURPackage]: + """ + parse RPC response to package list + :param response: RPC response json + :return: list of parsed packages + """ + response_type = response["type"] + if response_type == "error": + error_details = response.get("error", "Unknown API error") + raise InvalidPackageInfo(error_details) + return [AURPackage.from_json(package) for package in response["results"]] + + def make_request(self, request_type: str, *args: str, **kwargs: str) -> List[AURPackage]: + """ + perform request to AUR RPC + :param request_type: AUR request type, e.g. search, info + :param args: list of arguments to be passed as args query parameter + :param kwargs: list of additional named parameters like by + :return: response parsed to package list + """ + query: Dict[str, Any] = { + "type": request_type, + "v": self.rpc_version + } + + arg_query = "arg[]" if len(args) > 1 else "arg" + query[arg_query] = list(args) + + for key, value in kwargs.items(): + query[key] = value + + try: + response = requests.get(self.rpc_url, params=query) + response.raise_for_status() + return self.parse_response(response.json()) + except requests.HTTPError as e: + self.logger.exception( + "could not perform request by using type %s: %s", + request_type, + exception_response_text(e)) + raise + except Exception: + self.logger.exception("could not perform request by using type %s", request_type) + raise + + def package_info(self, package_name: str) -> AURPackage: + """ + get package info by its name + :param package_name: package name to search + :return: package which match the package name + """ + packages = self.make_request("info", package_name) + return next(package for package in packages if package.name == package_name) + + def package_search(self, *keywords: str, by: str = "name-desc") -> List[AURPackage]: + """ + search package in AUR web + :param keywords: keywords to search + :param by: search by the field + :return: list of packages which match the criteria + """ + return self.make_request("search", *keywords, by=by) diff --git a/src/ahriman/core/util.py b/src/ahriman/core/util.py index 0c5f928b..bed5aa65 100644 --- a/src/ahriman/core/util.py +++ b/src/ahriman/core/util.py @@ -17,7 +17,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -import aur # type: ignore import datetime import os import subprocess @@ -25,29 +24,11 @@ import requests from logging import Logger from pathlib import Path -from typing import Any, Dict, Generator, Iterable, List, Optional, Union +from typing import Any, Dict, Generator, Iterable, Optional, Union from ahriman.core.exceptions import InvalidOption, UnsafeRun -def aur_search(*terms: str) -> List[aur.Package]: - """ - search in AUR by using API with multiple words. This method is required in order to handle - https://bugs.archlinux.org/task/49133. In addition short words will be dropped - :param terms: search terms, e.g. "ahriman", "is", "cool" - :return: list of packages each of them matches all search terms - """ - packages: Dict[str, aur.Package] = {} - for term in filter(lambda word: len(word) > 3, terms): - portion = aur.search(term) - packages = { - package.package_base: package - for package in portion - if package.package_base in packages or not packages - } - return list(packages.values()) - - def check_output(*args: str, exception: Optional[Exception], cwd: Optional[Path] = None, input_data: Optional[str] = None, logger: Optional[Logger] = None) -> str: """ diff --git a/src/ahriman/models/aur_package.py b/src/ahriman/models/aur_package.py new file mode 100644 index 00000000..ee678fa3 --- /dev/null +++ b/src/ahriman/models/aur_package.py @@ -0,0 +1,109 @@ +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from __future__ import annotations + +import datetime +import inflection + +from dataclasses import dataclass, field, fields +from typing import Any, Callable, Dict, List, Optional, Type + +from ahriman.core.util import filter_json + + +@dataclass +class AURPackage: + """ + AUR package descriptor + :ivar id: package ID + :ivar name: package name + :ivar package_base_id: package base ID + :ivar version: package base version + :ivar description: package base description + :ivar url: package upstream URL + :ivar num_votes: number of votes for the package + :ivar polularity: package popularity + :ivar out_of_date: package out of date timestamp if any + :ivar maintainer: package maintainer + :ivar first_submitted: timestamp of the first package submission + :ivar last_modified: timestamp of the last package submission + :ivar url_path: AUR package path + :ivar depends: list of package dependencies + :ivar make_depends: list of package make dependencies + :ivar opt_depends: list of package optional dependencies + :ivar conflicts: conflicts list for the package + :ivar provides: list of packages which this package provides + :ivar license: list of package licenses + :ivar keywords: list of package keywords + """ + + id: int + name: str + package_base_id: int + package_base: str + version: str + description: str + num_votes: int + popularity: float + first_submitted: datetime.datetime + last_modified: datetime.datetime + url_path: str + url: Optional[str] = None + out_of_date: Optional[datetime.datetime] = None + maintainer: Optional[str] = None + depends: List[str] = field(default_factory=list) + make_depends: List[str] = field(default_factory=list) + opt_depends: List[str] = field(default_factory=list) + conflicts: List[str] = field(default_factory=list) + provides: List[str] = field(default_factory=list) + license: List[str] = field(default_factory=list) + keywords: List[str] = field(default_factory=list) + + @classmethod + def from_json(cls: Type[AURPackage], dump: Dict[str, Any]) -> AURPackage: + """ + construct package descriptor from RPC properties + :param dump: json dump body + :return: AUR package descriptor + """ + # filter to only known fields + known_fields = [pair.name for pair in fields(cls)] + properties = cls.convert(dump) + return cls(**filter_json(properties, known_fields)) + + @staticmethod + def convert(descriptor: Dict[str, Any]) -> Dict[str, Any]: + """ + covert AUR RPC key names to package keys + :param descriptor: RPC package descriptor + :return: package descriptor with names converted to snake case + """ + identity_mapper: Callable[[Any], Any] = lambda value: value + value_mapper: Dict[str, Callable[[Any], Any]] = { + "out_of_date": lambda value: datetime.datetime.utcfromtimestamp(value) if value is not None else None, + "first_submitted": datetime.datetime.utcfromtimestamp, + "last_modified": datetime.datetime.utcfromtimestamp, + } + + result: Dict[str, Any] = {} + for api_key, api_value in descriptor.items(): + property_key = inflection.underscore(api_key) + mapper = value_mapper.get(property_key, identity_mapper) + result[property_key] = mapper(api_value) + + return result diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 6a38c9be..63db4b66 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -19,7 +19,6 @@ # from __future__ import annotations -import aur # type: ignore import copy import logging @@ -29,6 +28,7 @@ from pyalpm import vercmp # type: ignore from srcinfo.parse import parse_srcinfo # type: ignore from typing import Any, Dict, Iterable, List, Optional, Set, Type +from ahriman.core.alpm.aur import AUR from ahriman.core.alpm.pacman import Pacman from ahriman.core.exceptions import InvalidPackageInfo from ahriman.core.util import check_output @@ -129,7 +129,7 @@ class Package: :param aur_url: AUR root url :return: package properties """ - package = aur.info(name) + package = AUR.info(name) return cls(package.package_base, package.version, aur_url, {package.name: PackageDescription()}) @classmethod diff --git a/src/ahriman/web/middlewares/auth_handler.py b/src/ahriman/web/middlewares/auth_handler.py index f2cbdc2c..481d0647 100644 --- a/src/ahriman/web/middlewares/auth_handler.py +++ b/src/ahriman/web/middlewares/auth_handler.py @@ -25,8 +25,8 @@ from aiohttp import web from aiohttp.web import middleware, Request from aiohttp.web_response import StreamResponse from aiohttp.web_urldispatcher import StaticResource -from aiohttp_session import setup as setup_session # type: ignore -from aiohttp_session.cookie_storage import EncryptedCookieStorage # type: ignore +from aiohttp_session import setup as setup_session +from aiohttp_session.cookie_storage import EncryptedCookieStorage from cryptography import fernet from typing import Optional diff --git a/src/ahriman/web/views/service/search.py b/src/ahriman/web/views/service/search.py index 1df3b9b8..f1fb7fe7 100644 --- a/src/ahriman/web/views/service/search.py +++ b/src/ahriman/web/views/service/search.py @@ -17,12 +17,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -import aur # type: ignore - from aiohttp.web import HTTPNotFound, Response, json_response from typing import Callable, List -from ahriman.core.util import aur_search +from ahriman.core.alpm.aur import AUR +from ahriman.models.aur_package import AURPackage from ahriman.models.user_access import UserAccess from ahriman.web.views.base import BaseView @@ -45,11 +44,11 @@ class SearchView(BaseView): :return: 200 with found package bases and descriptions sorted by base """ search: List[str] = self.request.query.getall("for", default=[]) - packages = aur_search(*search) + packages = AUR.multisearch(*search) if not packages: raise HTTPNotFound(reason=f"No packages found for terms: {search}") - comparator: Callable[[aur.Package], str] = lambda item: str(item.package_base) + comparator: Callable[[AURPackage], str] = lambda item: str(item.package_base) response = [ { "package": package.package_base, diff --git a/tests/ahriman/application/formatters/conftest.py b/tests/ahriman/application/formatters/conftest.py index 0394e05f..6a38c710 100644 --- a/tests/ahriman/application/formatters/conftest.py +++ b/tests/ahriman/application/formatters/conftest.py @@ -1,4 +1,3 @@ -import aur import pytest from ahriman.application.formatters.aur_printer import AurPrinter @@ -7,12 +6,13 @@ from ahriman.application.formatters.package_printer import PackagePrinter from ahriman.application.formatters.status_printer import StatusPrinter from ahriman.application.formatters.string_printer import StringPrinter from ahriman.application.formatters.update_printer import UpdatePrinter +from ahriman.models.aur_package import AURPackage from ahriman.models.build_status import BuildStatus from ahriman.models.package import Package @pytest.fixture -def aur_package_ahriman_printer(aur_package_ahriman: aur.Package) -> AurPrinter: +def aur_package_ahriman_printer(aur_package_ahriman: AURPackage) -> AurPrinter: """ fixture for AUR package printer :param aur_package_ahriman: AUR package fixture diff --git a/tests/ahriman/application/handlers/test_handler_search.py b/tests/ahriman/application/handlers/test_handler_search.py index 8300e0b9..b8e5bb81 100644 --- a/tests/ahriman/application/handlers/test_handler_search.py +++ b/tests/ahriman/application/handlers/test_handler_search.py @@ -1,5 +1,5 @@ import argparse -import aur +import dataclasses import pytest from pytest_mock import MockerFixture @@ -7,6 +7,7 @@ from pytest_mock import MockerFixture from ahriman.application.handlers import Search from ahriman.core.configuration import Configuration from ahriman.core.exceptions import InvalidOption +from ahriman.models.aur_package import AURPackage def _default_args(args: argparse.Namespace) -> argparse.Namespace: @@ -21,13 +22,13 @@ def _default_args(args: argparse.Namespace) -> argparse.Namespace: return args -def test_run(args: argparse.Namespace, configuration: Configuration, aur_package_ahriman: aur.Package, +def test_run(args: argparse.Namespace, configuration: Configuration, aur_package_ahriman: AURPackage, mocker: MockerFixture) -> None: """ must run command """ args = _default_args(args) - search_mock = mocker.patch("ahriman.application.handlers.search.aur_search", return_value=[aur_package_ahriman]) + search_mock = mocker.patch("ahriman.core.alpm.aur.AUR.multisearch", return_value=[aur_package_ahriman]) print_mock = mocker.patch("ahriman.application.formatters.printer.Printer.print") Search.run(args, "x86_64", configuration, True) @@ -35,38 +36,38 @@ def test_run(args: argparse.Namespace, configuration: Configuration, aur_package print_mock.assert_called_once() -def test_run_sort(args: argparse.Namespace, configuration: Configuration, aur_package_ahriman: aur.Package, +def test_run_sort(args: argparse.Namespace, configuration: Configuration, aur_package_ahriman: AURPackage, mocker: MockerFixture) -> None: """ must run command with sorting """ args = _default_args(args) - mocker.patch("ahriman.application.handlers.search.aur_search", return_value=[aur_package_ahriman]) + mocker.patch("ahriman.core.alpm.aur.AUR.multisearch", return_value=[aur_package_ahriman]) sort_mock = mocker.patch("ahriman.application.handlers.search.Search.sort") Search.run(args, "x86_64", configuration, True) sort_mock.assert_called_once_with([aur_package_ahriman], "name") -def test_run_sort_by(args: argparse.Namespace, configuration: Configuration, aur_package_ahriman: aur.Package, +def test_run_sort_by(args: argparse.Namespace, configuration: Configuration, aur_package_ahriman: AURPackage, mocker: MockerFixture) -> None: """ must run command with sorting by specified field """ args = _default_args(args) args.sort_by = "field" - mocker.patch("ahriman.application.handlers.search.aur_search", return_value=[aur_package_ahriman]) + mocker.patch("ahriman.core.alpm.aur.AUR.multisearch", return_value=[aur_package_ahriman]) sort_mock = mocker.patch("ahriman.application.handlers.search.Search.sort") Search.run(args, "x86_64", configuration, True) sort_mock.assert_called_once_with([aur_package_ahriman], "field") -def test_sort(aur_package_ahriman: aur.Package) -> None: +def test_sort(aur_package_ahriman: AURPackage) -> None: """ must sort package list """ - another = aur_package_ahriman._replace(name="1", package_base="base") + another = dataclasses.replace(aur_package_ahriman, name="1", package_base="base") # sort by name assert Search.sort([aur_package_ahriman, another], "name") == [another, aur_package_ahriman] # sort by another field @@ -75,7 +76,7 @@ def test_sort(aur_package_ahriman: aur.Package) -> None: assert Search.sort([aur_package_ahriman, another], "version") == [another, aur_package_ahriman] -def test_sort_exception(aur_package_ahriman: aur.Package) -> None: +def test_sort_exception(aur_package_ahriman: AURPackage) -> None: """ must raise an exception on unknown sorting field """ @@ -94,4 +95,5 @@ def test_sort_fields() -> None: """ must store valid field list which are allowed to be used for sorting """ - assert all(field in aur.Package._fields for field in Search.SORT_FIELDS) + expected = {pair.name for pair in dataclasses.fields(AURPackage)} + assert all(field in expected for field in Search.SORT_FIELDS) diff --git a/tests/ahriman/conftest.py b/tests/ahriman/conftest.py index 23fb8e69..96afe586 100644 --- a/tests/ahriman/conftest.py +++ b/tests/ahriman/conftest.py @@ -1,4 +1,4 @@ -import aur +import datetime import pytest from pathlib import Path @@ -10,6 +10,7 @@ from ahriman.core.auth.auth import Auth from ahriman.core.configuration import Configuration from ahriman.core.spawn import Spawn from ahriman.core.status.watcher import Watcher +from ahriman.models.aur_package import AURPackage from ahriman.models.package import Package from ahriman.models.package_description import PackageDescription from ahriman.models.repository_paths import RepositoryPaths @@ -48,28 +49,56 @@ def anyvar(cls: Type[T], strict: bool = False) -> T: # generic fixtures @pytest.fixture -def aur_package_ahriman(package_ahriman: Package) -> aur.Package: +def aur_package_ahriman() -> AURPackage: """ fixture for AUR package - :param package_ahriman: package fixture :return: AUR package test instance """ - return aur.Package( - num_votes=None, - description=package_ahriman.packages[package_ahriman.base].description, - url_path=package_ahriman.web_url, - last_modified=None, - name=package_ahriman.base, + return AURPackage( + id=1009791, + name="ahriman", + package_base_id=165427, + package_base="ahriman", + version="1.7.0-1", + description="ArcH Linux ReposItory MANager", + num_votes=0, + popularity=0, + first_submitted=datetime.datetime(2021, 4, 9, 22, 44, 45), + last_modified=datetime.datetime(2021, 12, 25, 23, 11, 11), + url_path="/cgit/aur.git/snapshot/ahriman.tar.gz", + url="https://github.com/arcan1s/ahriman", out_of_date=None, - id=None, - first_submitted=None, - maintainer=None, - version=package_ahriman.version, - license=package_ahriman.packages[package_ahriman.base].licenses, - url=None, - package_base=package_ahriman.base, - package_base_id=None, - category_id=None) + maintainer="arcanis", + depends=[ + "devtools", + "git", + "pyalpm", + "python-aur", + "python-passlib", + "python-srcinfo", + ], + make_depends=["python-pip"], + opt_depends=[ + "breezy", + "darcs", + "mercurial", + "python-aioauth-client", + "python-aiohttp", + "python-aiohttp-debugtoolbar", + "python-aiohttp-jinja2", + "python-aiohttp-security", + "python-aiohttp-session", + "python-boto3", + "python-cryptography", + "python-jinja", + "rsync", + "subversion", + ], + conflicts=[], + provides=[], + license=["GPL3"], + keywords=[], + ) @pytest.fixture @@ -103,7 +132,7 @@ def package_ahriman(package_description_ahriman: PackageDescription) -> Package: packages = {"ahriman": package_description_ahriman} return Package( base="ahriman", - version="0.12.1-1", + version="1.7.0-1", aur_url="https://aur.archlinux.org", packages=packages) @@ -139,9 +168,16 @@ def package_description_ahriman() -> PackageDescription: architecture="x86_64", archive_size=4200, build_date=42, - depends=["devtools", "git", "pyalpm", "python-aur", "python-srcinfo"], + depends=[ + "devtools", + "git", + "pyalpm", + "python-aur", + "python-passlib", + "python-srcinfo", + ], description="ArcH Linux ReposItory MANager", - filename="ahriman-0.12.1-1-any.pkg.tar.zst", + filename="ahriman-1.7.0-1-any.pkg.tar.zst", groups=[], installed_size=4200000, licenses=["GPL3"], diff --git a/tests/ahriman/core/alpm/conftest.py b/tests/ahriman/core/alpm/conftest.py new file mode 100644 index 00000000..6f0a021a --- /dev/null +++ b/tests/ahriman/core/alpm/conftest.py @@ -0,0 +1,12 @@ +import pytest + +from ahriman.core.alpm.aur import AUR + + +@pytest.fixture +def aur() -> AUR: + """ + aur helper fixture + :return: aur helper instance + """ + return AUR() diff --git a/tests/ahriman/core/alpm/test_aur.py b/tests/ahriman/core/alpm/test_aur.py new file mode 100644 index 00000000..bf9f9541 --- /dev/null +++ b/tests/ahriman/core/alpm/test_aur.py @@ -0,0 +1,173 @@ +import json +import pytest +import requests + +from pathlib import Path +from pytest_mock import MockerFixture +from unittest import mock +from unittest.mock import MagicMock + +from ahriman.core.alpm.aur import AUR +from ahriman.core.exceptions import InvalidPackageInfo +from ahriman.models.aur_package import AURPackage + + +def _get_response(resource_path_root: Path) -> str: + """ + load response from resource file + :param resource_path_root: path to resource root + :return: response text + """ + return (resource_path_root / "models" / "package_ahriman_aur").read_text() + + +def test_info(mocker: MockerFixture) -> None: + """ + must call info method + """ + info_mock = mocker.patch("ahriman.core.alpm.aur.AUR.package_info") + AUR.info("ahriman") + info_mock.assert_called_once_with("ahriman") + + +def test_multisearch(aur_package_ahriman: AURPackage, mocker: MockerFixture) -> None: + """ + must search in AUR with multiple words + """ + terms = ["ahriman", "is", "cool"] + search_mock = mocker.patch("ahriman.core.alpm.aur.AUR.search", return_value=[aur_package_ahriman]) + + assert AUR.multisearch(*terms) == [aur_package_ahriman] + search_mock.assert_has_calls([mock.call("ahriman"), mock.call("cool")]) + + +def test_multisearch_empty(mocker: MockerFixture) -> None: + """ + must return empty list if no long terms supplied + """ + terms = ["it", "is"] + search_mock = mocker.patch("ahriman.core.alpm.aur.AUR.search") + + assert AUR.multisearch(*terms) == [] + search_mock.assert_not_called() + + +def test_multisearch_single(aur_package_ahriman: AURPackage, mocker: MockerFixture) -> None: + """ + must search in AUR with one word + """ + search_mock = mocker.patch("ahriman.core.alpm.aur.AUR.search", return_value=[aur_package_ahriman]) + assert AUR.multisearch("ahriman") == [aur_package_ahriman] + search_mock.assert_called_once_with("ahriman") + + +def test_search(mocker: MockerFixture) -> None: + """ + must call search method + """ + search_mock = mocker.patch("ahriman.core.alpm.aur.AUR.package_search") + AUR.search("ahriman") + search_mock.assert_called_once_with("ahriman") + + +def test_parse_response(aur_package_ahriman: AURPackage, resource_path_root: Path) -> None: + """ + must parse success response + """ + response = _get_response(resource_path_root) + assert AUR.parse_response(json.loads(response)) == [aur_package_ahriman] + + +def test_parse_response_error(resource_path_root: Path) -> None: + """ + must raise exception on invalid response + """ + response = (resource_path_root / "models" / "aur_error").read_text() + with pytest.raises(InvalidPackageInfo, match="Incorrect request type specified."): + AUR.parse_response(json.loads(response)) + + +def test_parse_response_unknown_error(resource_path_root: Path) -> None: + """ + must raise exception on invalid response with empty error message + """ + with pytest.raises(InvalidPackageInfo, match="Unknown API error"): + AUR.parse_response({"type": "error"}) + + +def test_make_request(aur: AUR, aur_package_ahriman: AURPackage, + mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must perform request to AUR + """ + response_mock = MagicMock() + response_mock.json.return_value = json.loads(_get_response(resource_path_root)) + request_mock = mocker.patch("requests.get", return_value=response_mock) + + assert aur.make_request("info", "ahriman") == [aur_package_ahriman] + request_mock.assert_called_once_with( + "https://aur.archlinux.org/rpc", params={"v": "5", "type": "info", "arg": ["ahriman"]}) + + +def test_make_request_multi_arg(aur: AUR, aur_package_ahriman: AURPackage, + mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must perform request to AUR with multiple args + """ + response_mock = MagicMock() + response_mock.json.return_value = json.loads(_get_response(resource_path_root)) + request_mock = mocker.patch("requests.get", return_value=response_mock) + + assert aur.make_request("search", "ahriman", "is", "cool") == [aur_package_ahriman] + request_mock.assert_called_once_with( + "https://aur.archlinux.org/rpc", params={"v": "5", "type": "search", "arg[]": ["ahriman", "is", "cool"]}) + + +def test_make_request_with_kwargs(aur: AUR, aur_package_ahriman: AURPackage, + mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must perform request to AUR with named parameters + """ + response_mock = MagicMock() + response_mock.json.return_value = json.loads(_get_response(resource_path_root)) + request_mock = mocker.patch("requests.get", return_value=response_mock) + + assert aur.make_request("search", "ahriman", by="name") == [aur_package_ahriman] + request_mock.assert_called_once_with( + "https://aur.archlinux.org/rpc", params={"v": "5", "type": "search", "arg": ["ahriman"], "by": "name"}) + + +def test_make_request_failed(aur: AUR, mocker: MockerFixture) -> None: + """ + must reraise generic exception + """ + mocker.patch("requests.get", side_effect=Exception()) + with pytest.raises(Exception): + aur.make_request("info", "ahriman") + + +def test_make_request_failed_http_error(aur: AUR, mocker: MockerFixture) -> None: + """ + must reraise http exception + """ + mocker.patch("requests.get", side_effect=requests.exceptions.HTTPError()) + with pytest.raises(requests.exceptions.HTTPError): + aur.make_request("info", "ahriman") + + +def test_package_info(aur: AUR, aur_package_ahriman: AURPackage, mocker: MockerFixture) -> None: + """ + must make request for info + """ + request_mock = mocker.patch("ahriman.core.alpm.aur.AUR.make_request", return_value=[aur_package_ahriman]) + assert aur.package_info(aur_package_ahriman.name) == aur_package_ahriman + request_mock.assert_called_once_with("info", aur_package_ahriman.name) + + +def test_package_search(aur: AUR, aur_package_ahriman: AURPackage, mocker: MockerFixture) -> None: + """ + must make request for search + """ + request_mock = mocker.patch("ahriman.core.alpm.aur.AUR.make_request", return_value=[aur_package_ahriman]) + assert aur.package_search(aur_package_ahriman.name, by="name") == [aur_package_ahriman] + request_mock.assert_called_once_with("search", aur_package_ahriman.name, by="name") diff --git a/tests/ahriman/core/test_util.py b/tests/ahriman/core/test_util.py index 98b48d2f..b8df4a12 100644 --- a/tests/ahriman/core/test_util.py +++ b/tests/ahriman/core/test_util.py @@ -1,4 +1,3 @@ -import aur import datetime import logging import pytest @@ -6,45 +5,12 @@ import subprocess from pathlib import Path from pytest_mock import MockerFixture -from unittest import mock from ahriman.core.exceptions import InvalidOption, UnsafeRun -from ahriman.core.util import aur_search, check_output, check_user, filter_json, package_like, pretty_datetime, \ - pretty_size, walk +from ahriman.core.util import check_output, check_user, filter_json, package_like, pretty_datetime, pretty_size, walk from ahriman.models.package import Package -def test_aur_search(aur_package_ahriman: aur.Package, mocker: MockerFixture) -> None: - """ - must search in AUR with multiple words - """ - terms = ["ahriman", "is", "cool"] - search_mock = mocker.patch("aur.search", return_value=[aur_package_ahriman]) - - assert aur_search(*terms) == [aur_package_ahriman] - search_mock.assert_has_calls([mock.call("ahriman"), mock.call("cool")]) - - -def test_aur_search_empty(mocker: MockerFixture) -> None: - """ - must return empty list if no long terms supplied - """ - terms = ["it", "is"] - search_mock = mocker.patch("aur.search") - - assert aur_search(*terms) == [] - search_mock.assert_not_called() - - -def test_aur_search_single(aur_package_ahriman: aur.Package, mocker: MockerFixture) -> None: - """ - must search in AUR with one word - """ - search_mock = mocker.patch("aur.search", return_value=[aur_package_ahriman]) - assert aur_search("ahriman") == [aur_package_ahriman] - search_mock.assert_called_once_with("ahriman") - - def test_check_output(mocker: MockerFixture) -> None: """ must run command and log result @@ -127,7 +93,7 @@ def test_filter_json(package_ahriman: Package) -> None: def test_filter_json_empty_value(package_ahriman: Package) -> None: """ - must return empty values from object + must filter empty values from object """ probe = package_ahriman.view() probe["base"] = None @@ -238,8 +204,10 @@ def test_walk(resource_path_root: Path) -> None: expected = sorted([ resource_path_root / "core/ahriman.ini", resource_path_root / "core/logging.ini", + resource_path_root / "models/aur_error", resource_path_root / "models/big_file_checksum", resource_path_root / "models/empty_file_checksum", + resource_path_root / "models/package_ahriman_aur", resource_path_root / "models/package_ahriman_srcinfo", resource_path_root / "models/package_tpacpi-bat-git_srcinfo", resource_path_root / "models/package_yay_srcinfo", diff --git a/tests/ahriman/core/upload/test_http_upload.py b/tests/ahriman/core/upload/test_http_upload.py index 97846527..9e70fd9f 100644 --- a/tests/ahriman/core/upload/test_http_upload.py +++ b/tests/ahriman/core/upload/test_http_upload.py @@ -22,7 +22,7 @@ def test_calculate_hash_small(resource_path_root: Path) -> None: must calculate checksum for path which is single chunk """ path = resource_path_root / "models" / "package_ahriman_srcinfo" - assert HttpUpload.calculate_hash(path) == "a55f82198e56061295d405aeb58f4062" + assert HttpUpload.calculate_hash(path) == "c0aaf6ebf95ca9206dc8ba1d8ff10af3" def test_get_body_get_hashes() -> None: diff --git a/tests/ahriman/core/upload/test_s3.py b/tests/ahriman/core/upload/test_s3.py index ca6a24a7..a0256c47 100644 --- a/tests/ahriman/core/upload/test_s3.py +++ b/tests/ahriman/core/upload/test_s3.py @@ -31,7 +31,7 @@ def test_calculate_etag_small(resource_path_root: Path) -> None: must calculate checksum for path which is single chunk """ path = resource_path_root / "models" / "package_ahriman_srcinfo" - assert S3.calculate_etag(path, _chunk_size) == "a55f82198e56061295d405aeb58f4062" + assert S3.calculate_etag(path, _chunk_size) == "c0aaf6ebf95ca9206dc8ba1d8ff10af3" def test_files_remove(s3_remote_objects: List[Any]) -> None: diff --git a/tests/ahriman/models/test_aur_package.py b/tests/ahriman/models/test_aur_package.py new file mode 100644 index 00000000..96029b42 --- /dev/null +++ b/tests/ahriman/models/test_aur_package.py @@ -0,0 +1,47 @@ +import datetime +import json + +from dataclasses import asdict, fields +from pathlib import Path +from pytest_mock import MockerFixture +from typing import Any, Dict + +from ahriman.models.aur_package import AURPackage + + +def _get_data(resource_path_root: Path) -> Dict[str, Any]: + """ + load package description from resource file + :param resource_path_root: path to resource root + :return: json descriptor + """ + response = (resource_path_root / "models" / "package_ahriman_aur").read_text() + return json.loads(response)["results"][0] + + +def test_from_json(aur_package_ahriman: AURPackage, resource_path_root: Path) -> None: + """ + must load package from json + """ + model = _get_data(resource_path_root) + assert AURPackage.from_json(model) == aur_package_ahriman + + +def test_from_json_2(aur_package_ahriman: AURPackage, mocker: MockerFixture) -> None: + """ + must load the same package from json + """ + mocker.patch("ahriman.models.aur_package.AURPackage.convert", side_effect=lambda v: v) + assert AURPackage.from_json(asdict(aur_package_ahriman)) == aur_package_ahriman + + +def test_convert(aur_package_ahriman: AURPackage, resource_path_root: Path) -> None: + """ + must convert fields to snakecase and also apply converters + """ + model = _get_data(resource_path_root) + converted = AURPackage.convert(model) + known_fields = [pair.name for pair in fields(AURPackage)] + assert all(field in known_fields for field in converted) + assert isinstance(converted.get("first_submitted"), datetime.datetime) + assert isinstance(converted.get("last_modified"), datetime.datetime) diff --git a/tests/ahriman/models/test_package.py b/tests/ahriman/models/test_package.py index 110c8cc8..20e54a88 100644 --- a/tests/ahriman/models/test_package.py +++ b/tests/ahriman/models/test_package.py @@ -2,9 +2,10 @@ import pytest from pathlib import Path from pytest_mock import MockerFixture -from unittest.mock import MagicMock, PropertyMock +from unittest.mock import MagicMock from ahriman.core.exceptions import InvalidPackageInfo +from ahriman.models.aur_package import AURPackage from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.repository_paths import RepositoryPaths @@ -96,15 +97,11 @@ def test_from_archive(package_ahriman: Package, pyalpm_handle: MagicMock, mocker assert Package.from_archive(Path("path"), pyalpm_handle, package_ahriman.aur_url) == package_ahriman -def test_from_aur(package_ahriman: Package, mocker: MockerFixture) -> None: +def test_from_aur(package_ahriman: Package, aur_package_ahriman: AURPackage, mocker: MockerFixture) -> None: """ must construct package from aur """ - mock = MagicMock() - type(mock).name = PropertyMock(return_value=package_ahriman.base) - type(mock).package_base = PropertyMock(return_value=package_ahriman.base) - type(mock).version = PropertyMock(return_value=package_ahriman.version) - mocker.patch("aur.info", return_value=mock) + mocker.patch("ahriman.core.alpm.aur.AUR.info", return_value=aur_package_ahriman) package = Package.from_aur(package_ahriman.base, package_ahriman.aur_url) assert package_ahriman.base == package.base diff --git a/tests/ahriman/web/views/service/test_views_service_search.py b/tests/ahriman/web/views/service/test_views_service_search.py index b2c37964..7ba037e6 100644 --- a/tests/ahriman/web/views/service/test_views_service_search.py +++ b/tests/ahriman/web/views/service/test_views_service_search.py @@ -1,9 +1,9 @@ -import aur import pytest from aiohttp.test_utils import TestClient from pytest_mock import MockerFixture +from ahriman.models.aur_package import AURPackage from ahriman.models.user_access import UserAccess from ahriman.web.views.service.search import SearchView @@ -17,11 +17,11 @@ async def test_get_permission() -> None: assert await SearchView.get_permission(request) == UserAccess.Read -async def test_get(client: TestClient, aur_package_ahriman: aur.Package, mocker: MockerFixture) -> None: +async def test_get(client: TestClient, aur_package_ahriman: AURPackage, mocker: MockerFixture) -> None: """ must call get request correctly """ - mocker.patch("ahriman.web.views.service.search.aur_search", return_value=[aur_package_ahriman]) + mocker.patch("ahriman.core.alpm.aur.AUR.multisearch", return_value=[aur_package_ahriman]) response = await client.get("/service-api/v1/search", params={"for": "ahriman"}) assert response.ok @@ -33,7 +33,7 @@ async def test_get_exception(client: TestClient, mocker: MockerFixture) -> None: """ must raise 400 on empty search string """ - search_mock = mocker.patch("ahriman.web.views.service.search.aur_search", return_value=[]) + search_mock = mocker.patch("ahriman.core.alpm.aur.AUR.multisearch", return_value=[]) response = await client.get("/service-api/v1/search") assert response.status == 404 @@ -44,7 +44,7 @@ async def test_get_join(client: TestClient, mocker: MockerFixture) -> None: """ must join search args with space """ - search_mock = mocker.patch("ahriman.web.views.service.search.aur_search") + search_mock = mocker.patch("ahriman.core.alpm.aur.AUR.multisearch") response = await client.get("/service-api/v1/search", params=[("for", "ahriman"), ("for", "maybe")]) assert response.ok diff --git a/tests/ahriman/web/views/user/test_views_user_login.py b/tests/ahriman/web/views/user/test_views_user_login.py index de980e36..572a199e 100644 --- a/tests/ahriman/web/views/user/test_views_user_login.py +++ b/tests/ahriman/web/views/user/test_views_user_login.py @@ -1,4 +1,5 @@ import pytest + from aiohttp.test_utils import TestClient from pytest_mock import MockerFixture from unittest.mock import MagicMock @@ -31,7 +32,7 @@ async def test_get_redirect_to_oauth(client_with_auth: TestClient) -> None: must redirect to OAuth service provider in case if no code is supplied """ oauth = client_with_auth.app["validator"] = MagicMock(spec=OAuth) - oauth.get_oauth_url.return_value = "https://example.com" + oauth.get_oauth_url.return_value = "https://httpbin.org" get_response = await client_with_auth.get("/user-api/v1/login") assert get_response.ok @@ -43,7 +44,7 @@ async def test_get_redirect_to_oauth_empty_code(client_with_auth: TestClient) -> must redirect to OAuth service provider in case if empty code is supplied """ oauth = client_with_auth.app["validator"] = MagicMock(spec=OAuth) - oauth.get_oauth_url.return_value = "https://example.com" + oauth.get_oauth_url.return_value = "https://httpbin.org" get_response = await client_with_auth.get("/user-api/v1/login", params={"code": ""}) assert get_response.ok diff --git a/tests/testresources/models/aur_error b/tests/testresources/models/aur_error new file mode 100644 index 00000000..04655520 --- /dev/null +++ b/tests/testresources/models/aur_error @@ -0,0 +1,7 @@ +{ + "error": "Incorrect request type specified.", + "resultcount": 0, + "results": [], + "type": "error", + "version": 5 +} diff --git a/tests/testresources/models/package_ahriman_aur b/tests/testresources/models/package_ahriman_aur new file mode 100644 index 00000000..585df47a --- /dev/null +++ b/tests/testresources/models/package_ahriman_aur @@ -0,0 +1,54 @@ +{ + "resultcount": 1, + "results": [ + { + "Depends": [ + "devtools", + "git", + "pyalpm", + "python-aur", + "python-passlib", + "python-srcinfo" + ], + "Description": "ArcH Linux ReposItory MANager", + "FirstSubmitted": 1618008285, + "ID": 1009791, + "Keywords": [], + "LastModified": 1640473871, + "License": [ + "GPL3" + ], + "Maintainer": "arcanis", + "MakeDepends": [ + "python-pip" + ], + "Name": "ahriman", + "NumVotes": 0, + "OptDepends": [ + "breezy", + "darcs", + "mercurial", + "python-aioauth-client", + "python-aiohttp", + "python-aiohttp-debugtoolbar", + "python-aiohttp-jinja2", + "python-aiohttp-security", + "python-aiohttp-session", + "python-boto3", + "python-cryptography", + "python-jinja", + "rsync", + "subversion" + ], + "OutOfDate": null, + "PackageBase": "ahriman", + "PackageBaseID": 165427, + "Popularity": 0, + "URL": "https://github.com/arcan1s/ahriman", + "URLPath": "/cgit/aur.git/snapshot/ahriman.tar.gz", + "Version": "1.7.0-1" + } + ], + "type": "multiinfo", + "version": 5 +} diff --git a/tests/testresources/models/package_ahriman_srcinfo b/tests/testresources/models/package_ahriman_srcinfo index 57e15e39..2529d2ef 100644 --- a/tests/testresources/models/package_ahriman_srcinfo +++ b/tests/testresources/models/package_ahriman_srcinfo @@ -1,6 +1,6 @@ pkgbase = ahriman pkgdesc = ArcH Linux ReposItory MANager - pkgver = 0.12.1 + pkgver = 1.7.0 pkgrel = 1 url = https://github.com/arcan1s/ahriman arch = any @@ -10,6 +10,7 @@ pkgbase = ahriman depends = git depends = pyalpm depends = python-aur + depends = python-passlib depends = python-srcinfo optdepends = aws-cli: sync to s3 optdepends = breezy: -bzr packages support @@ -24,7 +25,7 @@ pkgbase = ahriman optdepends = subversion: -svn packages support backup = etc/ahriman.ini backup = etc/ahriman.ini.d/logging.ini - source = https://github.com/arcan1s/ahriman/releases/download/0.12.1/ahriman-0.12.1-src.tar.xz + source = https://github.com/arcan1s/ahriman/releases/download/1.7.0/ahriman-1.7.0-src.tar.xz source = ahriman.sysusers source = ahriman.tmpfiles sha512sums = 8acc57f937d587ca665c29092cadddbaf3ba0b80e870b80d1551e283aba8f21306f9030a26fec8c71ab5863316f5f5f061b7ddc63cdff9e6d5a885f28ef1893d