From 43696b99203d3fcb18795c1be298dc713bc5d2da Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Sun, 11 Sep 2022 01:26:30 +0300 Subject: [PATCH] Make optional dependencies trully optional (#67) The issue appears when there is no boto, jinja and some other libraries are not installed because the classes which use these libraries are still being imported inside the package file. The fix removes those imports from package root, because they should not be here, in fact, content of report and upload packages must be imported only inside the trigger class and only if they are actually required This commit also adds setuptools as required dependency since it is used for some parsers (previously it was provided dependency) --- .github/workflows/setup.sh | 2 +- .github/workflows/tests.sh | 2 +- Dockerfile | 2 +- docs/configuration.rst | 2 + package/archlinux/PKGBUILD | 2 +- setup.py | 1 + .../application/application_packages.py | 2 +- src/ahriman/core/alpm/remote/aur.py | 4 +- src/ahriman/core/alpm/remote/official.py | 7 +- src/ahriman/core/report/__init__.py | 7 -- src/ahriman/core/report/email.py | 3 +- src/ahriman/core/report/html.py | 3 +- src/ahriman/core/report/report.py | 8 +-- src/ahriman/core/report/telegram.py | 8 ++- src/ahriman/core/sign/gpg.py | 4 +- src/ahriman/core/upload/__init__.py | 6 -- src/ahriman/core/upload/github.py | 2 +- src/ahriman/core/upload/http_upload.py | 4 +- src/ahriman/core/upload/upload.py | 6 +- .../application/test_application_packages.py | 2 +- tests/ahriman/core/alpm/remote/test_aur.py | 12 +++- .../ahriman/core/alpm/remote/test_official.py | 6 +- tests/ahriman/core/report/test_console.py | 2 +- tests/ahriman/core/report/test_email.py | 12 ++-- tests/ahriman/core/report/test_html.py | 2 +- .../core/report/test_jinja_template.py | 2 +- tests/ahriman/core/report/test_report.py | 10 +-- tests/ahriman/core/report/test_telegram.py | 19 +++--- tests/ahriman/core/sign/test_gpg.py | 4 +- tests/ahriman/core/upload/conftest.py | 4 +- tests/ahriman/core/upload/test_github.py | 64 +++++++++---------- tests/ahriman/core/upload/test_http_upload.py | 5 +- tests/ahriman/core/upload/test_rsync.py | 4 +- tests/ahriman/core/upload/test_s3.py | 10 +-- tests/ahriman/core/upload/test_upload.py | 8 +-- 35 files changed, 131 insertions(+), 110 deletions(-) diff --git a/.github/workflows/setup.sh b/.github/workflows/setup.sh index fccb79f9..d7df4e73 100755 --- a/.github/workflows/setup.sh +++ b/.github/workflows/setup.sh @@ -8,7 +8,7 @@ echo -e '[arcanisrepo]\nServer = http://repo.arcanis.me/$arch\nSigLevel = Never' # refresh the image pacman --noconfirm -Syu # main dependencies -pacman --noconfirm -Sy base-devel devtools git pyalpm python-aur python-passlib python-srcinfo sudo +pacman --noconfirm -Sy base-devel devtools git pyalpm python-aur python-passlib python-setuptools python-srcinfo sudo # make dependencies pacman --noconfirm -Sy python-build python-installer python-wheel # optional dependencies diff --git a/.github/workflows/tests.sh b/.github/workflows/tests.sh index 864a8e0c..736a189b 100755 --- a/.github/workflows/tests.sh +++ b/.github/workflows/tests.sh @@ -4,7 +4,7 @@ set -ex # install dependencies -pacman --noconfirm -Syu base-devel python-pip python-tox +pacman --noconfirm -Syu base-devel python-pip python-setuptools python-tox # run test and check targets make check tests diff --git a/Dockerfile b/Dockerfile index 8449fb3f..a3508518 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN useradd -m -d /home/build -s /usr/bin/nologin build && \ COPY "docker/install-aur-package.sh" "/usr/local/bin/install-aur-package" ## install package dependencies ## darcs is not installed by reasons, because it requires a lot haskell packages which dramatically increase image size -RUN pacman --noconfirm -Sy devtools git pyalpm python-inflection python-passlib python-requests python-srcinfo && \ +RUN pacman --noconfirm -Sy devtools git pyalpm python-inflection python-passlib python-requests python-setuptools python-srcinfo && \ pacman --noconfirm -Sy python-build python-installer python-wheel && \ pacman --noconfirm -Sy breezy mercurial python-aiohttp python-boto3 python-cryptography python-jinja rsync subversion && \ runuser -u build -- install-aur-package python-aioauth-client python-aiohttp-jinja2 python-aiohttp-debugtoolbar \ diff --git a/docs/configuration.rst b/docs/configuration.rst index 6aa5cf26..1a36920b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -138,6 +138,7 @@ Section name must be either ``telegram`` (plus optional architecture name, e.g. * ``link_path`` - prefix for HTML links, string, required. * ``template_path`` - path to Jinja2 template, string, required. * ``template_type`` - ``parse_mode`` to be passed to telegram API, one of ``MarkdownV2``, ``HTML``, ``Markdown``, string, optional, default ``HTML``. +* ``timeout`` - HTTP request timeout in seconds, int, optional, default is ``30``. ``upload`` group ---------------- @@ -167,6 +168,7 @@ This feature requires Github key creation (see below). Section name must be eith #. Generate new token. Required scope is ``public_repo`` (or ``repo`` for private repository support). * ``repository`` - Github repository name, string, required. Repository must be created before any action and must have active branch (e.g. with readme). +* ``timeout`` - HTTP request timeout in seconds, int, optional, default is ``30``. * ``username`` - Github authorization user, string, required. Basically the same as ``owner``. ``rsync`` type diff --git a/package/archlinux/PKGBUILD b/package/archlinux/PKGBUILD index de3081ae..dec9819d 100644 --- a/package/archlinux/PKGBUILD +++ b/package/archlinux/PKGBUILD @@ -7,7 +7,7 @@ pkgdesc="ArcH linux ReposItory MANager" arch=('any') url="https://github.com/arcan1s/ahriman" license=('GPL3') -depends=('devtools' 'git' 'pyalpm' 'python-inflection' 'python-passlib' 'python-requests' 'python-srcinfo') +depends=('devtools' 'git' 'pyalpm' 'python-inflection' 'python-passlib' 'python-requests' 'python-setuptools' 'python-srcinfo') makedepends=('python-build' 'python-installer' 'python-wheel') optdepends=('breezy: -bzr packages support' 'darcs: -darcs packages support' diff --git a/setup.py b/setup.py index eaf3b083..986af05e 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,7 @@ setup( "inflection", "passlib", "requests", + "setuptools", "srcinfo", ], setup_requires=[ diff --git a/src/ahriman/application/application/application_packages.py b/src/ahriman/application/application/application_packages.py index 559612d6..a6766113 100644 --- a/src/ahriman/application/application/application_packages.py +++ b/src/ahriman/application/application/application_packages.py @@ -129,7 +129,7 @@ class ApplicationPackages(ApplicationProperties): source(str): remote URL of the package archive """ dst = self.repository.paths.packages / Path(source).name # URL is path, is not it? - response = requests.get(source, stream=True) + response = requests.get(source, stream=True, timeout=None) # timeout=None to suppress pylint warns response.raise_for_status() with dst.open("wb") as local_file: diff --git a/src/ahriman/core/alpm/remote/aur.py b/src/ahriman/core/alpm/remote/aur.py index cb521914..416bd7ff 100644 --- a/src/ahriman/core/alpm/remote/aur.py +++ b/src/ahriman/core/alpm/remote/aur.py @@ -36,11 +36,13 @@ class AUR(Remote): DEFAULT_AUR_URL(str): (class attribute) default AUR url DEFAULT_RPC_URL(str): (class attribute) default AUR RPC url DEFAULT_RPC_VERSION(str): (class attribute) default AUR RPC version + DEFAULT_TIMEOUT(int): (class attribute) HTTP request timeout in seconds """ DEFAULT_AUR_URL = "https://aur.archlinux.org" DEFAULT_RPC_URL = f"{DEFAULT_AUR_URL}/rpc" DEFAULT_RPC_VERSION = "5" + DEFAULT_TIMEOUT = 30 @staticmethod def parse_response(response: Dict[str, Any]) -> List[AURPackage]: @@ -113,7 +115,7 @@ class AUR(Remote): query[key] = value try: - response = requests.get(self.DEFAULT_RPC_URL, params=query) + response = requests.get(self.DEFAULT_RPC_URL, params=query, timeout=self.DEFAULT_TIMEOUT) response.raise_for_status() return self.parse_response(response.json()) except requests.HTTPError as e: diff --git a/src/ahriman/core/alpm/remote/official.py b/src/ahriman/core/alpm/remote/official.py index b39e78c1..58edee39 100644 --- a/src/ahriman/core/alpm/remote/official.py +++ b/src/ahriman/core/alpm/remote/official.py @@ -36,11 +36,13 @@ class Official(Remote): DEFAULT_ARCHLINUX_URL(str): (class attribute) default archlinux url DEFAULT_SEARCH_REPOSITORIES(List[str]): (class attribute) default list of repositories to search DEFAULT_RPC_URL(str): (class attribute) default archlinux repositories RPC url + DEFAULT_TIMEOUT(int): (class attribute) HTTP request timeout in seconds """ DEFAULT_ARCHLINUX_URL = "https://archlinux.org" DEFAULT_SEARCH_REPOSITORIES = ["Core", "Extra", "Multilib", "Community"] DEFAULT_RPC_URL = "https://archlinux.org/packages/search/json" + DEFAULT_TIMEOUT = 30 @staticmethod def parse_response(response: Dict[str, Any]) -> List[AURPackage]: @@ -101,7 +103,10 @@ class Official(Remote): List[AURPackage]: response parsed to package list """ try: - response = requests.get(self.DEFAULT_RPC_URL, params={by: args, "repo": self.DEFAULT_SEARCH_REPOSITORIES}) + response = requests.get( + self.DEFAULT_RPC_URL, + params={by: args, "repo": self.DEFAULT_SEARCH_REPOSITORIES}, + timeout=self.DEFAULT_TIMEOUT) response.raise_for_status() return self.parse_response(response.json()) except requests.HTTPError as e: diff --git a/src/ahriman/core/report/__init__.py b/src/ahriman/core/report/__init__.py index cca1d51f..ad4c2ae0 100644 --- a/src/ahriman/core/report/__init__.py +++ b/src/ahriman/core/report/__init__.py @@ -18,11 +18,4 @@ # along with this program. If not, see . # from ahriman.core.report.report import Report -from ahriman.core.report.jinja_template import JinjaTemplate - -from ahriman.core.report.console import Console -from ahriman.core.report.email import Email -from ahriman.core.report.html import HTML -from ahriman.core.report.telegram import Telegram - from ahriman.core.report.report_trigger import ReportTrigger diff --git a/src/ahriman/core/report/email.py b/src/ahriman/core/report/email.py index af5ce2c1..cfd413e7 100644 --- a/src/ahriman/core/report/email.py +++ b/src/ahriman/core/report/email.py @@ -25,7 +25,8 @@ from email.mime.text import MIMEText from typing import Dict, Iterable from ahriman.core.configuration import Configuration -from ahriman.core.report import JinjaTemplate, Report +from ahriman.core.report import Report +from ahriman.core.report.jinja_template import JinjaTemplate from ahriman.core.util import pretty_datetime from ahriman.models.package import Package from ahriman.models.result import Result diff --git a/src/ahriman/core/report/html.py b/src/ahriman/core/report/html.py index 63fb3756..34bd67a7 100644 --- a/src/ahriman/core/report/html.py +++ b/src/ahriman/core/report/html.py @@ -20,7 +20,8 @@ from typing import Iterable from ahriman.core.configuration import Configuration -from ahriman.core.report import JinjaTemplate, Report +from ahriman.core.report import Report +from ahriman.core.report.jinja_template import JinjaTemplate from ahriman.models.package import Package from ahriman.models.result import Result diff --git a/src/ahriman/core/report/report.py b/src/ahriman/core/report/report.py index 5bc0ebf3..50aeb9e6 100644 --- a/src/ahriman/core/report/report.py +++ b/src/ahriman/core/report/report.py @@ -84,16 +84,16 @@ class Report(LazyLogging): section, provider_name = configuration.gettype(target, architecture) provider = ReportSettings.from_option(provider_name) if provider == ReportSettings.HTML: - from ahriman.core.report import HTML + from ahriman.core.report.html import HTML return HTML(architecture, configuration, section) if provider == ReportSettings.Email: - from ahriman.core.report import Email + from ahriman.core.report.email import Email return Email(architecture, configuration, section) if provider == ReportSettings.Console: - from ahriman.core.report import Console + from ahriman.core.report.console import Console return Console(architecture, configuration, section) if provider == ReportSettings.Telegram: - from ahriman.core.report import Telegram + from ahriman.core.report.telegram import Telegram return Telegram(architecture, configuration, section) return cls(architecture, configuration) # should never happen diff --git a/src/ahriman/core/report/telegram.py b/src/ahriman/core/report/telegram.py index ce5a22ee..b4b724aa 100644 --- a/src/ahriman/core/report/telegram.py +++ b/src/ahriman/core/report/telegram.py @@ -23,7 +23,8 @@ import requests from typing import Iterable from ahriman.core.configuration import Configuration -from ahriman.core.report import JinjaTemplate, Report +from ahriman.core.report import Report +from ahriman.core.report.jinja_template import JinjaTemplate from ahriman.core.util import exception_response_text from ahriman.models.package import Package from ahriman.models.result import Result @@ -40,6 +41,7 @@ class Telegram(Report, JinjaTemplate): chat_id(str): chat id to post message, either string with @ or integer template_path(Path): path to template for built packages template_type(str): template message type to be used in parse mode, one of MarkdownV2, HTML, Markdown + timeout(int): HTTP request timeout in seconds """ TELEGRAM_API_URL = "https://api.telegram.org" @@ -61,6 +63,7 @@ class Telegram(Report, JinjaTemplate): self.chat_id = configuration.get(section, "chat_id") self.template_path = configuration.getpath(section, "template_path") self.template_type = configuration.get(section, "template_type", fallback="HTML") + self.timeout = configuration.getint(section, "timeout", fallback=30) def _send(self, text: str) -> None: """ @@ -72,7 +75,8 @@ class Telegram(Report, JinjaTemplate): try: response = requests.post( f"{self.TELEGRAM_API_URL}/bot{self.api_key}/sendMessage", - data={"chat_id": self.chat_id, "text": text, "parse_mode": self.template_type}) + data={"chat_id": self.chat_id, "text": text, "parse_mode": self.template_type}, + timeout=self.timeout) response.raise_for_status() except requests.HTTPError as e: self.logger.exception("could not perform request: %s", exception_response_text(e)) diff --git a/src/ahriman/core/sign/gpg.py b/src/ahriman/core/sign/gpg.py index 6e9c72fd..f8c20467 100644 --- a/src/ahriman/core/sign/gpg.py +++ b/src/ahriman/core/sign/gpg.py @@ -34,6 +34,7 @@ class GPG(LazyLogging): gnupg wrapper Attributes: + DEFAULT_TIMEOUT(int): (class attribute) HTTP request timeout in seconds architecture(str): repository architecture configuration(Configuration): configuration instance default_key(Optional[str]): default PGP key ID to use @@ -41,6 +42,7 @@ class GPG(LazyLogging): """ _check_output = check_output + DEFAULT_TIMEOUT = 30 def __init__(self, architecture: str, configuration: Configuration) -> None: """ @@ -120,7 +122,7 @@ class GPG(LazyLogging): "op": "get", "options": "mr", "search": key - }) + }, timeout=self.DEFAULT_TIMEOUT) response.raise_for_status() except requests.exceptions.HTTPError as e: self.logger.exception("could not download key %s from %s: %s", key, server, exception_response_text(e)) diff --git a/src/ahriman/core/upload/__init__.py b/src/ahriman/core/upload/__init__.py index 753a291a..24bdcb2e 100644 --- a/src/ahriman/core/upload/__init__.py +++ b/src/ahriman/core/upload/__init__.py @@ -18,10 +18,4 @@ # along with this program. If not, see . # from ahriman.core.upload.upload import Upload -from ahriman.core.upload.http_upload import HttpUpload - -from ahriman.core.upload.github import Github -from ahriman.core.upload.rsync import Rsync -from ahriman.core.upload.s3 import S3 - from ahriman.core.upload.upload_trigger import UploadTrigger diff --git a/src/ahriman/core/upload/github.py b/src/ahriman/core/upload/github.py index af87d99f..afeb6e0f 100644 --- a/src/ahriman/core/upload/github.py +++ b/src/ahriman/core/upload/github.py @@ -24,7 +24,7 @@ from pathlib import Path from typing import Any, Dict, Iterable, Optional from ahriman.core.configuration import Configuration -from ahriman.core.upload import HttpUpload +from ahriman.core.upload.http_upload import HttpUpload from ahriman.core.util import walk from ahriman.models.package import Package diff --git a/src/ahriman/core/upload/http_upload.py b/src/ahriman/core/upload/http_upload.py index 54b34077..5c1db486 100644 --- a/src/ahriman/core/upload/http_upload.py +++ b/src/ahriman/core/upload/http_upload.py @@ -34,6 +34,7 @@ class HttpUpload(Upload): Attributes: auth(Tuple[str, str]): HTTP auth object + timeout(int): HTTP request timeout in seconds """ def __init__(self, architecture: str, configuration: Configuration, section: str) -> None: @@ -49,6 +50,7 @@ class HttpUpload(Upload): password = configuration.get(section, "password") username = configuration.get(section, "username") self.auth = (password, username) + self.timeout = configuration.getint(section, "timeout", fallback=30) @staticmethod def calculate_hash(path: Path) -> str: @@ -108,7 +110,7 @@ class HttpUpload(Upload): requests.Response: request response object """ try: - response = requests.request(method, url, auth=self.auth, **kwargs) + response = requests.request(method, url, auth=self.auth, timeout=self.timeout, **kwargs) response.raise_for_status() except requests.HTTPError as e: self.logger.exception("could not perform %s request to %s: %s", method, url, exception_response_text(e)) diff --git a/src/ahriman/core/upload/upload.py b/src/ahriman/core/upload/upload.py index 2f0da5ad..ede890fb 100644 --- a/src/ahriman/core/upload/upload.py +++ b/src/ahriman/core/upload/upload.py @@ -83,13 +83,13 @@ class Upload(LazyLogging): section, provider_name = configuration.gettype(target, architecture) provider = UploadSettings.from_option(provider_name) if provider == UploadSettings.Rsync: - from ahriman.core.upload import Rsync + from ahriman.core.upload.rsync import Rsync return Rsync(architecture, configuration, section) if provider == UploadSettings.S3: - from ahriman.core.upload import S3 + from ahriman.core.upload.s3 import S3 return S3(architecture, configuration, section) if provider == UploadSettings.Github: - from ahriman.core.upload import Github + from ahriman.core.upload.github import Github return Github(architecture, configuration, section) return cls(architecture, configuration) # should never happen diff --git a/tests/ahriman/application/application/test_application_packages.py b/tests/ahriman/application/application/test_application_packages.py index 35053d99..52a29694 100644 --- a/tests/ahriman/application/application/test_application_packages.py +++ b/tests/ahriman/application/application/test_application_packages.py @@ -111,7 +111,7 @@ def test_add_remote(application_packages: ApplicationPackages, package_descripti application_packages._add_remote(url) open_mock.assert_called_once_with("wb") - request_mock.assert_called_once_with(url, stream=True) + request_mock.assert_called_once_with(url, stream=True, timeout=None) response_mock.raise_for_status.assert_called_once_with() diff --git a/tests/ahriman/core/alpm/remote/test_aur.py b/tests/ahriman/core/alpm/remote/test_aur.py index 4698c5f0..2cca99f0 100644 --- a/tests/ahriman/core/alpm/remote/test_aur.py +++ b/tests/ahriman/core/alpm/remote/test_aur.py @@ -78,7 +78,9 @@ def test_make_request(aur: AUR, aur_package_ahriman: AURPackage, assert aur.make_request("info", "ahriman") == [aur_package_ahriman] request_mock.assert_called_once_with( - "https://aur.archlinux.org/rpc", params={"v": "5", "type": "info", "arg": ["ahriman"]}) + "https://aur.archlinux.org/rpc", + params={"v": "5", "type": "info", "arg": ["ahriman"]}, + timeout=aur.DEFAULT_TIMEOUT) def test_make_request_multi_arg(aur: AUR, aur_package_ahriman: AURPackage, @@ -92,7 +94,9 @@ def test_make_request_multi_arg(aur: AUR, aur_package_ahriman: AURPackage, assert aur.make_request("search", "ahriman", "is", "cool") == [aur_package_ahriman] request_mock.assert_called_once_with( - "https://aur.archlinux.org/rpc", params={"v": "5", "type": "search", "arg[]": ["ahriman", "is", "cool"]}) + "https://aur.archlinux.org/rpc", + params={"v": "5", "type": "search", "arg[]": ["ahriman", "is", "cool"]}, + timeout=aur.DEFAULT_TIMEOUT) def test_make_request_with_kwargs(aur: AUR, aur_package_ahriman: AURPackage, @@ -106,7 +110,9 @@ def test_make_request_with_kwargs(aur: AUR, aur_package_ahriman: AURPackage, assert aur.make_request("search", "ahriman", by="name") == [aur_package_ahriman] request_mock.assert_called_once_with( - "https://aur.archlinux.org/rpc", params={"v": "5", "type": "search", "arg": ["ahriman"], "by": "name"}) + "https://aur.archlinux.org/rpc", + params={"v": "5", "type": "search", "arg": ["ahriman"], "by": "name"}, + timeout=aur.DEFAULT_TIMEOUT) def test_make_request_failed(aur: AUR, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/core/alpm/remote/test_official.py b/tests/ahriman/core/alpm/remote/test_official.py index 7d9716fd..28342d2e 100644 --- a/tests/ahriman/core/alpm/remote/test_official.py +++ b/tests/ahriman/core/alpm/remote/test_official.py @@ -84,8 +84,10 @@ def test_make_request(official: Official, aur_package_akonadi: AURPackage, request_mock = mocker.patch("requests.get", return_value=response_mock) assert official.make_request("akonadi", by="q") == [aur_package_akonadi] - request_mock.assert_called_once_with("https://archlinux.org/packages/search/json", - params={"q": ("akonadi",), "repo": Official.DEFAULT_SEARCH_REPOSITORIES}) + request_mock.assert_called_once_with( + "https://archlinux.org/packages/search/json", + params={"q": ("akonadi",), "repo": Official.DEFAULT_SEARCH_REPOSITORIES}, + timeout=official.DEFAULT_TIMEOUT) def test_make_request_failed(official: Official, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/core/report/test_console.py b/tests/ahriman/core/report/test_console.py index 7f2b64c0..83c8dab0 100644 --- a/tests/ahriman/core/report/test_console.py +++ b/tests/ahriman/core/report/test_console.py @@ -2,7 +2,7 @@ from pytest_mock import MockerFixture from unittest import mock from ahriman.core.configuration import Configuration -from ahriman.core.report import Console +from ahriman.core.report.console import Console from ahriman.models.package import Package from ahriman.models.result import Result diff --git a/tests/ahriman/core/report/test_email.py b/tests/ahriman/core/report/test_email.py index fc2a046d..f215702d 100644 --- a/tests/ahriman/core/report/test_email.py +++ b/tests/ahriman/core/report/test_email.py @@ -3,7 +3,7 @@ import pytest from pytest_mock import MockerFixture from ahriman.core.configuration import Configuration -from ahriman.core.report import Email +from ahriman.core.report.email import Email from ahriman.models.package import Package from ahriman.models.result import Result @@ -90,7 +90,7 @@ def test_generate(configuration: Configuration, package_ahriman: Package, mocker """ must generate report """ - send_mock = mocker.patch("ahriman.core.report.Email._send") + send_mock = mocker.patch("ahriman.core.report.email.Email._send") report = Email("x86_64", configuration, "email") report.generate([package_ahriman], Result()) @@ -102,7 +102,7 @@ def test_generate_with_built(configuration: Configuration, package_ahriman: Pack """ must generate report with built packages """ - send_mock = mocker.patch("ahriman.core.report.Email._send") + send_mock = mocker.patch("ahriman.core.report.email.Email._send") report = Email("x86_64", configuration, "email") report.generate([package_ahriman], result) @@ -117,7 +117,7 @@ def test_generate_with_built_and_full_path( """ must generate report with built packages and full packages lists """ - send_mock = mocker.patch("ahriman.core.report.Email._send") + send_mock = mocker.patch("ahriman.core.report.email.Email._send") report = Email("x86_64", configuration, "email") report.full_template_path = report.template_path @@ -130,7 +130,7 @@ def test_generate_no_empty(configuration: Configuration, package_ahriman: Packag must not generate report with built packages if no_empty_report is set """ configuration.set_option("email", "no_empty_report", "yes") - send_mock = mocker.patch("ahriman.core.report.Email._send") + send_mock = mocker.patch("ahriman.core.report.email.Email._send") report = Email("x86_64", configuration, "email") report.generate([package_ahriman], Result()) @@ -143,7 +143,7 @@ def test_generate_no_empty_with_built(configuration: Configuration, package_ahri must generate report with built packages if no_empty_report is set """ configuration.set_option("email", "no_empty_report", "yes") - send_mock = mocker.patch("ahriman.core.report.Email._send") + send_mock = mocker.patch("ahriman.core.report.email.Email._send") report = Email("x86_64", configuration, "email") report.generate([package_ahriman], result) diff --git a/tests/ahriman/core/report/test_html.py b/tests/ahriman/core/report/test_html.py index bb8c540e..b57e3b60 100644 --- a/tests/ahriman/core/report/test_html.py +++ b/tests/ahriman/core/report/test_html.py @@ -3,7 +3,7 @@ import pytest from pytest_mock import MockerFixture from ahriman.core.configuration import Configuration -from ahriman.core.report import HTML +from ahriman.core.report.html import HTML from ahriman.models.package import Package diff --git a/tests/ahriman/core/report/test_jinja_template.py b/tests/ahriman/core/report/test_jinja_template.py index 7a9783c0..93deccbf 100644 --- a/tests/ahriman/core/report/test_jinja_template.py +++ b/tests/ahriman/core/report/test_jinja_template.py @@ -1,5 +1,5 @@ from ahriman.core.configuration import Configuration -from ahriman.core.report import JinjaTemplate +from ahriman.core.report.jinja_template import JinjaTemplate from ahriman.models.package import Package from ahriman.models.result import Result diff --git a/tests/ahriman/core/report/test_report.py b/tests/ahriman/core/report/test_report.py index 5864c960..6cc269db 100644 --- a/tests/ahriman/core/report/test_report.py +++ b/tests/ahriman/core/report/test_report.py @@ -13,7 +13,7 @@ def test_report_failure(configuration: Configuration, mocker: MockerFixture) -> """ must raise ReportFailed on errors """ - mocker.patch("ahriman.core.report.HTML.generate", side_effect=Exception()) + mocker.patch("ahriman.core.report.html.HTML.generate", side_effect=Exception()) with pytest.raises(ReportFailed): Report.load("x86_64", configuration, "html").run([], Result()) @@ -32,7 +32,7 @@ def test_report_console(configuration: Configuration, result: Result, mocker: Mo """ must generate console report """ - report_mock = mocker.patch("ahriman.core.report.Console.generate") + report_mock = mocker.patch("ahriman.core.report.console.Console.generate") Report.load("x86_64", configuration, "console").run(result, []) report_mock.assert_called_once_with([], result) @@ -41,7 +41,7 @@ def test_report_email(configuration: Configuration, result: Result, mocker: Mock """ must generate email report """ - report_mock = mocker.patch("ahriman.core.report.Email.generate") + report_mock = mocker.patch("ahriman.core.report.email.Email.generate") Report.load("x86_64", configuration, "email").run(result, []) report_mock.assert_called_once_with([], result) @@ -50,7 +50,7 @@ def test_report_html(configuration: Configuration, result: Result, mocker: Mocke """ must generate html report """ - report_mock = mocker.patch("ahriman.core.report.HTML.generate") + report_mock = mocker.patch("ahriman.core.report.html.HTML.generate") Report.load("x86_64", configuration, "html").run(result, []) report_mock.assert_called_once_with([], result) @@ -59,6 +59,6 @@ def test_report_telegram(configuration: Configuration, result: Result, mocker: M """ must generate telegram report """ - report_mock = mocker.patch("ahriman.core.report.Telegram.generate") + report_mock = mocker.patch("ahriman.core.report.telegram.Telegram.generate") Report.load("x86_64", configuration, "telegram").run(result, []) report_mock.assert_called_once_with([], result) diff --git a/tests/ahriman/core/report/test_telegram.py b/tests/ahriman/core/report/test_telegram.py index 920e263c..ddd44afa 100644 --- a/tests/ahriman/core/report/test_telegram.py +++ b/tests/ahriman/core/report/test_telegram.py @@ -5,7 +5,7 @@ from pytest_mock import MockerFixture from unittest import mock from ahriman.core.configuration import Configuration -from ahriman.core.report import Telegram +from ahriman.core.report.telegram import Telegram from ahriman.models.package import Package from ahriman.models.result import Result @@ -20,7 +20,8 @@ def test_send(configuration: Configuration, mocker: MockerFixture) -> None: report._send("a text") request_mock.assert_called_once_with( pytest.helpers.anyvar(str, strict=True), - data={"chat_id": pytest.helpers.anyvar(str, strict=True), "text": "a text", "parse_mode": "HTML"}) + data={"chat_id": pytest.helpers.anyvar(str, strict=True), "text": "a text", "parse_mode": "HTML"}, + timeout=report.timeout) def test_send_failed(configuration: Configuration, mocker: MockerFixture) -> None: @@ -50,7 +51,7 @@ def test_generate(configuration: Configuration, package_ahriman: Package, result """ must generate report """ - send_mock = mocker.patch("ahriman.core.report.Telegram._send") + send_mock = mocker.patch("ahriman.core.report.telegram.Telegram._send") report = Telegram("x86_64", configuration, "telegram") report.generate([package_ahriman], result) @@ -62,7 +63,7 @@ def test_generate_big_text_without_spaces(configuration: Configuration, package_ """ must raise ValueError in case if there are no new lines in text """ - mocker.patch("ahriman.core.report.JinjaTemplate.make_html", return_value="ab" * 4096) + mocker.patch("ahriman.core.report.jinja_template.JinjaTemplate.make_html", return_value="ab" * 4096) report = Telegram("x86_64", configuration, "telegram") with pytest.raises(ValueError): @@ -74,8 +75,8 @@ def test_generate_big_text(configuration: Configuration, package_ahriman: Packag """ must generate report with big text """ - mocker.patch("ahriman.core.report.JinjaTemplate.make_html", return_value="a\n" * 4096) - send_mock = mocker.patch("ahriman.core.report.Telegram._send") + mocker.patch("ahriman.core.report.jinja_template.JinjaTemplate.make_html", return_value="a\n" * 4096) + send_mock = mocker.patch("ahriman.core.report.telegram.Telegram._send") report = Telegram("x86_64", configuration, "telegram") report.generate([package_ahriman], result) @@ -89,8 +90,8 @@ def test_generate_very_big_text(configuration: Configuration, package_ahriman: P """ must generate report with very big text """ - mocker.patch("ahriman.core.report.JinjaTemplate.make_html", return_value="ab\n" * 4096) - send_mock = mocker.patch("ahriman.core.report.Telegram._send") + mocker.patch("ahriman.core.report.jinja_template.JinjaTemplate.make_html", return_value="ab\n" * 4096) + send_mock = mocker.patch("ahriman.core.report.telegram.Telegram._send") report = Telegram("x86_64", configuration, "telegram") report.generate([package_ahriman], result) @@ -105,7 +106,7 @@ def test_generate_no_empty(configuration: Configuration, package_ahriman: Packag """ must generate report """ - send_mock = mocker.patch("ahriman.core.report.Telegram._send") + send_mock = mocker.patch("ahriman.core.report.telegram.Telegram._send") report = Telegram("x86_64", configuration, "telegram") report.generate([package_ahriman], Result()) diff --git a/tests/ahriman/core/sign/test_gpg.py b/tests/ahriman/core/sign/test_gpg.py index 33e9399f..34f64c24 100644 --- a/tests/ahriman/core/sign/test_gpg.py +++ b/tests/ahriman/core/sign/test_gpg.py @@ -83,7 +83,9 @@ def test_key_download(gpg: GPG, mocker: MockerFixture) -> None: requests_mock = mocker.patch("requests.get") gpg.key_download("pgp.mit.edu", "0xE989490C") requests_mock.assert_called_once_with( - "http://pgp.mit.edu/pks/lookup", params={"op": "get", "options": "mr", "search": "0xE989490C"}) + "http://pgp.mit.edu/pks/lookup", + params={"op": "get", "options": "mr", "search": "0xE989490C"}, + timeout=gpg.DEFAULT_TIMEOUT) def test_key_download_failure(gpg: GPG, mocker: MockerFixture) -> None: diff --git a/tests/ahriman/core/upload/conftest.py b/tests/ahriman/core/upload/conftest.py index 082749e7..24de8b5d 100644 --- a/tests/ahriman/core/upload/conftest.py +++ b/tests/ahriman/core/upload/conftest.py @@ -5,7 +5,9 @@ from typing import Any, Dict, List from unittest.mock import MagicMock from ahriman.core.configuration import Configuration -from ahriman.core.upload import Github, Rsync, S3 +from ahriman.core.upload.github import Github +from ahriman.core.upload.rsync import Rsync +from ahriman.core.upload.s3 import S3 _s3_object = namedtuple("s3_object", ["key", "e_tag", "delete"]) diff --git a/tests/ahriman/core/upload/test_github.py b/tests/ahriman/core/upload/test_github.py index 483efca9..8ff44fd0 100644 --- a/tests/ahriman/core/upload/test_github.py +++ b/tests/ahriman/core/upload/test_github.py @@ -6,14 +6,14 @@ from pytest_mock import MockerFixture from typing import Any, Dict from unittest import mock -from ahriman.core.upload import Github +from ahriman.core.upload.github import Github def test_asset_remove(github: Github, github_release: Dict[str, Any], mocker: MockerFixture) -> None: """ must remove asset from the release """ - request_mock = mocker.patch("ahriman.core.upload.Github._request") + request_mock = mocker.patch("ahriman.core.upload.github.Github._request") github.asset_remove(github_release, "asset_name") request_mock.assert_called_once_with("DELETE", "asset_url") @@ -22,7 +22,7 @@ def test_asset_remove_unknown(github: Github, github_release: Dict[str, Any], mo """ must not fail if no asset found """ - request_mock = mocker.patch("ahriman.core.upload.Github._request") + request_mock = mocker.patch("ahriman.core.upload.github.Github._request") github.asset_remove(github_release, "unknown_asset_name") request_mock.assert_not_called() @@ -32,8 +32,8 @@ def test_asset_upload(github: Github, github_release: Dict[str, Any], mocker: Mo must upload asset to the repository """ mocker.patch("pathlib.Path.open", return_value=b"") - request_mock = mocker.patch("ahriman.core.upload.Github._request") - remove_mock = mocker.patch("ahriman.core.upload.Github.asset_remove") + request_mock = mocker.patch("ahriman.core.upload.github.Github._request") + remove_mock = mocker.patch("ahriman.core.upload.github.Github.asset_remove") github.asset_upload(github_release, Path("/root/new.tar.xz")) request_mock.assert_called_once_with("POST", "upload_url", params={"name": "new.tar.xz"}, @@ -46,8 +46,8 @@ def test_asset_upload_with_removal(github: Github, github_release: Dict[str, Any must remove existing file before upload """ mocker.patch("pathlib.Path.open", return_value=b"") - mocker.patch("ahriman.core.upload.Github._request") - remove_mock = mocker.patch("ahriman.core.upload.Github.asset_remove") + mocker.patch("ahriman.core.upload.github.Github._request") + remove_mock = mocker.patch("ahriman.core.upload.github.Github.asset_remove") github.asset_upload(github_release, Path("asset_name")) github.asset_upload(github_release, Path("/root/asset_name")) @@ -62,9 +62,9 @@ def test_asset_upload_empty_mimetype(github: Github, github_release: Dict[str, A must upload asset to the repository with empty mime type if cannot guess it """ mocker.patch("pathlib.Path.open", return_value=b"") - mocker.patch("ahriman.core.upload.Github.asset_remove") + mocker.patch("ahriman.core.upload.github.Github.asset_remove") mocker.patch("mimetypes.guess_type", return_value=(None, None)) - request_mock = mocker.patch("ahriman.core.upload.Github._request") + request_mock = mocker.patch("ahriman.core.upload.github.Github._request") github.asset_upload(github_release, Path("/root/new.tar.xz")) request_mock.assert_called_once_with("POST", "upload_url", params={"name": "new.tar.xz"}, @@ -84,7 +84,7 @@ def test_files_remove(github: Github, github_release: Dict[str, Any], mocker: Mo """ must remove files from the remote """ - remove_mock = mocker.patch("ahriman.core.upload.Github.asset_remove") + remove_mock = mocker.patch("ahriman.core.upload.github.Github.asset_remove") github.files_remove(github_release, {Path("a"): "a"}, {"a": "a", "b": "b"}) remove_mock.assert_called_once_with(github_release, "b") @@ -93,7 +93,7 @@ def test_files_remove_empty(github: Github, github_release: Dict[str, Any], mock """ must remove nothing if nothing changed """ - remove_mock = mocker.patch("ahriman.core.upload.Github.asset_remove") + remove_mock = mocker.patch("ahriman.core.upload.github.Github.asset_remove") github.files_remove(github_release, {Path("a"): "a"}, {"a": "a"}) remove_mock.assert_not_called() @@ -102,7 +102,7 @@ def test_files_upload(github: Github, github_release: Dict[str, Any], mocker: Mo """ must upload files to the remote """ - upload_mock = mocker.patch("ahriman.core.upload.Github.asset_upload") + upload_mock = mocker.patch("ahriman.core.upload.github.Github.asset_upload") github.files_upload(github_release, {Path("a"): "a", Path("b"): "c", Path("c"): "c"}, {"a": "a", "b": "b"}) upload_mock.assert_has_calls([ mock.call(github_release, Path("b")), @@ -114,7 +114,7 @@ def test_files_upload_empty(github: Github, github_release: Dict[str, Any], mock """ must upload nothing if nothing changed """ - upload_mock = mocker.patch("ahriman.core.upload.Github.asset_upload") + upload_mock = mocker.patch("ahriman.core.upload.github.Github.asset_upload") github.files_upload(github_release, {Path("a"): "a"}, {"a": "a"}) upload_mock.assert_not_called() @@ -123,7 +123,7 @@ def test_release_create(github: Github, mocker: MockerFixture) -> None: """ must create release """ - request_mock = mocker.patch("ahriman.core.upload.Github._request") + request_mock = mocker.patch("ahriman.core.upload.github.Github._request") github.release_create() request_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True), json={"tag_name": github.architecture, "name": github.architecture}) @@ -133,7 +133,7 @@ def test_release_get(github: Github, mocker: MockerFixture) -> None: """ must get release """ - request_mock = mocker.patch("ahriman.core.upload.Github._request") + request_mock = mocker.patch("ahriman.core.upload.github.Github._request") github.release_get() request_mock.assert_called_once_with("GET", pytest.helpers.anyvar(str, True)) @@ -144,7 +144,7 @@ def test_release_get_empty(github: Github, mocker: MockerFixture) -> None: """ response = requests.Response() response.status_code = 404 - mocker.patch("ahriman.core.upload.Github._request", side_effect=requests.HTTPError(response=response)) + mocker.patch("ahriman.core.upload.github.Github._request", side_effect=requests.HTTPError(response=response)) assert github.release_get() is None @@ -152,7 +152,7 @@ def test_release_get_exception(github: Github, mocker: MockerFixture) -> None: """ must re-raise non HTTPError exception """ - mocker.patch("ahriman.core.upload.Github._request", side_effect=Exception()) + mocker.patch("ahriman.core.upload.github.Github._request", side_effect=Exception()) with pytest.raises(Exception): github.release_get() @@ -162,7 +162,7 @@ def test_release_get_exception_http_error(github: Github, mocker: MockerFixture) must re-raise HTTPError exception with code differs from 404 """ exception = requests.HTTPError(response=requests.Response()) - mocker.patch("ahriman.core.upload.Github._request", side_effect=exception) + mocker.patch("ahriman.core.upload.github.Github._request", side_effect=exception) with pytest.raises(requests.HTTPError): github.release_get() @@ -171,7 +171,7 @@ def test_release_update(github: Github, github_release: Dict[str, Any], mocker: """ must update release """ - request_mock = mocker.patch("ahriman.core.upload.Github._request") + request_mock = mocker.patch("ahriman.core.upload.github.Github._request") github.release_update(github_release, "body") request_mock.assert_called_once_with("POST", "release_url", json={"body": "body"}) @@ -180,12 +180,12 @@ def test_release_sync(github: Github, mocker: MockerFixture) -> None: """ must run sync command """ - release_get_mock = mocker.patch("ahriman.core.upload.Github.release_get", return_value={}) - get_hashes_mock = mocker.patch("ahriman.core.upload.Github.get_hashes", return_value={}) - get_local_files_mock = mocker.patch("ahriman.core.upload.Github.get_local_files", return_value={}) - files_upload_mock = mocker.patch("ahriman.core.upload.Github.files_upload") - files_remove_mock = mocker.patch("ahriman.core.upload.Github.files_remove") - release_update_mock = mocker.patch("ahriman.core.upload.Github.release_update") + release_get_mock = mocker.patch("ahriman.core.upload.github.Github.release_get", return_value={}) + get_hashes_mock = mocker.patch("ahriman.core.upload.github.Github.get_hashes", return_value={}) + get_local_files_mock = mocker.patch("ahriman.core.upload.github.Github.get_local_files", return_value={}) + files_upload_mock = mocker.patch("ahriman.core.upload.github.Github.files_upload") + files_remove_mock = mocker.patch("ahriman.core.upload.github.Github.files_remove") + release_update_mock = mocker.patch("ahriman.core.upload.github.Github.release_update") github.sync(Path("local"), []) release_get_mock.assert_called_once_with() @@ -200,13 +200,13 @@ def test_release_sync_create_release(github: Github, mocker: MockerFixture) -> N """ must create release in case if it does not exist """ - mocker.patch("ahriman.core.upload.Github.release_get", return_value=None) - mocker.patch("ahriman.core.upload.Github.get_hashes") - mocker.patch("ahriman.core.upload.Github.get_local_files") - mocker.patch("ahriman.core.upload.Github.files_upload") - mocker.patch("ahriman.core.upload.Github.files_remove") - mocker.patch("ahriman.core.upload.Github.release_update") - release_create_mock = mocker.patch("ahriman.core.upload.Github.release_create") + mocker.patch("ahriman.core.upload.github.Github.release_get", return_value=None) + mocker.patch("ahriman.core.upload.github.Github.get_hashes") + mocker.patch("ahriman.core.upload.github.Github.get_local_files") + mocker.patch("ahriman.core.upload.github.Github.files_upload") + mocker.patch("ahriman.core.upload.github.Github.files_remove") + mocker.patch("ahriman.core.upload.github.Github.release_update") + release_create_mock = mocker.patch("ahriman.core.upload.github.Github.release_create") github.sync(Path("local"), []) release_create_mock.assert_called_once_with() diff --git a/tests/ahriman/core/upload/test_http_upload.py b/tests/ahriman/core/upload/test_http_upload.py index c6bbb9b6..95a19765 100644 --- a/tests/ahriman/core/upload/test_http_upload.py +++ b/tests/ahriman/core/upload/test_http_upload.py @@ -5,7 +5,8 @@ from pathlib import Path from pytest_mock import MockerFixture from unittest.mock import MagicMock -from ahriman.core.upload import Github, HttpUpload +from ahriman.core.upload.github import Github +from ahriman.core.upload.http_upload import HttpUpload def test_calculate_hash_empty(resource_path_root: Path) -> None: @@ -49,7 +50,7 @@ def test_request(github: Github, mocker: MockerFixture) -> None: request_mock = mocker.patch("requests.request", return_value=response_mock) github._request("GET", "url", arg="arg") - request_mock.assert_called_once_with("GET", "url", auth=github.auth, arg="arg") + request_mock.assert_called_once_with("GET", "url", auth=github.auth, timeout=github.timeout, arg="arg") response_mock.raise_for_status.assert_called_once_with() diff --git a/tests/ahriman/core/upload/test_rsync.py b/tests/ahriman/core/upload/test_rsync.py index cdd7f6d7..52bf595a 100644 --- a/tests/ahriman/core/upload/test_rsync.py +++ b/tests/ahriman/core/upload/test_rsync.py @@ -1,13 +1,13 @@ from pathlib import Path from pytest_mock import MockerFixture -from ahriman.core.upload import Rsync +from ahriman.core.upload.rsync import Rsync def test_sync(rsync: Rsync, mocker: MockerFixture) -> None: """ must run sync command """ - check_output_mock = mocker.patch("ahriman.core.upload.Rsync._check_output") + check_output_mock = mocker.patch("ahriman.core.upload.rsync.Rsync._check_output") rsync.sync(Path("path"), []) check_output_mock.assert_called_once_with(*rsync.command, "path", rsync.remote, exception=None, logger=rsync.logger) diff --git a/tests/ahriman/core/upload/test_s3.py b/tests/ahriman/core/upload/test_s3.py index 0480909c..a4fa86d8 100644 --- a/tests/ahriman/core/upload/test_s3.py +++ b/tests/ahriman/core/upload/test_s3.py @@ -4,7 +4,7 @@ from typing import Any, List, Optional, Tuple from unittest import mock from unittest.mock import MagicMock -from ahriman.core.upload import S3 +from ahriman.core.upload.s3 import S3 _chunk_size = 8 * 1024 * 1024 @@ -104,10 +104,10 @@ def test_sync(s3: S3, mocker: MockerFixture) -> None: """ must run sync command """ - local_files_mock = mocker.patch("ahriman.core.upload.S3.get_local_files", return_value=["a"]) - remote_objects_mock = mocker.patch("ahriman.core.upload.S3.get_remote_objects", return_value=["b"]) - remove_files_mock = mocker.patch("ahriman.core.upload.S3.files_remove") - upload_files_mock = mocker.patch("ahriman.core.upload.S3.files_upload") + local_files_mock = mocker.patch("ahriman.core.upload.s3.S3.get_local_files", return_value=["a"]) + remote_objects_mock = mocker.patch("ahriman.core.upload.s3.S3.get_remote_objects", return_value=["b"]) + remove_files_mock = mocker.patch("ahriman.core.upload.s3.S3.files_remove") + upload_files_mock = mocker.patch("ahriman.core.upload.s3.S3.files_upload") s3.sync(Path("root"), []) remote_objects_mock.assert_called_once_with() diff --git a/tests/ahriman/core/upload/test_upload.py b/tests/ahriman/core/upload/test_upload.py index 396fe335..74f3d94d 100644 --- a/tests/ahriman/core/upload/test_upload.py +++ b/tests/ahriman/core/upload/test_upload.py @@ -13,7 +13,7 @@ def test_upload_failure(configuration: Configuration, mocker: MockerFixture) -> """ must raise SyncFailed on errors """ - mocker.patch("ahriman.core.upload.Rsync.sync", side_effect=Exception()) + mocker.patch("ahriman.core.upload.rsync.Rsync.sync", side_effect=Exception()) with pytest.raises(SyncFailed): Upload.load("x86_64", configuration, "rsync").run(Path("path"), []) @@ -32,7 +32,7 @@ def test_upload_rsync(configuration: Configuration, mocker: MockerFixture) -> No """ must upload via rsync """ - upload_mock = mocker.patch("ahriman.core.upload.Rsync.sync") + upload_mock = mocker.patch("ahriman.core.upload.rsync.Rsync.sync") Upload.load("x86_64", configuration, "rsync").run(Path("path"), []) upload_mock.assert_called_once_with(Path("path"), []) @@ -41,7 +41,7 @@ def test_upload_s3(configuration: Configuration, mocker: MockerFixture) -> None: """ must upload via s3 """ - upload_mock = mocker.patch("ahriman.core.upload.S3.sync") + upload_mock = mocker.patch("ahriman.core.upload.s3.S3.sync") Upload.load("x86_64", configuration, "customs3").run(Path("path"), []) upload_mock.assert_called_once_with(Path("path"), []) @@ -50,6 +50,6 @@ def test_upload_github(configuration: Configuration, mocker: MockerFixture) -> N """ must upload via github """ - upload_mock = mocker.patch("ahriman.core.upload.Github.sync") + upload_mock = mocker.patch("ahriman.core.upload.github.Github.sync") Upload.load("x86_64", configuration, "github").run(Path("path"), []) upload_mock.assert_called_once_with(Path("path"), [])