Compare commits

..

1 Commits

Author SHA1 Message Date
9ec566f095 feat: add retry policy 2026-02-20 02:49:32 +02:00
18 changed files with 1536 additions and 1650 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -97,15 +97,6 @@ libalpm and AUR related configuration. Group name can refer to architecture, e.g
* ``sync_files_database`` - download files database from mirror, boolean, required. * ``sync_files_database`` - download files database from mirror, boolean, required.
* ``use_ahriman_cache`` - use local pacman package cache instead of system one, boolean, required. With this option enabled you might want to refresh database periodically (available as additional flag for some subcommands). If set to ``no``, databases must be synchronized manually. * ``use_ahriman_cache`` - use local pacman package cache instead of system one, boolean, required. With this option enabled you might want to refresh database periodically (available as additional flag for some subcommands). If set to ``no``, databases must be synchronized manually.
``aur`` group
-------------
Archlinux User Repository related configuration.
* ``max_retries`` - maximum amount of retries of HTTP requests, integer, optional, default ``0``.
* ``retry_backoff`` - retry exponential backoff, float, optional, default ``0.0``.
* ``timeout`` - HTTP request timeout in seconds, integer, optional, default is ``30``.
``auth`` group ``auth`` group
-------------- --------------

View File

@@ -2,7 +2,7 @@
pkgbase='ahriman' pkgbase='ahriman'
pkgname=('ahriman' 'ahriman-core' 'ahriman-triggers' 'ahriman-web') pkgname=('ahriman' 'ahriman-core' 'ahriman-triggers' 'ahriman-web')
pkgver=2.20.0rc4 pkgver=2.20.0rc1
pkgrel=1 pkgrel=1
pkgdesc="ArcH linux ReposItory MANager" pkgdesc="ArcH linux ReposItory MANager"
arch=('any') arch=('any')

View File

@@ -23,14 +23,6 @@ sync_files_database = yes
; as additional option for some subcommands). If set to no, databases must be synchronized manually. ; as additional option for some subcommands). If set to no, databases must be synchronized manually.
use_ahriman_cache = yes use_ahriman_cache = yes
[aur]
; Maximum amount of retries of HTTP requests.
max_retries = 3
; Retry exponential backoff.
retry_backoff = 1.0
; HTTP request timeout in seconds.
;timeout = 30
[build] [build]
; List of additional flags passed to archbuild command. ; List of additional flags passed to archbuild command.
;archbuild_flags = ;archbuild_flags =

View File

@@ -1,4 +1,4 @@
.TH AHRIMAN "1" "2026\-02\-21" "ahriman 2.20.0rc4" "ArcH linux ReposItory MANager" .TH AHRIMAN "1" "2026\-02\-18" "ahriman 2.20.0rc1" "ArcH linux ReposItory MANager"
.SH NAME .SH NAME
ahriman \- ArcH linux ReposItory MANager ahriman \- ArcH linux ReposItory MANager
.SH SYNOPSIS .SH SYNOPSIS

View File

@@ -17,4 +17,4 @@
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
__version__ = "2.20.0rc4" __version__ = "2.20.0rc1"

View File

@@ -251,7 +251,7 @@ class Setup(Handler):
content = f"PACKAGER='{packager}'\n" content = f"PACKAGER='{packager}'\n"
if makeflags_jobs: if makeflags_jobs:
content += "MAKEFLAGS=\"-j$(nproc)\"\n" content += """MAKEFLAGS="-j$(nproc)"\n"""
uid, _ = paths.root_owner uid, _ = paths.root_owner
home_dir = Path(getpwuid(uid).pw_dir) home_dir = Path(getpwuid(uid).pw_dir)

View File

@@ -69,8 +69,8 @@ class OAuth(Mapping):
Returns: Returns:
str: login control as html code to insert str: login control as html code to insert
""" """
return f"<a class=\"nav-link\" href=\"/api/v1/login\" title=\"login via OAuth2\"><i class=\"bi bi-{ return f"""<a class="nav-link" href="/api/v1/login" title="login via OAuth2"><i class="bi bi-{
self.icon}\"></i> login</a>" self.icon}"></i> login</a>"""
@staticmethod @staticmethod
def get_provider(name: str) -> type[aioauth_client.OAuth2Client]: def get_provider(name: str) -> type[aioauth_client.OAuth2Client]:

View File

@@ -97,26 +97,6 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
}, },
}, },
}, },
"aur": {
"type": "dict",
"schema": {
"max_retries": {
"type": "integer",
"coerce": "integer",
"min": 0,
},
"retry_backoff": {
"type": "float",
"coerce": "float",
"min": 0,
},
"timeout": {
"type": "integer",
"coerce": "integer",
"min": 0,
},
},
},
"auth": { "auth": {
"type": "dict", "type": "dict",
"schema": { "schema": {

View File

@@ -60,7 +60,7 @@ class SyncAhrimanClient(SyncHttpClient):
return adapters return adapters
def on_session_creation(self, session: requests.Session) -> None: def start(self, session: requests.Session) -> None:
""" """
method which will be called on session creation method which will be called on session creation

View File

@@ -22,7 +22,7 @@ import sys
from functools import cached_property from functools import cached_property
from requests.adapters import BaseAdapter, HTTPAdapter from requests.adapters import BaseAdapter, HTTPAdapter
from typing import Any, ClassVar, IO, Literal from typing import Any, IO, Literal
from urllib3.util.retry import Retry from urllib3.util.retry import Retry
from ahriman import __version__ from ahriman import __version__
@@ -39,19 +39,11 @@ class SyncHttpClient(LazyLogging):
wrapper around requests library to reduce boilerplate wrapper around requests library to reduce boilerplate
Attributes: Attributes:
DEFAULT_MAX_RETRIES(int): (class attribute) default maximum amount of retries
DEFAULT_RETRY_BACKOFF(float): (class attribute) default retry exponential backoff
DEFAULT_TIMEOUT(int | None): (class attribute) default HTTP request timeout in seconds
auth(tuple[str, str] | None): HTTP basic auth object if set auth(tuple[str, str] | None): HTTP basic auth object if set
retry(Retry): retry policy of the HTTP client
suppress_errors(bool): suppress logging of request errors suppress_errors(bool): suppress logging of request errors
timeout(int | None): HTTP request timeout in seconds timeout(int | None): HTTP request timeout in seconds
""" """
DEFAULT_MAX_RETRIES: ClassVar[int] = 0
DEFAULT_RETRY_BACKOFF: ClassVar[float] = 0.0
DEFAULT_TIMEOUT: ClassVar[int | None] = 30
def __init__(self, configuration: Configuration | None = None, section: str | None = None, *, def __init__(self, configuration: Configuration | None = None, section: str | None = None, *,
suppress_errors: bool = False) -> None: suppress_errors: bool = False) -> None:
""" """
@@ -60,19 +52,26 @@ class SyncHttpClient(LazyLogging):
section(str | None, optional): settings section name (Default value = None) section(str | None, optional): settings section name (Default value = None)
suppress_errors(bool, optional): suppress logging of request errors (Default value = False) suppress_errors(bool, optional): suppress logging of request errors (Default value = False)
""" """
configuration = configuration or Configuration() # dummy configuration if configuration is None:
section = section or configuration.default_section configuration = Configuration() # dummy configuration
if section is None:
section = configuration.default_section
username = configuration.get(section, "username", fallback=None) username = configuration.get(section, "username", fallback=None)
password = configuration.get(section, "password", fallback=None) password = configuration.get(section, "password", fallback=None)
self.auth = (username, password) if username and password else None self.auth = (username, password) if username and password else None
self.timeout: int | None = configuration.getint(section, "timeout", fallback=30)
self.suppress_errors = suppress_errors self.suppress_errors = suppress_errors
self.timeout = configuration.getint(section, "timeout", fallback=self.DEFAULT_TIMEOUT) retries = configuration.getint(section, "max_retries", fallback=0)
self.retry = SyncHttpClient.retry_policy( self.retry = Retry(
max_retries=configuration.getint(section, "max_retries", fallback=self.DEFAULT_MAX_RETRIES), total=retries,
retry_backoff=configuration.getfloat(section, "retry_backoff", fallback=self.DEFAULT_RETRY_BACKOFF), connect=retries,
read=retries,
status=retries,
status_forcelist=[429, 500, 502, 503, 504],
backoff_factor=configuration.getfloat(section, "retry_backoff", fallback=0.0),
) )
@cached_property @cached_property
@@ -93,7 +92,7 @@ class SyncHttpClient(LazyLogging):
f"{requests.utils.default_user_agent()} " \ f"{requests.utils.default_user_agent()} " \
f"python/{python_version}" f"python/{python_version}"
self.on_session_creation(session) self.start(session)
return session return session
@@ -111,27 +110,6 @@ class SyncHttpClient(LazyLogging):
result: str = exception.response.text if exception.response is not None else "" result: str = exception.response.text if exception.response is not None else ""
return result return result
@staticmethod
def retry_policy(max_retries: int = 0, retry_backoff: float = 0.0) -> Retry:
"""
build retry policy for class
Args:
max_retries(int, optional): maximum amount of retries allowed (Default value = 0)
retry_backoff(float, optional): retry exponential backoff (Default value = 0.0)
Returns:
Retry: built retry policy
"""
return Retry(
total=max_retries,
connect=max_retries,
read=max_retries,
status=max_retries,
status_forcelist=[429, 500, 502, 503, 504],
backoff_factor=retry_backoff,
)
def adapters(self) -> dict[str, BaseAdapter]: def adapters(self) -> dict[str, BaseAdapter]:
""" """
get registered adapters get registered adapters
@@ -139,9 +117,10 @@ class SyncHttpClient(LazyLogging):
Returns: Returns:
dict[str, BaseAdapter]: map of protocol and adapter used for this protocol dict[str, BaseAdapter]: map of protocol and adapter used for this protocol
""" """
adapter = HTTPAdapter(max_retries=self.retry)
return { return {
"http://": HTTPAdapter(max_retries=self.retry), "http://": adapter,
"https://": HTTPAdapter(max_retries=self.retry), "https://": adapter,
} }
def make_request(self, method: Literal["DELETE", "GET", "HEAD", "POST", "PUT"], url: str, *, def make_request(self, method: Literal["DELETE", "GET", "HEAD", "POST", "PUT"], url: str, *,
@@ -192,7 +171,7 @@ class SyncHttpClient(LazyLogging):
self.logger.exception("could not perform http request") self.logger.exception("could not perform http request")
raise raise
def on_session_creation(self, session: requests.Session) -> None: def start(self, session: requests.Session) -> None:
""" """
method which will be called on session creation method which will be called on session creation

View File

@@ -21,7 +21,6 @@ from typing import Self
from ahriman.core import _Context, context from ahriman.core import _Context, context
from ahriman.core.alpm.pacman import Pacman from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote import AUR
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite from ahriman.core.database import SQLite
from ahriman.core.repository.executor import Executor from ahriman.core.repository.executor import Executor
@@ -74,24 +73,9 @@ class Repository(Executor, UpdateHandler):
""" """
instance = cls(repository_id, configuration, database, instance = cls(repository_id, configuration, database,
report=report, refresh_pacman_database=refresh_pacman_database) report=report, refresh_pacman_database=refresh_pacman_database)
instance._set_globals(configuration)
instance._set_context() instance._set_context()
return instance return instance
@staticmethod
def _set_globals(configuration: Configuration) -> None:
"""
set global settings based on configuration via class attributes
Args:
configuration(Configuration): configuration instance
"""
AUR.DEFAULT_MAX_RETRIES = configuration.getint("aur", "max_retries", fallback=0)
AUR.DEFAULT_RETRY_BACKOFF = configuration.getfloat("aur", "retry_backoff", fallback=0.0)
AUR.DEFAULT_TIMEOUT = configuration.getint("aur", "timeout", fallback=30)
def _set_context(self) -> None: def _set_context(self) -> None:
""" """
create context variables and set their values create context variables and set their values

View File

@@ -375,7 +375,7 @@ class Package(LazyLogging):
Returns: Returns:
str: print-friendly string str: print-friendly string
""" """
details = "" if self.is_single_package else f" ({" ".join(sorted(self.packages.keys()))})" details = "" if self.is_single_package else f""" ({" ".join(sorted(self.packages.keys()))})"""
return f"{self.base}{details}" return f"{self.base}{details}"
def vercmp(self, version: str) -> int: def vercmp(self, version: str) -> int:

View File

@@ -147,7 +147,7 @@ class PkgbuildPatch:
""" """
if "$" in source: if "$" in source:
# copy from library method with double quotes instead # copy from library method with double quotes instead
return f"\"{source.replace("\"", "'\"'")}\"" return f"""\"{source.replace("\"", "'\"'")}\""""
# otherwise just return normal call # otherwise just return normal call
return shlex.quote(source) return shlex.quote(source)
@@ -195,13 +195,13 @@ class PkgbuildPatch:
""" """
if isinstance(self.value, list): # list like if isinstance(self.value, list): # list like
value = " ".join(map(self.quote, self.value)) value = " ".join(map(self.quote, self.value))
return f"{self.key}=({value})" return f"""{self.key}=({value})"""
if self.is_plain_diff: # no additional logic for plain diffs if self.is_plain_diff: # no additional logic for plain diffs
return self.value return self.value
# we suppose that function values are only supported in string-like values # we suppose that function values are only supported in string-like values
if self.is_function: if self.is_function:
return f"{self.key} {self.value}" # no quoting enabled here return f"{self.key} {self.value}" # no quoting enabled here
return f"{self.key}={self.quote(self.value)}" return f"""{self.key}={self.quote(self.value)}"""
def substitute(self, variables: dict[str, str]) -> str | list[str]: def substitute(self, variables: dict[str, str]) -> str | list[str]:
""" """

View File

@@ -154,7 +154,7 @@ def setup_auth(application: Application, configuration: Configuration, validator
cookie_name="AHRIMAN", cookie_name="AHRIMAN",
max_age=validator.max_age, max_age=validator.max_age,
httponly=True, httponly=True,
samesite="Lax", samesite="Strict",
) )
setup_session(application, storage) setup_session(application, storage)

View File

@@ -30,7 +30,7 @@ def test_login_url(ahriman_client: SyncAhrimanClient) -> None:
assert ahriman_client._login_url().endswith("/api/v1/login") assert ahriman_client._login_url().endswith("/api/v1/login")
def test_on_session_creation(ahriman_client: SyncAhrimanClient, user: User, mocker: MockerFixture) -> None: def test_start(ahriman_client: SyncAhrimanClient, user: User, mocker: MockerFixture) -> None:
""" """
must log in user on start must log in user on start
""" """
@@ -42,17 +42,17 @@ def test_on_session_creation(ahriman_client: SyncAhrimanClient, user: User, mock
} }
session = requests.Session() session = requests.Session()
ahriman_client.on_session_creation(session) ahriman_client.start(session)
requests_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True), json=payload, session=session) requests_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True), json=payload, session=session)
def test_on_session_creation_failed(ahriman_client: SyncAhrimanClient, user: User, mocker: MockerFixture) -> None: def test_start_failed(ahriman_client: SyncAhrimanClient, user: User, mocker: MockerFixture) -> None:
""" """
must suppress any exception happened during session start must suppress any exception happened during session start
""" """
ahriman_client.user = user ahriman_client.user = user
mocker.patch("requests.Session.request", side_effect=Exception) mocker.patch("requests.Session.request", side_effect=Exception)
ahriman_client.on_session_creation(requests.Session()) ahriman_client.start(requests.Session())
def test_start_failed_http_error(ahriman_client: SyncAhrimanClient, user: User, mocker: MockerFixture) -> None: def test_start_failed_http_error(ahriman_client: SyncAhrimanClient, user: User, mocker: MockerFixture) -> None:
@@ -61,7 +61,7 @@ def test_start_failed_http_error(ahriman_client: SyncAhrimanClient, user: User,
""" """
ahriman_client.user = user ahriman_client.user = user
mocker.patch("requests.Session.request", side_effect=requests.HTTPError) mocker.patch("requests.Session.request", side_effect=requests.HTTPError)
ahriman_client.on_session_creation(requests.Session()) ahriman_client.start(requests.Session())
def test_start_skip(ahriman_client: SyncAhrimanClient, mocker: MockerFixture) -> None: def test_start_skip(ahriman_client: SyncAhrimanClient, mocker: MockerFixture) -> None:
@@ -69,5 +69,5 @@ def test_start_skip(ahriman_client: SyncAhrimanClient, mocker: MockerFixture) ->
must skip login if no user set must skip login if no user set
""" """
requests_mock = mocker.patch("requests.Session.request") requests_mock = mocker.patch("requests.Session.request")
ahriman_client.on_session_creation(requests.Session()) ahriman_client.start(requests.Session())
requests_mock.assert_not_called() requests_mock.assert_not_called()

View File

@@ -4,7 +4,6 @@ import requests
from pytest_mock import MockerFixture from pytest_mock import MockerFixture
from unittest.mock import MagicMock, call as MockCall from unittest.mock import MagicMock, call as MockCall
from ahriman.core.alpm.remote import AUR
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.http import SyncHttpClient from ahriman.core.http import SyncHttpClient
@@ -38,29 +37,11 @@ def test_session(mocker: MockerFixture) -> None:
""" """
must generate valid session must generate valid session
""" """
on_session_creation_mock = mocker.patch("ahriman.core.http.sync_http_client.SyncHttpClient.on_session_creation") start_mock = mocker.patch("ahriman.core.http.sync_http_client.SyncHttpClient.start")
session = SyncHttpClient().session session = SyncHttpClient().session
assert "User-Agent" in session.headers assert "User-Agent" in session.headers
on_session_creation_mock.assert_called_once_with(pytest.helpers.anyvar(int)) start_mock.assert_called_once_with(pytest.helpers.anyvar(int))
def test_retry_policy() -> None:
"""
must set retry policy
"""
SyncHttpClient.DEFAULT_MAX_RETRIES = 1
SyncHttpClient.DEFAULT_RETRY_BACKOFF = 2.0
AUR.DEFAULT_MAX_RETRIES = 3
AUR.DEFAULT_RETRY_BACKOFF = 4.0
client = SyncHttpClient()
assert client.retry.connect == 1
assert client.retry.backoff_factor == 2.0
aur = AUR()
assert aur.retry.connect == 3
assert aur.retry.backoff_factor == 4.0
def test_exception_response_text() -> None: def test_exception_response_text() -> None:
@@ -87,11 +68,11 @@ def test_adapters() -> None:
must create adapters with retry policy must create adapters with retry policy
""" """
client = SyncHttpClient() client = SyncHttpClient()
adapters = client.adapters() adapers = client.adapters()
assert "http://" in adapters assert "http://" in adapers
assert "https://" in adapters assert "https://" in adapers
assert all(adapter.max_retries == client.retry for adapter in adapters.values()) assert all(adapter.max_retries == client.retry for adapter in adapers.values())
def test_make_request(mocker: MockerFixture) -> None: def test_make_request(mocker: MockerFixture) -> None:
@@ -194,9 +175,9 @@ def test_make_request_session() -> None:
stream=None, auth=None, timeout=client.timeout) stream=None, auth=None, timeout=client.timeout)
def test_on_session_creation() -> None: def test_start() -> None:
""" """
must do nothing on start must do nothing on start
""" """
client = SyncHttpClient() client = SyncHttpClient()
client.on_session_creation(client.session) client.start(client.session)

View File

@@ -2,7 +2,6 @@ from pytest_mock import MockerFixture
from unittest.mock import call as MockCall from unittest.mock import call as MockCall
from ahriman.core.alpm.pacman import Pacman from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote import AUR
from ahriman.core.configuration import Configuration from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite from ahriman.core.database import SQLite
from ahriman.core.repository import Repository from ahriman.core.repository import Repository
@@ -14,28 +13,13 @@ def test_load(configuration: Configuration, database: SQLite, mocker: MockerFixt
""" """
must correctly load instance must correctly load instance
""" """
globals_mock = mocker.patch("ahriman.core.repository.Repository._set_globals")
context_mock = mocker.patch("ahriman.core.repository.Repository._set_context") context_mock = mocker.patch("ahriman.core.repository.Repository._set_context")
_, repository_id = configuration.check_loaded() _, repository_id = configuration.check_loaded()
Repository.load(repository_id, configuration, database, report=False) Repository.load(repository_id, configuration, database, report=False)
globals_mock.assert_called_once_with(configuration)
context_mock.assert_called_once_with() context_mock.assert_called_once_with()
def test_set_globals(configuration: Configuration) -> None:
"""
must correctly set globals
"""
configuration.set_option("aur", "timeout", "42")
configuration.set_option("aur", "max_retries", "10")
Repository._set_globals(configuration)
aur = AUR()
assert aur.timeout == 42
assert aur.retry.connect == 10
def test_set_context(configuration: Configuration, database: SQLite, mocker: MockerFixture) -> None: def test_set_context(configuration: Configuration, database: SQLite, mocker: MockerFixture) -> None:
""" """
must set context variables must set context variables