arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-07-21 17:59:55 +00:00
Code Issues Projects Releases Wiki Activity

use http client class for all http requests

Browse Source
This commit is contained in:
Evgenii Alekseev
2023-08-23 03:27:42 +03:00
parent 6dfe1b92f2
commit 33e68a59e2
40 changed files with 603 additions and 570 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/ahriman/core/alpm/remote/aur.py
View File

@ -17,14 +17,11 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import requests
from typing import Any
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote import Remote
from ahriman.core.exceptions import PackageInfoError, UnknownPackageError
from ahriman.core.util import exception_response_text
from ahriman.models.aur_package import AURPackage
@ -36,13 +33,11 @@ class AUR(Remote):
DEFAULT_AUR_URL(str): (class attribute) default AUR url
DEFAULT_RPC_URL(str): (class attribute) default AUR RPC url
DEFAULT_RPC_VERSION(str): (class attribute) default AUR RPC version
DEFAULT_TIMEOUT(int): (class attribute) HTTP request timeout in seconds
"""
DEFAULT_AUR_URL = "https://aur.archlinux.org"
DEFAULT_RPC_URL = f"{DEFAULT_AUR_URL}/rpc"
DEFAULT_RPC_VERSION = "5"
DEFAULT_TIMEOUT = 30
@classmethod
def remote_git_url(cls, package_base: str, repository: str) -> str:
@ -91,7 +86,7 @@ class AUR(Remote):
raise PackageInfoError(error_details)
return [AURPackage.from_json(package) for package in response["results"]]
def make_request(self, request_type: str, *args: str, **kwargs: str) -> list[AURPackage]:
def aur_request(self, request_type: str, *args: str, **kwargs: str) -> list[AURPackage]:
"""
perform request to AUR RPC
@ -103,34 +98,20 @@ class AUR(Remote):
Returns:
list[AURPackage]: response parsed to package list
"""
query: dict[str, Any] = {
"type": request_type,
"v": self.DEFAULT_RPC_VERSION
}
query: list[tuple[str, str]] = [
("type", request_type),
("v", self.DEFAULT_RPC_VERSION),
]
arg_query = "arg[]" if len(args) > 1 else "arg"
query[arg_query] = list(args)
for arg in args:
query.append((arg_query, arg))
for key, value in kwargs.items():
query[key] = value
query.append((key, value))
try:
response = requests.get(
self.DEFAULT_RPC_URL,
params=query,
headers={"User-Agent": self.DEFAULT_USER_AGENT},
timeout=self.DEFAULT_TIMEOUT)
response.raise_for_status()
return self.parse_response(response.json())
except requests.HTTPError as e:
self.logger.exception(
"could not perform request by using type %s: %s",
request_type,
exception_response_text(e))
raise
except Exception:
self.logger.exception("could not perform request by using type %s", request_type)
raise
response = self.make_request("GET", self.DEFAULT_RPC_URL, params=query)
return self.parse_response(response.json())
def package_info(self, package_name: str, *, pacman: Pacman) -> AURPackage:
"""
@ -146,7 +127,7 @@ class AUR(Remote):
Raises:
UnknownPackageError: package doesn't exist
"""
packages = self.make_request("info", package_name)
packages = self.aur_request("info", package_name)
try:
return next(package for package in packages if package.name == package_name)
except StopIteration:
@ -163,4 +144,4 @@ class AUR(Remote):
Returns:
list[AURPackage]: list of packages which match the criteria
"""
return self.make_request("search", *keywords, by="name-desc")
return self.aur_request("search", *keywords, by="name-desc")

34
src/ahriman/core/alpm/remote/official.py
View File

@ -17,14 +17,11 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import requests
from typing import Any
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote import Remote
from ahriman.core.exceptions import PackageInfoError, UnknownPackageError
from ahriman.core.util import exception_response_text
from ahriman.models.aur_package import AURPackage
@ -37,14 +34,12 @@ class Official(Remote):
DEFAULT_ARCHLINUX_GIT_URL(str): (class attribute) default url for git packages
DEFAULT_SEARCH_REPOSITORIES(list[str]): (class attribute) default list of repositories to search
DEFAULT_RPC_URL(str): (class attribute) default archlinux repositories RPC url
DEFAULT_TIMEOUT(int): (class attribute) HTTP request timeout in seconds
"""
DEFAULT_ARCHLINUX_GIT_URL = "https://gitlab.archlinux.org"
DEFAULT_ARCHLINUX_URL = "https://archlinux.org"
DEFAULT_SEARCH_REPOSITORIES = ["Core", "Extra", "Multilib"]
DEFAULT_RPC_URL = "https://archlinux.org/packages/search/json"
DEFAULT_TIMEOUT = 30
@classmethod
def remote_git_url(cls, package_base: str, repository: str) -> str:
@ -91,7 +86,7 @@ class Official(Remote):
raise PackageInfoError("API validation error")
return [AURPackage.from_repo(package) for package in response["results"]]
def make_request(self, *args: str, by: str) -> list[AURPackage]:
def arch_request(self, *args: str, by: str) -> list[AURPackage]:
"""
perform request to official repositories RPC
@ -102,20 +97,15 @@ class Official(Remote):
Returns:
list[AURPackage]: response parsed to package list
"""
try:
response = requests.get(
self.DEFAULT_RPC_URL,
params={by: args, "repo": self.DEFAULT_SEARCH_REPOSITORIES},
headers={"User-Agent": self.DEFAULT_USER_AGENT},
timeout=self.DEFAULT_TIMEOUT)
response.raise_for_status()
return self.parse_response(response.json())
except requests.HTTPError as e:
self.logger.exception("could not perform request: %s", exception_response_text(e))
raise
except Exception:
self.logger.exception("could not perform request")
raise
query: list[tuple[str, str]] = [
("repo", repository)
for repository in self.DEFAULT_SEARCH_REPOSITORIES
]
for arg in args:
query.append((by, arg))
response = self.make_request("GET", self.DEFAULT_RPC_URL, params=query)
return self.parse_response(response.json())
def package_info(self, package_name: str, *, pacman: Pacman) -> AURPackage:
"""
@ -131,7 +121,7 @@ class Official(Remote):
Raises:
UnknownPackageError: package doesn't exist
"""
packages = self.make_request(package_name, by="name")
packages = self.arch_request(package_name, by="name")
try:
return next(package for package in packages if package.name == package_name)
except StopIteration:
@ -148,4 +138,4 @@ class Official(Remote):
Returns:
list[AURPackage]: list of packages which match the criteria
"""
return self.make_request(*keywords, by="q")
return self.arch_request(*keywords, by="q")

10
src/ahriman/core/alpm/remote/remote.py
View File

@ -17,19 +17,15 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman import __version__
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.log import LazyLogging
from ahriman.core.http import SyncHttpClient
from ahriman.models.aur_package import AURPackage
class Remote(LazyLogging):
class Remote(SyncHttpClient):
"""
base class for remote package search
Attributes:
DEFAULT_USER_AGENT(str): (class attribute) default user agent
Examples:
These classes are designed to be used without instancing. In order to achieve it several class methods are
provided: ``info``, ``multisearch`` and ``search``. Thus, the basic flow is the following::
@ -43,8 +39,6 @@ class Remote(LazyLogging):
directly, whereas ``multisearch`` splits search one by one and finds intersection between search results.
"""
DEFAULT_USER_AGENT = f"ahriman/{__version__}"
@classmethod
def info(cls, package_name: str, *, pacman: Pacman) -> AURPackage:
"""

20
src/ahriman/core/http/__init__.py Normal file
View File

@ -0,0 +1,20 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.http.sync_http_client import MultipartType, SyncHttpClient

137
src/ahriman/core/http/sync_http_client.py Normal file
View File

@ -0,0 +1,137 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import requests
from functools import cached_property
from typing import Any, IO, Literal
from ahriman import __version__
from ahriman.core.configuration import Configuration
from ahriman.core.log import LazyLogging
# filename, file, content-type, headers
MultipartType = tuple[str, IO[bytes], str, dict[str, str]]
class SyncHttpClient(LazyLogging):
"""
wrapper around requests library to reduce boilerplate
Attributes:
auth(tuple[str, str] | None): HTTP basic auth object if set
suppress_errors(bool): suppress logging of request errors
timeout(int): HTTP request timeout in seconds
"""
def __init__(self, section: str | None = None, configuration: Configuration | None = None, *,
suppress_errors: bool = False) -> None:
"""
default constructor
Args:
section(str, optional): settings section name (Default value = None)
configuration(Configuration | None): configuration instance (Default value = None)
suppress_errors(bool, optional): suppress logging of request errors (Default value = False)
"""
if configuration is None:
configuration = Configuration() # dummy configuration
if section is None:
section = configuration.default_section
username = configuration.get(section, "username", fallback=None)
password = configuration.get(section, "password", fallback=None)
self.auth = (username, password) if username and password else None
self.timeout = configuration.getint(section, "timeout", fallback=30)
self.suppress_errors = suppress_errors
@cached_property
def session(self) -> requests.Session:
"""
get or create session
Returns:
request.Session: created session object
"""
session = requests.Session()
session.headers["User-Agent"] = f"ahriman/{__version__}"
return session
@staticmethod
def exception_response_text(exception: requests.exceptions.RequestException) -> str:
"""
safe response exception text generation
Args:
exception(requests.exceptions.RequestException): exception raised
Returns:
str: text of the response if it is not None and empty string otherwise
"""
result: str = exception.response.text if exception.response is not None else ""
return result
def make_request(self, method: Literal["DELETE", "GET", "POST", "PUT"], url: str, *,
headers: dict[str, str] | None = None,
params: list[tuple[str, str]] | None = None,
data: Any | None = None,
json: dict[str, Any] | None = None,
files: dict[str, MultipartType] | None = None,
session: requests.Session | None = None,
suppress_errors: bool | None = None) -> requests.Response:
"""
perform request with specified parameters
Args:
method(Literal["DELETE", "GET", "POST", "PUT"]): HTTP method to call
url(str): remote url to call
headers(dict[str, str] | None, optional): request headers (Default value = None)
params(list[tuple[str, str]] | None, optional): request query parameters (Default value = None)
data(Any | None, optional): request raw data parameters (Default value = None)
json(dict[str, Any] | None, optional): request json parameters (Default value = None)
files(dict[str, MultipartType] | None, optional): multipart upload (Default value = None)
session(requests.Session | None, optional): session object if any (Default value = None)
suppress_errors(bool | None, optional): suppress logging errors (e.g. if no web server available). If none
set, the instance-wide value will be used (Default value = None)
Returns:
requests.Response: response object
"""
# defaults
if suppress_errors is None:
suppress_errors = self.suppress_errors
if session is None:
session = self.session
try:
response = session.request(method, url, params=params, data=data, headers=headers, files=files, json=json,
auth=self.auth, timeout=self.timeout)
response.raise_for_status()
return response
except requests.HTTPError as ex:
if not suppress_errors:
self.logger.exception("could not perform http request: %s", self.exception_response_text(ex))
raise
except Exception:
if not suppress_errors:
self.logger.exception("could not perform http request")
raise

5
src/ahriman/core/report/remote_call.py
View File

@ -81,8 +81,9 @@ class RemoteCall(Report):
"""
try:
response = self.client.make_request("GET", f"/api/v1/service/process/{process_id}")
except requests.RequestException as e:
if e.response is not None and e.response.status_code == 404:
except requests.HTTPError as ex:
status_code = ex.response.status_code if ex.response is not None else None
if status_code == 404:
return False
raise

23
src/ahriman/core/report/telegram.py
View File

@ -17,17 +17,15 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import requests # technically we could use python-telegram-bot, but it is just a single request, c'mon
from ahriman.core.configuration import Configuration
from ahriman.core.http import SyncHttpClient
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.report.report import Report
from ahriman.core.util import exception_response_text
from ahriman.models.package import Package
from ahriman.models.result import Result
class Telegram(Report, JinjaTemplate):
class Telegram(Report, JinjaTemplate, SyncHttpClient):
"""
telegram report generator
@ -38,7 +36,6 @@ class Telegram(Report, JinjaTemplate):
chat_id(str): chat id to post message, either string with @ or integer
template_path(Path): path to template for built packages
template_type(str): template message type to be used in parse mode, one of MarkdownV2, HTML, Markdown
timeout(int): HTTP request timeout in seconds
"""
TELEGRAM_API_URL = "https://api.telegram.org"
@ -55,12 +52,12 @@ class Telegram(Report, JinjaTemplate):
"""
Report.__init__(self, architecture, configuration)
JinjaTemplate.__init__(self, section, configuration)
SyncHttpClient.__init__(self, section, configuration)
self.api_key = configuration.get(section, "api_key")
self.chat_id = configuration.get(section, "chat_id")
self.template_path = configuration.getpath(section, "template_path")
self.template_type = configuration.get(section, "template_type", fallback="HTML")
self.timeout = configuration.getint(section, "timeout", fallback=30)
def _send(self, text: str) -> None:
"""
@ -69,18 +66,8 @@ class Telegram(Report, JinjaTemplate):
Args:
text(str): message body text
"""
try:
response = requests.post(
f"{self.TELEGRAM_API_URL}/bot{self.api_key}/sendMessage",
data={"chat_id": self.chat_id, "text": text, "parse_mode": self.template_type},
timeout=self.timeout)
response.raise_for_status()
except requests.HTTPError as e:
self.logger.exception("could not perform request: %s", exception_response_text(e))
raise
except Exception:
self.logger.exception("could not perform request")
raise
self.make_request("POST", f"{self.TELEGRAM_API_URL}/bot{self.api_key}/sendMessage",
data={"chat_id": self.chat_id, "text": text, "parse_mode": self.template_type})
def generate(self, packages: list[Package], result: Result) -> None:
"""

26
src/ahriman/core/sign/gpg.py
View File

@ -17,30 +17,26 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import requests
from pathlib import Path
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import BuildError
from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output, exception_response_text
from ahriman.core.http import SyncHttpClient
from ahriman.core.util import check_output
from ahriman.models.sign_settings import SignSettings
class GPG(LazyLogging):
class GPG(SyncHttpClient):
"""
gnupg wrapper
Attributes:
DEFAULT_TIMEOUT(int): (class attribute) HTTP request timeout in seconds
configuration(Configuration): configuration instance
default_key(str | None): default PGP key ID to use
targets(set[SignSettings]): list of targets to sign (repository, package etc)
"""
_check_output = check_output
DEFAULT_TIMEOUT = 30
def __init__(self, configuration: Configuration) -> None:
"""
@ -49,6 +45,7 @@ class GPG(LazyLogging):
Args:
configuration(Configuration): configuration instance
"""
SyncHttpClient.__init__(self)
self.configuration = configuration
self.targets, self.default_key = self.sign_options(configuration)
@ -126,16 +123,11 @@ class GPG(LazyLogging):
str: key as plain text
"""
key = key if key.startswith("0x") else f"0x{key}"
try:
response = requests.get(f"https://{server}/pks/lookup", params={
"op": "get",
"options": "mr",
"search": key
}, timeout=self.DEFAULT_TIMEOUT)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
self.logger.exception("could not download key %s from %s: %s", key, server, exception_response_text(e))
raise
response = self.make_request("GET", f"https://{server}/pks/lookup", params=[
("op", "get"),
("options", "mr"),
("search", key),
])
return response.text
def key_export(self, key: str) -> str:

137
src/ahriman/core/status/web_client.py
View File

@ -22,39 +22,27 @@ import logging
import requests
from functools import cached_property
from typing import Any, IO, Literal
from urllib.parse import quote_plus as urlencode
from ahriman import __version__
from ahriman.core.configuration import Configuration
from ahriman.core.log import LazyLogging
from ahriman.core.http import SyncHttpClient
from ahriman.core.status.client import Client
from ahriman.core.util import exception_response_text
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.internal_status import InternalStatus
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
from ahriman.models.user import User
# filename, file, content-type, headers
MultipartType = tuple[str, IO[bytes], str, dict[str, str]]
class WebClient(Client, LazyLogging):
class WebClient(Client, SyncHttpClient):
"""
build status reporter web client
Attributes:
address(str): address of the web service
suppress_errors(bool): suppress logging errors (e.g. if no web server available)
user(User | None): web service user descriptor
use_unix_socket(bool): use websocket or not
"""
_login_url = "/api/v1/login"
_status_url = "/api/v1/status"
def __init__(self, configuration: Configuration) -> None:
"""
default constructor
@ -62,11 +50,10 @@ class WebClient(Client, LazyLogging):
Args:
configuration(Configuration): configuration instance
"""
suppress_errors = configuration.getboolean("settings", "suppress_http_log_errors", fallback=False)
SyncHttpClient.__init__(self, "web", configuration, suppress_errors=suppress_errors)
self.address, self.use_unix_socket = self.parse_address(configuration)
self.user = User.from_option(
configuration.get("web", "username", fallback=None),
configuration.get("web", "password", fallback=None))
self.suppress_errors = configuration.getboolean("settings", "suppress_http_log_errors", fallback=False)
@cached_property
def session(self) -> requests.Session:
@ -78,34 +65,6 @@ class WebClient(Client, LazyLogging):
"""
return self._create_session(use_unix_socket=self.use_unix_socket)
@staticmethod
def _logs_url(package_base: str) -> str:
"""
get url for the logs api
Args:
package_base(str): package base
Returns:
str: full url for web service for logs
"""
return f"/api/v1/packages/{package_base}/logs"
@staticmethod
def _package_url(package_base: str = "") -> str:
"""
url generator
Args:
package_base(str, optional): package base to generate url (Default value = "")
Returns:
str: full url of web service for specific package base
"""
# in case if unix socket is used we need to normalize url
suffix = f"/{package_base}" if package_base else ""
return f"/api/v1/packages{suffix}"
@staticmethod
def parse_address(configuration: Configuration) -> tuple[str, bool]:
"""
@ -157,56 +116,60 @@ class WebClient(Client, LazyLogging):
Args:
session(requests.Session): request session to login
"""
if self.user is None:
if self.auth is None:
return # no auth configured
username, password = self.auth
payload = {
"username": self.user.username,
"password": self.user.password
"username": username,
"password": password,
}
with contextlib.suppress(Exception):
self.make_request("POST", self._login_url, json=payload, session=session)
self.make_request("POST", self._login_url(), json=payload, session=session)
def make_request(self, method: Literal["DELETE", "GET", "POST"], url: str, *,
params: list[tuple[str, str]] | None = None,
json: dict[str, Any] | None = None,
files: dict[str, MultipartType] | None = None,
session: requests.Session | None = None,
suppress_errors: bool | None = None) -> requests.Response:
def _login_url(self) -> str:
"""
perform request with specified parameters
Args:
method(Literal["DELETE", "GET", "POST"]): HTTP method to call
url(str): remote url to call
params(list[tuple[str, str]] | None, optional): request query parameters (Default value = None)
json(dict[str, Any] | None, optional): request json parameters (Default value = None)
files(dict[str, MultipartType] | None, optional): multipart upload (Default value = None)
session(requests.Session | None, optional): session object if any (Default value = None)
suppress_errors(bool | None, optional): suppress logging errors (e.g. if no web server available). If none
set, the instance-wide value will be used (Default value = None)
get url for the login api
Returns:
requests.Response: response object
str: full url for web service to log in
"""
# defaults
if suppress_errors is None:
suppress_errors = self.suppress_errors
if session is None:
session = self.session
return f"{self.address}/api/v1/login"
try:
response = session.request(method, f"{self.address}{url}", params=params, json=json, files=files)
response.raise_for_status()
return response
except requests.RequestException as e:
if not suppress_errors:
self.logger.exception("could not perform http request: %s", exception_response_text(e))
raise
except Exception:
if not suppress_errors:
self.logger.exception("could not perform http request")
raise
def _logs_url(self, package_base: str) -> str:
"""
get url for the logs api
Args:
package_base(str): package base
Returns:
str: full url for web service for logs
"""
return f"{self.address}/api/v1/packages/{package_base}/logs"
def _package_url(self, package_base: str = "") -> str:
"""
url generator
Args:
package_base(str, optional): package base to generate url (Default value = "")
Returns:
str: full url of web service for specific package base
"""
# in case if unix socket is used we need to normalize url
suffix = f"/{package_base}" if package_base else ""
return f"{self.address}/api/v1/packages{suffix}"
def _status_url(self) -> str:
"""
get url for the status api
Returns:
str: full url for web service for status
"""
return f"{self.address}/api/v1/status"
def package_add(self, package: Package, status: BuildStatusEnum) -> None:
"""
@ -293,7 +256,7 @@ class WebClient(Client, LazyLogging):
InternalStatus: current internal (web) service status
"""
with contextlib.suppress(Exception):
response = self.make_request("GET", self._status_url)
response = self.make_request("GET", self._status_url())
response_json = response.json()
return InternalStatus.from_json(response_json)
@ -309,4 +272,4 @@ class WebClient(Client, LazyLogging):
"""
payload = {"status": status.value}
with contextlib.suppress(Exception):
self.make_request("POST", self._status_url, json=payload)
self.make_request("POST", self._status_url(), json=payload)

14
src/ahriman/core/upload/github.py
View File

@ -61,7 +61,7 @@ class Github(HttpUpload):
"""
try:
asset = next(asset for asset in release["assets"] if asset["name"] == name)
self._request("DELETE", asset["url"])
self.make_request("DELETE", asset["url"])
except StopIteration:
self.logger.info("no asset %s found in release %s", name, release["name"])
@ -81,7 +81,7 @@ class Github(HttpUpload):
headers = {"Content-Type": mime} if mime is not None else {"Content-Type": "application/octet-stream"}
with path.open("rb") as archive:
self._request("POST", url, params={"name": path.name}, data=archive, headers=headers)
self.make_request("POST", url, params=[("name", path.name)], data=archive, headers=headers)
def get_local_files(self, path: Path) -> dict[Path, str]:
"""
@ -136,7 +136,7 @@ class Github(HttpUpload):
dict[str, Any]: github API release object for the new release
"""
url = f"https://api.github.com/repos/{self.github_owner}/{self.github_repository}/releases"
response = self._request("POST", url, json={"tag_name": self.architecture, "name": self.architecture})
response = self.make_request("POST", url, json={"tag_name": self.architecture, "name": self.architecture})
release: dict[str, Any] = response.json()
return release
@ -149,11 +149,11 @@ class Github(HttpUpload):
"""
url = f"https://api.github.com/repos/{self.github_owner}/{self.github_repository}/releases/tags/{self.architecture}"
try:
response = self._request("GET", url)
response = self.make_request("GET", url)
release: dict[str, Any] = response.json()
return release
except requests.HTTPError as e:
status_code = e.response.status_code if e.response is not None else None
except requests.HTTPError as ex:
status_code = ex.response.status_code if ex.response is not None else None
if status_code == 404:
return None
raise
@ -166,7 +166,7 @@ class Github(HttpUpload):
release(dict[str, Any]): release object
body(str): new release body
"""
self._request("POST", release["url"], json={"body": body})
self.make_request("POST", release["url"], json={"body": body})
def sync(self, path: Path, built_packages: list[Package]) -> None:
"""

46
src/ahriman/core/upload/http_upload.py
View File

@ -18,24 +18,17 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import hashlib
import requests
from functools import cached_property
from pathlib import Path
from typing import Any
from ahriman.core.configuration import Configuration
from ahriman.core.http import SyncHttpClient
from ahriman.core.upload.upload import Upload
from ahriman.core.util import exception_response_text
class HttpUpload(Upload):
class HttpUpload(Upload, SyncHttpClient):
"""
helper for the http based uploads
Attributes:
auth(tuple[str, str] | None): HTTP auth object if set
timeout(int): HTTP request timeout in seconds
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
@ -48,20 +41,7 @@ class HttpUpload(Upload):
section(str): configuration section name
"""
Upload.__init__(self, architecture, configuration)
password = configuration.get(section, "password", fallback=None)
username = configuration.get(section, "username", fallback=None)
self.auth = (password, username) if password and username else None
self.timeout = configuration.getint(section, "timeout", fallback=30)
@cached_property
def session(self) -> requests.Session:
"""
get or create session
Returns:
request.Session: created session object
"""
return requests.Session()
SyncHttpClient.__init__(self, section, configuration)
@staticmethod
def calculate_hash(path: Path) -> str:
@ -107,23 +87,3 @@ class HttpUpload(Upload):
file, md5 = line.split()
files[file] = md5
return files
def _request(self, method: str, url: str, **kwargs: Any) -> requests.Response:
"""
request wrapper
Args:
method(str): request method
url(str): request url
**kwargs(Any): request parameters to be passed as is
Returns:
requests.Response: request response object
"""
try:
response = self.session.request(method, url, auth=self.auth, timeout=self.timeout, **kwargs)
response.raise_for_status()
except requests.HTTPError as e:
self.logger.exception("could not perform %s request to %s: %s", method, url, exception_response_text(e))
raise
return response

5
src/ahriman/core/upload/remote_service.py
View File

@ -23,8 +23,9 @@ from functools import cached_property
from pathlib import Path
from ahriman.core.configuration import Configuration
from ahriman.core.http import MultipartType
from ahriman.core.sign.gpg import GPG
from ahriman.core.status.web_client import MultipartType, WebClient
from ahriman.core.status.web_client import WebClient
from ahriman.core.upload.http_upload import HttpUpload
from ahriman.models.package import Package
@ -77,7 +78,7 @@ class RemoteService(HttpUpload):
if signature_path is not None:
files["signature"] = signature_path.name, signature_path.open("rb"), "application/octet-stream", {}
self._request("POST", f"{self.client.address}/api/v1/service/upload", files=files)
self.make_request("POST", f"{self.client.address}/api/v1/service/upload", files=files)
finally:
for _, fd, _, _ in files.values():
fd.close()

4
src/ahriman/core/upload/upload.py
View File

@ -51,8 +51,8 @@ class Upload(LazyLogging):
>>> try:
>>> upload.sync(configuration.repository_paths.repository, [])
>>> except Exception as exception:
>>> handle_exceptions(exception)
>>> except Exception as ex:
>>> handle_exceptions(ex)
"""
def __init__(self, architecture: str, configuration: Configuration) -> None:

16
src/ahriman/core/util.py
View File

@ -24,7 +24,6 @@ import itertools
import logging
import os
import re
import requests
import selectors
import subprocess
@ -44,7 +43,6 @@ __all__ = [
"check_user",
"dataclass_view",
"enum_values",
"exception_response_text",
"extract_user",
"filter_json",
"full_version",
@ -214,20 +212,6 @@ def enum_values(enum: type[Enum]) -> list[str]:
return [str(key.value) for key in enum] # explicit str conversion for typing
def exception_response_text(exception: requests.exceptions.RequestException) -> str:
"""
safe response exception text generation
Args:
exception(requests.exceptions.RequestException): exception raised
Returns:
str: text of the response if it is not None and empty string otherwise
"""
result: str = exception.response.text if exception.response is not None else ""
return result
def extract_user() -> str | None:
"""
extract user from system environment

18
src/ahriman/models/user.py
View File

@ -75,24 +75,6 @@ class User:
object.__setattr__(self, "packager_id", self.packager_id or None)
object.__setattr__(self, "key", self.key or None)
@classmethod
def from_option(cls, username: str | None, password: str | None,
access: UserAccess = UserAccess.Read) -> Self | None:
"""
build user descriptor from configuration options
Args:
username(str | None): username
password(str | None): password as string
access(UserAccess, optional): optional user access (Default value = UserAccess.Read)
Returns:
Self | None: generated user descriptor if all options are supplied and None otherwise
"""
if username is None or password is None:
return None
return cls(username=username, password=password, access=access, packager_id=None, key=None)
@staticmethod
def generate_password(length: int) -> str:
"""

34
src/ahriman/web/middlewares/exception_handler.py
View File

@ -60,31 +60,31 @@ def exception_handler(logger: logging.Logger) -> MiddlewareType:
async def handle(request: Request, handler: HandlerType) -> StreamResponse:
try:
return await handler(request)
except HTTPUnauthorized as e:
except HTTPUnauthorized as ex:
if _is_templated_unauthorized(request):
context = {"code": e.status_code, "reason": e.reason}
return aiohttp_jinja2.render_template("error.jinja2", request, context, status=e.status_code)
return json_response(data={"error": e.reason}, status=e.status_code)
except HTTPMethodNotAllowed as e:
if e.method == "OPTIONS":
context = {"code": ex.status_code, "reason": ex.reason}
return aiohttp_jinja2.render_template("error.jinja2", request, context, status=ex.status_code)
return json_response(data={"error": ex.reason}, status=ex.status_code)
except HTTPMethodNotAllowed as ex:
if ex.method == "OPTIONS":
# automatically handle OPTIONS method, idea comes from
# https://github.com/arcan1s/ffxivbis/blob/master/src/main/scala/me/arcanis/ffxivbis/http/api/v1/HttpHandler.scala#L32
raise HTTPNoContent(headers={"Allow": ",".join(sorted(e.allowed_methods))})
if e.method == "HEAD":
raise HTTPNoContent(headers={"Allow": ",".join(sorted(ex.allowed_methods))})
if ex.method == "HEAD":
# since we have special autogenerated HEAD method, we need to remove it from list of available
e.allowed_methods = {method for method in e.allowed_methods if method != "HEAD"}
e.headers["Allow"] = ",".join(sorted(e.allowed_methods))
raise e
ex.allowed_methods = {method for method in ex.allowed_methods if method != "HEAD"}
ex.headers["Allow"] = ",".join(sorted(ex.allowed_methods))
raise ex
raise
except HTTPClientError as e:
return json_response(data={"error": e.reason}, status=e.status_code)
except HTTPServerError as e:
except HTTPClientError as ex:
return json_response(data={"error": ex.reason}, status=ex.status_code)
except HTTPServerError as ex:
logger.exception("server exception during performing request to %s", request.path)
return json_response(data={"error": e.reason}, status=e.status_code)
return json_response(data={"error": ex.reason}, status=ex.status_code)
except HTTPException: # just raise 2xx and 3xx codes
raise
except Exception as e:
except Exception as ex:
logger.exception("unknown exception during performing request to %s", request.path)
return json_response(data={"error": str(e)}, status=500)
return json_response(data={"error": str(ex)}, status=500)
return handle

4
src/ahriman/web/views/service/add.py
View File

@ -64,8 +64,8 @@ class AddView(BaseView):
try:
data = await self.extract_data(["packages"])
packages = self.get_non_empty(lambda key: [package for package in data[key] if package], "packages")
except Exception as e:
raise HTTPBadRequest(reason=str(e))
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
username = await self.username()
process_id = self.spawner.packages_add(packages, username, now=True)

8
src/ahriman/web/views/service/pgp.py
View File

@ -68,8 +68,8 @@ class PGPView(BaseView):
try:
key = self.get_non_empty(self.request.query.getone, "key")
server = self.get_non_empty(self.request.query.getone, "server")
except Exception as e:
raise HTTPBadRequest(reason=str(e))
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
try:
key = self.service.repository.sign.key_download(server, key)
@ -107,8 +107,8 @@ class PGPView(BaseView):
try:
key = self.get_non_empty(data.get, "key")
except Exception as e:
raise HTTPBadRequest(reason=str(e))
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
process_id = self.spawner.key_import(key, data.get("server"))

4
src/ahriman/web/views/service/rebuild.py
View File

@ -65,8 +65,8 @@ class RebuildView(BaseView):
data = await self.extract_data(["packages"])
packages = self.get_non_empty(lambda key: [package for package in data[key] if package], "packages")
depends_on = next(iter(packages))
except Exception as e:
raise HTTPBadRequest(reason=str(e))
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
username = await self.username()
process_id = self.spawner.packages_rebuild(depends_on, username)

4
src/ahriman/web/views/service/remove.py
View File

@ -64,8 +64,8 @@ class RemoveView(BaseView):
try:
data = await self.extract_data(["packages"])
packages = self.get_non_empty(lambda key: [package for package in data[key] if package], "packages")
except Exception as e:
raise HTTPBadRequest(reason=str(e))
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
process_id = self.spawner.packages_remove(packages)

4
src/ahriman/web/views/service/request.py
View File

@ -64,8 +64,8 @@ class RequestView(BaseView):
try:
data = await self.extract_data(["packages"])
packages = self.get_non_empty(lambda key: [package for package in data[key] if package], "packages")
except Exception as e:
raise HTTPBadRequest(reason=str(e))
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
username = await self.username()
process_id = self.spawner.packages_add(packages, username, now=False)

4
src/ahriman/web/views/service/search.py
View File

@ -69,8 +69,8 @@ class SearchView(BaseView):
try:
search: list[str] = self.get_non_empty(lambda key: self.request.query.getall(key, default=[]), "for")
packages = AUR.multisearch(*search, pacman=self.service.repository.pacman)
except Exception as e:
raise HTTPBadRequest(reason=str(e))
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
if not packages:
raise HTTPNotFound(reason=f"No packages found for terms: {search}")

4
src/ahriman/web/views/service/update.py
View File

@ -63,8 +63,8 @@ class UpdateView(BaseView):
"""
try:
data = await self.extract_data()
except Exception as e:
raise HTTPBadRequest(reason=str(e))
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
username = await self.username()
process_id = self.spawner.packages_update(

4
src/ahriman/web/views/service/upload.py
View File

@ -120,8 +120,8 @@ class UploadView(BaseView):
try:
reader = await self.request.multipart()
except Exception as e:
raise HTTPBadRequest(reason=str(e))
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
max_body_size = self.configuration.getint("web", "max_body_size", fallback=None)
target = self.configuration.repository_paths.packages

4
src/ahriman/web/views/status/logs.py
View File

@ -138,8 +138,8 @@ class LogsView(BaseView):
created = data["created"]
record = data["message"]
version = data["version"]
except Exception as e:
raise HTTPBadRequest(reason=str(e))
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
self.service.logs_update(LogRecordId(package_base, version), created, record)

4
src/ahriman/web/views/status/package.py
View File

@ -138,8 +138,8 @@ class PackageView(BaseView):
try:
package = Package.from_json(data["package"]) if "package" in data else None
status = BuildStatusEnum(data["status"])
except Exception as e:
raise HTTPBadRequest(reason=str(e))
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
try:
self.service.package_update(package_base, status, package)

4
src/ahriman/web/views/status/status.py
View File

@ -99,8 +99,8 @@ class StatusView(BaseView):
try:
data = await self.extract_data()
status = BuildStatusEnum(data["status"])
except Exception as e:
raise HTTPBadRequest(reason=str(e))
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
self.service.status_update(status)