arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-07-15 06:55:48 +00:00
Code Issues Projects Releases Wiki Activity

add support of officiall repositories api

Browse Source
This commit is contained in:
Evgenii Alekseev
2022-04-07 04:19:37 +03:00
parent 4990ce4198
commit 9ce1c36f35
22 changed files with 689 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/ahriman/application/handlers/search.py
View File

@ -23,7 +23,8 @@ from dataclasses import fields
from typing import Callable, Iterable, List, Tuple, Type
from ahriman.application.handlers.handler import Handler
from ahriman.core.alpm.aur import AUR
from ahriman.core.alpm.remote.aur import AUR
from ahriman.core.alpm.remote.official import Official
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import InvalidOption
from ahriman.core.formatters.aur_printer import AurPrinter
@ -50,10 +51,14 @@ class Search(Handler):
:param no_report: force disable reporting
:param unsafe: if set no user check will be performed before path creation
"""
packages_list = AUR.multisearch(*args.search)
Search.check_if_empty(args.exit_code, not packages_list)
for package in Search.sort(packages_list, args.sort_by):
AurPrinter(package).print(args.info)
official_packages_list = Official.multisearch(*args.search)
aur_packages_list = AUR.multisearch(*args.search)
Search.check_if_empty(args.exit_code, not official_packages_list and not aur_packages_list)
for packages_list in (official_packages_list, aur_packages_list):
# keep sorting by packages source
for package in Search.sort(packages_list, args.sort_by):
AurPrinter(package).print(args.info)
@staticmethod
def sort(packages: Iterable[AURPackage], sort_by: str) -> List[AURPackage]:

19
src/ahriman/core/alpm/remote/__init__.py Normal file
View File

@ -0,0 +1,19 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#

53
src/ahriman/core/alpm/aur.py → src/ahriman/core/alpm/remote/aur.py
View File

@ -17,24 +17,21 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import annotations
import logging
import requests
from typing import Any, Dict, List, Optional, Type
from typing import Any, Dict, List, Optional
from ahriman.core.alpm.remote.remote import Remote
from ahriman.core.exceptions import InvalidPackageInfo
from ahriman.core.util import exception_response_text
from ahriman.models.aur_package import AURPackage
class AUR:
class AUR(Remote):
"""
AUR RPC wrapper
:cvar DEFAULT_RPC_URL: default AUR RPC url
:cvar DEFAULT_RPC_VERSION: default AUR RPC version
:ivar logger: class logger
:ivar rpc_url: AUR RPC url
:ivar rpc_version: AUR RPC version
"""
@ -48,46 +45,9 @@ class AUR:
:param rpc_url: AUR RPC url
:param rpc_version: AUR RPC version
"""
Remote.__init__(self)
self.rpc_url = rpc_url or self.DEFAULT_RPC_URL
self.rpc_version = rpc_version or self.DEFAULT_RPC_VERSION
self.logger = logging.getLogger("build_details")
@classmethod
def info(cls: Type[AUR], package_name: str) -> AURPackage:
"""
get package info by its name
:param package_name: package name to search
:return: package which match the package name
"""
return cls().package_info(package_name)
@classmethod
def multisearch(cls: Type[AUR], *keywords: str) -> List[AURPackage]:
"""
search in AUR by using API with multiple words. This method is required in order to handle
https://bugs.archlinux.org/task/49133. In addition short words will be dropped
:param keywords: search terms, e.g. "ahriman", "is", "cool"
:return: list of packages each of them matches all search terms
"""
instance = cls()
packages: Dict[str, AURPackage] = {}
for term in filter(lambda word: len(word) > 3, keywords):
portion = instance.search(term)
packages = {
package.package_base: package
for package in portion
if package.package_base in packages or not packages
}
return list(packages.values())
@classmethod
def search(cls: Type[AUR], *keywords: str) -> List[AURPackage]:
"""
search package in AUR web
:param keywords: keywords to search
:return: list of packages which match the criteria
"""
return cls().package_search(*keywords)
@staticmethod
def parse_response(response: Dict[str, Any]) -> List[AURPackage]:
@ -144,11 +104,10 @@ class AUR:
packages = self.make_request("info", package_name)
return next(package for package in packages if package.name == package_name)
def package_search(self, *keywords: str, by: str = "name-desc") -> List[AURPackage]:
def package_search(self, *keywords: str) -> List[AURPackage]:
"""
search package in AUR web
:param keywords: keywords to search
:param by: search by the field
:return: list of packages which match the criteria
"""
return self.make_request("search", *keywords, by=by)
return self.make_request("search", *keywords, by="name-desc")

91
src/ahriman/core/alpm/remote/official.py Normal file
View File

@ -0,0 +1,91 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import requests
from typing import Any, Dict, List, Optional
from ahriman.core.alpm.remote.remote import Remote
from ahriman.core.exceptions import InvalidPackageInfo
from ahriman.core.util import exception_response_text
from ahriman.models.aur_package import AURPackage
class Official(Remote):
"""
official repository RPC wrapper
:cvar DEFAULT_RPC_URL: default AUR RPC url
:ivar rpc_url: AUR RPC url
"""
DEFAULT_RPC_URL = "https://archlinux.org/packages/search/json"
def __init__(self, rpc_url: Optional[str] = None) -> None:
"""
default constructor
:param rpc_url: AUR RPC url
"""
Remote.__init__(self)
self.rpc_url = rpc_url or self.DEFAULT_RPC_URL
@staticmethod
def parse_response(response: Dict[str, Any]) -> List[AURPackage]:
"""
parse RPC response to package list
:param response: RPC response json
:return: list of parsed packages
"""
if not response["valid"]:
raise InvalidPackageInfo("API validation error")
return [AURPackage.from_repo(package) for package in response["results"]]
def make_request(self, *args: str, by: str) -> List[AURPackage]:
"""
perform request to official repositories RPC
:param args: list of arguments to be passed as args query parameter
:param by: search by the field
:return: response parsed to package list
"""
try:
response = requests.get(self.rpc_url, params={by: args})
response.raise_for_status()
return self.parse_response(response.json())
except requests.HTTPError as e:
self.logger.exception("could not perform request: %s", exception_response_text(e))
raise
except Exception:
self.logger.exception("could not perform request")
raise
def package_info(self, package_name: str) -> AURPackage:
"""
get package info by its name
:param package_name: package name to search
:return: package which match the package name
"""
packages = self.make_request(package_name, by="name")
return next(package for package in packages if package.name == package_name)
def package_search(self, *keywords: str) -> List[AURPackage]:
"""
search package in AUR web
:param keywords: keywords to search
:return: list of packages which match the criteria
"""
return self.make_request(*keywords, by="q")

92
src/ahriman/core/alpm/remote/remote.py Normal file
View File

@ -0,0 +1,92 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import annotations
import logging
from typing import Dict, List, Type
from ahriman.models.aur_package import AURPackage
class Remote:
"""
base class for remote package search
:ivar logger: class logger
"""
def __init__(self) -> None:
"""
default constructor
"""
self.logger = logging.getLogger("build_details")
@classmethod
def info(cls: Type[Remote], package_name: str) -> AURPackage:
"""
get package info by its name
:param package_name: package name to search
:return: package which match the package name
"""
return cls().package_info(package_name)
@classmethod
def multisearch(cls: Type[Remote], *keywords: str) -> List[AURPackage]:
"""
search in remote repository by using API with multiple words. This method is required in order to handle
https://bugs.archlinux.org/task/49133. In addition, short words will be dropped
:param keywords: search terms, e.g. "ahriman", "is", "cool"
:return: list of packages each of them matches all search terms
"""
instance = cls()
packages: Dict[str, AURPackage] = {}
for term in filter(lambda word: len(word) > 3, keywords):
portion = instance.search(term)
packages = {
package.name: package # not mistake to group them by name
for package in portion
if package.name in packages or not packages
}
return list(packages.values())
@classmethod
def search(cls: Type[Remote], *keywords: str) -> List[AURPackage]:
"""
search package in AUR web
:param keywords: keywords to search
:return: list of packages which match the criteria
"""
return cls().package_search(*keywords)
def package_info(self, package_name: str) -> AURPackage:
"""
get package info by its name
:param package_name: package name to search
:return: package which match the package name
"""
raise NotImplementedError
def package_search(self, *keywords: str) -> List[AURPackage]:
"""
search package in AUR web
:param keywords: keywords to search
:return: list of packages which match the criteria
"""
raise NotImplementedError

12
src/ahriman/core/util.py
View File

@ -116,6 +116,18 @@ def filter_json(source: Dict[str, Any], known_fields: Iterable[str]) -> Dict[str
return {key: value for key, value in source.items() if key in known_fields and value is not None}
def full_version(epoch: Union[str, int, None], pkgver: str, pkgrel: str) -> str:
"""
generate full version from components
:param epoch: package epoch if any
:param pkgver: package version
:param pkgrel: package release version (arch linux specific)
:return: generated version
"""
prefix = f"{epoch}:" if epoch else ""
return f"{prefix}{pkgver}-{pkgrel}"
def package_like(filename: Path) -> bool:
"""
check if file looks like package

37
src/ahriman/models/aur_package.py
View File

@ -25,7 +25,7 @@ import inflection
from dataclasses import dataclass, field, fields
from typing import Any, Callable, Dict, List, Optional, Type
from ahriman.core.util import filter_json
from ahriman.core.util import filter_json, full_version
@dataclass
@ -59,12 +59,12 @@ class AURPackage:
package_base_id: int
package_base: str
version: str
description: str
num_votes: int
popularity: float
first_submitted: datetime.datetime
last_modified: datetime.datetime
url_path: str
description: str = "" # despite the fact that the field is required some packages don't have it
url: Optional[str] = None
out_of_date: Optional[datetime.datetime] = None
maintainer: Optional[str] = None
@ -88,6 +88,39 @@ class AURPackage:
properties = cls.convert(dump)
return cls(**filter_json(properties, known_fields))
@classmethod
def from_repo(cls: Type[AURPackage], dump: Dict[str, Any]) -> AURPackage:
"""
construct package descriptor from official repository RPC properties
:param dump: json dump body
:return: AUR package descriptor
"""
return cls(
id=0,
name=dump["pkgname"],
package_base_id=0,
package_base=dump["pkgbase"],
version=full_version(dump["epoch"], dump["pkgver"], dump["pkgrel"]),
description=dump["pkgdesc"],
num_votes=0,
popularity=0.0,
first_submitted=datetime.datetime.utcfromtimestamp(0),
last_modified=datetime.datetime.strptime(dump["last_update"], "%Y-%m-%dT%H:%M:%S.%fZ"),
url_path="",
url=dump["url"],
out_of_date=datetime.datetime.strptime(
dump["flag_date"],
"%Y-%m-%dT%H:%M:%S.%fZ") if dump["flag_date"] is not None else None,
maintainer=next(iter(dump["maintainers"]), None),
depends=dump["depends"],
make_depends=dump["makedepends"],
opt_depends=dump["optdepends"],
conflicts=dump["conflicts"],
provides=dump["provides"],
license=dump["licenses"],
keywords=[],
)
@staticmethod
def convert(descriptor: Dict[str, Any]) -> Dict[str, Any]:
"""

36
src/ahriman/models/package.py
View File

@ -26,12 +26,13 @@ from dataclasses import asdict, dataclass
from pathlib import Path
from pyalpm import vercmp # type: ignore
from srcinfo.parse import parse_srcinfo # type: ignore
from typing import Any, Dict, Iterable, List, Optional, Set, Type
from typing import Any, Dict, Iterable, List, Set, Type
from ahriman.core.alpm.aur import AUR
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote.aur import AUR
from ahriman.core.alpm.remote.official import Official
from ahriman.core.exceptions import InvalidPackageInfo
from ahriman.core.util import check_output
from ahriman.core.util import check_output, full_version
from ahriman.models.package_description import PackageDescription
from ahriman.models.package_source import PackageSource
from ahriman.models.repository_paths import RepositoryPaths
@ -144,7 +145,7 @@ class Package:
if errors:
raise InvalidPackageInfo(errors)
packages = {key: PackageDescription() for key in srcinfo["packages"]}
version = cls.full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"])
version = full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"])
return cls(srcinfo["pkgbase"], version, aur_url, packages)
@ -165,6 +166,17 @@ class Package:
aur_url=dump["aur_url"],
packages=packages)
@classmethod
def from_official(cls: Type[Package], name: str, aur_url: str) -> Package:
"""
construct package properties from official repository page
:param name: package name (either base or normal name)
:param aur_url: AUR root url
:return: package properties
"""
package = Official.info(name)
return cls(package.package_base, package.version, aur_url, {package.name: PackageDescription()})
@classmethod
def load(cls: Type[Package], package: str, source: PackageSource, pacman: Pacman, aur_url: str) -> Package:
"""
@ -183,6 +195,8 @@ class Package:
return cls.from_aur(package, aur_url)
if resolved_source == PackageSource.Local:
return cls.from_build(Path(package), aur_url)
if resolved_source == PackageSource.Repository:
return cls.from_official(package, aur_url)
raise InvalidPackageInfo(f"Unsupported local package source {resolved_source}")
except InvalidPackageInfo:
raise
@ -215,18 +229,6 @@ class Package:
full_list = set(depends + makedepends) - packages
return {trim_version(package_name) for package_name in full_list}
@staticmethod
def full_version(epoch: Optional[str], pkgver: str, pkgrel: str) -> str:
"""
generate full version from components
:param epoch: package epoch if any
:param pkgver: package version
:param pkgrel: package release version (arch linux specific)
:return: generated version
"""
prefix = f"{epoch}:" if epoch else ""
return f"{prefix}{pkgver}-{pkgrel}"
def actual_version(self, paths: RepositoryPaths) -> str:
"""
additional method to handle VCS package versions
@ -252,7 +254,7 @@ class Package:
if errors:
raise InvalidPackageInfo(errors)
return self.full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"])
return full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"])
except Exception:
logger.exception("cannot determine version of VCS package, make sure that you have VCS tools installed")

2
src/ahriman/web/views/service/search.py
View File

@ -20,7 +20,7 @@
from aiohttp.web import HTTPNotFound, Response, json_response
from typing import Callable, List
from ahriman.core.alpm.aur import AUR
from ahriman.core.alpm.remote.aur import AUR
from ahriman.models.aur_package import AURPackage
from ahriman.models.user_access import UserAccess
from ahriman.web.views.base import BaseView