Complete official repository support (#59)

This commit is contained in:
2022-05-03 00:49:32 +03:00
committed by GitHub
parent 5030395025
commit 571f720ae2
68 changed files with 1206 additions and 407 deletions

View File

@ -80,12 +80,13 @@ class ApplicationPackages(ApplicationProperties):
known_packages(Set[str]): list of packages which are known by the service
without_dependencies(bool): if set, dependency check will be disabled
"""
package = Package.load(source, PackageSource.AUR, self.repository.pacman, self.repository.aur_url)
package = Package.from_aur(source, self.repository.pacman)
self.database.build_queue_insert(package)
self.database.remote_update(package)
with tmpdir() as local_path:
Sources.load(local_path, package.git_url, self.database.patches_get(package.base))
Sources.load(local_path, package.remote, self.database.patches_get(package.base))
self._process_dependencies(local_path, known_packages, without_dependencies)
def _add_directory(self, source: str, *_: Any) -> None:
@ -108,9 +109,10 @@ class ApplicationPackages(ApplicationProperties):
known_packages(Set[str]): list of packages which are known by the service
without_dependencies(bool): if set, dependency check will be disabled
"""
package = Package.load(source, PackageSource.Local, self.repository.pacman, self.repository.aur_url)
source_dir = Path(source)
package = Package.from_build(source_dir)
cache_dir = self.repository.paths.cache_for(package.base)
shutil.copytree(Path(source), cache_dir) # copy package to store in caches
shutil.copytree(source_dir, cache_dir) # copy package to store in caches
Sources.init(cache_dir) # we need to run init command in directory where we do have permissions
self.database.build_queue_insert(package)
@ -132,6 +134,18 @@ class ApplicationPackages(ApplicationProperties):
for chunk in response.iter_content(chunk_size=1024):
local_file.write(chunk)
def _add_repository(self, source: str, *_: Any) -> None:
"""
add package from official repository
Args:
source(str): package base name
"""
package = Package.from_official(source, self.repository.pacman)
self.database.build_queue_insert(package)
self.database.remote_update(package)
# repository packages must not depend on unknown packages, thus we are not going to process dependencies
def _process_dependencies(self, local_path: Path, known_packages: Set[str], without_dependencies: bool) -> None:
"""
process package dependencies

View File

@ -128,14 +128,14 @@ class ApplicationRepository(ApplicationProperties):
packages: List[str] = []
for single in probe.packages:
try:
_ = Package.from_aur(single, probe.aur_url)
_ = Package.from_aur(single, self.repository.pacman)
except Exception:
packages.append(single)
return packages
def unknown_local(probe: Package) -> List[str]:
cache_dir = self.repository.paths.cache_for(probe.base)
local = Package.from_build(cache_dir, probe.aur_url)
local = Package.from_build(cache_dir)
packages = set(probe.packages.keys()).difference(local.packages.keys())
return list(packages)

View File

@ -29,7 +29,6 @@ from ahriman.core.configuration import Configuration
from ahriman.core.formatters.string_printer import StringPrinter
from ahriman.models.action import Action
from ahriman.models.package import Package
from ahriman.models.package_source import PackageSource
class Patch(Handler):
@ -57,21 +56,20 @@ class Patch(Handler):
elif args.action == Action.Remove:
Patch.patch_set_remove(application, args.package)
elif args.action == Action.Update:
Patch.patch_set_create(application, args.package, args.track)
Patch.patch_set_create(application, Path(args.package), args.track)
@staticmethod
def patch_set_create(application: Application, sources_dir: str, track: List[str]) -> None:
def patch_set_create(application: Application, sources_dir: Path, track: List[str]) -> None:
"""
create patch set for the package base
Args:
application(Application): application instance
sources_dir(str): path to directory with the package sources
sources_dir(Path): path to directory with the package sources
track(List[str]): track files which match the glob before creating the patch
"""
package = Package.load(sources_dir, PackageSource.Local, application.repository.pacman,
application.repository.aur_url)
patch = Sources.patch_create(Path(sources_dir), *track)
package = Package.from_build(sources_dir)
patch = Sources.patch_create(sources_dir, *track)
application.database.patches_insert(package.base, patch)
@staticmethod

View File

@ -22,6 +22,7 @@ import argparse
from dataclasses import fields
from typing import Callable, Iterable, List, Tuple, Type
from ahriman.application.application import Application
from ahriman.application.handlers.handler import Handler
from ahriman.core.alpm.remote.aur import AUR
from ahriman.core.alpm.remote.official import Official
@ -55,8 +56,10 @@ class Search(Handler):
no_report(bool): force disable reporting
unsafe(bool): if set no user check will be performed before path creation
"""
official_packages_list = Official.multisearch(*args.search)
aur_packages_list = AUR.multisearch(*args.search)
application = Application(architecture, configuration, no_report, unsafe)
official_packages_list = Official.multisearch(*args.search, pacman=application.repository.pacman)
aur_packages_list = AUR.multisearch(*args.search, pacman=application.repository.pacman)
Search.check_if_empty(args.exit_code, not official_packages_list and not aur_packages_list)
for packages_list in (official_packages_list, aur_packages_list):

View File

@ -17,8 +17,8 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from pyalpm import Handle # type: ignore
from typing import Set
from pyalpm import Handle, Package, SIG_PACKAGE # type: ignore
from typing import Generator, Set
from ahriman.core.configuration import Configuration
@ -42,7 +42,7 @@ class Pacman:
pacman_root = configuration.getpath("alpm", "database")
self.handle = Handle(root, str(pacman_root))
for repository in configuration.getlist("alpm", "repositories"):
self.handle.register_syncdb(repository, 0) # 0 is pgp_level
self.handle.register_syncdb(repository, SIG_PACKAGE)
def all_packages(self) -> Set[str]:
"""
@ -58,3 +58,19 @@ class Pacman:
result.update(package.provides) # provides list for meta-packages
return result
def get(self, package_name: str) -> Generator[Package, None, None]:
"""
retrieve list of the packages from the repository by name
Args:
package_name(str): package name to search
Yields:
Package: list of packages which were returned by the query
"""
for database in self.handle.get_syncdbs():
package = database.get_pkg(package_name)
if package is None:
continue
yield package

View File

@ -19,8 +19,9 @@
#
import requests
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote.remote import Remote
from ahriman.core.exceptions import InvalidPackageInfo
from ahriman.core.util import exception_response_text
@ -32,27 +33,15 @@ class AUR(Remote):
AUR RPC wrapper
Attributes:
DEFAULT_AUR_URL(str): (class attribute) default AUR url
DEFAULT_RPC_URL(str): (class attribute) default AUR RPC url
DEFAULT_RPC_VERSION(str): (class attribute) default AUR RPC version
rpc_url(str): AUR RPC url
rpc_version(str): AUR RPC version
"""
DEFAULT_RPC_URL = "https://aur.archlinux.org/rpc"
DEFAULT_AUR_URL = "https://aur.archlinux.org"
DEFAULT_RPC_URL = f"{DEFAULT_AUR_URL}/rpc"
DEFAULT_RPC_VERSION = "5"
def __init__(self, rpc_url: Optional[str] = None, rpc_version: Optional[str] = None) -> None:
"""
default constructor
Args:
rpc_url(Optional[str], optional): AUR RPC url (Default value = None)
rpc_version(Optional[str], optional): AUR RPC version (Default value = None)
"""
Remote.__init__(self)
self.rpc_url = rpc_url or self.DEFAULT_RPC_URL
self.rpc_version = rpc_version or self.DEFAULT_RPC_VERSION
@staticmethod
def parse_response(response: Dict[str, Any]) -> List[AURPackage]:
"""
@ -73,6 +62,33 @@ class AUR(Remote):
raise InvalidPackageInfo(error_details)
return [AURPackage.from_json(package) for package in response["results"]]
@staticmethod
def remote_git_url(package_base: str, repository: str) -> str:
"""
generate remote git url from the package base
Args
package_base(str): package base
repository(str): repository name
Returns:
str: git url for the specific base
"""
return f"{AUR.DEFAULT_AUR_URL}/{package_base}.git"
@staticmethod
def remote_web_url(package_base: str) -> str:
"""
generate remote web url from the package base
Args
package_base(str): package base
Returns:
str: web url for the specific base
"""
return f"{AUR.DEFAULT_AUR_URL}/packages/{package_base}"
def make_request(self, request_type: str, *args: str, **kwargs: str) -> List[AURPackage]:
"""
perform request to AUR RPC
@ -87,7 +103,7 @@ class AUR(Remote):
"""
query: Dict[str, Any] = {
"type": request_type,
"v": self.rpc_version
"v": self.DEFAULT_RPC_VERSION
}
arg_query = "arg[]" if len(args) > 1 else "arg"
@ -97,7 +113,7 @@ class AUR(Remote):
query[key] = value
try:
response = requests.get(self.rpc_url, params=query)
response = requests.get(self.DEFAULT_RPC_URL, params=query)
response.raise_for_status()
return self.parse_response(response.json())
except requests.HTTPError as e:
@ -110,12 +126,13 @@ class AUR(Remote):
self.logger.exception("could not perform request by using type %s", request_type)
raise
def package_info(self, package_name: str) -> AURPackage:
def package_info(self, package_name: str, *, pacman: Pacman) -> AURPackage:
"""
get package info by its name
Args:
package_name(str): package name to search
pacman(Pacman): alpm wrapper instance
Returns:
AURPackage: package which match the package name
@ -123,12 +140,13 @@ class AUR(Remote):
packages = self.make_request("info", package_name)
return next(package for package in packages if package.name == package_name)
def package_search(self, *keywords: str) -> List[AURPackage]:
def package_search(self, *keywords: str, pacman: Pacman) -> List[AURPackage]:
"""
search package in AUR web
Args:
*keywords(str): keywords to search
pacman(Pacman): alpm wrapper instance
Returns:
List[AURPackage]: list of packages which match the criteria

View File

@ -19,8 +19,9 @@
#
import requests
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote.remote import Remote
from ahriman.core.exceptions import InvalidPackageInfo
from ahriman.core.util import exception_response_text
@ -32,22 +33,15 @@ class Official(Remote):
official repository RPC wrapper
Attributes:
DEFAULT_RPC_URL(str): (class attribute) default AUR RPC url
rpc_url(str): AUR RPC url
DEFAULT_ARCHLINUX_URL(str): (class attribute) default archlinux url
DEFAULT_SEARCH_REPOSITORIES(List[str]): (class attribute) default list of repositories to search
DEFAULT_RPC_URL(str): (class attribute) default archlinux repositories RPC url
"""
DEFAULT_ARCHLINUX_URL = "https://archlinux.org"
DEFAULT_SEARCH_REPOSITORIES = ["Core", "Extra", "Multilib", "Community"]
DEFAULT_RPC_URL = "https://archlinux.org/packages/search/json"
def __init__(self, rpc_url: Optional[str] = None) -> None:
"""
default constructor
Args:
rpc_url(Optional[str], optional): AUR RPC url (Default value = None)
"""
Remote.__init__(self)
self.rpc_url = rpc_url or self.DEFAULT_RPC_URL
@staticmethod
def parse_response(response: Dict[str, Any]) -> List[AURPackage]:
"""
@ -66,6 +60,35 @@ class Official(Remote):
raise InvalidPackageInfo("API validation error")
return [AURPackage.from_repo(package) for package in response["results"]]
@staticmethod
def remote_git_url(package_base: str, repository: str) -> str:
"""
generate remote git url from the package base
Args
package_base(str): package base
repository(str): repository name
Returns:
str: git url for the specific base
"""
if repository.lower() in ("core", "extra", "testing", "kde-unstable"):
return "https://github.com/archlinux/svntogit-packages.git" # hardcoded, ok
return "https://github.com/archlinux/svntogit-community.git"
@staticmethod
def remote_web_url(package_base: str) -> str:
"""
generate remote web url from the package base
Args
package_base(str): package base
Returns:
str: web url for the specific base
"""
return f"{Official.DEFAULT_ARCHLINUX_URL}/packages/{package_base}"
def make_request(self, *args: str, by: str) -> List[AURPackage]:
"""
perform request to official repositories RPC
@ -78,7 +101,7 @@ class Official(Remote):
List[AURPackage]: response parsed to package list
"""
try:
response = requests.get(self.rpc_url, params={by: args})
response = requests.get(self.DEFAULT_RPC_URL, params={by: args, "repo": self.DEFAULT_SEARCH_REPOSITORIES})
response.raise_for_status()
return self.parse_response(response.json())
except requests.HTTPError as e:
@ -88,12 +111,13 @@ class Official(Remote):
self.logger.exception("could not perform request")
raise
def package_info(self, package_name: str) -> AURPackage:
def package_info(self, package_name: str, *, pacman: Pacman) -> AURPackage:
"""
get package info by its name
Args:
package_name(str): package name to search
pacman(Pacman): alpm wrapper instance
Returns:
AURPackage: package which match the package name
@ -101,12 +125,13 @@ class Official(Remote):
packages = self.make_request(package_name, by="name")
return next(package for package in packages if package.name == package_name)
def package_search(self, *keywords: str) -> List[AURPackage]:
def package_search(self, *keywords: str, pacman: Pacman) -> List[AURPackage]:
"""
search package in AUR web
Args:
*keywords(str): keywords to search
pacman(Pacman): alpm wrapper instance
Returns:
List[AURPackage]: list of packages which match the criteria

View File

@ -0,0 +1,51 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote.official import Official
from ahriman.models.aur_package import AURPackage
class OfficialSyncdb(Official):
"""
official repository wrapper based on synchronized databases.
Despite the fact that official repository provides an API for the interaction according to the comment in issue
https://github.com/arcan1s/ahriman/pull/59#issuecomment-1106412297 we might face rate limits while requesting
updates.
This approach also has limitations, because we don't require superuser rights (neither going to download database
separately), the database file might be outdated and must be handled manually (or kind of). This behaviour might be
changed in the future.
Still we leave search function based on the official repositories RPC.
"""
def package_info(self, package_name: str, *, pacman: Pacman) -> AURPackage:
"""
get package info by its name
Args:
package_name(str): package name to search
pacman(Pacman): alpm wrapper instance
Returns:
AURPackage: package which match the package name
"""
return next(AURPackage.from_pacman(package) for package in pacman.get(package_name))

View File

@ -23,6 +23,7 @@ import logging
from typing import Dict, List, Type
from ahriman.core.alpm.pacman import Pacman
from ahriman.models.aur_package import AURPackage
@ -41,26 +42,28 @@ class Remote:
self.logger = logging.getLogger("build_details")
@classmethod
def info(cls: Type[Remote], package_name: str) -> AURPackage:
def info(cls: Type[Remote], package_name: str, *, pacman: Pacman) -> AURPackage:
"""
get package info by its name
Args:
package_name(str): package name to search
pacman(Pacman): alpm wrapper instance
Returns:
AURPackage: package which match the package name
"""
return cls().package_info(package_name)
return cls().package_info(package_name, pacman=pacman)
@classmethod
def multisearch(cls: Type[Remote], *keywords: str) -> List[AURPackage]:
def multisearch(cls: Type[Remote], *keywords: str, pacman: Pacman) -> List[AURPackage]:
"""
search in remote repository by using API with multiple words. This method is required in order to handle
https://bugs.archlinux.org/task/49133. In addition, short words will be dropped
Args:
*keywords(str): search terms, e.g. "ahriman", "is", "cool"
pacman(Pacman): alpm wrapper instance
Returns:
List[AURPackage]: list of packages each of them matches all search terms
@ -68,7 +71,7 @@ class Remote:
instance = cls()
packages: Dict[str, AURPackage] = {}
for term in filter(lambda word: len(word) > 3, keywords):
portion = instance.search(term)
portion = instance.search(term, pacman=pacman)
packages = {
package.name: package # not mistake to group them by name
for package in portion
@ -76,25 +79,60 @@ class Remote:
}
return list(packages.values())
@staticmethod
def remote_git_url(package_base: str, repository: str) -> str:
"""
generate remote git url from the package base
Args
package_base(str): package base
repository(str): repository name
Returns:
str: git url for the specific base
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
@staticmethod
def remote_web_url(package_base: str) -> str:
"""
generate remote web url from the package base
Args
package_base(str): package base
Returns:
str: web url for the specific base
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
@classmethod
def search(cls: Type[Remote], *keywords: str) -> List[AURPackage]:
def search(cls: Type[Remote], *keywords: str, pacman: Pacman) -> List[AURPackage]:
"""
search package in AUR web
Args:
*keywords(str): search terms, e.g. "ahriman", "is", "cool"
pacman(Pacman): alpm wrapper instance
Returns:
List[AURPackage]: list of packages which match the criteria
"""
return cls().package_search(*keywords)
return cls().package_search(*keywords, pacman=pacman)
def package_info(self, package_name: str) -> AURPackage:
def package_info(self, package_name: str, *, pacman: Pacman) -> AURPackage:
"""
get package info by its name
Args:
package_name(str): package name to search
pacman(Pacman): alpm wrapper instance
Returns:
AURPackage: package which match the package name
@ -104,12 +142,13 @@ class Remote:
"""
raise NotImplementedError
def package_search(self, *keywords: str) -> List[AURPackage]:
def package_search(self, *keywords: str, pacman: Pacman) -> List[AURPackage]:
"""
search package in AUR web
Args:
*keywords(str): keywords to search
pacman(Pacman): alpm wrapper instance
Returns:
List[AURPackage]: list of packages which match the criteria

View File

@ -18,11 +18,13 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
import shutil
from pathlib import Path
from typing import List, Optional
from ahriman.core.util import check_output
from ahriman.core.util import check_output, walk
from ahriman.models.remote_source import RemoteSource
class Sources:
@ -30,12 +32,14 @@ class Sources:
helper to download package sources (PKGBUILD etc)
Attributes:
DEFAULT_BRANCH(str): (class attribute) default branch to process git repositories.
Must be used only for local stored repositories, use RemoteSource descriptor instead for real packages
logger(logging.Logger): (class attribute) class logger
"""
DEFAULT_BRANCH = "master" # default fallback branch
logger = logging.getLogger("build_details")
_branch = "master" # in case if BLM would like to change it
_check_output = check_output
@staticmethod
@ -73,13 +77,13 @@ class Sources:
return Sources._check_output("git", "diff", exception=None, cwd=sources_dir, logger=Sources.logger)
@staticmethod
def fetch(sources_dir: Path, remote: Optional[str]) -> None:
def fetch(sources_dir: Path, remote: Optional[RemoteSource]) -> None:
"""
either clone repository or update it to origin/`branch`
Args:
sources_dir(Path): local path to fetch
remote(Optional[str]): remote target (from where to fetch)
remote(Optional[RemoteSource]): remote target (from where to fetch)
"""
# local directory exists and there is .git directory
is_initialized_git = (sources_dir / ".git").is_dir()
@ -88,22 +92,30 @@ class Sources:
Sources.logger.info("skip update at %s because there are no branches configured", sources_dir)
return
branch = remote.branch if remote is not None else Sources.DEFAULT_BRANCH
if is_initialized_git:
Sources.logger.info("update HEAD to remote at %s", sources_dir)
Sources._check_output("git", "fetch", "origin", Sources._branch,
Sources.logger.info("update HEAD to remote at %s using branch %s", sources_dir, branch)
Sources._check_output("git", "fetch", "origin", branch,
exception=None, cwd=sources_dir, logger=Sources.logger)
elif remote is not None:
Sources.logger.info("clone remote %s to %s using branch %s", remote.git_url, sources_dir, branch)
Sources._check_output("git", "clone", "--branch", branch, "--single-branch",
remote.git_url, str(sources_dir),
exception=None, cwd=sources_dir, logger=Sources.logger)
elif remote is None:
Sources.logger.warning("%s is not initialized, but no remote provided", sources_dir)
else:
Sources.logger.info("clone remote %s to %s", remote, sources_dir)
Sources._check_output("git", "clone", remote, str(sources_dir),
exception=None, cwd=sources_dir, logger=Sources.logger)
Sources.logger.warning("%s is not initialized, but no remote provided", sources_dir)
# and now force reset to our branch
Sources._check_output("git", "checkout", "--force", Sources._branch,
Sources._check_output("git", "checkout", "--force", branch,
exception=None, cwd=sources_dir, logger=Sources.logger)
Sources._check_output("git", "reset", "--hard", f"origin/{Sources._branch}",
Sources._check_output("git", "reset", "--hard", f"origin/{branch}",
exception=None, cwd=sources_dir, logger=Sources.logger)
# move content if required
# we are using full path to source directory in order to make append possible
pkgbuild_dir = remote.pkgbuild_dir if remote is not None else sources_dir.resolve()
Sources.move((sources_dir / pkgbuild_dir).resolve(), sources_dir)
@staticmethod
def has_remotes(sources_dir: Path) -> bool:
"""
@ -126,17 +138,17 @@ class Sources:
Args:
sources_dir(Path): local path to sources
"""
Sources._check_output("git", "init", "--initial-branch", Sources._branch,
Sources._check_output("git", "init", "--initial-branch", Sources.DEFAULT_BRANCH,
exception=None, cwd=sources_dir, logger=Sources.logger)
@staticmethod
def load(sources_dir: Path, remote: str, patch: Optional[str]) -> None:
def load(sources_dir: Path, remote: Optional[RemoteSource], patch: Optional[str]) -> None:
"""
fetch sources from remote and apply patches
Args:
sources_dir(Path): local path to fetch
remote(str): remote target (from where to fetch)
remote(Optional[RemoteSource]): remote target (from where to fetch)
patch(Optional[str]): optional patch to be applied
"""
Sources.fetch(sources_dir, remote)
@ -145,6 +157,21 @@ class Sources:
return
Sources.patch_apply(sources_dir, patch)
@staticmethod
def move(pkgbuild_dir: Path, sources_dir: Path) -> None:
"""
move content from pkgbuild_dir to sources_dir
Args:
pkgbuild_dir(Path): path to directory with pkgbuild from which need to move
sources_dir(Path): path to target directory
"""
if pkgbuild_dir == sources_dir:
return # directories are the same, no need to move
for src in walk(pkgbuild_dir):
dst = sources_dir / src.relative_to(pkgbuild_dir)
shutil.move(src, dst)
@staticmethod
def patch_apply(sources_dir: Path, patch: str) -> None:
"""

View File

@ -107,4 +107,4 @@ class Task:
if self.paths.cache_for(self.package.base).is_dir():
# no need to clone whole repository, just copy from cache first
shutil.copytree(self.paths.cache_for(self.package.base), path, dirs_exist_ok=True)
Sources.load(path, self.package.git_url, database.patches_get(self.package.base))
Sources.load(path, self.package.remote, database.patches_get(self.package.base))

View File

@ -20,6 +20,7 @@
from sqlite3 import Connection
from ahriman.core.configuration import Configuration
from ahriman.core.database.data.package_remotes import migrate_package_remotes
from ahriman.core.database.data.package_statuses import migrate_package_statuses
from ahriman.core.database.data.patches import migrate_patches
from ahriman.core.database.data.users import migrate_users_data
@ -43,3 +44,5 @@ def migrate_data(
migrate_package_statuses(connection, repository_paths)
migrate_patches(connection, repository_paths)
migrate_users_data(connection, configuration)
if result.old_version <= 1:
migrate_package_remotes(connection, repository_paths)

View File

@ -0,0 +1,61 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from ahriman.models.package_source import PackageSource
from ahriman.models.remote_source import RemoteSource
from ahriman.models.repository_paths import RepositoryPaths
# pylint: disable=protected-access
def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> None:
"""
perform migration for package remote sources
Args:
connection(Connection): database connection
paths(RepositoryPaths): repository paths instance
"""
from ahriman.core.database.operations.package_operations import PackageOperations
def insert_remote(base: str, remote: RemoteSource) -> None:
connection.execute(
"""
update package_bases set
branch = :branch, git_url = :git_url, path = :path,
web_url = :web_url, source = :source
where package_base = :package_base
""",
dict(
package_base=base,
branch=remote.branch, git_url=remote.git_url, path=remote.path,
web_url=remote.web_url, source=remote.source
)
)
packages = PackageOperations._packages_get_select_package_bases(connection)
for package_base, package in packages.items():
local_cache = paths.cache_for(package_base)
if local_cache.exists() and not package.is_vcs:
continue # skip packages which are not VCS and with local cache
remote_source = RemoteSource.from_remote(PackageSource.AUR, package_base, "aur")
if remote_source is None:
continue # should never happen
insert_remote(package_base, remote_source)

View File

@ -42,7 +42,7 @@ def migrate_package_statuses(connection: Connection, paths: RepositoryPaths) ->
values
(:package_base, :version, :aur_url)
""",
dict(package_base=metadata.base, version=metadata.version, aur_url=metadata.aur_url))
dict(package_base=metadata.base, version=metadata.version, aur_url=""))
connection.execute(
"""
insert into package_statuses

View File

@ -0,0 +1,40 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
steps = [
"""
alter table package_bases add column branch text
""",
"""
alter table package_bases add column git_url text
""",
"""
alter table package_bases add column path text
""",
"""
alter table package_bases add column web_url text
""",
"""
alter table package_bases add column source text
""",
"""
alter table package_bases drop column aur_url
""",
]

View File

@ -24,6 +24,7 @@ from ahriman.core.database.operations.operations import Operations
from ahriman.models.build_status import BuildStatus
from ahriman.models.package import Package
from ahriman.models.package_description import PackageDescription
from ahriman.models.remote_source import RemoteSource
class PackageOperations(Operations):
@ -75,13 +76,22 @@ class PackageOperations(Operations):
connection.execute(
"""
insert into package_bases
(package_base, version, aur_url)
(package_base, version, source, branch, git_url, path, web_url)
values
(:package_base, :version, :aur_url)
(:package_base, :version, :source, :branch, :git_url, :path, :web_url)
on conflict (package_base) do update set
version = :version, aur_url = :aur_url
version = :version, branch = :branch, git_url = :git_url, path = :path, web_url = :web_url, source = :source
""",
dict(package_base=package.base, version=package.version, aur_url=package.aur_url))
dict(
package_base=package.base,
version=package.version,
branch=package.remote.branch if package.remote is not None else None,
git_url=package.remote.git_url if package.remote is not None else None,
path=package.remote.path if package.remote is not None else None,
web_url=package.remote.web_url if package.remote is not None else None,
source=package.remote.source.value if package.remote is not None else None,
)
)
@staticmethod
def _package_update_insert_packages(connection: Connection, package: Package) -> None:
@ -144,7 +154,7 @@ class PackageOperations(Operations):
Dict[str, Package]: map of the package base to its descriptor (without packages themselves)
"""
return {
row["package_base"]: Package(row["package_base"], row["version"], row["aur_url"], {})
row["package_base"]: Package(row["package_base"], row["version"], RemoteSource.from_json(row), {})
for row in connection.execute("""select * from package_bases""")
}
@ -225,3 +235,28 @@ class PackageOperations(Operations):
yield package, statuses.get(package_base, BuildStatus())
return self.with_connection(lambda connection: list(run(connection)))
def remote_update(self, package: Package) -> None:
"""
update package remote source
Args:
package(Package): package properties
"""
return self.with_connection(
lambda connection: self._package_update_insert_base(connection, package),
commit=True)
def remotes_get(self) -> Dict[str, RemoteSource]:
"""
get packages remotes based on current settings
Returns:
Dict[str, RemoteSource]: map of package base to its remote sources
"""
packages = self.with_connection(self._packages_get_select_package_bases)
return {
package_base: package.remote
for package_base, package in packages.items()
if package.remote is not None
}

View File

@ -81,5 +81,5 @@ class SQLite(AuthOperations, BuildOperations, PackageOperations, PatchOperations
paths = configuration.repository_paths
self.with_connection(lambda conn: Migrations.migrate(conn, configuration))
self.with_connection(lambda connection: Migrations.migrate(connection, configuration))
paths.chown(self.path)

View File

@ -24,7 +24,6 @@ from ahriman.core.repository.executor import Executor
from ahriman.core.repository.update_handler import UpdateHandler
from ahriman.core.util import package_like
from ahriman.models.package import Package
from ahriman.models.package_source import PackageSource
class Repository(Executor, UpdateHandler):
@ -42,11 +41,15 @@ class Repository(Executor, UpdateHandler):
Returns:
List[Package]: list of read packages
"""
sources = self.database.remotes_get()
result: Dict[str, Package] = {}
# we are iterating over bases, not single packages
for full_path in packages:
try:
local = Package.load(str(full_path), PackageSource.Archive, self.pacman, self.aur_url)
local = Package.from_archive(full_path, self.pacman, None)
local.remote = sources.get(local.base)
current = result.setdefault(local.base, local)
if current.version != local.version:
# force version to max of them
@ -95,5 +98,5 @@ class Repository(Executor, UpdateHandler):
return [
package
for package in packages
if depends_on is None or depends_on.intersection(package.full_depends(self.pacman, packages))
if depends_on.intersection(package.full_depends(self.pacman, packages))
]

View File

@ -35,7 +35,6 @@ class RepositoryProperties:
Attributes:
architecture(str): repository architecture
aur_url(str): base AUR url
configuration(Configuration): configuration instance
database(SQLite): database instance
ignore_list(List[str]): package bases which will be ignored during auto updates
@ -65,7 +64,6 @@ class RepositoryProperties:
self.configuration = configuration
self.database = database
self.aur_url = configuration.get("alpm", "aur_url")
self.name = configuration.get("repository", "name")
self.paths = configuration.repository_paths

View File

@ -62,13 +62,18 @@ class UpdateHandler(Cleaner):
continue
if filter_packages and local.base not in filter_packages:
continue
source = local.remote.source if local.remote is not None else None
try:
remote = Package.load(local.base, PackageSource.AUR, self.pacman, self.aur_url)
if source == PackageSource.Repository:
remote = Package.from_official(local.base, self.pacman)
else:
remote = Package.from_aur(local.base, self.pacman)
if local.is_outdated(remote, self.paths):
self.reporter.set_pending(local.base)
result.append(remote)
self.reporter.set_success(local)
else:
self.reporter.set_success(local)
except Exception:
self.reporter.set_failed(local.base)
self.logger.exception("could not load remote package %s", local.base)
@ -89,7 +94,7 @@ class UpdateHandler(Cleaner):
for dirname in self.paths.cache.iterdir():
try:
Sources.fetch(dirname, remote=None)
remote = Package.load(str(dirname), PackageSource.Local, self.pacman, self.aur_url)
remote = Package.from_build(dirname)
local = packages.get(remote.base)
if local is None:

View File

@ -70,7 +70,7 @@ class Leaf:
Leaf: loaded class
"""
with tmpdir() as clone_dir:
Sources.load(clone_dir, package.git_url, database.patches_get(package.base))
Sources.load(clone_dir, package.remote, database.patches_get(package.base))
dependencies = Package.dependencies(clone_dir)
return cls(package, dependencies)

View File

@ -20,7 +20,7 @@
from enum import Enum
class Action(Enum):
class Action(str, Enum):
"""
base action enumeration

View File

@ -23,6 +23,7 @@ import datetime
import inflection
from dataclasses import dataclass, field, fields
from pyalpm import Package # type: ignore
from typing import Any, Callable, Dict, List, Optional, Type
from ahriman.core.util import filter_json, full_version
@ -47,6 +48,7 @@ class AURPackage:
first_submitted(datetime.datetime): timestamp of the first package submission
last_modified(datetime.datetime): timestamp of the last package submission
url_path(str): AUR package path
repository(str): repository name of the package
depends(List[str]): list of package dependencies
make_depends(List[str]): list of package make dependencies
opt_depends(List[str]): list of package optional dependencies
@ -70,6 +72,7 @@ class AURPackage:
url: Optional[str] = None
out_of_date: Optional[datetime.datetime] = None
maintainer: Optional[str] = None
repository: str = "aur"
depends: List[str] = field(default_factory=list)
make_depends: List[str] = field(default_factory=list)
opt_depends: List[str] = field(default_factory=list)
@ -94,6 +97,42 @@ class AURPackage:
properties = cls.convert(dump)
return cls(**filter_json(properties, known_fields))
@classmethod
def from_pacman(cls: Type[AURPackage], package: Package) -> AURPackage:
"""
construct package descriptor from official repository wrapper
Args:
package(Package): pyalpm package descriptor
Returns:
AURPackage: AUR package descriptor
"""
return cls(
id=0,
name=package.name,
package_base_id=0,
package_base=package.base,
version=package.version,
description=package.desc,
num_votes=0,
popularity=0.0,
first_submitted=datetime.datetime.utcfromtimestamp(0),
last_modified=datetime.datetime.utcfromtimestamp(package.builddate),
url_path="",
url=package.url,
out_of_date=None,
maintainer=None,
repository=package.db.name,
depends=package.depends,
make_depends=package.makedepends,
opt_depends=package.optdepends,
conflicts=package.conflicts,
provides=package.provides,
license=package.licenses,
keywords=[],
)
@classmethod
def from_repo(cls: Type[AURPackage], dump: Dict[str, Any]) -> AURPackage:
"""
@ -122,6 +161,7 @@ class AURPackage:
dump["flag_date"],
"%Y-%m-%dT%H:%M:%S.%fZ") if dump["flag_date"] is not None else None,
maintainer=next(iter(dump["maintainers"]), None),
repository=dump["repo"],
depends=dump["depends"],
make_depends=dump["makedepends"],
opt_depends=dump["optdepends"],

View File

@ -23,7 +23,7 @@ from enum import Enum
from typing import Type
class AuthSettings(Enum):
class AuthSettings(str, Enum):
"""
web authorization type

View File

@ -28,7 +28,7 @@ from typing import Any, Dict, Type
from ahriman.core.util import filter_json, pretty_datetime
class BuildStatusEnum(Enum):
class BuildStatusEnum(str, Enum):
"""
build status enumeration

View File

@ -26,15 +26,17 @@ from dataclasses import asdict, dataclass
from pathlib import Path
from pyalpm import vercmp # type: ignore
from srcinfo.parse import parse_srcinfo # type: ignore
from typing import Any, Dict, Iterable, List, Set, Type
from typing import Any, Dict, Iterable, List, Optional, Set, Type
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote.aur import AUR
from ahriman.core.alpm.remote.official import Official
from ahriman.core.alpm.remote.official_syncdb import OfficialSyncdb
from ahriman.core.exceptions import InvalidPackageInfo
from ahriman.core.util import check_output, full_version
from ahriman.models.package_description import PackageDescription
from ahriman.models.package_source import PackageSource
from ahriman.models.remote_source import RemoteSource
from ahriman.models.repository_paths import RepositoryPaths
@ -44,15 +46,16 @@ class Package:
package properties representation
Attributes:
aur_url(str): AUR root url
base(str): package base name
packages(Dict[str, PackageDescription): map of package names to their properties. Filled only on load from archive
packages(Dict[str, PackageDescription): map of package names to their properties.
Filled only on load from archive
remote(Optional[RemoteSource]): package remote source if applicable
version(str): package full version
"""
base: str
version: str
aur_url: str
remote: Optional[RemoteSource]
packages: Dict[str, PackageDescription]
_check_output = check_output
@ -67,16 +70,6 @@ class Package:
"""
return sorted(set(sum([package.depends for package in self.packages.values()], start=[])))
@property
def git_url(self) -> str:
"""
get git clone url
Returns:
str: package git url to clone
"""
return f"{self.aur_url}/{self.base}.git"
@property
def groups(self) -> List[str]:
"""
@ -122,56 +115,46 @@ class Package:
"""
return sorted(set(sum([package.licenses for package in self.packages.values()], start=[])))
@property
def web_url(self) -> str:
"""
get package url which can be used to see package in web
Returns:
str: package AUR url
"""
return f"{self.aur_url}/packages/{self.base}"
@classmethod
def from_archive(cls: Type[Package], path: Path, pacman: Pacman, aur_url: str) -> Package:
def from_archive(cls: Type[Package], path: Path, pacman: Pacman, remote: Optional[RemoteSource]) -> Package:
"""
construct package properties from package archive
Args:
path(Path): path to package archive
pacman(Pacman): alpm wrapper instance
aur_url(str): AUR root url
remote(RemoteSource): package remote source if applicable
Returns:
Package: package properties
"""
package = pacman.handle.load_pkg(str(path))
return cls(package.base, package.version, aur_url,
{package.name: PackageDescription.from_package(package, path)})
description = PackageDescription.from_package(package, path)
return cls(package.base, package.version, remote, {package.name: description})
@classmethod
def from_aur(cls: Type[Package], name: str, aur_url: str) -> Package:
def from_aur(cls: Type[Package], name: str, pacman: Pacman) -> Package:
"""
construct package properties from AUR page
Args:
name(str): package name (either base or normal name)
aur_url(str): AUR root url
pacman(Pacman): alpm wrapper instance
Returns:
Package: package properties
"""
package = AUR.info(name)
return cls(package.package_base, package.version, aur_url, {package.name: PackageDescription()})
package = AUR.info(name, pacman=pacman)
remote = RemoteSource.from_remote(PackageSource.AUR, package.package_base, package.repository)
return cls(package.package_base, package.version, remote, {package.name: PackageDescription()})
@classmethod
def from_build(cls: Type[Package], path: Path, aur_url: str) -> Package:
def from_build(cls: Type[Package], path: Path) -> Package:
"""
construct package properties from sources directory
Args:
path(Path): path to package sources directory
aur_url(str): AUR root url
Returns:
Package: package properties
@ -179,13 +162,14 @@ class Package:
Raises:
InvalidPackageInfo: if there are parsing errors
"""
srcinfo, errors = parse_srcinfo((path / ".SRCINFO").read_text())
srcinfo_source = Package._check_output("makepkg", "--printsrcinfo", exception=None, cwd=path)
srcinfo, errors = parse_srcinfo(srcinfo_source)
if errors:
raise InvalidPackageInfo(errors)
packages = {key: PackageDescription() for key in srcinfo["packages"]}
version = full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"])
return cls(srcinfo["pkgbase"], version, aur_url, packages)
return cls(srcinfo["pkgbase"], version, None, packages)
@classmethod
def from_json(cls: Type[Package], dump: Dict[str, Any]) -> Package:
@ -202,59 +186,29 @@ class Package:
key: PackageDescription.from_json(value)
for key, value in dump.get("packages", {}).items()
}
return Package(
remote = dump.get("remote", {})
return cls(
base=dump["base"],
version=dump["version"],
aur_url=dump["aur_url"],
remote=RemoteSource.from_json(remote),
packages=packages)
@classmethod
def from_official(cls: Type[Package], name: str, aur_url: str) -> Package:
def from_official(cls: Type[Package], name: str, pacman: Pacman, use_syncdb: bool = True) -> Package:
"""
construct package properties from official repository page
Args:
name(str): package name (either base or normal name)
aur_url(str): AUR root url
pacman(Pacman): alpm wrapper instance
use_syncdb(bool): use pacman databases instead of official repositories RPC (Default value = True)
Returns:
Package: package properties
"""
package = Official.info(name)
return cls(package.package_base, package.version, aur_url, {package.name: PackageDescription()})
@classmethod
def load(cls: Type[Package], package: str, source: PackageSource, pacman: Pacman, aur_url: str) -> Package:
"""
package constructor from available sources
Args:
package(str): one of path to sources directory, path to archive or package name/base
source(PackageSource): source of the package required to define the load method
pacman(Pacman): alpm wrapper instance (required to load from archive)
aur_url(str): AUR root url
Returns:
Package: package properties
Raises:
InvalidPackageInfo: if supplied package source is not valid
"""
try:
resolved_source = source.resolve(package)
if resolved_source == PackageSource.Archive:
return cls.from_archive(Path(package), pacman, aur_url)
if resolved_source == PackageSource.AUR:
return cls.from_aur(package, aur_url)
if resolved_source == PackageSource.Local:
return cls.from_build(Path(package), aur_url)
if resolved_source == PackageSource.Repository:
return cls.from_official(package, aur_url)
raise InvalidPackageInfo(f"Unsupported local package source {resolved_source}")
except InvalidPackageInfo:
raise
except Exception as e:
raise InvalidPackageInfo(str(e))
package = OfficialSyncdb.info(name, pacman=pacman) if use_syncdb else Official.info(name, pacman=pacman)
remote = RemoteSource.from_remote(PackageSource.Repository, package.package_base, package.repository)
return cls(package.package_base, package.version, remote, {package.name: PackageDescription()})
@staticmethod
def dependencies(path: Path) -> Set[str]:
@ -279,7 +233,8 @@ class Package:
package_name = package_name.split(symbol)[0]
return package_name
srcinfo, errors = parse_srcinfo((path / ".SRCINFO").read_text())
srcinfo_source = Package._check_output("makepkg", "--printsrcinfo", exception=None, cwd=path)
srcinfo, errors = parse_srcinfo(srcinfo_source)
if errors:
raise InvalidPackageInfo(errors)
makedepends = extract_packages(srcinfo.get("makedepends", []))
@ -310,7 +265,7 @@ class Package:
from ahriman.core.build_tools.sources import Sources
logger = logging.getLogger("build_details")
Sources.load(paths.cache_for(self.base), self.git_url, None)
Sources.load(paths.cache_for(self.base), self.remote, None)
try:
# update pkgver first

View File

@ -26,7 +26,7 @@ from urllib.parse import urlparse
from ahriman.core.util import package_like
class PackageSource(Enum):
class PackageSource(str, Enum):
"""
package source for addition enumeration

View File

@ -0,0 +1,124 @@
#
# Copyright (c) 2021-2022 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import annotations
from dataclasses import asdict, dataclass, fields
from pathlib import Path
from typing import Any, Dict, Optional, Type
from ahriman.core.util import filter_json
from ahriman.models.package_source import PackageSource
@dataclass
class RemoteSource:
"""
remote package source properties
Attributes:
branch(str): branch of the git repository
git_url(str): url of the git repository
path(str): path to directory with PKGBUILD inside the git repository
source(PackageSource): package source pointer used by some parsers
web_url(str): url of the package in the web interface
"""
git_url: str
web_url: str
path: str
branch: str
source: PackageSource
def __post_init__(self) -> None:
"""
convert source to enum type
"""
self.source = PackageSource(self.source)
@property
def pkgbuild_dir(self) -> Path:
"""
get path to directory with package sources (PKGBUILD etc)
Returns:
Path: path to directory with package sources based on settings
"""
return Path(self.path)
@classmethod
def from_json(cls: Type[RemoteSource], dump: Dict[str, Any]) -> Optional[RemoteSource]:
"""
construct remote source from the json dump (or database row)
Args:
dump(Dict[str, Any]): json dump body
Returns:
Optional[RemoteSource]: remote source
"""
# filter to only known fields
known_fields = [pair.name for pair in fields(cls)]
dump = filter_json(dump, known_fields)
if dump:
return cls(**dump)
return None
@classmethod
def from_remote(cls: Type[RemoteSource], source: PackageSource, package_base: str,
repository: str) -> Optional[RemoteSource]:
"""
generate remote source from the package base
Args:
source(PackageSource): source of the package
package_base(str): package base
repository(str): repository name
Returns:
Optional[RemoteSource]: generated remote source if any, None otherwise
"""
if source == PackageSource.AUR:
from ahriman.core.alpm.remote.aur import AUR
return RemoteSource(
git_url=AUR.remote_git_url(package_base, repository),
web_url=AUR.remote_web_url(package_base),
path=".",
branch="master",
source=source,
)
if source == PackageSource.Repository:
from ahriman.core.alpm.remote.official import Official
return RemoteSource(
git_url=Official.remote_git_url(package_base, repository),
web_url=Official.remote_web_url(package_base),
path="trunk",
branch=f"packages/{package_base}",
source=source,
)
return None
def view(self) -> Dict[str, Any]:
"""
generate json package remote view
Returns:
Dict[str, Any]: json-friendly dictionary
"""
return asdict(self)

View File

@ -23,7 +23,7 @@ from enum import Enum
from typing import Type
class ReportSettings(Enum):
class ReportSettings(str, Enum):
"""
report targets enumeration

View File

@ -23,7 +23,7 @@ from enum import Enum
from typing import Type
class SignSettings(Enum):
class SignSettings(str, Enum):
"""
sign targets enumeration

View File

@ -23,7 +23,7 @@ from enum import Enum
from typing import Type
class SmtpSSLSettings(Enum):
class SmtpSSLSettings(str, Enum):
"""
SMTP SSL mode enumeration

View File

@ -23,7 +23,7 @@ from enum import Enum
from typing import Type
class UploadSettings(Enum):
class UploadSettings(str, Enum):
"""
remote synchronization targets enumeration

View File

@ -20,7 +20,7 @@
from enum import Enum
class UserAccess(Enum):
class UserAccess(str, Enum):
"""
web user access enumeration

View File

@ -86,7 +86,7 @@ class IndexView(BaseView):
"status_color": status.status.bootstrap_color(),
"timestamp": pretty_datetime(status.timestamp),
"version": package.version,
"web_url": package.web_url,
"web_url": package.remote.web_url if package.remote is not None else None,
} for package, status in sorted(self.service.packages, key=lambda item: item[0].base)
]
service = {

View File

@ -50,7 +50,7 @@ class SearchView(BaseView):
HTTPNotFound: if no packages found
"""
search: List[str] = self.request.query.getall("for", default=[])
packages = AUR.multisearch(*search)
packages = AUR.multisearch(*search, pacman=self.service.repository.pacman)
if not packages:
raise HTTPNotFound(reason=f"No packages found for terms: {search}")