diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 1f4b5477..4713fdf8 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -446,7 +446,7 @@ def _set_patch_list_parser(root: SubParserAction) -> argparse.ArgumentParser: """ parser = root.add_parser("patch-list", help="list patch sets", description="list available patches for the package", formatter_class=_formatter) - parser.add_argument("package", help="package base", nargs="?") + parser.add_argument("package", help="package base") parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true") parser.add_argument("-v", "--variable", help="if set, show only patches for specified PKGBUILD variables", action="append") diff --git a/src/ahriman/application/application/application.py b/src/ahriman/application/application/application.py index 4ec45a61..11f1d758 100644 --- a/src/ahriman/application/application/application.py +++ b/src/ahriman/application/application/application.py @@ -161,8 +161,7 @@ class Application(ApplicationPackages, ApplicationRepository): package = Package.from_aur(package_name, username) with_dependencies[package.base] = package - # register package in local database - self.database.package_base_update(package) + # register package in the database self.repository.reporter.set_unknown(package) return list(with_dependencies.values()) diff --git a/src/ahriman/application/application/application_packages.py b/src/ahriman/application/application/application_packages.py index 5ff74826..45e62a89 100644 --- a/src/ahriman/application/application/application_packages.py +++ b/src/ahriman/application/application/application_packages.py @@ -65,7 +65,7 @@ class ApplicationPackages(ApplicationProperties): """ package = Package.from_aur(source, username) self.database.build_queue_insert(package) - self.database.package_base_update(package) + self.reporter.set_unknown(package) def _add_directory(self, source: str, *_: Any) -> None: """ @@ -139,7 +139,7 @@ class ApplicationPackages(ApplicationProperties): """ package = Package.from_official(source, self.repository.pacman, username) self.database.build_queue_insert(package) - self.database.package_base_update(package) + self.reporter.set_unknown(package) def add(self, names: Iterable[str], source: PackageSource, username: str | None = None) -> None: """ diff --git a/src/ahriman/application/application/application_properties.py b/src/ahriman/application/application/application_properties.py index 9d5ba208..55a88d09 100644 --- a/src/ahriman/application/application/application_properties.py +++ b/src/ahriman/application/application/application_properties.py @@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.database import SQLite from ahriman.core.log import LazyLogging from ahriman.core.repository import Repository +from ahriman.core.status.client import Client from ahriman.models.pacman_synchronization import PacmanSynchronization from ahriman.models.repository_id import RepositoryId @@ -63,3 +64,13 @@ class ApplicationProperties(LazyLogging): str: repository architecture """ return self.repository_id.architecture + + @property + def reporter(self) -> Client: + """ + instance of the web/database client + + Returns: + Client: repository reposter + """ + return self.repository.reporter diff --git a/src/ahriman/application/application/application_repository.py b/src/ahriman/application/application/application_repository.py index f4547c75..01e43e69 100644 --- a/src/ahriman/application/application/application_repository.py +++ b/src/ahriman/application/application/application_repository.py @@ -39,10 +39,8 @@ class ApplicationRepository(ApplicationProperties): Args: packages(Iterable[Package]): list of packages to retrieve changes """ - last_commit_hashes = self.database.hashes_get() - for package in packages: - last_commit_sha = last_commit_hashes.get(package.base) + last_commit_sha = self.reporter.package_changes_get(package.base).last_commit_sha if last_commit_sha is None: continue # skip check in case if we can't calculate diff diff --git a/src/ahriman/application/handlers/add.py b/src/ahriman/application/handlers/add.py index ac185e1c..d1591a29 100644 --- a/src/ahriman/application/handlers/add.py +++ b/src/ahriman/application/handlers/add.py @@ -50,7 +50,8 @@ class Add(Handler): application.add(args.package, args.source, args.username) patches = [PkgbuildPatch.from_env(patch) for patch in args.variable] if args.variable is not None else [] for package in args.package: # for each requested package insert patch - application.database.patches_insert(package, patches) + for patch in patches: + application.reporter.package_patches_add(package, patch) if not args.now: return diff --git a/src/ahriman/application/handlers/patch.py b/src/ahriman/application/handlers/patch.py index 0f131257..3fcaeea2 100644 --- a/src/ahriman/application/handlers/patch.py +++ b/src/ahriman/application/handlers/patch.py @@ -116,25 +116,29 @@ class Patch(Handler): package_base(str): package base patch(PkgbuildPatch): patch descriptor """ - application.database.patches_insert(package_base, [patch]) + application.reporter.package_patches_add(package_base, patch) @staticmethod - def patch_set_list(application: Application, package_base: str | None, variables: list[str] | None, + def patch_set_list(application: Application, package_base: str, variables: list[str] | None, exit_code: bool) -> None: """ list patches available for the package base Args: application(Application): application instance - package_base(str | None): package base + package_base(str): package base variables(list[str] | None): extract patches only for specified PKGBUILD variables exit_code(bool): exit with error on empty search result """ - patches = application.database.patches_list(package_base, variables) + patches = [] + if variables is not None: + for variable in variables: + patches.extend(application.reporter.package_patches_get(package_base, variable)) + else: + patches = application.reporter.package_patches_get(package_base, variables) Patch.check_if_empty(exit_code, not patches) - for base, patch in patches.items(): - PatchPrinter(base, patch)(verbose=True, separator=" = ") + PatchPrinter(package_base, patches)(verbose=True, separator=" = ") @staticmethod def patch_set_remove(application: Application, package_base: str, variables: list[str] | None) -> None: @@ -146,4 +150,8 @@ class Patch(Handler): package_base(str): package base variables(list[str] | None): remove patches only for specified PKGBUILD variables """ - application.database.patches_remove(package_base, variables) + if variables is not None: + for variable in variables: # iterate over single variable + application.reporter.package_patches_remove(package_base, variable) + else: + application.reporter.package_patches_remove(package_base, variables) # just pass as is diff --git a/src/ahriman/application/handlers/rebuild.py b/src/ahriman/application/handlers/rebuild.py index 5313c5b0..344b18a0 100644 --- a/src/ahriman/application/handlers/rebuild.py +++ b/src/ahriman/application/handlers/rebuild.py @@ -76,7 +76,7 @@ class Rebuild(Handler): if from_database: return [ package - for (package, last_status) in application.database.packages_get() + for (package, last_status) in application.reporter.package_get(None) if status is None or last_status.status == status ] diff --git a/src/ahriman/application/handlers/status_update.py b/src/ahriman/application/handlers/status_update.py index 3b1637b0..161e8772 100644 --- a/src/ahriman/application/handlers/status_update.py +++ b/src/ahriman/application/handlers/status_update.py @@ -56,7 +56,7 @@ class StatusUpdate(Handler): if (local := next((package for package in packages if package.base == base), None)) is not None: client.package_add(local, args.status) else: - client.package_update(base, args.status) + client.package_set(base, args.status) case Action.Update: # update service status client.status_update(args.status) diff --git a/src/ahriman/core/build_tools/task.py b/src/ahriman/core/build_tools/task.py index ea5f05bf..12bf1aa6 100644 --- a/src/ahriman/core/build_tools/task.py +++ b/src/ahriman/core/build_tools/task.py @@ -21,7 +21,6 @@ from pathlib import Path from ahriman.core.build_tools.sources import Sources from ahriman.core.configuration import Configuration -from ahriman.core.database import SQLite from ahriman.core.exceptions import BuildError from ahriman.core.log import LazyLogging from ahriman.core.util import check_output @@ -116,20 +115,20 @@ class Task(LazyLogging): # e.g. in some cases packagelist command produces debug packages which were not actually built return list(filter(lambda path: path.is_file(), map(Path, packages))) - def init(self, sources_dir: Path, database: SQLite, local_version: str | None) -> str | None: + def init(self, sources_dir: Path, patches: list[PkgbuildPatch], local_version: str | None) -> str | None: """ fetch package from git Args: sources_dir(Path): local path to fetch - database(SQLite): database instance + patches(list[PkgbuildPatch]): list of patches for the package local_version(str | None): local version of the package. If set and equal to current version, it will automatically bump pkgrel Returns: str | None: current commit sha if available """ - last_commit_sha = Sources.load(sources_dir, self.package, database.patches_get(self.package.base), self.paths) + last_commit_sha = Sources.load(sources_dir, self.package, patches, self.paths) if local_version is None: return last_commit_sha # there is no local package or pkgrel increment is disabled diff --git a/src/ahriman/core/database/operations/changes_operations.py b/src/ahriman/core/database/operations/changes_operations.py index 53fe6495..cba29cf3 100644 --- a/src/ahriman/core/database/operations/changes_operations.py +++ b/src/ahriman/core/database/operations/changes_operations.py @@ -117,27 +117,3 @@ class ChangesOperations(Operations): }) return self.with_connection(run, commit=True) - - def hashes_get(self, repository_id: RepositoryId | None = None) -> dict[str, str]: - """ - extract last commit hashes if available - - Args: - repository_id(RepositoryId, optional): repository unique identifier override (Default value = None) - - Returns: - dict[str, str]: map of package base to its last commit hash - """ - - repository_id = repository_id or self._repository_id - - def run(connection: Connection) -> dict[str, str]: - return { - row["package_base"]: row["last_commit_sha"] - for row in connection.execute( - """select package_base, last_commit_sha from package_changes where repository = :repository""", - {"repository": repository_id.id} - ) - } - - return self.with_connection(run) diff --git a/src/ahriman/core/database/operations/operations.py b/src/ahriman/core/database/operations/operations.py index 09dbee59..708658fe 100644 --- a/src/ahriman/core/database/operations/operations.py +++ b/src/ahriman/core/database/operations/operations.py @@ -25,7 +25,7 @@ from typing import Any, TypeVar from ahriman.core.log import LazyLogging from ahriman.models.repository_id import RepositoryId - +from ahriman.models.repository_paths import RepositoryPaths T = TypeVar("T") @@ -38,7 +38,7 @@ class Operations(LazyLogging): path(Path): path to the database file """ - def __init__(self, path: Path, repository_id: RepositoryId) -> None: + def __init__(self, path: Path, repository_id: RepositoryId, repository_paths: RepositoryPaths) -> None: """ default constructor @@ -48,6 +48,7 @@ class Operations(LazyLogging): """ self.path = path self._repository_id = repository_id + self._repository_paths = repository_paths @staticmethod def factory(cursor: sqlite3.Cursor, row: tuple[Any, ...]) -> dict[str, Any]: diff --git a/src/ahriman/core/database/operations/package_operations.py b/src/ahriman/core/database/operations/package_operations.py index 812d0009..78c2149d 100644 --- a/src/ahriman/core/database/operations/package_operations.py +++ b/src/ahriman/core/database/operations/package_operations.py @@ -150,34 +150,6 @@ class PackageOperations(Operations): """, package_list) - @staticmethod - def _package_update_insert_status(connection: Connection, package_base: str, status: BuildStatus, - repository_id: RepositoryId) -> None: - """ - insert base package status into table - - Args: - connection(Connection): database connection - package_base(str): package base name - status(BuildStatus): new build status - repository_id(RepositoryId): repository unique identifier - """ - connection.execute( - """ - insert into package_statuses - (package_base, status, last_updated, repository) - values - (:package_base, :status, :last_updated, :repository) - on conflict (package_base, repository) do update set - status = :status, last_updated = :last_updated - """, - { - "package_base": package_base, - "status": status.status.value, - "last_updated": status.timestamp, - "repository": repository_id.id, - }) - @staticmethod def _packages_get_select_package_bases(connection: Connection, repository_id: RepositoryId) -> dict[str, Package]: """ @@ -277,20 +249,18 @@ class PackageOperations(Operations): return self.with_connection(run, commit=True) - def package_update(self, package: Package, status: BuildStatus, repository_id: RepositoryId | None = None) -> None: + def package_update(self, package: Package, repository_id: RepositoryId | None = None) -> None: """ update package status Args: package(Package): package properties - status(BuildStatus): new build status repository_id(RepositoryId, optional): repository unique identifier override (Default value = None) """ repository_id = repository_id or self._repository_id def run(connection: Connection) -> None: self._package_update_insert_base(connection, package, repository_id) - self._package_update_insert_status(connection, package.base, status, repository_id) self._package_update_insert_packages(connection, package, repository_id) self._package_remove_packages(connection, package.base, package.packages.keys(), repository_id) @@ -336,3 +306,33 @@ class PackageOperations(Operations): package_base: package.remote for package_base, package in self.with_connection(run).items() } + + def status_update(self, package_base: str, status: BuildStatus, repository_id: RepositoryId | None = None) -> None: + """ + insert base package status into table + + Args: + package_base(str): package base name + status(BuildStatus): new build status + repository_id(RepositoryId, optional): repository unique identifier override (Default value = None) + """ + repository_id = repository_id or self._repository_id + + def run(connection: Connection) -> None: + connection.execute( + """ + insert into package_statuses + (package_base, status, last_updated, repository) + values + (:package_base, :status, :last_updated, :repository) + on conflict (package_base, repository) do update set + status = :status, last_updated = :last_updated + """, + { + "package_base": package_base, + "status": status.status.value, + "last_updated": status.timestamp, + "repository": repository_id.id, + }) + + return self.with_connection(run, commit=True) diff --git a/src/ahriman/core/database/sqlite.py b/src/ahriman/core/database/sqlite.py index 06bcd2bb..e6d26d7e 100644 --- a/src/ahriman/core/database/sqlite.py +++ b/src/ahriman/core/database/sqlite.py @@ -66,7 +66,7 @@ class SQLite( path = cls.database_path(configuration) _, repository_id = configuration.check_loaded() - database = cls(path, repository_id) + database = cls(path, repository_id, configuration.repository_paths) database.init(configuration) return database @@ -119,3 +119,6 @@ class SQLite( self.logs_remove(package_base, None) self.changes_remove(package_base) self.dependencies_remove(package_base) + + # remove local cache too + self._repository_paths.tree_clear(package_base) diff --git a/src/ahriman/core/gitremote/remote_push.py b/src/ahriman/core/gitremote/remote_push.py index d51cc913..b289b688 100644 --- a/src/ahriman/core/gitremote/remote_push.py +++ b/src/ahriman/core/gitremote/remote_push.py @@ -25,9 +25,9 @@ from tempfile import TemporaryDirectory from ahriman.core.build_tools.sources import Sources from ahriman.core.configuration import Configuration -from ahriman.core.database import SQLite from ahriman.core.exceptions import GitRemoteError from ahriman.core.log import LazyLogging +from ahriman.core.status.client import Client from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.remote_source import RemoteSource @@ -40,20 +40,20 @@ class RemotePush(LazyLogging): Attributes: commit_author(tuple[str, str] | None): optional commit author in form of git config - database(SQLite): database instance remote_source(RemoteSource): repository remote source (remote pull url and branch) + reporter(Client): reporter client used for additional information retrieval """ - def __init__(self, database: SQLite, configuration: Configuration, section: str) -> None: + def __init__(self, reporter: Client, configuration: Configuration, section: str) -> None: """ default constructor Args: - database(SQLite): database instance + reporter(Client): reporter client configuration(Configuration): configuration instance section(str): settings section name """ - self.database = database + self.reporter = reporter commit_email = configuration.get(section, "commit_email", fallback="ahriman@localhost") commit_user = configuration.get(section, "commit_user", fallback="ahriman") @@ -92,7 +92,7 @@ class RemotePush(LazyLogging): else: shutil.rmtree(git_file) # ...copy all patches... - for patch in self.database.patches_get(package.base): + for patch in self.reporter.package_patches_get(package.base, None): filename = f"ahriman-{package.base}.patch" if patch.key is None else f"ahriman-{patch.key}.patch" patch.write(package_target_dir / filename) # ...and finally return path to the copied directory diff --git a/src/ahriman/core/gitremote/remote_push_trigger.py b/src/ahriman/core/gitremote/remote_push_trigger.py index 44aa9166..d589001a 100644 --- a/src/ahriman/core/gitremote/remote_push_trigger.py +++ b/src/ahriman/core/gitremote/remote_push_trigger.py @@ -19,8 +19,8 @@ # from ahriman.core import context from ahriman.core.configuration import Configuration -from ahriman.core.database import SQLite from ahriman.core.gitremote.remote_push import RemotePush +from ahriman.core.status.client import Client from ahriman.core.triggers import Trigger from ahriman.models.package import Package from ahriman.models.repository_id import RepositoryId @@ -110,10 +110,10 @@ class RemotePushTrigger(Trigger): GitRemoteError: if database is not set in context """ ctx = context.get() - database = ctx.get(SQLite) + reporter = ctx.get(Client) for target in self.targets: section, _ = self.configuration.gettype( target, self.repository_id, fallback=self.CONFIGURATION_SCHEMA_FALLBACK) - runner = RemotePush(database, self.configuration, section) + runner = RemotePush(reporter, self.configuration, section) runner.run(result) diff --git a/src/ahriman/core/log/http_log_handler.py b/src/ahriman/core/log/http_log_handler.py index f2b0fde3..dba4245b 100644 --- a/src/ahriman/core/log/http_log_handler.py +++ b/src/ahriman/core/log/http_log_handler.py @@ -92,7 +92,7 @@ class HttpLogHandler(logging.Handler): return # in case if no package base supplied we need just skip log message try: - self.reporter.package_logs(log_record_id, record) + self.reporter.package_logs_add(log_record_id, record.created, record.getMessage()) except Exception: if self.suppress_errors: return diff --git a/src/ahriman/core/repository/executor.py b/src/ahriman/core/repository/executor.py index 43033ab1..a07257a1 100644 --- a/src/ahriman/core/repository/executor.py +++ b/src/ahriman/core/repository/executor.py @@ -58,7 +58,8 @@ class Executor(PackageInfo, Cleaner): self.reporter.set_building(package.base) task = Task(package, self.configuration, self.architecture, self.paths) local_version = local_versions.get(package.base) if bump_pkgrel else None - commit_sha = task.init(local_path, self.database, local_version) + patches = self.reporter.package_patches_get(package.base, None) + commit_sha = task.init(local_path, patches, local_version) built = task.build(local_path, PACKAGER=packager_id) for src in built: dst = self.paths.packages / src.name @@ -80,7 +81,7 @@ class Executor(PackageInfo, Cleaner): self.reporter.package_changes_set(single.base, Changes(last_commit_sha)) # update dependencies list dependencies = PackageArchive(self.paths.build_directory, single).depends_on() - self.database.dependencies_insert(dependencies) + self.reporter.package_dependencies_set(dependencies) # update result set result.add_updated(single) except Exception: @@ -102,9 +103,7 @@ class Executor(PackageInfo, Cleaner): """ def remove_base(package_base: str) -> None: try: - self.paths.tree_clear(package_base) # remove all internal files - self.database.package_clear(package_base) - self.reporter.package_remove(package_base) # we only update status page in case of base removal + self.reporter.package_remove(package_base) except Exception: self.logger.exception("could not remove base %s", package_base) diff --git a/src/ahriman/core/repository/package_info.py b/src/ahriman/core/repository/package_info.py index 56a6fef1..6c547adf 100644 --- a/src/ahriman/core/repository/package_info.py +++ b/src/ahriman/core/repository/package_info.py @@ -43,15 +43,14 @@ class PackageInfo(RepositoryProperties): Returns: list[Package]: list of read packages """ - sources = self.database.remotes_get() - result: dict[str, Package] = {} # we are iterating over bases, not single packages for full_path in packages: try: local = Package.from_archive(full_path, self.pacman) - if (source := sources.get(local.base)) is not None: - local.remote = source + remote, _ = next(iter(self.reporter.package_get(local.base)), (None, None)) + if remote is not None: # update source with remote + local.remote = remote.remote current = result.setdefault(local.base, local) if current.version != local.version: @@ -78,7 +77,8 @@ class PackageInfo(RepositoryProperties): """ with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name: dir_path = Path(dir_name) - current_commit_sha = Sources.load(dir_path, package, self.database.patches_get(package.base), self.paths) + patches = self.reporter.package_patches_get(package.base, None) + current_commit_sha = Sources.load(dir_path, package, patches, self.paths) changes: str | None = None if current_commit_sha != last_commit_sha: diff --git a/src/ahriman/core/repository/repository.py b/src/ahriman/core/repository/repository.py index f9c97907..a784e7a9 100644 --- a/src/ahriman/core/repository/repository.py +++ b/src/ahriman/core/repository/repository.py @@ -26,6 +26,7 @@ from ahriman.core.database import SQLite from ahriman.core.repository.executor import Executor from ahriman.core.repository.update_handler import UpdateHandler from ahriman.core.sign.gpg import GPG +from ahriman.core.status.client import Client from ahriman.models.pacman_synchronization import PacmanSynchronization from ahriman.models.repository_id import RepositoryId @@ -92,6 +93,7 @@ class Repository(Executor, UpdateHandler): ctx.set(Configuration, self.configuration) ctx.set(Pacman, self.pacman) ctx.set(GPG, self.sign) + ctx.set(Client, self.reporter) ctx.set(type(self), self) diff --git a/src/ahriman/core/repository/repository_properties.py b/src/ahriman/core/repository/repository_properties.py index 714b3c6d..42f30a15 100644 --- a/src/ahriman/core/repository/repository_properties.py +++ b/src/ahriman/core/repository/repository_properties.py @@ -75,7 +75,7 @@ class RepositoryProperties(LazyLogging): self.pacman = Pacman(repository_id, configuration, refresh_database=refresh_pacman_database) self.sign = GPG(configuration) self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args) - self.reporter = Client.load(repository_id, configuration, report=report) + self.reporter = Client.load(repository_id, configuration, database, report=report) self.triggers = TriggerLoader.load(repository_id, configuration) @property diff --git a/src/ahriman/core/repository/update_handler.py b/src/ahriman/core/repository/update_handler.py index ca79f8b4..cc8bea5c 100644 --- a/src/ahriman/core/repository/update_handler.py +++ b/src/ahriman/core/repository/update_handler.py @@ -98,18 +98,16 @@ class UpdateHandler(PackageInfo, Cleaner): return files - dependencies = {dependency.package_base: dependency for dependency in self.database.dependencies_get()} - result: list[Package] = [] for package in self.packages(filter_packages): - if package.base not in dependencies: + dependencies = next(iter(self.reporter.package_dependencies_get(package.base)), None) + if dependencies is None: continue # skip check if no package dependencies found - required = dependencies[package.base].paths - required_packages = {dep for dep_packages in required.values() for dep in dep_packages} + required_packages = {dep for dep_packages in dependencies.paths.values() for dep in dep_packages} filesystem = extract_files(required_packages) - for path, packages in required.items(): + for path, packages in dependencies.paths.items(): found = filesystem.get(path, set()) if found.intersection(packages): continue diff --git a/src/ahriman/core/status/client.py b/src/ahriman/core/status/client.py index 585f2aec..8a7a0362 100644 --- a/src/ahriman/core/status/client.py +++ b/src/ahriman/core/status/client.py @@ -17,16 +17,18 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # +# pylint: disable=too-many-public-methods from __future__ import annotations -import logging - from ahriman.core.configuration import Configuration +from ahriman.core.database import SQLite from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.changes import Changes +from ahriman.models.dependencies import Dependencies from ahriman.models.internal_status import InternalStatus from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package +from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.repository_id import RepositoryId @@ -36,22 +38,31 @@ class Client: """ @staticmethod - def load(repository_id: RepositoryId, configuration: Configuration, *, report: bool) -> Client: + def load(repository_id: RepositoryId, configuration: Configuration, database: SQLite | None = None, *, + report: bool = True) -> Client: """ load client from settings Args: repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance - report(bool): force enable or disable reporting + database(SQLite | None, optional): database instance (Default value = None) + report(bool, optional): force enable or disable reporting (Default value = True) Returns: Client: client according to current settings """ + def make_local_client() -> Client: + if database is None: + return Client() + + from ahriman.core.status.local_client import LocalClient + return LocalClient(repository_id, database) + if not report: - return Client() + return make_local_client() if not configuration.getboolean("status", "enabled", fallback=True): # global switch - return Client() + return make_local_client() # new-style section address = configuration.get("status", "address", fallback=None) @@ -65,7 +76,8 @@ class Client: if address or legacy_address or (host and port) or socket: from ahriman.core.status.web_client import WebClient return WebClient(repository_id, configuration) - return Client() + + return make_local_client() def package_add(self, package: Package, status: BuildStatusEnum) -> None: """ @@ -74,7 +86,11 @@ class Client: Args: package(Package): package properties status(BuildStatusEnum): current package build status + + Raises: + NotImplementedError: not implemented method """ + raise NotImplementedError def package_changes_get(self, package_base: str) -> Changes: """ @@ -85,9 +101,11 @@ class Client: Returns: Changes: package changes if available and empty object otherwise + + Raises: + NotImplementedError: not implemented method """ - del package_base - return Changes() + raise NotImplementedError def package_changes_set(self, package_base: str, changes: Changes) -> None: """ @@ -96,7 +114,38 @@ class Client: Args: package_base(str): package base to update changes(Changes): changes descriptor + + Raises: + NotImplementedError: not implemented method """ + raise NotImplementedError + + def package_dependencies_get(self, package_base: str | None) -> list[Dependencies]: + """ + get package dependencies + + Args: + package_base(str | None): package base to retrieve + + Returns: + list[Dependencies]: package implicit dependencies if available + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_dependencies_set(self, dependencies: Dependencies) -> None: + """ + update package dependencies + + Args: + dependencies(Dependencies): dependencies descriptor + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError def package_get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]: """ @@ -107,35 +156,118 @@ class Client: Returns: list[tuple[Package, BuildStatus]]: list of current package description and status if it has been found - """ - del package_base - return [] - def package_logs(self, log_record_id: LogRecordId, record: logging.LogRecord) -> None: + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: """ post log record Args: log_record_id(LogRecordId): log record id - record(logging.LogRecord): log record to post to api + created(float): log created timestamp + message(str): log message """ + def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: + """ + get package logs + + Args: + package_base(str): package base + limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1) + offset(int, optional): records offset (Default value = 0) + + Returns: + list[tuple[float, str]]: package logs + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_logs_remove(self, package_base: str, version: str | None) -> None: + """ + remove package logs + + Args: + package_base(str): package base + version(str | None): package version to remove logs. If None set, all logs will be removed + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_patches_add(self, package_base: str, patch: PkgbuildPatch) -> None: + """ + create or update package patch + + Args: + package_base(str): package base to update + patch(PkgbuildPatch): package patch + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_patches_get(self, package_base: str, variable: str | None) -> list[PkgbuildPatch]: + """ + get package patches + + Args: + package_base(str): package base to retrieve + variable(str | None): optional filter by patch variable + + Returns: + list[PkgbuildPatch]: list of patches for the specified package + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_patches_remove(self, package_base: str, variable: str | None) -> None: + """ + remove package patch + + Args: + package_base(str): package base to update + variable(str | None): patch name. If None set, all patches will be removed + + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + def package_remove(self, package_base: str) -> None: """ remove packages from watcher Args: package_base(str): package base to remove - """ - def package_update(self, package_base: str, status: BuildStatusEnum) -> None: + Raises: + NotImplementedError: not implemented method + """ + raise NotImplementedError + + def package_set(self, package_base: str, status: BuildStatusEnum) -> None: """ update package build status. Unlike :func:`package_add()` it does not update package properties Args: package_base(str): package base to update status(BuildStatusEnum): current package build status + + Raises: + NotImplementedError: not implemented method """ + raise NotImplementedError def set_building(self, package_base: str) -> None: """ @@ -144,7 +276,7 @@ class Client: Args: package_base(str): package base to update """ - return self.package_update(package_base, BuildStatusEnum.Building) + return self.package_set(package_base, BuildStatusEnum.Building) def set_failed(self, package_base: str) -> None: """ @@ -153,7 +285,7 @@ class Client: Args: package_base(str): package base to update """ - return self.package_update(package_base, BuildStatusEnum.Failed) + return self.package_set(package_base, BuildStatusEnum.Failed) def set_pending(self, package_base: str) -> None: """ @@ -162,7 +294,7 @@ class Client: Args: package_base(str): package base to update """ - return self.package_update(package_base, BuildStatusEnum.Pending) + return self.package_set(package_base, BuildStatusEnum.Pending) def set_success(self, package: Package) -> None: """ diff --git a/src/ahriman/core/status/local_client.py b/src/ahriman/core/status/local_client.py new file mode 100644 index 00000000..90b7e51a --- /dev/null +++ b/src/ahriman/core/status/local_client.py @@ -0,0 +1,207 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from ahriman.core.database import SQLite +from ahriman.core.status.client import Client +from ahriman.models.build_status import BuildStatus, BuildStatusEnum +from ahriman.models.changes import Changes +from ahriman.models.dependencies import Dependencies +from ahriman.models.log_record_id import LogRecordId +from ahriman.models.package import Package +from ahriman.models.pkgbuild_patch import PkgbuildPatch +from ahriman.models.repository_id import RepositoryId + + +class LocalClient(Client): + """ + local database handler + + Attributes: + database(SQLite): database instance + repository_id(RepositoryId): repository unique identifier + """ + + def __init__(self, repository_id: RepositoryId, database: SQLite) -> None: + """ + default constructor + + Args: + repository_id(RepositoryId): repository unique identifier + database(SQLite): database instance: + """ + self.database = database + self.repository_id = repository_id + + def package_add(self, package: Package, status: BuildStatusEnum) -> None: + """ + add new package with status + + Args: + package(Package): package properties + status(BuildStatusEnum): current package build status + """ + self.database.package_update(package, self.repository_id) + self.database.status_update(package.base, BuildStatus(status), self.repository_id) + + def package_changes_get(self, package_base: str) -> Changes: + """ + get package changes + + Args: + package_base(str): package base to retrieve + + Returns: + Changes: package changes if available and empty object otherwise + """ + return self.database.changes_get(package_base, self.repository_id) + + def package_changes_set(self, package_base: str, changes: Changes) -> None: + """ + update package changes + + Args: + package_base(str): package base to update + changes(Changes): changes descriptor + """ + self.database.changes_insert(package_base, changes, self.repository_id) + + def package_dependencies_get(self, package_base: str | None) -> list[Dependencies]: + """ + get package dependencies + + Args: + package_base(str | None): package base to retrieve + + Returns: + list[Dependencies]: package implicit dependencies if available + """ + return self.database.dependencies_get(package_base, self.repository_id) + + def package_dependencies_set(self, dependencies: Dependencies) -> None: + """ + update package dependencies + + Args: + dependencies(Dependencies): dependencies descriptor + """ + self.database.dependencies_insert(dependencies, self.repository_id) + + def package_get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]: + """ + get package status + + Args: + package_base(str | None): package base to get + + Returns: + list[tuple[Package, BuildStatus]]: list of current package description and status if it has been found + """ + packages = self.database.packages_get(self.repository_id) + if package_base is None: + return packages + return [(package, status) for package, status in packages if package.base == package_base] + + def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: + """ + post log record + + Args: + log_record_id(LogRecordId): log record id + created(float): log created timestamp + message(str): log message + """ + self.database.logs_insert(log_record_id, created, message, self.repository_id) + + def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: + """ + get package logs + + Args: + package_base(str): package base + limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1) + offset(int, optional): records offset (Default value = 0) + + Returns: + list[tuple[float, str]]: package logs + """ + return self.database.logs_get(package_base, limit, offset, self.repository_id) + + def package_logs_remove(self, package_base: str, version: str | None) -> None: + """ + remove package logs + + Args: + package_base(str): package base + version(str | None): package version to remove logs. If None set, all logs will be removed + """ + self.database.logs_remove(package_base, version, self.repository_id) + + def package_patches_add(self, package_base: str, patch: PkgbuildPatch) -> None: + """ + create or update package patch + + Args: + package_base(str): package base to update + patch(PkgbuildPatch): package patch + """ + self.database.patches_insert(package_base, [patch]) + + def package_patches_get(self, package_base: str, variable: str | None) -> list[PkgbuildPatch]: + """ + get package patches + + Args: + package_base(str): package base to retrieve + variable(str | None): optional filter by patch variable + + Returns: + list[PkgbuildPatch]: list of patches for the specified package + """ + variables = [variable] if variable is not None else None + return self.database.patches_list(package_base, variables).get(package_base, []) + + def package_patches_remove(self, package_base: str, variable: str | None) -> None: + """ + remove package patch + + Args: + package_base(str): package base to update + variable(str | None): patch name. If None set, all patches will be removed + """ + variables = [variable] if variable is not None else None + self.database.patches_remove(package_base, variables) + + def package_remove(self, package_base: str) -> None: + """ + remove packages from watcher + + Args: + package_base(str): package base to remove + """ + self.database.package_clear(package_base) + + def package_set(self, package_base: str, status: BuildStatusEnum) -> None: + """ + update package build status. Unlike :func:`package_add()` it does not update package properties + + Args: + package_base(str): package base to update + status(BuildStatusEnum): current package build status + """ + self.database.status_update(package_base, BuildStatus(status), self.repository_id) diff --git a/src/ahriman/core/status/watcher.py b/src/ahriman/core/status/watcher.py index 7b36dae8..e44316f3 100644 --- a/src/ahriman/core/status/watcher.py +++ b/src/ahriman/core/status/watcher.py @@ -19,15 +19,15 @@ # from threading import Lock -from ahriman.core.database import SQLite from ahriman.core.exceptions import UnknownPackageError from ahriman.core.log import LazyLogging +from ahriman.core.status.client import Client from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.changes import Changes +from ahriman.models.dependencies import Dependencies from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch -from ahriman.models.repository_id import RepositoryId class Watcher(LazyLogging): @@ -35,21 +35,18 @@ class Watcher(LazyLogging): package status watcher Attributes: - database(SQLite): database instance - repository_id(RepositoryId): repository unique identifier + client(Client): reporter instance status(BuildStatus): daemon status """ - def __init__(self, repository_id: RepositoryId, database: SQLite) -> None: + def __init__(self, client: Client) -> None: """ default constructor Args: - repository_id(RepositoryId): repository unique identifier - database(SQLite): database instance + client(Client): reporter instance """ - self.repository_id = repository_id - self.database = database + self.client = client self._lock = Lock() self._known: dict[str, tuple[Package, BuildStatus]] = {} @@ -76,7 +73,7 @@ class Watcher(LazyLogging): with self._lock: self._known = { package.base: (package, status) - for package, status in self.database.packages_get(self.repository_id) + for package, status in self.client.package_get(None) } def logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: @@ -91,8 +88,8 @@ class Watcher(LazyLogging): Returns: list[tuple[float, str]]: package logs """ - self.package_get(package_base) - return self.database.logs_get(package_base, limit, offset, self.repository_id) + _ = self.package_get(package_base) + return self.client.package_logs_get(package_base, limit, offset) def logs_remove(self, package_base: str, version: str | None) -> None: """ @@ -100,24 +97,24 @@ class Watcher(LazyLogging): Args: package_base(str): package base - version(str): package versio + version(str): package version """ - self.database.logs_remove(package_base, version, self.repository_id) + self.client.package_logs_remove(package_base, version) - def logs_update(self, log_record_id: LogRecordId, created: float, record: str) -> None: + def logs_update(self, log_record_id: LogRecordId, created: float, message: str) -> None: """ make new log record into database Args: log_record_id(LogRecordId): log record id created(float): log created timestamp - record(str): log record + message(str): log message """ if self._last_log_record_id != log_record_id: # there is new log record, so we remove old ones self.logs_remove(log_record_id.package_base, log_record_id.version) self._last_log_record_id = log_record_id - self.database.logs_insert(log_record_id, created, record, self.repository_id) + self.client.package_logs_add(log_record_id, created, message) def package_changes_get(self, package_base: str) -> Changes: """ @@ -129,8 +126,35 @@ class Watcher(LazyLogging): Returns: Changes: package changes if available """ - self.package_get(package_base) - return self.database.changes_get(package_base, self.repository_id) + _ = self.package_get(package_base) + return self.client.package_changes_get(package_base) + + def package_dependencies_get(self, package_base: str) -> Dependencies: + """ + retrieve package dependencies + + Args: + package_base(str): package base + + Returns: + Dependencies: package dependencies if available + """ + _ = self.package_get(package_base) + try: + return next(iter(self.client.package_dependencies_get(package_base))) + except StopIteration: + return Dependencies(package_base) + + def package_dependencies_set(self, package_base: str, dependencies: Dependencies) -> None: + """ + update package dependencies + + Args: + package_base(str): package base + dependencies(Dependencies): package dependencies + """ + _ = self.package_get(package_base) + self.client.package_dependencies_set(dependencies) def package_get(self, package_base: str) -> tuple[Package, BuildStatus]: """ @@ -160,7 +184,7 @@ class Watcher(LazyLogging): """ with self._lock: self._known.pop(package_base, None) - self.database.package_remove(package_base, self.repository_id) + self.client.package_remove(package_base) self.logs_remove(package_base, None) def package_update(self, package_base: str, status: BuildStatusEnum, package: Package | None) -> None: @@ -174,10 +198,9 @@ class Watcher(LazyLogging): """ if package is None: package, _ = self.package_get(package_base) - full_status = BuildStatus(status) with self._lock: - self._known[package_base] = (package, full_status) - self.database.package_update(package, full_status, self.repository_id) + self._known[package_base] = (package, BuildStatus(status)) + self.client.package_set(package_base, status) def patches_get(self, package_base: str, variable: str | None) -> list[PkgbuildPatch]: """ @@ -192,8 +215,7 @@ class Watcher(LazyLogging): """ # patches are package base based, we don't know (and don't differentiate) to which package does them belong # so here we skip checking if package exists or not - variables = [variable] if variable is not None else None - return self.database.patches_list(package_base, variables).get(package_base, []) + return self.client.package_patches_get(package_base, variable) def patches_remove(self, package_base: str, variable: str) -> None: """ @@ -203,7 +225,7 @@ class Watcher(LazyLogging): package_base(str): package base variable(str): patch variable name """ - self.database.patches_remove(package_base, [variable]) + self.client.package_patches_remove(package_base, variable) def patches_update(self, package_base: str, patch: PkgbuildPatch) -> None: """ @@ -213,7 +235,7 @@ class Watcher(LazyLogging): package_base(str): package base patch(PkgbuildPatch): package patch """ - self.database.patches_insert(package_base, [patch]) + self.client.package_patches_add(package_base, patch) def status_update(self, status: BuildStatusEnum) -> None: """ diff --git a/src/ahriman/core/status/web_client.py b/src/ahriman/core/status/web_client.py index 45a05a90..9fec2e86 100644 --- a/src/ahriman/core/status/web_client.py +++ b/src/ahriman/core/status/web_client.py @@ -18,7 +18,6 @@ # along with this program. If not, see . # import contextlib -import logging from urllib.parse import quote_plus as urlencode @@ -27,9 +26,11 @@ from ahriman.core.http import SyncAhrimanClient from ahriman.core.status.client import Client from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.changes import Changes +from ahriman.models.dependencies import Dependencies from ahriman.models.internal_status import InternalStatus from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package +from ahriman.models.pkgbuild_patch import PkgbuildPatch from ahriman.models.repository_id import RepositoryId @@ -92,10 +93,22 @@ class WebClient(Client, SyncAhrimanClient): package_base(str): package base Returns: - str: full url for web service for logs + str: full url for web service for changes """ return f"{self.address}/api/v1/packages/{urlencode(package_base)}/changes" + def _dependencies_url(self, package_base: str = "") -> str: + """ + get url for the dependencies api + + Args: + package_base(str, optional): package base (Default value = "") + + Returns: + str: full url for web service for dependencies + """ + return f"{self.address}/api/v1/dependencies/{urlencode(package_base)}" + def _logs_url(self, package_base: str) -> str: """ get url for the logs api @@ -110,7 +123,7 @@ class WebClient(Client, SyncAhrimanClient): def _package_url(self, package_base: str = "") -> str: """ - url generator + package url generator Args: package_base(str, optional): package base to generate url (Default value = "") @@ -121,6 +134,20 @@ class WebClient(Client, SyncAhrimanClient): suffix = f"/{urlencode(package_base)}" if package_base else "" return f"{self.address}/api/v1/packages{suffix}" + def _patches_url(self, package_base: str, variable: str = "") -> str: + """ + patches url generator + + Args: + package_base(str): package base + variable(str, optional): patch variable name to generate url (Default value = "") + + Returns: + str: full url of web service for the package patch + """ + suffix = f"/{urlencode(variable)}" if variable else "" + return f"{self.address}/api/v1/packages/{urlencode(package_base)}/patches{suffix}" + def _status_url(self) -> str: """ get url for the status api @@ -177,6 +204,37 @@ class WebClient(Client, SyncAhrimanClient): self.make_request("POST", self._changes_url(package_base), params=self.repository_id.query(), json=changes.view()) + def package_dependencies_get(self, package_base: str | None) -> list[Dependencies]: + """ + get package dependencies + + Args: + package_base(str | None): package base to retrieve + + Returns: + list[Dependencies]: package implicit dependencies if available + """ + with contextlib.suppress(Exception): + response = self.make_request("GET", self._dependencies_url(package_base or ""), + params=self.repository_id.query()) + response_json = response.json() + + dependencies = response_json if package_base is None else [response_json] + return [Dependencies.from_json(dependencies) for dependencies in dependencies] + + return [] + + def package_dependencies_set(self, dependencies: Dependencies) -> None: + """ + update package dependencies + + Args: + dependencies(Dependencies): dependencies descriptor + """ + with contextlib.suppress(Exception): + self.make_request("POST", self._dependencies_url(dependencies.package_base), + params=self.repository_id.query(), json=dependencies.view()) + def package_get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]: """ get package status @@ -199,17 +257,18 @@ class WebClient(Client, SyncAhrimanClient): return [] - def package_logs(self, log_record_id: LogRecordId, record: logging.LogRecord) -> None: + def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None: """ post log record Args: log_record_id(LogRecordId): log record id - record(logging.LogRecord): log record to post to api + created(float): log created timestamp + message(str): log message """ payload = { - "created": record.created, - "message": record.getMessage(), + "created": created, + "message": message, "version": log_record_id.version, } @@ -219,6 +278,83 @@ class WebClient(Client, SyncAhrimanClient): self.make_request("POST", self._logs_url(log_record_id.package_base), params=self.repository_id.query(), json=payload, suppress_errors=True) + def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]: + """ + get package logs + + Args: + package_base(str): package base + limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1) + offset(int, optional): records offset (Default value = 0) + + Returns: + list[tuple[float, str]]: package logs + """ + with contextlib.suppress(Exception): + query = self.repository_id.query() + [("limit", str(limit)), ("offset", str(offset))] + response = self.make_request("GET", self._logs_url(package_base), params=query) + response_json = response.json() + + return [(record["created"], record["message"]) for record in response_json] + + return [] + + def package_logs_remove(self, package_base: str, version: str | None) -> None: + """ + remove package logs + + Args: + package_base(str): package base + version(str | None): package version to remove logs. If None set, all logs will be removed + """ + with contextlib.suppress(Exception): + query = self.repository_id.query() + if version is not None: + query += [("version", version)] + self.make_request("DELETE", self._logs_url(package_base), params=query) + + def package_patches_add(self, package_base: str, patch: PkgbuildPatch) -> None: + """ + create or update package patch + + Args: + package_base(str): package base to update + patch(PkgbuildPatch): package patch + """ + with contextlib.suppress(Exception): + self.make_request("POST", self._patches_url(package_base), json=patch.view()) + + def package_patches_get(self, package_base: str, variable: str | None) -> list[PkgbuildPatch]: + """ + get package patches + + Args: + package_base(str): package base to retrieve + variable(str | None): optional filter by patch variable + + Returns: + list[PkgbuildPatch]: list of patches for the specified package + """ + with contextlib.suppress(Exception): + response = self.make_request("GET", self._patches_url(package_base, variable or "")) + response_json = response.json() + + patches = response_json if variable is None else [response_json] + return [PkgbuildPatch.from_json(patch) for patch in patches] + + return [] + + def package_patches_remove(self, package_base: str, variable: str | None) -> None: + """ + remove package patch + + Args: + package_base(str): package base to update + variable(str | None): patch name. If None set, all patches will be removed + """ + with contextlib.suppress(Exception): + self.make_request("DELETE", self._patches_url(package_base, variable or "")) + def package_remove(self, package_base: str) -> None: """ remove packages from watcher @@ -229,7 +365,7 @@ class WebClient(Client, SyncAhrimanClient): with contextlib.suppress(Exception): self.make_request("DELETE", self._package_url(package_base), params=self.repository_id.query()) - def package_update(self, package_base: str, status: BuildStatusEnum) -> None: + def package_set(self, package_base: str, status: BuildStatusEnum) -> None: """ update package build status. Unlike :func:`package_add()` it does not update package properties diff --git a/src/ahriman/core/support/package_creator.py b/src/ahriman/core/support/package_creator.py index d33248ca..28019dc0 100644 --- a/src/ahriman/core/support/package_creator.py +++ b/src/ahriman/core/support/package_creator.py @@ -18,13 +18,13 @@ # along with this program. If not, see . # import shutil +from pathlib import Path from ahriman.core import context from ahriman.core.build_tools.sources import Sources from ahriman.core.configuration import Configuration -from ahriman.core.database import SQLite +from ahriman.core.status.client import Client from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator -from ahriman.models.build_status import BuildStatus from ahriman.models.package import Package @@ -48,23 +48,39 @@ class PackageCreator: self.configuration = configuration self.generator = generator + def package_create(self, path: Path) -> None: + """ + create package files + + Args: + path(Path): path to directory with package files + """ + # clear old tree if any + shutil.rmtree(path, ignore_errors=True) + + # create local tree + path.mkdir(mode=0o755, parents=True, exist_ok=True) + self.generator.write_pkgbuild(path) + Sources.init(path) + + def package_register(self, path: Path) -> None: + """ + register package in build worker + + Args: + path(Path): path to directory with package files + """ + ctx = context.get() + reporter = ctx.get(Client) + _, repository_id = self.configuration.check_loaded() + package = Package.from_build(path, repository_id.architecture, None) + + reporter.set_unknown(package) + def run(self) -> None: """ create new local package """ local_path = self.configuration.repository_paths.cache_for(self.generator.pkgname) - - # clear old tree if any - shutil.rmtree(local_path, ignore_errors=True) - - # create local tree - local_path.mkdir(mode=0o755, parents=True, exist_ok=True) - self.generator.write_pkgbuild(local_path) - Sources.init(local_path) - - # register package - ctx = context.get() - database = ctx.get(SQLite) - _, repository_id = self.configuration.check_loaded() - package = Package.from_build(local_path, repository_id.architecture, None) - database.package_update(package, BuildStatus()) + self.package_create(local_path) + self.package_register(local_path) diff --git a/src/ahriman/models/dependencies.py b/src/ahriman/models/dependencies.py index fed89841..685e12b7 100644 --- a/src/ahriman/models/dependencies.py +++ b/src/ahriman/models/dependencies.py @@ -17,8 +17,11 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields from pathlib import Path +from typing import Any, Self + +from ahriman.core.util import dataclass_view, filter_json @dataclass(frozen=True) @@ -33,3 +36,27 @@ class Dependencies: package_base: str paths: dict[Path, list[str]] = field(default_factory=dict) + + @classmethod + def from_json(cls, dump: dict[str, Any]) -> Self: + """ + construct dependencies from the json dump + + Args: + dump(dict[str, Any]): json dump body + + Returns: + Self: dependencies object + """ + # filter to only known fields + known_fields = [pair.name for pair in fields(cls)] + return cls(**filter_json(dump, known_fields)) + + def view(self) -> dict[str, Any]: + """ + generate json dependencies view + + Returns: + dict[str, Any]: json-friendly dictionary + """ + return dataclass_view(self) diff --git a/src/ahriman/models/pkgbuild_patch.py b/src/ahriman/models/pkgbuild_patch.py index b0ba710b..def9b14e 100644 --- a/src/ahriman/models/pkgbuild_patch.py +++ b/src/ahriman/models/pkgbuild_patch.py @@ -19,11 +19,11 @@ # import shlex -from dataclasses import dataclass +from dataclasses import dataclass, fields from pathlib import Path from typing import Any, Generator, Self -from ahriman.core.util import dataclass_view +from ahriman.core.util import dataclass_view, filter_json @dataclass(frozen=True) @@ -84,6 +84,21 @@ class PkgbuildPatch: raw_value = next(iter(value_parts), "") # extract raw value return cls(key, cls.parse(raw_value)) + @classmethod + def from_json(cls, dump: dict[str, Any]) -> Self: + """ + construct patch descriptor from the json dump + + Args: + dump(dict[str, Any]): json dump body + + Returns: + Self: patch object + """ + # filter to only known fields + known_fields = [pair.name for pair in fields(cls)] + return cls(**filter_json(dump, known_fields)) + @staticmethod def parse(source: str) -> str | list[str]: """ diff --git a/src/ahriman/web/schemas/__init__.py b/src/ahriman/web/schemas/__init__.py index 3b3e4bd6..aac967f8 100644 --- a/src/ahriman/web/schemas/__init__.py +++ b/src/ahriman/web/schemas/__init__.py @@ -22,6 +22,7 @@ from ahriman.web.schemas.auth_schema import AuthSchema from ahriman.web.schemas.build_options_schema import BuildOptionsSchema from ahriman.web.schemas.changes_schema import ChangesSchema from ahriman.web.schemas.counters_schema import CountersSchema +from ahriman.web.schemas.dependencies_schema import DependenciesSchema from ahriman.web.schemas.error_schema import ErrorSchema from ahriman.web.schemas.file_schema import FileSchema from ahriman.web.schemas.info_schema import InfoSchema @@ -36,6 +37,7 @@ from ahriman.web.schemas.package_patch_schema import PackagePatchSchema from ahriman.web.schemas.package_properties_schema import PackagePropertiesSchema from ahriman.web.schemas.package_schema import PackageSchema from ahriman.web.schemas.package_status_schema import PackageStatusSchema, PackageStatusSimplifiedSchema +from ahriman.web.schemas.package_version_schema import PackageVersionSchema from ahriman.web.schemas.pagination_schema import PaginationSchema from ahriman.web.schemas.patch_name_schema import PatchNameSchema from ahriman.web.schemas.patch_schema import PatchSchema diff --git a/src/ahriman/web/schemas/dependencies_schema.py b/src/ahriman/web/schemas/dependencies_schema.py new file mode 100644 index 00000000..87cd496b --- /dev/null +++ b/src/ahriman/web/schemas/dependencies_schema.py @@ -0,0 +1,35 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from marshmallow import Schema, fields + + +class DependenciesSchema(Schema): + """ + request/response package dependencies schema + """ + + package_base = fields.String(metadata={ + "description": "Package base name", + "example": "ahriman", + }) + paths = fields.Dict( + keys=fields.String(), values=fields.List(fields.String()), required=True, metadata={ + "description": "Map of filesystem paths to packages which contain this path", + }) diff --git a/src/ahriman/web/schemas/package_version_schema.py b/src/ahriman/web/schemas/package_version_schema.py new file mode 100644 index 00000000..4b997b48 --- /dev/null +++ b/src/ahriman/web/schemas/package_version_schema.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from marshmallow import fields + +from ahriman import __version__ +from ahriman.web.schemas.repository_id_schema import RepositoryIdSchema + + +class PackageVersionSchema(RepositoryIdSchema): + """ + request package name schema + """ + + version = fields.String(required=True, metadata={ + "description": "Package version", + "example": __version__, + }) diff --git a/src/ahriman/web/views/v1/packages/changes.py b/src/ahriman/web/views/v1/packages/changes.py index b14eb282..7d113228 100644 --- a/src/ahriman/web/views/v1/packages/changes.py +++ b/src/ahriman/web/views/v1/packages/changes.py @@ -113,7 +113,6 @@ class ChangesView(StatusViewGuard, BaseView): raise HTTPBadRequest(reason=str(ex)) changes = Changes(last_commit_sha, change) - repository_id = self.repository_id() - self.service(repository_id).database.changes_insert(package_base, changes, repository_id) + self.service().client.package_changes_set(package_base, changes) raise HTTPNoContent diff --git a/src/ahriman/web/views/v1/packages/dependencies.py b/src/ahriman/web/views/v1/packages/dependencies.py new file mode 100644 index 00000000..b5d17fe7 --- /dev/null +++ b/src/ahriman/web/views/v1/packages/dependencies.py @@ -0,0 +1,66 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import aiohttp_apispec # type: ignore[import-untyped] + +from aiohttp.web import Response, json_response + +from ahriman.models.user_access import UserAccess +from ahriman.web.schemas import AuthSchema, DependenciesSchema, ErrorSchema, RepositoryIdSchema +from ahriman.web.views.base import BaseView +from ahriman.web.views.status_view_guard import StatusViewGuard + + +class DependenciesView(StatusViewGuard, BaseView): + """ + packages dependencies web view + + Attributes: + GET_PERMISSION(UserAccess): (class attribute) get permissions of self + """ + + GET_PERMISSION = UserAccess.Reporter + POST_PERMISSION = UserAccess.Full + ROUTES = ["/api/v1/dependencies"] + + @aiohttp_apispec.docs( + tags=["Build"], + summary="Get dependencies for all packages", + description="Retrieve implicit dependencies for all known packages", + responses={ + 200: {"description": "Success response", "schema": DependenciesSchema(many=True)}, + 401: {"description": "Authorization required", "schema": ErrorSchema}, + 403: {"description": "Access is forbidden", "schema": ErrorSchema}, + 404: {"description": "Package base and/or repository are unknown", "schema": ErrorSchema}, + 500: {"description": "Internal server error", "schema": ErrorSchema}, + }, + security=[{"token": [GET_PERMISSION]}], + ) + @aiohttp_apispec.cookies_schema(AuthSchema) + @aiohttp_apispec.querystring_schema(RepositoryIdSchema) + async def get(self) -> Response: + """ + get dependencies for all packages + + Returns: + Response: 200 with package implicit dependencies on success + """ + dependencies = self.service().client.package_dependencies_get(None) + + return json_response([dependency.view() for dependency in dependencies]) diff --git a/src/ahriman/web/views/v1/packages/dependency.py b/src/ahriman/web/views/v1/packages/dependency.py new file mode 100644 index 00000000..2b9ef7f6 --- /dev/null +++ b/src/ahriman/web/views/v1/packages/dependency.py @@ -0,0 +1,120 @@ +# +# Copyright (c) 2021-2024 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import aiohttp_apispec # type: ignore[import-untyped] + +from aiohttp.web import HTTPBadRequest, HTTPNoContent, HTTPNotFound, Response, json_response + +from ahriman.core.exceptions import UnknownPackageError +from ahriman.models.dependencies import Dependencies +from ahriman.models.user_access import UserAccess +from ahriman.web.schemas import AuthSchema, DependenciesSchema, ErrorSchema, PackageNameSchema, RepositoryIdSchema +from ahriman.web.views.base import BaseView +from ahriman.web.views.status_view_guard import StatusViewGuard + + +class DependencyView(StatusViewGuard, BaseView): + """ + package dependencies web view + + Attributes: + GET_PERMISSION(UserAccess): (class attribute) get permissions of self + POST_PERMISSION(UserAccess): (class attribute) post permissions of self + """ + + GET_PERMISSION = UserAccess.Reporter + POST_PERMISSION = UserAccess.Full + ROUTES = ["/api/v1/dependencies/{package}"] + + @aiohttp_apispec.docs( + tags=["Build"], + summary="Get package dependencies", + description="Retrieve package implicit dependencies", + responses={ + 200: {"description": "Success response", "schema": DependenciesSchema}, + 401: {"description": "Authorization required", "schema": ErrorSchema}, + 403: {"description": "Access is forbidden", "schema": ErrorSchema}, + 404: {"description": "Package base and/or repository are unknown", "schema": ErrorSchema}, + 500: {"description": "Internal server error", "schema": ErrorSchema}, + }, + security=[{"token": [GET_PERMISSION]}], + ) + @aiohttp_apispec.cookies_schema(AuthSchema) + @aiohttp_apispec.match_info_schema(PackageNameSchema) + @aiohttp_apispec.querystring_schema(RepositoryIdSchema) + async def get(self) -> Response: + """ + get package dependencies + + Returns: + Response: 200 with package implicit dependencies on success + + Raises: + HTTPNotFound: if package base is unknown + """ + package_base = self.request.match_info["package"] + + try: + dependencies = self.service().package_dependencies_get(package_base) + except UnknownPackageError: + raise HTTPNotFound(reason=f"Package {package_base} is unknown") + + return json_response(dependencies.view()) + + @aiohttp_apispec.docs( + tags=["Build"], + summary="Update package dependencies", + description="Set package implicit dependencies", + responses={ + 204: {"description": "Success response"}, + 400: {"description": "Bad data is supplied", "schema": ErrorSchema}, + 401: {"description": "Authorization required", "schema": ErrorSchema}, + 403: {"description": "Access is forbidden", "schema": ErrorSchema}, + 404: {"description": "Repository is unknown", "schema": ErrorSchema}, + 500: {"description": "Internal server error", "schema": ErrorSchema}, + }, + security=[{"token": [POST_PERMISSION]}], + ) + @aiohttp_apispec.cookies_schema(AuthSchema) + @aiohttp_apispec.match_info_schema(PackageNameSchema) + @aiohttp_apispec.querystring_schema(RepositoryIdSchema) + @aiohttp_apispec.json_schema(DependenciesSchema) + async def post(self) -> None: + """ + insert new package dependencies + + Raises: + HTTPBadRequest: if bad data is supplied + HTTPNoContent: in case of success response + """ + package_base = self.request.match_info["package"] + + try: + data = await self.request.json() + data["package_base"] = package_base # read from path instead of object + dependencies = Dependencies.from_json(data) + except Exception as ex: + raise HTTPBadRequest(reason=str(ex)) + + try: + self.service().package_dependencies_set(package_base, dependencies) + except UnknownPackageError: + raise HTTPNotFound(reason=f"Package {package_base} is unknown") + + raise HTTPNoContent diff --git a/src/ahriman/web/views/v1/packages/logs.py b/src/ahriman/web/views/v1/packages/logs.py index af527c46..68d61f4b 100644 --- a/src/ahriman/web/views/v1/packages/logs.py +++ b/src/ahriman/web/views/v1/packages/logs.py @@ -25,8 +25,8 @@ from ahriman.core.exceptions import UnknownPackageError from ahriman.core.util import pretty_datetime from ahriman.models.log_record_id import LogRecordId from ahriman.models.user_access import UserAccess -from ahriman.web.schemas import AuthSchema, ErrorSchema, LogsSchema, PackageNameSchema, RepositoryIdSchema, \ - VersionedLogSchema +from ahriman.web.schemas import AuthSchema, ErrorSchema, LogsSchema, PackageNameSchema, PackageVersionSchema, \ + RepositoryIdSchema, VersionedLogSchema from ahriman.web.views.base import BaseView from ahriman.web.views.status_view_guard import StatusViewGuard @@ -60,7 +60,7 @@ class LogsView(StatusViewGuard, BaseView): ) @aiohttp_apispec.cookies_schema(AuthSchema) @aiohttp_apispec.match_info_schema(PackageNameSchema) - @aiohttp_apispec.querystring_schema(RepositoryIdSchema) + @aiohttp_apispec.querystring_schema(PackageVersionSchema) async def delete(self) -> None: """ delete package logs @@ -69,7 +69,8 @@ class LogsView(StatusViewGuard, BaseView): HTTPNoContent: on success response """ package_base = self.request.match_info["package"] - self.service().logs_remove(package_base, None) + version = self.request.query.get("version") + self.service().logs_remove(package_base, version) raise HTTPNoContent diff --git a/src/ahriman/web/web.py b/src/ahriman/web/web.py index 64b48ae8..6d7fd315 100644 --- a/src/ahriman/web/web.py +++ b/src/ahriman/web/web.py @@ -30,6 +30,7 @@ from ahriman.core.database import SQLite from ahriman.core.distributed import WorkersCache from ahriman.core.exceptions import InitializeError from ahriman.core.spawn import Spawn +from ahriman.core.status.client import Client from ahriman.core.status.watcher import Watcher from ahriman.models.repository_id import RepositoryId from ahriman.web.apispec import setup_apispec @@ -167,7 +168,8 @@ def setup_server(configuration: Configuration, spawner: Spawn, repositories: lis watchers: dict[RepositoryId, Watcher] = {} for repository_id in repositories: application.logger.info("load repository %s", repository_id) - watchers[repository_id] = Watcher(repository_id, database) + client = Client.load(repository_id, configuration, database, report=False) # explicitly load local client + watchers[repository_id] = Watcher(client) application[WatcherKey] = watchers # workers cache application[WorkersKey] = WorkersCache(configuration)