diff --git a/docs/ahriman.1 b/docs/ahriman.1 index 5996281a..b640dad3 100644 --- a/docs/ahriman.1 +++ b/docs/ahriman.1 @@ -1,4 +1,4 @@ -.TH AHRIMAN "1" "2023\-01\-25" "ahriman" "Generated Python Manual" +.TH AHRIMAN "1" "2023\-01\-27" "ahriman" "Generated Python Manual" .SH NAME ahriman .SH SYNOPSIS @@ -156,7 +156,7 @@ web server .SH COMMAND \fI\,'ahriman aur\-search'\/\fR usage: ahriman aur\-search [\-h] [\-e] [\-\-info | \-\-no\-info] - [\-\-sort\-by {description,first_submitted,id,last_modified,maintainer,name,num_votes,out_of_date,package_base,package_base_id,popularity,repository,url,url_path,version}] + [\-\-sort\-by {description,first_submitted,id,last_modified,maintainer,name,num_votes,out_of_date,package_base,package_base_id,popularity,repository,submitter,url,url_path,version}] search [search ...] search for package in AUR using API @@ -175,7 +175,7 @@ return non\-zero exit status if result is empty show additional package information (default: False) .TP -\fB\-\-sort\-by\fR \fI\,{description,first_submitted,id,last_modified,maintainer,name,num_votes,out_of_date,package_base,package_base_id,popularity,repository,url,url_path,version}\/\fR +\fB\-\-sort\-by\fR \fI\,{description,first_submitted,id,last_modified,maintainer,name,num_votes,out_of_date,package_base,package_base_id,popularity,repository,submitter,url,url_path,version}\/\fR sort field by this field. In case if two packages have the same value of the specified field, they will be always sorted by name diff --git a/docs/ahriman.core.database.data.rst b/docs/ahriman.core.database.data.rst deleted file mode 100644 index 15cc9a3a..00000000 --- a/docs/ahriman.core.database.data.rst +++ /dev/null @@ -1,45 +0,0 @@ -ahriman.core.database.data package -================================== - -Submodules ----------- - -ahriman.core.database.data.package\_remotes module --------------------------------------------------- - -.. automodule:: ahriman.core.database.data.package_remotes - :members: - :no-undoc-members: - :show-inheritance: - -ahriman.core.database.data.package\_statuses module ---------------------------------------------------- - -.. automodule:: ahriman.core.database.data.package_statuses - :members: - :no-undoc-members: - :show-inheritance: - -ahriman.core.database.data.patches module ------------------------------------------ - -.. automodule:: ahriman.core.database.data.patches - :members: - :no-undoc-members: - :show-inheritance: - -ahriman.core.database.data.users module ---------------------------------------- - -.. automodule:: ahriman.core.database.data.users - :members: - :no-undoc-members: - :show-inheritance: - -Module contents ---------------- - -.. automodule:: ahriman.core.database.data - :members: - :no-undoc-members: - :show-inheritance: diff --git a/docs/ahriman.core.database.migrations.rst b/docs/ahriman.core.database.migrations.rst index 4a13a96e..2a1644d3 100644 --- a/docs/ahriman.core.database.migrations.rst +++ b/docs/ahriman.core.database.migrations.rst @@ -44,6 +44,14 @@ ahriman.core.database.migrations.m004\_logs module :no-undoc-members: :show-inheritance: +ahriman.core.database.migrations.m005\_make\_opt\_depends module +---------------------------------------------------------------- + +.. automodule:: ahriman.core.database.migrations.m005_make_opt_depends + :members: + :no-undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/ahriman.core.database.rst b/docs/ahriman.core.database.rst index bca3c50c..3c4c80db 100644 --- a/docs/ahriman.core.database.rst +++ b/docs/ahriman.core.database.rst @@ -7,7 +7,6 @@ Subpackages .. toctree:: :maxdepth: 4 - ahriman.core.database.data ahriman.core.database.migrations ahriman.core.database.operations diff --git a/docs/architecture.rst b/docs/architecture.rst index 33cfc6d0..e97bf267 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -114,7 +114,7 @@ Schema and data migrations The schema migration are applied according to current ``pragma user_info`` values, located at ``ahriman.core.database.migrations`` package and named as ``m000_migration_name.py`` (the preceding ``m`` is required in order to import migration content for tests). Additional class ``ahriman.core.database.migrations.Migrations`` reads all migrations automatically and applies them in alphabetical order. -There are also data migrations which are located at ``ahriman.core.database.data`` package and move data from old-style (e.g. json files in filesystem, directory trees, etc) to the database. They are also part of migration and (unlike schema migrations) are applied only at specific version breakpoints (e.g. if ``user_version`` is more than 0 no initial migration will be applied). +These migrations also contain data migrations. Though the recommended way is to migrate data directly from SQL requests, sometimes it is required to have external data (like packages list) in order to set correct data. To do so, special method `migrate_data` is used. Type conversions ^^^^^^^^^^^^^^^^ diff --git a/docs/completions/bash/_ahriman b/docs/completions/bash/_ahriman index 66ee0644..2d81c99f 100644 --- a/docs/completions/bash/_ahriman +++ b/docs/completions/bash/_ahriman @@ -71,8 +71,8 @@ _shtab_ahriman_web_option_strings=('-h' '--help') _shtab_ahriman_pos_0_choices=('aur-search' 'search' 'help' 'help-commands-unsafe' 'help-updates' 'help-version' 'version' 'package-add' 'add' 'package-update' 'package-remove' 'remove' 'package-status' 'status' 'package-status-remove' 'package-status-update' 'status-update' 'patch-add' 'patch-list' 'patch-remove' 'patch-set-add' 'repo-backup' 'repo-check' 'check' 'repo-daemon' 'daemon' 'repo-rebuild' 'rebuild' 'repo-remove-unknown' 'remove-unknown' 'repo-report' 'report' 'repo-restore' 'repo-sign' 'sign' 'repo-status-update' 'repo-sync' 'sync' 'repo-tree' 'repo-triggers' 'repo-update' 'update' 'service-clean' 'clean' 'repo-clean' 'service-config' 'config' 'repo-config' 'service-config-validate' 'config-validate' 'repo-config-validate' 'service-key-import' 'key-import' 'service-setup' 'init' 'repo-init' 'repo-setup' 'setup' 'service-shell' 'shell' 'user-add' 'user-list' 'user-remove' 'web') -_shtab_ahriman_aur_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'url' 'url_path' 'version') -_shtab_ahriman_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'url' 'url_path' 'version') +_shtab_ahriman_aur_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'submitter' 'url' 'url_path' 'version') +_shtab_ahriman_search___sort_by_choices=('description' 'first_submitted' 'id' 'last_modified' 'maintainer' 'name' 'num_votes' 'out_of_date' 'package_base' 'package_base_id' 'popularity' 'repository' 'submitter' 'url' 'url_path' 'version') _shtab_ahriman_package_add__s_choices=('auto' 'archive' 'aur' 'directory' 'local' 'remote' 'repository') _shtab_ahriman_package_add___source_choices=('auto' 'archive' 'aur' 'directory' 'local' 'remote' 'repository') _shtab_ahriman_add__s_choices=('auto' 'archive' 'aur' 'directory' 'local' 'remote' 'repository') diff --git a/docs/completions/zsh/_ahriman b/docs/completions/zsh/_ahriman index 37a1f979..a167a6b6 100644 --- a/docs/completions/zsh/_ahriman +++ b/docs/completions/zsh/_ahriman @@ -99,7 +99,7 @@ _shtab_ahriman_aur_search_options=( "(- : *)"{-h,--help}"[show this help message and exit]" {-e,--exit-code}"[return non-zero exit status if result is empty]" {--info,--no-info}"[show additional package information (default\: \%(default)s)]:info:" - "--sort-by[sort field by this field. In case if two packages have the same value of the specified field, they will be always sorted by name]:sort_by:(description first_submitted id last_modified maintainer name num_votes out_of_date package_base package_base_id popularity repository url url_path version)" + "--sort-by[sort field by this field. In case if two packages have the same value of the specified field, they will be always sorted by name]:sort_by:(description first_submitted id last_modified maintainer name num_votes out_of_date package_base package_base_id popularity repository submitter url url_path version)" "(*):search terms, can be specified multiple times, the result will match all terms:" ) @@ -408,7 +408,7 @@ _shtab_ahriman_search_options=( "(- : *)"{-h,--help}"[show this help message and exit]" {-e,--exit-code}"[return non-zero exit status if result is empty]" {--info,--no-info}"[show additional package information (default\: \%(default)s)]:info:" - "--sort-by[sort field by this field. In case if two packages have the same value of the specified field, they will be always sorted by name]:sort_by:(description first_submitted id last_modified maintainer name num_votes out_of_date package_base package_base_id popularity repository url url_path version)" + "--sort-by[sort field by this field. In case if two packages have the same value of the specified field, they will be always sorted by name]:sort_by:(description first_submitted id last_modified maintainer name num_votes out_of_date package_base package_base_id popularity repository submitter url url_path version)" "(*):search terms, can be specified multiple times, the result will match all terms:" ) diff --git a/src/ahriman/application/application/application_packages.py b/src/ahriman/application/application/application_packages.py index 0784ff41..5f5edf7c 100644 --- a/src/ahriman/application/application/application_packages.py +++ b/src/ahriman/application/application/application_packages.py @@ -21,7 +21,6 @@ import requests import shutil from pathlib import Path -from tempfile import TemporaryDirectory from typing import Any, Iterable, Set from ahriman.application.application.application_properties import ApplicationProperties @@ -62,9 +61,7 @@ class ApplicationPackages(ApplicationProperties): self.database.build_queue_insert(package) self.database.remote_update(package) - with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (local_dir := Path(dir_name)): - Sources.load(local_dir, package, self.database.patches_get(package.base), self.repository.paths) - self._process_dependencies(local_dir, known_packages, without_dependencies) + self._process_dependencies(package, known_packages, without_dependencies) def _add_directory(self, source: str, *_: Any) -> None: """ @@ -94,7 +91,7 @@ class ApplicationPackages(ApplicationProperties): self.database.build_queue_insert(package) - self._process_dependencies(cache_dir, known_packages, without_dependencies) + self._process_dependencies(package, known_packages, without_dependencies) def _add_remote(self, source: str, *_: Any) -> None: """ @@ -135,19 +132,19 @@ class ApplicationPackages(ApplicationProperties): """ raise NotImplementedError - def _process_dependencies(self, local_dir: Path, known_packages: Set[str], without_dependencies: bool) -> None: + def _process_dependencies(self, package: Package, known_packages: Set[str], without_dependencies: bool) -> None: """ process package dependencies Args: - local_dir(Path): path to local package sources (i.e. cloned AUR repository) + package(Package): source package of which dependencies have to be processed known_packages(Set[str]): list of packages which are known by the service without_dependencies(bool): if set, dependency check will be disabled """ if without_dependencies: return - dependencies = Package.dependencies(local_dir) + dependencies = package.depends_build self.add(dependencies.difference(known_packages), PackageSource.AUR, without_dependencies) def add(self, names: Iterable[str], source: PackageSource, without_dependencies: bool) -> None: diff --git a/src/ahriman/application/application/application_repository.py b/src/ahriman/application/application/application_repository.py index e8da4b09..eaf30295 100644 --- a/src/ahriman/application/application/application_repository.py +++ b/src/ahriman/application/application/application_repository.py @@ -145,7 +145,7 @@ class ApplicationRepository(ApplicationProperties): process_update(packages, build_result) # process manual packages - tree = Tree.resolve(updates, self.repository.paths, self.database) + tree = Tree.resolve(updates) for num, level in enumerate(tree): self.logger.info("processing level #%i %s", num, [package.base for package in level]) build_result = self.repository.process_build(level) @@ -183,7 +183,7 @@ class ApplicationRepository(ApplicationProperties): updated_packages = [package for _, package in sorted(updates.items())] # reorder updates according to the dependency tree - tree = Tree.resolve(updated_packages, self.repository.paths, self.database) + tree = Tree.resolve(updated_packages) for level in tree: for package in level: UpdatePrinter(package, local_versions.get(package.base)).print( diff --git a/src/ahriman/application/handlers/remove_unknown.py b/src/ahriman/application/handlers/remove_unknown.py index a54d812d..aca626e6 100644 --- a/src/ahriman/application/handlers/remove_unknown.py +++ b/src/ahriman/application/handlers/remove_unknown.py @@ -51,7 +51,7 @@ class RemoveUnknown(Handler): if args.dry_run: for package in sorted(unknown_packages): - StringPrinter(package).print(False) + StringPrinter(package).print(verbose=False) return application.remove(unknown_packages) diff --git a/src/ahriman/application/handlers/search.py b/src/ahriman/application/handlers/search.py index f59a5627..72f2f433 100644 --- a/src/ahriman/application/handlers/search.py +++ b/src/ahriman/application/handlers/search.py @@ -64,7 +64,7 @@ class Search(Handler): for packages_list in (official_packages_list, aur_packages_list): # keep sorting by packages source for package in Search.sort(packages_list, args.sort_by): - AurPrinter(package).print(args.info) + AurPrinter(package).print(verbose=args.info) @staticmethod def sort(packages: Iterable[AURPackage], sort_by: str) -> List[AURPackage]: diff --git a/src/ahriman/application/handlers/status.py b/src/ahriman/application/handlers/status.py index ea74a33e..396860ad 100644 --- a/src/ahriman/application/handlers/status.py +++ b/src/ahriman/application/handlers/status.py @@ -53,7 +53,7 @@ class Status(Handler): client = Application(architecture, configuration, report=True, unsafe=unsafe).repository.reporter if args.ahriman: service_status = client.get_internal() - StatusPrinter(service_status.status).print(args.info) + StatusPrinter(service_status.status).print(verbose=args.info) if args.package: packages: Iterable[Tuple[Package, BuildStatus]] = sum( (client.get(base) for base in args.package), @@ -67,4 +67,4 @@ class Status(Handler): filter_fn: Callable[[Tuple[Package, BuildStatus]], bool] =\ lambda item: args.status is None or item[1].status == args.status for package, package_status in sorted(filter(filter_fn, packages), key=comparator): - PackagePrinter(package, package_status).print(args.info) + PackagePrinter(package, package_status).print(verbose=args.info) diff --git a/src/ahriman/application/handlers/structure.py b/src/ahriman/application/handlers/structure.py index 7aa78bdf..df889165 100644 --- a/src/ahriman/application/handlers/structure.py +++ b/src/ahriman/application/handlers/structure.py @@ -51,6 +51,6 @@ class Structure(Handler): application = Application(architecture, configuration, report=report, unsafe=unsafe) packages = application.repository.packages() - tree = Tree.resolve(packages, application.repository.paths, application.database) + tree = Tree.resolve(packages) for num, level in enumerate(tree): TreePrinter(num, level).print(verbose=True, separator=" ") diff --git a/src/ahriman/core/database/data/__init__.py b/src/ahriman/core/database/data/__init__.py deleted file mode 100644 index 216b3270..00000000 --- a/src/ahriman/core/database/data/__init__.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright (c) 2021-2023 ahriman team. -# -# This file is part of ahriman -# (see https://github.com/arcan1s/ahriman). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -from sqlite3 import Connection - -from ahriman.core.configuration import Configuration -from ahriman.core.database.data.package_remotes import migrate_package_remotes -from ahriman.core.database.data.package_statuses import migrate_package_statuses -from ahriman.core.database.data.patches import migrate_patches -from ahriman.core.database.data.users import migrate_users_data -from ahriman.models.migration_result import MigrationResult - - -def migrate_data(result: MigrationResult, connection: Connection, configuration: Configuration) -> None: - """ - perform data migration - - Args: - result(MigrationResult): result of the schema migration - connection(Connection): database connection - configuration(Configuration): configuration instance - """ - # initial data migration - repository_paths = configuration.repository_paths - - if result.old_version <= 0: - migrate_package_statuses(connection, repository_paths) - migrate_patches(connection, repository_paths) - migrate_users_data(connection, configuration) - if result.old_version <= 1: - migrate_package_remotes(connection, repository_paths) diff --git a/src/ahriman/core/database/data/package_remotes.py b/src/ahriman/core/database/data/package_remotes.py deleted file mode 100644 index a62bef71..00000000 --- a/src/ahriman/core/database/data/package_remotes.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (c) 2021-2023 ahriman team. -# -# This file is part of ahriman -# (see https://github.com/arcan1s/ahriman). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -from sqlite3 import Connection - -from ahriman.models.package_source import PackageSource -from ahriman.models.remote_source import RemoteSource -from ahriman.models.repository_paths import RepositoryPaths - - -__all__ = ["migrate_package_remotes"] - - -# pylint: disable=protected-access -def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> None: - """ - perform migration for package remote sources - - Args: - connection(Connection): database connection - paths(RepositoryPaths): repository paths instance - """ - from ahriman.core.database.operations import PackageOperations - - def insert_remote(base: str, remote: RemoteSource) -> None: - connection.execute( - """ - update package_bases set - branch = :branch, git_url = :git_url, path = :path, - web_url = :web_url, source = :source - where package_base = :package_base - """, - dict( - package_base=base, - branch=remote.branch, git_url=remote.git_url, path=remote.path, - web_url=remote.web_url, source=remote.source - ) - ) - - packages = PackageOperations._packages_get_select_package_bases(connection) - for package_base, package in packages.items(): - local_cache = paths.cache_for(package_base) - if local_cache.exists() and not package.is_vcs: - continue # skip packages which are not VCS and with local cache - remote_source = RemoteSource.from_source(PackageSource.AUR, package_base, "aur") - if remote_source is None: - continue # should never happen - insert_remote(package_base, remote_source) diff --git a/src/ahriman/core/database/data/package_statuses.py b/src/ahriman/core/database/data/package_statuses.py deleted file mode 100644 index eb802097..00000000 --- a/src/ahriman/core/database/data/package_statuses.py +++ /dev/null @@ -1,82 +0,0 @@ -# -# Copyright (c) 2021-2023 ahriman team. -# -# This file is part of ahriman -# (see https://github.com/arcan1s/ahriman). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -import json - -from sqlite3 import Connection - -from ahriman.models.build_status import BuildStatus -from ahriman.models.package import Package -from ahriman.models.repository_paths import RepositoryPaths - - -__all__ = ["migrate_package_statuses"] - - -def migrate_package_statuses(connection: Connection, paths: RepositoryPaths) -> None: - """ - perform migration for package statuses - - Args: - connection(Connection): database connection - paths(RepositoryPaths): repository paths instance - """ - def insert_base(metadata: Package, last_status: BuildStatus) -> None: - connection.execute( - """ - insert into package_bases - (package_base, version, aur_url) - values - (:package_base, :version, :aur_url) - """, - dict(package_base=metadata.base, version=metadata.version, aur_url="")) - connection.execute( - """ - insert into package_statuses - (package_base, status, last_updated) - values - (:package_base, :status, :last_updated)""", - dict(package_base=metadata.base, status=last_status.status.value, last_updated=last_status.timestamp)) - - def insert_packages(metadata: Package) -> None: - package_list = [] - for name, description in metadata.packages.items(): - package_list.append(dict(package=name, package_base=metadata.base, **description.view())) - connection.executemany( - """ - insert into packages - (package, package_base, architecture, archive_size, build_date, depends, description, - filename, "groups", installed_size, licenses, provides, url) - values - (:package, :package_base, :architecture, :archive_size, :build_date, :depends, :description, - :filename, :groups, :installed_size, :licenses, :provides, :url) - """, - package_list) - - cache_path = paths.root / "status_cache.json" - if not cache_path.is_file(): - return # no file found - with cache_path.open() as cache: - dump = json.load(cache) - - for item in dump.get("packages", []): - package = Package.from_json(item["package"]) - status = BuildStatus.from_json(item["status"]) - insert_base(package, status) - insert_packages(package) diff --git a/src/ahriman/core/database/data/patches.py b/src/ahriman/core/database/data/patches.py deleted file mode 100644 index f2d2ec79..00000000 --- a/src/ahriman/core/database/data/patches.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright (c) 2021-2023 ahriman team. -# -# This file is part of ahriman -# (see https://github.com/arcan1s/ahriman). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -from sqlite3 import Connection - -from ahriman.models.repository_paths import RepositoryPaths - - -__all__ = ["migrate_patches"] - - -def migrate_patches(connection: Connection, paths: RepositoryPaths) -> None: - """ - perform migration for patches - - Args: - connection(Connection): database connection - paths(RepositoryPaths): repository paths instance - """ - root = paths.root / "patches" - if not root.is_dir(): - return # no directory found - - for package in root.iterdir(): - patch_path = package / "00-main.patch" - if not patch_path.is_file(): - continue # not exist - content = patch_path.read_text(encoding="utf8") - connection.execute( - """insert into patches (package_base, patch) values (:package_base, :patch)""", - {"package_base": package.name, "patch": content}) diff --git a/src/ahriman/core/database/data/users.py b/src/ahriman/core/database/data/users.py deleted file mode 100644 index 815cd878..00000000 --- a/src/ahriman/core/database/data/users.py +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright (c) 2021-2023 ahriman team. -# -# This file is part of ahriman -# (see https://github.com/arcan1s/ahriman). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -from sqlite3 import Connection - -from ahriman.core.configuration import Configuration - - -__all__ = ["migrate_users_data"] - - -def migrate_users_data(connection: Connection, configuration: Configuration) -> None: - """ - perform migration for users - - Args: - connection(Connection): database connection - configuration(Configuration): configuration instance - """ - for section in configuration.sections(): - for option, value in configuration[section].items(): - if not section.startswith("auth:"): - continue - access = section[5:] - connection.execute( - """insert into users (username, access, password) values (:username, :access, :password)""", - {"username": option.lower(), "access": access, "password": value}) diff --git a/src/ahriman/core/database/migrations/__init__.py b/src/ahriman/core/database/migrations/__init__.py index 923551e9..76bc8d7b 100644 --- a/src/ahriman/core/database/migrations/__init__.py +++ b/src/ahriman/core/database/migrations/__init__.py @@ -17,16 +17,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -from __future__ import annotations - from importlib import import_module from pathlib import Path from pkgutil import iter_modules -from sqlite3 import Connection -from typing import List, Type +from sqlite3 import Connection, Cursor +from typing import Callable, List from ahriman.core.configuration import Configuration -from ahriman.core.database.data import migrate_data from ahriman.core.log import LazyLogging from ahriman.models.migration import Migration from ahriman.models.migration_result import MigrationResult @@ -53,8 +50,8 @@ class Migrations(LazyLogging): self.connection = connection self.configuration = configuration - @classmethod - def migrate(cls: Type[Migrations], connection: Connection, configuration: Configuration) -> MigrationResult: + @staticmethod + def migrate(connection: Connection, configuration: Configuration) -> MigrationResult: """ perform migrations implicitly @@ -65,7 +62,26 @@ class Migrations(LazyLogging): Returns: MigrationResult: current schema version """ - return cls(connection, configuration).run() + return Migrations(connection, configuration).run() + + def migration(self, cursor: Cursor, migration: Migration) -> None: + """ + perform single migration + + Args: + cursor(Cursor): connection cursor + migration(Migration): single migration to perform + """ + self.logger.info("applying table migration %s at index %s", migration.name, migration.index) + for statement in migration.steps: + cursor.execute(statement) + self.logger.info("table migration %s at index %s has been applied", migration.name, migration.index) + + self.logger.info("perform data migration %s at index %s", migration.name, migration.index) + migration.migrate_data(self.connection, self.configuration) + self.logger.info( + "data migration %s at index %s has been performed", + migration.name, migration.index) def migrations(self) -> List[Migration]: """ @@ -81,9 +97,21 @@ class Migrations(LazyLogging): for index, module_name in enumerate(sorted(modules)): module = import_module(f"{__name__}.{module_name}") + steps: List[str] = getattr(module, "steps", []) self.logger.debug("found migration %s at index %s with steps count %s", module_name, index, len(steps)) - migrations.append(Migration(index=index, name=module_name, steps=steps)) + + migrate_data: Callable[[Connection, Configuration], None] = \ + getattr(module, "migrate_data", lambda *args: None) + + migrations.append( + Migration( + index=index, + name=module_name, + steps=steps, + migrate_data=migrate_data + ) + ) return migrations @@ -110,13 +138,7 @@ class Migrations(LazyLogging): try: cursor.execute("begin exclusive") for migration in migrations[current_version:]: - self.logger.info("applying migration %s at index %s", migration.name, migration.index) - for statement in migration.steps: - cursor.execute(statement) - self.logger.info("migration %s at index %s has been applied", migration.name, migration.index) - - migrate_data(result, self.connection, self.configuration) - + self.migration(cursor, migration) cursor.execute(f"pragma user_version = {expected_version}") # no support for ? placeholders except Exception: self.logger.exception("migration failed with exception") diff --git a/src/ahriman/core/database/migrations/m000_initial.py b/src/ahriman/core/database/migrations/m000_initial.py index 54895191..f2c655a1 100644 --- a/src/ahriman/core/database/migrations/m000_initial.py +++ b/src/ahriman/core/database/migrations/m000_initial.py @@ -17,7 +17,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -__all__ = ["steps"] +import json + +from sqlite3 import Connection + +from ahriman.core.configuration import Configuration +from ahriman.models.build_status import BuildStatus +from ahriman.models.package import Package +from ahriman.models.repository_paths import RepositoryPaths + + +__all__ = ["migrate_data", "steps"] steps = [ @@ -73,3 +83,109 @@ steps = [ ) """, ] + + +def migrate_data(connection: Connection, configuration: Configuration) -> None: + """ + perform data migration + + Args: + connection(Connection): database connection + configuration(Configuration): configuration instance + """ + migrate_package_statuses(connection, configuration.repository_paths) + migrate_patches(connection, configuration.repository_paths) + migrate_users_data(connection, configuration) + + +def migrate_package_statuses(connection: Connection, paths: RepositoryPaths) -> None: + """ + perform migration for package statuses + + Args: + connection(Connection): database connection + paths(RepositoryPaths): repository paths instance + """ + def insert_base(metadata: Package, last_status: BuildStatus) -> None: + connection.execute( + """ + insert into package_bases + (package_base, version, aur_url) + values + (:package_base, :version, :aur_url) + """, + dict(package_base=metadata.base, version=metadata.version, aur_url="")) + connection.execute( + """ + insert into package_statuses + (package_base, status, last_updated) + values + (:package_base, :status, :last_updated)""", + dict(package_base=metadata.base, status=last_status.status.value, last_updated=last_status.timestamp)) + + def insert_packages(metadata: Package) -> None: + package_list = [] + for name, description in metadata.packages.items(): + package_list.append(dict(package=name, package_base=metadata.base, **description.view())) + connection.executemany( + """ + insert into packages + (package, package_base, architecture, archive_size, build_date, depends, description, + filename, "groups", installed_size, licenses, provides, url) + values + (:package, :package_base, :architecture, :archive_size, :build_date, :depends, :description, + :filename, :groups, :installed_size, :licenses, :provides, :url) + """, + package_list) + + cache_path = paths.root / "status_cache.json" + if not cache_path.is_file(): + return # no file found + with cache_path.open() as cache: + dump = json.load(cache) + + for item in dump.get("packages", []): + package = Package.from_json(item["package"]) + status = BuildStatus.from_json(item["status"]) + insert_base(package, status) + insert_packages(package) + + +def migrate_patches(connection: Connection, paths: RepositoryPaths) -> None: + """ + perform migration for patches + + Args: + connection(Connection): database connection + paths(RepositoryPaths): repository paths instance + """ + root = paths.root / "patches" + if not root.is_dir(): + return # no directory found + + for package in root.iterdir(): + patch_path = package / "00-main.patch" + if not patch_path.is_file(): + continue # not exist + content = patch_path.read_text(encoding="utf8") + connection.execute( + """insert into patches (package_base, patch) values (:package_base, :patch)""", + {"package_base": package.name, "patch": content}) + + +def migrate_users_data(connection: Connection, configuration: Configuration) -> None: + """ + perform migration for users + + Args: + connection(Connection): database connection + configuration(Configuration): configuration instance + """ + for section in configuration.sections(): + for option, value in configuration[section].items(): + if not section.startswith("auth:"): + continue + access = section[5:] + connection.execute( + """insert into users (username, access, password) values (:username, :access, :password)""", + {"username": option.lower(), "access": access, "password": value}) diff --git a/src/ahriman/core/database/migrations/m001_package_source.py b/src/ahriman/core/database/migrations/m001_package_source.py index 2d6f0516..c0589d41 100644 --- a/src/ahriman/core/database/migrations/m001_package_source.py +++ b/src/ahriman/core/database/migrations/m001_package_source.py @@ -17,7 +17,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -__all__ = ["steps"] +from sqlite3 import Connection + +from ahriman.core.configuration import Configuration +from ahriman.models.package_source import PackageSource +from ahriman.models.remote_source import RemoteSource +from ahriman.models.repository_paths import RepositoryPaths + + +__all__ = ["migrate_data", "steps"] steps = [ @@ -40,3 +48,51 @@ steps = [ alter table package_bases drop column aur_url """, ] + + +def migrate_data(connection: Connection, configuration: Configuration) -> None: + """ + perform data migration + + Args: + connection(Connection): database connection + configuration(Configuration): configuration instance + """ + migrate_package_remotes(connection, configuration.repository_paths) + + +# pylint: disable=protected-access +def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> None: + """ + perform migration for package remote sources + + Args: + connection(Connection): database connection + paths(RepositoryPaths): repository paths instance + """ + from ahriman.core.database.operations import PackageOperations + + def insert_remote(base: str, remote: RemoteSource) -> None: + connection.execute( + """ + update package_bases set + branch = :branch, git_url = :git_url, path = :path, + web_url = :web_url, source = :source + where package_base = :package_base + """, + dict( + package_base=base, + branch=remote.branch, git_url=remote.git_url, path=remote.path, + web_url=remote.web_url, source=remote.source + ) + ) + + packages = PackageOperations._packages_get_select_package_bases(connection) + for package_base, package in packages.items(): + local_cache = paths.cache_for(package_base) + if local_cache.exists() and not package.is_vcs: + continue # skip packages which are not VCS and with local cache + remote_source = RemoteSource.from_source(PackageSource.AUR, package_base, "aur") + if remote_source is None: + continue # should never happen + insert_remote(package_base, remote_source) diff --git a/src/ahriman/core/database/migrations/m005_make_opt_depends.py b/src/ahriman/core/database/migrations/m005_make_opt_depends.py new file mode 100644 index 00000000..add68775 --- /dev/null +++ b/src/ahriman/core/database/migrations/m005_make_opt_depends.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from sqlite3 import Connection + +from ahriman.core.alpm.pacman import Pacman +from ahriman.core.configuration import Configuration +from ahriman.core.util import package_like +from ahriman.models.package import Package + + +__all__ = ["migrate_data", "steps"] + + +steps = [ + """ + alter table packages add column make_depends json + """, + """ + alter table packages add column opt_depends json + """, +] + + +def migrate_data(connection: Connection, configuration: Configuration) -> None: + """ + perform data migration + + Args: + connection(Connection): database connection + configuration(Configuration): configuration instance + """ + migrate_package_depends(connection, configuration) + + +def migrate_package_depends(connection: Connection, configuration: Configuration) -> None: + """ + migrate package opt and make depends fields + + Args: + connection(Connection): database connection + configuration(Configuration): configuration instance + """ + if not configuration.repository_paths.repository.is_dir(): + return + + _, architecture = configuration.check_loaded() + pacman = Pacman(architecture, configuration, refresh_database=False) + + package_list = [] + for full_path in filter(package_like, configuration.repository_paths.repository.iterdir()): + base = Package.from_archive(full_path, pacman, remote=None) + for package, description in base.packages.items(): + package_list.append({ + "make_depends": description.make_depends, + "opt_depends": description.opt_depends, + "package": package, + }) + + connection.executemany( + """ + update packages set + make_depends = :make_depends, opt_depends = :opt_depends + where package = :package + """, + package_list + ) diff --git a/src/ahriman/core/database/operations/package_operations.py b/src/ahriman/core/database/operations/package_operations.py index b4aa5f49..595522b3 100644 --- a/src/ahriman/core/database/operations/package_operations.py +++ b/src/ahriman/core/database/operations/package_operations.py @@ -110,15 +110,18 @@ class PackageOperations(Operations): insert into packages (package, package_base, architecture, archive_size, build_date, depends, description, filename, - "groups", installed_size, licenses, provides, url) + "groups", installed_size, licenses, provides, + url, make_depends, opt_depends) values (:package, :package_base, :architecture, :archive_size, :build_date, :depends, :description, :filename, - :groups, :installed_size, :licenses, :provides, :url) + :groups, :installed_size, :licenses, :provides, + :url, :make_depends, :opt_depends) on conflict (package, architecture) do update set package_base = :package_base, archive_size = :archive_size, build_date = :build_date, depends = :depends, description = :description, filename = :filename, - "groups" = :groups, installed_size = :installed_size, licenses = :licenses, provides = :provides, url = :url + "groups" = :groups, installed_size = :installed_size, licenses = :licenses, provides = :provides, + url = :url, make_depends = :make_depends, opt_depends = :opt_depends """, package_list) diff --git a/src/ahriman/core/formatters/printer.py b/src/ahriman/core/formatters/printer.py index ab517f65..e496cf6d 100644 --- a/src/ahriman/core/formatters/printer.py +++ b/src/ahriman/core/formatters/printer.py @@ -27,7 +27,7 @@ class Printer: base class for formatters """ - def print(self, verbose: bool, log_fn: Callable[[str], None] = print, separator: str = ": ") -> None: + def print(self, *, verbose: bool, log_fn: Callable[[str], None] = print, separator: str = ": ") -> None: """ print content diff --git a/src/ahriman/core/tree.py b/src/ahriman/core/tree.py index 41cc9c39..c9109dfb 100644 --- a/src/ahriman/core/tree.py +++ b/src/ahriman/core/tree.py @@ -21,14 +21,9 @@ from __future__ import annotations import itertools -from pathlib import Path -from tempfile import TemporaryDirectory -from typing import Callable, Iterable, List, Set, Tuple, Type +from typing import Callable, Iterable, List, Tuple -from ahriman.core.build_tools.sources import Sources -from ahriman.core.database import SQLite from ahriman.models.package import Package -from ahriman.models.repository_paths import RepositoryPaths class Leaf: @@ -40,16 +35,15 @@ class Leaf: package(Package): leaf package properties """ - def __init__(self, package: Package, dependencies: Set[str]) -> None: + def __init__(self, package: Package) -> None: """ default constructor Args: package(Package): package properties - dependencies(Set[str]): package dependencies """ self.package = package - self.dependencies = dependencies + self.dependencies = package.depends_build @property def items(self) -> Iterable[str]: @@ -61,24 +55,6 @@ class Leaf: """ return self.package.packages.keys() - @classmethod - def load(cls: Type[Leaf], package: Package, paths: RepositoryPaths, database: SQLite) -> Leaf: - """ - load leaf from package with dependencies - - Args: - package(Package): package properties - paths(RepositoryPaths): repository paths instance - database(SQLite): database instance - - Returns: - Leaf: loaded class - """ - with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (clone_dir := Path(dir_name)): - Sources.load(clone_dir, package, database.patches_get(package.base), paths) - dependencies = Package.dependencies(clone_dir) - return cls(package, dependencies) - def is_dependency(self, packages: Iterable[Leaf]) -> bool: """ check if the package is dependency of any other package from list or not @@ -130,7 +106,7 @@ class Tree: >>> repository = Repository.load("x86_64", configuration, database, report=True, unsafe=False) >>> packages = repository.packages() >>> - >>> tree = Tree.resolve(packages, configuration.repository_paths, database) + >>> tree = Tree.resolve(packages) >>> for tree_level in tree: >>> for package in tree_level: >>> print(package.base) @@ -138,14 +114,8 @@ class Tree: The direct constructor call is also possible but requires tree leaves to be instantioned in advance, e.g.:: - >>> leaves = [Leaf.load(package, database) for package in packages] + >>> leaves = [Leaf(package) for package in packages] >>> tree = Tree(leaves) - - Using the default ``Leaf()`` method is possible, but not really recommended because it requires from the user to - build the dependency list by himself:: - - >>> leaf = Leaf(package, dependecies) - >>> tree = Tree([leaf]) """ def __init__(self, leaves: List[Leaf]) -> None: @@ -157,23 +127,20 @@ class Tree: """ self.leaves = leaves - @classmethod - def resolve(cls: Type[Tree], packages: Iterable[Package], paths: RepositoryPaths, - database: SQLite) -> List[List[Package]]: + @staticmethod + def resolve(packages: Iterable[Package]) -> List[List[Package]]: """ resolve dependency tree Args: packages(Iterable[Package]): packages list - paths(RepositoryPaths): repository paths instance - database(SQLite): database instance Returns: List[List[Package]]: list of packages lists based on their dependencies """ - leaves = [Leaf.load(package, paths, database) for package in packages] - tree = cls(leaves) - return tree.levels() + leaves = [Leaf(package) for package in packages] + instance = Tree(leaves) + return instance.levels() def levels(self) -> List[List[Package]]: """ diff --git a/src/ahriman/core/util.py b/src/ahriman/core/util.py index 38b9e623..591390fc 100644 --- a/src/ahriman/core/util.py +++ b/src/ahriman/core/util.py @@ -35,7 +35,7 @@ from ahriman.models.repository_paths import RepositoryPaths __all__ = ["check_output", "check_user", "enum_values", "exception_response_text", "filter_json", "full_version", - "package_like", "pretty_datetime", "pretty_size", "safe_filename", "utcnow", "walk"] + "package_like", "pretty_datetime", "pretty_size", "safe_filename", "trim_package", "utcnow", "walk"] def check_output(*args: str, exception: Optional[Exception] = None, cwd: Optional[Path] = None, @@ -295,6 +295,23 @@ def safe_filename(source: str) -> str: return re.sub(r"[^A-Za-z\d\-._~:\[\]@]", "-", source) +def trim_package(package_name: str) -> str: + """ + remove version bound and description from package name. Pacman allows to specify version bound (=, <=, >= etc) for + packages in dependencies and also allows to specify description (via :); this function removes trailing parts and + return exact package name + + Args: + package_name(str): source package name + + Returns: + str: package name without description or version bound + """ + for symbol in ("<", "=", ">", ":"): + package_name = package_name.partition(symbol)[0] + return package_name + + def utcnow() -> datetime.datetime: """ get current time diff --git a/src/ahriman/models/aur_package.py b/src/ahriman/models/aur_package.py index ee0ef9c3..65c8c0d9 100644 --- a/src/ahriman/models/aur_package.py +++ b/src/ahriman/models/aur_package.py @@ -45,6 +45,7 @@ class AURPackage: popularity(float): package popularity out_of_date(Optional[datetime.datetime]): package out of date timestamp if any maintainer(Optional[str]): package maintainer + submitter(Optional[str]): package first submitter first_submitted(datetime.datetime): timestamp of the first package submission last_modified(datetime.datetime): timestamp of the last package submission url_path(str): AUR package path @@ -89,6 +90,7 @@ class AURPackage: url: Optional[str] = None out_of_date: Optional[datetime.datetime] = None maintainer: Optional[str] = None + submitter: Optional[str] = None repository: str = "aur" depends: List[str] = field(default_factory=list) make_depends: List[str] = field(default_factory=list) @@ -140,6 +142,7 @@ class AURPackage: url=package.url, out_of_date=None, maintainer=None, + submitter=None, repository=package.db.name, depends=package.depends, make_depends=package.makedepends, @@ -178,6 +181,7 @@ class AURPackage: dump["flag_date"], "%Y-%m-%dT%H:%M:%S.%fZ") if dump["flag_date"] is not None else None, maintainer=next(iter(dump["maintainers"]), None), + submitter=None, repository=dump["repo"], depends=dump["depends"], make_depends=dump["makedepends"], diff --git a/src/ahriman/models/internal_status.py b/src/ahriman/models/internal_status.py index a43b97ab..444424c1 100644 --- a/src/ahriman/models/internal_status.py +++ b/src/ahriman/models/internal_status.py @@ -57,7 +57,8 @@ class InternalStatus: InternalStatus: internal status """ counters = Counters.from_json(dump["packages"]) if "packages" in dump else Counters(total=0) - return cls(status=BuildStatus.from_json(dump.get("status", {})), + build_status = dump.get("status") or {} + return cls(status=BuildStatus.from_json(build_status), architecture=dump.get("architecture"), packages=counters, repository=dump.get("repository"), diff --git a/src/ahriman/models/migration.py b/src/ahriman/models/migration.py index 752dbe53..6af20158 100644 --- a/src/ahriman/models/migration.py +++ b/src/ahriman/models/migration.py @@ -18,7 +18,10 @@ # along with this program. If not, see . # from dataclasses import dataclass -from typing import List +from sqlite3 import Connection +from typing import Callable, List + +from ahriman.core.configuration import Configuration @dataclass(frozen=True, kw_only=True) @@ -30,8 +33,10 @@ class Migration: index(int): migration position name(str): migration name steps(List[str]): migration steps + migrate_data(Callable[[Connection, Configuration], None]): data migration callback """ index: int name: str steps: List[str] + migrate_data: Callable[[Connection, Configuration], None] diff --git a/src/ahriman/models/package.py b/src/ahriman/models/package.py index 9ac4ff9f..5c43a255 100644 --- a/src/ahriman/models/package.py +++ b/src/ahriman/models/package.py @@ -88,6 +88,36 @@ class Package(LazyLogging): """ return sorted(set(sum((package.depends for package in self.packages.values()), start=[]))) + @property + def depends_build(self) -> Set[str]: + """ + get full list of external dependencies which has to be installed for build process + + Returns: + Set[str]: full dependencies list used by devtools + """ + return (set(self.depends) | set(self.depends_make)) - self.packages.keys() + + @property + def depends_make(self) -> List[str]: + """ + get package make dependencies + + Returns: + List[str]: sum of make dependencies per each package + """ + return sorted(set(sum((package.make_depends for package in self.packages.values()), start=[]))) + + @property + def depends_opt(self) -> List[str]: + """ + get package optional dependencies + + Returns: + List[str]: sum of optional dependencies per each package + """ + return sorted(set(sum((package.opt_depends for package in self.packages.values()), start=[]))) + @property def groups(self) -> List[str]: """ @@ -168,7 +198,7 @@ class Package(LazyLogging): base=package.package_base, version=package.version, remote=remote, - packages={package.name: PackageDescription()}) + packages={package.name: PackageDescription.from_aur(package)}) @classmethod def from_build(cls: Type[Package], path: Path) -> Package: @@ -188,7 +218,18 @@ class Package(LazyLogging): srcinfo, errors = parse_srcinfo(srcinfo_source) if errors: raise PackageInfoError(errors) - packages = {key: PackageDescription() for key in srcinfo["packages"]} + + def get_property(key: str, properties: Dict[str, Any], default: Any) -> Any: + return properties.get(key, srcinfo.get(key, default)) + + packages = { + package: PackageDescription( + depends=get_property("depends", properties, []), + make_depends=get_property("makedepends", properties, []), + opt_depends=get_property("optdepends", properties, []), + ) + for package, properties in srcinfo["packages"].items() + } version = full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"]) return cls(base=srcinfo["pkgbase"], version=version, remote=None, packages=packages) @@ -204,11 +245,12 @@ class Package(LazyLogging): Returns: Package: package properties """ + packages_json = dump.get("packages") or {} packages = { key: PackageDescription.from_json(value) - for key, value in dump.get("packages", {}).items() + for key, value in packages_json.items() } - remote = dump.get("remote", {}) + remote = dump.get("remote") or {} return cls(base=dump["base"], version=dump["version"], remote=RemoteSource.from_json(remote), packages=packages) @classmethod @@ -230,43 +272,7 @@ class Package(LazyLogging): base=package.package_base, version=package.version, remote=remote, - packages={package.name: PackageDescription()}) - - @staticmethod - def dependencies(path: Path) -> Set[str]: - """ - load dependencies from package sources - - Args: - path(Path): path to package sources directory - - Returns: - Set[str]: list of package dependencies including makedepends array, but excluding packages from this base - - Raises: - InvalidPackageInfo: if there are parsing errors - """ - # additional function to remove versions from dependencies - def extract_packages(raw_packages_list: List[str]) -> Set[str]: - return {trim_version(package_name) for package_name in raw_packages_list} - - def trim_version(package_name: str) -> str: - for symbol in ("<", "=", ">"): - package_name = package_name.split(symbol)[0] - return package_name - - srcinfo_source = Package._check_output("makepkg", "--printsrcinfo", cwd=path) - srcinfo, errors = parse_srcinfo(srcinfo_source) - if errors: - raise PackageInfoError(errors) - makedepends = extract_packages(srcinfo.get("makedepends", [])) - # sum over each package - depends = extract_packages(srcinfo.get("depends", [])) - for package in srcinfo["packages"].values(): - depends |= extract_packages(package.get("depends", [])) - # we are not interested in dependencies inside pkgbase - packages = set(srcinfo["packages"].keys()) - return (depends | makedepends) - packages + packages={package.name: PackageDescription.from_aur(package)}) @staticmethod def supported_architectures(path: Path) -> Set[str]: diff --git a/src/ahriman/models/package_description.py b/src/ahriman/models/package_description.py index b264a553..4aebf465 100644 --- a/src/ahriman/models/package_description.py +++ b/src/ahriman/models/package_description.py @@ -24,7 +24,8 @@ from pathlib import Path from pyalpm import Package # type: ignore from typing import Any, Dict, List, Optional, Type -from ahriman.core.util import filter_json +from ahriman.core.util import filter_json, trim_package +from ahriman.models.aur_package import AURPackage @dataclass(kw_only=True) @@ -37,6 +38,8 @@ class PackageDescription: archive_size(Optional[int]): package archive size build_date(Optional[int]): package build date depends(List[str]): package dependencies list + opt_depends(List[str]): optional package dependencies list + make_depends(List[str]): package dependencies list used for building description(Optional[str]): package description filename(Optional[str]): package archive name groups(List[str]): package groups @@ -67,6 +70,8 @@ class PackageDescription: archive_size: Optional[int] = None build_date: Optional[int] = None depends: List[str] = field(default_factory=list) + make_depends: List[str] = field(default_factory=list) + opt_depends: List[str] = field(default_factory=list) description: Optional[str] = None filename: Optional[str] = None groups: List[str] = field(default_factory=list) @@ -75,6 +80,14 @@ class PackageDescription: provides: List[str] = field(default_factory=list) url: Optional[str] = None + def __post_init__(self) -> None: + """ + update dependencies list accordingly + """ + self.depends = [trim_package(package) for package in self.depends] + self.opt_depends = [trim_package(package) for package in self.opt_depends] + self.make_depends = [trim_package(package) for package in self.make_depends] + @property def filepath(self) -> Optional[Path]: """ @@ -85,6 +98,27 @@ class PackageDescription: """ return Path(self.filename) if self.filename is not None else None + @classmethod + def from_aur(cls: Type[PackageDescription], package: AURPackage) -> PackageDescription: + """ + construct properties from AUR package model + + Args: + package(AURPackage): AUR package model + + Returns: + PackageDescription: package properties based on source AUR package + """ + return cls( + depends=package.depends, + make_depends=package.make_depends, + opt_depends=package.opt_depends, + description=package.description, + licenses=package.license, + provides=package.provides, + url=package.url, + ) + @classmethod def from_json(cls: Type[PackageDescription], dump: Dict[str, Any]) -> PackageDescription: """ @@ -117,13 +151,16 @@ class PackageDescription: archive_size=package.size, build_date=package.builddate, depends=package.depends, + make_depends=package.makedepends, + opt_depends=package.optdepends, description=package.desc, filename=path.name, groups=package.groups, installed_size=package.isize, licenses=package.licenses, provides=package.provides, - url=package.url) + url=package.url, + ) def view(self) -> Dict[str, Any]: """ diff --git a/tests/ahriman/application/application/test_application_packages.py b/tests/ahriman/application/application/test_application_packages.py index b2c8f971..1f45031a 100644 --- a/tests/ahriman/application/application/test_application_packages.py +++ b/tests/ahriman/application/application/test_application_packages.py @@ -2,7 +2,7 @@ import pytest from pathlib import Path from pytest_mock import MockerFixture -from unittest.mock import MagicMock +from unittest.mock import MagicMock, call as MockCall from ahriman.application.application.application_packages import ApplicationPackages from ahriman.models.package import Package @@ -29,18 +29,12 @@ def test_add_aur(application_packages: ApplicationPackages, package_ahriman: Pac must add package from AUR """ mocker.patch("ahriman.models.package.Package.from_aur", return_value=package_ahriman) - load_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.load") dependencies_mock = mocker.patch( "ahriman.application.application.application_packages.ApplicationPackages._process_dependencies") build_queue_mock = mocker.patch("ahriman.core.database.SQLite.build_queue_insert") update_remote_mock = mocker.patch("ahriman.core.database.SQLite.remote_update") application_packages._add_aur(package_ahriman.base, set(), False) - load_mock.assert_called_once_with( - pytest.helpers.anyvar(int), - package_ahriman, - pytest.helpers.anyvar(int), - application_packages.repository.paths) dependencies_mock.assert_called_once_with(pytest.helpers.anyvar(int), set(), False) build_queue_mock.assert_called_once_with(package_ahriman) update_remote_mock.assert_called_once_with(package_ahriman) @@ -121,43 +115,37 @@ def test_known_packages(application_packages: ApplicationPackages) -> None: application_packages._known_packages() -def test_process_dependencies(application_packages: ApplicationPackages, mocker: MockerFixture) -> None: +def test_process_dependencies(application_packages: ApplicationPackages, package_ahriman: Package, + mocker: MockerFixture) -> None: """ must process dependencies addition """ - missing = {"python"} - path = Path("local") - dependencies_mock = mocker.patch("ahriman.models.package.Package.dependencies", return_value=missing) add_mock = mocker.patch("ahriman.application.application.application_packages.ApplicationPackages.add") - application_packages._process_dependencies(path, set(), False) - dependencies_mock.assert_called_once_with(path) - add_mock.assert_called_once_with(missing, PackageSource.AUR, False) + application_packages._process_dependencies(package_ahriman, set(), False) + add_mock.assert_called_once_with(package_ahriman.depends_build, PackageSource.AUR, False) -def test_process_dependencies_missing(application_packages: ApplicationPackages, mocker: MockerFixture) -> None: +def test_process_dependencies_missing(application_packages: ApplicationPackages, package_ahriman: Package, + mocker: MockerFixture) -> None: """ must process dependencies addition only for missing packages """ - path = Path("local") - dependencies_mock = mocker.patch("ahriman.models.package.Package.dependencies", - return_value={"python", "python-aiohttp"}) + missing = {"devtools"} add_mock = mocker.patch("ahriman.application.application.application_packages.ApplicationPackages.add") - application_packages._process_dependencies(path, {"python"}, False) - dependencies_mock.assert_called_once_with(path) - add_mock.assert_called_once_with({"python-aiohttp"}, PackageSource.AUR, False) + application_packages._process_dependencies( + package_ahriman, package_ahriman.depends_build.difference(missing), False) + add_mock.assert_called_once_with(missing, PackageSource.AUR, False) -def test_process_dependencies_skip(application_packages: ApplicationPackages, mocker: MockerFixture) -> None: +def test_process_dependencies_skip(application_packages: ApplicationPackages, package_ahriman: Package, + mocker: MockerFixture) -> None: """ must skip dependencies processing """ - dependencies_mock = mocker.patch("ahriman.models.package.Package.dependencies") add_mock = mocker.patch("ahriman.application.application.application_packages.ApplicationPackages.add") - - application_packages._process_dependencies(Path("local"), set(), True) - dependencies_mock.assert_not_called() + application_packages._process_dependencies(package_ahriman, set(), True) add_mock.assert_not_called() diff --git a/tests/ahriman/application/application/test_application_repository.py b/tests/ahriman/application/application/test_application_repository.py index 15feb6c9..d0b80cd3 100644 --- a/tests/ahriman/application/application/test_application_repository.py +++ b/tests/ahriman/application/application/test_application_repository.py @@ -161,7 +161,7 @@ def test_update(application_repository: ApplicationRepository, package_ahriman: must process package updates """ paths = [package.filepath for package in package_ahriman.packages.values()] - tree = Tree([Leaf(package_ahriman, set())]) + tree = Tree([Leaf(package_ahriman)]) mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels()) mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=paths) @@ -181,7 +181,7 @@ def test_update_empty(application_repository: ApplicationRepository, package_ahr """ must skip updating repository if no packages supplied """ - tree = Tree([Leaf(package_ahriman, set())]) + tree = Tree([Leaf(package_ahriman)]) mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels()) mocker.patch("ahriman.core.repository.repository.Repository.packages_built", return_value=[]) @@ -197,7 +197,7 @@ def test_updates_all(application_repository: ApplicationRepository, package_ahri """ must get updates for all """ - tree = Tree([Leaf(package_ahriman, set())]) + tree = Tree([Leaf(package_ahriman)]) mocker.patch("ahriman.core.tree.Tree.resolve", return_value=tree.levels()) mocker.patch("ahriman.core.repository.repository.Repository.packages", return_value=[]) diff --git a/tests/ahriman/application/handlers/test_handler_remove_unknown.py b/tests/ahriman/application/handlers/test_handler_remove_unknown.py index 5ce5ed7f..c2eeeb5e 100644 --- a/tests/ahriman/application/handlers/test_handler_remove_unknown.py +++ b/tests/ahriman/application/handlers/test_handler_remove_unknown.py @@ -56,4 +56,4 @@ def test_run_dry_run(args: argparse.Namespace, configuration: Configuration, rep RemoveUnknown.run(args, "x86_64", configuration, report=False, unsafe=False) application_mock.assert_called_once_with() remove_mock.assert_not_called() - print_mock.assert_called_once_with(False) + print_mock.assert_called_once_with(verbose=False) diff --git a/tests/ahriman/application/handlers/test_handler_search.py b/tests/ahriman/application/handlers/test_handler_search.py index f1373056..25ee83ae 100644 --- a/tests/ahriman/application/handlers/test_handler_search.py +++ b/tests/ahriman/application/handlers/test_handler_search.py @@ -46,7 +46,7 @@ def test_run(args: argparse.Namespace, configuration: Configuration, repository: aur_search_mock.assert_called_once_with("ahriman", pacman=pytest.helpers.anyvar(int)) official_search_mock.assert_called_once_with("ahriman", pacman=pytest.helpers.anyvar(int)) check_mock.assert_called_once_with(False, False) - print_mock.assert_has_calls([MockCall(False), MockCall(False)]) + print_mock.assert_has_calls([MockCall(verbose=False), MockCall(verbose=False)]) def test_run_empty_exception(args: argparse.Namespace, configuration: Configuration, repository: Repository, diff --git a/tests/ahriman/application/handlers/test_handler_status.py b/tests/ahriman/application/handlers/test_handler_status.py index f6c8a58e..d2fe7cc1 100644 --- a/tests/ahriman/application/handlers/test_handler_status.py +++ b/tests/ahriman/application/handlers/test_handler_status.py @@ -47,7 +47,7 @@ def test_run(args: argparse.Namespace, configuration: Configuration, repository: application_mock.assert_called_once_with() packages_mock.assert_called_once_with(None) check_mock.assert_called_once_with(False, False) - print_mock.assert_has_calls([MockCall(False) for _ in range(3)]) + print_mock.assert_has_calls([MockCall(verbose=False) for _ in range(3)]) def test_run_empty_exception(args: argparse.Namespace, configuration: Configuration, repository: Repository, @@ -79,7 +79,7 @@ def test_run_verbose(args: argparse.Namespace, configuration: Configuration, rep print_mock = mocker.patch("ahriman.core.formatters.Printer.print") Status.run(args, "x86_64", configuration, report=False, unsafe=False) - print_mock.assert_has_calls([MockCall(True) for _ in range(2)]) + print_mock.assert_has_calls([MockCall(verbose=True) for _ in range(2)]) def test_run_with_package_filter(args: argparse.Namespace, configuration: Configuration, repository: Repository, @@ -111,7 +111,7 @@ def test_run_by_status(args: argparse.Namespace, configuration: Configuration, r print_mock = mocker.patch("ahriman.core.formatters.Printer.print") Status.run(args, "x86_64", configuration, report=False, unsafe=False) - print_mock.assert_has_calls([MockCall(False) for _ in range(2)]) + print_mock.assert_has_calls([MockCall(verbose=False) for _ in range(2)]) def test_imply_with_report(args: argparse.Namespace, configuration: Configuration, database: SQLite, diff --git a/tests/ahriman/application/handlers/test_handler_structure.py b/tests/ahriman/application/handlers/test_handler_structure.py index 08b37205..4b9c5c57 100644 --- a/tests/ahriman/application/handlers/test_handler_structure.py +++ b/tests/ahriman/application/handlers/test_handler_structure.py @@ -1,5 +1,4 @@ import argparse -import pytest from pytest_mock import MockerFixture @@ -20,7 +19,7 @@ def test_run(args: argparse.Namespace, configuration: Configuration, repository: print_mock = mocker.patch("ahriman.core.formatters.Printer.print") Structure.run(args, "x86_64", configuration, report=False, unsafe=False) - application_mock.assert_called_once_with([package_ahriman], repository.paths, pytest.helpers.anyvar(int)) + application_mock.assert_called_once_with([package_ahriman]) print_mock.assert_called_once_with(verbose=True, separator=" ") diff --git a/tests/ahriman/conftest.py b/tests/ahriman/conftest.py index 113358a7..131ea780 100644 --- a/tests/ahriman/conftest.py +++ b/tests/ahriman/conftest.py @@ -98,29 +98,37 @@ def aur_package_ahriman() -> AURPackage: AURPackage: AUR package test instance """ return AURPackage( - id=1009791, + id=1197565, name="ahriman", package_base_id=165427, package_base="ahriman", - version="1.7.0-1", + version="2.6.0-1", description="ArcH linux ReposItory MANager", num_votes=0, popularity=0, first_submitted=datetime.datetime.utcfromtimestamp(1618008285), - last_modified=datetime.datetime.utcfromtimestamp(1640473871), + last_modified=datetime.datetime.utcfromtimestamp(1673826351), url_path="/cgit/aur.git/snapshot/ahriman.tar.gz", url="https://github.com/arcan1s/ahriman", out_of_date=None, maintainer="arcanis", + submitter="arcanis", depends=[ "devtools", "git", "pyalpm", - "python-aur", + "python-cerberus", + "python-inflection", "python-passlib", + "python-requests", + "python-setuptools", "python-srcinfo", ], - make_depends=["python-pip"], + make_depends=[ + "python-build", + "python-installer", + "python-wheel", + ], opt_depends=[ "breezy", "darcs", @@ -133,6 +141,7 @@ def aur_package_ahriman() -> AURPackage: "python-aiohttp-session", "python-boto3", "python-cryptography", + "python-requests-unixsocket", "python-jinja", "rsync", "subversion", @@ -251,7 +260,7 @@ def package_ahriman(package_description_ahriman: PackageDescription, remote_sour packages = {"ahriman": package_description_ahriman} return Package( base="ahriman", - version="1.7.0-1", + version="2.6.0-1", remote=remote_source, packages=packages) @@ -297,12 +306,37 @@ def package_description_ahriman() -> PackageDescription: "devtools", "git", "pyalpm", - "python-aur", + "python-cerberus", + "python-inflection", "python-passlib", + "python-requests", + "python-setuptools", "python-srcinfo", ], + make_depends=[ + "python-build", + "python-installer", + "python-wheel", + ], + opt_depends=[ + "breezy", + "darcs", + "mercurial", + "python-aioauth-client", + "python-aiohttp", + "python-aiohttp-debugtoolbar", + "python-aiohttp-jinja2", + "python-aiohttp-security", + "python-aiohttp-session", + "python-boto3", + "python-cryptography", + "python-requests-unixsocket", + "python-jinja", + "rsync", + "subversion", + ], description="ArcH linux ReposItory MANager", - filename="ahriman-1.7.0-1-any.pkg.tar.zst", + filename="ahriman-2.6.0-1-any.pkg.tar.zst", groups=[], installed_size=4200000, licenses=["GPL3"], diff --git a/tests/ahriman/core/conftest.py b/tests/ahriman/core/conftest.py index dff526c3..515b8c16 100644 --- a/tests/ahriman/core/conftest.py +++ b/tests/ahriman/core/conftest.py @@ -20,7 +20,7 @@ def leaf_ahriman(package_ahriman: Package) -> Leaf: Returns: Leaf: tree leaf test instance """ - return Leaf(package_ahriman, set()) + return Leaf(package_ahriman) @pytest.fixture @@ -34,7 +34,7 @@ def leaf_python_schedule(package_python_schedule: Package) -> Leaf: Returns: Leaf: tree leaf test instance """ - return Leaf(package_python_schedule, set()) + return Leaf(package_python_schedule) @pytest.fixture diff --git a/tests/ahriman/core/database/data/test_data_init.py b/tests/ahriman/core/database/data/test_data_init.py deleted file mode 100644 index 44016e8c..00000000 --- a/tests/ahriman/core/database/data/test_data_init.py +++ /dev/null @@ -1,47 +0,0 @@ -from pytest_mock import MockerFixture -from sqlite3 import Connection - -from ahriman.core.configuration import Configuration -from ahriman.core.database.data import migrate_data -from ahriman.models.migration_result import MigrationResult -from ahriman.models.repository_paths import RepositoryPaths - - -def test_migrate_data_initial(connection: Connection, configuration: Configuration, - repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: - """ - must perform initial migration - """ - packages = mocker.patch("ahriman.core.database.data.migrate_package_statuses") - patches = mocker.patch("ahriman.core.database.data.migrate_patches") - users = mocker.patch("ahriman.core.database.data.migrate_users_data") - remotes = mocker.patch("ahriman.core.database.data.migrate_package_remotes") - - migrate_data(MigrationResult(old_version=0, new_version=900), connection, configuration) - packages.assert_called_once_with(connection, repository_paths) - patches.assert_called_once_with(connection, repository_paths) - users.assert_called_once_with(connection, configuration) - remotes.assert_called_once_with(connection, repository_paths) - - -def test_migrate_data_remotes(connection: Connection, configuration: Configuration, - repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: - """ - must perform initial migration - """ - remotes = mocker.patch("ahriman.core.database.data.migrate_package_remotes") - - migrate_data(MigrationResult(old_version=1, new_version=900), connection, configuration) - remotes.assert_called_once_with(connection, repository_paths) - - -def test_migrate_data_skip(connection: Connection, configuration: Configuration, mocker: MockerFixture) -> None: - """ - must not migrate data if version is up-to-date - """ - packages = mocker.patch("ahriman.core.database.data.migrate_package_statuses") - users = mocker.patch("ahriman.core.database.data.migrate_users_data") - - migrate_data(MigrationResult(old_version=900, new_version=900), connection, configuration) - packages.assert_not_called() - users.assert_not_called() diff --git a/tests/ahriman/core/database/data/test_package_remotes.py b/tests/ahriman/core/database/data/test_package_remotes.py deleted file mode 100644 index 51a6fe25..00000000 --- a/tests/ahriman/core/database/data/test_package_remotes.py +++ /dev/null @@ -1,66 +0,0 @@ -import pytest - -from pytest_mock import MockerFixture -from sqlite3 import Connection - -from ahriman.core.database.data import migrate_package_remotes -from ahriman.models.package import Package -from ahriman.models.repository_paths import RepositoryPaths - - -def test_migrate_package_remotes(package_ahriman: Package, connection: Connection, repository_paths: RepositoryPaths, - mocker: MockerFixture) -> None: - """ - must put package remotes to database - """ - mocker.patch( - "ahriman.core.database.operations.PackageOperations._packages_get_select_package_bases", - return_value={package_ahriman.base: package_ahriman}) - mocker.patch("pathlib.Path.exists", return_value=False) - - migrate_package_remotes(connection, repository_paths) - connection.execute.assert_called_once_with(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)) - - -def test_migrate_package_remotes_has_local(package_ahriman: Package, connection: Connection, - repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: - """ - must skip processing for packages which have local cache - """ - mocker.patch( - "ahriman.core.database.operations.PackageOperations._packages_get_select_package_bases", - return_value={package_ahriman.base: package_ahriman}) - mocker.patch("pathlib.Path.exists", return_value=True) - - migrate_package_remotes(connection, repository_paths) - connection.execute.assert_not_called() - - -def test_migrate_package_remotes_vcs(package_ahriman: Package, connection: Connection, - repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: - """ - must process VCS packages with local cache - """ - mocker.patch( - "ahriman.core.database.operations.PackageOperations._packages_get_select_package_bases", - return_value={package_ahriman.base: package_ahriman}) - mocker.patch("pathlib.Path.exists", return_value=True) - mocker.patch.object(Package, "is_vcs", True) - - migrate_package_remotes(connection, repository_paths) - connection.execute.assert_called_once_with(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)) - - -def test_migrate_package_remotes_no_remotes(package_ahriman: Package, connection: Connection, - repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: - """ - must skip processing in case if no remotes generated (should never happen) - """ - mocker.patch( - "ahriman.core.database.operations.PackageOperations._packages_get_select_package_bases", - return_value={package_ahriman.base: package_ahriman}) - mocker.patch("pathlib.Path.exists", return_value=False) - mocker.patch("ahriman.models.remote_source.RemoteSource.from_source", return_value=None) - - migrate_package_remotes(connection, repository_paths) - connection.execute.assert_not_called() diff --git a/tests/ahriman/core/database/data/test_package_statuses.py b/tests/ahriman/core/database/data/test_package_statuses.py deleted file mode 100644 index 1d2803cf..00000000 --- a/tests/ahriman/core/database/data/test_package_statuses.py +++ /dev/null @@ -1,39 +0,0 @@ -import pytest - -from pytest_mock import MockerFixture -from sqlite3 import Connection -from unittest.mock import call as MockCall - -from ahriman.core.database.data import migrate_package_statuses -from ahriman.models.package import Package -from ahriman.models.repository_paths import RepositoryPaths - - -def test_migrate_package_statuses(connection: Connection, package_ahriman: Package, repository_paths: RepositoryPaths, - mocker: MockerFixture) -> None: - """ - must migrate packages to database - """ - response = {"packages": [pytest.helpers.get_package_status_extended(package_ahriman)]} - - mocker.patch("pathlib.Path.is_file", return_value=True) - mocker.patch("pathlib.Path.open") - mocker.patch("json.load", return_value=response) - - migrate_package_statuses(connection, repository_paths) - connection.execute.assert_has_calls([ - MockCall(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)), - MockCall(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)), - ]) - connection.executemany.assert_has_calls([ - MockCall(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)), - ]) - - -def test_migrate_package_statuses_skip(connection: Connection, repository_paths: RepositoryPaths, - mocker: MockerFixture) -> None: - """ - must skip packages migration if no cache file found - """ - mocker.patch("pathlib.Path.is_file", return_value=False) - migrate_package_statuses(connection, repository_paths) diff --git a/tests/ahriman/core/database/data/test_patches.py b/tests/ahriman/core/database/data/test_patches.py deleted file mode 100644 index 5e184d43..00000000 --- a/tests/ahriman/core/database/data/test_patches.py +++ /dev/null @@ -1,52 +0,0 @@ -import pytest - -from pathlib import Path -from pytest_mock import MockerFixture -from sqlite3 import Connection - -from ahriman.core.database.data import migrate_patches -from ahriman.models.package import Package -from ahriman.models.repository_paths import RepositoryPaths - - -def test_migrate_patches(connection: Connection, repository_paths: RepositoryPaths, - package_ahriman: Package, mocker: MockerFixture) -> None: - """ - must perform migration for patches - """ - mocker.patch("pathlib.Path.is_dir", return_value=True) - mocker.patch("pathlib.Path.is_file", return_value=True) - iterdir_mock = mocker.patch("pathlib.Path.iterdir", return_value=[Path(package_ahriman.base)]) - read_mock = mocker.patch("pathlib.Path.read_text", return_value="patch") - - migrate_patches(connection, repository_paths) - iterdir_mock.assert_called_once_with() - read_mock.assert_called_once_with(encoding="utf8") - connection.execute.assert_called_once_with(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)) - - -def test_migrate_patches_skip(connection: Connection, repository_paths: RepositoryPaths, - mocker: MockerFixture) -> None: - """ - must skip patches migration in case if no root found - """ - mocker.patch("pathlib.Path.is_dir", return_value=False) - iterdir_mock = mocker.patch("pathlib.Path.iterdir") - - migrate_patches(connection, repository_paths) - iterdir_mock.assert_not_called() - - -def test_migrate_patches_no_patch(connection: Connection, repository_paths: RepositoryPaths, - package_ahriman: Package, mocker: MockerFixture) -> None: - """ - must skip package if no match found - """ - mocker.patch("pathlib.Path.is_dir", return_value=True) - mocker.patch("pathlib.Path.is_file", return_value=False) - iterdir_mock = mocker.patch("pathlib.Path.iterdir", return_value=[Path(package_ahriman.base)]) - read_mock = mocker.patch("pathlib.Path.read_text") - - migrate_patches(connection, repository_paths) - iterdir_mock.assert_called_once_with() - read_mock.assert_not_called() diff --git a/tests/ahriman/core/database/data/test_users.py b/tests/ahriman/core/database/data/test_users.py deleted file mode 100644 index 8956214c..00000000 --- a/tests/ahriman/core/database/data/test_users.py +++ /dev/null @@ -1,21 +0,0 @@ -import pytest - -from sqlite3 import Connection -from unittest.mock import call as MockCall - -from ahriman.core.configuration import Configuration -from ahriman.core.database.data import migrate_users_data - - -def test_migrate_users_data(connection: Connection, configuration: Configuration) -> None: - """ - must put users to database - """ - configuration.set_option("auth:read", "user1", "password1") - configuration.set_option("auth:write", "user2", "password2") - - migrate_users_data(connection, configuration) - connection.execute.assert_has_calls([ - MockCall(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)), - MockCall(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)), - ]) diff --git a/tests/ahriman/core/database/migrations/test_m000_initial.py b/tests/ahriman/core/database/migrations/test_m000_initial.py index b73f47b0..b335f0eb 100644 --- a/tests/ahriman/core/database/migrations/test_m000_initial.py +++ b/tests/ahriman/core/database/migrations/test_m000_initial.py @@ -1,4 +1,15 @@ -from ahriman.core.database.migrations.m000_initial import steps +import pytest + +from pathlib import Path +from pytest_mock import MockerFixture +from sqlite3 import Connection +from unittest.mock import call as MockCall + +from ahriman.core.configuration import Configuration +from ahriman.core.database.migrations.m000_initial import migrate_data, migrate_package_statuses, migrate_patches, \ + migrate_users_data, steps +from ahriman.models.package import Package +from ahriman.models.repository_paths import RepositoryPaths def test_migration_initial() -> None: @@ -6,3 +17,104 @@ def test_migration_initial() -> None: migration must not be empty """ assert steps + + +def test_migrate_data(connection: Connection, configuration: Configuration, mocker: MockerFixture) -> None: + """ + must perform data migration + """ + packages = mocker.patch("ahriman.core.database.migrations.m000_initial.migrate_package_statuses") + patches = mocker.patch("ahriman.core.database.migrations.m000_initial.migrate_patches") + users = mocker.patch("ahriman.core.database.migrations.m000_initial.migrate_users_data") + + migrate_data(connection, configuration) + packages.assert_called_once_with(connection, configuration.repository_paths) + patches.assert_called_once_with(connection, configuration.repository_paths) + users.assert_called_once_with(connection, configuration) + + +def test_migrate_package_statuses(connection: Connection, package_ahriman: Package, repository_paths: RepositoryPaths, + mocker: MockerFixture) -> None: + """ + must migrate packages to database + """ + response = {"packages": [pytest.helpers.get_package_status_extended(package_ahriman)]} + + mocker.patch("pathlib.Path.is_file", return_value=True) + mocker.patch("pathlib.Path.open") + mocker.patch("json.load", return_value=response) + + migrate_package_statuses(connection, repository_paths) + connection.execute.assert_has_calls([ + MockCall(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)), + MockCall(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)), + ]) + connection.executemany.assert_has_calls([ + MockCall(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)), + ]) + + +def test_migrate_package_statuses_skip(connection: Connection, repository_paths: RepositoryPaths, + mocker: MockerFixture) -> None: + """ + must skip packages migration if no cache file found + """ + mocker.patch("pathlib.Path.is_file", return_value=False) + migrate_package_statuses(connection, repository_paths) + + +def test_migrate_patches(connection: Connection, repository_paths: RepositoryPaths, + package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must perform migration for patches + """ + mocker.patch("pathlib.Path.is_dir", return_value=True) + mocker.patch("pathlib.Path.is_file", return_value=True) + iterdir_mock = mocker.patch("pathlib.Path.iterdir", return_value=[Path(package_ahriman.base)]) + read_mock = mocker.patch("pathlib.Path.read_text", return_value="patch") + + migrate_patches(connection, repository_paths) + iterdir_mock.assert_called_once_with() + read_mock.assert_called_once_with(encoding="utf8") + connection.execute.assert_called_once_with(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)) + + +def test_migrate_patches_skip(connection: Connection, repository_paths: RepositoryPaths, + mocker: MockerFixture) -> None: + """ + must skip patches migration in case if no root found + """ + mocker.patch("pathlib.Path.is_dir", return_value=False) + iterdir_mock = mocker.patch("pathlib.Path.iterdir") + + migrate_patches(connection, repository_paths) + iterdir_mock.assert_not_called() + + +def test_migrate_patches_no_patch(connection: Connection, repository_paths: RepositoryPaths, + package_ahriman: Package, mocker: MockerFixture) -> None: + """ + must skip package if no match found + """ + mocker.patch("pathlib.Path.is_dir", return_value=True) + mocker.patch("pathlib.Path.is_file", return_value=False) + iterdir_mock = mocker.patch("pathlib.Path.iterdir", return_value=[Path(package_ahriman.base)]) + read_mock = mocker.patch("pathlib.Path.read_text") + + migrate_patches(connection, repository_paths) + iterdir_mock.assert_called_once_with() + read_mock.assert_not_called() + + +def test_migrate_users_data(connection: Connection, configuration: Configuration) -> None: + """ + must put users to database + """ + configuration.set_option("auth:read", "user1", "password1") + configuration.set_option("auth:write", "user2", "password2") + + migrate_users_data(connection, configuration) + connection.execute.assert_has_calls([ + MockCall(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)), + MockCall(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)), + ]) diff --git a/tests/ahriman/core/database/migrations/test_m001_package_source.py b/tests/ahriman/core/database/migrations/test_m001_package_source.py index a227db36..e5e67329 100644 --- a/tests/ahriman/core/database/migrations/test_m001_package_source.py +++ b/tests/ahriman/core/database/migrations/test_m001_package_source.py @@ -1,4 +1,12 @@ -from ahriman.core.database.migrations.m001_package_source import steps +import pytest + +from sqlite3 import Connection +from pytest_mock import MockerFixture + +from ahriman.core.configuration import Configuration +from ahriman.core.database.migrations.m001_package_source import migrate_data, migrate_package_remotes, steps +from ahriman.models.package import Package +from ahriman.models.repository_paths import RepositoryPaths def test_migration_package_source() -> None: @@ -6,3 +14,70 @@ def test_migration_package_source() -> None: migration must not be empty """ assert steps + + +def test_migrate_data(connection: Connection, configuration: Configuration, mocker: MockerFixture) -> None: + """ + must perform data migration + """ + remotes = mocker.patch("ahriman.core.database.migrations.m001_package_source.migrate_package_remotes") + migrate_data(connection, configuration) + remotes.assert_called_once_with(connection, configuration.repository_paths) + + +def test_migrate_package_remotes(package_ahriman: Package, connection: Connection, repository_paths: RepositoryPaths, + mocker: MockerFixture) -> None: + """ + must put package remotes to database + """ + mocker.patch( + "ahriman.core.database.operations.PackageOperations._packages_get_select_package_bases", + return_value={package_ahriman.base: package_ahriman}) + mocker.patch("pathlib.Path.exists", return_value=False) + + migrate_package_remotes(connection, repository_paths) + connection.execute.assert_called_once_with(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)) + + +def test_migrate_package_remotes_has_local(package_ahriman: Package, connection: Connection, + repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: + """ + must skip processing for packages which have local cache + """ + mocker.patch( + "ahriman.core.database.operations.PackageOperations._packages_get_select_package_bases", + return_value={package_ahriman.base: package_ahriman}) + mocker.patch("pathlib.Path.exists", return_value=True) + + migrate_package_remotes(connection, repository_paths) + connection.execute.assert_not_called() + + +def test_migrate_package_remotes_vcs(package_ahriman: Package, connection: Connection, + repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: + """ + must process VCS packages with local cache + """ + mocker.patch( + "ahriman.core.database.operations.PackageOperations._packages_get_select_package_bases", + return_value={package_ahriman.base: package_ahriman}) + mocker.patch("pathlib.Path.exists", return_value=True) + mocker.patch.object(Package, "is_vcs", True) + + migrate_package_remotes(connection, repository_paths) + connection.execute.assert_called_once_with(pytest.helpers.anyvar(str, strict=True), pytest.helpers.anyvar(int)) + + +def test_migrate_package_remotes_no_remotes(package_ahriman: Package, connection: Connection, + repository_paths: RepositoryPaths, mocker: MockerFixture) -> None: + """ + must skip processing in case if no remotes generated (should never happen) + """ + mocker.patch( + "ahriman.core.database.operations.PackageOperations._packages_get_select_package_bases", + return_value={package_ahriman.base: package_ahriman}) + mocker.patch("pathlib.Path.exists", return_value=False) + mocker.patch("ahriman.models.remote_source.RemoteSource.from_source", return_value=None) + + migrate_package_remotes(connection, repository_paths) + connection.execute.assert_not_called() diff --git a/tests/ahriman/core/database/migrations/test_m005_make_opt_depends.py b/tests/ahriman/core/database/migrations/test_m005_make_opt_depends.py new file mode 100644 index 00000000..c392a540 --- /dev/null +++ b/tests/ahriman/core/database/migrations/test_m005_make_opt_depends.py @@ -0,0 +1,53 @@ +import pytest + +from pytest_mock import MockerFixture +from sqlite3 import Connection + +from ahriman.core.configuration import Configuration +from ahriman.core.database.migrations.m005_make_opt_depends import migrate_data, migrate_package_depends, steps +from ahriman.models.package import Package + + +def test_migration_make_opt_depends() -> None: + """ + migration must not be empty + """ + assert steps + + +def test_migrate_data(connection: Connection, configuration: Configuration, mocker: MockerFixture) -> None: + """ + must perform data migration + """ + depends_mock = mocker.patch("ahriman.core.database.migrations.m005_make_opt_depends.migrate_package_depends") + migrate_data(connection, configuration) + depends_mock.assert_called_once_with(connection, configuration) + + +def test_migrate_package_depends(connection: Connection, configuration: Configuration, package_ahriman: Package, + mocker: MockerFixture) -> None: + """ + must update make and opt depends list + """ + mocker.patch("pathlib.Path.is_dir", return_value=True) + mocker.patch("pathlib.Path.iterdir", return_value=[package_ahriman.packages[package_ahriman.base].filepath]) + package_mock = mocker.patch("ahriman.models.package.Package.from_archive", return_value=package_ahriman) + + migrate_package_depends(connection, configuration) + package_mock.assert_called_once_with( + package_ahriman.packages[package_ahriman.base].filepath, pytest.helpers.anyvar(int), remote=None) + connection.executemany.assert_called_once_with(pytest.helpers.anyvar(str, strict=True), [{ + "make_depends": package_ahriman.packages[package_ahriman.base].make_depends, + "opt_depends": package_ahriman.packages[package_ahriman.base].opt_depends, + "package": package_ahriman.base, + }]) + + +def test_migrate_package_depends_skip(connection: Connection, configuration: Configuration, + mocker: MockerFixture) -> None: + """ + must skip update make and opt depends list if no repository directory found + """ + mocker.patch("pathlib.Path.is_dir", return_value=False) + migrate_package_depends(connection, configuration) + connection.executemany.assert_not_called() diff --git a/tests/ahriman/core/database/migrations/test_migrations_init.py b/tests/ahriman/core/database/migrations/test_migrations_init.py index 7d643bb1..508a55df 100644 --- a/tests/ahriman/core/database/migrations/test_migrations_init.py +++ b/tests/ahriman/core/database/migrations/test_migrations_init.py @@ -19,6 +19,19 @@ def test_migrate(connection: Connection, configuration: Configuration, mocker: M run_mock.assert_called_once_with() +def test_migration(migrations: Migrations, connection: Connection) -> None: + """ + must perform single migration + """ + migrate_data_mock = MagicMock() + cursor = MagicMock() + migration = Migration(index=0, name="test", steps=["select 1"], migrate_data=migrate_data_mock) + + migrations.migration(cursor, migration) + cursor.execute.assert_called_once_with("select 1") + migrate_data_mock.assert_called_once_with(migrations.connection, migrations.configuration) + + def test_migrations(migrations: Migrations) -> None: """ must retrieve migrations @@ -40,25 +53,23 @@ def test_run(migrations: Migrations, mocker: MockerFixture) -> None: """ must run migration """ + migration = Migration(index=0, name="test", steps=["select 1"], migrate_data=MagicMock()) cursor = MagicMock() mocker.patch("ahriman.core.database.migrations.Migrations.user_version", return_value=0) - mocker.patch("ahriman.core.database.migrations.Migrations.migrations", - return_value=[Migration(index=0, name="test", steps=["select 1"])]) + mocker.patch("ahriman.core.database.migrations.Migrations.migrations", return_value=[migration]) migrations.connection.cursor.return_value = cursor + migration_mock = mocker.patch("ahriman.core.database.migrations.Migrations.migration") validate_mock = mocker.patch("ahriman.models.migration_result.MigrationResult.validate") - migrate_data_mock = mocker.patch("ahriman.core.database.migrations.migrate_data") migrations.run() validate_mock.assert_called_once_with() cursor.execute.assert_has_calls([ MockCall("begin exclusive"), - MockCall("select 1"), MockCall("pragma user_version = 1"), MockCall("commit"), ]) cursor.close.assert_called_once_with() - migrate_data_mock.assert_called_once_with( - MigrationResult(old_version=0, new_version=1), migrations.connection, migrations.configuration) + migration_mock.assert_called_once_with(cursor, migration) def test_run_migration_exception(migrations: Migrations, mocker: MockerFixture) -> None: @@ -69,7 +80,7 @@ def test_run_migration_exception(migrations: Migrations, mocker: MockerFixture) mocker.patch("logging.Logger.info", side_effect=Exception()) mocker.patch("ahriman.core.database.migrations.Migrations.user_version", return_value=0) mocker.patch("ahriman.core.database.migrations.Migrations.migrations", - return_value=[Migration(index=0, name="test", steps=["select 1"])]) + return_value=[Migration(index=0, name="test", steps=["select 1"], migrate_data=MagicMock())]) mocker.patch("ahriman.models.migration_result.MigrationResult.validate") migrations.connection.cursor.return_value = cursor @@ -90,7 +101,7 @@ def test_run_sql_exception(migrations: Migrations, mocker: MockerFixture) -> Non cursor.execute.side_effect = Exception() mocker.patch("ahriman.core.database.migrations.Migrations.user_version", return_value=0) mocker.patch("ahriman.core.database.migrations.Migrations.migrations", - return_value=[Migration(index=0, name="test", steps=["select 1"])]) + return_value=[Migration(index=0, name="test", steps=["select 1"], migrate_data=MagicMock())]) mocker.patch("ahriman.models.migration_result.MigrationResult.validate") migrations.connection.cursor.return_value = cursor diff --git a/tests/ahriman/core/repository/test_repository.py b/tests/ahriman/core/repository/test_repository.py index ee05c55a..e958a324 100644 --- a/tests/ahriman/core/repository/test_repository.py +++ b/tests/ahriman/core/repository/test_repository.py @@ -125,7 +125,7 @@ def test_packages_depend_on(repository: Repository, package_ahriman: Package, pa """ mocker.patch("ahriman.core.repository.repository.Repository.packages", return_value=[package_ahriman, package_python_schedule]) - assert repository.packages_depend_on([package_ahriman], ["python-aur"]) == [package_ahriman] + assert repository.packages_depend_on([package_ahriman], ["python-srcinfo"]) == [package_ahriman] def test_packages_depend_on_empty(repository: Repository, package_ahriman: Package, package_python_schedule: Package, diff --git a/tests/ahriman/core/test_tree.py b/tests/ahriman/core/test_tree.py index d1ff6c32..cc227cbb 100644 --- a/tests/ahriman/core/test_tree.py +++ b/tests/ahriman/core/test_tree.py @@ -1,12 +1,6 @@ -import pytest - -from pytest_mock import MockerFixture - -from ahriman.core.database import SQLite from ahriman.core.tree import Leaf, Tree from ahriman.models.package import Package from ahriman.models.package_description import PackageDescription -from ahriman.models.repository_paths import RepositoryPaths def test_leaf_is_root_empty(leaf_ahriman: Leaf) -> None: @@ -39,29 +33,11 @@ def test_leaf_is_root_true(leaf_ahriman: Leaf, leaf_python_schedule: Leaf) -> No assert not leaf_ahriman.is_root([leaf_python_schedule]) -def test_leaf_load(package_ahriman: Package, repository_paths: RepositoryPaths, - database: SQLite, mocker: MockerFixture) -> None: - """ - must load with dependencies - """ - load_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.load") - dependencies_mock = mocker.patch("ahriman.models.package.Package.dependencies", return_value={"ahriman-dependency"}) - - leaf = Leaf.load(package_ahriman, repository_paths, database) - assert leaf.package == package_ahriman - assert leaf.dependencies == {"ahriman-dependency"} - load_mock.assert_called_once_with(pytest.helpers.anyvar(int), package_ahriman, [], repository_paths) - dependencies_mock.assert_called_once_with(pytest.helpers.anyvar(int)) - - -def test_tree_resolve(package_ahriman: Package, package_python_schedule: Package, repository_paths: RepositoryPaths, - database: SQLite, mocker: MockerFixture) -> None: +def test_tree_resolve(package_ahriman: Package, package_python_schedule: Package) -> None: """ must resolve denendecnies tree """ - mocker.patch("ahriman.core.tree.Leaf.load", side_effect=lambda package, p, d: Leaf(package, set(package.depends))) - - tree = Tree.resolve([package_ahriman, package_python_schedule], repository_paths, database) + tree = Tree.resolve([package_ahriman, package_python_schedule]) assert len(tree) == 1 assert len(tree[0]) == 2 @@ -87,36 +63,32 @@ def test_tree_levels_sorted() -> None: base="package1", version="1.0.0", remote=None, - packages={"package1": PackageDescription()} - ), - dependencies=set() + packages={"package1": PackageDescription(depends=[])} + ) ) leaf2 = Leaf( Package( base="package2", version="1.0.0", remote=None, - packages={"package2": PackageDescription()} - ), - dependencies={"package1"} + packages={"package2": PackageDescription(depends=["package1"])} + ) ) leaf3 = Leaf( Package( base="package3", version="1.0.0", remote=None, - packages={"package3": PackageDescription()} - ), - dependencies={"package1"} + packages={"package3": PackageDescription(depends=["package1"])} + ) ) leaf4 = Leaf( Package( base="package4", version="1.0.0", remote=None, - packages={"package4": PackageDescription()} - ), - dependencies={"package3"} + packages={"package4": PackageDescription(depends=["package3"])} + ) ) tree = Tree([leaf1, leaf2, leaf3, leaf4]) diff --git a/tests/ahriman/core/test_util.py b/tests/ahriman/core/test_util.py index 20828a7b..a0f674ad 100644 --- a/tests/ahriman/core/test_util.py +++ b/tests/ahriman/core/test_util.py @@ -12,7 +12,7 @@ from unittest.mock import MagicMock from ahriman.core.exceptions import BuildError, OptionError, UnsafeRunError from ahriman.core.util import check_output, check_user, enum_values, exception_response_text, filter_json, \ - full_version, package_like, pretty_datetime, pretty_size, safe_filename, utcnow, walk + full_version, package_like, pretty_datetime, pretty_size, safe_filename, trim_package, utcnow, walk from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.repository_paths import RepositoryPaths @@ -322,6 +322,18 @@ def test_safe_filename() -> None: assert safe_filename("tolua++-1.0.93-4-x86_64.pkg.tar.zst") == "tolua---1.0.93-4-x86_64.pkg.tar.zst" +def test_trim_package() -> None: + """ + must trim package version + """ + assert trim_package("package=1") == "package" + assert trim_package("package>=1") == "package" + assert trim_package("package>1") == "package" + assert trim_package("package<1") == "package" + assert trim_package("package<=1") == "package" + assert trim_package("package: a description") == "package" + + def test_utcnow() -> None: """ must generate correct timestamp diff --git a/tests/ahriman/core/upload/test_http_upload.py b/tests/ahriman/core/upload/test_http_upload.py index 95a19765..e6e977f5 100644 --- a/tests/ahriman/core/upload/test_http_upload.py +++ b/tests/ahriman/core/upload/test_http_upload.py @@ -22,7 +22,7 @@ def test_calculate_hash_small(resource_path_root: Path) -> None: must calculate checksum for path which is single chunk """ path = resource_path_root / "models" / "package_ahriman_srcinfo" - assert HttpUpload.calculate_hash(path) == "fcfc0f2522b0ee92de89fcedc7e56010" + assert HttpUpload.calculate_hash(path) == "79b0f84e0232ed34fd191a85c383ecc5" def test_get_body_get_hashes() -> None: diff --git a/tests/ahriman/core/upload/test_s3.py b/tests/ahriman/core/upload/test_s3.py index f352e0a3..33815891 100644 --- a/tests/ahriman/core/upload/test_s3.py +++ b/tests/ahriman/core/upload/test_s3.py @@ -30,7 +30,7 @@ def test_calculate_etag_small(resource_path_root: Path) -> None: must calculate checksum for path which is single chunk """ path = resource_path_root / "models" / "package_ahriman_srcinfo" - assert S3.calculate_etag(path, _chunk_size) == "fcfc0f2522b0ee92de89fcedc7e56010" + assert S3.calculate_etag(path, _chunk_size) == "79b0f84e0232ed34fd191a85c383ecc5" def test_files_remove(s3_remote_objects: List[Any]) -> None: diff --git a/tests/ahriman/models/conftest.py b/tests/ahriman/models/conftest.py index 14003859..a25234b8 100644 --- a/tests/ahriman/models/conftest.py +++ b/tests/ahriman/models/conftest.py @@ -139,6 +139,8 @@ def pyalpm_package_description_ahriman(package_description_ahriman: PackageDescr type(mock).arch = PropertyMock(return_value=package_description_ahriman.architecture) type(mock).builddate = PropertyMock(return_value=package_description_ahriman.build_date) type(mock).depends = PropertyMock(return_value=package_description_ahriman.depends) + type(mock).makedepends = PropertyMock(return_value=package_description_ahriman.make_depends) + type(mock).optdepends = PropertyMock(return_value=package_description_ahriman.opt_depends) type(mock).desc = PropertyMock(return_value=package_description_ahriman.description) type(mock).groups = PropertyMock(return_value=package_description_ahriman.groups) type(mock).isize = PropertyMock(return_value=package_description_ahriman.installed_size) diff --git a/tests/ahriman/models/test_aur_package.py b/tests/ahriman/models/test_aur_package.py index 9923c549..c9cb2753 100644 --- a/tests/ahriman/models/test_aur_package.py +++ b/tests/ahriman/models/test_aur_package.py @@ -66,6 +66,7 @@ def test_from_pacman(pyalpm_package_ahriman: pyalpm.Package, aur_package_ahriman object.__setattr__(model, "first_submitted", aur_package_ahriman.first_submitted) object.__setattr__(model, "url_path", aur_package_ahriman.url_path) object.__setattr__(model, "maintainer", aur_package_ahriman.maintainer) + object.__setattr__(model, "submitter", aur_package_ahriman.submitter) assert model == aur_package_ahriman diff --git a/tests/ahriman/models/test_package.py b/tests/ahriman/models/test_package.py index 7e14a6cb..b9d0898b 100644 --- a/tests/ahriman/models/test_package.py +++ b/tests/ahriman/models/test_package.py @@ -9,6 +9,7 @@ from ahriman.core.exceptions import PackageInfoError from ahriman.core.util import utcnow from ahriman.models.aur_package import AURPackage from ahriman.models.package import Package +from ahriman.models.package_description import PackageDescription from ahriman.models.repository_paths import RepositoryPaths @@ -22,6 +23,58 @@ def test_depends(package_python_schedule: Package) -> None: ) +def test_depends_build(package_ahriman: Package, package_python_schedule: Package) -> None: + """ + must return full list of packages required for build + """ + assert all( + set(package_ahriman.depends_build).intersection(package.depends) + for package in package_ahriman.packages.values() + ) + assert all( + set(package_ahriman.depends_build).intersection(package.make_depends) + for package in package_ahriman.packages.values() + ) + + assert all( + set(package_python_schedule.depends_build).intersection(package.depends) + for package in package_python_schedule.packages.values() + ) + # there is no make dependencies for python-schedule + + +def test_depends_build_with_version_and_overlap(mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must load correct list of dependencies with version + """ + + srcinfo = (resource_path_root / "models" / "package_gcc10_srcinfo").read_text() + mocker.patch("ahriman.models.package.Package._check_output", return_value=srcinfo) + + package_gcc10 = Package.from_build(Path("local")) + assert package_gcc10.depends_build == {"glibc", "doxygen", "binutils", "git", "libmpc", "python", "zstd"} + + +def test_depends_make(package_ahriman: Package) -> None: + """ + must return list of make dependencies + """ + assert all( + set(package_ahriman.depends_make).intersection(package.make_depends) + for package in package_ahriman.packages.values() + ) + + +def test_depends_opt(package_ahriman: Package) -> None: + """ + must return list of optional dependencies + """ + assert all( + set(package_ahriman.depends_opt).intersection(package.opt_depends) + for package in package_ahriman.packages.values() + ) + + def test_groups(package_ahriman: Package) -> None: """ must return list of groups for each package @@ -108,6 +161,33 @@ def test_from_build(package_ahriman: Package, mocker: MockerFixture, resource_pa assert package_ahriman == package +def test_from_build_multiple_packages(mocker: MockerFixture, resource_path_root: Path) -> None: + """ + must construct package from srcinfo with dependencies per-package overrides + """ + srcinfo = (resource_path_root / "models" / "package_gcc10_srcinfo").read_text() + mocker.patch("ahriman.models.package.Package._check_output", return_value=srcinfo) + + package = Package.from_build(Path("path")) + assert package.packages == { + "gcc10": PackageDescription( + depends=["gcc10-libs=10.3.0-2", "binutils>=2.28", "libmpc", "zstd"], + make_depends=["binutils", "doxygen", "git", "libmpc", "python"], + opt_depends=[], + ), + "gcc10-libs": PackageDescription( + depends=["glibc>=2.27"], + make_depends=["binutils", "doxygen", "git", "libmpc", "python"], + opt_depends=[], + ), + "gcc10-fortran": PackageDescription( + depends=["gcc10=10.3.0-2"], + make_depends=["binutils", "doxygen", "git", "libmpc", "python"], + opt_depends=[], + ), + } + + def test_from_build_failed(package_ahriman: Package, mocker: MockerFixture) -> None: """ must raise exception if there are errors during srcinfo load @@ -153,37 +233,6 @@ def test_from_official(package_ahriman: Package, aur_package_ahriman: AURPackage assert package_ahriman.packages.keys() == package.packages.keys() -def test_dependencies_failed(mocker: MockerFixture) -> None: - """ - must raise exception if there are errors during srcinfo load for dependencies - """ - mocker.patch("ahriman.models.package.Package._check_output", return_value="") - mocker.patch("ahriman.models.package.parse_srcinfo", return_value=({"packages": {}}, ["an error"])) - - with pytest.raises(PackageInfoError): - Package.dependencies(Path("path")) - - -def test_dependencies_with_version(mocker: MockerFixture, resource_path_root: Path) -> None: - """ - must load correct list of dependencies with version - """ - srcinfo = (resource_path_root / "models" / "package_yay_srcinfo").read_text() - mocker.patch("ahriman.models.package.Package._check_output", return_value=srcinfo) - - assert Package.dependencies(Path("path")) == {"git", "go", "pacman"} - - -def test_dependencies_with_version_and_overlap(mocker: MockerFixture, resource_path_root: Path) -> None: - """ - must load correct list of dependencies with version - """ - srcinfo = (resource_path_root / "models" / "package_gcc10_srcinfo").read_text() - mocker.patch("ahriman.models.package.Package._check_output", return_value=srcinfo) - - assert Package.dependencies(Path("path")) == {"glibc", "doxygen", "binutils", "git", "libmpc", "python", "zstd"} - - def test_supported_architectures(mocker: MockerFixture, resource_path_root: Path) -> None: """ must generate list of available architectures diff --git a/tests/ahriman/models/test_package_description.py b/tests/ahriman/models/test_package_description.py index 3a84c020..30c72a69 100644 --- a/tests/ahriman/models/test_package_description.py +++ b/tests/ahriman/models/test_package_description.py @@ -1,8 +1,17 @@ from unittest.mock import MagicMock +from ahriman.models.aur_package import AURPackage from ahriman.models.package_description import PackageDescription +def test_post_init() -> None: + """ + must trim versions and descriptions from dependencies list + """ + assert PackageDescription(depends=["a=1"], make_depends=["b>=3"], opt_depends=["c: a description"]) == \ + PackageDescription(depends=["a"], make_depends=["b"], opt_depends=["c"]) + + def test_filepath(package_description_ahriman: PackageDescription) -> None: """ must generate correct filepath if set @@ -19,6 +28,19 @@ def test_filepath_empty(package_description_ahriman: PackageDescription) -> None assert package_description_ahriman.filepath is None +def test_from_aur(package_description_ahriman: PackageDescription, aur_package_ahriman: AURPackage) -> None: + """ + must construct package description from AUR descriptor + """ + actual = PackageDescription.from_aur(aur_package_ahriman) + # missing attributes + actual.architecture = package_description_ahriman.architecture + actual.archive_size = package_description_ahriman.archive_size + actual.build_date = package_description_ahriman.build_date + actual.filename = package_description_ahriman.filename + actual.installed_size = package_description_ahriman.installed_size + + def test_from_json(package_description_ahriman: PackageDescription) -> None: """ must construct description from json object diff --git a/tests/testresources/models/package_ahriman_aur b/tests/testresources/models/package_ahriman_aur index cf5bf80a..9b5d5818 100644 --- a/tests/testresources/models/package_ahriman_aur +++ b/tests/testresources/models/package_ahriman_aur @@ -6,21 +6,26 @@ "devtools", "git", "pyalpm", - "python-aur", + "python-cerberus", + "python-inflection", "python-passlib", + "python-requests", + "python-setuptools", "python-srcinfo" ], "Description": "ArcH linux ReposItory MANager", "FirstSubmitted": 1618008285, - "ID": 1009791, + "ID": 1197565, "Keywords": [], - "LastModified": 1640473871, + "LastModified": 1673826351, "License": [ "GPL3" ], "Maintainer": "arcanis", "MakeDepends": [ - "python-pip" + "python-build", + "python-installer", + "python-wheel" ], "Name": "ahriman", "NumVotes": 0, @@ -36,6 +41,7 @@ "python-aiohttp-session", "python-boto3", "python-cryptography", + "python-requests-unixsocket", "python-jinja", "rsync", "subversion" @@ -44,11 +50,12 @@ "PackageBase": "ahriman", "PackageBaseID": 165427, "Popularity": 0, + "Submitter": "arcanis", "URL": "https://github.com/arcan1s/ahriman", "URLPath": "/cgit/aur.git/snapshot/ahriman.tar.gz", - "Version": "1.7.0-1" + "Version": "2.6.0-1" } ], "type": "multiinfo", "version": 5 -} +} \ No newline at end of file diff --git a/tests/testresources/models/package_ahriman_srcinfo b/tests/testresources/models/package_ahriman_srcinfo index 4a6c8b45..a0f854b0 100644 --- a/tests/testresources/models/package_ahriman_srcinfo +++ b/tests/testresources/models/package_ahriman_srcinfo @@ -1,35 +1,44 @@ pkgbase = ahriman - pkgdesc = ArcH linux ReposItory MANager - pkgver = 1.7.0 - pkgrel = 1 - url = https://github.com/arcan1s/ahriman - arch = any - license = GPL3 - makedepends = python-pip - depends = devtools - depends = git - depends = pyalpm - depends = python-aur - depends = python-passlib - depends = python-srcinfo - optdepends = aws-cli: sync to s3 - optdepends = breezy: -bzr packages support - optdepends = darcs: -darcs packages support - optdepends = gnupg: package and repository sign - optdepends = mercurial: -hg packages support - optdepends = python-aiohttp: web server - optdepends = python-aiohttp-jinja2: web server - optdepends = python-jinja: html report generation - optdepends = python-requests: web server - optdepends = rsync: sync by using rsync - optdepends = subversion: -svn packages support - backup = etc/ahriman.ini - backup = etc/ahriman.ini.d/logging.ini - source = https://github.com/arcan1s/ahriman/releases/download/1.7.0/ahriman-1.7.0-src.tar.xz - source = ahriman.sysusers - source = ahriman.tmpfiles - sha512sums = 8acc57f937d587ca665c29092cadddbaf3ba0b80e870b80d1551e283aba8f21306f9030a26fec8c71ab5863316f5f5f061b7ddc63cdff9e6d5a885f28ef1893d - sha512sums = 13718afec2c6786a18f0b223ef8e58dccf0688bca4cdbe203f14071f5031ed20120eb0ce38b52c76cfd6e8b6581a9c9eaa2743eb11abbaca637451a84c33f075 - sha512sums = 55b20f6da3d66e7bbf2add5d95a3b60632df121717d25a993e56e737d14f51fe063eb6f1b38bd81cc32e05db01c0c1d80aaa720c45cde87f238d8b46cdb8cbc4 + pkgdesc = ArcH linux ReposItory MANager + pkgver = 2.6.0 + pkgrel = 1 + url = https://github.com/arcan1s/ahriman + arch = any + license = GPL3 + makedepends = python-build + makedepends = python-installer + makedepends = python-wheel + depends = devtools + depends = git + depends = pyalpm + depends = python-cerberus + depends = python-inflection + depends = python-passlib + depends = python-requests + depends = python-setuptools + depends = python-srcinfo + optdepends = breezy: -bzr packages support + optdepends = darcs: -darcs packages support + optdepends = mercurial: -hg packages support + optdepends = python-aioauth-client: web server with OAuth2 authorization + optdepends = python-aiohttp: web server + optdepends = python-aiohttp-debugtoolbar: web server with enabled debug panel + optdepends = python-aiohttp-jinja2: web server + optdepends = python-aiohttp-security: web server with authorization + optdepends = python-aiohttp-session: web server with authorization + optdepends = python-boto3: sync to s3 + optdepends = python-cryptography: web server with authorization + optdepends = python-requests-unixsocket: client report to web server by unix socket + optdepends = python-jinja: html report generation + optdepends = rsync: sync by using rsync + optdepends = subversion: -svn packages support + backup = etc/ahriman.ini + backup = etc/ahriman.ini.d/logging.ini + source = https://github.com/arcan1s/ahriman/releases/download/2.6.0/ahriman-2.6.0-src.tar.xz + source = ahriman.sysusers + source = ahriman.tmpfiles + sha512sums = ec1f64e463455761d72be7f7b8b51b3b4424685c96a2d5eee6afa1c93780c8d7f8a39487a2f2f3bd83d2b58a93279e1392a965a4b905795e58ca686fb21123a1 + sha512sums = 53d37efec812afebf86281716259f9ea78a307b83897166c72777251c3eebcb587ecee375d907514781fb2a5c808cbb24ef9f3f244f12740155d0603bf213131 + sha512sums = 62b2eccc352d33853ef243c9cddd63663014aa97b87242f1b5bc5099a7dbd69ff3821f24ffc58e1b7f2387bd4e9e9712cc4c67f661b1724ad99cdf09b3717794 -pkgname = ahriman +pkgname = ahriman \ No newline at end of file