arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-07-14 22:45:47 +00:00
Code Issues Projects Releases Wiki Activity

calculate dependencies based on package information (#89)

Browse Source
This commit is contained in:
Evgenii Alekseev
2023-01-30 16:28:05 +02:00
committed by GitHub
parent 34fe8128aa
commit c1718b3862
59 changed files with 970 additions and 856 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/ahriman/application/application/application_packages.py
View File

@ -21,7 +21,6 @@ import requests
import shutil
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any, Iterable, Set
from ahriman.application.application.application_properties import ApplicationProperties
@ -62,9 +61,7 @@ class ApplicationPackages(ApplicationProperties):
self.database.build_queue_insert(package)
self.database.remote_update(package)
with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (local_dir := Path(dir_name)):
Sources.load(local_dir, package, self.database.patches_get(package.base), self.repository.paths)
self._process_dependencies(local_dir, known_packages, without_dependencies)
self._process_dependencies(package, known_packages, without_dependencies)
def _add_directory(self, source: str, *_: Any) -> None:
"""
@ -94,7 +91,7 @@ class ApplicationPackages(ApplicationProperties):
self.database.build_queue_insert(package)
self._process_dependencies(cache_dir, known_packages, without_dependencies)
self._process_dependencies(package, known_packages, without_dependencies)
def _add_remote(self, source: str, *_: Any) -> None:
"""
@ -135,19 +132,19 @@ class ApplicationPackages(ApplicationProperties):
"""
raise NotImplementedError
def _process_dependencies(self, local_dir: Path, known_packages: Set[str], without_dependencies: bool) -> None:
def _process_dependencies(self, package: Package, known_packages: Set[str], without_dependencies: bool) -> None:
"""
process package dependencies
Args:
local_dir(Path): path to local package sources (i.e. cloned AUR repository)
package(Package): source package of which dependencies have to be processed
known_packages(Set[str]): list of packages which are known by the service
without_dependencies(bool): if set, dependency check will be disabled
"""
if without_dependencies:
return
dependencies = Package.dependencies(local_dir)
dependencies = package.depends_build
self.add(dependencies.difference(known_packages), PackageSource.AUR, without_dependencies)
def add(self, names: Iterable[str], source: PackageSource, without_dependencies: bool) -> None:

4
src/ahriman/application/application/application_repository.py
View File

@ -145,7 +145,7 @@ class ApplicationRepository(ApplicationProperties):
process_update(packages, build_result)
# process manual packages
tree = Tree.resolve(updates, self.repository.paths, self.database)
tree = Tree.resolve(updates)
for num, level in enumerate(tree):
self.logger.info("processing level #%i %s", num, [package.base for package in level])
build_result = self.repository.process_build(level)
@ -183,7 +183,7 @@ class ApplicationRepository(ApplicationProperties):
updated_packages = [package for _, package in sorted(updates.items())]
# reorder updates according to the dependency tree
tree = Tree.resolve(updated_packages, self.repository.paths, self.database)
tree = Tree.resolve(updated_packages)
for level in tree:
for package in level:
UpdatePrinter(package, local_versions.get(package.base)).print(

2
src/ahriman/application/handlers/remove_unknown.py
View File

@ -51,7 +51,7 @@ class RemoveUnknown(Handler):
if args.dry_run:
for package in sorted(unknown_packages):
StringPrinter(package).print(False)
StringPrinter(package).print(verbose=False)
return
application.remove(unknown_packages)

2
src/ahriman/application/handlers/search.py
View File

@ -64,7 +64,7 @@ class Search(Handler):
for packages_list in (official_packages_list, aur_packages_list):
# keep sorting by packages source
for package in Search.sort(packages_list, args.sort_by):
AurPrinter(package).print(args.info)
AurPrinter(package).print(verbose=args.info)
@staticmethod
def sort(packages: Iterable[AURPackage], sort_by: str) -> List[AURPackage]:

4
src/ahriman/application/handlers/status.py
View File

@ -53,7 +53,7 @@ class Status(Handler):
client = Application(architecture, configuration, report=True, unsafe=unsafe).repository.reporter
if args.ahriman:
service_status = client.get_internal()
StatusPrinter(service_status.status).print(args.info)
StatusPrinter(service_status.status).print(verbose=args.info)
if args.package:
packages: Iterable[Tuple[Package, BuildStatus]] = sum(
(client.get(base) for base in args.package),
@ -67,4 +67,4 @@ class Status(Handler):
filter_fn: Callable[[Tuple[Package, BuildStatus]], bool] =\
lambda item: args.status is None or item[1].status == args.status
for package, package_status in sorted(filter(filter_fn, packages), key=comparator):
PackagePrinter(package, package_status).print(args.info)
PackagePrinter(package, package_status).print(verbose=args.info)

2
src/ahriman/application/handlers/structure.py
View File

@ -51,6 +51,6 @@ class Structure(Handler):
application = Application(architecture, configuration, report=report, unsafe=unsafe)
packages = application.repository.packages()
tree = Tree.resolve(packages, application.repository.paths, application.database)
tree = Tree.resolve(packages)
for num, level in enumerate(tree):
TreePrinter(num, level).print(verbose=True, separator=" ")

47
src/ahriman/core/database/data/__init__.py
View File

@ -1,47 +0,0 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from ahriman.core.configuration import Configuration
from ahriman.core.database.data.package_remotes import migrate_package_remotes
from ahriman.core.database.data.package_statuses import migrate_package_statuses
from ahriman.core.database.data.patches import migrate_patches
from ahriman.core.database.data.users import migrate_users_data
from ahriman.models.migration_result import MigrationResult
def migrate_data(result: MigrationResult, connection: Connection, configuration: Configuration) -> None:
"""
perform data migration
Args:
result(MigrationResult): result of the schema migration
connection(Connection): database connection
configuration(Configuration): configuration instance
"""
# initial data migration
repository_paths = configuration.repository_paths
if result.old_version <= 0:
migrate_package_statuses(connection, repository_paths)
migrate_patches(connection, repository_paths)
migrate_users_data(connection, configuration)
if result.old_version <= 1:
migrate_package_remotes(connection, repository_paths)

64
src/ahriman/core/database/data/package_remotes.py
View File

@ -1,64 +0,0 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from ahriman.models.package_source import PackageSource
from ahriman.models.remote_source import RemoteSource
from ahriman.models.repository_paths import RepositoryPaths
__all__ = ["migrate_package_remotes"]
# pylint: disable=protected-access
def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> None:
"""
perform migration for package remote sources
Args:
connection(Connection): database connection
paths(RepositoryPaths): repository paths instance
"""
from ahriman.core.database.operations import PackageOperations
def insert_remote(base: str, remote: RemoteSource) -> None:
connection.execute(
"""
update package_bases set
branch = :branch, git_url = :git_url, path = :path,
web_url = :web_url, source = :source
where package_base = :package_base
""",
dict(
package_base=base,
branch=remote.branch, git_url=remote.git_url, path=remote.path,
web_url=remote.web_url, source=remote.source
)
)
packages = PackageOperations._packages_get_select_package_bases(connection)
for package_base, package in packages.items():
local_cache = paths.cache_for(package_base)
if local_cache.exists() and not package.is_vcs:
continue # skip packages which are not VCS and with local cache
remote_source = RemoteSource.from_source(PackageSource.AUR, package_base, "aur")
if remote_source is None:
continue # should never happen
insert_remote(package_base, remote_source)

82
src/ahriman/core/database/data/package_statuses.py
View File

@ -1,82 +0,0 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import json
from sqlite3 import Connection
from ahriman.models.build_status import BuildStatus
from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths
__all__ = ["migrate_package_statuses"]
def migrate_package_statuses(connection: Connection, paths: RepositoryPaths) -> None:
"""
perform migration for package statuses
Args:
connection(Connection): database connection
paths(RepositoryPaths): repository paths instance
"""
def insert_base(metadata: Package, last_status: BuildStatus) -> None:
connection.execute(
"""
insert into package_bases
(package_base, version, aur_url)
values
(:package_base, :version, :aur_url)
""",
dict(package_base=metadata.base, version=metadata.version, aur_url=""))
connection.execute(
"""
insert into package_statuses
(package_base, status, last_updated)
values
(:package_base, :status, :last_updated)""",
dict(package_base=metadata.base, status=last_status.status.value, last_updated=last_status.timestamp))
def insert_packages(metadata: Package) -> None:
package_list = []
for name, description in metadata.packages.items():
package_list.append(dict(package=name, package_base=metadata.base, **description.view()))
connection.executemany(
"""
insert into packages
(package, package_base, architecture, archive_size, build_date, depends, description,
filename, "groups", installed_size, licenses, provides, url)
values
(:package, :package_base, :architecture, :archive_size, :build_date, :depends, :description,
:filename, :groups, :installed_size, :licenses, :provides, :url)
""",
package_list)
cache_path = paths.root / "status_cache.json"
if not cache_path.is_file():
return # no file found
with cache_path.open() as cache:
dump = json.load(cache)
for item in dump.get("packages", []):
package = Package.from_json(item["package"])
status = BuildStatus.from_json(item["status"])
insert_base(package, status)
insert_packages(package)

47
src/ahriman/core/database/data/patches.py
View File

@ -1,47 +0,0 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from ahriman.models.repository_paths import RepositoryPaths
__all__ = ["migrate_patches"]
def migrate_patches(connection: Connection, paths: RepositoryPaths) -> None:
"""
perform migration for patches
Args:
connection(Connection): database connection
paths(RepositoryPaths): repository paths instance
"""
root = paths.root / "patches"
if not root.is_dir():
return # no directory found
for package in root.iterdir():
patch_path = package / "00-main.patch"
if not patch_path.is_file():
continue # not exist
content = patch_path.read_text(encoding="utf8")
connection.execute(
"""insert into patches (package_base, patch) values (:package_base, :patch)""",
{"package_base": package.name, "patch": content})

43
src/ahriman/core/database/data/users.py
View File

@ -1,43 +0,0 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from ahriman.core.configuration import Configuration
__all__ = ["migrate_users_data"]
def migrate_users_data(connection: Connection, configuration: Configuration) -> None:
"""
perform migration for users
Args:
connection(Connection): database connection
configuration(Configuration): configuration instance
"""
for section in configuration.sections():
for option, value in configuration[section].items():
if not section.startswith("auth:"):
continue
access = section[5:]
connection.execute(
"""insert into users (username, access, password) values (:username, :access, :password)""",
{"username": option.lower(), "access": access, "password": value})

54
src/ahriman/core/database/migrations/__init__.py
View File

@ -17,16 +17,13 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from __future__ import annotations
from importlib import import_module
from pathlib import Path
from pkgutil import iter_modules
from sqlite3 import Connection
from typing import List, Type
from sqlite3 import Connection, Cursor
from typing import Callable, List
from ahriman.core.configuration import Configuration
from ahriman.core.database.data import migrate_data
from ahriman.core.log import LazyLogging
from ahriman.models.migration import Migration
from ahriman.models.migration_result import MigrationResult
@ -53,8 +50,8 @@ class Migrations(LazyLogging):
self.connection = connection
self.configuration = configuration
@classmethod
def migrate(cls: Type[Migrations], connection: Connection, configuration: Configuration) -> MigrationResult:
@staticmethod
def migrate(connection: Connection, configuration: Configuration) -> MigrationResult:
"""
perform migrations implicitly
@ -65,7 +62,26 @@ class Migrations(LazyLogging):
Returns:
MigrationResult: current schema version
"""
return cls(connection, configuration).run()
return Migrations(connection, configuration).run()
def migration(self, cursor: Cursor, migration: Migration) -> None:
"""
perform single migration
Args:
cursor(Cursor): connection cursor
migration(Migration): single migration to perform
"""
self.logger.info("applying table migration %s at index %s", migration.name, migration.index)
for statement in migration.steps:
cursor.execute(statement)
self.logger.info("table migration %s at index %s has been applied", migration.name, migration.index)
self.logger.info("perform data migration %s at index %s", migration.name, migration.index)
migration.migrate_data(self.connection, self.configuration)
self.logger.info(
"data migration %s at index %s has been performed",
migration.name, migration.index)
def migrations(self) -> List[Migration]:
"""
@ -81,9 +97,21 @@ class Migrations(LazyLogging):
for index, module_name in enumerate(sorted(modules)):
module = import_module(f"{__name__}.{module_name}")
steps: List[str] = getattr(module, "steps", [])
self.logger.debug("found migration %s at index %s with steps count %s", module_name, index, len(steps))
migrations.append(Migration(index=index, name=module_name, steps=steps))
migrate_data: Callable[[Connection, Configuration], None] = \
getattr(module, "migrate_data", lambda *args: None)
migrations.append(
Migration(
index=index,
name=module_name,
steps=steps,
migrate_data=migrate_data
)
)
return migrations
@ -110,13 +138,7 @@ class Migrations(LazyLogging):
try:
cursor.execute("begin exclusive")
for migration in migrations[current_version:]:
self.logger.info("applying migration %s at index %s", migration.name, migration.index)
for statement in migration.steps:
cursor.execute(statement)
self.logger.info("migration %s at index %s has been applied", migration.name, migration.index)
migrate_data(result, self.connection, self.configuration)
self.migration(cursor, migration)
cursor.execute(f"pragma user_version = {expected_version}") # no support for ? placeholders
except Exception:
self.logger.exception("migration failed with exception")

118
src/ahriman/core/database/migrations/m000_initial.py
View File

@ -17,7 +17,17 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ["steps"]
import json
from sqlite3 import Connection
from ahriman.core.configuration import Configuration
from ahriman.models.build_status import BuildStatus
from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths
__all__ = ["migrate_data", "steps"]
steps = [
@ -73,3 +83,109 @@ steps = [
)
""",
]
def migrate_data(connection: Connection, configuration: Configuration) -> None:
"""
perform data migration
Args:
connection(Connection): database connection
configuration(Configuration): configuration instance
"""
migrate_package_statuses(connection, configuration.repository_paths)
migrate_patches(connection, configuration.repository_paths)
migrate_users_data(connection, configuration)
def migrate_package_statuses(connection: Connection, paths: RepositoryPaths) -> None:
"""
perform migration for package statuses
Args:
connection(Connection): database connection
paths(RepositoryPaths): repository paths instance
"""
def insert_base(metadata: Package, last_status: BuildStatus) -> None:
connection.execute(
"""
insert into package_bases
(package_base, version, aur_url)
values
(:package_base, :version, :aur_url)
""",
dict(package_base=metadata.base, version=metadata.version, aur_url=""))
connection.execute(
"""
insert into package_statuses
(package_base, status, last_updated)
values
(:package_base, :status, :last_updated)""",
dict(package_base=metadata.base, status=last_status.status.value, last_updated=last_status.timestamp))
def insert_packages(metadata: Package) -> None:
package_list = []
for name, description in metadata.packages.items():
package_list.append(dict(package=name, package_base=metadata.base, **description.view()))
connection.executemany(
"""
insert into packages
(package, package_base, architecture, archive_size, build_date, depends, description,
filename, "groups", installed_size, licenses, provides, url)
values
(:package, :package_base, :architecture, :archive_size, :build_date, :depends, :description,
:filename, :groups, :installed_size, :licenses, :provides, :url)
""",
package_list)
cache_path = paths.root / "status_cache.json"
if not cache_path.is_file():
return # no file found
with cache_path.open() as cache:
dump = json.load(cache)
for item in dump.get("packages", []):
package = Package.from_json(item["package"])
status = BuildStatus.from_json(item["status"])
insert_base(package, status)
insert_packages(package)
def migrate_patches(connection: Connection, paths: RepositoryPaths) -> None:
"""
perform migration for patches
Args:
connection(Connection): database connection
paths(RepositoryPaths): repository paths instance
"""
root = paths.root / "patches"
if not root.is_dir():
return # no directory found
for package in root.iterdir():
patch_path = package / "00-main.patch"
if not patch_path.is_file():
continue # not exist
content = patch_path.read_text(encoding="utf8")
connection.execute(
"""insert into patches (package_base, patch) values (:package_base, :patch)""",
{"package_base": package.name, "patch": content})
def migrate_users_data(connection: Connection, configuration: Configuration) -> None:
"""
perform migration for users
Args:
connection(Connection): database connection
configuration(Configuration): configuration instance
"""
for section in configuration.sections():
for option, value in configuration[section].items():
if not section.startswith("auth:"):
continue
access = section[5:]
connection.execute(
"""insert into users (username, access, password) values (:username, :access, :password)""",
{"username": option.lower(), "access": access, "password": value})

58
src/ahriman/core/database/migrations/m001_package_source.py
View File

@ -17,7 +17,15 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
__all__ = ["steps"]
from sqlite3 import Connection
from ahriman.core.configuration import Configuration
from ahriman.models.package_source import PackageSource
from ahriman.models.remote_source import RemoteSource
from ahriman.models.repository_paths import RepositoryPaths
__all__ = ["migrate_data", "steps"]
steps = [
@ -40,3 +48,51 @@ steps = [
alter table package_bases drop column aur_url
""",
]
def migrate_data(connection: Connection, configuration: Configuration) -> None:
"""
perform data migration
Args:
connection(Connection): database connection
configuration(Configuration): configuration instance
"""
migrate_package_remotes(connection, configuration.repository_paths)
# pylint: disable=protected-access
def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> None:
"""
perform migration for package remote sources
Args:
connection(Connection): database connection
paths(RepositoryPaths): repository paths instance
"""
from ahriman.core.database.operations import PackageOperations
def insert_remote(base: str, remote: RemoteSource) -> None:
connection.execute(
"""
update package_bases set
branch = :branch, git_url = :git_url, path = :path,
web_url = :web_url, source = :source
where package_base = :package_base
""",
dict(
package_base=base,
branch=remote.branch, git_url=remote.git_url, path=remote.path,
web_url=remote.web_url, source=remote.source
)
)
packages = PackageOperations._packages_get_select_package_bases(connection)
for package_base, package in packages.items():
local_cache = paths.cache_for(package_base)
if local_cache.exists() and not package.is_vcs:
continue # skip packages which are not VCS and with local cache
remote_source = RemoteSource.from_source(PackageSource.AUR, package_base, "aur")
if remote_source is None:
continue # should never happen
insert_remote(package_base, remote_source)

83
src/ahriman/core/database/migrations/m005_make_opt_depends.py Normal file
View File

@ -0,0 +1,83 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from ahriman.core.alpm.pacman import Pacman
from ahriman.core.configuration import Configuration
from ahriman.core.util import package_like
from ahriman.models.package import Package
__all__ = ["migrate_data", "steps"]
steps = [
"""
alter table packages add column make_depends json
""",
"""
alter table packages add column opt_depends json
""",
]
def migrate_data(connection: Connection, configuration: Configuration) -> None:
"""
perform data migration
Args:
connection(Connection): database connection
configuration(Configuration): configuration instance
"""
migrate_package_depends(connection, configuration)
def migrate_package_depends(connection: Connection, configuration: Configuration) -> None:
"""
migrate package opt and make depends fields
Args:
connection(Connection): database connection
configuration(Configuration): configuration instance
"""
if not configuration.repository_paths.repository.is_dir():
return
_, architecture = configuration.check_loaded()
pacman = Pacman(architecture, configuration, refresh_database=False)
package_list = []
for full_path in filter(package_like, configuration.repository_paths.repository.iterdir()):
base = Package.from_archive(full_path, pacman, remote=None)
for package, description in base.packages.items():
package_list.append({
"make_depends": description.make_depends,
"opt_depends": description.opt_depends,
"package": package,
})
connection.executemany(
"""
update packages set
make_depends = :make_depends, opt_depends = :opt_depends
where package = :package
""",
package_list
)

9
src/ahriman/core/database/operations/package_operations.py
View File

@ -110,15 +110,18 @@ class PackageOperations(Operations):
insert into packages
(package, package_base, architecture, archive_size,
build_date, depends, description, filename,
"groups", installed_size, licenses, provides, url)
"groups", installed_size, licenses, provides,
url, make_depends, opt_depends)
values
(:package, :package_base, :architecture, :archive_size,
:build_date, :depends, :description, :filename,
:groups, :installed_size, :licenses, :provides, :url)
:groups, :installed_size, :licenses, :provides,
:url, :make_depends, :opt_depends)
on conflict (package, architecture) do update set
package_base = :package_base, archive_size = :archive_size,
build_date = :build_date, depends = :depends, description = :description, filename = :filename,
"groups" = :groups, installed_size = :installed_size, licenses = :licenses, provides = :provides, url = :url
"groups" = :groups, installed_size = :installed_size, licenses = :licenses, provides = :provides,
url = :url, make_depends = :make_depends, opt_depends = :opt_depends
""",
package_list)

2
src/ahriman/core/formatters/printer.py
View File

@ -27,7 +27,7 @@ class Printer:
base class for formatters
"""
def print(self, verbose: bool, log_fn: Callable[[str], None] = print, separator: str = ": ") -> None:
def print(self, *, verbose: bool, log_fn: Callable[[str], None] = print, separator: str = ": ") -> None:
"""
print content

53
src/ahriman/core/tree.py
View File

@ -21,14 +21,9 @@ from __future__ import annotations
import itertools
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Callable, Iterable, List, Set, Tuple, Type
from typing import Callable, Iterable, List, Tuple
from ahriman.core.build_tools.sources import Sources
from ahriman.core.database import SQLite
from ahriman.models.package import Package
from ahriman.models.repository_paths import RepositoryPaths
class Leaf:
@ -40,16 +35,15 @@ class Leaf:
package(Package): leaf package properties
"""
def __init__(self, package: Package, dependencies: Set[str]) -> None:
def __init__(self, package: Package) -> None:
"""
default constructor
Args:
package(Package): package properties
dependencies(Set[str]): package dependencies
"""
self.package = package
self.dependencies = dependencies
self.dependencies = package.depends_build
@property
def items(self) -> Iterable[str]:
@ -61,24 +55,6 @@ class Leaf:
"""
return self.package.packages.keys()
@classmethod
def load(cls: Type[Leaf], package: Package, paths: RepositoryPaths, database: SQLite) -> Leaf:
"""
load leaf from package with dependencies
Args:
package(Package): package properties
paths(RepositoryPaths): repository paths instance
database(SQLite): database instance
Returns:
Leaf: loaded class
"""
with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (clone_dir := Path(dir_name)):
Sources.load(clone_dir, package, database.patches_get(package.base), paths)
dependencies = Package.dependencies(clone_dir)
return cls(package, dependencies)
def is_dependency(self, packages: Iterable[Leaf]) -> bool:
"""
check if the package is dependency of any other package from list or not
@ -130,7 +106,7 @@ class Tree:
>>> repository = Repository.load("x86_64", configuration, database, report=True, unsafe=False)
>>> packages = repository.packages()
>>>
>>> tree = Tree.resolve(packages, configuration.repository_paths, database)
>>> tree = Tree.resolve(packages)
>>> for tree_level in tree:
>>> for package in tree_level:
>>> print(package.base)
@ -138,14 +114,8 @@ class Tree:
The direct constructor call is also possible but requires tree leaves to be instantioned in advance, e.g.::
>>> leaves = [Leaf.load(package, database) for package in packages]
>>> leaves = [Leaf(package) for package in packages]
>>> tree = Tree(leaves)
Using the default ``Leaf()`` method is possible, but not really recommended because it requires from the user to
build the dependency list by himself::
>>> leaf = Leaf(package, dependecies)
>>> tree = Tree([leaf])
"""
def __init__(self, leaves: List[Leaf]) -> None:
@ -157,23 +127,20 @@ class Tree:
"""
self.leaves = leaves
@classmethod
def resolve(cls: Type[Tree], packages: Iterable[Package], paths: RepositoryPaths,
database: SQLite) -> List[List[Package]]:
@staticmethod
def resolve(packages: Iterable[Package]) -> List[List[Package]]:
"""
resolve dependency tree
Args:
packages(Iterable[Package]): packages list
paths(RepositoryPaths): repository paths instance
database(SQLite): database instance
Returns:
List[List[Package]]: list of packages lists based on their dependencies
"""
leaves = [Leaf.load(package, paths, database) for package in packages]
tree = cls(leaves)
return tree.levels()
leaves = [Leaf(package) for package in packages]
instance = Tree(leaves)
return instance.levels()
def levels(self) -> List[List[Package]]:
"""

19
src/ahriman/core/util.py
View File

@ -35,7 +35,7 @@ from ahriman.models.repository_paths import RepositoryPaths
__all__ = ["check_output", "check_user", "enum_values", "exception_response_text", "filter_json", "full_version",
"package_like", "pretty_datetime", "pretty_size", "safe_filename", "utcnow", "walk"]
"package_like", "pretty_datetime", "pretty_size", "safe_filename", "trim_package", "utcnow", "walk"]
def check_output(*args: str, exception: Optional[Exception] = None, cwd: Optional[Path] = None,
@ -295,6 +295,23 @@ def safe_filename(source: str) -> str:
return re.sub(r"[^A-Za-z\d\-._~:\[\]@]", "-", source)
def trim_package(package_name: str) -> str:
"""
remove version bound and description from package name. Pacman allows to specify version bound (=, <=, >= etc) for
packages in dependencies and also allows to specify description (via :); this function removes trailing parts and
return exact package name
Args:
package_name(str): source package name
Returns:
str: package name without description or version bound
"""
for symbol in ("<", "=", ">", ":"):
package_name = package_name.partition(symbol)[0]
return package_name
def utcnow() -> datetime.datetime:
"""
get current time

4
src/ahriman/models/aur_package.py
View File

@ -45,6 +45,7 @@ class AURPackage:
popularity(float): package popularity
out_of_date(Optional[datetime.datetime]): package out of date timestamp if any
maintainer(Optional[str]): package maintainer
submitter(Optional[str]): package first submitter
first_submitted(datetime.datetime): timestamp of the first package submission
last_modified(datetime.datetime): timestamp of the last package submission
url_path(str): AUR package path
@ -89,6 +90,7 @@ class AURPackage:
url: Optional[str] = None
out_of_date: Optional[datetime.datetime] = None
maintainer: Optional[str] = None
submitter: Optional[str] = None
repository: str = "aur"
depends: List[str] = field(default_factory=list)
make_depends: List[str] = field(default_factory=list)
@ -140,6 +142,7 @@ class AURPackage:
url=package.url,
out_of_date=None,
maintainer=None,
submitter=None,
repository=package.db.name,
depends=package.depends,
make_depends=package.makedepends,
@ -178,6 +181,7 @@ class AURPackage:
dump["flag_date"],
"%Y-%m-%dT%H:%M:%S.%fZ") if dump["flag_date"] is not None else None,
maintainer=next(iter(dump["maintainers"]), None),
submitter=None,
repository=dump["repo"],
depends=dump["depends"],
make_depends=dump["makedepends"],

3
src/ahriman/models/internal_status.py
View File

@ -57,7 +57,8 @@ class InternalStatus:
InternalStatus: internal status
"""
counters = Counters.from_json(dump["packages"]) if "packages" in dump else Counters(total=0)
return cls(status=BuildStatus.from_json(dump.get("status", {})),
build_status = dump.get("status") or {}
return cls(status=BuildStatus.from_json(build_status),
architecture=dump.get("architecture"),
packages=counters,
repository=dump.get("repository"),

7
src/ahriman/models/migration.py
View File

@ -18,7 +18,10 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass
from typing import List
from sqlite3 import Connection
from typing import Callable, List
from ahriman.core.configuration import Configuration
@dataclass(frozen=True, kw_only=True)
@ -30,8 +33,10 @@ class Migration:
index(int): migration position
name(str): migration name
steps(List[str]): migration steps
migrate_data(Callable[[Connection, Configuration], None]): data migration callback
"""
index: int
name: str
steps: List[str]
migrate_data: Callable[[Connection, Configuration], None]

88
src/ahriman/models/package.py
View File

@ -88,6 +88,36 @@ class Package(LazyLogging):
"""
return sorted(set(sum((package.depends for package in self.packages.values()), start=[])))
@property
def depends_build(self) -> Set[str]:
"""
get full list of external dependencies which has to be installed for build process
Returns:
Set[str]: full dependencies list used by devtools
"""
return (set(self.depends) | set(self.depends_make)) - self.packages.keys()
@property
def depends_make(self) -> List[str]:
"""
get package make dependencies
Returns:
List[str]: sum of make dependencies per each package
"""
return sorted(set(sum((package.make_depends for package in self.packages.values()), start=[])))
@property
def depends_opt(self) -> List[str]:
"""
get package optional dependencies
Returns:
List[str]: sum of optional dependencies per each package
"""
return sorted(set(sum((package.opt_depends for package in self.packages.values()), start=[])))
@property
def groups(self) -> List[str]:
"""
@ -168,7 +198,7 @@ class Package(LazyLogging):
base=package.package_base,
version=package.version,
remote=remote,
packages={package.name: PackageDescription()})
packages={package.name: PackageDescription.from_aur(package)})
@classmethod
def from_build(cls: Type[Package], path: Path) -> Package:
@ -188,7 +218,18 @@ class Package(LazyLogging):
srcinfo, errors = parse_srcinfo(srcinfo_source)
if errors:
raise PackageInfoError(errors)
packages = {key: PackageDescription() for key in srcinfo["packages"]}
def get_property(key: str, properties: Dict[str, Any], default: Any) -> Any:
return properties.get(key, srcinfo.get(key, default))
packages = {
package: PackageDescription(
depends=get_property("depends", properties, []),
make_depends=get_property("makedepends", properties, []),
opt_depends=get_property("optdepends", properties, []),
)
for package, properties in srcinfo["packages"].items()
}
version = full_version(srcinfo.get("epoch"), srcinfo["pkgver"], srcinfo["pkgrel"])
return cls(base=srcinfo["pkgbase"], version=version, remote=None, packages=packages)
@ -204,11 +245,12 @@ class Package(LazyLogging):
Returns:
Package: package properties
"""
packages_json = dump.get("packages") or {}
packages = {
key: PackageDescription.from_json(value)
for key, value in dump.get("packages", {}).items()
for key, value in packages_json.items()
}
remote = dump.get("remote", {})
remote = dump.get("remote") or {}
return cls(base=dump["base"], version=dump["version"], remote=RemoteSource.from_json(remote), packages=packages)
@classmethod
@ -230,43 +272,7 @@ class Package(LazyLogging):
base=package.package_base,
version=package.version,
remote=remote,
packages={package.name: PackageDescription()})
@staticmethod
def dependencies(path: Path) -> Set[str]:
"""
load dependencies from package sources
Args:
path(Path): path to package sources directory
Returns:
Set[str]: list of package dependencies including makedepends array, but excluding packages from this base
Raises:
InvalidPackageInfo: if there are parsing errors
"""
# additional function to remove versions from dependencies
def extract_packages(raw_packages_list: List[str]) -> Set[str]:
return {trim_version(package_name) for package_name in raw_packages_list}
def trim_version(package_name: str) -> str:
for symbol in ("<", "=", ">"):
package_name = package_name.split(symbol)[0]
return package_name
srcinfo_source = Package._check_output("makepkg", "--printsrcinfo", cwd=path)
srcinfo, errors = parse_srcinfo(srcinfo_source)
if errors:
raise PackageInfoError(errors)
makedepends = extract_packages(srcinfo.get("makedepends", []))
# sum over each package
depends = extract_packages(srcinfo.get("depends", []))
for package in srcinfo["packages"].values():
depends |= extract_packages(package.get("depends", []))
# we are not interested in dependencies inside pkgbase
packages = set(srcinfo["packages"].keys())
return (depends | makedepends) - packages
packages={package.name: PackageDescription.from_aur(package)})
@staticmethod
def supported_architectures(path: Path) -> Set[str]:

41
src/ahriman/models/package_description.py
View File

@ -24,7 +24,8 @@ from pathlib import Path
from pyalpm import Package # type: ignore
from typing import Any, Dict, List, Optional, Type
from ahriman.core.util import filter_json
from ahriman.core.util import filter_json, trim_package
from ahriman.models.aur_package import AURPackage
@dataclass(kw_only=True)
@ -37,6 +38,8 @@ class PackageDescription:
archive_size(Optional[int]): package archive size
build_date(Optional[int]): package build date
depends(List[str]): package dependencies list
opt_depends(List[str]): optional package dependencies list
make_depends(List[str]): package dependencies list used for building
description(Optional[str]): package description
filename(Optional[str]): package archive name
groups(List[str]): package groups
@ -67,6 +70,8 @@ class PackageDescription:
archive_size: Optional[int] = None
build_date: Optional[int] = None
depends: List[str] = field(default_factory=list)
make_depends: List[str] = field(default_factory=list)
opt_depends: List[str] = field(default_factory=list)
description: Optional[str] = None
filename: Optional[str] = None
groups: List[str] = field(default_factory=list)
@ -75,6 +80,14 @@ class PackageDescription:
provides: List[str] = field(default_factory=list)
url: Optional[str] = None
def __post_init__(self) -> None:
"""
update dependencies list accordingly
"""
self.depends = [trim_package(package) for package in self.depends]
self.opt_depends = [trim_package(package) for package in self.opt_depends]
self.make_depends = [trim_package(package) for package in self.make_depends]
@property
def filepath(self) -> Optional[Path]:
"""
@ -85,6 +98,27 @@ class PackageDescription:
"""
return Path(self.filename) if self.filename is not None else None
@classmethod
def from_aur(cls: Type[PackageDescription], package: AURPackage) -> PackageDescription:
"""
construct properties from AUR package model
Args:
package(AURPackage): AUR package model
Returns:
PackageDescription: package properties based on source AUR package
"""
return cls(
depends=package.depends,
make_depends=package.make_depends,
opt_depends=package.opt_depends,
description=package.description,
licenses=package.license,
provides=package.provides,
url=package.url,
)
@classmethod
def from_json(cls: Type[PackageDescription], dump: Dict[str, Any]) -> PackageDescription:
"""
@ -117,13 +151,16 @@ class PackageDescription:
archive_size=package.size,
build_date=package.builddate,
depends=package.depends,
make_depends=package.makedepends,
opt_depends=package.optdepends,
description=package.desc,
filename=path.name,
groups=package.groups,
installed_size=package.isize,
licenses=package.licenses,
provides=package.provides,
url=package.url)
url=package.url,
)
def view(self) -> Dict[str, Any]:
"""