Compare commits

...

2 Commits

Author SHA1 Message Date
cd1b2b171c implement local reporter mode 2024-05-10 17:47:00 +03:00
cd4516d6e8 feat: allow to use simplified keys for context
Initial implementation requires explicit context key name to be set.
Though it is still useful sometimes (e.g. if there should be two
variables with the same type), in the most used scenarios internally
only type is required. This commit extends set and get methods to allow
to construct ContextKey from type directly

Also it breaks old keys, since - in order to reduce amount of possible
mistakes - internal classes uses this generation method
2024-05-10 17:31:54 +03:00
46 changed files with 1059 additions and 216 deletions

View File

@ -446,7 +446,7 @@ def _set_patch_list_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
parser = root.add_parser("patch-list", help="list patch sets",
description="list available patches for the package", formatter_class=_formatter)
parser.add_argument("package", help="package base", nargs="?")
parser.add_argument("package", help="package base")
parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
parser.add_argument("-v", "--variable", help="if set, show only patches for specified PKGBUILD variables",
action="append")

View File

@ -161,8 +161,7 @@ class Application(ApplicationPackages, ApplicationRepository):
package = Package.from_aur(package_name, username)
with_dependencies[package.base] = package
# register package in local database
self.database.package_base_update(package)
# register package in the database
self.repository.reporter.set_unknown(package)
return list(with_dependencies.values())

View File

@ -65,7 +65,7 @@ class ApplicationPackages(ApplicationProperties):
"""
package = Package.from_aur(source, username)
self.database.build_queue_insert(package)
self.database.package_base_update(package)
self.reporter.set_unknown(package)
def _add_directory(self, source: str, *_: Any) -> None:
"""
@ -139,7 +139,7 @@ class ApplicationPackages(ApplicationProperties):
"""
package = Package.from_official(source, self.repository.pacman, username)
self.database.build_queue_insert(package)
self.database.package_base_update(package)
self.reporter.set_unknown(package)
def add(self, names: Iterable[str], source: PackageSource, username: str | None = None) -> None:
"""

View File

@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.log import LazyLogging
from ahriman.core.repository import Repository
from ahriman.core.status.client import Client
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.repository_id import RepositoryId
@ -63,3 +64,13 @@ class ApplicationProperties(LazyLogging):
str: repository architecture
"""
return self.repository_id.architecture
@property
def reporter(self) -> Client:
"""
instance of the web/database client
Returns:
Client: repository reposter
"""
return self.repository.reporter

View File

@ -39,10 +39,8 @@ class ApplicationRepository(ApplicationProperties):
Args:
packages(Iterable[Package]): list of packages to retrieve changes
"""
last_commit_hashes = self.database.hashes_get()
for package in packages:
last_commit_sha = last_commit_hashes.get(package.base)
last_commit_sha = self.reporter.package_changes_get(package.base).last_commit_sha
if last_commit_sha is None:
continue # skip check in case if we can't calculate diff

View File

@ -50,7 +50,8 @@ class Add(Handler):
application.add(args.package, args.source, args.username)
patches = [PkgbuildPatch.from_env(patch) for patch in args.variable] if args.variable is not None else []
for package in args.package: # for each requested package insert patch
application.database.patches_insert(package, patches)
for patch in patches:
application.reporter.package_patches_add(package, patch)
if not args.now:
return

View File

@ -116,25 +116,29 @@ class Patch(Handler):
package_base(str): package base
patch(PkgbuildPatch): patch descriptor
"""
application.database.patches_insert(package_base, [patch])
application.reporter.package_patches_add(package_base, patch)
@staticmethod
def patch_set_list(application: Application, package_base: str | None, variables: list[str] | None,
def patch_set_list(application: Application, package_base: str, variables: list[str] | None,
exit_code: bool) -> None:
"""
list patches available for the package base
Args:
application(Application): application instance
package_base(str | None): package base
package_base(str): package base
variables(list[str] | None): extract patches only for specified PKGBUILD variables
exit_code(bool): exit with error on empty search result
"""
patches = application.database.patches_list(package_base, variables)
patches = []
if variables is not None:
for variable in variables:
patches.extend(application.reporter.package_patches_get(package_base, variable))
else:
patches = application.reporter.package_patches_get(package_base, variables)
Patch.check_if_empty(exit_code, not patches)
for base, patch in patches.items():
PatchPrinter(base, patch)(verbose=True, separator=" = ")
PatchPrinter(package_base, patches)(verbose=True, separator=" = ")
@staticmethod
def patch_set_remove(application: Application, package_base: str, variables: list[str] | None) -> None:
@ -146,4 +150,8 @@ class Patch(Handler):
package_base(str): package base
variables(list[str] | None): remove patches only for specified PKGBUILD variables
"""
application.database.patches_remove(package_base, variables)
if variables is not None:
for variable in variables: # iterate over single variable
application.reporter.package_patches_remove(package_base, variable)
else:
application.reporter.package_patches_remove(package_base, variables) # just pass as is

View File

@ -76,7 +76,7 @@ class Rebuild(Handler):
if from_database:
return [
package
for (package, last_status) in application.database.packages_get()
for (package, last_status) in application.reporter.package_get(None)
if status is None or last_status.status == status
]

View File

@ -56,7 +56,7 @@ class StatusUpdate(Handler):
if (local := next((package for package in packages if package.base == base), None)) is not None:
client.package_add(local, args.status)
else:
client.package_update(base, args.status)
client.package_set(base, args.status)
case Action.Update:
# update service status
client.status_update(args.status)

View File

@ -38,12 +38,12 @@ class _Context:
"""
self._content: dict[str, Any] = {}
def get(self, key: ContextKey[T]) -> T:
def get(self, key: ContextKey[T] | type[T]) -> T:
"""
get value for the specified key
Args:
key(ContextKey[T]): context key name
key(ContextKey[T] | type[T]): context key name
Returns:
T: value associated with the key
@ -52,29 +52,37 @@ class _Context:
KeyError: in case if the specified context variable was not found
ValueError: in case if type of value is not an instance of specified return type
"""
if not isinstance(key, ContextKey):
key = ContextKey.from_type(key)
if key.key not in self._content:
raise KeyError(key.key)
value = self._content[key.key]
if not isinstance(value, key.return_type):
raise ValueError(f"Value {value} is not an instance of {key.return_type}")
return value
def set(self, key: ContextKey[T], value: T) -> None:
def set(self, key: ContextKey[T] | type[T], value: T) -> None:
"""
set value for the specified key
Args:
key(ContextKey[T]): context key name
key(ContextKey[T] | type[T]): context key name
value(T): context value associated with the specified key
Raises:
KeyError: in case if the specified context variable already exists
ValueError: in case if type of value is not an instance of specified return type
"""
if not isinstance(key, ContextKey):
key = ContextKey.from_type(key)
if key.key in self._content:
raise KeyError(key.key)
if not isinstance(value, key.return_type):
raise ValueError(f"Value {value} is not an instance of {key.return_type}")
self._content[key.key] = value
def __iter__(self) -> Iterator[str]:

View File

@ -21,7 +21,6 @@ from pathlib import Path
from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.exceptions import BuildError
from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output
@ -116,20 +115,20 @@ class Task(LazyLogging):
# e.g. in some cases packagelist command produces debug packages which were not actually built
return list(filter(lambda path: path.is_file(), map(Path, packages)))
def init(self, sources_dir: Path, database: SQLite, local_version: str | None) -> str | None:
def init(self, sources_dir: Path, patches: list[PkgbuildPatch], local_version: str | None) -> str | None:
"""
fetch package from git
Args:
sources_dir(Path): local path to fetch
database(SQLite): database instance
patches(list[PkgbuildPatch]): list of patches for the package
local_version(str | None): local version of the package. If set and equal to current version, it will
automatically bump pkgrel
Returns:
str | None: current commit sha if available
"""
last_commit_sha = Sources.load(sources_dir, self.package, database.patches_get(self.package.base), self.paths)
last_commit_sha = Sources.load(sources_dir, self.package, patches, self.paths)
if local_version is None:
return last_commit_sha # there is no local package or pkgrel increment is disabled

View File

@ -117,27 +117,3 @@ class ChangesOperations(Operations):
})
return self.with_connection(run, commit=True)
def hashes_get(self, repository_id: RepositoryId | None = None) -> dict[str, str]:
"""
extract last commit hashes if available
Args:
repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
Returns:
dict[str, str]: map of package base to its last commit hash
"""
repository_id = repository_id or self._repository_id
def run(connection: Connection) -> dict[str, str]:
return {
row["package_base"]: row["last_commit_sha"]
for row in connection.execute(
"""select package_base, last_commit_sha from package_changes where repository = :repository""",
{"repository": repository_id.id}
)
}
return self.with_connection(run)

View File

@ -25,7 +25,7 @@ from typing import Any, TypeVar
from ahriman.core.log import LazyLogging
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
T = TypeVar("T")
@ -38,7 +38,7 @@ class Operations(LazyLogging):
path(Path): path to the database file
"""
def __init__(self, path: Path, repository_id: RepositoryId) -> None:
def __init__(self, path: Path, repository_id: RepositoryId, repository_paths: RepositoryPaths) -> None:
"""
default constructor
@ -48,6 +48,7 @@ class Operations(LazyLogging):
"""
self.path = path
self._repository_id = repository_id
self._repository_paths = repository_paths
@staticmethod
def factory(cursor: sqlite3.Cursor, row: tuple[Any, ...]) -> dict[str, Any]:

View File

@ -150,34 +150,6 @@ class PackageOperations(Operations):
""",
package_list)
@staticmethod
def _package_update_insert_status(connection: Connection, package_base: str, status: BuildStatus,
repository_id: RepositoryId) -> None:
"""
insert base package status into table
Args:
connection(Connection): database connection
package_base(str): package base name
status(BuildStatus): new build status
repository_id(RepositoryId): repository unique identifier
"""
connection.execute(
"""
insert into package_statuses
(package_base, status, last_updated, repository)
values
(:package_base, :status, :last_updated, :repository)
on conflict (package_base, repository) do update set
status = :status, last_updated = :last_updated
""",
{
"package_base": package_base,
"status": status.status.value,
"last_updated": status.timestamp,
"repository": repository_id.id,
})
@staticmethod
def _packages_get_select_package_bases(connection: Connection, repository_id: RepositoryId) -> dict[str, Package]:
"""
@ -277,20 +249,18 @@ class PackageOperations(Operations):
return self.with_connection(run, commit=True)
def package_update(self, package: Package, status: BuildStatus, repository_id: RepositoryId | None = None) -> None:
def package_update(self, package: Package, repository_id: RepositoryId | None = None) -> None:
"""
update package status
Args:
package(Package): package properties
status(BuildStatus): new build status
repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
"""
repository_id = repository_id or self._repository_id
def run(connection: Connection) -> None:
self._package_update_insert_base(connection, package, repository_id)
self._package_update_insert_status(connection, package.base, status, repository_id)
self._package_update_insert_packages(connection, package, repository_id)
self._package_remove_packages(connection, package.base, package.packages.keys(), repository_id)
@ -336,3 +306,33 @@ class PackageOperations(Operations):
package_base: package.remote
for package_base, package in self.with_connection(run).items()
}
def status_update(self, package_base: str, status: BuildStatus, repository_id: RepositoryId | None = None) -> None:
"""
insert base package status into table
Args:
package_base(str): package base name
status(BuildStatus): new build status
repository_id(RepositoryId, optional): repository unique identifier override (Default value = None)
"""
repository_id = repository_id or self._repository_id
def run(connection: Connection) -> None:
connection.execute(
"""
insert into package_statuses
(package_base, status, last_updated, repository)
values
(:package_base, :status, :last_updated, :repository)
on conflict (package_base, repository) do update set
status = :status, last_updated = :last_updated
""",
{
"package_base": package_base,
"status": status.status.value,
"last_updated": status.timestamp,
"repository": repository_id.id,
})
return self.with_connection(run, commit=True)

View File

@ -66,7 +66,7 @@ class SQLite(
path = cls.database_path(configuration)
_, repository_id = configuration.check_loaded()
database = cls(path, repository_id)
database = cls(path, repository_id, configuration.repository_paths)
database.init(configuration)
return database
@ -119,3 +119,6 @@ class SQLite(
self.logs_remove(package_base, None)
self.changes_remove(package_base)
self.dependencies_remove(package_base)
# remove local cache too
self._repository_paths.tree_clear(package_base)

View File

@ -25,9 +25,9 @@ from tempfile import TemporaryDirectory
from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.exceptions import GitRemoteError
from ahriman.core.log import LazyLogging
from ahriman.core.status.client import Client
from ahriman.models.package import Package
from ahriman.models.package_source import PackageSource
from ahriman.models.remote_source import RemoteSource
@ -40,20 +40,20 @@ class RemotePush(LazyLogging):
Attributes:
commit_author(tuple[str, str] | None): optional commit author in form of git config
database(SQLite): database instance
remote_source(RemoteSource): repository remote source (remote pull url and branch)
reporter(Client): reporter client used for additional information retrieval
"""
def __init__(self, database: SQLite, configuration: Configuration, section: str) -> None:
def __init__(self, reporter: Client, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
database(SQLite): database instance
reporter(Client): reporter client
configuration(Configuration): configuration instance
section(str): settings section name
"""
self.database = database
self.reporter = reporter
commit_email = configuration.get(section, "commit_email", fallback="ahriman@localhost")
commit_user = configuration.get(section, "commit_user", fallback="ahriman")
@ -92,7 +92,7 @@ class RemotePush(LazyLogging):
else:
shutil.rmtree(git_file)
# ...copy all patches...
for patch in self.database.patches_get(package.base):
for patch in self.reporter.package_patches_get(package.base, None):
filename = f"ahriman-{package.base}.patch" if patch.key is None else f"ahriman-{patch.key}.patch"
patch.write(package_target_dir / filename)
# ...and finally return path to the copied directory

View File

@ -19,10 +19,9 @@
#
from ahriman.core import context
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.gitremote.remote_push import RemotePush
from ahriman.core.status.client import Client
from ahriman.core.triggers import Trigger
from ahriman.models.context_key import ContextKey
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -111,10 +110,10 @@ class RemotePushTrigger(Trigger):
GitRemoteError: if database is not set in context
"""
ctx = context.get()
database = ctx.get(ContextKey("database", SQLite))
reporter = ctx.get(Client)
for target in self.targets:
section, _ = self.configuration.gettype(
target, self.repository_id, fallback=self.CONFIGURATION_SCHEMA_FALLBACK)
runner = RemotePush(database, self.configuration, section)
runner = RemotePush(reporter, self.configuration, section)
runner.run(result)

View File

@ -92,7 +92,7 @@ class HttpLogHandler(logging.Handler):
return # in case if no package base supplied we need just skip log message
try:
self.reporter.package_logs(log_record_id, record)
self.reporter.package_logs_add(log_record_id, record.created, record.getMessage())
except Exception:
if self.suppress_errors:
return

View File

@ -58,7 +58,8 @@ class Executor(PackageInfo, Cleaner):
self.reporter.set_building(package.base)
task = Task(package, self.configuration, self.architecture, self.paths)
local_version = local_versions.get(package.base) if bump_pkgrel else None
commit_sha = task.init(local_path, self.database, local_version)
patches = self.reporter.package_patches_get(package.base, None)
commit_sha = task.init(local_path, patches, local_version)
built = task.build(local_path, PACKAGER=packager_id)
for src in built:
dst = self.paths.packages / src.name
@ -80,7 +81,7 @@ class Executor(PackageInfo, Cleaner):
self.reporter.package_changes_set(single.base, Changes(last_commit_sha))
# update dependencies list
dependencies = PackageArchive(self.paths.build_directory, single).depends_on()
self.database.dependencies_insert(dependencies)
self.reporter.package_dependencies_set(dependencies)
# update result set
result.add_updated(single)
except Exception:
@ -102,9 +103,7 @@ class Executor(PackageInfo, Cleaner):
"""
def remove_base(package_base: str) -> None:
try:
self.paths.tree_clear(package_base) # remove all internal files
self.database.package_clear(package_base)
self.reporter.package_remove(package_base) # we only update status page in case of base removal
self.reporter.package_remove(package_base)
except Exception:
self.logger.exception("could not remove base %s", package_base)

View File

@ -43,15 +43,14 @@ class PackageInfo(RepositoryProperties):
Returns:
list[Package]: list of read packages
"""
sources = self.database.remotes_get()
result: dict[str, Package] = {}
# we are iterating over bases, not single packages
for full_path in packages:
try:
local = Package.from_archive(full_path, self.pacman)
if (source := sources.get(local.base)) is not None:
local.remote = source
remote, _ = next(iter(self.reporter.package_get(local.base)), (None, None))
if remote is not None: # update source with remote
local.remote = remote.remote
current = result.setdefault(local.base, local)
if current.version != local.version:
@ -78,7 +77,8 @@ class PackageInfo(RepositoryProperties):
"""
with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name:
dir_path = Path(dir_name)
current_commit_sha = Sources.load(dir_path, package, self.database.patches_get(package.base), self.paths)
patches = self.reporter.package_patches_get(package.base, None)
current_commit_sha = Sources.load(dir_path, package, patches, self.paths)
changes: str | None = None
if current_commit_sha != last_commit_sha:

View File

@ -26,7 +26,7 @@ from ahriman.core.database import SQLite
from ahriman.core.repository.executor import Executor
from ahriman.core.repository.update_handler import UpdateHandler
from ahriman.core.sign.gpg import GPG
from ahriman.models.context_key import ContextKey
from ahriman.core.status.client import Client
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.repository_id import RepositoryId
@ -89,11 +89,12 @@ class Repository(Executor, UpdateHandler):
# directly without loader
ctx = _Context()
ctx.set(ContextKey("database", SQLite), self.database)
ctx.set(ContextKey("configuration", Configuration), self.configuration)
ctx.set(ContextKey("pacman", Pacman), self.pacman)
ctx.set(ContextKey("sign", GPG), self.sign)
ctx.set(SQLite, self.database)
ctx.set(Configuration, self.configuration)
ctx.set(Pacman, self.pacman)
ctx.set(GPG, self.sign)
ctx.set(Client, self.reporter)
ctx.set(ContextKey("repository", type(self)), self)
ctx.set(type(self), self)
context.set(ctx)

View File

@ -75,7 +75,7 @@ class RepositoryProperties(LazyLogging):
self.pacman = Pacman(repository_id, configuration, refresh_database=refresh_pacman_database)
self.sign = GPG(configuration)
self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
self.reporter = Client.load(repository_id, configuration, report=report)
self.reporter = Client.load(repository_id, configuration, database, report=report)
self.triggers = TriggerLoader.load(repository_id, configuration)
@property

View File

@ -98,18 +98,16 @@ class UpdateHandler(PackageInfo, Cleaner):
return files
dependencies = {dependency.package_base: dependency for dependency in self.database.dependencies_get()}
result: list[Package] = []
for package in self.packages(filter_packages):
if package.base not in dependencies:
dependencies = next(iter(self.reporter.package_dependencies_get(package.base)), None)
if dependencies is None:
continue # skip check if no package dependencies found
required = dependencies[package.base].paths
required_packages = {dep for dep_packages in required.values() for dep in dep_packages}
required_packages = {dep for dep_packages in dependencies.paths.values() for dep in dep_packages}
filesystem = extract_files(required_packages)
for path, packages in required.items():
for path, packages in dependencies.paths.items():
found = filesystem.get(path, set())
if found.intersection(packages):
continue

View File

@ -17,16 +17,18 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# pylint: disable=too-many-public-methods
from __future__ import annotations
import logging
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.changes import Changes
from ahriman.models.dependencies import Dependencies
from ahriman.models.internal_status import InternalStatus
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_id import RepositoryId
@ -36,22 +38,31 @@ class Client:
"""
@staticmethod
def load(repository_id: RepositoryId, configuration: Configuration, *, report: bool) -> Client:
def load(repository_id: RepositoryId, configuration: Configuration, database: SQLite | None = None, *,
report: bool = True) -> Client:
"""
load client from settings
Args:
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
database(SQLite | None, optional): database instance (Default value = None)
report(bool, optional): force enable or disable reporting (Default value = True)
Returns:
Client: client according to current settings
"""
def make_local_client() -> Client:
if database is None:
return Client()
from ahriman.core.status.local_client import LocalClient
return LocalClient(repository_id, database)
if not report:
return Client()
return make_local_client()
if not configuration.getboolean("status", "enabled", fallback=True): # global switch
return Client()
return make_local_client()
# new-style section
address = configuration.get("status", "address", fallback=None)
@ -65,7 +76,8 @@ class Client:
if address or legacy_address or (host and port) or socket:
from ahriman.core.status.web_client import WebClient
return WebClient(repository_id, configuration)
return Client()
return make_local_client()
def package_add(self, package: Package, status: BuildStatusEnum) -> None:
"""
@ -74,7 +86,11 @@ class Client:
Args:
package(Package): package properties
status(BuildStatusEnum): current package build status
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
def package_changes_get(self, package_base: str) -> Changes:
"""
@ -85,9 +101,11 @@ class Client:
Returns:
Changes: package changes if available and empty object otherwise
Raises:
NotImplementedError: not implemented method
"""
del package_base
return Changes()
raise NotImplementedError
def package_changes_set(self, package_base: str, changes: Changes) -> None:
"""
@ -96,7 +114,38 @@ class Client:
Args:
package_base(str): package base to update
changes(Changes): changes descriptor
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
def package_dependencies_get(self, package_base: str | None) -> list[Dependencies]:
"""
get package dependencies
Args:
package_base(str | None): package base to retrieve
Returns:
list[Dependencies]: package implicit dependencies if available
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
def package_dependencies_set(self, dependencies: Dependencies) -> None:
"""
update package dependencies
Args:
dependencies(Dependencies): dependencies descriptor
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
def package_get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]:
"""
@ -107,35 +156,118 @@ class Client:
Returns:
list[tuple[Package, BuildStatus]]: list of current package description and status if it has been found
"""
del package_base
return []
def package_logs(self, log_record_id: LogRecordId, record: logging.LogRecord) -> None:
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None:
"""
post log record
Args:
log_record_id(LogRecordId): log record id
record(logging.LogRecord): log record to post to api
created(float): log created timestamp
message(str): log message
"""
def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]:
"""
get package logs
Args:
package_base(str): package base
limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1)
offset(int, optional): records offset (Default value = 0)
Returns:
list[tuple[float, str]]: package logs
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
def package_logs_remove(self, package_base: str, version: str | None) -> None:
"""
remove package logs
Args:
package_base(str): package base
version(str | None): package version to remove logs. If None set, all logs will be removed
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
def package_patches_add(self, package_base: str, patch: PkgbuildPatch) -> None:
"""
create or update package patch
Args:
package_base(str): package base to update
patch(PkgbuildPatch): package patch
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
def package_patches_get(self, package_base: str, variable: str | None) -> list[PkgbuildPatch]:
"""
get package patches
Args:
package_base(str): package base to retrieve
variable(str | None): optional filter by patch variable
Returns:
list[PkgbuildPatch]: list of patches for the specified package
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
def package_patches_remove(self, package_base: str, variable: str | None) -> None:
"""
remove package patch
Args:
package_base(str): package base to update
variable(str | None): patch name. If None set, all patches will be removed
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
def package_remove(self, package_base: str) -> None:
"""
remove packages from watcher
Args:
package_base(str): package base to remove
"""
def package_update(self, package_base: str, status: BuildStatusEnum) -> None:
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
def package_set(self, package_base: str, status: BuildStatusEnum) -> None:
"""
update package build status. Unlike :func:`package_add()` it does not update package properties
Args:
package_base(str): package base to update
status(BuildStatusEnum): current package build status
Raises:
NotImplementedError: not implemented method
"""
raise NotImplementedError
def set_building(self, package_base: str) -> None:
"""
@ -144,7 +276,7 @@ class Client:
Args:
package_base(str): package base to update
"""
return self.package_update(package_base, BuildStatusEnum.Building)
return self.package_set(package_base, BuildStatusEnum.Building)
def set_failed(self, package_base: str) -> None:
"""
@ -153,7 +285,7 @@ class Client:
Args:
package_base(str): package base to update
"""
return self.package_update(package_base, BuildStatusEnum.Failed)
return self.package_set(package_base, BuildStatusEnum.Failed)
def set_pending(self, package_base: str) -> None:
"""
@ -162,7 +294,7 @@ class Client:
Args:
package_base(str): package base to update
"""
return self.package_update(package_base, BuildStatusEnum.Pending)
return self.package_set(package_base, BuildStatusEnum.Pending)
def set_success(self, package: Package) -> None:
"""

View File

@ -0,0 +1,207 @@
#
# Copyright (c) 2021-2024 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.database import SQLite
from ahriman.core.status.client import Client
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.changes import Changes
from ahriman.models.dependencies import Dependencies
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_id import RepositoryId
class LocalClient(Client):
"""
local database handler
Attributes:
database(SQLite): database instance
repository_id(RepositoryId): repository unique identifier
"""
def __init__(self, repository_id: RepositoryId, database: SQLite) -> None:
"""
default constructor
Args:
repository_id(RepositoryId): repository unique identifier
database(SQLite): database instance:
"""
self.database = database
self.repository_id = repository_id
def package_add(self, package: Package, status: BuildStatusEnum) -> None:
"""
add new package with status
Args:
package(Package): package properties
status(BuildStatusEnum): current package build status
"""
self.database.package_update(package, self.repository_id)
self.database.status_update(package.base, BuildStatus(status), self.repository_id)
def package_changes_get(self, package_base: str) -> Changes:
"""
get package changes
Args:
package_base(str): package base to retrieve
Returns:
Changes: package changes if available and empty object otherwise
"""
return self.database.changes_get(package_base, self.repository_id)
def package_changes_set(self, package_base: str, changes: Changes) -> None:
"""
update package changes
Args:
package_base(str): package base to update
changes(Changes): changes descriptor
"""
self.database.changes_insert(package_base, changes, self.repository_id)
def package_dependencies_get(self, package_base: str | None) -> list[Dependencies]:
"""
get package dependencies
Args:
package_base(str | None): package base to retrieve
Returns:
list[Dependencies]: package implicit dependencies if available
"""
return self.database.dependencies_get(package_base, self.repository_id)
def package_dependencies_set(self, dependencies: Dependencies) -> None:
"""
update package dependencies
Args:
dependencies(Dependencies): dependencies descriptor
"""
self.database.dependencies_insert(dependencies, self.repository_id)
def package_get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]:
"""
get package status
Args:
package_base(str | None): package base to get
Returns:
list[tuple[Package, BuildStatus]]: list of current package description and status if it has been found
"""
packages = self.database.packages_get(self.repository_id)
if package_base is None:
return packages
return [(package, status) for package, status in packages if package.base == package_base]
def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None:
"""
post log record
Args:
log_record_id(LogRecordId): log record id
created(float): log created timestamp
message(str): log message
"""
self.database.logs_insert(log_record_id, created, message, self.repository_id)
def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]:
"""
get package logs
Args:
package_base(str): package base
limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1)
offset(int, optional): records offset (Default value = 0)
Returns:
list[tuple[float, str]]: package logs
"""
return self.database.logs_get(package_base, limit, offset, self.repository_id)
def package_logs_remove(self, package_base: str, version: str | None) -> None:
"""
remove package logs
Args:
package_base(str): package base
version(str | None): package version to remove logs. If None set, all logs will be removed
"""
self.database.logs_remove(package_base, version, self.repository_id)
def package_patches_add(self, package_base: str, patch: PkgbuildPatch) -> None:
"""
create or update package patch
Args:
package_base(str): package base to update
patch(PkgbuildPatch): package patch
"""
self.database.patches_insert(package_base, [patch])
def package_patches_get(self, package_base: str, variable: str | None) -> list[PkgbuildPatch]:
"""
get package patches
Args:
package_base(str): package base to retrieve
variable(str | None): optional filter by patch variable
Returns:
list[PkgbuildPatch]: list of patches for the specified package
"""
variables = [variable] if variable is not None else None
return self.database.patches_list(package_base, variables).get(package_base, [])
def package_patches_remove(self, package_base: str, variable: str | None) -> None:
"""
remove package patch
Args:
package_base(str): package base to update
variable(str | None): patch name. If None set, all patches will be removed
"""
variables = [variable] if variable is not None else None
self.database.patches_remove(package_base, variables)
def package_remove(self, package_base: str) -> None:
"""
remove packages from watcher
Args:
package_base(str): package base to remove
"""
self.database.package_clear(package_base)
def package_set(self, package_base: str, status: BuildStatusEnum) -> None:
"""
update package build status. Unlike :func:`package_add()` it does not update package properties
Args:
package_base(str): package base to update
status(BuildStatusEnum): current package build status
"""
self.database.status_update(package_base, BuildStatus(status), self.repository_id)

View File

@ -19,15 +19,15 @@
#
from threading import Lock
from ahriman.core.database import SQLite
from ahriman.core.exceptions import UnknownPackageError
from ahriman.core.log import LazyLogging
from ahriman.core.status.client import Client
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.changes import Changes
from ahriman.models.dependencies import Dependencies
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_id import RepositoryId
class Watcher(LazyLogging):
@ -35,21 +35,18 @@ class Watcher(LazyLogging):
package status watcher
Attributes:
database(SQLite): database instance
repository_id(RepositoryId): repository unique identifier
client(Client): reporter instance
status(BuildStatus): daemon status
"""
def __init__(self, repository_id: RepositoryId, database: SQLite) -> None:
def __init__(self, client: Client) -> None:
"""
default constructor
Args:
repository_id(RepositoryId): repository unique identifier
database(SQLite): database instance
client(Client): reporter instance
"""
self.repository_id = repository_id
self.database = database
self.client = client
self._lock = Lock()
self._known: dict[str, tuple[Package, BuildStatus]] = {}
@ -76,7 +73,7 @@ class Watcher(LazyLogging):
with self._lock:
self._known = {
package.base: (package, status)
for package, status in self.database.packages_get(self.repository_id)
for package, status in self.client.package_get(None)
}
def logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]:
@ -91,8 +88,8 @@ class Watcher(LazyLogging):
Returns:
list[tuple[float, str]]: package logs
"""
self.package_get(package_base)
return self.database.logs_get(package_base, limit, offset, self.repository_id)
_ = self.package_get(package_base)
return self.client.package_logs_get(package_base, limit, offset)
def logs_remove(self, package_base: str, version: str | None) -> None:
"""
@ -100,24 +97,24 @@ class Watcher(LazyLogging):
Args:
package_base(str): package base
version(str): package versio
version(str): package version
"""
self.database.logs_remove(package_base, version, self.repository_id)
self.client.package_logs_remove(package_base, version)
def logs_update(self, log_record_id: LogRecordId, created: float, record: str) -> None:
def logs_update(self, log_record_id: LogRecordId, created: float, message: str) -> None:
"""
make new log record into database
Args:
log_record_id(LogRecordId): log record id
created(float): log created timestamp
record(str): log record
message(str): log message
"""
if self._last_log_record_id != log_record_id:
# there is new log record, so we remove old ones
self.logs_remove(log_record_id.package_base, log_record_id.version)
self._last_log_record_id = log_record_id
self.database.logs_insert(log_record_id, created, record, self.repository_id)
self.client.package_logs_add(log_record_id, created, message)
def package_changes_get(self, package_base: str) -> Changes:
"""
@ -129,8 +126,35 @@ class Watcher(LazyLogging):
Returns:
Changes: package changes if available
"""
self.package_get(package_base)
return self.database.changes_get(package_base, self.repository_id)
_ = self.package_get(package_base)
return self.client.package_changes_get(package_base)
def package_dependencies_get(self, package_base: str) -> Dependencies:
"""
retrieve package dependencies
Args:
package_base(str): package base
Returns:
Dependencies: package dependencies if available
"""
_ = self.package_get(package_base)
try:
return next(iter(self.client.package_dependencies_get(package_base)))
except StopIteration:
return Dependencies(package_base)
def package_dependencies_set(self, package_base: str, dependencies: Dependencies) -> None:
"""
update package dependencies
Args:
package_base(str): package base
dependencies(Dependencies): package dependencies
"""
_ = self.package_get(package_base)
self.client.package_dependencies_set(dependencies)
def package_get(self, package_base: str) -> tuple[Package, BuildStatus]:
"""
@ -160,7 +184,7 @@ class Watcher(LazyLogging):
"""
with self._lock:
self._known.pop(package_base, None)
self.database.package_remove(package_base, self.repository_id)
self.client.package_remove(package_base)
self.logs_remove(package_base, None)
def package_update(self, package_base: str, status: BuildStatusEnum, package: Package | None) -> None:
@ -174,10 +198,9 @@ class Watcher(LazyLogging):
"""
if package is None:
package, _ = self.package_get(package_base)
full_status = BuildStatus(status)
with self._lock:
self._known[package_base] = (package, full_status)
self.database.package_update(package, full_status, self.repository_id)
self._known[package_base] = (package, BuildStatus(status))
self.client.package_set(package_base, status)
def patches_get(self, package_base: str, variable: str | None) -> list[PkgbuildPatch]:
"""
@ -192,8 +215,7 @@ class Watcher(LazyLogging):
"""
# patches are package base based, we don't know (and don't differentiate) to which package does them belong
# so here we skip checking if package exists or not
variables = [variable] if variable is not None else None
return self.database.patches_list(package_base, variables).get(package_base, [])
return self.client.package_patches_get(package_base, variable)
def patches_remove(self, package_base: str, variable: str) -> None:
"""
@ -203,7 +225,7 @@ class Watcher(LazyLogging):
package_base(str): package base
variable(str): patch variable name
"""
self.database.patches_remove(package_base, [variable])
self.client.package_patches_remove(package_base, variable)
def patches_update(self, package_base: str, patch: PkgbuildPatch) -> None:
"""
@ -213,7 +235,7 @@ class Watcher(LazyLogging):
package_base(str): package base
patch(PkgbuildPatch): package patch
"""
self.database.patches_insert(package_base, [patch])
self.client.package_patches_add(package_base, patch)
def status_update(self, status: BuildStatusEnum) -> None:
"""

View File

@ -18,7 +18,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import contextlib
import logging
from urllib.parse import quote_plus as urlencode
@ -27,9 +26,11 @@ from ahriman.core.http import SyncAhrimanClient
from ahriman.core.status.client import Client
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.changes import Changes
from ahriman.models.dependencies import Dependencies
from ahriman.models.internal_status import InternalStatus
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_id import RepositoryId
@ -92,10 +93,22 @@ class WebClient(Client, SyncAhrimanClient):
package_base(str): package base
Returns:
str: full url for web service for logs
str: full url for web service for changes
"""
return f"{self.address}/api/v1/packages/{urlencode(package_base)}/changes"
def _dependencies_url(self, package_base: str = "") -> str:
"""
get url for the dependencies api
Args:
package_base(str, optional): package base (Default value = "")
Returns:
str: full url for web service for dependencies
"""
return f"{self.address}/api/v1/dependencies/{urlencode(package_base)}"
def _logs_url(self, package_base: str) -> str:
"""
get url for the logs api
@ -110,7 +123,7 @@ class WebClient(Client, SyncAhrimanClient):
def _package_url(self, package_base: str = "") -> str:
"""
url generator
package url generator
Args:
package_base(str, optional): package base to generate url (Default value = "")
@ -121,6 +134,20 @@ class WebClient(Client, SyncAhrimanClient):
suffix = f"/{urlencode(package_base)}" if package_base else ""
return f"{self.address}/api/v1/packages{suffix}"
def _patches_url(self, package_base: str, variable: str = "") -> str:
"""
patches url generator
Args:
package_base(str): package base
variable(str, optional): patch variable name to generate url (Default value = "")
Returns:
str: full url of web service for the package patch
"""
suffix = f"/{urlencode(variable)}" if variable else ""
return f"{self.address}/api/v1/packages/{urlencode(package_base)}/patches{suffix}"
def _status_url(self) -> str:
"""
get url for the status api
@ -177,6 +204,37 @@ class WebClient(Client, SyncAhrimanClient):
self.make_request("POST", self._changes_url(package_base),
params=self.repository_id.query(), json=changes.view())
def package_dependencies_get(self, package_base: str | None) -> list[Dependencies]:
"""
get package dependencies
Args:
package_base(str | None): package base to retrieve
Returns:
list[Dependencies]: package implicit dependencies if available
"""
with contextlib.suppress(Exception):
response = self.make_request("GET", self._dependencies_url(package_base or ""),
params=self.repository_id.query())
response_json = response.json()
dependencies = response_json if package_base is None else [response_json]
return [Dependencies.from_json(dependencies) for dependencies in dependencies]
return []
def package_dependencies_set(self, dependencies: Dependencies) -> None:
"""
update package dependencies
Args:
dependencies(Dependencies): dependencies descriptor
"""
with contextlib.suppress(Exception):
self.make_request("POST", self._dependencies_url(dependencies.package_base),
params=self.repository_id.query(), json=dependencies.view())
def package_get(self, package_base: str | None) -> list[tuple[Package, BuildStatus]]:
"""
get package status
@ -199,17 +257,18 @@ class WebClient(Client, SyncAhrimanClient):
return []
def package_logs(self, log_record_id: LogRecordId, record: logging.LogRecord) -> None:
def package_logs_add(self, log_record_id: LogRecordId, created: float, message: str) -> None:
"""
post log record
Args:
log_record_id(LogRecordId): log record id
record(logging.LogRecord): log record to post to api
created(float): log created timestamp
message(str): log message
"""
payload = {
"created": record.created,
"message": record.getMessage(),
"created": created,
"message": message,
"version": log_record_id.version,
}
@ -219,6 +278,83 @@ class WebClient(Client, SyncAhrimanClient):
self.make_request("POST", self._logs_url(log_record_id.package_base),
params=self.repository_id.query(), json=payload, suppress_errors=True)
def package_logs_get(self, package_base: str, limit: int = -1, offset: int = 0) -> list[tuple[float, str]]:
"""
get package logs
Args:
package_base(str): package base
limit(int, optional): limit records to the specified count, -1 means unlimited (Default value = -1)
offset(int, optional): records offset (Default value = 0)
Returns:
list[tuple[float, str]]: package logs
"""
with contextlib.suppress(Exception):
query = self.repository_id.query() + [("limit", str(limit)), ("offset", str(offset))]
response = self.make_request("GET", self._logs_url(package_base), params=query)
response_json = response.json()
return [(record["created"], record["message"]) for record in response_json]
return []
def package_logs_remove(self, package_base: str, version: str | None) -> None:
"""
remove package logs
Args:
package_base(str): package base
version(str | None): package version to remove logs. If None set, all logs will be removed
"""
with contextlib.suppress(Exception):
query = self.repository_id.query()
if version is not None:
query += [("version", version)]
self.make_request("DELETE", self._logs_url(package_base), params=query)
def package_patches_add(self, package_base: str, patch: PkgbuildPatch) -> None:
"""
create or update package patch
Args:
package_base(str): package base to update
patch(PkgbuildPatch): package patch
"""
with contextlib.suppress(Exception):
self.make_request("POST", self._patches_url(package_base), json=patch.view())
def package_patches_get(self, package_base: str, variable: str | None) -> list[PkgbuildPatch]:
"""
get package patches
Args:
package_base(str): package base to retrieve
variable(str | None): optional filter by patch variable
Returns:
list[PkgbuildPatch]: list of patches for the specified package
"""
with contextlib.suppress(Exception):
response = self.make_request("GET", self._patches_url(package_base, variable or ""))
response_json = response.json()
patches = response_json if variable is None else [response_json]
return [PkgbuildPatch.from_json(patch) for patch in patches]
return []
def package_patches_remove(self, package_base: str, variable: str | None) -> None:
"""
remove package patch
Args:
package_base(str): package base to update
variable(str | None): patch name. If None set, all patches will be removed
"""
with contextlib.suppress(Exception):
self.make_request("DELETE", self._patches_url(package_base, variable or ""))
def package_remove(self, package_base: str) -> None:
"""
remove packages from watcher
@ -229,7 +365,7 @@ class WebClient(Client, SyncAhrimanClient):
with contextlib.suppress(Exception):
self.make_request("DELETE", self._package_url(package_base), params=self.repository_id.query())
def package_update(self, package_base: str, status: BuildStatusEnum) -> None:
def package_set(self, package_base: str, status: BuildStatusEnum) -> None:
"""
update package build status. Unlike :func:`package_add()` it does not update package properties

View File

@ -24,7 +24,6 @@ from ahriman.core.sign.gpg import GPG
from ahriman.core.support.package_creator import PackageCreator
from ahriman.core.support.pkgbuild.keyring_generator import KeyringGenerator
from ahriman.core.triggers import Trigger
from ahriman.models.context_key import ContextKey
from ahriman.models.repository_id import RepositoryId
@ -134,8 +133,8 @@ class KeyringTrigger(Trigger):
trigger action which will be called at the start of the application
"""
ctx = context.get()
sign = ctx.get(ContextKey("sign", GPG))
database = ctx.get(ContextKey("database", SQLite))
sign = ctx.get(GPG)
database = ctx.get(SQLite)
for target in self.targets:
generator = KeyringGenerator(database, sign, self.repository_id, self.configuration, target)

View File

@ -18,14 +18,13 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import shutil
from pathlib import Path
from ahriman.core import context
from ahriman.core.build_tools.sources import Sources
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.status.client import Client
from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator
from ahriman.models.build_status import BuildStatus
from ahriman.models.context_key import ContextKey
from ahriman.models.package import Package
@ -49,23 +48,39 @@ class PackageCreator:
self.configuration = configuration
self.generator = generator
def package_create(self, path: Path) -> None:
"""
create package files
Args:
path(Path): path to directory with package files
"""
# clear old tree if any
shutil.rmtree(path, ignore_errors=True)
# create local tree
path.mkdir(mode=0o755, parents=True, exist_ok=True)
self.generator.write_pkgbuild(path)
Sources.init(path)
def package_register(self, path: Path) -> None:
"""
register package in build worker
Args:
path(Path): path to directory with package files
"""
ctx = context.get()
reporter = ctx.get(Client)
_, repository_id = self.configuration.check_loaded()
package = Package.from_build(path, repository_id.architecture, None)
reporter.set_unknown(package)
def run(self) -> None:
"""
create new local package
"""
local_path = self.configuration.repository_paths.cache_for(self.generator.pkgname)
# clear old tree if any
shutil.rmtree(local_path, ignore_errors=True)
# create local tree
local_path.mkdir(mode=0o755, parents=True, exist_ok=True)
self.generator.write_pkgbuild(local_path)
Sources.init(local_path)
# register package
ctx = context.get()
database: SQLite = ctx.get(ContextKey("database", SQLite))
_, repository_id = self.configuration.check_loaded()
package = Package.from_build(local_path, repository_id.architecture, None)
database.package_update(package, BuildStatus())
self.package_create(local_path)
self.package_register(local_path)

View File

@ -18,7 +18,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass
from typing import Generic, TypeVar
from typing import Generic, Self, TypeVar
T = TypeVar("T")
@ -35,3 +35,16 @@ class ContextKey(Generic[T]):
"""
key: str
return_type: type[T]
@classmethod
def from_type(cls, return_type: type[T]) -> Self:
"""
construct key from type
Args:
return_type(type[T]): return type used for the specified context key
Returns:
Self: context key with autogenerated
"""
return cls(return_type.__name__, return_type)

View File

@ -17,8 +17,11 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass, field
from dataclasses import dataclass, field, fields
from pathlib import Path
from typing import Any, Self
from ahriman.core.util import dataclass_view, filter_json
@dataclass(frozen=True)
@ -33,3 +36,27 @@ class Dependencies:
package_base: str
paths: dict[Path, list[str]] = field(default_factory=dict)
@classmethod
def from_json(cls, dump: dict[str, Any]) -> Self:
"""
construct dependencies from the json dump
Args:
dump(dict[str, Any]): json dump body
Returns:
Self: dependencies object
"""
# filter to only known fields
known_fields = [pair.name for pair in fields(cls)]
return cls(**filter_json(dump, known_fields))
def view(self) -> dict[str, Any]:
"""
generate json dependencies view
Returns:
dict[str, Any]: json-friendly dictionary
"""
return dataclass_view(self)

View File

@ -19,11 +19,11 @@
#
import shlex
from dataclasses import dataclass
from dataclasses import dataclass, fields
from pathlib import Path
from typing import Any, Generator, Self
from ahriman.core.util import dataclass_view
from ahriman.core.util import dataclass_view, filter_json
@dataclass(frozen=True)
@ -84,6 +84,21 @@ class PkgbuildPatch:
raw_value = next(iter(value_parts), "") # extract raw value
return cls(key, cls.parse(raw_value))
@classmethod
def from_json(cls, dump: dict[str, Any]) -> Self:
"""
construct patch descriptor from the json dump
Args:
dump(dict[str, Any]): json dump body
Returns:
Self: patch object
"""
# filter to only known fields
known_fields = [pair.name for pair in fields(cls)]
return cls(**filter_json(dump, known_fields))
@staticmethod
def parse(source: str) -> str | list[str]:
"""

View File

@ -22,6 +22,7 @@ from ahriman.web.schemas.auth_schema import AuthSchema
from ahriman.web.schemas.build_options_schema import BuildOptionsSchema
from ahriman.web.schemas.changes_schema import ChangesSchema
from ahriman.web.schemas.counters_schema import CountersSchema
from ahriman.web.schemas.dependencies_schema import DependenciesSchema
from ahriman.web.schemas.error_schema import ErrorSchema
from ahriman.web.schemas.file_schema import FileSchema
from ahriman.web.schemas.info_schema import InfoSchema
@ -36,6 +37,7 @@ from ahriman.web.schemas.package_patch_schema import PackagePatchSchema
from ahriman.web.schemas.package_properties_schema import PackagePropertiesSchema
from ahriman.web.schemas.package_schema import PackageSchema
from ahriman.web.schemas.package_status_schema import PackageStatusSchema, PackageStatusSimplifiedSchema
from ahriman.web.schemas.package_version_schema import PackageVersionSchema
from ahriman.web.schemas.pagination_schema import PaginationSchema
from ahriman.web.schemas.patch_name_schema import PatchNameSchema
from ahriman.web.schemas.patch_schema import PatchSchema

View File

@ -0,0 +1,35 @@
#
# Copyright (c) 2021-2024 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from marshmallow import Schema, fields
class DependenciesSchema(Schema):
"""
request/response package dependencies schema
"""
package_base = fields.String(metadata={
"description": "Package base name",
"example": "ahriman",
})
paths = fields.Dict(
keys=fields.String(), values=fields.List(fields.String()), required=True, metadata={
"description": "Map of filesystem paths to packages which contain this path",
})

View File

@ -0,0 +1,34 @@
#
# Copyright (c) 2021-2024 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from marshmallow import fields
from ahriman import __version__
from ahriman.web.schemas.repository_id_schema import RepositoryIdSchema
class PackageVersionSchema(RepositoryIdSchema):
"""
request package name schema
"""
version = fields.String(required=True, metadata={
"description": "Package version",
"example": __version__,
})

View File

@ -113,7 +113,6 @@ class ChangesView(StatusViewGuard, BaseView):
raise HTTPBadRequest(reason=str(ex))
changes = Changes(last_commit_sha, change)
repository_id = self.repository_id()
self.service(repository_id).database.changes_insert(package_base, changes, repository_id)
self.service().client.package_changes_set(package_base, changes)
raise HTTPNoContent

View File

@ -0,0 +1,66 @@
#
# Copyright (c) 2021-2024 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import aiohttp_apispec # type: ignore[import-untyped]
from aiohttp.web import Response, json_response
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, DependenciesSchema, ErrorSchema, RepositoryIdSchema
from ahriman.web.views.base import BaseView
from ahriman.web.views.status_view_guard import StatusViewGuard
class DependenciesView(StatusViewGuard, BaseView):
"""
packages dependencies web view
Attributes:
GET_PERMISSION(UserAccess): (class attribute) get permissions of self
"""
GET_PERMISSION = UserAccess.Reporter
POST_PERMISSION = UserAccess.Full
ROUTES = ["/api/v1/dependencies"]
@aiohttp_apispec.docs(
tags=["Build"],
summary="Get dependencies for all packages",
description="Retrieve implicit dependencies for all known packages",
responses={
200: {"description": "Success response", "schema": DependenciesSchema(many=True)},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
404: {"description": "Package base and/or repository are unknown", "schema": ErrorSchema},
500: {"description": "Internal server error", "schema": ErrorSchema},
},
security=[{"token": [GET_PERMISSION]}],
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.querystring_schema(RepositoryIdSchema)
async def get(self) -> Response:
"""
get dependencies for all packages
Returns:
Response: 200 with package implicit dependencies on success
"""
dependencies = self.service().client.package_dependencies_get(None)
return json_response([dependency.view() for dependency in dependencies])

View File

@ -0,0 +1,120 @@
#
# Copyright (c) 2021-2024 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import aiohttp_apispec # type: ignore[import-untyped]
from aiohttp.web import HTTPBadRequest, HTTPNoContent, HTTPNotFound, Response, json_response
from ahriman.core.exceptions import UnknownPackageError
from ahriman.models.dependencies import Dependencies
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, DependenciesSchema, ErrorSchema, PackageNameSchema, RepositoryIdSchema
from ahriman.web.views.base import BaseView
from ahriman.web.views.status_view_guard import StatusViewGuard
class DependencyView(StatusViewGuard, BaseView):
"""
package dependencies web view
Attributes:
GET_PERMISSION(UserAccess): (class attribute) get permissions of self
POST_PERMISSION(UserAccess): (class attribute) post permissions of self
"""
GET_PERMISSION = UserAccess.Reporter
POST_PERMISSION = UserAccess.Full
ROUTES = ["/api/v1/dependencies/{package}"]
@aiohttp_apispec.docs(
tags=["Build"],
summary="Get package dependencies",
description="Retrieve package implicit dependencies",
responses={
200: {"description": "Success response", "schema": DependenciesSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
404: {"description": "Package base and/or repository are unknown", "schema": ErrorSchema},
500: {"description": "Internal server error", "schema": ErrorSchema},
},
security=[{"token": [GET_PERMISSION]}],
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.match_info_schema(PackageNameSchema)
@aiohttp_apispec.querystring_schema(RepositoryIdSchema)
async def get(self) -> Response:
"""
get package dependencies
Returns:
Response: 200 with package implicit dependencies on success
Raises:
HTTPNotFound: if package base is unknown
"""
package_base = self.request.match_info["package"]
try:
dependencies = self.service().package_dependencies_get(package_base)
except UnknownPackageError:
raise HTTPNotFound(reason=f"Package {package_base} is unknown")
return json_response(dependencies.view())
@aiohttp_apispec.docs(
tags=["Build"],
summary="Update package dependencies",
description="Set package implicit dependencies",
responses={
204: {"description": "Success response"},
400: {"description": "Bad data is supplied", "schema": ErrorSchema},
401: {"description": "Authorization required", "schema": ErrorSchema},
403: {"description": "Access is forbidden", "schema": ErrorSchema},
404: {"description": "Repository is unknown", "schema": ErrorSchema},
500: {"description": "Internal server error", "schema": ErrorSchema},
},
security=[{"token": [POST_PERMISSION]}],
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.match_info_schema(PackageNameSchema)
@aiohttp_apispec.querystring_schema(RepositoryIdSchema)
@aiohttp_apispec.json_schema(DependenciesSchema)
async def post(self) -> None:
"""
insert new package dependencies
Raises:
HTTPBadRequest: if bad data is supplied
HTTPNoContent: in case of success response
"""
package_base = self.request.match_info["package"]
try:
data = await self.request.json()
data["package_base"] = package_base # read from path instead of object
dependencies = Dependencies.from_json(data)
except Exception as ex:
raise HTTPBadRequest(reason=str(ex))
try:
self.service().package_dependencies_set(package_base, dependencies)
except UnknownPackageError:
raise HTTPNotFound(reason=f"Package {package_base} is unknown")
raise HTTPNoContent

View File

@ -25,8 +25,8 @@ from ahriman.core.exceptions import UnknownPackageError
from ahriman.core.util import pretty_datetime
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.user_access import UserAccess
from ahriman.web.schemas import AuthSchema, ErrorSchema, LogsSchema, PackageNameSchema, RepositoryIdSchema, \
VersionedLogSchema
from ahriman.web.schemas import AuthSchema, ErrorSchema, LogsSchema, PackageNameSchema, PackageVersionSchema, \
RepositoryIdSchema, VersionedLogSchema
from ahriman.web.views.base import BaseView
from ahriman.web.views.status_view_guard import StatusViewGuard
@ -60,7 +60,7 @@ class LogsView(StatusViewGuard, BaseView):
)
@aiohttp_apispec.cookies_schema(AuthSchema)
@aiohttp_apispec.match_info_schema(PackageNameSchema)
@aiohttp_apispec.querystring_schema(RepositoryIdSchema)
@aiohttp_apispec.querystring_schema(PackageVersionSchema)
async def delete(self) -> None:
"""
delete package logs
@ -69,7 +69,8 @@ class LogsView(StatusViewGuard, BaseView):
HTTPNoContent: on success response
"""
package_base = self.request.match_info["package"]
self.service().logs_remove(package_base, None)
version = self.request.query.get("version")
self.service().logs_remove(package_base, version)
raise HTTPNoContent

View File

@ -30,6 +30,7 @@ from ahriman.core.database import SQLite
from ahriman.core.distributed import WorkersCache
from ahriman.core.exceptions import InitializeError
from ahriman.core.spawn import Spawn
from ahriman.core.status.client import Client
from ahriman.core.status.watcher import Watcher
from ahriman.models.repository_id import RepositoryId
from ahriman.web.apispec import setup_apispec
@ -167,7 +168,8 @@ def setup_server(configuration: Configuration, spawner: Spawn, repositories: lis
watchers: dict[RepositoryId, Watcher] = {}
for repository_id in repositories:
application.logger.info("load repository %s", repository_id)
watchers[repository_id] = Watcher(repository_id, database)
client = Client.load(repository_id, configuration, database, report=False) # explicitly load local client
watchers[repository_id] = Watcher(client)
application[WatcherKey] = watchers
# workers cache
application[WorkersKey] = WorkersCache(configuration)

View File

@ -3,7 +3,6 @@ from pytest_mock import MockerFixture
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.gitremote import RemotePushTrigger
from ahriman.models.context_key import ContextKey
from ahriman.models.package import Package
from ahriman.models.result import Result
@ -30,5 +29,5 @@ def test_on_result(configuration: Configuration, result: Result, package_ahriman
trigger = RemotePushTrigger(repository_id, configuration)
trigger.on_result(result, [package_ahriman])
database_mock.assert_called_once_with(ContextKey("database", SQLite))
database_mock.assert_called_once_with(SQLite)
run_mock.assert_called_once_with(result)

View File

@ -6,7 +6,6 @@ from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.repository import Repository
from ahriman.core.sign.gpg import GPG
from ahriman.models.context_key import ContextKey
def test_load(configuration: Configuration, database: SQLite, mocker: MockerFixture) -> None:
@ -29,9 +28,9 @@ def test_set_context(configuration: Configuration, database: SQLite, mocker: Moc
instance = Repository.load(repository_id, configuration, database, report=False)
set_mock.assert_has_calls([
MockCall(ContextKey("database", SQLite), instance.database),
MockCall(ContextKey("configuration", Configuration), instance.configuration),
MockCall(ContextKey("pacman", Pacman), instance.pacman),
MockCall(ContextKey("sign", GPG), instance.sign),
MockCall(ContextKey("repository", Repository), instance),
MockCall(SQLite, instance.database),
MockCall(Configuration, instance.configuration),
MockCall(Pacman, instance.pacman),
MockCall(GPG, instance.sign),
MockCall(Repository, instance),
])

View File

@ -5,7 +5,6 @@ from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.core.sign.gpg import GPG
from ahriman.core.support import KeyringTrigger
from ahriman.models.context_key import ContextKey
def test_configuration_sections(configuration: Configuration) -> None:
@ -29,5 +28,5 @@ def test_on_start(configuration: Configuration, mocker: MockerFixture) -> None:
trigger = KeyringTrigger(repository_id, configuration)
trigger.on_start()
context_mock.assert_has_calls([MockCall(ContextKey("sign", GPG)), MockCall(ContextKey("database", SQLite))])
context_mock.assert_has_calls([MockCall(GPG), MockCall(SQLite)])
run_mock.assert_called_once_with()

View File

@ -4,7 +4,6 @@ from pytest_mock import MockerFixture
from ahriman.core.database import SQLite
from ahriman.core.support.package_creator import PackageCreator
from ahriman.models.context_key import ContextKey
from ahriman.models.package import Package
from ahriman.models.package_description import PackageDescription
from ahriman.models.package_source import PackageSource
@ -38,5 +37,5 @@ def test_run(package_creator: PackageCreator, database: SQLite, mocker: MockerFi
init_mock.assert_called_once_with(local_path)
package_mock.assert_called_once_with(local_path, "x86_64", None)
database_mock.assert_called_once_with(ContextKey("database", SQLite))
database_mock.assert_called_once_with(SQLite)
insert_mock.assert_called_once_with(package, pytest.helpers.anyvar(int))

View File

@ -15,6 +15,18 @@ def test_get_set() -> None:
assert ctx.get(key) == value
def test_get_set_type() -> None:
"""
must set and get variable by type
"""
key, value = int, 42
ctx = _Context()
ctx.set(key, value)
assert ctx.get(key) == value
assert ctx.get(ContextKey.from_type(int)) == value
def test_get_key_exception() -> None:
"""
must raise KeyError in case if key was not found

View File

@ -0,0 +1,9 @@
from ahriman.models.context_key import ContextKey
def test_from_type() -> None:
"""
must construct key from type
"""
assert ContextKey.from_type(int) == ContextKey("int", int)
assert ContextKey.from_type(ContextKey) == ContextKey("ContextKey", ContextKey)