feat: allow to use one application for multiple repositories (#111)

* allow to use one application for multiple repositories

* update tests

* handle None append argument everywhere

* rewrite repository definition logic

* drop optional flags from docs

* support of new schema in systemd units

* add migration docs and ability to migrate tree automatically

* use repostory id instead

* verbose multiarchitectureerror

* object path support for s3 sync

* fix tests after rebase
This commit is contained in:
2023-09-08 03:42:28 +03:00
parent 99eecdebf3
commit 59356e905a
191 changed files with 3441 additions and 1319 deletions

View File

@ -79,9 +79,14 @@ def _parser() -> argparse.ArgumentParser:
parser.add_argument("--log-handler", help="explicit log handler specification. If none set, the handler will be "
"guessed from environment",
type=LogHandler, choices=enum_values(LogHandler))
parser.add_argument("-q", "--quiet", help="force disable any logging", action="store_true")
parser.add_argument("--report", help="force enable or disable reporting to web service",
action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("-q", "--quiet", help="force disable any logging", action="store_true")
parser.add_argument("-r", "--repository", help="target repository. For several subcommands it can be used "
"multiple times", action="append")
# special secret argument for systemd unit. The issue is that systemd doesn't allow multiple arguments to template
# name. This parameter accepts [[arch]-repo] in order to keep backward compatibility
parser.add_argument("--repository-id", help=argparse.SUPPRESS)
parser.add_argument("--unsafe", help="allow to run ahriman as non-ahriman user. Some actions might be unavailable",
action="store_true")
parser.add_argument("--wait-timeout", help="wait for lock to be free. Negative value will lead to "
@ -127,6 +132,7 @@ def _parser() -> argparse.ArgumentParser:
_set_service_key_import_parser(subparsers)
_set_service_setup_parser(subparsers)
_set_service_shell_parser(subparsers)
_set_service_tree_migrate_parser(subparsers)
_set_user_add_parser(subparsers)
_set_user_list_parser(subparsers)
_set_user_remove_parser(subparsers)
@ -155,7 +161,8 @@ def _set_aur_search_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser.add_argument("--sort-by", help="sort field by this field. In case if two packages have the same value of "
"the specified field, they will be always sorted by name",
default="name", choices=sorted(handlers.Search.SORT_FIELDS))
parser.set_defaults(handler=handlers.Search, architecture=[""], lock=None, report=False, quiet=True, unsafe=True)
parser.set_defaults(handler=handlers.Search, architecture=[""], lock=None, quiet=True, report=False,
repository=[""], unsafe=True)
return parser
@ -173,8 +180,8 @@ def _set_help_parser(root: SubParserAction) -> argparse.ArgumentParser:
description="show help message for application or command and exit",
formatter_class=_formatter)
parser.add_argument("command", help="show help message for specific command", nargs="?")
parser.set_defaults(handler=handlers.Help, architecture=[""], lock=None, report=False, quiet=True, unsafe=True,
parser=_parser)
parser.set_defaults(handler=handlers.Help, architecture=[""], lock=None, quiet=True, report=False, repository=[""],
unsafe=True, parser=_parser)
return parser
@ -192,8 +199,8 @@ def _set_help_commands_unsafe_parser(root: SubParserAction) -> argparse.Argument
description="list unsafe commands as defined in default args", formatter_class=_formatter)
parser.add_argument("command", help="instead of showing commands, just test command line for unsafe subcommand "
"and return 0 in case if command is safe and 1 otherwise", nargs="*")
parser.set_defaults(handler=handlers.UnsafeCommands, architecture=[""], lock=None, report=False, quiet=True,
unsafe=True, parser=_parser)
parser.set_defaults(handler=handlers.UnsafeCommands, architecture=[""], lock=None, quiet=True, report=False,
repository=[""], unsafe=True, parser=_parser)
return parser
@ -211,8 +218,8 @@ def _set_help_updates_parser(root: SubParserAction) -> argparse.ArgumentParser:
description="request AUR for current version and compare with current service version",
formatter_class=_formatter)
parser.add_argument("-e", "--exit-code", help="return non-zero exit code if updates available", action="store_true")
parser.set_defaults(handler=handlers.ServiceUpdates, architecture=[""], lock=None, report=False, quiet=True,
unsafe=True)
parser.set_defaults(handler=handlers.ServiceUpdates, architecture=[""], lock=None, quiet=True, report=False,
repository=[""], unsafe=True)
return parser
@ -228,7 +235,8 @@ def _set_help_version_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
parser = root.add_parser("help-version", aliases=["version"], help="application version",
description="print application and its dependencies versions", formatter_class=_formatter)
parser.set_defaults(handler=handlers.Versions, architecture=[""], lock=None, report=False, quiet=True, unsafe=True)
parser.set_defaults(handler=handlers.Versions, architecture=[""], lock=None, quiet=True, report=False,
repository=[""], unsafe=True)
return parser
@ -310,7 +318,7 @@ def _set_package_status_parser(root: SubParserAction) -> argparse.ArgumentParser
action=argparse.BooleanOptionalAction, default=False)
parser.add_argument("-s", "--status", help="filter packages by status",
type=BuildStatusEnum, choices=enum_values(BuildStatusEnum))
parser.set_defaults(handler=handlers.Status, lock=None, report=False, quiet=True, unsafe=True)
parser.set_defaults(handler=handlers.Status, lock=None, quiet=True, report=False, unsafe=True)
return parser
@ -330,7 +338,7 @@ def _set_package_status_remove_parser(root: SubParserAction) -> argparse.Argumen
"clears the status page.",
formatter_class=_formatter)
parser.add_argument("package", help="remove specified packages from status page", nargs="+")
parser.set_defaults(handler=handlers.StatusUpdate, action=Action.Remove, lock=None, report=False, quiet=True,
parser.set_defaults(handler=handlers.StatusUpdate, action=Action.Remove, lock=None, quiet=True, report=False,
unsafe=True)
return parser
@ -352,7 +360,7 @@ def _set_package_status_update_parser(root: SubParserAction) -> argparse.Argumen
nargs="*")
parser.add_argument("-s", "--status", help="new package build status",
type=BuildStatusEnum, choices=enum_values(BuildStatusEnum), default=BuildStatusEnum.Success)
parser.set_defaults(handler=handlers.StatusUpdate, action=Action.Update, lock=None, report=False, quiet=True,
parser.set_defaults(handler=handlers.StatusUpdate, action=Action.Update, lock=None, quiet=True, report=False,
unsafe=True)
return parser
@ -379,7 +387,8 @@ def _set_patch_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
"it must end with ()")
parser.add_argument("patch", help="path to file which contains function or variable value. If not set, "
"the value will be read from stdin", type=Path, nargs="?")
parser.set_defaults(handler=handlers.Patch, action=Action.Update, architecture=[""], lock=None, report=False)
parser.set_defaults(handler=handlers.Patch, action=Action.Update, architecture=[""], lock=None, report=False,
repository=[""])
return parser
@ -400,7 +409,7 @@ def _set_patch_list_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser.add_argument("-v", "--variable", help="if set, show only patches for specified PKGBUILD variables",
action="append")
parser.set_defaults(handler=handlers.Patch, action=Action.List, architecture=[""], lock=None, report=False,
unsafe=True)
repository=[""], unsafe=True)
return parser
@ -421,7 +430,8 @@ def _set_patch_remove_parser(root: SubParserAction) -> argparse.ArgumentParser:
"to remove only specified PKGBUILD variables. In case if not set, "
"it will remove all patches related to the package",
action="append")
parser.set_defaults(handler=handlers.Patch, action=Action.Remove, architecture=[""], lock=None, report=False)
parser.set_defaults(handler=handlers.Patch, action=Action.Remove, architecture=[""], lock=None, report=False,
repository=[""])
return parser
@ -446,7 +456,7 @@ def _set_patch_set_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser.add_argument("-t", "--track", help="files which has to be tracked", action="append",
default=["*.diff", "*.patch"])
parser.set_defaults(handler=handlers.Patch, action=Action.Update, architecture=[""], lock=None, report=False,
variable=None)
repository=[""], variable=None)
return parser
@ -463,7 +473,8 @@ def _set_repo_backup_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser = root.add_parser("repo-backup", help="backup repository data",
description="backup repository settings and database", formatter_class=_formatter)
parser.add_argument("path", help="path of the output archive", type=Path)
parser.set_defaults(handler=handlers.Backup, architecture=[""], lock=None, report=False, unsafe=True)
parser.set_defaults(handler=handlers.Backup, architecture=[""], lock=None, report=False, repository=[""],
unsafe=True)
return parser
@ -642,7 +653,8 @@ def _set_repo_restore_parser(root: SubParserAction) -> argparse.ArgumentParser:
description="restore settings and database", formatter_class=_formatter)
parser.add_argument("path", help="path of the input archive", type=Path)
parser.add_argument("-o", "--output", help="root path of the extracted files", type=Path, default=Path("/"))
parser.set_defaults(handler=handlers.Restore, architecture=[""], lock=None, report=False, unsafe=True)
parser.set_defaults(handler=handlers.Restore, architecture=[""], lock=None, report=False, repository=[""],
unsafe=True)
return parser
@ -679,8 +691,8 @@ def _set_repo_status_update_parser(root: SubParserAction) -> argparse.ArgumentPa
description="update repository status on the status page", formatter_class=_formatter)
parser.add_argument("-s", "--status", help="new status",
type=BuildStatusEnum, choices=enum_values(BuildStatusEnum), default=BuildStatusEnum.Success)
parser.set_defaults(handler=handlers.StatusUpdate, action=Action.Update, lock=None, report=False, package=[],
quiet=True, unsafe=True)
parser.set_defaults(handler=handlers.StatusUpdate, action=Action.Update, lock=None, package=[], quiet=True,
report=False, unsafe=True)
return parser
@ -717,7 +729,7 @@ def _set_repo_tree_parser(root: SubParserAction) -> argparse.ArgumentParser:
formatter_class=_formatter)
parser.add_argument("-p", "--partitions", help="also divide packages by independent partitions",
type=int, default=1)
parser.set_defaults(handler=handlers.Structure, lock=None, report=False, quiet=True, unsafe=True)
parser.set_defaults(handler=handlers.Structure, lock=None, quiet=True, report=False, unsafe=True)
return parser
@ -820,7 +832,7 @@ def _set_service_config_parser(root: SubParserAction) -> argparse.ArgumentParser
formatter_class=_formatter)
parser.add_argument("--secure", help="hide passwords and secrets from output",
action=argparse.BooleanOptionalAction, default=True)
parser.set_defaults(handler=handlers.Dump, lock=None, report=False, quiet=True, unsafe=True)
parser.set_defaults(handler=handlers.Dump, lock=None, quiet=True, report=False, unsafe=True)
return parser
@ -840,7 +852,7 @@ def _set_service_config_validate_parser(root: SubParserAction) -> argparse.Argum
formatter_class=_formatter)
parser.add_argument("-e", "--exit-code", help="return non-zero exit status if configuration is invalid",
action="store_true")
parser.set_defaults(handler=handlers.Validate, lock=None, report=False, quiet=True, unsafe=True)
parser.set_defaults(handler=handlers.Validate, lock=None, quiet=True, report=False, unsafe=True)
return parser
@ -863,7 +875,7 @@ def _set_service_key_import_parser(root: SubParserAction) -> argparse.ArgumentPa
formatter_class=_formatter)
parser.add_argument("--key-server", help="key server for key import", default="keyserver.ubuntu.com")
parser.add_argument("key", help="PGP key to import from public server")
parser.set_defaults(handler=handlers.KeyImport, architecture=[""], lock=None, report=False)
parser.set_defaults(handler=handlers.KeyImport, architecture=[""], lock=None, report=False, repository=[""])
return parser
@ -883,7 +895,6 @@ def _set_service_setup_parser(root: SubParserAction) -> argparse.ArgumentParser:
epilog="Create _minimal_ configuration for the service according to provided options.",
formatter_class=_formatter)
parser.add_argument("--build-as-user", help="force makepkg user to the specific one")
parser.add_argument("--build-command", help="build command prefix", default="ahriman")
parser.add_argument("--from-configuration", help="path to default devtools pacman configuration",
type=Path, default=Path("/usr") / "share" / "devtools" / "pacman.conf.d" / "extra.conf")
parser.add_argument("--generate-salt", help="generate salt for user passwords",
@ -894,14 +905,13 @@ def _set_service_setup_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser.add_argument("--multilib", help="add or do not multilib repository",
action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("--packager", help="packager name and email", required=True)
parser.add_argument("--repository", help="repository name", required=True)
parser.add_argument("--server", help="server to be used for devtools. If none set, local files will be used")
parser.add_argument("--sign-key", help="sign key id")
parser.add_argument("--sign-target", help="sign options", action="append",
type=SignSettings.from_option, choices=enum_values(SignSettings))
parser.add_argument("--web-port", help="port of the web service", type=int)
parser.add_argument("--web-unix-socket", help="path to unix socket used for interprocess communications", type=Path)
parser.set_defaults(handler=handlers.Setup, lock=None, report=False, quiet=True, unsafe=True)
parser.set_defaults(handler=handlers.Setup, lock=None, quiet=True, report=False, unsafe=True)
return parser
@ -923,6 +933,22 @@ def _set_service_shell_parser(root: SubParserAction) -> argparse.ArgumentParser:
return parser
def _set_service_tree_migrate_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for tree migration subcommand
Args:
root(SubParserAction): subparsers for the commands
Returns:
argparse.ArgumentParser: created argument parser
"""
parser = root.add_parser("service-tree-migrate", help="migrate repository tree",
description="migrate repository tree between versions", formatter_class=_formatter)
parser.set_defaults(handler=handlers.TreeMigrate, lock=None, quiet=True, report=False)
return parser
def _set_user_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
"""
add parser for create user subcommand
@ -943,10 +969,10 @@ def _set_user_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
"`Name Surname <mail@example.com>`")
parser.add_argument("-p", "--password", help="user password. Blank password will be treated as empty password, "
"which is in particular must be used for OAuth2 authorization type.")
parser.add_argument("-r", "--role", help="user access level",
parser.add_argument("-R", "--role", help="user access level",
type=UserAccess, choices=enum_values(UserAccess), default=UserAccess.Read)
parser.set_defaults(handler=handlers.Users, action=Action.Update, architecture=[""], lock=None, report=False,
quiet=True)
parser.set_defaults(handler=handlers.Users, action=Action.Update, architecture=[""], lock=None, quiet=True,
report=False, repository=[""])
return parser
@ -965,9 +991,9 @@ def _set_user_list_parser(root: SubParserAction) -> argparse.ArgumentParser:
formatter_class=_formatter)
parser.add_argument("username", help="filter users by username", nargs="?")
parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
parser.add_argument("-r", "--role", help="filter users by role", type=UserAccess, choices=enum_values(UserAccess))
parser.set_defaults(handler=handlers.Users, action=Action.List, architecture=[""], lock=None, report=False,
quiet=True, unsafe=True)
parser.add_argument("-R", "--role", help="filter users by role", type=UserAccess, choices=enum_values(UserAccess))
parser.set_defaults(handler=handlers.Users, action=Action.List, architecture=[""], lock=None, quiet=True,
report=False, repository=[""], unsafe=True)
return parser
@ -985,8 +1011,8 @@ def _set_user_remove_parser(root: SubParserAction) -> argparse.ArgumentParser:
description="remove user from the user mapping and update the configuration",
formatter_class=_formatter)
parser.add_argument("username", help="username for web service")
parser.set_defaults(handler=handlers.Users, action=Action.Remove, architecture=[""], lock=None, report=False,
quiet=True)
parser.set_defaults(handler=handlers.Users, action=Action.Remove, architecture=[""], lock=None, quiet=True,
report=False, repository=[""])
return parser

View File

@ -37,9 +37,10 @@ class Application(ApplicationPackages, ApplicationRepository):
>>> from ahriman.core.configuration import Configuration
>>> from ahriman.models.package_source import PackageSource
>>> from ahriman.models.repository_id import RepositoryId
>>>
>>> configuration = Configuration()
>>> application = Application("x86_64", configuration, report=True)
>>> application = Application(RepositoryId("x86_64", "x86_64"), configuration, report=True)
>>> # add packages to build queue
>>> application.add(["ahriman"], PackageSource.AUR)
>>>

View File

@ -22,6 +22,7 @@ from ahriman.core.database import SQLite
from ahriman.core.log import LazyLogging
from ahriman.core.repository import Repository
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.repository_id import RepositoryId
class ApplicationProperties(LazyLogging):
@ -29,26 +30,36 @@ class ApplicationProperties(LazyLogging):
application base properties class
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
database(SQLite): database instance
repository(Repository): repository instance
repository_id(RepositoryId): repository unique identifier
"""
def __init__(self, architecture: str, configuration: Configuration, *, report: bool,
def __init__(self, repository_id: RepositoryId, configuration: Configuration, *, report: bool,
refresh_pacman_database: PacmanSynchronization = PacmanSynchronization.Disabled) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
refresh_pacman_database(PacmanSynchronization, optional): pacman database synchronization level
(Default value = PacmanSynchronization.Disabled)
"""
self.configuration = configuration
self.architecture = architecture
self.repository_id = repository_id
self.database = SQLite.load(configuration)
self.repository = Repository.load(architecture, configuration, self.database, report=report,
self.repository = Repository.load(repository_id, configuration, self.database, report=report,
refresh_pacman_database=refresh_pacman_database)
@property
def architecture(self) -> str:
"""
repository architecture for backward compatibility
Returns:
str: repository architecture
"""
return self.repository_id.architecture

View File

@ -39,6 +39,7 @@ from ahriman.application.handlers.sign import Sign
from ahriman.application.handlers.status import Status
from ahriman.application.handlers.status_update import StatusUpdate
from ahriman.application.handlers.structure import Structure
from ahriman.application.handlers.tree_migrate import TreeMigrate
from ahriman.application.handlers.triggers import Triggers
from ahriman.application.handlers.unsafe_commands import UnsafeCommands
from ahriman.application.handlers.update import Update

View File

@ -23,6 +23,7 @@ from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.packagers import Packagers
from ahriman.models.repository_id import RepositoryId
class Add(Handler):
@ -31,17 +32,18 @@ class Add(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report, refresh_pacman_database=args.refresh)
application = Application(repository_id, configuration, report=report, refresh_pacman_database=args.refresh)
application.on_start()
application.add(args.package, args.source, args.username)
if not args.now:

View File

@ -26,6 +26,7 @@ from tarfile import TarFile
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.models.repository_id import RepositoryId
class Backup(Handler):
@ -33,16 +34,17 @@ class Backup(Handler):
backup packages handler
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
ALLOW_MULTI_ARCHITECTURE_RUN = False # system-wide action
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class Clean(Handler):
@ -30,17 +31,18 @@ class Clean(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
application.on_start()
application.clean(cache=args.cache, chroot=args.chroot, manual=args.manual, packages=args.packages,
pacman=args.pacman)

View File

@ -22,6 +22,7 @@ import threading
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class Daemon(Handler):
@ -30,19 +31,20 @@ class Daemon(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
from ahriman.application.handlers import Update
Update.run(args, architecture, configuration, report=report)
timer = threading.Timer(args.interval, Daemon.run, args=[args, architecture, configuration],
Update.run(args, repository_id, configuration, report=report)
timer = threading.Timer(args.interval, Daemon.run, args=[args, repository_id, configuration],
kwargs={"report": report})
timer.start()
timer.join()

View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import ConfigurationPathsPrinter, ConfigurationPrinter, StringPrinter
from ahriman.models.repository_id import RepositoryId
class Dump(Handler):
@ -29,16 +30,17 @@ class Dump(Handler):
dump configuration handler
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False
ALLOW_MULTI_ARCHITECTURE_RUN = False # conflicting io
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

View File

@ -25,7 +25,8 @@ from multiprocessing import Pool
from ahriman.application.lock import Lock
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import ExitCode, MissingArchitectureError, MultipleArchitecturesError
from ahriman.core.log import Log
from ahriman.core.log.log_loader import LogLoader
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
@ -34,7 +35,6 @@ class Handler:
base handler class for command callbacks
Attributes:
ALLOW_AUTO_ARCHITECTURE_RUN(bool): (class attribute) allow defining architecture from existing repositories
ALLOW_MULTI_ARCHITECTURE_RUN(bool): (class attribute) allow running with multiple architectures
Examples:
@ -46,60 +46,28 @@ class Handler:
>>> Add.execute(args)
"""
ALLOW_AUTO_ARCHITECTURE_RUN = True
ALLOW_MULTI_ARCHITECTURE_RUN = True
@classmethod
def architectures_extract(cls, args: argparse.Namespace) -> list[str]:
"""
get known architectures
Args:
args(argparse.Namespace): command line args
Returns:
list[str]: list of architectures for which tree is created
Raises:
MissingArchitectureError: if no architecture set and automatic detection is not allowed or failed
"""
if not cls.ALLOW_AUTO_ARCHITECTURE_RUN and args.architecture is None:
# for some parsers (e.g. config) we need to run with specific architecture
# for those cases architecture must be set explicitly
raise MissingArchitectureError(args.command)
if args.architecture: # architecture is specified explicitly
return sorted(set(args.architecture))
configuration = Configuration()
configuration.load(args.configuration)
# wtf???
root = configuration.getpath("repository", "root") # pylint: disable=assignment-from-no-return
architectures = RepositoryPaths.known_architectures(root)
if not architectures: # well we did not find anything
raise MissingArchitectureError(args.command)
return sorted(architectures)
@classmethod
def call(cls, args: argparse.Namespace, architecture: str) -> bool:
def call(cls, args: argparse.Namespace, repository_id: RepositoryId) -> bool:
"""
additional function to wrap all calls for multiprocessing library
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
Returns:
bool: True on success, False otherwise
"""
try:
configuration = Configuration.from_path(args.configuration, architecture)
configuration = Configuration.from_path(args.configuration, repository_id)
log_handler = Log.handler(args.log_handler)
Log.load(configuration, log_handler, quiet=args.quiet, report=args.report)
log_handler = LogLoader.handler(args.log_handler)
LogLoader.load(configuration, log_handler, quiet=args.quiet, report=args.report)
with Lock(args, architecture, configuration):
cls.run(args, architecture, configuration, report=args.report)
with Lock(args, repository_id, configuration):
cls.run(args, repository_id, configuration, report=args.report)
return True
except ExitCode:
@ -123,28 +91,82 @@ class Handler:
Raises:
MultipleArchitecturesError: if more than one architecture supplied and no multi architecture supported
"""
architectures = cls.architectures_extract(args)
repositories = cls.repositories_extract(args)
# actually we do not have to spawn another process if it is single-process application, do we?
if len(architectures) > 1:
if len(repositories) > 1:
if not cls.ALLOW_MULTI_ARCHITECTURE_RUN:
raise MultipleArchitecturesError(args.command)
raise MultipleArchitecturesError(args.command, repositories)
with Pool(len(architectures)) as pool:
result = pool.starmap(cls.call, [(args, architecture) for architecture in architectures])
with Pool(len(repositories)) as pool:
result = pool.starmap(cls.call, [(args, repository_id) for repository_id in repositories])
else:
result = [cls.call(args, architectures.pop())]
result = [cls.call(args, repositories.pop())]
return 0 if all(result) else 1
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def repositories_extract(cls, args: argparse.Namespace) -> list[RepositoryId]:
"""
get known architectures
Args:
args(argparse.Namespace): command line args
Returns:
list[RepositoryId]: list of repository names and architectures for which tree is created
Raises:
MissingArchitectureError: if no architecture set and automatic detection is not allowed or failed
"""
configuration = Configuration()
configuration.load(args.configuration)
# pylint, wtf???
root = configuration.getpath("repository", "root") # pylint: disable=assignment-from-no-return
# preparse systemd repository-id argument
# we are using unescaped values, so / is not allowed here, because it is impossible to separate if from dashes
if args.repository_id is not None:
# repository parts is optional for backward compatibility
architecture, *repository_parts = args.repository_id.split("/")
args.architecture = [architecture]
if repository_parts:
args.repository = ["-".join(repository_parts)] # replace slash with dash
# extract repository names first
names = args.repository
if names is None: # try to read file system first
names = RepositoryPaths.known_repositories(root)
if not names: # try to read configuration now
names = [configuration.get("repository", "name")]
# extract architecture names
if (architectures := args.architecture) is not None:
repositories = set(
RepositoryId(architecture, name)
for name in names
for architecture in architectures
)
else: # try to read from file system
repositories = set(
RepositoryId(architecture, name)
for name in names
for architecture in RepositoryPaths.known_architectures(root, name)
)
if not repositories:
raise MissingArchitectureError(args.command)
return sorted(repositories)
@classmethod
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting

View File

@ -21,6 +21,7 @@ import argparse
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class Help(Handler):
@ -28,16 +29,17 @@ class Help(Handler):
help handler
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
ALLOW_MULTI_ARCHITECTURE_RUN = False # system-wide action
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class KeyImport(Handler):
@ -29,18 +30,19 @@ class KeyImport(Handler):
key import packages handler
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
ALLOW_MULTI_ARCHITECTURE_RUN = False # system-wide action
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
application.repository.sign.key_import(args.key_server, args.key)

View File

@ -30,6 +30,7 @@ from ahriman.core.formatters import PatchPrinter
from ahriman.models.action import Action
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_id import RepositoryId
class Patch(Handler):
@ -37,18 +38,21 @@ class Patch(Handler):
patch control handler
"""
ALLOW_MULTI_ARCHITECTURE_RUN = False # system-wide action
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
application.on_start()
match args.action:
@ -56,7 +60,7 @@ class Patch(Handler):
patch = Patch.patch_create_from_function(args.variable, args.patch)
Patch.patch_set_create(application, args.package, patch)
case Action.Update:
package_base, patch = Patch.patch_create_from_diff(args.package, architecture, args.track)
package_base, patch = Patch.patch_create_from_diff(args.package, repository_id.architecture, args.track)
Patch.patch_set_create(application, package_base, patch)
case Action.List:
Patch.patch_set_list(application, args.package, args.variable, args.exit_code)
@ -114,7 +118,7 @@ class Patch(Handler):
application.database.patches_insert(package_base, patch)
@staticmethod
def patch_set_list(application: Application, package_base: str | None, variables: list[str],
def patch_set_list(application: Application, package_base: str | None, variables: list[str] | None,
exit_code: bool) -> None:
"""
list patches available for the package base
@ -122,7 +126,7 @@ class Patch(Handler):
Args:
application(Application): application instance
package_base(str | None): package base
variables(list[str]): extract patches only for specified PKGBUILD variables
variables(list[str] | None): extract patches only for specified PKGBUILD variables
exit_code(bool): exit with error on empty search result
"""
patches = application.database.patches_list(package_base, variables)
@ -132,13 +136,13 @@ class Patch(Handler):
PatchPrinter(base, patch).print(verbose=True, separator=" = ")
@staticmethod
def patch_set_remove(application: Application, package_base: str, variables: list[str]) -> None:
def patch_set_remove(application: Application, package_base: str, variables: list[str] | None) -> None:
"""
remove patch set for the package base
Args:
application(Application): application instance
package_base(str): package base
variables(list[str]): remove patches only for specified PKGBUILD variables
variables(list[str] | None): remove patches only for specified PKGBUILD variables
"""
application.database.patches_remove(package_base, variables)

View File

@ -24,6 +24,7 @@ from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class Rebuild(Handler):
@ -32,21 +33,22 @@ class Rebuild(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
application.on_start()
packages = Rebuild.extract_packages(application, args.status, from_database=args.from_database)
updates = application.repository.packages_depend_on(packages, args.depends_on or None)
updates = application.repository.packages_depend_on(packages, args.depends_on)
Rebuild.check_if_empty(args.exit_code, not updates)
if args.dry_run:

View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class Remove(Handler):
@ -30,16 +31,17 @@ class Remove(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
application.on_start()
application.remove(args.package)

View File

@ -23,6 +23,7 @@ from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import StringPrinter
from ahriman.models.repository_id import RepositoryId
class RemoveUnknown(Handler):
@ -31,17 +32,18 @@ class RemoveUnknown(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
application.on_start()
unknown_packages = application.unknown()

View File

@ -23,6 +23,7 @@ from tarfile import TarFile
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class Restore(Handler):
@ -30,16 +31,17 @@ class Restore(Handler):
restore packages handler
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
ALLOW_MULTI_ARCHITECTURE_RUN = False # system-wide action
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

View File

@ -29,6 +29,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import OptionError
from ahriman.core.formatters import AurPrinter
from ahriman.models.aur_package import AURPackage
from ahriman.models.repository_id import RepositoryId
class Search(Handler):
@ -39,7 +40,7 @@ class Search(Handler):
SORT_FIELDS(set[str]): (class attribute) allowed fields to sort the package list
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
ALLOW_MULTI_ARCHITECTURE_RUN = False # system-wide action
SORT_FIELDS = {
field.name
for field in fields(AURPackage)
@ -47,17 +48,18 @@ class Search(Handler):
}
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
official_packages_list = Official.multisearch(*args.search, pacman=application.repository.pacman)
aur_packages_list = AUR.multisearch(*args.search, pacman=application.repository.pacman)

View File

@ -25,6 +25,7 @@ from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import UpdatePrinter
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class ServiceUpdates(Handler):
@ -32,20 +33,21 @@ class ServiceUpdates(Handler):
service updates handler
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
ALLOW_MULTI_ARCHITECTURE_RUN = False # system-wide action
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
remote = Package.from_aur("ahriman", application.repository.pacman, None)
_, release = remote.version.rsplit("-", 1) # we don't store pkgrel locally, so we just append it

View File

@ -25,6 +25,8 @@ from pwd import getpwuid
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import MissingArchitectureError
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
from ahriman.models.user import User
@ -39,88 +41,92 @@ class Setup(Handler):
SUDOERS_DIR_PATH(Path): (class attribute) path to sudoers.d includes directory
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False
ALLOW_MULTI_ARCHITECTURE_RUN = False # conflicting io
ARCHBUILD_COMMAND_PATH = Path("/usr") / "bin" / "archbuild"
MIRRORLIST_PATH = Path("/etc") / "pacman.d" / "mirrorlist"
SUDOERS_DIR_PATH = Path("/etc") / "sudoers.d"
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
Setup.configuration_create_ahriman(args, architecture, args.repository, configuration)
# special check for args to avoid auto definition for setup command
if args.architecture is None or args.repository is None:
raise MissingArchitectureError(args.command)
Setup.configuration_create_ahriman(args, repository_id, configuration)
configuration.reload()
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
Setup.configuration_create_makepkg(args.packager, args.makeflags_jobs, application.repository.paths)
Setup.executable_create(application.repository.paths, args.build_command, architecture)
Setup.executable_create(application.repository.paths, repository_id)
repository_server = f"file://{application.repository.paths.repository}" if args.server is None else args.server
Setup.configuration_create_devtools(args.build_command, architecture, args.from_configuration, args.mirror,
args.multilib, args.repository, repository_server)
Setup.configuration_create_sudo(application.repository.paths, args.build_command, architecture)
Setup.configuration_create_devtools(
repository_id, args.from_configuration, args.mirror, args.multilib, repository_server)
Setup.configuration_create_sudo(application.repository.paths, repository_id)
application.repository.repo.init()
# lazy database sync
application.repository.pacman.handle # pylint: disable=pointless-statement
@staticmethod
def build_command(root: Path, prefix: str, architecture: str) -> Path:
def build_command(root: Path, repository_id: RepositoryId) -> Path:
"""
generate build command name
Args:
root(Path): root directory for the build command (must be root of the repository)
prefix(str): command prefix in {prefix}-{architecture}-build
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
Returns:
Path: valid devtools command name
"""
return root / f"{prefix}-{architecture}-build"
return root / f"{repository_id.name}-{repository_id.architecture}-build"
@staticmethod
def configuration_create_ahriman(args: argparse.Namespace, architecture: str, repository: str,
def configuration_create_ahriman(args: argparse.Namespace, repository_id: RepositoryId,
root: Configuration) -> None:
"""
create service specific configuration
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository(str): repository name
repository_id(RepositoryId): repository unique identifier
root(Configuration): root configuration instance
"""
configuration = Configuration()
section = Configuration.section_name("build", architecture)
build_command = Setup.build_command(root.repository_paths.root, args.build_command, architecture)
section = Configuration.section_name("build", repository_id.name, repository_id.architecture)
build_command = Setup.build_command(root.repository_paths.root, repository_id)
configuration.set_option(section, "build_command", str(build_command))
configuration.set_option("repository", "name", repository)
configuration.set_option("repository", "name", repository_id.name) # backward compatibility for docker
if args.build_as_user is not None:
configuration.set_option(section, "makechrootpkg_flags", f"-U {args.build_as_user}")
section = Configuration.section_name("alpm", architecture)
section = Configuration.section_name("alpm", repository_id.name, repository_id.architecture)
if args.mirror is not None:
configuration.set_option(section, "mirror", args.mirror)
if not args.multilib:
repositories = filter(lambda r: r != "multilib", root.getlist("alpm", "repositories"))
configuration.set_option(section, "repositories", " ".join(repositories))
section = Configuration.section_name("sign", architecture)
section = Configuration.section_name("sign", repository_id.name, repository_id.architecture)
if args.sign_key is not None:
configuration.set_option(section, "target", " ".join([target.name.lower() for target in args.sign_target]))
sign_targets = args.sign_target or []
configuration.set_option(section, "target", " ".join([target.name.lower() for target in sign_targets]))
configuration.set_option(section, "key", args.sign_key)
section = Configuration.section_name("web", architecture)
section = Configuration.section_name("web", repository_id.name, repository_id.architecture)
if args.web_port is not None:
configuration.set_option(section, "port", str(args.web_port))
if args.web_unix_socket is not None:
@ -134,8 +140,8 @@ class Setup(Handler):
configuration.write(ahriman_configuration)
@staticmethod
def configuration_create_devtools(prefix: str, architecture: str, source: Path, mirror: str | None,
multilib: bool, repository: str, repository_server: str) -> None:
def configuration_create_devtools(repository_id: RepositoryId, source: Path, mirror: str | None,
multilib: bool, repository_server: str) -> None:
"""
create configuration for devtools based on ``source`` configuration
@ -143,12 +149,10 @@ class Setup(Handler):
devtools does not allow to specify the pacman configuration, thus we still have to use configuration in /usr
Args:
prefix(str): command prefix in {prefix}-{architecture}-build
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
source(Path): path to source configuration file
mirror(str | None): link to package server mirror
multilib(bool): add or do not multilib repository to the configuration
repository(str): repository name
repository_server(str): url of the repository
"""
# allow_no_value=True is required because pacman uses boolean configuration in which just keys present
@ -163,7 +167,7 @@ class Setup(Handler):
configuration.read(source)
# set our architecture now
configuration.set_option("options", "Architecture", architecture)
configuration.set_option("options", "Architecture", repository_id.architecture)
# add multilib
if multilib:
@ -178,10 +182,10 @@ class Setup(Handler):
configuration.set_option(section, "Server", mirror)
# add repository itself
configuration.set_option(repository, "SigLevel", "Never") # we don't care
configuration.set_option(repository, "Server", repository_server)
configuration.set_option(repository_id.name, "SigLevel", "Never") # we don't care
configuration.set_option(repository_id.name, "Server", repository_server)
target = source.parent / f"{prefix}-{architecture}.conf"
target = source.parent / f"{repository_id.name}-{repository_id.architecture}.conf"
with target.open("w") as devtools_configuration:
configuration.write(devtools_configuration)
@ -205,31 +209,29 @@ class Setup(Handler):
(home_dir / ".makepkg.conf").write_text(content, encoding="utf8")
@staticmethod
def configuration_create_sudo(paths: RepositoryPaths, prefix: str, architecture: str) -> None:
def configuration_create_sudo(paths: RepositoryPaths, repository_id: RepositoryId) -> None:
"""
create configuration to run build command with sudo without password
Args:
paths(RepositoryPaths): repository paths instance
prefix(str): command prefix in {prefix}-{architecture}-build
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
"""
command = Setup.build_command(paths.root, prefix, architecture)
sudoers_file = Setup.build_command(Setup.SUDOERS_DIR_PATH, prefix, architecture)
command = Setup.build_command(paths.root, repository_id)
sudoers_file = Setup.build_command(Setup.SUDOERS_DIR_PATH, repository_id)
sudoers_file.write_text(f"ahriman ALL=(ALL) NOPASSWD:SETENV: {command} *\n", encoding="utf8")
sudoers_file.chmod(0o400) # security!
@staticmethod
def executable_create(paths: RepositoryPaths, prefix: str, architecture: str) -> None:
def executable_create(paths: RepositoryPaths, repository_id: RepositoryId) -> None:
"""
create executable for the service
Args:
paths(RepositoryPaths): repository paths instance
prefix(str): command prefix in {prefix}-{architecture}-build
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
"""
command = Setup.build_command(paths.root, prefix, architecture)
command = Setup.build_command(paths.root, repository_id)
command.unlink(missing_ok=True)
command.symlink_to(Setup.ARCHBUILD_COMMAND_PATH)
paths.chown(command) # we would like to keep owner inside ahriman's home

View File

@ -26,6 +26,7 @@ from pathlib import Path
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import StringPrinter
from ahriman.models.repository_id import RepositoryId
class Shell(Handler):
@ -33,16 +34,17 @@ class Shell(Handler):
python shell handler
"""
ALLOW_MULTI_ARCHITECTURE_RUN = False
ALLOW_MULTI_ARCHITECTURE_RUN = False # conflicting io
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
@ -50,7 +52,13 @@ class Shell(Handler):
# licensed by https://creativecommons.org/licenses/by-sa/3.0
path = Path(sys.prefix) / "share" / "ahriman" / "templates" / "shell"
StringPrinter(path.read_text(encoding="utf8")).print(verbose=False)
local_variables = {"architecture": architecture, "configuration": configuration}
local_variables = {
"architecture": repository_id.architecture,
"configuration": configuration,
"repository_id": repository_id,
}
if args.code is None:
code.interact(local=local_variables)
else:

View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class Sign(Handler):
@ -30,14 +31,15 @@ class Sign(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
Application(architecture, configuration, report=report).sign(args.package)
Application(repository_id, configuration, report=report).sign(args.package)

View File

@ -27,6 +27,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.formatters import PackagePrinter, StatusPrinter
from ahriman.models.build_status import BuildStatus
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class Status(Handler):
@ -34,21 +35,22 @@ class Status(Handler):
package status handler
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False
ALLOW_MULTI_ARCHITECTURE_RUN = False # conflicting io
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
# we are using reporter here
client = Application(architecture, configuration, report=True).repository.reporter
client = Application(repository_id, configuration, report=True).repository.reporter
if args.ahriman:
service_status = client.status_get()
StatusPrinter(service_status.status).print(verbose=args.info)

View File

@ -23,6 +23,7 @@ from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.action import Action
from ahriman.models.repository_id import RepositoryId
class StatusUpdate(Handler):
@ -30,21 +31,22 @@ class StatusUpdate(Handler):
status update handler
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False
ALLOW_MULTI_ARCHITECTURE_RUN = False # conflicting io
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
# we are using reporter here
client = Application(architecture, configuration, report=True).repository.reporter
client = Application(repository_id, configuration, report=True).repository.reporter
match args.action:
case Action.Update if args.package:

View File

@ -24,6 +24,7 @@ from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import StringPrinter, TreePrinter
from ahriman.core.tree import Tree
from ahriman.models.repository_id import RepositoryId
class Structure(Handler):
@ -31,20 +32,21 @@ class Structure(Handler):
dump repository structure handler
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False
ALLOW_MULTI_ARCHITECTURE_RUN = False # conflicting io
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
partitions = Tree.partition(application.repository.packages(), count=args.partitions)
for partition_id, partition in enumerate(partitions):

View File

@ -0,0 +1,68 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import argparse
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
class TreeMigrate(Handler):
"""
tree migration handler
"""
@classmethod
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
current_tree = configuration.repository_paths
target_tree = RepositoryPaths(current_tree.root, current_tree.repository_id, _force_current_tree=True)
# create repository tree
target_tree.tree_create()
# perform migration
TreeMigrate.tree_move(current_tree, target_tree)
@staticmethod
def tree_move(from_tree: RepositoryPaths, to_tree: RepositoryPaths) -> None:
"""
move files between trees. Trees must be created in advance
Args:
from_tree(RepositoryPaths): old repository tree
to_tree(RepositoryPaths): new repository tree
"""
# we don't care about devtools chroot
for attribute in (
RepositoryPaths.packages,
RepositoryPaths.pacman,
RepositoryPaths.repository,
):
attribute.fget(from_tree).rename(attribute.fget(to_tree)) # type: ignore[attr-defined]

View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -31,19 +32,20 @@ class Triggers(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
if args.trigger:
loader = application.repository.triggers
loader.triggers = [loader.load_trigger(trigger, architecture, configuration) for trigger in args.trigger]
loader.triggers = [loader.load_trigger(trigger, repository_id, configuration) for trigger in args.trigger]
application.on_start()
application.on_result(Result())

View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import StringPrinter
from ahriman.models.repository_id import RepositoryId
class UnsafeCommands(Handler):
@ -29,16 +30,17 @@ class UnsafeCommands(Handler):
unsafe command help parser
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
ALLOW_MULTI_ARCHITECTURE_RUN = False # system-wide action
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

View File

@ -25,6 +25,7 @@ from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.packagers import Packagers
from ahriman.models.repository_id import RepositoryId
class Update(Handler):
@ -33,17 +34,18 @@ class Update(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report, refresh_pacman_database=args.refresh)
application = Application(repository_id, configuration, report=report, refresh_pacman_database=args.refresh)
application.on_start()
packages = application.updates(args.package, aur=args.aur, local=args.local, manual=args.manual, vcs=args.vcs)
Update.check_if_empty(args.exit_code, not packages)

View File

@ -26,6 +26,7 @@ from ahriman.core.database import SQLite
from ahriman.core.exceptions import PasswordError
from ahriman.core.formatters import UserPrinter
from ahriman.models.action import Action
from ahriman.models.repository_id import RepositoryId
from ahriman.models.user import User
@ -34,16 +35,17 @@ class Users(Handler):
user management handler
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
ALLOW_MULTI_ARCHITECTURE_RUN = False # system-wide action
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

View File

@ -29,6 +29,7 @@ from ahriman.core.configuration.validator import Validator
from ahriman.core.exceptions import ExtensionError
from ahriman.core.formatters import ValidationPrinter
from ahriman.core.triggers import TriggerLoader
from ahriman.models.repository_id import RepositoryId
class Validate(Handler):
@ -36,20 +37,21 @@ class Validate(Handler):
configuration validator handler
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False
ALLOW_MULTI_ARCHITECTURE_RUN = False # conflicting io
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
schema = Validate.schema(architecture, configuration)
schema = Validate.schema(repository_id, configuration)
validator = Validator(configuration=configuration, schema=schema)
if validator.validate(configuration.dump()):
@ -61,12 +63,12 @@ class Validate(Handler):
Validate.check_if_empty(args.exit_code, True)
@staticmethod
def schema(architecture: str, configuration: Configuration) -> ConfigurationSchema:
def schema(repository_id: RepositoryId, configuration: Configuration) -> ConfigurationSchema:
"""
get schema with triggers
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
Returns:
@ -85,12 +87,12 @@ class Validate(Handler):
continue
# default settings if any
for schema_name, schema in trigger_class.configuration_schema(architecture, None).items():
for schema_name, schema in trigger_class.configuration_schema(repository_id, None).items():
erased = Validate.schema_erase_required(copy.deepcopy(schema))
root[schema_name] = Validate.schema_merge(root.get(schema_name, {}), erased)
# settings according to enabled triggers
for schema_name, schema in trigger_class.configuration_schema(architecture, configuration).items():
for schema_name, schema in trigger_class.configuration_schema(repository_id, configuration).items():
root[schema_name] = Validate.schema_merge(root.get(schema_name, {}), copy.deepcopy(schema))
return root

View File

@ -28,6 +28,7 @@ from ahriman import __version__
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import VersionPrinter
from ahriman.models.repository_id import RepositoryId
class Versions(Handler):
@ -38,17 +39,18 @@ class Versions(Handler):
PEP423_PACKAGE_NAME(str): (class attribute) special regex for valid PEP423 package name
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
ALLOW_MULTI_ARCHITECTURE_RUN = False # system-wide action
PEP423_PACKAGE_NAME = re.compile(r"^[A-Za-z0-9._-]+")
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

View File

@ -24,6 +24,7 @@ from collections.abc import Generator
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.spawn import Spawn
from ahriman.models.repository_id import RepositoryId
class Web(Handler):
@ -31,28 +32,28 @@ class Web(Handler):
web server handler
"""
ALLOW_AUTO_ARCHITECTURE_RUN = False
ALLOW_MULTI_ARCHITECTURE_RUN = False # required to be able to spawn external processes
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
# we are using local import for optional dependencies
from ahriman.web.web import run_server, setup_service
spawner_args = Web.extract_arguments(args, architecture, configuration)
spawner = Spawn(args.parser(), architecture, list(spawner_args))
spawner_args = Web.extract_arguments(args, repository_id, configuration)
spawner = Spawn(args.parser(), repository_id, list(spawner_args))
spawner.start()
application = setup_service(architecture, configuration, spawner)
application = setup_service(repository_id, configuration, spawner)
run_server(application)
# terminate spawn process at the last
@ -60,21 +61,22 @@ class Web(Handler):
spawner.join()
@staticmethod
def extract_arguments(args: argparse.Namespace, architecture: str,
def extract_arguments(args: argparse.Namespace, repository_id: RepositoryId,
configuration: Configuration) -> Generator[str, None, None]:
"""
extract list of arguments used for current command, except for command specific ones
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
Returns:
Generator[str, None, None]: command line arguments which were used for this specific command
"""
# read architecture from the same argument list
yield from ["--architecture", architecture]
yield from ["--architecture", repository_id.architecture]
yield from ["--repository", repository_id.name]
# read configuration path from current settings
if (configuration_path := configuration.path) is not None:
yield from ["--configuration", str(configuration_path)]

View File

@ -30,6 +30,7 @@ from ahriman.core.log import LazyLogging
from ahriman.core.status.client import Client
from ahriman.core.util import check_user
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.repository_id import RepositoryId
from ahriman.models.waiter import Waiter
@ -50,26 +51,29 @@ class Lock(LazyLogging):
The common flow is to create instance in ``with`` block and handle exceptions after all::
>>> from ahriman.core.configuration import Configuration
>>> from ahriman.models.repository_id import RepositoryId
>>>
>>> configuration = Configuration()
>>> try:
>>> with Lock(args, "x86_64", configuration):
>>> with Lock(args, RepositoryId("x86_64", "aur-clone"), configuration):
>>> perform_actions()
>>> except Exception as exception:
>>> handle_exceptions(exception)
"""
def __init__(self, args: argparse.Namespace, architecture: str, configuration: Configuration) -> None:
def __init__(self, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
lock_suffix = f"{repository_id.name}_{repository_id.architecture}"
self.path: Path | None = \
args.lock.with_stem(f"{args.lock.stem}_{architecture}") if args.lock is not None else None
args.lock.with_stem(f"{args.lock.stem}_{lock_suffix}") if args.lock is not None else None
self.force: bool = args.force
self.unsafe: bool = args.unsafe
self.wait_timeout: int = args.wait_timeout

View File

@ -23,11 +23,13 @@ from collections.abc import Callable, Generator
from functools import cached_property
from pathlib import Path
from pyalpm import DB, Handle, Package, SIG_PACKAGE, error as PyalpmError # type: ignore[import]
from string import Template
from ahriman.core.configuration import Configuration
from ahriman.core.log import LazyLogging
from ahriman.core.util import trim_package
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
@ -36,18 +38,18 @@ class Pacman(LazyLogging):
alpm wrapper
"""
def __init__(self, architecture: str, configuration: Configuration, *,
def __init__(self, repository_id: RepositoryId, configuration: Configuration, *,
refresh_database: PacmanSynchronization) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
refresh_database(PacmanSynchronization): synchronize local cache to remote
"""
self.__create_handle_fn: Callable[[], Handle] = lambda: self.__create_handle(
architecture, configuration, refresh_database=refresh_database)
repository_id, configuration, refresh_database=refresh_database)
@cached_property
def handle(self) -> Handle:
@ -59,13 +61,13 @@ class Pacman(LazyLogging):
"""
return self.__create_handle_fn()
def __create_handle(self, architecture: str, configuration: Configuration, *,
def __create_handle(self, repository_id: RepositoryId, configuration: Configuration, *,
refresh_database: PacmanSynchronization) -> Handle:
"""
create lazy handle function
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
refresh_database(PacmanSynchronization): synchronize local cache to remote
@ -81,7 +83,7 @@ class Pacman(LazyLogging):
handle = Handle(str(root), str(database_path))
for repository in configuration.getlist("alpm", "repositories"):
database = self.database_init(handle, repository, mirror, architecture)
database = self.database_init(handle, repository, mirror, repository_id.architecture)
self.database_copy(handle, database, pacman_root, paths, use_ahriman_cache=use_ahriman_cache)
if use_ahriman_cache and refresh_database:
@ -113,6 +115,7 @@ class Pacman(LazyLogging):
dst = repository_database(pacman_db_path)
if dst.is_file():
return # file already exists, do not copy
dst.parent.mkdir(mode=0o755, exist_ok=True) # create sync directory if it doesn't exist
src = repository_database(pacman_root)
if not src.is_file():
self.logger.warning("repository %s is set to be used, however, no working copy was found", database.name)
@ -136,8 +139,14 @@ class Pacman(LazyLogging):
"""
self.logger.info("loading pacman database %s", repository)
database: DB = handle.register_syncdb(repository, SIG_PACKAGE)
# replace variables in mirror address
database.servers = [mirror.replace("$repo", repository).replace("$arch", architecture)]
variables = {
"arch": architecture,
"repo": repository,
}
database.servers = [Template(mirror).safe_substitute(variables)]
return database
def database_sync(self, handle: Handle, *, force: bool) -> None:

View File

@ -154,7 +154,7 @@ class Sources(LazyLogging):
shutil.copytree(cache_dir, sources_dir, dirs_exist_ok=True)
instance.fetch(sources_dir, package.remote)
patches.extend(instance.extend_architectures(sources_dir, paths.architecture))
patches.extend(instance.extend_architectures(sources_dir, paths.repository_id.architecture))
for patch in patches:
instance.patch_apply(sources_dir, patch)

View File

@ -27,6 +27,7 @@ from typing import Any, Self
from ahriman.core.configuration.shell_interpolator import ShellInterpolator
from ahriman.core.exceptions import InitializeError
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
@ -38,9 +39,9 @@ class Configuration(configparser.RawConfigParser):
ARCHITECTURE_SPECIFIC_SECTIONS(list[str]): (class attribute) known sections which can be architecture specific.
Required by dump and merging functions
SYSTEM_CONFIGURATION_PATH(Path): (class attribute) default system configuration path distributed by package
architecture(str | None): repository architecture
includes(list[Path]): list of includes which were read
path(Path | None): path to root configuration file
repository_id(RepositoryId | None): repository unique identifier
Examples:
Configuration class provides additional method in order to handle application configuration. Since this class is
@ -49,7 +50,7 @@ class Configuration(configparser.RawConfigParser):
>>> from pathlib import Path
>>>
>>> configuration = Configuration.from_path(Path("/etc/ahriman.ini"), "x86_64")
>>> configuration = Configuration.from_path(Path("/etc/ahriman.ini"), RepositoryId("x86_64", "aur-clone"))
>>> repository_name = configuration.get("repository", "name")
>>> makepkg_flags = configuration.getlist("build", "makepkg_flags")
@ -59,7 +60,7 @@ class Configuration(configparser.RawConfigParser):
In order to get current settings, the ``check_loaded`` method can be used. This method will raise an
``InitializeError`` in case if configuration was not yet loaded::
>>> path, architecture = configuration.check_loaded()
>>> path, repository_id = configuration.check_loaded()
"""
ARCHITECTURE_SPECIFIC_SECTIONS = ["alpm", "build", "sign", "web"]
@ -84,10 +85,21 @@ class Configuration(configparser.RawConfigParser):
}
)
self.architecture: str | None = None
self.repository_id: RepositoryId | None = None
self.path: Path | None = None
self.includes: list[Path] = []
@property
def architecture(self) -> str:
"""
repository architecture for backward compatibility
Returns:
str: repository architecture
"""
_, repository_id = self.check_loaded()
return repository_id.architecture
@property
def include(self) -> Path:
"""
@ -111,12 +123,13 @@ class Configuration(configparser.RawConfigParser):
@property
def repository_name(self) -> str:
"""
repository name as defined by configuration
repository name for backward compatibility
Returns:
str: repository name from configuration
str: repository name
"""
return self.get("repository", "name")
_, repository_id = self.check_loaded()
return repository_id.name
@property
def repository_paths(self) -> RepositoryPaths:
@ -126,39 +139,60 @@ class Configuration(configparser.RawConfigParser):
Returns:
RepositoryPaths: repository paths instance
"""
_, architecture = self.check_loaded()
return RepositoryPaths(self.getpath("repository", "root"), architecture)
_, repository_id = self.check_loaded()
return RepositoryPaths(self.getpath("repository", "root"), repository_id)
@classmethod
def from_path(cls, path: Path, architecture: str) -> Self:
def from_path(cls, path: Path, repository_id: RepositoryId) -> Self:
"""
constructor with full object initialization
Args:
path(Path): path to root configuration file
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
Returns:
Self: configuration instance
"""
configuration = cls()
configuration.load(path)
configuration.merge_sections(architecture)
configuration.merge_sections(repository_id)
return configuration
@staticmethod
def section_name(section: str, suffix: str) -> str:
def override_sections(section: str, repository_id: RepositoryId) -> list[str]:
"""
extract override sections
Args:
section(str): section name
repository_id(RepositoryId): repository unique identifier
Returns:
list[str]: architecture and repository specific sections in correct order
"""
# the valid order is global < per architecture < per repository < per repository and architecture
return [
Configuration.section_name(section, repository_id.architecture), # architecture specific override
Configuration.section_name(section, repository_id.name), # override with repository name
Configuration.section_name(section, repository_id.name, repository_id.architecture), # both
]
@staticmethod
def section_name(section: str, *suffixes: str | None) -> str:
"""
generate section name for sections which depends on context
Args:
section(str): section name
suffix(str): session suffix, e.g. repository architecture
*suffixes(str | None): session suffix, e.g. repository architecture
Returns:
str: correct section name for repository specific section
"""
return f"{section}:{suffix}"
for suffix in filter(bool, suffixes):
section = f"{section}:{suffix}"
return section
def _convert_path(self, value: str) -> Path:
"""
@ -175,19 +209,19 @@ class Configuration(configparser.RawConfigParser):
return path
return self.path.parent / path
def check_loaded(self) -> tuple[Path, str]:
def check_loaded(self) -> tuple[Path, RepositoryId]:
"""
check if service was actually loaded
Returns:
tuple[Path, str]: configuration root path and architecture if loaded
tuple[Path, RepositoryId]: configuration root path and architecture if loaded
Raises:
InitializeError: in case if architecture and/or path are not set
"""
if self.path is None or self.architecture is None:
raise InitializeError("Configuration path and/or architecture are not set")
return self.path, self.architecture
if self.path is None or self.repository_id is None:
raise InitializeError("Configuration path and/or repository id are not set")
return self.path, self.repository_id
def dump(self) -> dict[str, dict[str, str]]:
"""
@ -207,14 +241,14 @@ class Configuration(configparser.RawConfigParser):
def getpath(self, *args: Any, **kwargs: Any) -> Path: ... # type: ignore[empty-body]
def gettype(self, section: str, architecture: str, *, fallback: str | None = None) -> tuple[str, str]:
def gettype(self, section: str, repository_id: RepositoryId, *, fallback: str | None = None) -> tuple[str, str]:
"""
get type variable with fallback to old logic. Despite the fact that it has same semantics as other get* methods,
but it has different argument list
Args:
section(str): section name
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
fallback(str | None, optional): optional fallback type if any. If set, second element of the tuple will
be always set to this value (Default value = None)
@ -227,9 +261,9 @@ class Configuration(configparser.RawConfigParser):
if (group_type := self.get(section, "type", fallback=fallback)) is not None:
return section, group_type # new-style logic
# okay lets check for the section with architecture name
full_section = self.section_name(section, architecture)
if self.has_section(full_section):
return full_section, section
for specific in self.override_sections(section, repository_id):
if self.has_section(specific):
return specific, section
# okay lets just use section as type
if self.has_section(section):
return section, section
@ -262,23 +296,24 @@ class Configuration(configparser.RawConfigParser):
except (FileNotFoundError, configparser.NoOptionError, configparser.NoSectionError):
pass
def merge_sections(self, architecture: str) -> None:
def merge_sections(self, repository_id: RepositoryId) -> None:
"""
merge architecture specific sections into main configuration
merge architecture and repository specific sections into main configuration
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
"""
self.architecture = architecture
self.repository_id = repository_id
for section in self.ARCHITECTURE_SPECIFIC_SECTIONS:
# get overrides
specific = self.section_name(section, architecture)
if self.has_section(specific):
# if there is no such section it means that there is no overrides for this arch,
# but we anyway will have to delete sections for others architectures
for key, value in self[specific].items():
self.set_option(section, key, value)
# remove any arch specific section
for specific in self.override_sections(section, repository_id):
if self.has_section(specific):
# if there is no such section it means that there is no overrides for this arch,
# but we anyway will have to delete sections for others architectures
for key, value in self[specific].items():
self.set_option(section, key, value)
# remove any arch/repo specific section
for foreign in self.sections():
# we would like to use lambda filter here, but pylint is too dumb
if not foreign.startswith(f"{section}:"):
@ -289,11 +324,11 @@ class Configuration(configparser.RawConfigParser):
"""
reload configuration if possible or raise exception otherwise
"""
path, architecture = self.check_loaded()
path, repository_id = self.check_loaded()
for section in self.sections(): # clear current content
self.remove_section(section)
self.load(path)
self.merge_sections(architecture)
self.merge_sections(repository_id)
def set_option(self, section: str, option: str, value: str) -> None:
"""

View File

@ -187,7 +187,6 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"schema": {
"name": {
"type": "string",
"required": True,
},
"root": {
"type": "string",

View File

@ -71,7 +71,18 @@ def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> N
paths(RepositoryPaths): repository paths instance
"""
from ahriman.core.alpm.remote import AUR
from ahriman.core.database.operations import PackageOperations
from ahriman.models.package import Package
def get_packages() -> dict[str, Package]:
return {
row["package_base"]: Package(
base=row["package_base"],
version=row["version"],
remote=RemoteSource.from_json(row),
packages={},
packager=row.get("packager") or None,
) for row in connection.execute("""select * from package_bases""")
}
def insert_remote(base: str, remote: RemoteSource) -> None:
connection.execute(
@ -88,8 +99,7 @@ def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> N
}
)
packages = PackageOperations._packages_get_select_package_bases(connection)
for package_base, package in packages.items():
for package_base, package in get_packages().items():
local_cache = paths.cache_for(package_base)
if local_cache.exists() and not package.is_vcs:
continue # skip packages which are not VCS and with local cache

View File

@ -61,8 +61,8 @@ def migrate_package_depends(connection: Connection, configuration: Configuration
if not configuration.repository_paths.repository.is_dir():
return
_, architecture = configuration.check_loaded()
pacman = Pacman(architecture, configuration, refresh_database=PacmanSynchronization.Disabled)
_, repository_id = configuration.check_loaded()
pacman = Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled)
package_list = []
for full_path in filter(package_like, configuration.repository_paths.repository.iterdir()):

View File

@ -45,9 +45,9 @@ steps = [
)
""",
"""
insert into packages select * from packages_ where architecture is not null;
insert into packages select * from packages_ where architecture is not null
""",
"""
drop table packages_;
drop table packages_
""",
]

View File

@ -58,8 +58,8 @@ def migrate_package_check_depends(connection: Connection, configuration: Configu
if not configuration.repository_paths.repository.is_dir():
return
_, architecture = configuration.check_loaded()
pacman = Pacman(architecture, configuration, refresh_database=PacmanSynchronization.Disabled)
_, repository_id = configuration.check_loaded()
pacman = Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled)
package_list = []
for full_path in filter(package_like, configuration.repository_paths.repository.iterdir()):

View File

@ -64,8 +64,8 @@ def migrate_package_base_packager(connection: Connection, configuration: Configu
if not configuration.repository_paths.repository.is_dir():
return
_, architecture = configuration.check_loaded()
pacman = Pacman(architecture, configuration, refresh_database=PacmanSynchronization.Disabled)
_, repository_id = configuration.check_loaded()
pacman = Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled)
package_list = []
for full_path in filter(package_like, configuration.repository_paths.repository.iterdir()):

View File

@ -0,0 +1,211 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from ahriman.core.configuration import Configuration
__all__ = ["migrate_data", "steps"]
steps = [
# set correct types for schema
"""
alter table users rename to users_
""",
"""
create table users (
username text not null unique,
access text not null,
password text,
packager_id text,
key_id text
)
""",
"""
insert into users select * from users_
""",
"""
drop table users_
""",
# update base tables
# build_queue
"""
alter table build_queue add column repository text not null default ''
""",
"""
alter table build_queue rename to build_queue_
""",
"""
create table build_queue (
package_base text not null,
properties json not null,
repository text not null,
unique (package_base, repository)
)
""",
"""
insert into build_queue select * from build_queue_
""",
"""
drop table build_queue_
""",
# package_bases
"""
alter table package_bases add column repository text not null default ''
""",
"""
alter table package_bases rename to package_bases_
""",
"""
create table package_bases (
package_base text not null,
version text not null,
branch text,
git_url text,
path text,
web_url text,
source text,
packager text,
repository text not null,
unique (package_base, repository)
)
""",
"""
insert into package_bases select * from package_bases_
""",
"""
drop table package_bases_
""",
# package_statuses
"""
alter table package_statuses add column repository text not null default ''
""",
"""
alter table package_statuses rename to package_statuses_
""",
"""
create table package_statuses (
package_base text not null,
status text not null,
last_updated integer,
repository text not null,
unique (package_base, repository)
)
""",
"""
insert into package_statuses select * from package_statuses_
""",
"""
drop table package_statuses_
""",
# packages
"""
alter table packages add column repository text not null default ''
""",
"""
alter table packages rename to packages_
""",
"""
create table packages (
package text not null,
package_base text not null,
architecture text not null,
archive_size integer,
build_date integer,
depends json,
description text,
filename text,
"groups" json,
installed_size integer,
licenses json,
provides json,
url text,
make_depends json,
opt_depends json,
check_depends json,
repository text not null,
unique (package, architecture, repository)
)
""",
"""
insert into packages select * from packages_
""",
"""
drop table packages_
""",
# logs
"""
alter table logs add column repository text not null default ''
""",
"""
drop index logs_package_base_version
""",
"""
alter table logs rename to logs_
""",
"""
create table logs (
package_base text not null,
created real not null,
record text,
version text not null,
repository text not null
)
""",
"""
insert into logs select * from logs_
""",
"""
create index logs_package_base_version_repository
on logs (package_base, version, repository)
""",
"""
drop table logs_
""",
]
def migrate_data(connection: Connection, configuration: Configuration) -> None:
"""
perform data migration
Args:
connection(Connection): database connection
configuration(Configuration): configuration instance
"""
migrate_package_repository(connection, configuration)
def migrate_package_repository(connection: Connection, configuration: Configuration) -> None:
"""
update repository name from current settings
Args:
connection(Connection): database connection
configuration(Configuration): configuration instance
"""
_, repository_id = configuration.check_loaded()
connection.execute("""update build_queue set repository = :repository""", {"repository": repository_id.id})
connection.execute("""update package_bases set repository = :repository""", {"repository": repository_id.id})
connection.execute("""update package_statuses set repository = :repository""", {"repository": repository_id.id})
connection.execute("""update packages set repository = :repository""", {"repository": repository_id.id})
connection.execute("""update logs set repository = :repository""", {"repository": repository_id.id})

View File

@ -39,9 +39,12 @@ class BuildOperations(Operations):
connection.execute(
"""
delete from build_queue
where :package_base is null or package_base = :package_base
where (:package_base is null or package_base = :package_base) and repository = :repository
""",
{"package_base": package_base})
{
"package_base": package_base,
"repository": self.repository_id.id,
})
return self.with_connection(run, commit=True)
@ -55,7 +58,10 @@ class BuildOperations(Operations):
def run(connection: Connection) -> list[Package]:
return [
Package.from_json(row["properties"])
for row in connection.execute("""select * from build_queue""")
for row in connection.execute(
"""select properties from build_queue where repository = :repository""",
{"repository": self.repository_id.id}
)
]
return self.with_connection(run)
@ -71,12 +77,16 @@ class BuildOperations(Operations):
connection.execute(
"""
insert into build_queue
(package_base, properties)
(package_base, properties, repository)
values
(:package_base, :properties)
on conflict (package_base) do update set
(:package_base, :properties, :repository)
on conflict (package_base, repository) do update set
properties = :properties
""",
{"package_base": package.base, "properties": package.view()})
{
"package_base": package.base,
"properties": package.view(),
"repository": self.repository_id.id,
})
return self.with_connection(run, commit=True)

View File

@ -45,11 +45,13 @@ class LogsOperations(Operations):
(row["created"], row["record"])
for row in connection.execute(
"""
select created, record from logs where package_base = :package_base
select created, record from logs
where package_base = :package_base and repository = :repository
order by created limit :limit offset :offset
""",
{
"package_base": package_base,
"repository": self.repository_id.id,
"limit": limit,
"offset": offset,
})
@ -70,15 +72,16 @@ class LogsOperations(Operations):
connection.execute(
"""
insert into logs
(package_base, version, created, record)
(package_base, version, created, record, repository)
values
(:package_base, :version, :created, :record)
(:package_base, :version, :created, :record, :repository)
""",
{
"package_base": log_record_id.package_base,
"version": log_record_id.version,
"created": created,
"record": record,
"repository": self.repository_id.id,
}
)
@ -97,9 +100,15 @@ class LogsOperations(Operations):
connection.execute(
"""
delete from logs
where package_base = :package_base and (:version is null or version <> :version)
where package_base = :package_base
and repository = :repository
and (:version is null or version <> :version)
""",
{"package_base": package_base, "version": version}
{
"package_base": package_base,
"version": version,
"repository": self.repository_id.id,
}
)
return self.with_connection(run, commit=True)

View File

@ -24,6 +24,7 @@ from pathlib import Path
from typing import Any, TypeVar
from ahriman.core.log import LazyLogging
from ahriman.models.repository_id import RepositoryId
T = TypeVar("T")
@ -35,16 +36,19 @@ class Operations(LazyLogging):
Attributes:
path(Path): path to the database file
repository_id(RepositoryId): repository unique identifier to perform implicit filtering
"""
def __init__(self, path: Path) -> None:
def __init__(self, path: Path, repository_id: RepositoryId) -> None:
"""
default constructor
Args:
path(Path): path to the database file
repository_id(RepositoryId): repository unique identifier
"""
self.path = path
self.repository_id = repository_id
@staticmethod
def factory(cursor: sqlite3.Cursor, row: tuple[Any, ...]) -> dict[str, Any]:

View File

@ -32,8 +32,7 @@ class PackageOperations(Operations):
package operations
"""
@staticmethod
def _package_remove_package_base(connection: Connection, package_base: str) -> None:
def _package_remove_package_base(self, connection: Connection, package_base: str) -> None:
"""
remove package base information
@ -41,13 +40,15 @@ class PackageOperations(Operations):
connection(Connection): database connection
package_base(str): package base name
"""
connection.execute("""delete from package_statuses where package_base = :package_base""",
{"package_base": package_base})
connection.execute("""delete from package_bases where package_base = :package_base""",
{"package_base": package_base})
connection.execute(
"""delete from package_statuses where package_base = :package_base and repository = :repository""",
{"package_base": package_base, "repository": self.repository_id.id})
connection.execute(
"""delete from package_bases where package_base = :package_base and repository = :repository""",
{"package_base": package_base, "repository": self.repository_id.id})
@staticmethod
def _package_remove_packages(connection: Connection, package_base: str, current_packages: Iterable[str]) -> None:
def _package_remove_packages(self, connection: Connection, package_base: str,
current_packages: Iterable[str]) -> None:
"""
remove packages belong to the package base
@ -59,13 +60,17 @@ class PackageOperations(Operations):
packages = [
package
for package in connection.execute(
"""select package from packages where package_base = :package_base""", {"package_base": package_base})
"""
select package, repository from packages
where package_base = :package_base and repository = :repository""",
{"package_base": package_base, "repository": self.repository_id.id})
if package["package"] not in current_packages
]
connection.executemany("""delete from packages where package = :package""", packages)
connection.executemany(
"""delete from packages where package = :package and repository = :repository""",
packages)
@staticmethod
def _package_update_insert_base(connection: Connection, package: Package) -> None:
def _package_update_insert_base(self, connection: Connection, package: Package) -> None:
"""
insert base package into table
@ -76,10 +81,10 @@ class PackageOperations(Operations):
connection.execute(
"""
insert into package_bases
(package_base, version, source, branch, git_url, path, web_url, packager)
(package_base, version, source, branch, git_url, path, web_url, packager, repository)
values
(:package_base, :version, :source, :branch, :git_url, :path, :web_url, :packager)
on conflict (package_base) do update set
(:package_base, :version, :source, :branch, :git_url, :path, :web_url, :packager, :repository)
on conflict (package_base, repository) do update set
version = :version, branch = :branch, git_url = :git_url, path = :path, web_url = :web_url,
source = :source, packager = :packager
""",
@ -92,11 +97,11 @@ class PackageOperations(Operations):
"web_url": package.remote.web_url,
"source": package.remote.source.value,
"packager": package.packager,
"repository": self.repository_id.id,
}
)
@staticmethod
def _package_update_insert_packages(connection: Connection, package: Package) -> None:
def _package_update_insert_packages(self, connection: Connection, package: Package) -> None:
"""
insert packages into table
@ -108,20 +113,27 @@ class PackageOperations(Operations):
for name, description in package.packages.items():
if description.architecture is None:
continue # architecture is required
package_list.append({"package": name, "package_base": package.base, **description.view()})
package_list.append({
"package": name,
"package_base": package.base,
"repository": self.repository_id.id,
**description.view(),
})
connection.executemany(
"""
insert into packages
(package, package_base, architecture, archive_size,
build_date, depends, description, filename,
"groups", installed_size, licenses, provides,
url, make_depends, opt_depends, check_depends)
url, make_depends, opt_depends, check_depends,
repository)
values
(:package, :package_base, :architecture, :archive_size,
:build_date, :depends, :description, :filename,
:groups, :installed_size, :licenses, :provides,
:url, :make_depends, :opt_depends, :check_depends)
on conflict (package, architecture) do update set
:url, :make_depends, :opt_depends, :check_depends,
:repository)
on conflict (package, architecture, repository) do update set
package_base = :package_base, archive_size = :archive_size,
build_date = :build_date, depends = :depends, description = :description, filename = :filename,
"groups" = :groups, installed_size = :installed_size, licenses = :licenses, provides = :provides,
@ -129,8 +141,7 @@ class PackageOperations(Operations):
""",
package_list)
@staticmethod
def _package_update_insert_status(connection: Connection, package_base: str, status: BuildStatus) -> None:
def _package_update_insert_status(self, connection: Connection, package_base: str, status: BuildStatus) -> None:
"""
insert base package status into table
@ -141,16 +152,21 @@ class PackageOperations(Operations):
"""
connection.execute(
"""
insert into package_statuses (package_base, status, last_updated)
insert into package_statuses
(package_base, status, last_updated, repository)
values
(:package_base, :status, :last_updated)
on conflict (package_base) do update set
(:package_base, :status, :last_updated, :repository)
on conflict (package_base, repository) do update set
status = :status, last_updated = :last_updated
""",
{"package_base": package_base, "status": status.status.value, "last_updated": status.timestamp})
{
"package_base": package_base,
"status": status.status.value,
"last_updated": status.timestamp,
"repository": self.repository_id.id,
})
@staticmethod
def _packages_get_select_package_bases(connection: Connection) -> dict[str, Package]:
def _packages_get_select_package_bases(self, connection: Connection) -> dict[str, Package]:
"""
select package bases from the table
@ -167,11 +183,13 @@ class PackageOperations(Operations):
remote=RemoteSource.from_json(row),
packages={},
packager=row["packager"] or None,
) for row in connection.execute("""select * from package_bases""")
) for row in connection.execute(
"""select * from package_bases where repository = :repository""",
{"repository": self.repository_id.id}
)
}
@staticmethod
def _packages_get_select_packages(connection: Connection, packages: dict[str, Package]) -> dict[str, Package]:
def _packages_get_select_packages(self, connection: Connection, packages: dict[str, Package]) -> dict[str, Package]:
"""
select packages from the table
@ -182,14 +200,16 @@ class PackageOperations(Operations):
Returns:
dict[str, Package]: map of the package base to its descriptor including individual packages
"""
for row in connection.execute("""select * from packages"""):
for row in connection.execute(
"""select * from packages where repository = :repository""",
{"repository": self.repository_id.id}
):
if row["package_base"] not in packages:
continue # normally must never happen though
packages[row["package_base"]].packages[row["package"]] = PackageDescription.from_json(row)
return packages
@staticmethod
def _packages_get_select_statuses(connection: Connection) -> dict[str, BuildStatus]:
def _packages_get_select_statuses(self, connection: Connection) -> dict[str, BuildStatus]:
"""
select package build statuses from the table
@ -201,7 +221,10 @@ class PackageOperations(Operations):
"""
return {
row["package_base"]: BuildStatus.from_json({"status": row["status"], "timestamp": row["last_updated"]})
for row in connection.execute("""select * from package_statuses""")
for row in connection.execute(
"""select * from package_statuses where repository = :repository""",
{"repository": self.repository_id.id}
)
}
def package_remove(self, package_base: str) -> None:

View File

@ -40,7 +40,7 @@ class PatchOperations(Operations):
Returns:
list[PkgbuildPatch]: plain text patch for the package
"""
return self.patches_list(package_base, []).get(package_base, [])
return self.patches_list(package_base, None).get(package_base, [])
def patches_insert(self, package_base: str, patch: PkgbuildPatch) -> None:
"""
@ -64,13 +64,13 @@ class PatchOperations(Operations):
return self.with_connection(run, commit=True)
def patches_list(self, package_base: str | None, variables: list[str]) -> dict[str, list[PkgbuildPatch]]:
def patches_list(self, package_base: str | None, variables: list[str] | None) -> dict[str, list[PkgbuildPatch]]:
"""
extract all patches
Args:
package_base(str | None): optional filter by package base
variables(list[str]): extract patches only for specified PKGBUILD variables
variables(list[str] | None): extract patches only for specified PKGBUILD variables
Returns:
dict[str, list[PkgbuildPatch]]: map of package base to patch content
@ -86,29 +86,30 @@ class PatchOperations(Operations):
# we could use itertools & operator but why?
patches: dict[str, list[PkgbuildPatch]] = defaultdict(list)
for package, patch in self.with_connection(run):
if variables and patch.key not in variables:
if variables is not None and patch.key not in variables:
continue
patches[package].append(patch)
return dict(patches)
def patches_remove(self, package_base: str, variables: list[str]) -> None:
def patches_remove(self, package_base: str, variables: list[str] | None) -> None:
"""
remove patch set
Args:
package_base(str): package base to clear patches
variables(list[str]): remove patches only for specified PKGBUILD variables
variables(list[str] | None): remove patches only for specified PKGBUILD variables
"""
def run_many(connection: Connection) -> None:
patches = variables or [] # suppress mypy warning
connection.executemany(
"""delete from patches where package_base = :package_base and variable = :variable""",
[{"package_base": package_base, "variable": variable} for variable in variables])
[{"package_base": package_base, "variable": variable} for variable in patches])
def run(connection: Connection) -> None:
connection.execute(
"""delete from patches where package_base = :package_base""",
{"package_base": package_base})
if variables:
if variables is not None:
return self.with_connection(run_many, commit=True)
return self.with_connection(run, commit=True)

View File

@ -56,8 +56,11 @@ class SQLite(AuthOperations, BuildOperations, LogsOperations, PackageOperations,
Self: fully initialized instance of the database
"""
path = cls.database_path(configuration)
database = cls(path)
_, repository_id = configuration.check_loaded()
database = cls(path, repository_id)
database.init(configuration)
return database
@staticmethod

View File

@ -23,6 +23,8 @@ from collections.abc import Callable
from pathlib import Path
from typing import Any, Self
from ahriman.models.repository_id import RepositoryId
class BuildError(RuntimeError):
"""
@ -165,7 +167,7 @@ class MissingArchitectureError(ValueError):
Args:
command(str): command name which throws exception
"""
ValueError.__init__(self, f"Architecture required for subcommand {command}, but missing")
ValueError.__init__(self, f"Architecture/repository required for subcommand {command}, but missing")
class MultipleArchitecturesError(ValueError):
@ -173,14 +175,18 @@ class MultipleArchitecturesError(ValueError):
exception which will be raised if multiple architectures are not supported by the handler
"""
def __init__(self, command: str) -> None:
def __init__(self, command: str, repositories: list[RepositoryId] | None = None) -> None:
"""
default constructor
Args:
command(str): command name which throws exception
repositories(list[RepositoryId] | None, optional): found repository list (Default value = None)
"""
ValueError.__init__(self, f"Multiple architectures are not supported by subcommand {command}")
message = f"Multiple architectures/repositories are not supported by subcommand {command}"
if repositories is not None:
message += f", got {repositories}"
ValueError.__init__(self, message)
class OptionError(ValueError):

View File

@ -30,6 +30,7 @@ from ahriman.core.util import walk
from ahriman.models.package import Package
from ahriman.models.package_source import PackageSource
from ahriman.models.remote_source import RemoteSource
from ahriman.models.repository_id import RepositoryId
class RemotePull(LazyLogging):
@ -42,13 +43,13 @@ class RemotePull(LazyLogging):
repository_paths(RepositoryPaths): repository paths instance
"""
def __init__(self, configuration: Configuration, architecture: str, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
architecture(str): repository architecture
section(str): settings section name
"""
self.remote_source = RemoteSource(
@ -58,7 +59,7 @@ class RemotePull(LazyLogging):
branch=configuration.get(section, "pull_branch", fallback="master"),
source=PackageSource.Local,
)
self.architecture = architecture
self.architecture = repository_id.architecture
self.repository_paths = configuration.repository_paths
def package_copy(self, pkgbuild_path: Path) -> None:

View File

@ -20,6 +20,7 @@
from ahriman.core.configuration import Configuration
from ahriman.core.gitremote.remote_pull import RemotePull
from ahriman.core.triggers import Trigger
from ahriman.models.repository_id import RepositoryId
class RemotePullTrigger(Trigger):
@ -56,15 +57,15 @@ class RemotePullTrigger(Trigger):
}
CONFIGURATION_SCHEMA_FALLBACK = "gitremote"
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, architecture, configuration)
Trigger.__init__(self, repository_id, configuration)
self.targets = self.configuration_sections(configuration)
@classmethod
@ -86,6 +87,6 @@ class RemotePullTrigger(Trigger):
"""
for target in self.targets:
section, _ = self.configuration.gettype(
target, self.architecture, fallback=self.CONFIGURATION_SCHEMA_FALLBACK)
runner = RemotePull(self.configuration, self.architecture, section)
target, self.repository_id, fallback=self.CONFIGURATION_SCHEMA_FALLBACK)
runner = RemotePull(self.repository_id, self.configuration, section)
runner.run()

View File

@ -24,6 +24,7 @@ from ahriman.core.gitremote.remote_push import RemotePush
from ahriman.core.triggers import Trigger
from ahriman.models.context_key import ContextKey
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -67,15 +68,15 @@ class RemotePushTrigger(Trigger):
}
CONFIGURATION_SCHEMA_FALLBACK = "gitremote"
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, architecture, configuration)
Trigger.__init__(self, repository_id, configuration)
self.targets = self.configuration_sections(configuration)
@classmethod
@ -107,6 +108,6 @@ class RemotePushTrigger(Trigger):
for target in self.targets:
section, _ = self.configuration.gettype(
target, self.architecture, fallback=self.CONFIGURATION_SCHEMA_FALLBACK)
target, self.repository_id, fallback=self.CONFIGURATION_SCHEMA_FALLBACK)
runner = RemotePush(database, self.configuration, section)
runner.run(result)

View File

@ -41,14 +41,14 @@ class SyncHttpClient(LazyLogging):
timeout(int): HTTP request timeout in seconds
"""
def __init__(self, section: str | None = None, configuration: Configuration | None = None, *,
def __init__(self, configuration: Configuration | None = None, section: str | None = None, *,
suppress_errors: bool = False) -> None:
"""
default constructor
Args:
section(str, optional): settings section name (Default value = None)
configuration(Configuration | None): configuration instance (Default value = None)
section(str, optional): settings section name (Default value = None)
suppress_errors(bool, optional): suppress logging of request errors (Default value = False)
"""
if configuration is None:

View File

@ -18,4 +18,3 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.log.lazy_logging import LazyLogging
from ahriman.core.log.log import Log

View File

@ -27,7 +27,7 @@ from ahriman.core.log.http_log_handler import HttpLogHandler
from ahriman.models.log_handler import LogHandler
class Log:
class LogLoader:
"""
simple static method class which setups application loggers
@ -63,7 +63,7 @@ class Log:
del JournalHandler
return LogHandler.Journald # journald import was found
except ImportError:
if Log.DEFAULT_SYSLOG_DEVICE.exists():
if LogLoader.DEFAULT_SYSLOG_DEVICE.exists():
return LogHandler.Syslog
return LogHandler.Console
@ -94,8 +94,7 @@ class Log:
fileConfig(log_configuration, disable_existing_loggers=True)
logging.debug("using %s logger", default_handler)
except Exception:
logging.basicConfig(filename=None, format=Log.DEFAULT_LOG_FORMAT,
level=Log.DEFAULT_LOG_LEVEL)
logging.basicConfig(filename=None, format=LogLoader.DEFAULT_LOG_FORMAT, level=LogLoader.DEFAULT_LOG_LEVEL)
logging.exception("could not load logging from configuration, fallback to stderr")
HttpLogHandler.load(configuration, report=report)

View File

@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.formatters import BuildPrinter
from ahriman.core.report.report import Report
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -32,16 +33,16 @@ class Console(Report):
use_utf(bool): print utf8 symbols instead of ASCII
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Report.__init__(self, architecture, configuration)
Report.__init__(self, repository_id, configuration)
self.use_utf = configuration.getboolean(section, "use_utf", fallback=True)
def generate(self, packages: list[Package], result: Result) -> None:

View File

@ -27,6 +27,7 @@ from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.report.report import Report
from ahriman.core.util import pretty_datetime, utcnow
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
from ahriman.models.smtp_ssl_settings import SmtpSSLSettings
@ -48,17 +49,17 @@ class Email(Report, JinjaTemplate):
user(str | None): username to authenticate via SMTP
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Report.__init__(self, architecture, configuration)
JinjaTemplate.__init__(self, section, configuration)
Report.__init__(self, repository_id, configuration)
JinjaTemplate.__init__(self, repository_id, configuration, section)
self.full_template_path = configuration.getpath(section, "full_template_path", fallback=None)
self.template_path = configuration.getpath(section, "template_path")

View File

@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.report.report import Report
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -33,17 +34,17 @@ class HTML(Report, JinjaTemplate):
template_path(Path): path to template for full package list
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Report.__init__(self, architecture, configuration)
JinjaTemplate.__init__(self, section, configuration)
Report.__init__(self, repository_id, configuration)
JinjaTemplate.__init__(self, repository_id, configuration, section)
self.report_path = configuration.getpath(section, "path")
self.template_path = configuration.getpath(section, "template_path")

View File

@ -25,6 +25,7 @@ from pathlib import Path
from ahriman.core.configuration import Configuration
from ahriman.core.sign.gpg import GPG
from ahriman.core.util import pretty_datetime, pretty_size
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
from ahriman.models.sign_settings import SignSettings
@ -63,19 +64,20 @@ class JinjaTemplate:
sign_targets(set[SignSettings]): targets to sign enabled in configuration
"""
def __init__(self, section: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
section(str): settings section name
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
self.link_path = configuration.get(section, "link_path")
# base template vars
self.homepage = configuration.get(section, "homepage", fallback=None)
self.name = configuration.repository_name
self.name = repository_id.name
self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration)

View File

@ -23,6 +23,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.report.report import Report
from ahriman.core.status.web_client import WebClient
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
from ahriman.models.waiter import Waiter
@ -39,16 +40,16 @@ class RemoteCall(Report):
wait_timeout(int): timeout to wait external process
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Report.__init__(self, architecture, configuration)
Report.__init__(self, repository_id, configuration)
self.client = WebClient(configuration)

View File

@ -24,6 +24,7 @@ from ahriman.core.exceptions import ReportError
from ahriman.core.log import LazyLogging
from ahriman.models.package import Package
from ahriman.models.report_settings import ReportSettings
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -32,17 +33,15 @@ class Report(LazyLogging):
base report generator
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
repository_id(RepositoryId): repository unique identifier
Examples:
``Report`` classes provide several method in order to operate with the report generation and additional class
method ``load`` which can be used in order to determine right report instance::
>>> from ahriman.core.configuration import Configuration
>>>
>>> configuration = Configuration()
>>> report = Report.load("x86_64", configuration, "email")
>>> report = Report.load(RepositoryId("x86_64", "aur-clone"), configuration, "email")
The ``generate`` method can be used in order to perform the report itself, whereas ``run`` method handles
exception and raises ``ReportFailed`` instead::
@ -55,49 +54,49 @@ class Report(LazyLogging):
>>> report.run(Result(), [])
"""
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
self.architecture = architecture
self.repository_id = repository_id
self.configuration = configuration
@staticmethod
def load(architecture: str, configuration: Configuration, target: str) -> Report:
def load(repository_id: RepositoryId, configuration: Configuration, target: str) -> Report:
"""
load client from settings
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
target(str): target to generate report aka section name (e.g. html)
Returns:
Report: client according to current settings
"""
section, provider_name = configuration.gettype(target, architecture)
section, provider_name = configuration.gettype(target, repository_id)
match ReportSettings.from_option(provider_name):
case ReportSettings.HTML:
from ahriman.core.report.html import HTML
return HTML(architecture, configuration, section)
return HTML(repository_id, configuration, section)
case ReportSettings.Email:
from ahriman.core.report.email import Email
return Email(architecture, configuration, section)
return Email(repository_id, configuration, section)
case ReportSettings.Console:
from ahriman.core.report.console import Console
return Console(architecture, configuration, section)
return Console(repository_id, configuration, section)
case ReportSettings.Telegram:
from ahriman.core.report.telegram import Telegram
return Telegram(architecture, configuration, section)
return Telegram(repository_id, configuration, section)
case ReportSettings.RemoteCall:
from ahriman.core.report.remote_call import RemoteCall
return RemoteCall(architecture, configuration, section)
return RemoteCall(repository_id, configuration, section)
case _:
return Report(architecture, configuration) # should never happen
return Report(repository_id, configuration) # should never happen
def generate(self, packages: list[Package], result: Result) -> None:
"""

View File

@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.triggers import Trigger
from ahriman.core.report.report import Report
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -218,15 +219,15 @@ class ReportTrigger(Trigger):
}
}
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, architecture, configuration)
Trigger.__init__(self, repository_id, configuration)
self.targets = self.configuration_sections(configuration)
@classmethod
@ -251,5 +252,5 @@ class ReportTrigger(Trigger):
packages(list[Package]): list of all available packages
"""
for target in self.targets:
runner = Report.load(self.architecture, self.configuration, target)
runner = Report.load(self.repository_id, self.configuration, target)
runner.run(result, packages)

View File

@ -22,6 +22,7 @@ from ahriman.core.http import SyncHttpClient
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.report.report import Report
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -41,18 +42,18 @@ class Telegram(Report, JinjaTemplate, SyncHttpClient):
TELEGRAM_API_URL = "https://api.telegram.org"
TELEGRAM_MAX_CONTENT_LENGTH = 4096
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Report.__init__(self, architecture, configuration)
JinjaTemplate.__init__(self, section, configuration)
SyncHttpClient.__init__(self, section, configuration)
Report.__init__(self, repository_id, configuration)
JinjaTemplate.__init__(self, repository_id, configuration, section)
SyncHttpClient.__init__(self, configuration, section)
self.api_key = configuration.get(section, "api_key")
self.chat_id = configuration.get(section, "chat_id")

View File

@ -32,6 +32,7 @@ from ahriman.core.util import package_like
from ahriman.models.context_key import ContextKey
from ahriman.models.package import Package
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.repository_id import RepositoryId
class Repository(Executor, UpdateHandler):
@ -47,7 +48,7 @@ class Repository(Executor, UpdateHandler):
>>>
>>> configuration = Configuration()
>>> database = SQLite.load(configuration)
>>> repository = Repository.load("x86_64", configuration, database, report=True)
>>> repository = Repository.load(RepositoryId("x86_64", "x86_64"), configuration, database, report=True)
>>> known_packages = repository.packages()
>>>
>>> build_result = repository.process_build(known_packages)
@ -58,13 +59,13 @@ class Repository(Executor, UpdateHandler):
"""
@classmethod
def load(cls, architecture: str, configuration: Configuration, database: SQLite, *, report: bool,
def load(cls, repository_id: RepositoryId, configuration: Configuration, database: SQLite, *, report: bool,
refresh_pacman_database: PacmanSynchronization = PacmanSynchronization.Disabled) -> Self:
"""
load instance from argument list
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
database(SQLite): database instance
report(bool): force enable or disable reporting
@ -74,7 +75,7 @@ class Repository(Executor, UpdateHandler):
Returns:
Self: fully loaded repository class instance
"""
instance = cls(architecture, configuration, database,
instance = cls(repository_id, configuration, database,
report=report, refresh_pacman_database=refresh_pacman_database)
instance._set_context()
return instance

View File

@ -27,6 +27,7 @@ from ahriman.core.status.client import Client
from ahriman.core.triggers import TriggerLoader
from ahriman.models.packagers import Packagers
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
from ahriman.models.user import User
from ahriman.models.user_access import UserAccess
@ -37,47 +38,65 @@ class RepositoryProperties(LazyLogging):
repository internal objects holder
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
database(SQLite): database instance
ignore_list(list[str]): package bases which will be ignored during auto updates
name(str): repository name
pacman(Pacman): alpm wrapper instance
paths(RepositoryPaths): repository paths instance
repo(Repo): repo commands wrapper instance
reporter(Client): build status reporter instance
repository_id(RepositoryId): repository unique identifier
sign(GPG): GPG wrapper instance
triggers(TriggerLoader): triggers holder
vcs_allowed_age(int): maximal age of the VCS packages before they will be checked
"""
def __init__(self, architecture: str, configuration: Configuration, database: SQLite, *, report: bool,
def __init__(self, repository_id: RepositoryId, configuration: Configuration, database: SQLite, *, report: bool,
refresh_pacman_database: PacmanSynchronization) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
database(SQLite): database instance
report(bool): force enable or disable reporting
refresh_pacman_database(PacmanSynchronization): pacman database synchronization level
"""
self.architecture = architecture
self.repository_id = repository_id
self.configuration = configuration
self.database = database
self.name = configuration.repository_name
self.vcs_allowed_age = configuration.getint("build", "vcs_allowed_age", fallback=0)
self.paths: RepositoryPaths = configuration.repository_paths # additional workaround for pycharm typing
self.ignore_list = configuration.getlist("build", "ignore_packages", fallback=[])
self.pacman = Pacman(architecture, configuration, refresh_database=refresh_pacman_database)
self.pacman = Pacman(repository_id, configuration, refresh_database=refresh_pacman_database)
self.sign = GPG(configuration)
self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
self.reporter = Client.load(configuration, report=report)
self.triggers = TriggerLoader.load(architecture, configuration)
self.triggers = TriggerLoader.load(repository_id, configuration)
@property
def architecture(self) -> str:
"""
repository architecture for backward compatibility
Returns:
str: repository architecture
"""
return self.repository_id.architecture
@property
def name(self) -> str:
"""
repository name for backward compatibility
Returns:
str: repository name
"""
return self.repository_id.name
def packager(self, packagers: Packagers, package_base: str) -> User:
"""

View File

@ -28,6 +28,7 @@ from multiprocessing import Process, Queue
from threading import Lock, Thread
from ahriman.core.log import LazyLogging
from ahriman.models.repository_id import RepositoryId
class Spawn(Thread, LazyLogging):
@ -37,22 +38,23 @@ class Spawn(Thread, LazyLogging):
Attributes:
active(dict[str, Process]): map of active child processes required to avoid zombies
architecture(str): repository architecture
command_arguments(list[str]): base command line arguments
queue(Queue[tuple[str, bool, int]]): multiprocessing queue to read updates from processes
repository_id(RepositoryId): repository unique identifier
"""
def __init__(self, args_parser: argparse.ArgumentParser, architecture: str, command_arguments: list[str]) -> None:
def __init__(self, args_parser: argparse.ArgumentParser, repository_id: RepositoryId,
command_arguments: list[str]) -> None:
"""
default constructor
Args:
args_parser(argparse.ArgumentParser): command line parser for the application
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
command_arguments(list[str]): base command line arguments
"""
Thread.__init__(self, name="spawn")
self.architecture = architecture
self.repository_id = repository_id
self.args_parser = args_parser
self.command_arguments = command_arguments
@ -77,20 +79,20 @@ class Spawn(Thread, LazyLogging):
return name if value else f"no-{name}"
@staticmethod
def process(callback: Callable[[argparse.Namespace, str], bool], args: argparse.Namespace, architecture: str,
process_id: str, queue: Queue[tuple[str, bool, int]]) -> None: # pylint: disable=unsubscriptable-object
def process(callback: Callable[[argparse.Namespace, RepositoryId], bool], args: argparse.Namespace,
repository_id: RepositoryId, process_id: str, queue: Queue[tuple[str, bool, int]]) -> None: # pylint: disable=unsubscriptable-object
"""
helper to run external process
Args:
callback(Callable[[argparse.Namespace, str], bool]): application run function (i.e. Handler.run method)
args(argparse.Namespace): command line arguments
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
process_id(str): process unique identifier
queue(Queue[tuple[str, bool, int]]): output queue
"""
start_time = time.monotonic()
result = callback(args, architecture)
result = callback(args, repository_id)
stop_time = time.monotonic()
consumed_time = int(1000 * (stop_time - start_time))
@ -128,7 +130,7 @@ class Spawn(Thread, LazyLogging):
callback = parsed.handler.call
process = Process(target=self.process,
args=(callback, parsed, self.architecture, process_id, self.queue),
args=(callback, parsed, self.repository_id, process_id, self.queue),
daemon=True)
process.start()

View File

@ -25,6 +25,7 @@ from ahriman.core.repository import Repository
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class Watcher(LazyLogging):
@ -32,26 +33,26 @@ class Watcher(LazyLogging):
package status watcher
Attributes:
architecture(str): repository architecture
database(SQLite): database instance
known(dict[str, tuple[Package, BuildStatus]]): list of known packages. For the most cases ``packages`` should
be used instead
repository(Repository): repository object
repository_id(RepositoryId): repository unique identifier
status(BuildStatus): daemon status
"""
def __init__(self, architecture: str, configuration: Configuration, database: SQLite) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, database: SQLite) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
database(SQLite): database instance
"""
self.architecture = architecture
self.repository_id = repository_id
self.database = database
self.repository = Repository.load(architecture, configuration, database, report=False)
self.repository = Repository.load(repository_id, configuration, database, report=False)
self.known: dict[str, tuple[Package, BuildStatus]] = {}
self.status = BuildStatus()

View File

@ -51,7 +51,7 @@ class WebClient(Client, SyncHttpClient):
configuration(Configuration): configuration instance
"""
suppress_errors = configuration.getboolean("settings", "suppress_http_log_errors", fallback=False)
SyncHttpClient.__init__(self, "web", configuration, suppress_errors=suppress_errors)
SyncHttpClient.__init__(self, configuration, "web", suppress_errors=suppress_errors)
self.address, self.use_unix_socket = self.parse_address(configuration)

View File

@ -25,6 +25,7 @@ from ahriman.core.support.package_creator import PackageCreator
from ahriman.core.support.pkgbuild.keyring_generator import KeyringGenerator
from ahriman.core.triggers import Trigger
from ahriman.models.context_key import ContextKey
from ahriman.models.repository_id import RepositoryId
class KeyringTrigger(Trigger):
@ -82,15 +83,15 @@ class KeyringTrigger(Trigger):
},
}
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, architecture, configuration)
Trigger.__init__(self, repository_id, configuration)
self.targets = self.configuration_sections(configuration)
@classmethod
@ -115,6 +116,6 @@ class KeyringTrigger(Trigger):
database = ctx.get(ContextKey("database", SQLite))
for target in self.targets:
generator = KeyringGenerator(database, sign, self.configuration, target)
generator = KeyringGenerator(database, sign, self.repository_id, self.configuration, target)
runner = PackageCreator(self.configuration, generator)
runner.run()

View File

@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.support.package_creator import PackageCreator
from ahriman.core.support.pkgbuild.mirrorlist_generator import MirrorlistGenerator
from ahriman.core.triggers import Trigger
from ahriman.models.repository_id import RepositoryId
class MirrorlistTrigger(Trigger):
@ -75,15 +76,15 @@ class MirrorlistTrigger(Trigger):
},
}
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, architecture, configuration)
Trigger.__init__(self, repository_id, configuration)
self.targets = self.configuration_sections(configuration)
@classmethod
@ -104,6 +105,6 @@ class MirrorlistTrigger(Trigger):
trigger action which will be called at the start of the application
"""
for target in self.targets:
generator = MirrorlistGenerator(self.configuration, target)
generator = MirrorlistGenerator(self.repository_id, self.configuration, target)
runner = PackageCreator(self.configuration, generator)
runner.run()

View File

@ -66,6 +66,6 @@ class PackageCreator:
# register package
ctx = context.get()
database: SQLite = ctx.get(ContextKey("database", SQLite))
_, architecture = self.configuration.check_loaded()
package = Package.from_build(local_path, architecture, None)
_, repository_id = self.configuration.check_loaded()
package = Package.from_build(local_path, repository_id.architecture, None)
database.package_update(package, BuildStatus())

View File

@ -25,6 +25,7 @@ from ahriman.core.database import SQLite
from ahriman.core.exceptions import PkgbuildGeneratorError
from ahriman.core.sign.gpg import GPG
from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator
from ahriman.models.repository_id import RepositoryId
class KeyringGenerator(PkgbuildGenerator):
@ -43,18 +44,20 @@ class KeyringGenerator(PkgbuildGenerator):
trusted(list[str]): lif of trusted PGP keys
"""
def __init__(self, database: SQLite, sign: GPG, configuration: Configuration, section: str) -> None:
def __init__(self, database: SQLite, sign: GPG, repository_id: RepositoryId,
configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
database(SQLite): database instance
sign(GPG): GPG wrapper instance
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
self.sign = sign
self.name = configuration.repository_name
self.name = repository_id.name
# configuration fields
packager_keys = [packager.key for packager in database.user_list(None, None) if packager.key is not None]

View File

@ -23,6 +23,7 @@ from pathlib import Path
from ahriman.core.configuration import Configuration
from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_id import RepositoryId
class MirrorlistGenerator(PkgbuildGenerator):
@ -38,24 +39,24 @@ class MirrorlistGenerator(PkgbuildGenerator):
servers(list[str]): list of mirror servers
"""
def __init__(self, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
name = configuration.repository_name
# configuration fields
self.servers = configuration.getlist(section, "servers")
self.path = configuration.getpath(section, "path", fallback=Path("/etc") / "pacman.d" / f"{name}-mirrorlist")
self.path = configuration.getpath(
section, "path", fallback=Path("/etc") / "pacman.d" / f"{repository_id.name}-mirrorlist")
self.path = self.path.relative_to("/") # in pkgbuild we are always operating with relative to / path
# pkgbuild description fields
self.pkgbuild_pkgname = configuration.get(section, "package", fallback=f"{name}-mirrorlist")
self.pkgbuild_pkgname = configuration.get(section, "package", fallback=f"{repository_id.name}-mirrorlist")
self.pkgbuild_pkgdesc = configuration.get(
section, "description", fallback=f"{name} mirror list for use by pacman")
section, "description", fallback=f"{repository_id.name} mirror list for use by pacman")
self.pkgbuild_license = configuration.getlist(section, "license", fallback=["Unlicense"])
self.pkgbuild_url = configuration.get(section, "homepage", fallback="")

View File

@ -19,9 +19,8 @@
#
from __future__ import annotations
import functools
from collections.abc import Iterable
from functools import partial
from ahriman.core.exceptions import PartitionError
from ahriman.core.util import minmax, partition
@ -102,10 +101,11 @@ class Tree:
>>> from ahriman.core.configuration import Configuration
>>> from ahriman.core.database import SQLite
>>> from ahriman.core.repository import Repository
>>> from ahriman.models.repository_id import RepositoryId
>>>
>>> configuration = Configuration()
>>> database = SQLite.load(configuration)
>>> repository = Repository.load("x86_64", configuration, database, report=True)
>>> repository = Repository.load(RepositoryId("x86_64", "aur-clone"), configuration, database, report=True)
>>> packages = repository.packages()
>>>
>>> tree = Tree.resolve(packages)
@ -242,7 +242,7 @@ class Tree:
unprocessed = self.leaves[:]
while unprocessed:
# additional workaround with partial in order to hide cell-var-from-loop pylint warning
predicate = functools.partial(Leaf.is_root, packages=unprocessed)
predicate = partial(Leaf.is_root, packages=unprocessed)
new_level, unprocessed = partition(unprocessed, predicate)
unsorted.append(new_level)
@ -252,7 +252,7 @@ class Tree:
next_level = unsorted[next_num]
# change lists inside the collection
predicate = functools.partial(Leaf.is_dependency, packages=next_level)
predicate = partial(Leaf.is_dependency, packages=next_level)
unsorted[current_num], to_be_moved = partition(current_level, predicate)
unsorted[next_num].extend(to_be_moved)
@ -279,12 +279,12 @@ class Tree:
while True: # python doesn't allow to use walrus operator to unpack tuples
# get packages which depend on packages in chunk
predicate = functools.partial(Leaf.is_root, packages=chunk)
predicate = partial(Leaf.is_root, packages=chunk)
unprocessed, new_dependent = partition(unprocessed, predicate)
chunk.extend(new_dependent)
# get packages which are dependency of packages in chunk
predicate = functools.partial(Leaf.is_dependency, packages=chunk)
predicate = partial(Leaf.is_dependency, packages=chunk)
new_dependencies, unprocessed = partition(unprocessed, predicate)
chunk.extend(new_dependencies)

View File

@ -23,6 +23,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.configuration.schema import ConfigurationSchema
from ahriman.core.log import LazyLogging
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -34,8 +35,8 @@ class Trigger(LazyLogging):
CONFIGURATION_SCHEMA(ConfigurationSchema): (class attribute) configuration schema template
CONFIGURATION_SCHEMA_FALLBACK(str | None): (class attribute) optional fallback option for defining
configuration schema type used
architecture(str): repository architecture
configuration(Configuration): configuration instance
repository_id(RepositoryId): repository unique identifier
Examples:
This class must be used in order to create own extension. Basically idea is the following::
@ -51,26 +52,37 @@ class Trigger(LazyLogging):
>>> configuration = Configuration()
>>> configuration.set_option("build", "triggers", "my.awesome.package.CustomTrigger")
>>>
>>> loader = TriggerLoader.load("x86_64", configuration)
>>> loader = TriggerLoader.load(RepositoryId("x86_64", "aur-clone"), configuration)
>>> loader.on_result(Result(), [])
"""
CONFIGURATION_SCHEMA: ConfigurationSchema = {}
CONFIGURATION_SCHEMA_FALLBACK: str | None = None
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
self.architecture = architecture
self.repository_id = repository_id
self.configuration = configuration
@property
def architecture(self) -> str:
"""
repository architecture for backward compatibility
Returns:
str: repository architecture
"""
return self.repository_id.architecture
@classmethod
def configuration_schema(cls, architecture: str, configuration: Configuration | None) -> ConfigurationSchema:
def configuration_schema(cls, repository_id: RepositoryId,
configuration: Configuration | None) -> ConfigurationSchema:
"""
configuration schema based on supplied service configuration
@ -78,7 +90,7 @@ class Trigger(LazyLogging):
Schema must be in cerberus format, for details and examples you can check built-in triggers.
Args:
architecture(str): repository architecture
repository_id(str): repository unique identifier
configuration(Configuration | None): configuration instance. If set to None, the default schema
should be returned
@ -93,7 +105,7 @@ class Trigger(LazyLogging):
if not configuration.has_section(target):
continue
section, schema_name = configuration.gettype(
target, architecture, fallback=cls.CONFIGURATION_SCHEMA_FALLBACK)
target, repository_id, fallback=cls.CONFIGURATION_SCHEMA_FALLBACK)
if schema_name not in cls.CONFIGURATION_SCHEMA:
continue
result[section] = cls.CONFIGURATION_SCHEMA[schema_name]

View File

@ -31,6 +31,7 @@ from ahriman.core.exceptions import ExtensionError
from ahriman.core.log import LazyLogging
from ahriman.core.triggers import Trigger
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -49,7 +50,7 @@ class TriggerLoader(LazyLogging):
Having such configuration you can create instance of the loader::
>>> loader = TriggerLoader.load("x86_64", configuration)
>>> loader = TriggerLoader.load(RepositoryId("x86_64", "aur-clone"), configuration)
>>> print(loader.triggers)
After that you are free to run triggers::
@ -65,12 +66,12 @@ class TriggerLoader(LazyLogging):
self.triggers: list[Trigger] = []
@classmethod
def load(cls, architecture: str, configuration: Configuration) -> Self:
def load(cls, repository_id: RepositoryId, configuration: Configuration) -> Self:
"""
create instance from configuration
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
Returns:
@ -78,7 +79,7 @@ class TriggerLoader(LazyLogging):
"""
instance = cls()
instance.triggers = [
instance.load_trigger(trigger, architecture, configuration)
instance.load_trigger(trigger, repository_id, configuration)
for trigger in instance.selected_triggers(configuration)
]
@ -166,13 +167,13 @@ class TriggerLoader(LazyLogging):
except ModuleNotFoundError:
raise ExtensionError(f"Module {package} not found") from None
def load_trigger(self, module_path: str, architecture: str, configuration: Configuration) -> Trigger:
def load_trigger(self, module_path: str, repository_id: RepositoryId, configuration: Configuration) -> Trigger:
"""
load trigger by module path
Args:
module_path(str): module import path to load
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
Returns:
@ -183,7 +184,7 @@ class TriggerLoader(LazyLogging):
"""
trigger_type = self.load_trigger_class(module_path)
try:
trigger = trigger_type(architecture, configuration)
trigger = trigger_type(repository_id, configuration)
except Exception:
raise ExtensionError(f"Could not load instance of trigger from {trigger_type} loaded from {module_path}")

View File

@ -25,32 +25,44 @@ from typing import Any
from ahriman.core.configuration import Configuration
from ahriman.core.upload.http_upload import HttpUpload
from ahriman.core.upload.upload import Upload
from ahriman.core.util import walk
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class Github(HttpUpload):
class GitHub(Upload, HttpUpload):
"""
upload files to GitHub releases
Attributes:
github_owner(str): GitHub repository owner
github_release_tag(str): GitHub release tag
github_release_tag_name(str): GitHub release tag name
github_repository(str): GitHub repository name
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
HttpUpload.__init__(self, architecture, configuration, section)
Upload.__init__(self, repository_id, configuration)
HttpUpload.__init__(self, configuration, section)
self.github_owner = configuration.get(section, "owner")
self.github_repository = configuration.get(section, "repository")
if configuration.getboolean(section, "use_full_release_name", fallback=False):
self.github_release_tag = f"{repository_id.name}-{repository_id.architecture}"
self.github_release_tag_name = f"{repository_id.name} {repository_id.architecture}"
else:
self.github_release_tag_name = self.github_release_tag = repository_id.architecture
def asset_remove(self, release: dict[str, Any], name: str) -> None:
"""
remove asset from the release by name
@ -136,7 +148,10 @@ class Github(HttpUpload):
dict[str, Any]: GitHub API release object for the new release
"""
url = f"https://api.github.com/repos/{self.github_owner}/{self.github_repository}/releases"
response = self.make_request("POST", url, json={"tag_name": self.architecture, "name": self.architecture})
response = self.make_request("POST", url, json={
"tag_name": self.github_release_tag,
"name": self.github_release_tag_name,
})
release: dict[str, Any] = response.json()
return release
@ -147,7 +162,7 @@ class Github(HttpUpload):
Returns:
dict[str, Any] | None: GitHub API release object if release found and None otherwise
"""
url = f"https://api.github.com/repos/{self.github_owner}/{self.github_repository}/releases/tags/{self.architecture}"
url = f"https://api.github.com/repos/{self.github_owner}/{self.github_repository}/releases/tags/{self.github_release_tag}"
try:
response = self.make_request("GET", url)
release: dict[str, Any] = response.json()

View File

@ -21,28 +21,14 @@ import hashlib
from pathlib import Path
from ahriman.core.configuration import Configuration
from ahriman.core.http import SyncHttpClient
from ahriman.core.upload.upload import Upload
class HttpUpload(Upload, SyncHttpClient):
class HttpUpload(SyncHttpClient):
"""
helper for the http based uploads
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
configuration(Configuration): configuration instance
section(str): configuration section name
"""
Upload.__init__(self, architecture, configuration)
SyncHttpClient.__init__(self, section, configuration)
@staticmethod
def calculate_hash(path: Path) -> str:
"""

View File

@ -27,10 +27,12 @@ from ahriman.core.http import MultipartType
from ahriman.core.sign.gpg import GPG
from ahriman.core.status.web_client import WebClient
from ahriman.core.upload.http_upload import HttpUpload
from ahriman.core.upload.upload import Upload
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class RemoteService(HttpUpload):
class RemoteService(Upload, HttpUpload):
"""
upload files to another server instance
@ -38,16 +40,17 @@ class RemoteService(HttpUpload):
client(WebClient): web client instance
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
HttpUpload.__init__(self, architecture, configuration, section)
Upload.__init__(self, repository_id, configuration)
HttpUpload.__init__(self, configuration, section)
self.client = WebClient(configuration)
@cached_property

View File

@ -23,6 +23,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.upload.upload import Upload
from ahriman.core.util import check_output
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class Rsync(Upload):
@ -36,16 +37,16 @@ class Rsync(Upload):
_check_output = check_output
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Upload.__init__(self, architecture, configuration)
Upload.__init__(self, repository_id, configuration)
self.command = configuration.getlist(section, "command")
self.remote = configuration.get(section, "remote")

View File

@ -28,6 +28,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.upload.upload import Upload
from ahriman.core.util import walk
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class S3(Upload):
@ -37,21 +38,29 @@ class S3(Upload):
Attributes
bucket(Any): boto3 S3 bucket object
chunk_size(int): chunk size for calculating checksums
object_path(Path): relative path to which packages will be uploaded
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Upload.__init__(self, architecture, configuration)
Upload.__init__(self, repository_id, configuration)
self.bucket = self.get_bucket(configuration, section)
self.chunk_size = configuration.getint(section, "chunk_size", fallback=8 * 1024 * 1024)
if (object_path := configuration.get(section, "object_path", fallback=None)) is not None:
# we need to avoid path conversion here, hence the string
self.object_path = Path(object_path)
else:
paths = configuration.repository_paths
self.object_path = paths.repository.relative_to(paths.root / "repository")
@staticmethod
def calculate_etag(path: Path, chunk_size: int) -> str:
"""
@ -127,7 +136,7 @@ class S3(Upload):
continue
local_path = path / local_file
remote_path = Path(self.architecture) / local_file
remote_path = self.object_path / local_file.name
(mime, _) = mimetypes.guess_type(local_path)
extra_args = {"ContentType": mime} if mime is not None else None
@ -155,8 +164,8 @@ class S3(Upload):
Returns:
dict[Path, Any]: map of path object to the remote s3 object
"""
objects = self.bucket.objects.filter(Prefix=self.architecture)
return {Path(item.key).relative_to(self.architecture): item for item in objects}
objects = self.bucket.objects.filter(Prefix=str(self.object_path))
return {Path(item.key).relative_to(self.object_path): item for item in objects}
def sync(self, path: Path, built_packages: list[Package]) -> None:
"""

View File

@ -25,6 +25,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import SynchronizationError
from ahriman.core.log import LazyLogging
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.upload_settings import UploadSettings
@ -33,18 +34,16 @@ class Upload(LazyLogging):
base remote sync class
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
repository_id(RepositoryId): repository unique identifier
Examples:
These classes provide the way to upload packages to remote sources as it is described in their implementations.
Basic flow includes class instantiating by using the ``load`` method and then calling the ``run`` method which
wraps any internal exceptions into the ``SyncFailed`` exception::
>>> from ahriman.core.configuration import Configuration
>>>
>>> configuration = Configuration()
>>> upload = Upload.load("x86_64", configuration, "s3")
>>> upload = Upload.load(RepositoryId("x86_64", "aur-clone"), configuration, "s3")
>>> upload.run(configuration.repository_paths.repository, [])
Or in case if direct access to exception is required, the ``sync`` method can be used::
@ -55,46 +54,46 @@ class Upload(LazyLogging):
>>> handle_exceptions(ex)
"""
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
self.architecture = architecture
self.repository_id = repository_id
self.configuration = configuration
@staticmethod
def load(architecture: str, configuration: Configuration, target: str) -> Upload:
def load(repository_id: RepositoryId, configuration: Configuration, target: str) -> Upload:
"""
load client from settings
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
target(str): target to run sync (e.g. s3)
Returns:
Upload: client according to current settings
"""
section, provider_name = configuration.gettype(target, architecture)
section, provider_name = configuration.gettype(target, repository_id)
match UploadSettings.from_option(provider_name):
case UploadSettings.Rsync:
from ahriman.core.upload.rsync import Rsync
return Rsync(architecture, configuration, section)
return Rsync(repository_id, configuration, section)
case UploadSettings.S3:
from ahriman.core.upload.s3 import S3
return S3(architecture, configuration, section)
return S3(repository_id, configuration, section)
case UploadSettings.GitHub:
from ahriman.core.upload.github import Github
return Github(architecture, configuration, section)
from ahriman.core.upload.github import GitHub
return GitHub(repository_id, configuration, section)
case UploadSettings.RemoteService:
from ahriman.core.upload.remote_service import RemoteService
return RemoteService(architecture, configuration, section)
return RemoteService(repository_id, configuration, section)
case _:
return Upload(architecture, configuration) # should never happen
return Upload(repository_id, configuration) # should never happen
def run(self, path: Path, built_packages: list[Package]) -> None:
"""

View File

@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.triggers import Trigger
from ahriman.core.upload.upload import Upload
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -56,7 +57,6 @@ class UploadTrigger(Trigger):
},
"password": {
"type": "string",
"required": True,
},
"repository": {
"type": "string",
@ -67,6 +67,10 @@ class UploadTrigger(Trigger):
"coerce": "integer",
"min": 0,
},
"use_full_release_name": {
"type": "boolean",
"coerce": "boolean",
},
"username": {
"type": "string",
},
@ -126,6 +130,9 @@ class UploadTrigger(Trigger):
"coerce": "integer",
"min": 0,
},
"object_path": {
"type": "string",
},
"region": {
"type": "string",
"required": True,
@ -138,15 +145,15 @@ class UploadTrigger(Trigger):
},
}
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, architecture, configuration)
Trigger.__init__(self, repository_id, configuration)
self.targets = self.configuration_sections(configuration)
@classmethod
@ -171,5 +178,5 @@ class UploadTrigger(Trigger):
packages(list[Package]): list of all available packages
"""
for target in self.targets:
runner = Upload.load(self.architecture, self.configuration, target)
runner = Upload.load(self.repository_id, self.configuration, target)
runner.run(self.configuration.repository_paths.repository, result.success)

View File

@ -68,9 +68,10 @@ class AURPackage:
>>>
>>> from ahriman.core.alpm.pacman import Pacman
>>> from ahriman.core.configuration import Configuration
>>> from ahriman.models.repository_id import RepositoryId
>>>
>>> configuration = Configuration()
>>> pacman = Pacman("x86_64", configuration)
>>> pacman = Pacman(RepositoryId("x86_64", "aur-clone"), configuration)
>>> metadata = pacman.package_get("pacman")
>>> package = AURPackage.from_pacman(next(metadata)) # load package from pyalpm wrapper
"""

View File

@ -56,9 +56,10 @@ class PackageDescription:
>>> from pathlib import Path
>>> from ahriman.core.alpm.pacman import Pacman
>>> from ahriman.core.configuration import Configuration
>>> from ahriman.models.repository_id import RepositoryId
>>>
>>> configuration = Configuration()
>>> pacman = Pacman("x86_64", configuration)
>>> pacman = Pacman(RepositoryId("x86_64", "aur-clone"), configuration)
>>> pyalpm_description = next(package for package in pacman.package_get("pacman"))
>>> description = PackageDescription.from_package(
>>> pyalpm_description, Path("/var/cache/pacman/pkg/pacman-6.0.1-4-x86_64.pkg.tar.zst"))

View File

@ -0,0 +1,73 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass
from typing import Any
@dataclass(frozen=True)
class RepositoryId:
"""
unique identifier of the repository
Attributes:
architecture(str): repository architecture
name(str): repository name
"""
architecture: str
name: str
@property
def is_empty(self) -> bool:
"""
check if all data is supplied for the loading
Returns:
bool: True in case if architecture or name are not set and False otherwise
"""
return not self.architecture or not self.name
@property
def id(self) -> str:
"""
get repository id to be used for databases
Returns:
str: unique id for this repository
"""
return f"{self.architecture}-{self.name}" # basically the same as used for command line
def __lt__(self, other: Any) -> bool:
"""
comparison operator for sorting
Args:
other(Any): other object to compare
Returns:
bool: True in case if this is less than other and False otherwise
Raises:
TypeError: if other is different from RepositoryId type
"""
if not isinstance(other, RepositoryId):
raise ValueError(f"'<' not supported between instances of '{type(self)}' and '{type(other)}'")
return (self.name, self.architecture) < (other.name, other.architecture)

View File

@ -20,25 +20,29 @@
import os
import shutil
from dataclasses import dataclass
from collections.abc import Generator
from dataclasses import dataclass, field
from functools import cached_property
from pathlib import Path
from ahriman.core.exceptions import PathError
from ahriman.core.log import LazyLogging
from ahriman.models.repository_id import RepositoryId
@dataclass(frozen=True)
class RepositoryPaths:
class RepositoryPaths(LazyLogging):
"""
repository paths holder. For the most operations with paths you want to use this object
Attributes:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
root(Path): repository root (i.e. ahriman home)
Examples:
This class can be used in order to access the repository tree structure::
>>> paths = RepositoryPaths(Path("/var/lib/ahriman"), "x86_64")
>>> paths = RepositoryPaths(Path("/var/lib/ahriman"), RepositoryId("x86_64", "aur-clone"))
Additional methods can be used in order to ensure that tree is created::
@ -51,7 +55,33 @@ class RepositoryPaths:
"""
root: Path
architecture: str
repository_id: RepositoryId
_force_current_tree: bool = field(default=False, kw_only=True)
@property
def _repository_root(self) -> Path:
"""
repository root which can be used for invalid (not fully loaded instances)
Returns:
Path: root path to repositories
"""
return self.root / "repository"
@cached_property
def _suffix(self) -> Path:
"""
suffix of the paths as defined by repository structure
Returns:
Path: relative path which contains only architecture segment in case if legacy tree is used and repository
name and architecture otherwise
"""
if not self._force_current_tree:
if (self._repository_root / self.repository_id.architecture).is_dir():
self.logger.warning("using legacy per architecture tree")
return Path(self.repository_id.architecture) # legacy tree suffix
return Path(self.repository_id.name) / self.repository_id.architecture
@property
def cache(self) -> Path:
@ -72,7 +102,7 @@ class RepositoryPaths:
Path: full patch to devtools chroot directory
"""
# for the chroot directory devtools will create own tree, and we don"t have to specify architecture here
return self.root / "chroot"
return self.root / "chroot" / self.repository_id.name
@property
def packages(self) -> Path:
@ -82,7 +112,7 @@ class RepositoryPaths:
Returns:
Path: full path to built packages directory
"""
return self.root / "packages" / self.architecture
return self.root / "packages" / self._suffix
@property
def pacman(self) -> Path:
@ -92,7 +122,7 @@ class RepositoryPaths:
Returns:
Path: full path to pacman local database cache
"""
return self.root / "pacman" / self.architecture
return self.root / "pacman" / self._suffix
@property
def repository(self) -> Path:
@ -102,7 +132,7 @@ class RepositoryPaths:
Returns:
Path: full path to the repository directory
"""
return self.root / "repository" / self.architecture
return self._repository_root / self._suffix
@property
def root_owner(self) -> tuple[int, int]:
@ -114,23 +144,57 @@ class RepositoryPaths:
"""
return self.owner(self.root)
# TODO see https://github.com/python/mypy/issues/12534, remove type: ignore after release
# pylint: disable=protected-access
@classmethod
def known_architectures(cls, root: Path) -> set[str]:
def known_architectures(cls, root: Path, name: str = "") -> set[str]: # type: ignore[return]
"""
get known architectures
get known architecture names
Args:
root(Path): repository root
name(str, optional): repository name (Default value = "")
Returns:
set[str]: list of repository architectures for which there is created tree
"""
def walk(repository_dir: Path) -> Generator[str, None, None]:
for architecture in filter(lambda path: path.is_dir(), repository_dir.iterdir()):
yield architecture.name
instance = cls(root, RepositoryId("", ""))
match (instance._repository_root / name):
case full_tree if full_tree.is_dir():
return set(walk(full_tree)) # actually works for legacy too in case if name is set to empty string
case _ if instance._repository_root.is_dir():
return set(walk(instance._repository_root)) # legacy only tree
case _:
return set() # no tree detected at all
# pylint: disable=protected-access
@classmethod
def known_repositories(cls, root: Path) -> set[str]:
"""
get known repository names
Args:
root(Path): repository root
Returns:
set[str]: list of architectures for which tree is created
set[str]: list of repository names for which there is created tree. Returns empty set in case if repository
is loaded in legacy mode
"""
paths = cls(root, "")
return {
path.name
for path in paths.repository.iterdir()
if path.is_dir()
}
# simply walk through the root. In case if there are subdirectories, emit the name
def walk(paths: RepositoryPaths) -> Generator[str, None, None]:
for repository in filter(lambda path: path.is_dir(), paths._repository_root.iterdir()):
if any(path.is_dir() for path in repository.iterdir()):
yield repository.name
instance = cls(root, RepositoryId("", ""))
if not instance._repository_root.is_dir():
return set() # no tree created
return set(walk(instance))
@staticmethod
def owner(path: Path) -> tuple[int, int]:
@ -197,11 +261,13 @@ class RepositoryPaths:
"""
create ahriman working tree
"""
if self.repository_id.is_empty:
return # do not even try to create tree in case if no repository id set
for directory in (
self.cache,
self.chroot,
self.packages,
self.pacman / "sync", # we need sync directory in order to be able to copy databases
self.pacman,
self.repository,
):
directory.mkdir(mode=0o755, parents=True, exist_ok=True)

View File

@ -65,9 +65,9 @@ class StatusView(BaseView):
counters = Counters.from_packages(self.service.packages)
status = InternalStatus(
status=self.service.status,
architecture=self.service.architecture,
architecture=self.service.repository_id.architecture,
packages=counters,
repository=self.service.repository.name,
repository=self.service.repository_id.name,
version=__version__,
)

View File

@ -31,6 +31,7 @@ from ahriman.core.exceptions import InitializeError
from ahriman.core.log.filtered_access_logger import FilteredAccessLogger
from ahriman.core.spawn import Spawn
from ahriman.core.status.watcher import Watcher
from ahriman.models.repository_id import RepositoryId
from ahriman.web.apispec import setup_apispec
from ahriman.web.cors import setup_cors
from ahriman.web.middlewares.exception_handler import exception_handler
@ -120,12 +121,12 @@ def run_server(application: Application) -> None:
access_log=logging.getLogger("http"), access_log_class=FilteredAccessLogger)
def setup_service(architecture: str, configuration: Configuration, spawner: Spawn) -> Application:
def setup_service(repository_id: RepositoryId, configuration: Configuration, spawner: Spawn) -> Application:
"""
create web application
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
spawner(Spawn): spawner thread
@ -155,7 +156,7 @@ def setup_service(architecture: str, configuration: Configuration, spawner: Spaw
database = application["database"] = SQLite.load(configuration)
application.logger.info("setup watcher")
application["watcher"] = Watcher(architecture, configuration, database)
application["watcher"] = Watcher(repository_id, configuration, database)
application.logger.info("setup process spawner")
application["spawn"] = spawner