diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 371989d9..76d10e1f 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -17,6 +17,7 @@ host = $AHRIMAN_HOST EOF AHRIMAN_DEFAULT_ARGS=("--architecture" "$AHRIMAN_ARCHITECTURE") +AHRIMAN_DEFAULT_ARGS+=("--repository" "$AHRIMAN_REPOSITORY") if [ -n "$AHRIMAN_OUTPUT" ]; then AHRIMAN_DEFAULT_ARGS+=("--log-handler" "$AHRIMAN_OUTPUT") fi @@ -33,19 +34,18 @@ chown "$AHRIMAN_USER":"$AHRIMAN_USER" "$AHRIMAN_GNUPG_HOME" # run built-in setup command AHRIMAN_SETUP_ARGS=("--build-as-user" "$AHRIMAN_USER") AHRIMAN_SETUP_ARGS+=("--packager" "$AHRIMAN_PACKAGER") -AHRIMAN_SETUP_ARGS+=("--repository" "$AHRIMAN_REPOSITORY") if [ -z "$AHRIMAN_MULTILIB" ]; then AHRIMAN_SETUP_ARGS+=("--no-multilib") fi if [ -n "$AHRIMAN_PACMAN_MIRROR" ]; then AHRIMAN_SETUP_ARGS+=("--mirror" "$AHRIMAN_PACMAN_MIRROR") fi -if [ -n "$AHRIMAN_PORT" ]; then - AHRIMAN_SETUP_ARGS+=("--web-port" "$AHRIMAN_PORT") -fi if [ -n "$AHRIMAN_REPOSITORY_SERVER" ]; then AHRIMAN_SETUP_ARGS+=("--server" "$AHRIMAN_REPOSITORY_SERVER") fi +if [ -n "$AHRIMAN_PORT" ]; then + AHRIMAN_SETUP_ARGS+=("--web-port" "$AHRIMAN_PORT") +fi if [ -n "$AHRIMAN_UNIX_SOCKET" ]; then AHRIMAN_SETUP_ARGS+=("--web-unix-socket" "$AHRIMAN_UNIX_SOCKET") fi diff --git a/docs/configuration.rst b/docs/configuration.rst index deaea442..b1f0a6dd 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1,7 +1,12 @@ Configuration ============= -Some groups can be specified for each architecture separately. E.g. if there are ``build`` and ``build:x86_64`` groups it will use an option from ``build:x86_64`` for the ``x86_64`` architecture and ``build`` for any other (architecture specific group has higher priority). In case if both groups are presented, architecture specific options will be merged into global ones overriding them. +Some groups can be specified for each architecture and/or repository separately. E.g. if there are ``build`` and ``build:x86_64`` groups it will use an option from ``build:x86_64`` for the ``x86_64`` architecture and ``build`` for any other (architecture specific group has higher priority). In case if both groups are presented, architecture specific options will be merged into global ones overriding them. The order which will be used for option resolution is the following: + +1. Repository and architecture specific, e.g. ``build:aur-clone:x86_64``. +2. Repository specific, e.g. ``build:aur-clone``. +2. Architecture specific, e.g. ``build:x86_64``. +2. Default section, e.g. ``build``. There are two variable types which have been added to default ones, they are paths and lists. List values will be read in the same way as shell does: @@ -87,7 +92,6 @@ Build related configuration. Group name can refer to architecture, e.g. ``build: Base repository settings. -* ``name`` - repository name, string, required. * ``root`` - root path for application, string, required. ``sign:*`` groups @@ -292,20 +296,21 @@ Type will be read from several sources: ``github`` type ^^^^^^^^^^^^^^^ -This feature requires Github key creation (see below). Section name must be either ``github`` (plus optional architecture name, e.g. ``github:x86_64``) or random name with ``type`` set. +This feature requires GitHub key creation (see below). Section name must be either ``github`` (plus optional architecture name, e.g. ``github:x86_64``) or random name with ``type`` set. * ``type`` - type of the upload, string, optional, must be set to ``github`` if exists. -* ``owner`` - Github repository owner, string, required. -* ``password`` - created Github API key. In order to create it do the following: +* ``owner`` - GitHub repository owner, string, required. +* ``password`` - created GitHub API key. In order to create it do the following: #. Go to `settings page `_. #. Switch to `developers settings `_. #. Switch to `personal access tokens `_. #. Generate new token. Required scope is ``public_repo`` (or ``repo`` for private repository support). -* ``repository`` - Github repository name, string, required. Repository must be created before any action and must have active branch (e.g. with readme). +* ``repository`` - GitHub repository name, string, required. Repository must be created before any action and must have active branch (e.g. with readme). * ``timeout`` - HTTP request timeout in seconds, int, optional, default is ``30``. -* ``username`` - Github authorization user, string, required. Basically the same as ``owner``. +* ``use_full_release_name`` - if set to ``yes``, the release will contain both repository name and architecture, and only architecture otherwise, boolean, optional, default ``no``. +* ``username`` - GitHub authorization user, string, required. Basically the same as ``owner``. ``remote-service`` type ^^^^^^^^^^^^^^^^^^^^^^^ @@ -329,7 +334,7 @@ Requires ``rsync`` package to be installed. Do not forget to configure ssh for u Requires ``boto3`` library to be installed. Section name must be either ``s3`` (plus optional architecture name, e.g. ``s3:x86_64``) or random name with ``type`` set. -* ``type`` - type of the upload, string, optional, must be set to ``github`` if exists. +* ``type`` - type of the upload, string, optional, must be set to ``s3`` if exists. * ``access_key`` - AWS access key ID, string, required. * ``bucket`` - bucket name (e.g. ``bucket``), string, required. * ``chunk_size`` - chunk size for calculating entity tags, int, optional, default 8 * 1024 * 1024. diff --git a/docs/faq.rst b/docs/faq.rst index 62c3da76..8c92911f 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -17,7 +17,7 @@ TL;DR .. code-block:: shell yay -S ahriman - ahriman -a x86_64 service-setup --packager "ahriman bot " --repository "repository" + ahriman -a x86_64 -r aur-clone service-setup --packager "ahriman bot " systemctl enable --now ahriman@x86_64.timer Long answer @@ -32,7 +32,7 @@ There is special command which can be used in order to validate current configur .. code-block:: shell - ahriman -a x86_64 service-config-validate --exit-code + ahriman -a x86_64 -r aur-clone service-config-validate --exit-code This command will print found errors, based on `cerberus `_, e.g.: @@ -317,7 +317,7 @@ Add the following lines to your ``pacman.conf``: .. code-block:: ini [repository] - Server = file:///var/lib/ahriman/repository/x86_64 + Server = file:///var/lib/ahriman/repository/$repo/$arch (You might need to add ``SigLevel`` option according to the pacman documentation.) @@ -554,8 +554,8 @@ There are several choices: .. code-block:: server { - location /x86_64 { - root /var/lib/ahriman/repository/x86_64; + location /aur-clone/x86_64 { + root /var/lib/ahriman/repository/aur-clone/x86_64; autoindex on; } } @@ -571,7 +571,7 @@ There are several choices: [rsync] remote = 192.168.0.1:/srv/repo - After that just add ``/srv/repo`` to the ``pacman.conf`` as usual. You can also upload to S3 (e.g. ``Server = https://s3.eu-central-1.amazonaws.com/repository/x86_64``) or to Github (e.g. ``Server = https://github.com/ahriman/repository/releases/download/x86_64``). + After that just add ``/srv/repo`` to the ``pacman.conf`` as usual. You can also upload to S3 (e.g. ``Server = https://s3.eu-central-1.amazonaws.com/repository/aur-clone/x86_64``) or to Github (e.g. ``Server = https://github.com/ahriman/repository/releases/download/aur-clone-x86_64``). How to sync to S3 ^^^^^^^^^^^^^^^^^ @@ -676,7 +676,7 @@ How to report by email [email] host = smtp.example.com - link_path = http://example.com/x86_64 + link_path = http://example.com/aur-clone/x86_64 password = ... port = 465 receivers = me@example.com @@ -702,8 +702,8 @@ How to generate index page for S3 target = html [html] - path = /var/lib/ahriman/repository/x86_64/index.html - link_path = http://example.com/x86_64 + path = /var/lib/ahriman/repository/aur-clone/x86_64/index.html + link_path = http://example.com/aur-clone/x86_64 After these steps ``index.html`` file will be automatically synced to S3 @@ -741,7 +741,7 @@ How to post build report to telegram [telegram] api_key = aaAAbbBBccCC chat_id = @ahriman - link_path = http://example.com/x86_64 + link_path = http://example.com/aur-clone/x86_64 ``api_key`` is the one sent by `@BotFather `_, ``chat_id`` is the value retrieved from previous step. diff --git a/docs/setup.rst b/docs/setup.rst index 9b1a673c..410aa712 100644 --- a/docs/setup.rst +++ b/docs/setup.rst @@ -10,7 +10,7 @@ Initial setup .. code-block:: shell - sudo ahriman -a x86_64 service-setup ... + sudo ahriman -a x86_64 -r aur-clone service-setup ... ``service-setup`` literally does the following steps: @@ -29,26 +29,26 @@ Initial setup .. code-block:: shell - ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build + ln -s /usr/bin/archbuild /usr/local/bin/aur-clone-x86_64-build #. Create configuration file (same as previous ``{name}.conf``): .. code-block:: shell - cp /usr/share/devtools/pacman.conf.d/{extra,ahriman}.conf + cp /usr/share/devtools/pacman.conf.d/{extra,aur-clone}.conf #. Change configuration file, add your own repository, add multilib repository etc: .. code-block:: shell - echo '[multilib]' | tee -a /usr/share/devtools/pacman-ahriman.conf - echo 'Include = /etc/pacman.d/mirrorlist' | tee -a /usr/share/devtools/pacman.conf.d/ahriman.conf + echo '[multilib]' | tee -a /usr/share/devtools/pacman.conf.d/aur-clone-x86_64.conf + echo 'Include = /etc/pacman.d/mirrorlist' | tee -a /usr/share/devtools/pacman.conf.d/aur-clone-x86_64.conf - echo '[aur-clone]' | tee -a /usr/share/devtools/pacman-ahriman.conf - echo 'SigLevel = Optional TrustAll' | tee -a /usr/share/devtools/pacman.conf.d/ahriman.conf - echo 'Server = file:///var/lib/ahriman/repository/$arch' | tee -a /usr/share/devtools/pacman.conf.d/ahriman.conf + echo '[aur-clone]' | tee -a /usr/share/devtools/pacman.conf.d/aur-clone-x86_64.conf + echo 'SigLevel = Optional TrustAll' | tee -a /usr/share/devtools/pacman.conf.d/aur-clone-x86_64.conf + echo 'Server = file:///var/lib/ahriman/repository/$repo/$arch' | tee -a /usr/share/devtools/pacman.conf.d/aur-clone-x86_64.conf #. Set ``build_command`` option to point to your command: @@ -56,14 +56,14 @@ Initial setup .. code-block:: shell echo '[build]' | tee -a /etc/ahriman.ini.d/build.ini - echo 'build_command = ahriman-x86_64-build' | tee -a /etc/ahriman.ini.d/build.ini + echo 'build_command = aur-clone-x86_64-build' | tee -a /etc/ahriman.ini.d/build.ini #. Configure ``/etc/sudoers.d/ahriman`` to allow running command without a password: .. code-block:: shell - echo 'Cmnd_Alias CARCHBUILD_CMD = /usr/local/bin/ahriman-x86_64-build *' | tee -a /etc/sudoers.d/ahriman + echo 'Cmnd_Alias CARCHBUILD_CMD = /usr/local/bin/aur-clone-x86_64-build *' | tee -a /etc/sudoers.d/ahriman echo 'ahriman ALL=(ALL) NOPASSWD:SETENV: CARCHBUILD_CMD' | tee -a /etc/sudoers.d/ahriman chmod 400 /etc/sudoers.d/ahriman @@ -88,6 +88,6 @@ Initial setup .. code-block:: shell - sudo -u ahriman ahriman -a x86_64 package-add ahriman --now --refresh + sudo -u ahriman ahriman package-add ahriman --now --refresh The ``--refresh`` flag is required in order to handle local database update. diff --git a/package/share/ahriman/settings/ahriman.ini b/package/share/ahriman/settings/ahriman.ini index 56f96743..440660eb 100644 --- a/package/share/ahriman/settings/ahriman.ini +++ b/package/share/ahriman/settings/ahriman.ini @@ -21,7 +21,6 @@ allow_read_only = yes [build] archbuild_flags = -build_command = extra-x86_64-build ignore_packages = makechrootpkg_flags = makepkg_flags = --nocolor --ignorearch @@ -30,7 +29,6 @@ triggers_known = ahriman.core.gitremote.RemotePullTrigger ahriman.core.gitremote vcs_allowed_age = 604800 [repository] -name = aur-clone root = /var/lib/ahriman [sign] diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index bca4c4d6..ea6efbdb 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -81,6 +81,8 @@ def _parser() -> argparse.ArgumentParser: type=LogHandler, choices=enum_values(LogHandler)) parser.add_argument("--report", help="force enable or disable reporting to web service", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("-r", "--repository", help="target repository. For several subcommands it can be used " + "multiple times", action="append") parser.add_argument("-q", "--quiet", help="force disable any logging", action="store_true") parser.add_argument("--unsafe", help="allow to run ahriman as non-ahriman user. Some actions might be unavailable", action="store_true") @@ -883,7 +885,6 @@ def _set_service_setup_parser(root: SubParserAction) -> argparse.ArgumentParser: epilog="Create _minimal_ configuration for the service according to provided options.", formatter_class=_formatter) parser.add_argument("--build-as-user", help="force makepkg user to the specific one") - parser.add_argument("--build-command", help="build command prefix", default="ahriman") parser.add_argument("--from-configuration", help="path to default devtools pacman configuration", type=Path, default=Path("/usr") / "share" / "devtools" / "pacman.conf.d" / "extra.conf") parser.add_argument("--generate-salt", help="generate salt for user passwords", @@ -894,7 +895,6 @@ def _set_service_setup_parser(root: SubParserAction) -> argparse.ArgumentParser: parser.add_argument("--multilib", help="add or do not multilib repository", action=argparse.BooleanOptionalAction, default=True) parser.add_argument("--packager", help="packager name and email", required=True) - parser.add_argument("--repository", help="repository name", required=True) parser.add_argument("--server", help="server to be used for devtools. If none set, local files will be used") parser.add_argument("--sign-key", help="sign key id") parser.add_argument("--sign-target", help="sign options", action="append", @@ -943,7 +943,7 @@ def _set_user_add_parser(root: SubParserAction) -> argparse.ArgumentParser: "`Name Surname `") parser.add_argument("-p", "--password", help="user password. Blank password will be treated as empty password, " "which is in particular must be used for OAuth2 authorization type.") - parser.add_argument("-r", "--role", help="user access level", + parser.add_argument("-R", "--role", help="user access level", type=UserAccess, choices=enum_values(UserAccess), default=UserAccess.Read) parser.set_defaults(handler=handlers.Users, action=Action.Update, architecture=[""], lock=None, report=False, quiet=True) @@ -965,7 +965,7 @@ def _set_user_list_parser(root: SubParserAction) -> argparse.ArgumentParser: formatter_class=_formatter) parser.add_argument("username", help="filter users by username", nargs="?") parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true") - parser.add_argument("-r", "--role", help="filter users by role", type=UserAccess, choices=enum_values(UserAccess)) + parser.add_argument("-R", "--role", help="filter users by role", type=UserAccess, choices=enum_values(UserAccess)) parser.set_defaults(handler=handlers.Users, action=Action.List, architecture=[""], lock=None, report=False, quiet=True, unsafe=True) return parser diff --git a/src/ahriman/application/application/application.py b/src/ahriman/application/application/application.py index a17bc3a6..2a2baed8 100644 --- a/src/ahriman/application/application/application.py +++ b/src/ahriman/application/application/application.py @@ -37,9 +37,10 @@ class Application(ApplicationPackages, ApplicationRepository): >>> from ahriman.core.configuration import Configuration >>> from ahriman.models.package_source import PackageSource + >>> from ahriman.models.repository_id import RepositoryId >>> >>> configuration = Configuration() - >>> application = Application("x86_64", configuration, report=True) + >>> application = Application(RepositoryId("x86_64", "x86_64"), configuration, report=True) >>> # add packages to build queue >>> application.add(["ahriman"], PackageSource.AUR) >>> diff --git a/src/ahriman/application/application/application_properties.py b/src/ahriman/application/application/application_properties.py index 380b39fa..f5aa0229 100644 --- a/src/ahriman/application/application/application_properties.py +++ b/src/ahriman/application/application/application_properties.py @@ -22,6 +22,7 @@ from ahriman.core.database import SQLite from ahriman.core.log import LazyLogging from ahriman.core.repository import Repository from ahriman.models.pacman_synchronization import PacmanSynchronization +from ahriman.models.repository_id import RepositoryId class ApplicationProperties(LazyLogging): @@ -29,26 +30,36 @@ class ApplicationProperties(LazyLogging): application base properties class Attributes: - architecture(str): repository architecture configuration(Configuration): configuration instance database(SQLite): database instance repository(Repository): repository instance + repository_id(RepositoryId): repository unique identifier """ - def __init__(self, architecture: str, configuration: Configuration, *, report: bool, + def __init__(self, repository_id: RepositoryId, configuration: Configuration, *, report: bool, refresh_pacman_database: PacmanSynchronization = PacmanSynchronization.Disabled) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting refresh_pacman_database(PacmanSynchronization, optional): pacman database synchronization level (Default value = PacmanSynchronization.Disabled) """ self.configuration = configuration - self.architecture = architecture + self.repository_id = repository_id self.database = SQLite.load(configuration) - self.repository = Repository.load(architecture, configuration, self.database, report=report, + self.repository = Repository.load(repository_id, configuration, self.database, report=report, refresh_pacman_database=refresh_pacman_database) + + @property + def architecture(self) -> str: + """ + repository architecture for backward compatibility + + Returns: + str: repository architecture + """ + return self.repository_id.architecture diff --git a/src/ahriman/application/handlers/add.py b/src/ahriman/application/handlers/add.py index b42e1ee4..2892f004 100644 --- a/src/ahriman/application/handlers/add.py +++ b/src/ahriman/application/handlers/add.py @@ -23,6 +23,7 @@ from ahriman.application.application import Application from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.models.packagers import Packagers +from ahriman.models.repository_id import RepositoryId class Add(Handler): @@ -31,17 +32,18 @@ class Add(Handler): """ @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - application = Application(architecture, configuration, report=report, refresh_pacman_database=args.refresh) + application = Application(repository_id, configuration, report=report, refresh_pacman_database=args.refresh) application.on_start() application.add(args.package, args.source, args.username) if not args.now: diff --git a/src/ahriman/application/handlers/backup.py b/src/ahriman/application/handlers/backup.py index 4bac517d..ec10b913 100644 --- a/src/ahriman/application/handlers/backup.py +++ b/src/ahriman/application/handlers/backup.py @@ -26,6 +26,7 @@ from tarfile import TarFile from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.core.database import SQLite +from ahriman.models.repository_id import RepositoryId class Backup(Handler): @@ -36,13 +37,14 @@ class Backup(Handler): ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture" @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ diff --git a/src/ahriman/application/handlers/clean.py b/src/ahriman/application/handlers/clean.py index 3487d96b..34443e3e 100644 --- a/src/ahriman/application/handlers/clean.py +++ b/src/ahriman/application/handlers/clean.py @@ -22,6 +22,7 @@ import argparse from ahriman.application.application import Application from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration +from ahriman.models.repository_id import RepositoryId class Clean(Handler): @@ -30,17 +31,18 @@ class Clean(Handler): """ @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - application = Application(architecture, configuration, report=report) + application = Application(repository_id, configuration, report=report) application.on_start() application.clean(cache=args.cache, chroot=args.chroot, manual=args.manual, packages=args.packages, pacman=args.pacman) diff --git a/src/ahriman/application/handlers/daemon.py b/src/ahriman/application/handlers/daemon.py index 3878ee06..33c00bf7 100644 --- a/src/ahriman/application/handlers/daemon.py +++ b/src/ahriman/application/handlers/daemon.py @@ -22,6 +22,7 @@ import threading from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration +from ahriman.models.repository_id import RepositoryId class Daemon(Handler): @@ -30,19 +31,20 @@ class Daemon(Handler): """ @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ from ahriman.application.handlers import Update - Update.run(args, architecture, configuration, report=report) - timer = threading.Timer(args.interval, Daemon.run, args=[args, architecture, configuration], + Update.run(args, repository_id, configuration, report=report) + timer = threading.Timer(args.interval, Daemon.run, args=[args, repository_id, configuration], kwargs={"report": report}) timer.start() timer.join() diff --git a/src/ahriman/application/handlers/dump.py b/src/ahriman/application/handlers/dump.py index d212b36b..63fa6358 100644 --- a/src/ahriman/application/handlers/dump.py +++ b/src/ahriman/application/handlers/dump.py @@ -22,6 +22,7 @@ import argparse from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.core.formatters import ConfigurationPathsPrinter, ConfigurationPrinter, StringPrinter +from ahriman.models.repository_id import RepositoryId class Dump(Handler): @@ -32,13 +33,14 @@ class Dump(Handler): ALLOW_AUTO_ARCHITECTURE_RUN = False @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ diff --git a/src/ahriman/application/handlers/handler.py b/src/ahriman/application/handlers/handler.py index c5c61be2..a74a6d15 100644 --- a/src/ahriman/application/handlers/handler.py +++ b/src/ahriman/application/handlers/handler.py @@ -25,7 +25,8 @@ from multiprocessing import Pool from ahriman.application.lock import Lock from ahriman.core.configuration import Configuration from ahriman.core.exceptions import ExitCode, MissingArchitectureError, MultipleArchitecturesError -from ahriman.core.log import Log +from ahriman.core.log.log_loader import LogLoader +from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_paths import RepositoryPaths @@ -50,56 +51,25 @@ class Handler: ALLOW_MULTI_ARCHITECTURE_RUN = True @classmethod - def architectures_extract(cls, args: argparse.Namespace) -> list[str]: - """ - get known architectures - - Args: - args(argparse.Namespace): command line args - - Returns: - list[str]: list of architectures for which tree is created - - Raises: - MissingArchitectureError: if no architecture set and automatic detection is not allowed or failed - """ - if not cls.ALLOW_AUTO_ARCHITECTURE_RUN and args.architecture is None: - # for some parsers (e.g. config) we need to run with specific architecture - # for those cases architecture must be set explicitly - raise MissingArchitectureError(args.command) - if args.architecture: # architecture is specified explicitly - return sorted(set(args.architecture)) - - configuration = Configuration() - configuration.load(args.configuration) - # wtf??? - root = configuration.getpath("repository", "root") # pylint: disable=assignment-from-no-return - architectures = RepositoryPaths.known_architectures(root) - - if not architectures: # well we did not find anything - raise MissingArchitectureError(args.command) - return sorted(architectures) - - @classmethod - def call(cls, args: argparse.Namespace, architecture: str) -> bool: + def call(cls, args: argparse.Namespace, repository_id: RepositoryId) -> bool: """ additional function to wrap all calls for multiprocessing library Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier Returns: bool: True on success, False otherwise """ try: - configuration = Configuration.from_path(args.configuration, architecture) + configuration = Configuration.from_path(args.configuration, repository_id) - log_handler = Log.handler(args.log_handler) - Log.load(configuration, log_handler, quiet=args.quiet, report=args.report) + log_handler = LogLoader.handler(args.log_handler) + LogLoader.load(configuration, log_handler, quiet=args.quiet, report=args.report) - with Lock(args, architecture, configuration): - cls.run(args, architecture, configuration, report=args.report) + with Lock(args, repository_id, configuration): + cls.run(args, repository_id, configuration, report=args.report) return True except ExitCode: @@ -123,28 +93,70 @@ class Handler: Raises: MultipleArchitecturesError: if more than one architecture supplied and no multi architecture supported """ - architectures = cls.architectures_extract(args) + repositories = cls.repositories_extract(args) # actually we do not have to spawn another process if it is single-process application, do we? - if len(architectures) > 1: + if len(repositories) > 1: if not cls.ALLOW_MULTI_ARCHITECTURE_RUN: raise MultipleArchitecturesError(args.command) - with Pool(len(architectures)) as pool: - result = pool.starmap(cls.call, [(args, architecture) for architecture in architectures]) + with Pool(len(repositories)) as pool: + result = pool.starmap(cls.call, [(args, repository_id) for repository_id in repositories]) else: - result = [cls.call(args, architectures.pop())] + result = [cls.call(args, repositories.pop())] return 0 if all(result) else 1 @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def repositories_extract(cls, args: argparse.Namespace) -> list[RepositoryId]: + """ + get known architectures + + Args: + args(argparse.Namespace): command line args + + Returns: + tuple[str | None, str]: list of repository names and architectures for which tree is created + + Raises: + MissingArchitectureError: if no architecture set and automatic detection is not allowed or failed + """ + if not cls.ALLOW_AUTO_ARCHITECTURE_RUN and args.architecture is None: + # for some parsers (e.g. config) we need to run with specific architecture + # for those cases architecture must be set explicitly + raise MissingArchitectureError(args.command) + + configuration = Configuration() + configuration.load(args.configuration) + name = configuration.get("repository", "name", fallback="") # will only be used for legacy mode + + if args.architecture: # architecture is specified explicitly + repositories = args.repository or [name] # fallback for legacy mode + return sorted( + set( + RepositoryId(architecture, repository) + for architecture in args.architecture + for repository in repositories + ) + ) + + # wtf??? + root = configuration.getpath("repository", "root") # pylint: disable=assignment-from-no-return + architectures = RepositoryPaths.known_architectures(root, name) + + if not architectures: # well we did not find anything + raise MissingArchitectureError(args.command) + return sorted(architectures) + + @classmethod + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting diff --git a/src/ahriman/application/handlers/help.py b/src/ahriman/application/handlers/help.py index b7f7aeed..ba0354f0 100644 --- a/src/ahriman/application/handlers/help.py +++ b/src/ahriman/application/handlers/help.py @@ -21,6 +21,7 @@ import argparse from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration +from ahriman.models.repository_id import RepositoryId class Help(Handler): @@ -31,13 +32,14 @@ class Help(Handler): ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture" @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ diff --git a/src/ahriman/application/handlers/key_import.py b/src/ahriman/application/handlers/key_import.py index 4424d910..2d4ed7f7 100644 --- a/src/ahriman/application/handlers/key_import.py +++ b/src/ahriman/application/handlers/key_import.py @@ -22,6 +22,7 @@ import argparse from ahriman.application.application import Application from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration +from ahriman.models.repository_id import RepositoryId class KeyImport(Handler): @@ -32,15 +33,16 @@ class KeyImport(Handler): ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture" @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - application = Application(architecture, configuration, report=report) + application = Application(repository_id, configuration, report=report) application.repository.sign.key_import(args.key_server, args.key) diff --git a/src/ahriman/application/handlers/patch.py b/src/ahriman/application/handlers/patch.py index 30c2ec13..264fbd89 100644 --- a/src/ahriman/application/handlers/patch.py +++ b/src/ahriman/application/handlers/patch.py @@ -30,6 +30,7 @@ from ahriman.core.formatters import PatchPrinter from ahriman.models.action import Action from ahriman.models.package import Package from ahriman.models.pkgbuild_patch import PkgbuildPatch +from ahriman.models.repository_id import RepositoryId class Patch(Handler): @@ -38,17 +39,18 @@ class Patch(Handler): """ @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - application = Application(architecture, configuration, report=report) + application = Application(repository_id, configuration, report=report) application.on_start() match args.action: @@ -56,7 +58,7 @@ class Patch(Handler): patch = Patch.patch_create_from_function(args.variable, args.patch) Patch.patch_set_create(application, args.package, patch) case Action.Update: - package_base, patch = Patch.patch_create_from_diff(args.package, architecture, args.track) + package_base, patch = Patch.patch_create_from_diff(args.package, repository_id.architecture, args.track) Patch.patch_set_create(application, package_base, patch) case Action.List: Patch.patch_set_list(application, args.package, args.variable, args.exit_code) diff --git a/src/ahriman/application/handlers/rebuild.py b/src/ahriman/application/handlers/rebuild.py index 595ef8ae..8ee39b52 100644 --- a/src/ahriman/application/handlers/rebuild.py +++ b/src/ahriman/application/handlers/rebuild.py @@ -24,6 +24,7 @@ from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.models.build_status import BuildStatusEnum from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId class Rebuild(Handler): @@ -32,17 +33,18 @@ class Rebuild(Handler): """ @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - application = Application(architecture, configuration, report=report) + application = Application(repository_id, configuration, report=report) application.on_start() packages = Rebuild.extract_packages(application, args.status, from_database=args.from_database) diff --git a/src/ahriman/application/handlers/remove.py b/src/ahriman/application/handlers/remove.py index 1c65cb85..349075ba 100644 --- a/src/ahriman/application/handlers/remove.py +++ b/src/ahriman/application/handlers/remove.py @@ -22,6 +22,7 @@ import argparse from ahriman.application.application import Application from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration +from ahriman.models.repository_id import RepositoryId class Remove(Handler): @@ -30,16 +31,17 @@ class Remove(Handler): """ @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - application = Application(architecture, configuration, report=report) + application = Application(repository_id, configuration, report=report) application.on_start() application.remove(args.package) diff --git a/src/ahriman/application/handlers/remove_unknown.py b/src/ahriman/application/handlers/remove_unknown.py index 2fda7576..f6f01f36 100644 --- a/src/ahriman/application/handlers/remove_unknown.py +++ b/src/ahriman/application/handlers/remove_unknown.py @@ -23,6 +23,7 @@ from ahriman.application.application import Application from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.core.formatters import StringPrinter +from ahriman.models.repository_id import RepositoryId class RemoveUnknown(Handler): @@ -31,17 +32,18 @@ class RemoveUnknown(Handler): """ @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - application = Application(architecture, configuration, report=report) + application = Application(repository_id, configuration, report=report) application.on_start() unknown_packages = application.unknown() diff --git a/src/ahriman/application/handlers/restore.py b/src/ahriman/application/handlers/restore.py index 966b43ce..05901904 100644 --- a/src/ahriman/application/handlers/restore.py +++ b/src/ahriman/application/handlers/restore.py @@ -23,6 +23,7 @@ from tarfile import TarFile from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration +from ahriman.models.repository_id import RepositoryId class Restore(Handler): @@ -33,13 +34,14 @@ class Restore(Handler): ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture" @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ diff --git a/src/ahriman/application/handlers/search.py b/src/ahriman/application/handlers/search.py index 6fabcb89..c9607775 100644 --- a/src/ahriman/application/handlers/search.py +++ b/src/ahriman/application/handlers/search.py @@ -29,6 +29,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.exceptions import OptionError from ahriman.core.formatters import AurPrinter from ahriman.models.aur_package import AURPackage +from ahriman.models.repository_id import RepositoryId class Search(Handler): @@ -47,17 +48,18 @@ class Search(Handler): } @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - application = Application(architecture, configuration, report=report) + application = Application(repository_id, configuration, report=report) official_packages_list = Official.multisearch(*args.search, pacman=application.repository.pacman) aur_packages_list = AUR.multisearch(*args.search, pacman=application.repository.pacman) diff --git a/src/ahriman/application/handlers/service_updates.py b/src/ahriman/application/handlers/service_updates.py index 96e2f550..758c4a5b 100644 --- a/src/ahriman/application/handlers/service_updates.py +++ b/src/ahriman/application/handlers/service_updates.py @@ -25,6 +25,7 @@ from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.core.formatters import UpdatePrinter from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId class ServiceUpdates(Handler): @@ -35,17 +36,18 @@ class ServiceUpdates(Handler): ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture" @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - application = Application(architecture, configuration, report=report) + application = Application(repository_id, configuration, report=report) remote = Package.from_aur("ahriman", application.repository.pacman, None) _, release = remote.version.rsplit("-", 1) # we don't store pkgrel locally, so we just append it diff --git a/src/ahriman/application/handlers/setup.py b/src/ahriman/application/handlers/setup.py index 3515d9cf..8700c4ee 100644 --- a/src/ahriman/application/handlers/setup.py +++ b/src/ahriman/application/handlers/setup.py @@ -25,6 +25,7 @@ from pwd import getpwuid from ahriman.application.application import Application from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration +from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_paths import RepositoryPaths from ahriman.models.user import User @@ -46,81 +47,80 @@ class Setup(Handler): SUDOERS_DIR_PATH = Path("/etc") / "sudoers.d" @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - Setup.configuration_create_ahriman(args, architecture, args.repository, configuration) + Setup.configuration_create_ahriman(args, repository_id, configuration) configuration.reload() - application = Application(architecture, configuration, report=report) + application = Application(repository_id, configuration, report=report) Setup.configuration_create_makepkg(args.packager, args.makeflags_jobs, application.repository.paths) - Setup.executable_create(application.repository.paths, args.build_command, architecture) + Setup.executable_create(application.repository.paths, repository_id) repository_server = f"file://{application.repository.paths.repository}" if args.server is None else args.server - Setup.configuration_create_devtools(args.build_command, architecture, args.from_configuration, args.mirror, - args.multilib, args.repository, repository_server) - Setup.configuration_create_sudo(application.repository.paths, args.build_command, architecture) + Setup.configuration_create_devtools(repository_id, args.from_configuration, args.mirror, args.multilib, + repository_server) + Setup.configuration_create_sudo(application.repository.paths, repository_id) application.repository.repo.init() # lazy database sync application.repository.pacman.handle # pylint: disable=pointless-statement @staticmethod - def build_command(root: Path, prefix: str, architecture: str) -> Path: + def build_command(root: Path, repository_id: RepositoryId) -> Path: """ generate build command name Args: root(Path): root directory for the build command (must be root of the repository) - prefix(str): command prefix in {prefix}-{architecture}-build - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier Returns: Path: valid devtools command name """ - return root / f"{prefix}-{architecture}-build" + return root / f"{repository_id.name}-{repository_id.architecture}-build" @staticmethod - def configuration_create_ahriman(args: argparse.Namespace, architecture: str, repository: str, + def configuration_create_ahriman(args: argparse.Namespace, repository_id: RepositoryId, root: Configuration) -> None: """ create service specific configuration Args: args(argparse.Namespace): command line args - architecture(str): repository architecture - repository(str): repository name + repository_id(RepositoryId): repository unique identifier root(Configuration): root configuration instance """ configuration = Configuration() - section = Configuration.section_name("build", architecture) - build_command = Setup.build_command(root.repository_paths.root, args.build_command, architecture) + section = Configuration.section_name("build", repository_id.name, repository_id.architecture) + build_command = Setup.build_command(root.repository_paths.root, repository_id) configuration.set_option(section, "build_command", str(build_command)) - configuration.set_option("repository", "name", repository) + configuration.set_option("repository", "name", repository_id.name) # backward compatibility for docker if args.build_as_user is not None: configuration.set_option(section, "makechrootpkg_flags", f"-U {args.build_as_user}") - section = Configuration.section_name("alpm", architecture) + section = Configuration.section_name("alpm", repository_id.name, repository_id.architecture) if args.mirror is not None: configuration.set_option(section, "mirror", args.mirror) if not args.multilib: repositories = filter(lambda r: r != "multilib", root.getlist("alpm", "repositories")) configuration.set_option(section, "repositories", " ".join(repositories)) - section = Configuration.section_name("sign", architecture) + section = Configuration.section_name("sign", repository_id.name, repository_id.architecture) if args.sign_key is not None: configuration.set_option(section, "target", " ".join([target.name.lower() for target in args.sign_target])) configuration.set_option(section, "key", args.sign_key) - section = Configuration.section_name("web", architecture) + section = Configuration.section_name("web", repository_id.name, repository_id.architecture) if args.web_port is not None: configuration.set_option(section, "port", str(args.web_port)) if args.web_unix_socket is not None: @@ -134,8 +134,8 @@ class Setup(Handler): configuration.write(ahriman_configuration) @staticmethod - def configuration_create_devtools(prefix: str, architecture: str, source: Path, mirror: str | None, - multilib: bool, repository: str, repository_server: str) -> None: + def configuration_create_devtools(repository_id: RepositoryId, source: Path, mirror: str | None, + multilib: bool, repository_server: str) -> None: """ create configuration for devtools based on ``source`` configuration @@ -143,12 +143,10 @@ class Setup(Handler): devtools does not allow to specify the pacman configuration, thus we still have to use configuration in /usr Args: - prefix(str): command prefix in {prefix}-{architecture}-build - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier source(Path): path to source configuration file mirror(str | None): link to package server mirror multilib(bool): add or do not multilib repository to the configuration - repository(str): repository name repository_server(str): url of the repository """ # allow_no_value=True is required because pacman uses boolean configuration in which just keys present @@ -163,7 +161,7 @@ class Setup(Handler): configuration.read(source) # set our architecture now - configuration.set_option("options", "Architecture", architecture) + configuration.set_option("options", "Architecture", repository_id.architecture) # add multilib if multilib: @@ -178,10 +176,10 @@ class Setup(Handler): configuration.set_option(section, "Server", mirror) # add repository itself - configuration.set_option(repository, "SigLevel", "Never") # we don't care - configuration.set_option(repository, "Server", repository_server) + configuration.set_option(repository_id.name, "SigLevel", "Never") # we don't care + configuration.set_option(repository_id.name, "Server", repository_server) - target = source.parent / f"{prefix}-{architecture}.conf" + target = source.parent / f"{repository_id.name}-{repository_id.architecture}.conf" with target.open("w") as devtools_configuration: configuration.write(devtools_configuration) @@ -205,31 +203,29 @@ class Setup(Handler): (home_dir / ".makepkg.conf").write_text(content, encoding="utf8") @staticmethod - def configuration_create_sudo(paths: RepositoryPaths, prefix: str, architecture: str) -> None: + def configuration_create_sudo(paths: RepositoryPaths, repository_id: RepositoryId) -> None: """ create configuration to run build command with sudo without password Args: paths(RepositoryPaths): repository paths instance - prefix(str): command prefix in {prefix}-{architecture}-build - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier """ - command = Setup.build_command(paths.root, prefix, architecture) - sudoers_file = Setup.build_command(Setup.SUDOERS_DIR_PATH, prefix, architecture) + command = Setup.build_command(paths.root, repository_id) + sudoers_file = Setup.build_command(Setup.SUDOERS_DIR_PATH, repository_id) sudoers_file.write_text(f"ahriman ALL=(ALL) NOPASSWD:SETENV: {command} *\n", encoding="utf8") sudoers_file.chmod(0o400) # security! @staticmethod - def executable_create(paths: RepositoryPaths, prefix: str, architecture: str) -> None: + def executable_create(paths: RepositoryPaths, repository_id: RepositoryId) -> None: """ create executable for the service Args: paths(RepositoryPaths): repository paths instance - prefix(str): command prefix in {prefix}-{architecture}-build - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier """ - command = Setup.build_command(paths.root, prefix, architecture) + command = Setup.build_command(paths.root, repository_id) command.unlink(missing_ok=True) command.symlink_to(Setup.ARCHBUILD_COMMAND_PATH) paths.chown(command) # we would like to keep owner inside ahriman's home diff --git a/src/ahriman/application/handlers/shell.py b/src/ahriman/application/handlers/shell.py index ef9fb531..9e1ef7cb 100644 --- a/src/ahriman/application/handlers/shell.py +++ b/src/ahriman/application/handlers/shell.py @@ -26,6 +26,7 @@ from pathlib import Path from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.core.formatters import StringPrinter +from ahriman.models.repository_id import RepositoryId class Shell(Handler): @@ -36,13 +37,14 @@ class Shell(Handler): ALLOW_MULTI_ARCHITECTURE_RUN = False @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ @@ -50,7 +52,13 @@ class Shell(Handler): # licensed by https://creativecommons.org/licenses/by-sa/3.0 path = Path(sys.prefix) / "share" / "ahriman" / "templates" / "shell" StringPrinter(path.read_text(encoding="utf8")).print(verbose=False) - local_variables = {"architecture": architecture, "configuration": configuration} + + local_variables = { + "architecture": repository_id.architecture, + "configuration": configuration, + "repository_id": repository_id, + } + if args.code is None: code.interact(local=local_variables) else: diff --git a/src/ahriman/application/handlers/sign.py b/src/ahriman/application/handlers/sign.py index e572d55b..4d1ac575 100644 --- a/src/ahriman/application/handlers/sign.py +++ b/src/ahriman/application/handlers/sign.py @@ -22,6 +22,7 @@ import argparse from ahriman.application.application import Application from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration +from ahriman.models.repository_id import RepositoryId class Sign(Handler): @@ -30,14 +31,15 @@ class Sign(Handler): """ @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - Application(architecture, configuration, report=report).sign(args.package) + Application(repository_id, configuration, report=report).sign(args.package) diff --git a/src/ahriman/application/handlers/status.py b/src/ahriman/application/handlers/status.py index 477d0a3f..75776e94 100644 --- a/src/ahriman/application/handlers/status.py +++ b/src/ahriman/application/handlers/status.py @@ -27,6 +27,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.formatters import PackagePrinter, StatusPrinter from ahriman.models.build_status import BuildStatus from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId class Status(Handler): @@ -37,18 +38,19 @@ class Status(Handler): ALLOW_AUTO_ARCHITECTURE_RUN = False @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ # we are using reporter here - client = Application(architecture, configuration, report=True).repository.reporter + client = Application(repository_id, configuration, report=True).repository.reporter if args.ahriman: service_status = client.status_get() StatusPrinter(service_status.status).print(verbose=args.info) diff --git a/src/ahriman/application/handlers/status_update.py b/src/ahriman/application/handlers/status_update.py index 803ca8e5..3df723ec 100644 --- a/src/ahriman/application/handlers/status_update.py +++ b/src/ahriman/application/handlers/status_update.py @@ -23,6 +23,7 @@ from ahriman.application.application import Application from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.models.action import Action +from ahriman.models.repository_id import RepositoryId class StatusUpdate(Handler): @@ -33,18 +34,19 @@ class StatusUpdate(Handler): ALLOW_AUTO_ARCHITECTURE_RUN = False @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ # we are using reporter here - client = Application(architecture, configuration, report=True).repository.reporter + client = Application(repository_id, configuration, report=True).repository.reporter match args.action: case Action.Update if args.package: diff --git a/src/ahriman/application/handlers/structure.py b/src/ahriman/application/handlers/structure.py index 596a0344..f3ba279c 100644 --- a/src/ahriman/application/handlers/structure.py +++ b/src/ahriman/application/handlers/structure.py @@ -24,6 +24,7 @@ from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.core.formatters import StringPrinter, TreePrinter from ahriman.core.tree import Tree +from ahriman.models.repository_id import RepositoryId class Structure(Handler): @@ -34,17 +35,18 @@ class Structure(Handler): ALLOW_AUTO_ARCHITECTURE_RUN = False @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - application = Application(architecture, configuration, report=report) + application = Application(repository_id, configuration, report=report) partitions = Tree.partition(application.repository.packages(), count=args.partitions) for partition_id, partition in enumerate(partitions): diff --git a/src/ahriman/application/handlers/triggers.py b/src/ahriman/application/handlers/triggers.py index f47b4081..23d63034 100644 --- a/src/ahriman/application/handlers/triggers.py +++ b/src/ahriman/application/handlers/triggers.py @@ -22,6 +22,7 @@ import argparse from ahriman.application.application import Application from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration +from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result @@ -31,19 +32,20 @@ class Triggers(Handler): """ @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - application = Application(architecture, configuration, report=report) + application = Application(repository_id, configuration, report=report) if args.trigger: loader = application.repository.triggers - loader.triggers = [loader.load_trigger(trigger, architecture, configuration) for trigger in args.trigger] + loader.triggers = [loader.load_trigger(trigger, repository_id, configuration) for trigger in args.trigger] application.on_start() application.on_result(Result()) diff --git a/src/ahriman/application/handlers/unsafe_commands.py b/src/ahriman/application/handlers/unsafe_commands.py index a40e9eec..e6d384b9 100644 --- a/src/ahriman/application/handlers/unsafe_commands.py +++ b/src/ahriman/application/handlers/unsafe_commands.py @@ -22,6 +22,7 @@ import argparse from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.core.formatters import StringPrinter +from ahriman.models.repository_id import RepositoryId class UnsafeCommands(Handler): @@ -32,13 +33,14 @@ class UnsafeCommands(Handler): ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture" @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ diff --git a/src/ahriman/application/handlers/update.py b/src/ahriman/application/handlers/update.py index f2fa9efd..9824fda9 100644 --- a/src/ahriman/application/handlers/update.py +++ b/src/ahriman/application/handlers/update.py @@ -25,6 +25,7 @@ from ahriman.application.application import Application from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.models.packagers import Packagers +from ahriman.models.repository_id import RepositoryId class Update(Handler): @@ -33,17 +34,18 @@ class Update(Handler): """ @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - application = Application(architecture, configuration, report=report, refresh_pacman_database=args.refresh) + application = Application(repository_id, configuration, report=report, refresh_pacman_database=args.refresh) application.on_start() packages = application.updates(args.package, aur=args.aur, local=args.local, manual=args.manual, vcs=args.vcs) Update.check_if_empty(args.exit_code, not packages) diff --git a/src/ahriman/application/handlers/users.py b/src/ahriman/application/handlers/users.py index 92a33b70..11357e21 100644 --- a/src/ahriman/application/handlers/users.py +++ b/src/ahriman/application/handlers/users.py @@ -26,6 +26,7 @@ from ahriman.core.database import SQLite from ahriman.core.exceptions import PasswordError from ahriman.core.formatters import UserPrinter from ahriman.models.action import Action +from ahriman.models.repository_id import RepositoryId from ahriman.models.user import User @@ -37,13 +38,14 @@ class Users(Handler): ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture" @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ diff --git a/src/ahriman/application/handlers/validate.py b/src/ahriman/application/handlers/validate.py index 64214dff..741c436d 100644 --- a/src/ahriman/application/handlers/validate.py +++ b/src/ahriman/application/handlers/validate.py @@ -29,6 +29,7 @@ from ahriman.core.configuration.validator import Validator from ahriman.core.exceptions import ExtensionError from ahriman.core.formatters import ValidationPrinter from ahriman.core.triggers import TriggerLoader +from ahriman.models.repository_id import RepositoryId class Validate(Handler): @@ -39,17 +40,18 @@ class Validate(Handler): ALLOW_AUTO_ARCHITECTURE_RUN = False @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ - schema = Validate.schema(architecture, configuration) + schema = Validate.schema(repository_id, configuration) validator = Validator(configuration=configuration, schema=schema) if validator.validate(configuration.dump()): @@ -61,12 +63,12 @@ class Validate(Handler): Validate.check_if_empty(args.exit_code, True) @staticmethod - def schema(architecture: str, configuration: Configuration) -> ConfigurationSchema: + def schema(repository_id: RepositoryId, configuration: Configuration) -> ConfigurationSchema: """ get schema with triggers Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance Returns: @@ -85,12 +87,12 @@ class Validate(Handler): continue # default settings if any - for schema_name, schema in trigger_class.configuration_schema(architecture, None).items(): + for schema_name, schema in trigger_class.configuration_schema(repository_id, None).items(): erased = Validate.schema_erase_required(copy.deepcopy(schema)) root[schema_name] = Validate.schema_merge(root.get(schema_name, {}), erased) # settings according to enabled triggers - for schema_name, schema in trigger_class.configuration_schema(architecture, configuration).items(): + for schema_name, schema in trigger_class.configuration_schema(repository_id, configuration).items(): root[schema_name] = Validate.schema_merge(root.get(schema_name, {}), copy.deepcopy(schema)) return root diff --git a/src/ahriman/application/handlers/versions.py b/src/ahriman/application/handlers/versions.py index dbde2dbb..c966fd18 100644 --- a/src/ahriman/application/handlers/versions.py +++ b/src/ahriman/application/handlers/versions.py @@ -28,6 +28,7 @@ from ahriman import __version__ from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.core.formatters import VersionPrinter +from ahriman.models.repository_id import RepositoryId class Versions(Handler): @@ -42,13 +43,14 @@ class Versions(Handler): PEP423_PACKAGE_NAME = re.compile(r"^[A-Za-z0-9._-]+") @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ diff --git a/src/ahriman/application/handlers/web.py b/src/ahriman/application/handlers/web.py index 6cac79b5..50baf7b0 100644 --- a/src/ahriman/application/handlers/web.py +++ b/src/ahriman/application/handlers/web.py @@ -24,6 +24,7 @@ from collections.abc import Generator from ahriman.application.handlers import Handler from ahriman.core.configuration import Configuration from ahriman.core.spawn import Spawn +from ahriman.models.repository_id import RepositoryId class Web(Handler): @@ -35,24 +36,25 @@ class Web(Handler): ALLOW_MULTI_ARCHITECTURE_RUN = False # required to be able to spawn external processes @classmethod - def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None: + def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *, + report: bool) -> None: """ callback for command line Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance report(bool): force enable or disable reporting """ # we are using local import for optional dependencies from ahriman.web.web import run_server, setup_service - spawner_args = Web.extract_arguments(args, architecture, configuration) - spawner = Spawn(args.parser(), architecture, list(spawner_args)) + spawner_args = Web.extract_arguments(args, repository_id, configuration) + spawner = Spawn(args.parser(), repository_id, list(spawner_args)) spawner.start() - application = setup_service(architecture, configuration, spawner) + application = setup_service(repository_id, configuration, spawner) run_server(application) # terminate spawn process at the last @@ -60,21 +62,23 @@ class Web(Handler): spawner.join() @staticmethod - def extract_arguments(args: argparse.Namespace, architecture: str, + def extract_arguments(args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration) -> Generator[str, None, None]: """ extract list of arguments used for current command, except for command specific ones Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance Returns: Generator[str, None, None]: command line arguments which were used for this specific command """ # read architecture from the same argument list - yield from ["--architecture", architecture] + yield from ["--architecture", repository_id.architecture] + if repository_id.name is not None: + yield from ["--repository", repository_id.name] # read configuration path from current settings if (configuration_path := configuration.path) is not None: yield from ["--configuration", str(configuration_path)] diff --git a/src/ahriman/application/lock.py b/src/ahriman/application/lock.py index a2724673..f9203089 100644 --- a/src/ahriman/application/lock.py +++ b/src/ahriman/application/lock.py @@ -30,6 +30,7 @@ from ahriman.core.log import LazyLogging from ahriman.core.status.client import Client from ahriman.core.util import check_user from ahriman.models.build_status import BuildStatusEnum +from ahriman.models.repository_id import RepositoryId from ahriman.models.waiter import Waiter @@ -50,26 +51,29 @@ class Lock(LazyLogging): The common flow is to create instance in ``with`` block and handle exceptions after all:: >>> from ahriman.core.configuration import Configuration + >>> from ahriman.models.repository_id import RepositoryId >>> >>> configuration = Configuration() >>> try: - >>> with Lock(args, "x86_64", configuration): + >>> with Lock(args, RepositoryId("x86_64", "aur-clone"), configuration): >>> perform_actions() >>> except Exception as exception: >>> handle_exceptions(exception) """ - def __init__(self, args: argparse.Namespace, architecture: str, configuration: Configuration) -> None: + def __init__(self, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration) -> None: """ default constructor Args: args(argparse.Namespace): command line args - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance """ + lock_suffix = f"{repository_id.name}_{repository_id.architecture}" if repository_id.name is not None else repository_id.architecture self.path: Path | None = \ - args.lock.with_stem(f"{args.lock.stem}_{architecture}") if args.lock is not None else None + args.lock.with_stem(f"{args.lock.stem}_{lock_suffix}") if args.lock is not None else None + self.force: bool = args.force self.unsafe: bool = args.unsafe self.wait_timeout: int = args.wait_timeout diff --git a/src/ahriman/core/alpm/pacman.py b/src/ahriman/core/alpm/pacman.py index 6fb56540..90eeb68b 100644 --- a/src/ahriman/core/alpm/pacman.py +++ b/src/ahriman/core/alpm/pacman.py @@ -23,11 +23,13 @@ from collections.abc import Callable, Generator from functools import cached_property from pathlib import Path from pyalpm import DB, Handle, Package, SIG_PACKAGE, error as PyalpmError # type: ignore[import] +from string import Template from ahriman.core.configuration import Configuration from ahriman.core.log import LazyLogging from ahriman.core.util import trim_package from ahriman.models.pacman_synchronization import PacmanSynchronization +from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_paths import RepositoryPaths @@ -36,18 +38,18 @@ class Pacman(LazyLogging): alpm wrapper """ - def __init__(self, architecture: str, configuration: Configuration, *, + def __init__(self, repository_id: RepositoryId, configuration: Configuration, *, refresh_database: PacmanSynchronization) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance refresh_database(PacmanSynchronization): synchronize local cache to remote """ self.__create_handle_fn: Callable[[], Handle] = lambda: self.__create_handle( - architecture, configuration, refresh_database=refresh_database) + repository_id, configuration, refresh_database=refresh_database) @cached_property def handle(self) -> Handle: @@ -59,13 +61,13 @@ class Pacman(LazyLogging): """ return self.__create_handle_fn() - def __create_handle(self, architecture: str, configuration: Configuration, *, + def __create_handle(self, repository_id: RepositoryId, configuration: Configuration, *, refresh_database: PacmanSynchronization) -> Handle: """ create lazy handle function Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance refresh_database(PacmanSynchronization): synchronize local cache to remote @@ -81,7 +83,7 @@ class Pacman(LazyLogging): handle = Handle(str(root), str(database_path)) for repository in configuration.getlist("alpm", "repositories"): - database = self.database_init(handle, repository, mirror, architecture) + database = self.database_init(handle, repository, mirror, repository_id.architecture) self.database_copy(handle, database, pacman_root, paths, use_ahriman_cache=use_ahriman_cache) if use_ahriman_cache and refresh_database: @@ -136,8 +138,14 @@ class Pacman(LazyLogging): """ self.logger.info("loading pacman database %s", repository) database: DB = handle.register_syncdb(repository, SIG_PACKAGE) + # replace variables in mirror address - database.servers = [mirror.replace("$repo", repository).replace("$arch", architecture)] + variables = { + "arch": architecture, + "repo": repository, + } + database.servers = [Template(mirror).safe_substitute(variables)] + return database def database_sync(self, handle: Handle, *, force: bool) -> None: diff --git a/src/ahriman/core/build_tools/sources.py b/src/ahriman/core/build_tools/sources.py index ea7291dd..49d0b195 100644 --- a/src/ahriman/core/build_tools/sources.py +++ b/src/ahriman/core/build_tools/sources.py @@ -154,7 +154,7 @@ class Sources(LazyLogging): shutil.copytree(cache_dir, sources_dir, dirs_exist_ok=True) instance.fetch(sources_dir, package.remote) - patches.extend(instance.extend_architectures(sources_dir, paths.architecture)) + patches.extend(instance.extend_architectures(sources_dir, paths.repository_id.architecture)) for patch in patches: instance.patch_apply(sources_dir, patch) diff --git a/src/ahriman/core/configuration/configuration.py b/src/ahriman/core/configuration/configuration.py index 85ad33df..562fed4c 100644 --- a/src/ahriman/core/configuration/configuration.py +++ b/src/ahriman/core/configuration/configuration.py @@ -27,6 +27,7 @@ from typing import Any, Self from ahriman.core.configuration.shell_interpolator import ShellInterpolator from ahriman.core.exceptions import InitializeError +from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_paths import RepositoryPaths @@ -38,9 +39,9 @@ class Configuration(configparser.RawConfigParser): ARCHITECTURE_SPECIFIC_SECTIONS(list[str]): (class attribute) known sections which can be architecture specific. Required by dump and merging functions SYSTEM_CONFIGURATION_PATH(Path): (class attribute) default system configuration path distributed by package - architecture(str | None): repository architecture includes(list[Path]): list of includes which were read path(Path | None): path to root configuration file + repository_id(RepositoryId | None): repository unique identifier Examples: Configuration class provides additional method in order to handle application configuration. Since this class is @@ -49,7 +50,7 @@ class Configuration(configparser.RawConfigParser): >>> from pathlib import Path >>> - >>> configuration = Configuration.from_path(Path("/etc/ahriman.ini"), "x86_64") + >>> configuration = Configuration.from_path(Path("/etc/ahriman.ini"), RepositoryId("x86_64", "aur-clone")) >>> repository_name = configuration.get("repository", "name") >>> makepkg_flags = configuration.getlist("build", "makepkg_flags") @@ -59,7 +60,7 @@ class Configuration(configparser.RawConfigParser): In order to get current settings, the ``check_loaded`` method can be used. This method will raise an ``InitializeError`` in case if configuration was not yet loaded:: - >>> path, architecture = configuration.check_loaded() + >>> path, repository_id = configuration.check_loaded() """ ARCHITECTURE_SPECIFIC_SECTIONS = ["alpm", "build", "sign", "web"] @@ -84,7 +85,7 @@ class Configuration(configparser.RawConfigParser): } ) - self.architecture: str | None = None + self.repository_id: RepositoryId | None = None self.path: Path | None = None self.includes: list[Path] = [] @@ -111,12 +112,13 @@ class Configuration(configparser.RawConfigParser): @property def repository_name(self) -> str: """ - repository name as defined by configuration + repository name for backward compatibility Returns: - str: repository name from configuration + str: repository name """ - return self.get("repository", "name") + _, repository_id = self.check_loaded() + return repository_id.name @property def repository_paths(self) -> RepositoryPaths: @@ -126,39 +128,60 @@ class Configuration(configparser.RawConfigParser): Returns: RepositoryPaths: repository paths instance """ - _, architecture = self.check_loaded() - return RepositoryPaths(self.getpath("repository", "root"), architecture) + _, repository_id = self.check_loaded() + return RepositoryPaths(self.getpath("repository", "root"), repository_id) @classmethod - def from_path(cls, path: Path, architecture: str) -> Self: + def from_path(cls, path: Path, repository_id: RepositoryId) -> Self: """ constructor with full object initialization Args: path(Path): path to root configuration file - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier Returns: Self: configuration instance """ configuration = cls() configuration.load(path) - configuration.merge_sections(architecture) + configuration.merge_sections(repository_id) return configuration @staticmethod - def section_name(section: str, suffix: str) -> str: + def override_sections(section: str, repository_id: RepositoryId) -> list[str]: + """ + extract override sections + + Args: + section(str): section name + repository_id(RepositoryId): repository unique identifier + + Returns: + list[str]: architecture and repository specific sections in correct order + """ + # the valid order is global < per architecture < per repository < per repository and architecture + return [ + Configuration.section_name(section, repository_id.architecture), # architecture specific override + Configuration.section_name(section, repository_id.name), + Configuration.section_name(section, repository_id.name, repository_id.architecture), + ] + + @staticmethod + def section_name(section: str, *suffixes: str) -> str: """ generate section name for sections which depends on context Args: section(str): section name - suffix(str): session suffix, e.g. repository architecture + *suffixes(str): session suffix, e.g. repository architecture Returns: str: correct section name for repository specific section """ - return f"{section}:{suffix}" + for suffix in suffixes: + section = f"{section}:{suffix}" + return section def _convert_path(self, value: str) -> Path: """ @@ -175,19 +198,19 @@ class Configuration(configparser.RawConfigParser): return path return self.path.parent / path - def check_loaded(self) -> tuple[Path, str]: + def check_loaded(self) -> tuple[Path, RepositoryId]: """ check if service was actually loaded Returns: - tuple[Path, str]: configuration root path and architecture if loaded + tuple[Path, RepositoryId]: configuration root path and architecture if loaded Raises: InitializeError: in case if architecture and/or path are not set """ - if self.path is None or self.architecture is None: - raise InitializeError("Configuration path and/or architecture are not set") - return self.path, self.architecture + if self.path is None or self.repository_id is None: + raise InitializeError("Configuration path and/or repository id are not set") + return self.path, self.repository_id def dump(self) -> dict[str, dict[str, str]]: """ @@ -207,14 +230,14 @@ class Configuration(configparser.RawConfigParser): def getpath(self, *args: Any, **kwargs: Any) -> Path: ... # type: ignore[empty-body] - def gettype(self, section: str, architecture: str, *, fallback: str | None = None) -> tuple[str, str]: + def gettype(self, section: str, repository_id: RepositoryId, *, fallback: str | None = None) -> tuple[str, str]: """ get type variable with fallback to old logic. Despite the fact that it has same semantics as other get* methods, but it has different argument list Args: section(str): section name - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier fallback(str | None, optional): optional fallback type if any. If set, second element of the tuple will be always set to this value (Default value = None) @@ -227,9 +250,9 @@ class Configuration(configparser.RawConfigParser): if (group_type := self.get(section, "type", fallback=fallback)) is not None: return section, group_type # new-style logic # okay lets check for the section with architecture name - full_section = self.section_name(section, architecture) - if self.has_section(full_section): - return full_section, section + for specific in self.override_sections(section, repository_id): + if self.has_section(specific): + return specific, section # okay lets just use section as type if self.has_section(section): return section, section @@ -262,23 +285,24 @@ class Configuration(configparser.RawConfigParser): except (FileNotFoundError, configparser.NoOptionError, configparser.NoSectionError): pass - def merge_sections(self, architecture: str) -> None: + def merge_sections(self, repository_id: RepositoryId) -> None: """ - merge architecture specific sections into main configuration + merge architecture and repository specific sections into main configuration Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier """ - self.architecture = architecture + self.repository_id = repository_id + for section in self.ARCHITECTURE_SPECIFIC_SECTIONS: - # get overrides - specific = self.section_name(section, architecture) - if self.has_section(specific): - # if there is no such section it means that there is no overrides for this arch, - # but we anyway will have to delete sections for others architectures - for key, value in self[specific].items(): - self.set_option(section, key, value) - # remove any arch specific section + for specific in self.override_sections(section, repository_id): + if self.has_section(specific): + # if there is no such section it means that there is no overrides for this arch, + # but we anyway will have to delete sections for others architectures + for key, value in self[specific].items(): + self.set_option(section, key, value) + + # remove any arch/repo specific section for foreign in self.sections(): # we would like to use lambda filter here, but pylint is too dumb if not foreign.startswith(f"{section}:"): @@ -289,11 +313,11 @@ class Configuration(configparser.RawConfigParser): """ reload configuration if possible or raise exception otherwise """ - path, architecture = self.check_loaded() + path, repository_id = self.check_loaded() for section in self.sections(): # clear current content self.remove_section(section) self.load(path) - self.merge_sections(architecture) + self.merge_sections(repository_id) def set_option(self, section: str, option: str, value: str) -> None: """ diff --git a/src/ahriman/core/configuration/schema.py b/src/ahriman/core/configuration/schema.py index 98cb7549..5b10dee7 100644 --- a/src/ahriman/core/configuration/schema.py +++ b/src/ahriman/core/configuration/schema.py @@ -187,7 +187,6 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = { "schema": { "name": { "type": "string", - "required": True, }, "root": { "type": "string", diff --git a/src/ahriman/core/database/migrations/m001_package_source.py b/src/ahriman/core/database/migrations/m001_package_source.py index ece66537..6719a198 100644 --- a/src/ahriman/core/database/migrations/m001_package_source.py +++ b/src/ahriman/core/database/migrations/m001_package_source.py @@ -71,7 +71,18 @@ def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> N paths(RepositoryPaths): repository paths instance """ from ahriman.core.alpm.remote import AUR - from ahriman.core.database.operations import PackageOperations + from ahriman.models.package import Package + + def get_packages() -> dict[str, Package]: + return { + row["package_base"]: Package( + base=row["package_base"], + version=row["version"], + remote=RemoteSource.from_json(row), + packages={}, + packager=row["packager"] or None, + ) for row in connection.execute("""select * from package_bases""") + } def insert_remote(base: str, remote: RemoteSource) -> None: connection.execute( @@ -88,8 +99,7 @@ def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> N } ) - packages = PackageOperations._packages_get_select_package_bases(connection) - for package_base, package in packages.items(): + for package_base, package in get_packages().items(): local_cache = paths.cache_for(package_base) if local_cache.exists() and not package.is_vcs: continue # skip packages which are not VCS and with local cache diff --git a/src/ahriman/core/database/migrations/m005_make_opt_depends.py b/src/ahriman/core/database/migrations/m005_make_opt_depends.py index ed9f6084..be91665c 100644 --- a/src/ahriman/core/database/migrations/m005_make_opt_depends.py +++ b/src/ahriman/core/database/migrations/m005_make_opt_depends.py @@ -61,8 +61,8 @@ def migrate_package_depends(connection: Connection, configuration: Configuration if not configuration.repository_paths.repository.is_dir(): return - _, architecture = configuration.check_loaded() - pacman = Pacman(architecture, configuration, refresh_database=PacmanSynchronization.Disabled) + _, repository_id = configuration.check_loaded() + pacman = Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled) package_list = [] for full_path in filter(package_like, configuration.repository_paths.repository.iterdir()): diff --git a/src/ahriman/core/database/migrations/m006_packages_architecture_required.py b/src/ahriman/core/database/migrations/m006_packages_architecture_required.py index a5438286..a2530239 100644 --- a/src/ahriman/core/database/migrations/m006_packages_architecture_required.py +++ b/src/ahriman/core/database/migrations/m006_packages_architecture_required.py @@ -45,9 +45,9 @@ steps = [ ) """, """ - insert into packages select * from packages_ where architecture is not null; + insert into packages select * from packages_ where architecture is not null """, """ - drop table packages_; + drop table packages_ """, ] diff --git a/src/ahriman/core/database/migrations/m007_check_depends.py b/src/ahriman/core/database/migrations/m007_check_depends.py index 9769bd01..870aa91e 100644 --- a/src/ahriman/core/database/migrations/m007_check_depends.py +++ b/src/ahriman/core/database/migrations/m007_check_depends.py @@ -58,8 +58,8 @@ def migrate_package_check_depends(connection: Connection, configuration: Configu if not configuration.repository_paths.repository.is_dir(): return - _, architecture = configuration.check_loaded() - pacman = Pacman(architecture, configuration, refresh_database=PacmanSynchronization.Disabled) + _, repository_id = configuration.check_loaded() + pacman = Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled) package_list = [] for full_path in filter(package_like, configuration.repository_paths.repository.iterdir()): diff --git a/src/ahriman/core/database/migrations/m008_packagers.py b/src/ahriman/core/database/migrations/m008_packagers.py index df62c21f..1f5f6251 100644 --- a/src/ahriman/core/database/migrations/m008_packagers.py +++ b/src/ahriman/core/database/migrations/m008_packagers.py @@ -64,8 +64,8 @@ def migrate_package_base_packager(connection: Connection, configuration: Configu if not configuration.repository_paths.repository.is_dir(): return - _, architecture = configuration.check_loaded() - pacman = Pacman(architecture, configuration, refresh_database=PacmanSynchronization.Disabled) + _, repository_id = configuration.check_loaded() + pacman = Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled) package_list = [] for full_path in filter(package_like, configuration.repository_paths.repository.iterdir()): diff --git a/src/ahriman/core/database/migrations/m011_repository_name.py b/src/ahriman/core/database/migrations/m011_repository_name.py new file mode 100644 index 00000000..1e929017 --- /dev/null +++ b/src/ahriman/core/database/migrations/m011_repository_name.py @@ -0,0 +1,245 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from sqlite3 import Connection + +from ahriman.core.configuration import Configuration + + +__all__ = ["migrate_data", "steps"] + + +steps = [ + # set correct types for schema + """ + alter table users rename to users_ + """, + """ + create table users ( + username text not null unique, + access text not null, + password text, + packager_id text, + key_id text + ) + """, + """ + insert into users select * from users_ + """, + """ + drop table users_ + """, + # update base tables + # build_queue + """ + alter table build_queue add column repository text not null default '' + """, + """ + alter table build_queue rename to build_queue_ + """, + """ + create table build_queue ( + package_base text not null, + properties json not null, + repository text not null, + primary key (package_base, repository) + ) + """, + """ + insert into build_queue select * from build_queue_ + """, + """ + drop table build_queue_ + """, + # package_bases + """ + alter table package_bases add column repository text not null default '' + """, + """ + alter table package_bases rename to package_bases_ + """, + """ + create table package_bases ( + package_base text not null, + version text not null, + branch text, + git_url text, + path text, + web_url text, + source text, + packager text, + repository text not null, + primary key (package_base, repository) + ) + """, + """ + insert into package_bases select * from package_bases_ + """, + """ + drop table package_bases_ + """, + # package_statuses + """ + alter table package_statuses add column repository text not null default '' + """, + """ + alter table package_statuses rename to package_statuses_ + """, + """ + create table package_statuses ( + package_base text not null, + status text not null, + last_updated integer, + repository text not null, + primary key (package_base, repository) + ) + """, + """ + insert into package_statuses select * from package_statuses_ + """, + """ + drop table package_statuses_ + """, + # packages + """ + alter table packages add column repository text not null default '' + """, + """ + alter table packages rename to packages_ + """, + """ + create table packages ( + package text not null, + package_base text not null, + architecture text not null, + archive_size integer, + build_date integer, + depends json, + description text, + filename text, + "groups" json, + installed_size integer, + licenses json, + provides json, + url text, + make_depends json, + opt_depends json, + check_depends json, + repository text not null, + primary key (package, architecture, repository) + ) + """, + """ + insert into packages select * from packages_ + """, + """ + drop table packages_ + """, + # patches + """ + alter table patches add column repository text not null default '' + """, + """ + drop index patches_package_base_variable + """, + """ + alter table patches rename to patches_ + """, + """ + create table patches ( + package_base text not null, + variable text, + patch blob not null, + repository text not null + ) + """, + """ + create unique index patches_package_base_variable_repository + on patches (package_base, coalesce(variable, ''), repository) + """, + """ + insert into patches select * from patches_ + """, + """ + drop table patches_ + """, + # logs + """ + alter table logs add column repository text not null default '' + """, + """ + drop index logs_package_base_version + """, + """ + alter table logs rename to logs_ + """, + """ + create table logs ( + package_base text not null, + created real not null, + record text, + version text not null, + repository text not null + ) + """, + """ + insert into logs select * from logs_ + """, + """ + create index logs_package_base_version on logs (package_base, version) + """, + """ + drop table logs_ + """, +] + + +def migrate_data(connection: Connection, configuration: Configuration) -> None: + """ + perform data migration + + Args: + connection(Connection): database connection + configuration(Configuration): configuration instance + """ + migrate_package_repository(connection, configuration) + + +def migrate_package_repository(connection: Connection, configuration: Configuration) -> None: + """ + update repository name from current settings + + Args: + connection(Connection): database connection + configuration(Configuration): configuration instance + """ + _, repository_id = configuration.check_loaded() + + connection.execute("""update build_queue set repository = :repository""", + {"repository": repository_id.name, }) + connection.execute("""update package_bases set repository = :repository""", + {"repository": repository_id.name, }) + connection.execute("""update package_statuses set repository = :repository""", + {"repository": repository_id.name, }) + connection.execute("""update packages set repository = :repository""", + {"repository": repository_id.name, }) + connection.execute("""update patches set repository = :repository""", + {"repository": repository_id.name, }) + connection.execute("""update logs set repository = :repository""", + {"repository": repository_id.name, }) diff --git a/src/ahriman/core/database/operations/build_operations.py b/src/ahriman/core/database/operations/build_operations.py index 13e33e1d..cfbab902 100644 --- a/src/ahriman/core/database/operations/build_operations.py +++ b/src/ahriman/core/database/operations/build_operations.py @@ -39,9 +39,9 @@ class BuildOperations(Operations): connection.execute( """ delete from build_queue - where :package_base is null or package_base = :package_base + where (:package_base is null or package_base = :package_base) and repository = :repository """, - {"package_base": package_base}) + {"package_base": package_base, "repository": self.repository_id.name}) return self.with_connection(run, commit=True) @@ -55,7 +55,10 @@ class BuildOperations(Operations): def run(connection: Connection) -> list[Package]: return [ Package.from_json(row["properties"]) - for row in connection.execute("""select * from build_queue""") + for row in connection.execute( + """select properties from build_queue where repository = :repository""", + {"repository": self.repository_id.name} + ) ] return self.with_connection(run) @@ -71,12 +74,12 @@ class BuildOperations(Operations): connection.execute( """ insert into build_queue - (package_base, properties) + (package_base, properties, repository) values - (:package_base, :properties) - on conflict (package_base) do update set + (:package_base, :properties, :repository) + on conflict (package_base, repository) do update set properties = :properties """, - {"package_base": package.base, "properties": package.view()}) + {"package_base": package.base, "properties": package.view(), "repository": self.repository_id.name}) return self.with_connection(run, commit=True) diff --git a/src/ahriman/core/database/operations/logs_operations.py b/src/ahriman/core/database/operations/logs_operations.py index 1ecd2563..5cd29878 100644 --- a/src/ahriman/core/database/operations/logs_operations.py +++ b/src/ahriman/core/database/operations/logs_operations.py @@ -45,14 +45,11 @@ class LogsOperations(Operations): (row["created"], row["record"]) for row in connection.execute( """ - select created, record from logs where package_base = :package_base + select created, record from logs + where package_base = :package_base and repository = :repository order by created limit :limit offset :offset """, - { - "package_base": package_base, - "limit": limit, - "offset": offset, - }) + {"package_base": package_base, "repository": self.repository_id.name, "limit": limit, "offset": offset}) ] return self.with_connection(run) @@ -70,15 +67,16 @@ class LogsOperations(Operations): connection.execute( """ insert into logs - (package_base, version, created, record) + (package_base, version, created, record, repository) values - (:package_base, :version, :created, :record) + (:package_base, :version, :created, :record, :repository) """, { "package_base": log_record_id.package_base, "version": log_record_id.version, "created": created, "record": record, + "repository": self.repository_id.name, } ) @@ -97,9 +95,10 @@ class LogsOperations(Operations): connection.execute( """ delete from logs - where package_base = :package_base and (:version is null or version <> :version) + where package_base = :package_base and repository = :repository + and (:version is null or version <> :version) """, - {"package_base": package_base, "version": version} + {"package_base": package_base, "version": version, "repository": self.repository_id.name} ) return self.with_connection(run, commit=True) diff --git a/src/ahriman/core/database/operations/operations.py b/src/ahriman/core/database/operations/operations.py index 9a447adf..4fbe4805 100644 --- a/src/ahriman/core/database/operations/operations.py +++ b/src/ahriman/core/database/operations/operations.py @@ -24,6 +24,7 @@ from pathlib import Path from typing import Any, TypeVar from ahriman.core.log import LazyLogging +from ahriman.models.repository_id import RepositoryId T = TypeVar("T") @@ -35,16 +36,19 @@ class Operations(LazyLogging): Attributes: path(Path): path to the database file + repository_id(RepositoryId): repository unique identifier to perform implicit filtering """ - def __init__(self, path: Path) -> None: + def __init__(self, path: Path, repository_id: RepositoryId) -> None: """ default constructor Args: path(Path): path to the database file + repository_id(RepositoryId): repository unique identifier """ self.path = path + self.repository_id = repository_id @staticmethod def factory(cursor: sqlite3.Cursor, row: tuple[Any, ...]) -> dict[str, Any]: diff --git a/src/ahriman/core/database/operations/package_operations.py b/src/ahriman/core/database/operations/package_operations.py index acf28703..4278e1bc 100644 --- a/src/ahriman/core/database/operations/package_operations.py +++ b/src/ahriman/core/database/operations/package_operations.py @@ -32,8 +32,7 @@ class PackageOperations(Operations): package operations """ - @staticmethod - def _package_remove_package_base(connection: Connection, package_base: str) -> None: + def _package_remove_package_base(self, connection: Connection, package_base: str) -> None: """ remove package base information @@ -41,13 +40,20 @@ class PackageOperations(Operations): connection(Connection): database connection package_base(str): package base name """ - connection.execute("""delete from package_statuses where package_base = :package_base""", - {"package_base": package_base}) - connection.execute("""delete from package_bases where package_base = :package_base""", - {"package_base": package_base}) + connection.execute( + """ + delete from package_statuses + where package_base = :package_base and repository = :repository + """, + {"package_base": package_base, "repository": self.repository_id.name}) + connection.execute( + """ + delete from package_bases + where package_base = :package_base and repository = :repository""", + {"package_base": package_base, "repository": self.repository_id.name}) - @staticmethod - def _package_remove_packages(connection: Connection, package_base: str, current_packages: Iterable[str]) -> None: + def _package_remove_packages(self, connection: Connection, package_base: str, + current_packages: Iterable[str]) -> None: """ remove packages belong to the package base @@ -59,13 +65,20 @@ class PackageOperations(Operations): packages = [ package for package in connection.execute( - """select package from packages where package_base = :package_base""", {"package_base": package_base}) + """ + select package, repository from packages + where package_base = :package_base and repository = :repository""", + {"package_base": package_base, "repository": self.repository_id.name}) if package["package"] not in current_packages ] - connection.executemany("""delete from packages where package = :package""", packages) + connection.executemany( + """ + delete from packages + where package = :package and repository = :repository + """, + packages) - @staticmethod - def _package_update_insert_base(connection: Connection, package: Package) -> None: + def _package_update_insert_base(self, connection: Connection, package: Package) -> None: """ insert base package into table @@ -76,10 +89,10 @@ class PackageOperations(Operations): connection.execute( """ insert into package_bases - (package_base, version, source, branch, git_url, path, web_url, packager) + (package_base, version, source, branch, git_url, path, web_url, packager, repository) values - (:package_base, :version, :source, :branch, :git_url, :path, :web_url, :packager) - on conflict (package_base) do update set + (:package_base, :version, :source, :branch, :git_url, :path, :web_url, :packager, :repository) + on conflict (package_base, repository) do update set version = :version, branch = :branch, git_url = :git_url, path = :path, web_url = :web_url, source = :source, packager = :packager """, @@ -92,11 +105,11 @@ class PackageOperations(Operations): "web_url": package.remote.web_url, "source": package.remote.source.value, "packager": package.packager, + "repository": self.repository_id.name, } ) - @staticmethod - def _package_update_insert_packages(connection: Connection, package: Package) -> None: + def _package_update_insert_packages(self, connection: Connection, package: Package) -> None: """ insert packages into table @@ -108,20 +121,27 @@ class PackageOperations(Operations): for name, description in package.packages.items(): if description.architecture is None: continue # architecture is required - package_list.append({"package": name, "package_base": package.base, **description.view()}) + package_list.append({ + "package": name, + "package_base": package.base, + "repository": self.repository_id.name, + **description.view(), + }) connection.executemany( """ insert into packages (package, package_base, architecture, archive_size, build_date, depends, description, filename, "groups", installed_size, licenses, provides, - url, make_depends, opt_depends, check_depends) + url, make_depends, opt_depends, check_depends, + repository) values (:package, :package_base, :architecture, :archive_size, :build_date, :depends, :description, :filename, :groups, :installed_size, :licenses, :provides, - :url, :make_depends, :opt_depends, :check_depends) - on conflict (package, architecture) do update set + :url, :make_depends, :opt_depends, :check_depends, + :repository) + on conflict (package, architecture, repository) do update set package_base = :package_base, archive_size = :archive_size, build_date = :build_date, depends = :depends, description = :description, filename = :filename, "groups" = :groups, installed_size = :installed_size, licenses = :licenses, provides = :provides, @@ -129,8 +149,7 @@ class PackageOperations(Operations): """, package_list) - @staticmethod - def _package_update_insert_status(connection: Connection, package_base: str, status: BuildStatus) -> None: + def _package_update_insert_status(self, connection: Connection, package_base: str, status: BuildStatus) -> None: """ insert base package status into table @@ -141,16 +160,21 @@ class PackageOperations(Operations): """ connection.execute( """ - insert into package_statuses (package_base, status, last_updated) + insert into package_statuses + (package_base, status, last_updated, repository) values - (:package_base, :status, :last_updated) - on conflict (package_base) do update set + (:package_base, :status, :last_updated, :repository) + on conflict (package_base, repository) do update set status = :status, last_updated = :last_updated """, - {"package_base": package_base, "status": status.status.value, "last_updated": status.timestamp}) + { + "package_base": package_base, + "status": status.status.value, + "last_updated": status.timestamp, + "repository": self.repository_id.name, + }) - @staticmethod - def _packages_get_select_package_bases(connection: Connection) -> dict[str, Package]: + def _packages_get_select_package_bases(self, connection: Connection) -> dict[str, Package]: """ select package bases from the table @@ -167,11 +191,13 @@ class PackageOperations(Operations): remote=RemoteSource.from_json(row), packages={}, packager=row["packager"] or None, - ) for row in connection.execute("""select * from package_bases""") + ) for row in connection.execute( + """select * from package_bases where repository = :repository""", + {"repository": self.repository_id.name} + ) } - @staticmethod - def _packages_get_select_packages(connection: Connection, packages: dict[str, Package]) -> dict[str, Package]: + def _packages_get_select_packages(self, connection: Connection, packages: dict[str, Package]) -> dict[str, Package]: """ select packages from the table @@ -182,14 +208,15 @@ class PackageOperations(Operations): Returns: dict[str, Package]: map of the package base to its descriptor including individual packages """ - for row in connection.execute("""select * from packages"""): + for row in connection.execute( + """select * from packages where repository = :repository""", + {"repository": self.repository_id.name}): if row["package_base"] not in packages: continue # normally must never happen though packages[row["package_base"]].packages[row["package"]] = PackageDescription.from_json(row) return packages - @staticmethod - def _packages_get_select_statuses(connection: Connection) -> dict[str, BuildStatus]: + def _packages_get_select_statuses(self, connection: Connection) -> dict[str, BuildStatus]: """ select package build statuses from the table @@ -201,7 +228,10 @@ class PackageOperations(Operations): """ return { row["package_base"]: BuildStatus.from_json({"status": row["status"], "timestamp": row["last_updated"]}) - for row in connection.execute("""select * from package_statuses""") + for row in connection.execute( + """select * from package_statuses where repository = :repository""", + {"repository": self.repository_id.name} + ) } def package_remove(self, package_base: str) -> None: diff --git a/src/ahriman/core/database/operations/patch_operations.py b/src/ahriman/core/database/operations/patch_operations.py index 98cbbfc4..064ad6ce 100644 --- a/src/ahriman/core/database/operations/patch_operations.py +++ b/src/ahriman/core/database/operations/patch_operations.py @@ -54,13 +54,19 @@ class PatchOperations(Operations): connection.execute( """ insert into patches - (package_base, variable, patch) + (package_base, variable, patch, repository) values - (:package_base, :variable, :patch) - on conflict (package_base, coalesce(variable, '')) do update set + (:package_base, :variable, :patch, :repository) + on conflict (package_base, coalesce(variable, ''), repository) do update set patch = :patch """, - {"package_base": package_base, "variable": patch.key, "patch": patch.value}) + { + "package_base": package_base, + "variable": patch.key, + "patch": patch.value, + "repository": self.repository_id.name, + } + ) return self.with_connection(run, commit=True) @@ -79,8 +85,10 @@ class PatchOperations(Operations): return [ (row["package_base"], PkgbuildPatch(row["variable"], row["patch"])) for row in connection.execute( - """select * from patches where :package_base is null or package_base = :package_base""", - {"package_base": package_base}) + """ + select * from patches + where (:package_base is null or package_base = :package_base) and repository = :repository""", + {"package_base": package_base, "repository": self.repository_id.name}) ] # we could use itertools & operator but why? @@ -101,13 +109,23 @@ class PatchOperations(Operations): """ def run_many(connection: Connection) -> None: connection.executemany( - """delete from patches where package_base = :package_base and variable = :variable""", - [{"package_base": package_base, "variable": variable} for variable in variables]) + """ + delete from patches + where package_base = :package_base and variable = :variable and repository = :repository + """, + [ + { + "package_base": package_base, + "variable": variable, + "repository": self.repository_id.name, + } for variable in variables + ] + ) def run(connection: Connection) -> None: connection.execute( - """delete from patches where package_base = :package_base""", - {"package_base": package_base}) + """delete from patches where package_base = :package_base and repository = :repository""", + {"package_base": package_base, "repository": self.repository_id.name}) if variables: return self.with_connection(run_many, commit=True) diff --git a/src/ahriman/core/database/sqlite.py b/src/ahriman/core/database/sqlite.py index 9a668983..500f63cc 100644 --- a/src/ahriman/core/database/sqlite.py +++ b/src/ahriman/core/database/sqlite.py @@ -56,8 +56,11 @@ class SQLite(AuthOperations, BuildOperations, LogsOperations, PackageOperations, Self: fully initialized instance of the database """ path = cls.database_path(configuration) - database = cls(path) + _, repository_id = configuration.check_loaded() + + database = cls(path, repository_id) database.init(configuration) + return database @staticmethod diff --git a/src/ahriman/core/gitremote/remote_pull.py b/src/ahriman/core/gitremote/remote_pull.py index f5d5718a..7536a59d 100644 --- a/src/ahriman/core/gitremote/remote_pull.py +++ b/src/ahriman/core/gitremote/remote_pull.py @@ -30,6 +30,7 @@ from ahriman.core.util import walk from ahriman.models.package import Package from ahriman.models.package_source import PackageSource from ahriman.models.remote_source import RemoteSource +from ahriman.models.repository_id import RepositoryId class RemotePull(LazyLogging): @@ -42,13 +43,13 @@ class RemotePull(LazyLogging): repository_paths(RepositoryPaths): repository paths instance """ - def __init__(self, configuration: Configuration, architecture: str, section: str) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None: """ default constructor Args: + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance - architecture(str): repository architecture section(str): settings section name """ self.remote_source = RemoteSource( @@ -58,7 +59,7 @@ class RemotePull(LazyLogging): branch=configuration.get(section, "pull_branch", fallback="master"), source=PackageSource.Local, ) - self.architecture = architecture + self.architecture = repository_id.architecture self.repository_paths = configuration.repository_paths def package_copy(self, pkgbuild_path: Path) -> None: diff --git a/src/ahriman/core/gitremote/remote_pull_trigger.py b/src/ahriman/core/gitremote/remote_pull_trigger.py index b850cc77..4fef34d5 100644 --- a/src/ahriman/core/gitremote/remote_pull_trigger.py +++ b/src/ahriman/core/gitremote/remote_pull_trigger.py @@ -20,6 +20,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.gitremote.remote_pull import RemotePull from ahriman.core.triggers import Trigger +from ahriman.models.repository_id import RepositoryId class RemotePullTrigger(Trigger): @@ -56,15 +57,15 @@ class RemotePullTrigger(Trigger): } CONFIGURATION_SCHEMA_FALLBACK = "gitremote" - def __init__(self, architecture: str, configuration: Configuration) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance """ - Trigger.__init__(self, architecture, configuration) + Trigger.__init__(self, repository_id, configuration) self.targets = self.configuration_sections(configuration) @classmethod @@ -86,6 +87,6 @@ class RemotePullTrigger(Trigger): """ for target in self.targets: section, _ = self.configuration.gettype( - target, self.architecture, fallback=self.CONFIGURATION_SCHEMA_FALLBACK) - runner = RemotePull(self.configuration, self.architecture, section) + target, self.repository_id, fallback=self.CONFIGURATION_SCHEMA_FALLBACK) + runner = RemotePull(self.repository_id, self.configuration, section) runner.run() diff --git a/src/ahriman/core/gitremote/remote_push_trigger.py b/src/ahriman/core/gitremote/remote_push_trigger.py index a2a9d47d..a46713e6 100644 --- a/src/ahriman/core/gitremote/remote_push_trigger.py +++ b/src/ahriman/core/gitremote/remote_push_trigger.py @@ -24,6 +24,7 @@ from ahriman.core.gitremote.remote_push import RemotePush from ahriman.core.triggers import Trigger from ahriman.models.context_key import ContextKey from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result @@ -67,15 +68,15 @@ class RemotePushTrigger(Trigger): } CONFIGURATION_SCHEMA_FALLBACK = "gitremote" - def __init__(self, architecture: str, configuration: Configuration) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance """ - Trigger.__init__(self, architecture, configuration) + Trigger.__init__(self, repository_id, configuration) self.targets = self.configuration_sections(configuration) @classmethod @@ -107,6 +108,6 @@ class RemotePushTrigger(Trigger): for target in self.targets: section, _ = self.configuration.gettype( - target, self.architecture, fallback=self.CONFIGURATION_SCHEMA_FALLBACK) + target, self.repository_id, fallback=self.CONFIGURATION_SCHEMA_FALLBACK) runner = RemotePush(database, self.configuration, section) runner.run(result) diff --git a/src/ahriman/core/http/sync_http_client.py b/src/ahriman/core/http/sync_http_client.py index 731e0b6f..6b670c31 100644 --- a/src/ahriman/core/http/sync_http_client.py +++ b/src/ahriman/core/http/sync_http_client.py @@ -41,14 +41,14 @@ class SyncHttpClient(LazyLogging): timeout(int): HTTP request timeout in seconds """ - def __init__(self, section: str | None = None, configuration: Configuration | None = None, *, + def __init__(self, configuration: Configuration | None = None, section: str | None = None, *, suppress_errors: bool = False) -> None: """ default constructor Args: - section(str, optional): settings section name (Default value = None) configuration(Configuration | None): configuration instance (Default value = None) + section(str, optional): settings section name (Default value = None) suppress_errors(bool, optional): suppress logging of request errors (Default value = False) """ if configuration is None: diff --git a/src/ahriman/core/log/__init__.py b/src/ahriman/core/log/__init__.py index 97b65d75..c0ba3e7d 100644 --- a/src/ahriman/core/log/__init__.py +++ b/src/ahriman/core/log/__init__.py @@ -18,4 +18,3 @@ # along with this program. If not, see . # from ahriman.core.log.lazy_logging import LazyLogging -from ahriman.core.log.log import Log diff --git a/src/ahriman/core/log/log.py b/src/ahriman/core/log/log_loader.py similarity index 95% rename from src/ahriman/core/log/log.py rename to src/ahriman/core/log/log_loader.py index 7125d973..57189d44 100644 --- a/src/ahriman/core/log/log.py +++ b/src/ahriman/core/log/log_loader.py @@ -27,7 +27,7 @@ from ahriman.core.log.http_log_handler import HttpLogHandler from ahriman.models.log_handler import LogHandler -class Log: +class LogLoader: """ simple static method class which setups application loggers @@ -63,7 +63,7 @@ class Log: del JournalHandler return LogHandler.Journald # journald import was found except ImportError: - if Log.DEFAULT_SYSLOG_DEVICE.exists(): + if LogLoader.DEFAULT_SYSLOG_DEVICE.exists(): return LogHandler.Syslog return LogHandler.Console @@ -94,8 +94,7 @@ class Log: fileConfig(log_configuration, disable_existing_loggers=True) logging.debug("using %s logger", default_handler) except Exception: - logging.basicConfig(filename=None, format=Log.DEFAULT_LOG_FORMAT, - level=Log.DEFAULT_LOG_LEVEL) + logging.basicConfig(filename=None, format=LogLoader.DEFAULT_LOG_FORMAT, level=LogLoader.DEFAULT_LOG_LEVEL) logging.exception("could not load logging from configuration, fallback to stderr") HttpLogHandler.load(configuration, report=report) diff --git a/src/ahriman/core/report/console.py b/src/ahriman/core/report/console.py index 8476a129..d8cd4b4e 100644 --- a/src/ahriman/core/report/console.py +++ b/src/ahriman/core/report/console.py @@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.formatters import BuildPrinter from ahriman.core.report.report import Report from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result @@ -32,16 +33,16 @@ class Console(Report): use_utf(bool): print utf8 symbols instead of ASCII """ - def __init__(self, architecture: str, configuration: Configuration, section: str) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance section(str): settings section name """ - Report.__init__(self, architecture, configuration) + Report.__init__(self, repository_id, configuration) self.use_utf = configuration.getboolean(section, "use_utf", fallback=True) def generate(self, packages: list[Package], result: Result) -> None: diff --git a/src/ahriman/core/report/email.py b/src/ahriman/core/report/email.py index e51825a4..ad5360a9 100644 --- a/src/ahriman/core/report/email.py +++ b/src/ahriman/core/report/email.py @@ -27,6 +27,7 @@ from ahriman.core.report.jinja_template import JinjaTemplate from ahriman.core.report.report import Report from ahriman.core.util import pretty_datetime, utcnow from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result from ahriman.models.smtp_ssl_settings import SmtpSSLSettings @@ -48,17 +49,17 @@ class Email(Report, JinjaTemplate): user(str | None): username to authenticate via SMTP """ - def __init__(self, architecture: str, configuration: Configuration, section: str) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance section(str): settings section name """ - Report.__init__(self, architecture, configuration) - JinjaTemplate.__init__(self, section, configuration) + Report.__init__(self, repository_id, configuration) + JinjaTemplate.__init__(self, repository_id, configuration, section) self.full_template_path = configuration.getpath(section, "full_template_path", fallback=None) self.template_path = configuration.getpath(section, "template_path") diff --git a/src/ahriman/core/report/html.py b/src/ahriman/core/report/html.py index ffc1ff08..11d89a87 100644 --- a/src/ahriman/core/report/html.py +++ b/src/ahriman/core/report/html.py @@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.report.jinja_template import JinjaTemplate from ahriman.core.report.report import Report from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result @@ -33,17 +34,17 @@ class HTML(Report, JinjaTemplate): template_path(Path): path to template for full package list """ - def __init__(self, architecture: str, configuration: Configuration, section: str) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance section(str): settings section name """ - Report.__init__(self, architecture, configuration) - JinjaTemplate.__init__(self, section, configuration) + Report.__init__(self, repository_id, configuration) + JinjaTemplate.__init__(self, repository_id, configuration, section) self.report_path = configuration.getpath(section, "path") self.template_path = configuration.getpath(section, "template_path") diff --git a/src/ahriman/core/report/jinja_template.py b/src/ahriman/core/report/jinja_template.py index a525ecf6..4826d29c 100644 --- a/src/ahriman/core/report/jinja_template.py +++ b/src/ahriman/core/report/jinja_template.py @@ -25,6 +25,7 @@ from pathlib import Path from ahriman.core.configuration import Configuration from ahriman.core.sign.gpg import GPG from ahriman.core.util import pretty_datetime, pretty_size +from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result from ahriman.models.sign_settings import SignSettings @@ -63,19 +64,20 @@ class JinjaTemplate: sign_targets(set[SignSettings]): targets to sign enabled in configuration """ - def __init__(self, section: str, configuration: Configuration) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None: """ default constructor Args: - section(str): settings section name + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance + section(str): settings section name """ self.link_path = configuration.get(section, "link_path") # base template vars self.homepage = configuration.get(section, "homepage", fallback=None) - self.name = configuration.repository_name + self.name = repository_id.name self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration) diff --git a/src/ahriman/core/report/remote_call.py b/src/ahriman/core/report/remote_call.py index 94d5feb3..55062686 100644 --- a/src/ahriman/core/report/remote_call.py +++ b/src/ahriman/core/report/remote_call.py @@ -23,6 +23,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.report.report import Report from ahriman.core.status.web_client import WebClient from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result from ahriman.models.waiter import Waiter @@ -39,16 +40,16 @@ class RemoteCall(Report): wait_timeout(int): timeout to wait external process """ - def __init__(self, architecture: str, configuration: Configuration, section: str) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance section(str): settings section name """ - Report.__init__(self, architecture, configuration) + Report.__init__(self, repository_id, configuration) self.client = WebClient(configuration) diff --git a/src/ahriman/core/report/report.py b/src/ahriman/core/report/report.py index 392837fb..c11898af 100644 --- a/src/ahriman/core/report/report.py +++ b/src/ahriman/core/report/report.py @@ -24,6 +24,7 @@ from ahriman.core.exceptions import ReportError from ahriman.core.log import LazyLogging from ahriman.models.package import Package from ahriman.models.report_settings import ReportSettings +from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result @@ -32,17 +33,15 @@ class Report(LazyLogging): base report generator Attributes: - architecture(str): repository architecture configuration(Configuration): configuration instance + repository_id(RepositoryId): repository unique identifier Examples: ``Report`` classes provide several method in order to operate with the report generation and additional class method ``load`` which can be used in order to determine right report instance:: - >>> from ahriman.core.configuration import Configuration - >>> >>> configuration = Configuration() - >>> report = Report.load("x86_64", configuration, "email") + >>> report = Report.load(RepositoryId("x86_64", "aur-clone"), configuration, "email") The ``generate`` method can be used in order to perform the report itself, whereas ``run`` method handles exception and raises ``ReportFailed`` instead:: @@ -55,49 +54,49 @@ class Report(LazyLogging): >>> report.run(Result(), []) """ - def __init__(self, architecture: str, configuration: Configuration) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance """ - self.architecture = architecture + self.repository_id = repository_id self.configuration = configuration @staticmethod - def load(architecture: str, configuration: Configuration, target: str) -> Report: + def load(repository_id: RepositoryId, configuration: Configuration, target: str) -> Report: """ load client from settings Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance target(str): target to generate report aka section name (e.g. html) Returns: Report: client according to current settings """ - section, provider_name = configuration.gettype(target, architecture) + section, provider_name = configuration.gettype(target, repository_id) match ReportSettings.from_option(provider_name): case ReportSettings.HTML: from ahriman.core.report.html import HTML - return HTML(architecture, configuration, section) + return HTML(repository_id, configuration, section) case ReportSettings.Email: from ahriman.core.report.email import Email - return Email(architecture, configuration, section) + return Email(repository_id, configuration, section) case ReportSettings.Console: from ahriman.core.report.console import Console - return Console(architecture, configuration, section) + return Console(repository_id, configuration, section) case ReportSettings.Telegram: from ahriman.core.report.telegram import Telegram - return Telegram(architecture, configuration, section) + return Telegram(repository_id, configuration, section) case ReportSettings.RemoteCall: from ahriman.core.report.remote_call import RemoteCall - return RemoteCall(architecture, configuration, section) + return RemoteCall(repository_id, configuration, section) case _: - return Report(architecture, configuration) # should never happen + return Report(repository_id, configuration) # should never happen def generate(self, packages: list[Package], result: Result) -> None: """ diff --git a/src/ahriman/core/report/report_trigger.py b/src/ahriman/core/report/report_trigger.py index a5d441eb..19ac542d 100644 --- a/src/ahriman/core/report/report_trigger.py +++ b/src/ahriman/core/report/report_trigger.py @@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.triggers import Trigger from ahriman.core.report.report import Report from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result @@ -218,15 +219,15 @@ class ReportTrigger(Trigger): } } - def __init__(self, architecture: str, configuration: Configuration) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance """ - Trigger.__init__(self, architecture, configuration) + Trigger.__init__(self, repository_id, configuration) self.targets = self.configuration_sections(configuration) @classmethod @@ -251,5 +252,5 @@ class ReportTrigger(Trigger): packages(list[Package]): list of all available packages """ for target in self.targets: - runner = Report.load(self.architecture, self.configuration, target) + runner = Report.load(self.repository_id, self.configuration, target) runner.run(result, packages) diff --git a/src/ahriman/core/report/telegram.py b/src/ahriman/core/report/telegram.py index bab40311..46145150 100644 --- a/src/ahriman/core/report/telegram.py +++ b/src/ahriman/core/report/telegram.py @@ -22,6 +22,7 @@ from ahriman.core.http import SyncHttpClient from ahriman.core.report.jinja_template import JinjaTemplate from ahriman.core.report.report import Report from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result @@ -41,18 +42,18 @@ class Telegram(Report, JinjaTemplate, SyncHttpClient): TELEGRAM_API_URL = "https://api.telegram.org" TELEGRAM_MAX_CONTENT_LENGTH = 4096 - def __init__(self, architecture: str, configuration: Configuration, section: str) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance section(str): settings section name """ - Report.__init__(self, architecture, configuration) - JinjaTemplate.__init__(self, section, configuration) - SyncHttpClient.__init__(self, section, configuration) + Report.__init__(self, repository_id, configuration) + JinjaTemplate.__init__(self, repository_id, configuration, section) + SyncHttpClient.__init__(self, configuration, section) self.api_key = configuration.get(section, "api_key") self.chat_id = configuration.get(section, "chat_id") diff --git a/src/ahriman/core/repository/repository.py b/src/ahriman/core/repository/repository.py index 90f519d4..0017289a 100644 --- a/src/ahriman/core/repository/repository.py +++ b/src/ahriman/core/repository/repository.py @@ -32,6 +32,7 @@ from ahriman.core.util import package_like from ahriman.models.context_key import ContextKey from ahriman.models.package import Package from ahriman.models.pacman_synchronization import PacmanSynchronization +from ahriman.models.repository_id import RepositoryId class Repository(Executor, UpdateHandler): @@ -47,7 +48,7 @@ class Repository(Executor, UpdateHandler): >>> >>> configuration = Configuration() >>> database = SQLite.load(configuration) - >>> repository = Repository.load("x86_64", configuration, database, report=True) + >>> repository = Repository.load(RepositoryId("x86_64", "x86_64"), configuration, database, report=True) >>> known_packages = repository.packages() >>> >>> build_result = repository.process_build(known_packages) @@ -58,13 +59,13 @@ class Repository(Executor, UpdateHandler): """ @classmethod - def load(cls, architecture: str, configuration: Configuration, database: SQLite, *, report: bool, + def load(cls, repository_id: RepositoryId, configuration: Configuration, database: SQLite, *, report: bool, refresh_pacman_database: PacmanSynchronization = PacmanSynchronization.Disabled) -> Self: """ load instance from argument list Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance database(SQLite): database instance report(bool): force enable or disable reporting @@ -74,7 +75,7 @@ class Repository(Executor, UpdateHandler): Returns: Self: fully loaded repository class instance """ - instance = cls(architecture, configuration, database, + instance = cls(repository_id, configuration, database, report=report, refresh_pacman_database=refresh_pacman_database) instance._set_context() return instance diff --git a/src/ahriman/core/repository/repository_properties.py b/src/ahriman/core/repository/repository_properties.py index a83a8412..26a78315 100644 --- a/src/ahriman/core/repository/repository_properties.py +++ b/src/ahriman/core/repository/repository_properties.py @@ -27,6 +27,7 @@ from ahriman.core.status.client import Client from ahriman.core.triggers import TriggerLoader from ahriman.models.packagers import Packagers from ahriman.models.pacman_synchronization import PacmanSynchronization +from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_paths import RepositoryPaths from ahriman.models.user import User from ahriman.models.user_access import UserAccess @@ -37,47 +38,65 @@ class RepositoryProperties(LazyLogging): repository internal objects holder Attributes: - architecture(str): repository architecture configuration(Configuration): configuration instance database(SQLite): database instance ignore_list(list[str]): package bases which will be ignored during auto updates - name(str): repository name pacman(Pacman): alpm wrapper instance paths(RepositoryPaths): repository paths instance repo(Repo): repo commands wrapper instance reporter(Client): build status reporter instance + repository_id(RepositoryId): repository unique identifier sign(GPG): GPG wrapper instance triggers(TriggerLoader): triggers holder vcs_allowed_age(int): maximal age of the VCS packages before they will be checked """ - def __init__(self, architecture: str, configuration: Configuration, database: SQLite, *, report: bool, + def __init__(self, repository_id: RepositoryId, configuration: Configuration, database: SQLite, *, report: bool, refresh_pacman_database: PacmanSynchronization) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance database(SQLite): database instance report(bool): force enable or disable reporting refresh_pacman_database(PacmanSynchronization): pacman database synchronization level """ - self.architecture = architecture + self.repository_id = repository_id self.configuration = configuration self.database = database - self.name = configuration.repository_name self.vcs_allowed_age = configuration.getint("build", "vcs_allowed_age", fallback=0) self.paths: RepositoryPaths = configuration.repository_paths # additional workaround for pycharm typing self.ignore_list = configuration.getlist("build", "ignore_packages", fallback=[]) - self.pacman = Pacman(architecture, configuration, refresh_database=refresh_pacman_database) + self.pacman = Pacman(repository_id, configuration, refresh_database=refresh_pacman_database) self.sign = GPG(configuration) self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args) self.reporter = Client.load(configuration, report=report) - self.triggers = TriggerLoader.load(architecture, configuration) + self.triggers = TriggerLoader.load(repository_id, configuration) + + @property + def architecture(self) -> str: + """ + repository architecture for backward compatibility + + Returns: + str: repository architecture + """ + return self.repository_id.architecture + + @property + def name(self) -> str: + """ + repository name for backward compatibility + + Returns: + str: repository name + """ + return self.repository_id.name def packager(self, packagers: Packagers, package_base: str) -> User: """ diff --git a/src/ahriman/core/spawn.py b/src/ahriman/core/spawn.py index ed550a25..d50801b8 100644 --- a/src/ahriman/core/spawn.py +++ b/src/ahriman/core/spawn.py @@ -28,6 +28,7 @@ from multiprocessing import Process, Queue from threading import Lock, Thread from ahriman.core.log import LazyLogging +from ahriman.models.repository_id import RepositoryId class Spawn(Thread, LazyLogging): @@ -37,22 +38,23 @@ class Spawn(Thread, LazyLogging): Attributes: active(dict[str, Process]): map of active child processes required to avoid zombies - architecture(str): repository architecture command_arguments(list[str]): base command line arguments queue(Queue[tuple[str, bool, int]]): multiprocessing queue to read updates from processes + repository_id(RepositoryId): repository unique identifier """ - def __init__(self, args_parser: argparse.ArgumentParser, architecture: str, command_arguments: list[str]) -> None: + def __init__(self, args_parser: argparse.ArgumentParser, repository_id: RepositoryId, + command_arguments: list[str]) -> None: """ default constructor Args: args_parser(argparse.ArgumentParser): command line parser for the application - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier command_arguments(list[str]): base command line arguments """ Thread.__init__(self, name="spawn") - self.architecture = architecture + self.repository_id = repository_id self.args_parser = args_parser self.command_arguments = command_arguments @@ -77,20 +79,20 @@ class Spawn(Thread, LazyLogging): return name if value else f"no-{name}" @staticmethod - def process(callback: Callable[[argparse.Namespace, str], bool], args: argparse.Namespace, architecture: str, - process_id: str, queue: Queue[tuple[str, bool, int]]) -> None: # pylint: disable=unsubscriptable-object + def process(callback: Callable[[argparse.Namespace, RepositoryId], bool], args: argparse.Namespace, + repository_id: RepositoryId, process_id: str, queue: Queue[tuple[str, bool, int]]) -> None: # pylint: disable=unsubscriptable-object """ helper to run external process Args: callback(Callable[[argparse.Namespace, str], bool]): application run function (i.e. Handler.run method) args(argparse.Namespace): command line arguments - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier process_id(str): process unique identifier queue(Queue[tuple[str, bool, int]]): output queue """ start_time = time.monotonic() - result = callback(args, architecture) + result = callback(args, repository_id) stop_time = time.monotonic() consumed_time = int(1000 * (stop_time - start_time)) @@ -128,7 +130,7 @@ class Spawn(Thread, LazyLogging): callback = parsed.handler.call process = Process(target=self.process, - args=(callback, parsed, self.architecture, process_id, self.queue), + args=(callback, parsed, self.repository_id, process_id, self.queue), daemon=True) process.start() diff --git a/src/ahriman/core/status/watcher.py b/src/ahriman/core/status/watcher.py index 3ea03c4c..05b35419 100644 --- a/src/ahriman/core/status/watcher.py +++ b/src/ahriman/core/status/watcher.py @@ -25,6 +25,7 @@ from ahriman.core.repository import Repository from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.log_record_id import LogRecordId from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId class Watcher(LazyLogging): @@ -32,26 +33,26 @@ class Watcher(LazyLogging): package status watcher Attributes: - architecture(str): repository architecture database(SQLite): database instance known(dict[str, tuple[Package, BuildStatus]]): list of known packages. For the most cases ``packages`` should be used instead repository(Repository): repository object + repository_id(RepositoryId): repository unique identifier status(BuildStatus): daemon status """ - def __init__(self, architecture: str, configuration: Configuration, database: SQLite) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration, database: SQLite) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance database(SQLite): database instance """ - self.architecture = architecture + self.repository_id = repository_id self.database = database - self.repository = Repository.load(architecture, configuration, database, report=False) + self.repository = Repository.load(repository_id, configuration, database, report=False) self.known: dict[str, tuple[Package, BuildStatus]] = {} self.status = BuildStatus() diff --git a/src/ahriman/core/status/web_client.py b/src/ahriman/core/status/web_client.py index 76e2a2b7..d63b79e0 100644 --- a/src/ahriman/core/status/web_client.py +++ b/src/ahriman/core/status/web_client.py @@ -51,7 +51,7 @@ class WebClient(Client, SyncHttpClient): configuration(Configuration): configuration instance """ suppress_errors = configuration.getboolean("settings", "suppress_http_log_errors", fallback=False) - SyncHttpClient.__init__(self, "web", configuration, suppress_errors=suppress_errors) + SyncHttpClient.__init__(self, configuration, "web", suppress_errors=suppress_errors) self.address, self.use_unix_socket = self.parse_address(configuration) diff --git a/src/ahriman/core/support/keyring_trigger.py b/src/ahriman/core/support/keyring_trigger.py index 583b3beb..6fbc0a06 100644 --- a/src/ahriman/core/support/keyring_trigger.py +++ b/src/ahriman/core/support/keyring_trigger.py @@ -25,6 +25,7 @@ from ahriman.core.support.package_creator import PackageCreator from ahriman.core.support.pkgbuild.keyring_generator import KeyringGenerator from ahriman.core.triggers import Trigger from ahriman.models.context_key import ContextKey +from ahriman.models.repository_id import RepositoryId class KeyringTrigger(Trigger): @@ -82,15 +83,15 @@ class KeyringTrigger(Trigger): }, } - def __init__(self, architecture: str, configuration: Configuration) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance """ - Trigger.__init__(self, architecture, configuration) + Trigger.__init__(self, repository_id, configuration) self.targets = self.configuration_sections(configuration) @classmethod @@ -115,6 +116,6 @@ class KeyringTrigger(Trigger): database = ctx.get(ContextKey("database", SQLite)) for target in self.targets: - generator = KeyringGenerator(database, sign, self.configuration, target) + generator = KeyringGenerator(database, sign, self.repository_id, self.configuration, target) runner = PackageCreator(self.configuration, generator) runner.run() diff --git a/src/ahriman/core/support/mirrorlist_trigger.py b/src/ahriman/core/support/mirrorlist_trigger.py index a9e182ca..37c3886a 100644 --- a/src/ahriman/core/support/mirrorlist_trigger.py +++ b/src/ahriman/core/support/mirrorlist_trigger.py @@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.support.package_creator import PackageCreator from ahriman.core.support.pkgbuild.mirrorlist_generator import MirrorlistGenerator from ahriman.core.triggers import Trigger +from ahriman.models.repository_id import RepositoryId class MirrorlistTrigger(Trigger): @@ -75,15 +76,15 @@ class MirrorlistTrigger(Trigger): }, } - def __init__(self, architecture: str, configuration: Configuration) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance """ - Trigger.__init__(self, architecture, configuration) + Trigger.__init__(self, repository_id, configuration) self.targets = self.configuration_sections(configuration) @classmethod @@ -104,6 +105,6 @@ class MirrorlistTrigger(Trigger): trigger action which will be called at the start of the application """ for target in self.targets: - generator = MirrorlistGenerator(self.configuration, target) + generator = MirrorlistGenerator(self.repository_id, self.configuration, target) runner = PackageCreator(self.configuration, generator) runner.run() diff --git a/src/ahriman/core/support/package_creator.py b/src/ahriman/core/support/package_creator.py index 4048d27b..44aee94d 100644 --- a/src/ahriman/core/support/package_creator.py +++ b/src/ahriman/core/support/package_creator.py @@ -66,6 +66,6 @@ class PackageCreator: # register package ctx = context.get() database: SQLite = ctx.get(ContextKey("database", SQLite)) - _, architecture = self.configuration.check_loaded() - package = Package.from_build(local_path, architecture, None) + _, repository_id = self.configuration.check_loaded() + package = Package.from_build(local_path, repository_id.architecture, None) database.package_update(package, BuildStatus()) diff --git a/src/ahriman/core/support/pkgbuild/keyring_generator.py b/src/ahriman/core/support/pkgbuild/keyring_generator.py index 3374eb2b..5e4d320b 100644 --- a/src/ahriman/core/support/pkgbuild/keyring_generator.py +++ b/src/ahriman/core/support/pkgbuild/keyring_generator.py @@ -25,6 +25,7 @@ from ahriman.core.database import SQLite from ahriman.core.exceptions import PkgbuildGeneratorError from ahriman.core.sign.gpg import GPG from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator +from ahriman.models.repository_id import RepositoryId class KeyringGenerator(PkgbuildGenerator): @@ -43,18 +44,20 @@ class KeyringGenerator(PkgbuildGenerator): trusted(list[str]): lif of trusted PGP keys """ - def __init__(self, database: SQLite, sign: GPG, configuration: Configuration, section: str) -> None: + def __init__(self, database: SQLite, sign: GPG, repository_id: RepositoryId, + configuration: Configuration, section: str) -> None: """ default constructor Args: database(SQLite): database instance sign(GPG): GPG wrapper instance + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance section(str): settings section name """ self.sign = sign - self.name = configuration.repository_name + self.name = repository_id.name # configuration fields packager_keys = [packager.key for packager in database.user_list(None, None) if packager.key is not None] diff --git a/src/ahriman/core/support/pkgbuild/mirrorlist_generator.py b/src/ahriman/core/support/pkgbuild/mirrorlist_generator.py index 36f3df08..0ca18dbe 100644 --- a/src/ahriman/core/support/pkgbuild/mirrorlist_generator.py +++ b/src/ahriman/core/support/pkgbuild/mirrorlist_generator.py @@ -23,6 +23,7 @@ from pathlib import Path from ahriman.core.configuration import Configuration from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator from ahriman.models.pkgbuild_patch import PkgbuildPatch +from ahriman.models.repository_id import RepositoryId class MirrorlistGenerator(PkgbuildGenerator): @@ -38,24 +39,24 @@ class MirrorlistGenerator(PkgbuildGenerator): servers(list[str]): list of mirror servers """ - def __init__(self, configuration: Configuration, section: str) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None: """ default constructor Args: + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance section(str): settings section name """ - name = configuration.repository_name - # configuration fields self.servers = configuration.getlist(section, "servers") - self.path = configuration.getpath(section, "path", fallback=Path("/etc") / "pacman.d" / f"{name}-mirrorlist") + self.path = configuration.getpath( + section, "path", fallback=Path("/etc") / "pacman.d" / f"{repository_id.name}-mirrorlist") self.path = self.path.relative_to("/") # in pkgbuild we are always operating with relative to / path # pkgbuild description fields - self.pkgbuild_pkgname = configuration.get(section, "package", fallback=f"{name}-mirrorlist") + self.pkgbuild_pkgname = configuration.get(section, "package", fallback=f"{repository_id.name}-mirrorlist") self.pkgbuild_pkgdesc = configuration.get( - section, "description", fallback=f"{name} mirror list for use by pacman") + section, "description", fallback=f"{repository_id.name} mirror list for use by pacman") self.pkgbuild_license = configuration.getlist(section, "license", fallback=["Unlicense"]) self.pkgbuild_url = configuration.get(section, "homepage", fallback="") diff --git a/src/ahriman/core/tree.py b/src/ahriman/core/tree.py index eb964f6c..07c0064c 100644 --- a/src/ahriman/core/tree.py +++ b/src/ahriman/core/tree.py @@ -102,10 +102,11 @@ class Tree: >>> from ahriman.core.configuration import Configuration >>> from ahriman.core.database import SQLite >>> from ahriman.core.repository import Repository + >>> from ahriman.models.repository_id import RepositoryId >>> >>> configuration = Configuration() >>> database = SQLite.load(configuration) - >>> repository = Repository.load("x86_64", configuration, database, report=True) + >>> repository = Repository.load(RepositoryId("x86_64", "aur-clone"), configuration, database, report=True) >>> packages = repository.packages() >>> >>> tree = Tree.resolve(packages) diff --git a/src/ahriman/core/triggers/trigger.py b/src/ahriman/core/triggers/trigger.py index b443c513..d16df3c5 100644 --- a/src/ahriman/core/triggers/trigger.py +++ b/src/ahriman/core/triggers/trigger.py @@ -23,6 +23,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.configuration.schema import ConfigurationSchema from ahriman.core.log import LazyLogging from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result @@ -34,8 +35,8 @@ class Trigger(LazyLogging): CONFIGURATION_SCHEMA(ConfigurationSchema): (class attribute) configuration schema template CONFIGURATION_SCHEMA_FALLBACK(str | None): (class attribute) optional fallback option for defining configuration schema type used - architecture(str): repository architecture configuration(Configuration): configuration instance + repository_id(RepositoryId): repository unique identifier Examples: This class must be used in order to create own extension. Basically idea is the following:: @@ -51,26 +52,37 @@ class Trigger(LazyLogging): >>> configuration = Configuration() >>> configuration.set_option("build", "triggers", "my.awesome.package.CustomTrigger") >>> - >>> loader = TriggerLoader.load("x86_64", configuration) + >>> loader = TriggerLoader.load(RepositoryId("x86_64", "aur-clone"), configuration) >>> loader.on_result(Result(), []) """ CONFIGURATION_SCHEMA: ConfigurationSchema = {} CONFIGURATION_SCHEMA_FALLBACK: str | None = None - def __init__(self, architecture: str, configuration: Configuration) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance """ - self.architecture = architecture + self.repository_id = repository_id self.configuration = configuration + @property + def architecture(self) -> str: + """ + repository architecture for backward compatibility + + Returns: + str: repository architecture + """ + return self.repository_id.architecture + @classmethod - def configuration_schema(cls, architecture: str, configuration: Configuration | None) -> ConfigurationSchema: + def configuration_schema(cls, repository_id: RepositoryId, + configuration: Configuration | None) -> ConfigurationSchema: """ configuration schema based on supplied service configuration @@ -78,7 +90,7 @@ class Trigger(LazyLogging): Schema must be in cerberus format, for details and examples you can check built-in triggers. Args: - architecture(str): repository architecture + repository_id(str): repository unique identifier configuration(Configuration | None): configuration instance. If set to None, the default schema should be returned @@ -93,7 +105,7 @@ class Trigger(LazyLogging): if not configuration.has_section(target): continue section, schema_name = configuration.gettype( - target, architecture, fallback=cls.CONFIGURATION_SCHEMA_FALLBACK) + target, repository_id, fallback=cls.CONFIGURATION_SCHEMA_FALLBACK) if schema_name not in cls.CONFIGURATION_SCHEMA: continue result[section] = cls.CONFIGURATION_SCHEMA[schema_name] diff --git a/src/ahriman/core/triggers/trigger_loader.py b/src/ahriman/core/triggers/trigger_loader.py index 82d4405f..d92586f7 100644 --- a/src/ahriman/core/triggers/trigger_loader.py +++ b/src/ahriman/core/triggers/trigger_loader.py @@ -31,6 +31,7 @@ from ahriman.core.exceptions import ExtensionError from ahriman.core.log import LazyLogging from ahriman.core.triggers import Trigger from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result @@ -49,7 +50,7 @@ class TriggerLoader(LazyLogging): Having such configuration you can create instance of the loader:: - >>> loader = TriggerLoader.load("x86_64", configuration) + >>> loader = TriggerLoader.load(RepositoryId("x86_64", "aur-clone"), configuration) >>> print(loader.triggers) After that you are free to run triggers:: @@ -65,12 +66,12 @@ class TriggerLoader(LazyLogging): self.triggers: list[Trigger] = [] @classmethod - def load(cls, architecture: str, configuration: Configuration) -> Self: + def load(cls, repository_id: RepositoryId, configuration: Configuration) -> Self: """ create instance from configuration Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance Returns: @@ -78,7 +79,7 @@ class TriggerLoader(LazyLogging): """ instance = cls() instance.triggers = [ - instance.load_trigger(trigger, architecture, configuration) + instance.load_trigger(trigger, repository_id, configuration) for trigger in instance.selected_triggers(configuration) ] @@ -166,13 +167,13 @@ class TriggerLoader(LazyLogging): except ModuleNotFoundError: raise ExtensionError(f"Module {package} not found") from None - def load_trigger(self, module_path: str, architecture: str, configuration: Configuration) -> Trigger: + def load_trigger(self, module_path: str, repository_id: RepositoryId, configuration: Configuration) -> Trigger: """ load trigger by module path Args: module_path(str): module import path to load - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance Returns: @@ -183,7 +184,7 @@ class TriggerLoader(LazyLogging): """ trigger_type = self.load_trigger_class(module_path) try: - trigger = trigger_type(architecture, configuration) + trigger = trigger_type(repository_id, configuration) except Exception: raise ExtensionError(f"Could not load instance of trigger from {trigger_type} loaded from {module_path}") diff --git a/src/ahriman/core/upload/github.py b/src/ahriman/core/upload/github.py index 5002b2a6..35be8fa1 100644 --- a/src/ahriman/core/upload/github.py +++ b/src/ahriman/core/upload/github.py @@ -25,32 +25,44 @@ from typing import Any from ahriman.core.configuration import Configuration from ahriman.core.upload.http_upload import HttpUpload +from ahriman.core.upload.upload import Upload from ahriman.core.util import walk from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId -class Github(HttpUpload): +class GitHub(Upload, HttpUpload): """ upload files to GitHub releases Attributes: github_owner(str): GitHub repository owner + github_release_tag(str): GitHub release tag + github_release_tag_name(str): GitHub release tag name github_repository(str): GitHub repository name """ - def __init__(self, architecture: str, configuration: Configuration, section: str) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance section(str): settings section name """ - HttpUpload.__init__(self, architecture, configuration, section) + Upload.__init__(self, repository_id, configuration) + HttpUpload.__init__(self, configuration, section) + self.github_owner = configuration.get(section, "owner") self.github_repository = configuration.get(section, "repository") + if configuration.getboolean(section, "use_full_release_name", fallback=False): + self.github_release_tag = f"{repository_id.name}-{repository_id.architecture}" + self.github_release_tag_name = f"{repository_id.name} {repository_id.architecture}" + else: + self.github_release_tag_name = self.github_release_tag = repository_id.architecture + def asset_remove(self, release: dict[str, Any], name: str) -> None: """ remove asset from the release by name @@ -136,7 +148,10 @@ class Github(HttpUpload): dict[str, Any]: GitHub API release object for the new release """ url = f"https://api.github.com/repos/{self.github_owner}/{self.github_repository}/releases" - response = self.make_request("POST", url, json={"tag_name": self.architecture, "name": self.architecture}) + response = self.make_request("POST", url, json={ + "tag_name": self.github_release_tag, + "name": self.github_release_tag_name, + }) release: dict[str, Any] = response.json() return release @@ -147,7 +162,7 @@ class Github(HttpUpload): Returns: dict[str, Any] | None: GitHub API release object if release found and None otherwise """ - url = f"https://api.github.com/repos/{self.github_owner}/{self.github_repository}/releases/tags/{self.architecture}" + url = f"https://api.github.com/repos/{self.github_owner}/{self.github_repository}/releases/tags/{self.github_release_tag}" try: response = self.make_request("GET", url) release: dict[str, Any] = response.json() diff --git a/src/ahriman/core/upload/http_upload.py b/src/ahriman/core/upload/http_upload.py index 68cc5c52..9c7d4d8e 100644 --- a/src/ahriman/core/upload/http_upload.py +++ b/src/ahriman/core/upload/http_upload.py @@ -21,28 +21,14 @@ import hashlib from pathlib import Path -from ahriman.core.configuration import Configuration from ahriman.core.http import SyncHttpClient -from ahriman.core.upload.upload import Upload -class HttpUpload(Upload, SyncHttpClient): +class HttpUpload(SyncHttpClient): """ helper for the http based uploads """ - def __init__(self, architecture: str, configuration: Configuration, section: str) -> None: - """ - default constructor - - Args: - architecture(str): repository architecture - configuration(Configuration): configuration instance - section(str): configuration section name - """ - Upload.__init__(self, architecture, configuration) - SyncHttpClient.__init__(self, section, configuration) - @staticmethod def calculate_hash(path: Path) -> str: """ diff --git a/src/ahriman/core/upload/remote_service.py b/src/ahriman/core/upload/remote_service.py index 5ea2e03a..507e7383 100644 --- a/src/ahriman/core/upload/remote_service.py +++ b/src/ahriman/core/upload/remote_service.py @@ -27,10 +27,12 @@ from ahriman.core.http import MultipartType from ahriman.core.sign.gpg import GPG from ahriman.core.status.web_client import WebClient from ahriman.core.upload.http_upload import HttpUpload +from ahriman.core.upload.upload import Upload from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId -class RemoteService(HttpUpload): +class RemoteService(Upload, HttpUpload): """ upload files to another server instance @@ -38,16 +40,17 @@ class RemoteService(HttpUpload): client(WebClient): web client instance """ - def __init__(self, architecture: str, configuration: Configuration, section: str) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance section(str): settings section name """ - HttpUpload.__init__(self, architecture, configuration, section) + Upload.__init__(self, repository_id, configuration) + HttpUpload.__init__(self, configuration, section) self.client = WebClient(configuration) @cached_property diff --git a/src/ahriman/core/upload/rsync.py b/src/ahriman/core/upload/rsync.py index fd5c66ea..8f5db29d 100644 --- a/src/ahriman/core/upload/rsync.py +++ b/src/ahriman/core/upload/rsync.py @@ -23,6 +23,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.upload.upload import Upload from ahriman.core.util import check_output from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId class Rsync(Upload): @@ -36,16 +37,16 @@ class Rsync(Upload): _check_output = check_output - def __init__(self, architecture: str, configuration: Configuration, section: str) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance section(str): settings section name """ - Upload.__init__(self, architecture, configuration) + Upload.__init__(self, repository_id, configuration) self.command = configuration.getlist(section, "command") self.remote = configuration.get(section, "remote") diff --git a/src/ahriman/core/upload/s3.py b/src/ahriman/core/upload/s3.py index dbf3bc4b..c2e3e5e3 100644 --- a/src/ahriman/core/upload/s3.py +++ b/src/ahriman/core/upload/s3.py @@ -28,6 +28,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.upload.upload import Upload from ahriman.core.util import walk from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId class S3(Upload): @@ -37,21 +38,25 @@ class S3(Upload): Attributes bucket(Any): boto3 S3 bucket object chunk_size(int): chunk size for calculating checksums + remote_root(Path): relative path to which packages will be uploaded """ - def __init__(self, architecture: str, configuration: Configuration, section: str) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance section(str): settings section name """ - Upload.__init__(self, architecture, configuration) + Upload.__init__(self, repository_id, configuration) self.bucket = self.get_bucket(configuration, section) self.chunk_size = configuration.getint(section, "chunk_size", fallback=8 * 1024 * 1024) + paths = configuration.repository_paths + self.remote_root = paths.repository.relative_to(paths.root / "repository") + @staticmethod def calculate_etag(path: Path, chunk_size: int) -> str: """ @@ -127,7 +132,7 @@ class S3(Upload): continue local_path = path / local_file - remote_path = Path(self.architecture) / local_file + remote_path = self.remote_root / local_file (mime, _) = mimetypes.guess_type(local_path) extra_args = {"ContentType": mime} if mime is not None else None @@ -155,8 +160,8 @@ class S3(Upload): Returns: dict[Path, Any]: map of path object to the remote s3 object """ - objects = self.bucket.objects.filter(Prefix=self.architecture) - return {Path(item.key).relative_to(self.architecture): item for item in objects} + objects = self.bucket.objects.filter(Prefix=str(self.remote_root)) + return {Path(item.key).relative_to(self.remote_root): item for item in objects} def sync(self, path: Path, built_packages: list[Package]) -> None: """ diff --git a/src/ahriman/core/upload/upload.py b/src/ahriman/core/upload/upload.py index 83df31ae..f12c6534 100644 --- a/src/ahriman/core/upload/upload.py +++ b/src/ahriman/core/upload/upload.py @@ -25,6 +25,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.exceptions import SynchronizationError from ahriman.core.log import LazyLogging from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId from ahriman.models.upload_settings import UploadSettings @@ -33,18 +34,16 @@ class Upload(LazyLogging): base remote sync class Attributes: - architecture(str): repository architecture configuration(Configuration): configuration instance + repository_id(RepositoryId): repository unique identifier Examples: These classes provide the way to upload packages to remote sources as it is described in their implementations. Basic flow includes class instantiating by using the ``load`` method and then calling the ``run`` method which wraps any internal exceptions into the ``SyncFailed`` exception:: - >>> from ahriman.core.configuration import Configuration - >>> >>> configuration = Configuration() - >>> upload = Upload.load("x86_64", configuration, "s3") + >>> upload = Upload.load(RepositoryId("x86_64", "aur-clone"), configuration, "s3") >>> upload.run(configuration.repository_paths.repository, []) Or in case if direct access to exception is required, the ``sync`` method can be used:: @@ -55,46 +54,46 @@ class Upload(LazyLogging): >>> handle_exceptions(ex) """ - def __init__(self, architecture: str, configuration: Configuration) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance """ - self.architecture = architecture + self.repository_id = repository_id self.configuration = configuration @staticmethod - def load(architecture: str, configuration: Configuration, target: str) -> Upload: + def load(repository_id: RepositoryId, configuration: Configuration, target: str) -> Upload: """ load client from settings Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance target(str): target to run sync (e.g. s3) Returns: Upload: client according to current settings """ - section, provider_name = configuration.gettype(target, architecture) + section, provider_name = configuration.gettype(target, repository_id) match UploadSettings.from_option(provider_name): case UploadSettings.Rsync: from ahriman.core.upload.rsync import Rsync - return Rsync(architecture, configuration, section) + return Rsync(repository_id, configuration, section) case UploadSettings.S3: from ahriman.core.upload.s3 import S3 - return S3(architecture, configuration, section) + return S3(repository_id, configuration, section) case UploadSettings.GitHub: - from ahriman.core.upload.github import Github - return Github(architecture, configuration, section) + from ahriman.core.upload.github import GitHub + return GitHub(repository_id, configuration, section) case UploadSettings.RemoteService: from ahriman.core.upload.remote_service import RemoteService - return RemoteService(architecture, configuration, section) + return RemoteService(repository_id, configuration, section) case _: - return Upload(architecture, configuration) # should never happen + return Upload(repository_id, configuration) # should never happen def run(self, path: Path, built_packages: list[Package]) -> None: """ diff --git a/src/ahriman/core/upload/upload_trigger.py b/src/ahriman/core/upload/upload_trigger.py index 0ccdfe15..f07dce14 100644 --- a/src/ahriman/core/upload/upload_trigger.py +++ b/src/ahriman/core/upload/upload_trigger.py @@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.triggers import Trigger from ahriman.core.upload.upload import Upload from ahriman.models.package import Package +from ahriman.models.repository_id import RepositoryId from ahriman.models.result import Result @@ -67,6 +68,10 @@ class UploadTrigger(Trigger): "coerce": "integer", "min": 0, }, + "use_full_release_name": { + "type": "boolean", + "coerce": "boolean", + }, "username": { "type": "string", }, @@ -138,15 +143,15 @@ class UploadTrigger(Trigger): }, } - def __init__(self, architecture: str, configuration: Configuration) -> None: + def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None: """ default constructor Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance """ - Trigger.__init__(self, architecture, configuration) + Trigger.__init__(self, repository_id, configuration) self.targets = self.configuration_sections(configuration) @classmethod @@ -171,5 +176,5 @@ class UploadTrigger(Trigger): packages(list[Package]): list of all available packages """ for target in self.targets: - runner = Upload.load(self.architecture, self.configuration, target) + runner = Upload.load(self.repository_id, self.configuration, target) runner.run(self.configuration.repository_paths.repository, result.success) diff --git a/src/ahriman/models/aur_package.py b/src/ahriman/models/aur_package.py index 8b9e620e..789d93ad 100644 --- a/src/ahriman/models/aur_package.py +++ b/src/ahriman/models/aur_package.py @@ -68,9 +68,10 @@ class AURPackage: >>> >>> from ahriman.core.alpm.pacman import Pacman >>> from ahriman.core.configuration import Configuration + >>> from ahriman.models.repository_id import RepositoryId >>> >>> configuration = Configuration() - >>> pacman = Pacman("x86_64", configuration) + >>> pacman = Pacman(RepositoryId("x86_64", "aur-clone"), configuration) >>> metadata = pacman.package_get("pacman") >>> package = AURPackage.from_pacman(next(metadata)) # load package from pyalpm wrapper """ diff --git a/src/ahriman/models/package_description.py b/src/ahriman/models/package_description.py index 7154e217..bbb08c31 100644 --- a/src/ahriman/models/package_description.py +++ b/src/ahriman/models/package_description.py @@ -56,9 +56,10 @@ class PackageDescription: >>> from pathlib import Path >>> from ahriman.core.alpm.pacman import Pacman >>> from ahriman.core.configuration import Configuration + >>> from ahriman.models.repository_id import RepositoryId >>> >>> configuration = Configuration() - >>> pacman = Pacman("x86_64", configuration) + >>> pacman = Pacman(RepositoryId("x86_64", "aur-clone"), configuration) >>> pyalpm_description = next(package for package in pacman.package_get("pacman")) >>> description = PackageDescription.from_package( >>> pyalpm_description, Path("/var/cache/pacman/pkg/pacman-6.0.1-4-x86_64.pkg.tar.zst")) diff --git a/src/ahriman/models/repository_id.py b/src/ahriman/models/repository_id.py new file mode 100644 index 00000000..67ca8478 --- /dev/null +++ b/src/ahriman/models/repository_id.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2021-2023 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +from dataclasses import dataclass +from typing import Any + +from ahriman.core.exceptions import InitializeError + + +@dataclass(frozen=True) +class RepositoryId: + """ + unique identifier of the repository + + Attributes: + architecture(str): repository architecture + name(str): repository name + """ + + architecture: str + name: str + + def __post_init__(self) -> None: + """ + check that name is set + + Raises: + InitializeError: in case if name is not set + """ + if not self.name: + raise InitializeError("Repository name is not set") + + def __lt__(self, other: Any) -> bool: + """ + comparison operator for sorting + + Args: + other(Any): other object to compare + + Returns: + bool: True in case if this is less than other and False otherwise + + Raises: + TypeError: if other is different from RepositoryId type + """ + if not isinstance(other, RepositoryId): + raise ValueError(f"'<' not supported between instances of '{type(self)}' and '{type(other)}'") + + return self.name <= other.name and self.architecture < other.architecture diff --git a/src/ahriman/models/repository_paths.py b/src/ahriman/models/repository_paths.py index b7fbd4f1..8295905c 100644 --- a/src/ahriman/models/repository_paths.py +++ b/src/ahriman/models/repository_paths.py @@ -20,25 +20,29 @@ import os import shutil +from collections.abc import Generator from dataclasses import dataclass +from functools import cached_property from pathlib import Path from ahriman.core.exceptions import PathError +from ahriman.core.log import LazyLogging +from ahriman.models.repository_id import RepositoryId @dataclass(frozen=True) -class RepositoryPaths: +class RepositoryPaths(LazyLogging): """ repository paths holder. For the most operations with paths you want to use this object Attributes: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier root(Path): repository root (i.e. ahriman home) Examples: This class can be used in order to access the repository tree structure:: - >>> paths = RepositoryPaths(Path("/var/lib/ahriman"), "x86_64") + >>> paths = RepositoryPaths(Path("/var/lib/ahriman"), RepositoryId("x86_64", "aur-clone")) Additional methods can be used in order to ensure that tree is created:: @@ -51,7 +55,21 @@ class RepositoryPaths: """ root: Path - architecture: str + repository_id: RepositoryId + + @cached_property + def _suffix(self) -> Path: + """ + suffix of the paths as defined by repository structure + + Returns: + Path: relative path which contains only architecture segment in case if legacy tree is used and repository + name and architecture otherwise + """ + if (self.root / "repository" / self.repository_id.architecture).is_dir(): + self.logger.warning("legacy single repository tree has been found") + return Path(self.repository_id.architecture) + return Path(self.repository_id.name) / self.repository_id.architecture @property def cache(self) -> Path: @@ -72,7 +90,7 @@ class RepositoryPaths: Path: full patch to devtools chroot directory """ # for the chroot directory devtools will create own tree, and we don"t have to specify architecture here - return self.root / "chroot" + return self.root / "chroot" / self.repository_id.name @property def packages(self) -> Path: @@ -82,7 +100,7 @@ class RepositoryPaths: Returns: Path: full path to built packages directory """ - return self.root / "packages" / self.architecture + return self.root / "packages" / self._suffix @property def pacman(self) -> Path: @@ -92,7 +110,7 @@ class RepositoryPaths: Returns: Path: full path to pacman local database cache """ - return self.root / "pacman" / self.architecture + return self.root / "pacman" / self._suffix @property def repository(self) -> Path: @@ -102,7 +120,7 @@ class RepositoryPaths: Returns: Path: full path to the repository directory """ - return self.root / "repository" / self.architecture + return self.root / "repository" / self._suffix @property def root_owner(self) -> tuple[int, int]: @@ -115,22 +133,28 @@ class RepositoryPaths: return self.owner(self.root) @classmethod - def known_architectures(cls, root: Path) -> set[str]: + def known_architectures(cls, root: Path, name: str) -> set[RepositoryId]: """ get known architectures Args: root(Path): repository root + name(str): repository name from configuration Returns: - set[str]: list of architectures for which tree is created + set[RepositoryId]: list of tuple of repository name and architectures for which tree is created """ - paths = cls(root, "") - return { - path.name - for path in paths.repository.iterdir() - if path.is_dir() - } + def walk(repository_path: Path, repository_name: str) -> Generator[RepositoryId, None, None]: + for architecture in filter(lambda path: path.is_dir(), repository_path.iterdir()): + yield RepositoryId(architecture.name, repository_name) + + def walk_with_name(paths: RepositoryPaths) -> Generator[RepositoryId, None, None]: + for repository in filter(lambda path: path.is_dir(), paths.repository.iterdir()): + yield from walk(repository, repository.name) + + instance = cls(root, RepositoryId("", "")) + # try to get list per repository first and then fallback to old schema if nothing found + return set(walk_with_name(instance)) or set(walk(instance.repository, name)) @staticmethod def owner(path: Path) -> tuple[int, int]: diff --git a/src/ahriman/web/views/v1/status/status.py b/src/ahriman/web/views/v1/status/status.py index c30dcc41..8bd4feef 100644 --- a/src/ahriman/web/views/v1/status/status.py +++ b/src/ahriman/web/views/v1/status/status.py @@ -65,9 +65,9 @@ class StatusView(BaseView): counters = Counters.from_packages(self.service.packages) status = InternalStatus( status=self.service.status, - architecture=self.service.architecture, + architecture=self.service.repository_id.architecture, packages=counters, - repository=self.service.repository.name, + repository=self.service.repository_id.name, version=__version__, ) diff --git a/src/ahriman/web/web.py b/src/ahriman/web/web.py index 040343dd..25d937a6 100644 --- a/src/ahriman/web/web.py +++ b/src/ahriman/web/web.py @@ -31,6 +31,7 @@ from ahriman.core.exceptions import InitializeError from ahriman.core.log.filtered_access_logger import FilteredAccessLogger from ahriman.core.spawn import Spawn from ahriman.core.status.watcher import Watcher +from ahriman.models.repository_id import RepositoryId from ahriman.web.apispec import setup_apispec from ahriman.web.cors import setup_cors from ahriman.web.middlewares.exception_handler import exception_handler @@ -120,12 +121,12 @@ def run_server(application: Application) -> None: access_log=logging.getLogger("http"), access_log_class=FilteredAccessLogger) -def setup_service(architecture: str, configuration: Configuration, spawner: Spawn) -> Application: +def setup_service(repository_id: RepositoryId, configuration: Configuration, spawner: Spawn) -> Application: """ create web application Args: - architecture(str): repository architecture + repository_id(RepositoryId): repository unique identifier configuration(Configuration): configuration instance spawner(Spawn): spawner thread @@ -155,7 +156,7 @@ def setup_service(architecture: str, configuration: Configuration, spawner: Spaw database = application["database"] = SQLite.load(configuration) application.logger.info("setup watcher") - application["watcher"] = Watcher(architecture, configuration, database) + application["watcher"] = Watcher(repository_id, configuration, database) application.logger.info("setup process spawner") application["spawn"] = spawner diff --git a/tests/ahriman/application/application/conftest.py b/tests/ahriman/application/application/conftest.py index dbaef770..9b607430 100644 --- a/tests/ahriman/application/application/conftest.py +++ b/tests/ahriman/application/application/conftest.py @@ -27,7 +27,8 @@ def application_packages(configuration: Configuration, database: SQLite, reposit """ mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) mocker.patch("ahriman.core.database.SQLite.load", return_value=database) - return ApplicationPackages("x86_64", configuration, report=False) + _, repository_id = configuration.check_loaded() + return ApplicationPackages(repository_id, configuration, report=False) @pytest.fixture @@ -47,7 +48,8 @@ def application_properties(configuration: Configuration, database: SQLite, repos """ mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) mocker.patch("ahriman.core.database.SQLite.load", return_value=database) - return ApplicationProperties("x86_64", configuration, report=False) + _, repository_id = configuration.check_loaded() + return ApplicationProperties(repository_id, configuration, report=False) @pytest.fixture @@ -67,4 +69,5 @@ def application_repository(configuration: Configuration, database: SQLite, repos """ mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) mocker.patch("ahriman.core.database.SQLite.load", return_value=database) - return ApplicationRepository("x86_64", configuration, report=False) + _, repository_id = configuration.check_loaded() + return ApplicationRepository(repository_id, configuration, report=False) diff --git a/tests/ahriman/application/conftest.py b/tests/ahriman/application/conftest.py index 56a82310..a2fd871e 100644 --- a/tests/ahriman/application/conftest.py +++ b/tests/ahriman/application/conftest.py @@ -28,7 +28,8 @@ def application(configuration: Configuration, repository: Repository, database: """ mocker.patch("ahriman.core.database.SQLite.load", return_value=database) mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) - return Application("x86_64", configuration, report=False) + _, repository_id = configuration.check_loaded() + return Application(repository_id, configuration, report=False) @pytest.fixture @@ -54,7 +55,8 @@ def lock(args: argparse.Namespace, configuration: Configuration) -> Lock: Returns: Lock: file lock test instance """ - return Lock(args, "x86_64", configuration) + _, repository_id = configuration.check_loaded() + return Lock(args, repository_id, configuration) @pytest.fixture diff --git a/tests/ahriman/conftest.py b/tests/ahriman/conftest.py index 12b96d86..7a91d465 100644 --- a/tests/ahriman/conftest.py +++ b/tests/ahriman/conftest.py @@ -19,7 +19,9 @@ from ahriman.models.build_status import BuildStatus, BuildStatusEnum from ahriman.models.package import Package from ahriman.models.package_description import PackageDescription from ahriman.models.package_source import PackageSource +from ahriman.models.pacman_synchronization import PacmanSynchronization from ahriman.models.remote_source import RemoteSource +from ahriman.models.repository_id import RepositoryId from ahriman.models.repository_paths import RepositoryPaths from ahriman.models.result import Result from ahriman.models.user import User @@ -243,18 +245,19 @@ def auth(configuration: Configuration) -> Auth: @pytest.fixture -def configuration(resource_path_root: Path) -> Configuration: +def configuration(repository_id: RepositoryId, resource_path_root: Path) -> Configuration: """ configuration fixture Args: + repository_id(RepositoryId): repository identifier fixture resource_path_root(Path): resource path root directory Returns: Configuration: configuration test instance """ path = resource_path_root / "core" / "ahriman.ini" - return Configuration.from_path(path=path, architecture="x86_64") + return Configuration.from_path(path, repository_id) @pytest.fixture @@ -434,7 +437,8 @@ def pacman(configuration: Configuration) -> Pacman: Returns: Pacman: pacman wrapper test instance """ - return Pacman("x86_64", configuration, refresh_database=0) + _, repository_id = configuration.check_loaded() + return Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled) @pytest.fixture @@ -481,7 +485,19 @@ def repository(configuration: Configuration, database: SQLite, mocker: MockerFix Repository: repository test instance """ mocker.patch("ahriman.core.repository.Repository._set_context") - return Repository.load("x86_64", configuration, database, report=False) + _, repository_id = configuration.check_loaded() + return Repository.load(repository_id, configuration, database, report=False) + + +@pytest.fixture +def repository_id() -> RepositoryId: + """ + fixture for repository identifier + + Returns: + RepositoryId: repository identifier test instance + """ + return RepositoryId("x86_64", "aur-clone") @pytest.fixture @@ -525,7 +541,12 @@ def spawner(configuration: Configuration) -> Spawn: Returns: Spawn: spawner fixture """ - return Spawn(MagicMock(), "x86_64", ["--architecture", "x86_64", "--configuration", str(configuration.path)]) + _, repository_id = configuration.check_loaded() + return Spawn(MagicMock(), repository_id, [ + "--architecture", "x86_64", + "--repository", repository_id.name, + "--configuration", str(configuration.path), + ]) @pytest.fixture @@ -554,4 +575,5 @@ def watcher(configuration: Configuration, database: SQLite, repository: Reposito Watcher: package status watcher test instance """ mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) - return Watcher("x86_64", configuration, database) + _, repository_id = configuration.check_loaded() + return Watcher(repository_id, configuration, database) diff --git a/tests/ahriman/core/log/test_log.py b/tests/ahriman/core/log/test_log_loader.py similarity index 72% rename from tests/ahriman/core/log/test_log.py rename to tests/ahriman/core/log/test_log_loader.py index b6652481..4e12eb26 100644 --- a/tests/ahriman/core/log/test_log.py +++ b/tests/ahriman/core/log/test_log_loader.py @@ -7,7 +7,7 @@ from pytest_mock import MockerFixture from systemd.journal import JournalHandler from ahriman.core.configuration import Configuration -from ahriman.core.log import Log +from ahriman.core.log.log_loader import LogLoader from ahriman.models.log_handler import LogHandler @@ -15,14 +15,14 @@ def test_handler() -> None: """ must extract journald handler if available """ - assert Log.handler(None) == LogHandler.Journald + assert LogLoader.handler(None) == LogHandler.Journald def test_handler_selected() -> None: """ must return selected log handler """ - assert Log.handler(LogHandler.Console) == LogHandler.Console + assert LogLoader.handler(LogHandler.Console) == LogHandler.Console def test_handler_syslog(mocker: MockerFixture) -> None: @@ -31,7 +31,7 @@ def test_handler_syslog(mocker: MockerFixture) -> None: """ mocker.patch("pathlib.Path.exists", return_value=True) mocker.patch.dict(sys.modules, {"systemd.journal": None}) - assert Log.handler(None) == LogHandler.Syslog + assert LogLoader.handler(None) == LogHandler.Syslog def test_handler_console(mocker: MockerFixture) -> None: @@ -40,17 +40,17 @@ def test_handler_console(mocker: MockerFixture) -> None: """ mocker.patch("pathlib.Path.exists", return_value=False) mocker.patch.dict(sys.modules, {"systemd.journal": None}) - assert Log.handler(None) == LogHandler.Console + assert LogLoader.handler(None) == LogHandler.Console def test_load(configuration: Configuration, mocker: MockerFixture) -> None: """ must load logging """ - logging_mock = mocker.patch("ahriman.core.log.log.fileConfig", side_effect=fileConfig) + logging_mock = mocker.patch("ahriman.core.log.log_loader.fileConfig", side_effect=fileConfig) http_log_mock = mocker.patch("ahriman.core.log.http_log_handler.HttpLogHandler.load") - Log.load(configuration, LogHandler.Journald, quiet=False, report=False) + LogLoader.load(configuration, LogHandler.Journald, quiet=False, report=False) logging_mock.assert_called_once_with(pytest.helpers.anyvar(int), disable_existing_loggers=True) http_log_mock.assert_called_once_with(configuration, report=False) assert all(isinstance(handler, JournalHandler) for handler in logging.getLogger().handlers) @@ -60,8 +60,8 @@ def test_load_fallback(configuration: Configuration, mocker: MockerFixture) -> N """ must fall back to stderr without errors """ - mocker.patch("ahriman.core.log.log.fileConfig", side_effect=PermissionError()) - Log.load(configuration, LogHandler.Journald, quiet=False, report=False) + mocker.patch("ahriman.core.log.log_loader.fileConfig", side_effect=PermissionError()) + LogLoader.load(configuration, LogHandler.Journald, quiet=False, report=False) def test_load_quiet(configuration: Configuration, mocker: MockerFixture) -> None: @@ -69,5 +69,5 @@ def test_load_quiet(configuration: Configuration, mocker: MockerFixture) -> None must disable logging in case if quiet flag set """ disable_mock = mocker.patch("logging.disable") - Log.load(configuration, LogHandler.Journald, quiet=True, report=False) + LogLoader.load(configuration, LogHandler.Journald, quiet=True, report=False) disable_mock.assert_called_once_with(logging.WARNING) diff --git a/tests/ahriman/core/report/conftest.py b/tests/ahriman/core/report/conftest.py index a2927ba0..0a6ffb37 100644 --- a/tests/ahriman/core/report/conftest.py +++ b/tests/ahriman/core/report/conftest.py @@ -17,4 +17,5 @@ def remote_call(configuration: Configuration) -> RemoteCall: """ configuration.set_option("web", "host", "localhost") configuration.set_option("web", "port", "8080") - return RemoteCall("x86_64", configuration, "remote-call") + _, repository_id = configuration.check_loaded() + return RemoteCall(repository_id, configuration, "remote-call") diff --git a/tests/ahriman/core/repository/conftest.py b/tests/ahriman/core/repository/conftest.py index fa602a10..439cf473 100644 --- a/tests/ahriman/core/repository/conftest.py +++ b/tests/ahriman/core/repository/conftest.py @@ -22,7 +22,8 @@ def cleaner(configuration: Configuration, database: SQLite) -> Cleaner: Returns: Cleaner: cleaner test instance """ - return Cleaner("x86_64", configuration, database, report=False, + _, repository_id = configuration.check_loaded() + return Cleaner(repository_id, configuration, database, report=False, refresh_pacman_database=PacmanSynchronization.Disabled) @@ -43,7 +44,8 @@ def executor(configuration: Configuration, database: SQLite, mocker: MockerFixtu mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_chroot") mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_packages") mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_queue") - return Executor("x86_64", configuration, database, report=False, + _, repository_id = configuration.check_loaded() + return Executor(repository_id, configuration, database, report=False, refresh_pacman_database=PacmanSynchronization.Disabled) @@ -64,5 +66,6 @@ def update_handler(configuration: Configuration, database: SQLite, mocker: Mocke mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_chroot") mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_packages") mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_queue") - return UpdateHandler("x86_64", configuration, database, report=False, + _, repository_id = configuration.check_loaded() + return UpdateHandler(repository_id, configuration, database, report=False, refresh_pacman_database=PacmanSynchronization.Disabled) diff --git a/tests/ahriman/core/triggers/conftest.py b/tests/ahriman/core/triggers/conftest.py index 386136f0..dcfb3483 100644 --- a/tests/ahriman/core/triggers/conftest.py +++ b/tests/ahriman/core/triggers/conftest.py @@ -15,7 +15,8 @@ def trigger(configuration: Configuration) -> Trigger: Returns: Trigger: trigger test instance """ - return Trigger("x86_64", configuration) + _, repository_id = configuration.check_loaded() + return Trigger(repository_id, configuration) @pytest.fixture @@ -28,4 +29,5 @@ def trigger_loader(configuration: Configuration) -> TriggerLoader: Returns: TriggerLoader: trigger loader test instance """ - return TriggerLoader.load("x86_64", configuration) + _, repository_id = configuration.check_loaded() + return TriggerLoader.load(repository_id, configuration) diff --git a/tests/ahriman/core/upload/conftest.py b/tests/ahriman/core/upload/conftest.py index f3439b07..380fac6e 100644 --- a/tests/ahriman/core/upload/conftest.py +++ b/tests/ahriman/core/upload/conftest.py @@ -4,33 +4,34 @@ from typing import Any from unittest.mock import MagicMock from ahriman.core.configuration import Configuration -from ahriman.core.upload.github import Github +from ahriman.core.upload.github import GitHub from ahriman.core.upload.remote_service import RemoteService from ahriman.core.upload.rsync import Rsync from ahriman.core.upload.s3 import S3 @pytest.fixture -def github(configuration: Configuration) -> Github: +def github(configuration: Configuration) -> GitHub: """ - fixture for github synchronization + fixture for GitHub synchronization Args: configuration(Configuration): configuration fixture Returns: - Github: github test instance + GitHub: GitHub test instance """ - return Github("x86_64", configuration, "github:x86_64") + _, repository_id = configuration.check_loaded() + return GitHub(repository_id, configuration, "github:x86_64") @pytest.fixture def github_release() -> dict[str, Any]: """ - fixture for the github release object + fixture for the GitHub release object Returns: - dict[str, Any]: github test release object + dict[str, Any]: GitHub test release object """ return { "url": "release_url", @@ -59,7 +60,8 @@ def remote_service(configuration: Configuration) -> RemoteService: """ configuration.set_option("web", "host", "localhost") configuration.set_option("web", "port", "8080") - return RemoteService("x86_64", configuration, "remote-service") + _, repository_id = configuration.check_loaded() + return RemoteService(repository_id, configuration, "remote-service") @pytest.fixture @@ -73,7 +75,8 @@ def rsync(configuration: Configuration) -> Rsync: Returns: Rsync: rsync test instance """ - return Rsync("x86_64", configuration, "rsync") + _, repository_id = configuration.check_loaded() + return Rsync(repository_id, configuration, "rsync") @pytest.fixture @@ -87,7 +90,8 @@ def s3(configuration: Configuration) -> S3: Returns: S3: S3 test instance """ - return S3("x86_64", configuration, "customs3") + _, repository_id = configuration.check_loaded() + return S3(repository_id, configuration, "customs3") @pytest.fixture diff --git a/tests/ahriman/core/upload/test_github.py b/tests/ahriman/core/upload/test_github.py index 477f8751..57bb12e6 100644 --- a/tests/ahriman/core/upload/test_github.py +++ b/tests/ahriman/core/upload/test_github.py @@ -6,10 +6,10 @@ from pytest_mock import MockerFixture from typing import Any from unittest.mock import call as MockCall -from ahriman.core.upload.github import Github +from ahriman.core.upload.github import GitHub -def test_asset_remove(github: Github, github_release: dict[str, Any], mocker: MockerFixture) -> None: +def test_asset_remove(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None: """ must remove asset from the release """ @@ -18,7 +18,7 @@ def test_asset_remove(github: Github, github_release: dict[str, Any], mocker: Mo request_mock.assert_called_once_with("DELETE", "asset_url") -def test_asset_remove_unknown(github: Github, github_release: dict[str, Any], mocker: MockerFixture) -> None: +def test_asset_remove_unknown(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None: """ must not fail if no asset found """ @@ -27,7 +27,7 @@ def test_asset_remove_unknown(github: Github, github_release: dict[str, Any], mo request_mock.assert_not_called() -def test_asset_upload(github: Github, github_release: dict[str, Any], mocker: MockerFixture) -> None: +def test_asset_upload(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None: """ must upload asset to the repository """ @@ -42,7 +42,7 @@ def test_asset_upload(github: Github, github_release: dict[str, Any], mocker: Mo remove_mock.assert_not_called() -def test_asset_upload_with_removal(github: Github, github_release: dict[str, Any], mocker: MockerFixture) -> None: +def test_asset_upload_with_removal(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None: """ must remove existing file before upload """ @@ -58,7 +58,7 @@ def test_asset_upload_with_removal(github: Github, github_release: dict[str, Any ]) -def test_asset_upload_empty_mimetype(github: Github, github_release: dict[str, Any], mocker: MockerFixture) -> None: +def test_asset_upload_empty_mimetype(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None: """ must upload asset to the repository with empty mime type if the library cannot guess it """ @@ -73,7 +73,7 @@ def test_asset_upload_empty_mimetype(github: Github, github_release: dict[str, A headers={"Content-Type": "application/octet-stream"}) -def test_get_local_files(github: Github, resource_path_root: Path, mocker: MockerFixture) -> None: +def test_get_local_files(github: GitHub, resource_path_root: Path, mocker: MockerFixture) -> None: """ must get all local files recursively """ @@ -82,7 +82,7 @@ def test_get_local_files(github: Github, resource_path_root: Path, mocker: Mocke walk_mock.assert_called() -def test_files_remove(github: Github, github_release: dict[str, Any], mocker: MockerFixture) -> None: +def test_files_remove(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None: """ must remove files from the remote """ @@ -91,7 +91,7 @@ def test_files_remove(github: Github, github_release: dict[str, Any], mocker: Mo remove_mock.assert_called_once_with(github_release, "b") -def test_files_remove_empty(github: Github, github_release: dict[str, Any], mocker: MockerFixture) -> None: +def test_files_remove_empty(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None: """ must remove nothing if nothing changed """ @@ -100,7 +100,7 @@ def test_files_remove_empty(github: Github, github_release: dict[str, Any], mock remove_mock.assert_not_called() -def test_files_upload(github: Github, github_release: dict[str, Any], mocker: MockerFixture) -> None: +def test_files_upload(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None: """ must upload files to the remote """ @@ -112,7 +112,7 @@ def test_files_upload(github: Github, github_release: dict[str, Any], mocker: Mo ]) -def test_files_upload_empty(github: Github, github_release: dict[str, Any], mocker: MockerFixture) -> None: +def test_files_upload_empty(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None: """ must upload nothing if nothing changed """ @@ -121,7 +121,7 @@ def test_files_upload_empty(github: Github, github_release: dict[str, Any], mock upload_mock.assert_not_called() -def test_release_create(github: Github, mocker: MockerFixture) -> None: +def test_release_create(github: GitHub, mocker: MockerFixture) -> None: """ must create release """ @@ -131,7 +131,7 @@ def test_release_create(github: Github, mocker: MockerFixture) -> None: json={"tag_name": github.architecture, "name": github.architecture}) -def test_release_get(github: Github, mocker: MockerFixture) -> None: +def test_release_get(github: GitHub, mocker: MockerFixture) -> None: """ must get release """ @@ -140,7 +140,7 @@ def test_release_get(github: Github, mocker: MockerFixture) -> None: request_mock.assert_called_once_with("GET", pytest.helpers.anyvar(str, True)) -def test_release_get_empty(github: Github, mocker: MockerFixture) -> None: +def test_release_get_empty(github: GitHub, mocker: MockerFixture) -> None: """ must return nothing in case of 404 status code """ @@ -151,7 +151,7 @@ def test_release_get_empty(github: Github, mocker: MockerFixture) -> None: assert github.release_get() is None -def test_release_get_exception(github: Github, mocker: MockerFixture) -> None: +def test_release_get_exception(github: GitHub, mocker: MockerFixture) -> None: """ must re-raise non HTTPError exception """ @@ -160,7 +160,7 @@ def test_release_get_exception(github: Github, mocker: MockerFixture) -> None: github.release_get() -def test_release_get_exception_http_error(github: Github, mocker: MockerFixture) -> None: +def test_release_get_exception_http_error(github: GitHub, mocker: MockerFixture) -> None: """ must re-raise HTTPError exception with code differs from 404 """ @@ -170,7 +170,7 @@ def test_release_get_exception_http_error(github: Github, mocker: MockerFixture) github.release_get() -def test_release_update(github: Github, github_release: dict[str, Any], mocker: MockerFixture) -> None: +def test_release_update(github: GitHub, github_release: dict[str, Any], mocker: MockerFixture) -> None: """ must update release """ @@ -179,7 +179,7 @@ def test_release_update(github: Github, github_release: dict[str, Any], mocker: request_mock.assert_called_once_with("POST", "release_url", json={"body": "body"}) -def test_release_sync(github: Github, mocker: MockerFixture) -> None: +def test_release_sync(github: GitHub, mocker: MockerFixture) -> None: """ must run sync command """ @@ -199,7 +199,7 @@ def test_release_sync(github: Github, mocker: MockerFixture) -> None: release_update_mock.assert_called_once_with({}, pytest.helpers.anyvar(int)) -def test_release_sync_create_release(github: Github, mocker: MockerFixture) -> None: +def test_release_sync_create_release(github: GitHub, mocker: MockerFixture) -> None: """ must create release in case if it does not exist """ diff --git a/tests/ahriman/web/conftest.py b/tests/ahriman/web/conftest.py index 665bbdac..cc53c7b9 100644 --- a/tests/ahriman/web/conftest.py +++ b/tests/ahriman/web/conftest.py @@ -16,6 +16,7 @@ from ahriman.core.configuration import Configuration from ahriman.core.database import SQLite from ahriman.core.repository import Repository from ahriman.core.spawn import Spawn +from ahriman.models.repository_id import RepositoryId from ahriman.models.user import User from ahriman.web.web import setup_service @@ -112,7 +113,9 @@ def application(configuration: Configuration, spawner: Spawn, database: SQLite, mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) mocker.patch("aiohttp_apispec.setup_aiohttp_apispec") mocker.patch.object(ahriman.core.auth.helpers, "_has_aiohttp_security", False) - return setup_service("x86_64", configuration, spawner) + _, repository_id = configuration.check_loaded() + + return setup_service(repository_id, configuration, spawner) @pytest.fixture @@ -138,7 +141,8 @@ def application_with_auth(configuration: Configuration, user: User, spawner: Spa mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) mocker.patch("aiohttp_apispec.setup_aiohttp_apispec") mocker.patch.object(ahriman.core.auth.helpers, "_has_aiohttp_security", True) - application = setup_service("x86_64", configuration, spawner) + _, repository_id = configuration.check_loaded() + application = setup_service(repository_id, configuration, spawner) generated = user.hash_password(application["validator"].salt) mocker.patch("ahriman.core.database.SQLite.user_get", return_value=generated) @@ -169,7 +173,9 @@ def application_with_debug(configuration: Configuration, user: User, spawner: Sp mocker.patch("ahriman.core.repository.Repository.load", return_value=repository) mocker.patch("aiohttp_apispec.setup_aiohttp_apispec") mocker.patch.object(ahriman.core.auth.helpers, "_has_aiohttp_security", False) - return setup_service("x86_64", configuration, spawner) + _, repository_id = configuration.check_loaded() + + return setup_service(repository_id, configuration, spawner) @pytest.fixture