arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-04-24 15:27:17 +00:00
Code Issues Projects Releases Wiki Activity

allow to use one application for multiple repositories

Browse Source
This commit is contained in:
Evgenii Alekseev 2023-08-28 19:24:42 +03:00
parent c915d68c97
commit 32b00de42c
103 changed files with 1229 additions and 599 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docker/entrypoint.sh
View File

@ -17,6 +17,7 @@ host = $AHRIMAN_HOST
EOF
AHRIMAN_DEFAULT_ARGS=("--architecture" "$AHRIMAN_ARCHITECTURE")
AHRIMAN_DEFAULT_ARGS+=("--repository" "$AHRIMAN_REPOSITORY")
if [ -n "$AHRIMAN_OUTPUT" ]; then
AHRIMAN_DEFAULT_ARGS+=("--log-handler" "$AHRIMAN_OUTPUT")
fi
@ -33,19 +34,18 @@ chown "$AHRIMAN_USER":"$AHRIMAN_USER" "$AHRIMAN_GNUPG_HOME"
# run built-in setup command
AHRIMAN_SETUP_ARGS=("--build-as-user" "$AHRIMAN_USER")
AHRIMAN_SETUP_ARGS+=("--packager" "$AHRIMAN_PACKAGER")
AHRIMAN_SETUP_ARGS+=("--repository" "$AHRIMAN_REPOSITORY")
if [ -z "$AHRIMAN_MULTILIB" ]; then
AHRIMAN_SETUP_ARGS+=("--no-multilib")
fi
if [ -n "$AHRIMAN_PACMAN_MIRROR" ]; then
AHRIMAN_SETUP_ARGS+=("--mirror" "$AHRIMAN_PACMAN_MIRROR")
fi
if [ -n "$AHRIMAN_PORT" ]; then
AHRIMAN_SETUP_ARGS+=("--web-port" "$AHRIMAN_PORT")
fi
if [ -n "$AHRIMAN_REPOSITORY_SERVER" ]; then
AHRIMAN_SETUP_ARGS+=("--server" "$AHRIMAN_REPOSITORY_SERVER")
fi
if [ -n "$AHRIMAN_PORT" ]; then
AHRIMAN_SETUP_ARGS+=("--web-port" "$AHRIMAN_PORT")
fi
if [ -n "$AHRIMAN_UNIX_SOCKET" ]; then
AHRIMAN_SETUP_ARGS+=("--web-unix-socket" "$AHRIMAN_UNIX_SOCKET")
fi

21
docs/configuration.rst
View File

@ -1,7 +1,12 @@
Configuration
=============
Some groups can be specified for each architecture separately. E.g. if there are ``build`` and ``build:x86_64`` groups it will use an option from ``build:x86_64`` for the ``x86_64`` architecture and ``build`` for any other (architecture specific group has higher priority). In case if both groups are presented, architecture specific options will be merged into global ones overriding them.
Some groups can be specified for each architecture and/or repository separately. E.g. if there are ``build`` and ``build:x86_64`` groups it will use an option from ``build:x86_64`` for the ``x86_64`` architecture and ``build`` for any other (architecture specific group has higher priority). In case if both groups are presented, architecture specific options will be merged into global ones overriding them. The order which will be used for option resolution is the following:
1. Repository and architecture specific, e.g. ``build:aur-clone:x86_64``.
2. Repository specific, e.g. ``build:aur-clone``.
2. Architecture specific, e.g. ``build:x86_64``.
2. Default section, e.g. ``build``.
There are two variable types which have been added to default ones, they are paths and lists. List values will be read in the same way as shell does:
@ -87,7 +92,6 @@ Build related configuration. Group name can refer to architecture, e.g. ``build:
Base repository settings.
* ``name`` - repository name, string, required.
* ``root`` - root path for application, string, required.
``sign:*`` groups
@ -292,20 +296,21 @@ Type will be read from several sources:
``github`` type
^^^^^^^^^^^^^^^
This feature requires Github key creation (see below). Section name must be either ``github`` (plus optional architecture name, e.g. ``github:x86_64``) or random name with ``type`` set.
This feature requires GitHub key creation (see below). Section name must be either ``github`` (plus optional architecture name, e.g. ``github:x86_64``) or random name with ``type`` set.
* ``type`` - type of the upload, string, optional, must be set to ``github`` if exists.
* ``owner`` - Github repository owner, string, required.
* ``password`` - created Github API key. In order to create it do the following:
* ``owner`` - GitHub repository owner, string, required.
* ``password`` - created GitHub API key. In order to create it do the following:
#. Go to `settings page <https://github.com/settings/profile>`_.
#. Switch to `developers settings <https://github.com/settings/apps>`_.
#. Switch to `personal access tokens <https://github.com/settings/tokens>`_.
#. Generate new token. Required scope is ``public_repo`` (or ``repo`` for private repository support).
* ``repository`` - Github repository name, string, required. Repository must be created before any action and must have active branch (e.g. with readme).
* ``repository`` - GitHub repository name, string, required. Repository must be created before any action and must have active branch (e.g. with readme).
* ``timeout`` - HTTP request timeout in seconds, int, optional, default is ``30``.
* ``username`` - Github authorization user, string, required. Basically the same as ``owner``.
* ``use_full_release_name`` - if set to ``yes``, the release will contain both repository name and architecture, and only architecture otherwise, boolean, optional, default ``no``.
* ``username`` - GitHub authorization user, string, required. Basically the same as ``owner``.
``remote-service`` type
^^^^^^^^^^^^^^^^^^^^^^^
@ -329,7 +334,7 @@ Requires ``rsync`` package to be installed. Do not forget to configure ssh for u
Requires ``boto3`` library to be installed. Section name must be either ``s3`` (plus optional architecture name, e.g. ``s3:x86_64``) or random name with ``type`` set.
* ``type`` - type of the upload, string, optional, must be set to ``github`` if exists.
* ``type`` - type of the upload, string, optional, must be set to ``s3`` if exists.
* ``access_key`` - AWS access key ID, string, required.
* ``bucket`` - bucket name (e.g. ``bucket``), string, required.
* ``chunk_size`` - chunk size for calculating entity tags, int, optional, default 8 * 1024 * 1024.

20
docs/faq.rst
View File

@ -17,7 +17,7 @@ TL;DR
.. code-block:: shell
yay -S ahriman
ahriman -a x86_64 service-setup --packager "ahriman bot <ahriman@example.com>" --repository "repository"
ahriman -a x86_64 -r aur-clone service-setup --packager "ahriman bot <ahriman@example.com>"
systemctl enable --now ahriman@x86_64.timer
Long answer
@ -32,7 +32,7 @@ There is special command which can be used in order to validate current configur
.. code-block:: shell
ahriman -a x86_64 service-config-validate --exit-code
ahriman -a x86_64 -r aur-clone service-config-validate --exit-code
This command will print found errors, based on `cerberus <https://docs.python-cerberus.org/>`_, e.g.:
@ -317,7 +317,7 @@ Add the following lines to your ``pacman.conf``:
.. code-block:: ini
[repository]
Server = file:///var/lib/ahriman/repository/x86_64
Server = file:///var/lib/ahriman/repository/$repo/$arch
(You might need to add ``SigLevel`` option according to the pacman documentation.)
@ -554,8 +554,8 @@ There are several choices:
.. code-block::
server {
location /x86_64 {
root /var/lib/ahriman/repository/x86_64;
location /aur-clone/x86_64 {
root /var/lib/ahriman/repository/aur-clone/x86_64;
autoindex on;
}
}
@ -571,7 +571,7 @@ There are several choices:
[rsync]
remote = 192.168.0.1:/srv/repo
After that just add ``/srv/repo`` to the ``pacman.conf`` as usual. You can also upload to S3 (e.g. ``Server = https://s3.eu-central-1.amazonaws.com/repository/x86_64``) or to Github (e.g. ``Server = https://github.com/ahriman/repository/releases/download/x86_64``).
After that just add ``/srv/repo`` to the ``pacman.conf`` as usual. You can also upload to S3 (e.g. ``Server = https://s3.eu-central-1.amazonaws.com/repository/aur-clone/x86_64``) or to Github (e.g. ``Server = https://github.com/ahriman/repository/releases/download/aur-clone-x86_64``).
How to sync to S3
^^^^^^^^^^^^^^^^^
@ -676,7 +676,7 @@ How to report by email
[email]
host = smtp.example.com
link_path = http://example.com/x86_64
link_path = http://example.com/aur-clone/x86_64
password = ...
port = 465
receivers = me@example.com
@ -702,8 +702,8 @@ How to generate index page for S3
target = html
[html]
path = /var/lib/ahriman/repository/x86_64/index.html
link_path = http://example.com/x86_64
path = /var/lib/ahriman/repository/aur-clone/x86_64/index.html
link_path = http://example.com/aur-clone/x86_64
After these steps ``index.html`` file will be automatically synced to S3
@ -741,7 +741,7 @@ How to post build report to telegram
[telegram]
api_key = aaAAbbBBccCC
chat_id = @ahriman
link_path = http://example.com/x86_64
link_path = http://example.com/aur-clone/x86_64
``api_key`` is the one sent by `@BotFather <https://t.me/botfather>`_, ``chat_id`` is the value retrieved from previous step.

22
docs/setup.rst
View File

@ -10,7 +10,7 @@ Initial setup
.. code-block:: shell
sudo ahriman -a x86_64 service-setup ...
sudo ahriman -a x86_64 -r aur-clone service-setup ...
``service-setup`` literally does the following steps:
@ -29,26 +29,26 @@ Initial setup
.. code-block:: shell
ln -s /usr/bin/archbuild /usr/local/bin/ahriman-x86_64-build
ln -s /usr/bin/archbuild /usr/local/bin/aur-clone-x86_64-build
#.
Create configuration file (same as previous ``{name}.conf``):
.. code-block:: shell
cp /usr/share/devtools/pacman.conf.d/{extra,ahriman}.conf
cp /usr/share/devtools/pacman.conf.d/{extra,aur-clone}.conf
#.
Change configuration file, add your own repository, add multilib repository etc:
.. code-block:: shell
echo '[multilib]' | tee -a /usr/share/devtools/pacman-ahriman.conf
echo 'Include = /etc/pacman.d/mirrorlist' | tee -a /usr/share/devtools/pacman.conf.d/ahriman.conf
echo '[multilib]' | tee -a /usr/share/devtools/pacman.conf.d/aur-clone-x86_64.conf
echo 'Include = /etc/pacman.d/mirrorlist' | tee -a /usr/share/devtools/pacman.conf.d/aur-clone-x86_64.conf
echo '[aur-clone]' | tee -a /usr/share/devtools/pacman-ahriman.conf
echo 'SigLevel = Optional TrustAll' | tee -a /usr/share/devtools/pacman.conf.d/ahriman.conf
echo 'Server = file:///var/lib/ahriman/repository/$arch' | tee -a /usr/share/devtools/pacman.conf.d/ahriman.conf
echo '[aur-clone]' | tee -a /usr/share/devtools/pacman.conf.d/aur-clone-x86_64.conf
echo 'SigLevel = Optional TrustAll' | tee -a /usr/share/devtools/pacman.conf.d/aur-clone-x86_64.conf
echo 'Server = file:///var/lib/ahriman/repository/$repo/$arch' | tee -a /usr/share/devtools/pacman.conf.d/aur-clone-x86_64.conf
#.
Set ``build_command`` option to point to your command:
@ -56,14 +56,14 @@ Initial setup
.. code-block:: shell
echo '[build]' | tee -a /etc/ahriman.ini.d/build.ini
echo 'build_command = ahriman-x86_64-build' | tee -a /etc/ahriman.ini.d/build.ini
echo 'build_command = aur-clone-x86_64-build' | tee -a /etc/ahriman.ini.d/build.ini
#.
Configure ``/etc/sudoers.d/ahriman`` to allow running command without a password:
.. code-block:: shell
echo 'Cmnd_Alias CARCHBUILD_CMD = /usr/local/bin/ahriman-x86_64-build *' | tee -a /etc/sudoers.d/ahriman
echo 'Cmnd_Alias CARCHBUILD_CMD = /usr/local/bin/aur-clone-x86_64-build *' | tee -a /etc/sudoers.d/ahriman
echo 'ahriman ALL=(ALL) NOPASSWD:SETENV: CARCHBUILD_CMD' | tee -a /etc/sudoers.d/ahriman
chmod 400 /etc/sudoers.d/ahriman
@ -88,6 +88,6 @@ Initial setup
.. code-block:: shell
sudo -u ahriman ahriman -a x86_64 package-add ahriman --now --refresh
sudo -u ahriman ahriman package-add ahriman --now --refresh
The ``--refresh`` flag is required in order to handle local database update.

2
package/share/ahriman/settings/ahriman.ini
View File

@ -21,7 +21,6 @@ allow_read_only = yes
[build]
archbuild_flags =
build_command = extra-x86_64-build
ignore_packages =
makechrootpkg_flags =
makepkg_flags = --nocolor --ignorearch
@ -30,7 +29,6 @@ triggers_known = ahriman.core.gitremote.RemotePullTrigger ahriman.core.gitremote
vcs_allowed_age = 604800
[repository]
name = aur-clone
root = /var/lib/ahriman
[sign]

8
src/ahriman/application/ahriman.py
View File

@ -81,6 +81,8 @@ def _parser() -> argparse.ArgumentParser:
type=LogHandler, choices=enum_values(LogHandler))
parser.add_argument("--report", help="force enable or disable reporting to web service",
action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("-r", "--repository", help="target repository. For several subcommands it can be used "
"multiple times", action="append")
parser.add_argument("-q", "--quiet", help="force disable any logging", action="store_true")
parser.add_argument("--unsafe", help="allow to run ahriman as non-ahriman user. Some actions might be unavailable",
action="store_true")
@ -883,7 +885,6 @@ def _set_service_setup_parser(root: SubParserAction) -> argparse.ArgumentParser:
epilog="Create _minimal_ configuration for the service according to provided options.",
formatter_class=_formatter)
parser.add_argument("--build-as-user", help="force makepkg user to the specific one")
parser.add_argument("--build-command", help="build command prefix", default="ahriman")
parser.add_argument("--from-configuration", help="path to default devtools pacman configuration",
type=Path, default=Path("/usr") / "share" / "devtools" / "pacman.conf.d" / "extra.conf")
parser.add_argument("--generate-salt", help="generate salt for user passwords",
@ -894,7 +895,6 @@ def _set_service_setup_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser.add_argument("--multilib", help="add or do not multilib repository",
action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("--packager", help="packager name and email", required=True)
parser.add_argument("--repository", help="repository name", required=True)
parser.add_argument("--server", help="server to be used for devtools. If none set, local files will be used")
parser.add_argument("--sign-key", help="sign key id")
parser.add_argument("--sign-target", help="sign options", action="append",
@ -943,7 +943,7 @@ def _set_user_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
"`Name Surname <mail@example.com>`")
parser.add_argument("-p", "--password", help="user password. Blank password will be treated as empty password, "
"which is in particular must be used for OAuth2 authorization type.")
parser.add_argument("-r", "--role", help="user access level",
parser.add_argument("-R", "--role", help="user access level",
type=UserAccess, choices=enum_values(UserAccess), default=UserAccess.Read)
parser.set_defaults(handler=handlers.Users, action=Action.Update, architecture=[""], lock=None, report=False,
quiet=True)
@ -965,7 +965,7 @@ def _set_user_list_parser(root: SubParserAction) -> argparse.ArgumentParser:
formatter_class=_formatter)
parser.add_argument("username", help="filter users by username", nargs="?")
parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
parser.add_argument("-r", "--role", help="filter users by role", type=UserAccess, choices=enum_values(UserAccess))
parser.add_argument("-R", "--role", help="filter users by role", type=UserAccess, choices=enum_values(UserAccess))
parser.set_defaults(handler=handlers.Users, action=Action.List, architecture=[""], lock=None, report=False,
quiet=True, unsafe=True)
return parser

3
src/ahriman/application/application/application.py
View File

@ -37,9 +37,10 @@ class Application(ApplicationPackages, ApplicationRepository):
>>> from ahriman.core.configuration import Configuration
>>> from ahriman.models.package_source import PackageSource
>>> from ahriman.models.repository_id import RepositoryId
>>>
>>> configuration = Configuration()
>>> application = Application("x86_64", configuration, report=True)
>>> application = Application(RepositoryId("x86_64", "x86_64"), configuration, report=True)
>>> # add packages to build queue
>>> application.add(["ahriman"], PackageSource.AUR)
>>>

21
src/ahriman/application/application/application_properties.py
View File

@ -22,6 +22,7 @@ from ahriman.core.database import SQLite
from ahriman.core.log import LazyLogging
from ahriman.core.repository import Repository
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.repository_id import RepositoryId
class ApplicationProperties(LazyLogging):
@ -29,26 +30,36 @@ class ApplicationProperties(LazyLogging):
application base properties class
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
database(SQLite): database instance
repository(Repository): repository instance
repository_id(RepositoryId): repository unique identifier
"""
def __init__(self, architecture: str, configuration: Configuration, *, report: bool,
def __init__(self, repository_id: RepositoryId, configuration: Configuration, *, report: bool,
refresh_pacman_database: PacmanSynchronization = PacmanSynchronization.Disabled) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
refresh_pacman_database(PacmanSynchronization, optional): pacman database synchronization level
(Default value = PacmanSynchronization.Disabled)
"""
self.configuration = configuration
self.architecture = architecture
self.repository_id = repository_id
self.database = SQLite.load(configuration)
self.repository = Repository.load(architecture, configuration, self.database, report=report,
self.repository = Repository.load(repository_id, configuration, self.database, report=report,
refresh_pacman_database=refresh_pacman_database)
@property
def architecture(self) -> str:
"""
repository architecture for backward compatibility
Returns:
str: repository architecture
"""
return self.repository_id.architecture

8
src/ahriman/application/handlers/add.py
View File

@ -23,6 +23,7 @@ from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.packagers import Packagers
from ahriman.models.repository_id import RepositoryId
class Add(Handler):
@ -31,17 +32,18 @@ class Add(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report, refresh_pacman_database=args.refresh)
application = Application(repository_id, configuration, report=report, refresh_pacman_database=args.refresh)
application.on_start()
application.add(args.package, args.source, args.username)
if not args.now:

6
src/ahriman/application/handlers/backup.py
View File

@ -26,6 +26,7 @@ from tarfile import TarFile
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.database import SQLite
from ahriman.models.repository_id import RepositoryId
class Backup(Handler):
@ -36,13 +37,14 @@ class Backup(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

8
src/ahriman/application/handlers/clean.py
View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class Clean(Handler):
@ -30,17 +31,18 @@ class Clean(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
application.on_start()
application.clean(cache=args.cache, chroot=args.chroot, manual=args.manual, packages=args.packages,
pacman=args.pacman)

10
src/ahriman/application/handlers/daemon.py
View File

@ -22,6 +22,7 @@ import threading
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class Daemon(Handler):
@ -30,19 +31,20 @@ class Daemon(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
from ahriman.application.handlers import Update
Update.run(args, architecture, configuration, report=report)
timer = threading.Timer(args.interval, Daemon.run, args=[args, architecture, configuration],
Update.run(args, repository_id, configuration, report=report)
timer = threading.Timer(args.interval, Daemon.run, args=[args, repository_id, configuration],
kwargs={"report": report})
timer.start()
timer.join()

6
src/ahriman/application/handlers/dump.py
View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import ConfigurationPathsPrinter, ConfigurationPrinter, StringPrinter
from ahriman.models.repository_id import RepositoryId
class Dump(Handler):
@ -32,13 +33,14 @@ class Dump(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

104
src/ahriman/application/handlers/handler.py
View File

@ -25,7 +25,8 @@ from multiprocessing import Pool
from ahriman.application.lock import Lock
from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import ExitCode, MissingArchitectureError, MultipleArchitecturesError
from ahriman.core.log import Log
from ahriman.core.log.log_loader import LogLoader
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
@ -50,56 +51,25 @@ class Handler:
ALLOW_MULTI_ARCHITECTURE_RUN = True
@classmethod
def architectures_extract(cls, args: argparse.Namespace) -> list[str]:
"""
get known architectures
Args:
args(argparse.Namespace): command line args
Returns:
list[str]: list of architectures for which tree is created
Raises:
MissingArchitectureError: if no architecture set and automatic detection is not allowed or failed
"""
if not cls.ALLOW_AUTO_ARCHITECTURE_RUN and args.architecture is None:
# for some parsers (e.g. config) we need to run with specific architecture
# for those cases architecture must be set explicitly
raise MissingArchitectureError(args.command)
if args.architecture: # architecture is specified explicitly
return sorted(set(args.architecture))
configuration = Configuration()
configuration.load(args.configuration)
# wtf???
root = configuration.getpath("repository", "root") # pylint: disable=assignment-from-no-return
architectures = RepositoryPaths.known_architectures(root)
if not architectures: # well we did not find anything
raise MissingArchitectureError(args.command)
return sorted(architectures)
@classmethod
def call(cls, args: argparse.Namespace, architecture: str) -> bool:
def call(cls, args: argparse.Namespace, repository_id: RepositoryId) -> bool:
"""
additional function to wrap all calls for multiprocessing library
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
Returns:
bool: True on success, False otherwise
"""
try:
configuration = Configuration.from_path(args.configuration, architecture)
configuration = Configuration.from_path(args.configuration, repository_id)
log_handler = Log.handler(args.log_handler)
Log.load(configuration, log_handler, quiet=args.quiet, report=args.report)
log_handler = LogLoader.handler(args.log_handler)
LogLoader.load(configuration, log_handler, quiet=args.quiet, report=args.report)
with Lock(args, architecture, configuration):
cls.run(args, architecture, configuration, report=args.report)
with Lock(args, repository_id, configuration):
cls.run(args, repository_id, configuration, report=args.report)
return True
except ExitCode:
@ -123,28 +93,70 @@ class Handler:
Raises:
MultipleArchitecturesError: if more than one architecture supplied and no multi architecture supported
"""
architectures = cls.architectures_extract(args)
repositories = cls.repositories_extract(args)
# actually we do not have to spawn another process if it is single-process application, do we?
if len(architectures) > 1:
if len(repositories) > 1:
if not cls.ALLOW_MULTI_ARCHITECTURE_RUN:
raise MultipleArchitecturesError(args.command)
with Pool(len(architectures)) as pool:
result = pool.starmap(cls.call, [(args, architecture) for architecture in architectures])
with Pool(len(repositories)) as pool:
result = pool.starmap(cls.call, [(args, repository_id) for repository_id in repositories])
else:
result = [cls.call(args, architectures.pop())]
result = [cls.call(args, repositories.pop())]
return 0 if all(result) else 1
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def repositories_extract(cls, args: argparse.Namespace) -> list[RepositoryId]:
"""
get known architectures
Args:
args(argparse.Namespace): command line args
Returns:
tuple[str | None, str]: list of repository names and architectures for which tree is created
Raises:
MissingArchitectureError: if no architecture set and automatic detection is not allowed or failed
"""
if not cls.ALLOW_AUTO_ARCHITECTURE_RUN and args.architecture is None:
# for some parsers (e.g. config) we need to run with specific architecture
# for those cases architecture must be set explicitly
raise MissingArchitectureError(args.command)
configuration = Configuration()
configuration.load(args.configuration)
name = configuration.get("repository", "name", fallback="") # will only be used for legacy mode
if args.architecture: # architecture is specified explicitly
repositories = args.repository or [name] # fallback for legacy mode
return sorted(
set(
RepositoryId(architecture, repository)
for architecture in args.architecture
for repository in repositories
)
)
# wtf???
root = configuration.getpath("repository", "root") # pylint: disable=assignment-from-no-return
architectures = RepositoryPaths.known_architectures(root, name)
if not architectures: # well we did not find anything
raise MissingArchitectureError(args.command)
return sorted(architectures)
@classmethod
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting

6
src/ahriman/application/handlers/help.py
View File

@ -21,6 +21,7 @@ import argparse
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class Help(Handler):
@ -31,13 +32,14 @@ class Help(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

8
src/ahriman/application/handlers/key_import.py
View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class KeyImport(Handler):
@ -32,15 +33,16 @@ class KeyImport(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
application.repository.sign.key_import(args.key_server, args.key)

10
src/ahriman/application/handlers/patch.py
View File

@ -30,6 +30,7 @@ from ahriman.core.formatters import PatchPrinter
from ahriman.models.action import Action
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_id import RepositoryId
class Patch(Handler):
@ -38,17 +39,18 @@ class Patch(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
application.on_start()
match args.action:
@ -56,7 +58,7 @@ class Patch(Handler):
patch = Patch.patch_create_from_function(args.variable, args.patch)
Patch.patch_set_create(application, args.package, patch)
case Action.Update:
package_base, patch = Patch.patch_create_from_diff(args.package, architecture, args.track)
package_base, patch = Patch.patch_create_from_diff(args.package, repository_id.architecture, args.track)
Patch.patch_set_create(application, package_base, patch)
case Action.List:
Patch.patch_set_list(application, args.package, args.variable, args.exit_code)

8
src/ahriman/application/handlers/rebuild.py
View File

@ -24,6 +24,7 @@ from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class Rebuild(Handler):
@ -32,17 +33,18 @@ class Rebuild(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
application.on_start()
packages = Rebuild.extract_packages(application, args.status, from_database=args.from_database)

8
src/ahriman/application/handlers/remove.py
View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class Remove(Handler):
@ -30,16 +31,17 @@ class Remove(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
application.on_start()
application.remove(args.package)

8
src/ahriman/application/handlers/remove_unknown.py
View File

@ -23,6 +23,7 @@ from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import StringPrinter
from ahriman.models.repository_id import RepositoryId
class RemoveUnknown(Handler):
@ -31,17 +32,18 @@ class RemoveUnknown(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
application.on_start()
unknown_packages = application.unknown()

6
src/ahriman/application/handlers/restore.py
View File

@ -23,6 +23,7 @@ from tarfile import TarFile
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class Restore(Handler):
@ -33,13 +34,14 @@ class Restore(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

8
src/ahriman/application/handlers/search.py
View File

@ -29,6 +29,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import OptionError
from ahriman.core.formatters import AurPrinter
from ahriman.models.aur_package import AURPackage
from ahriman.models.repository_id import RepositoryId
class Search(Handler):
@ -47,17 +48,18 @@ class Search(Handler):
}
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
official_packages_list = Official.multisearch(*args.search, pacman=application.repository.pacman)
aur_packages_list = AUR.multisearch(*args.search, pacman=application.repository.pacman)

8
src/ahriman/application/handlers/service_updates.py
View File

@ -25,6 +25,7 @@ from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import UpdatePrinter
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class ServiceUpdates(Handler):
@ -35,17 +36,18 @@ class ServiceUpdates(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
remote = Package.from_aur("ahriman", application.repository.pacman, None)
_, release = remote.version.rsplit("-", 1) # we don't store pkgrel locally, so we just append it

74
src/ahriman/application/handlers/setup.py
View File

@ -25,6 +25,7 @@ from pwd import getpwuid
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
from ahriman.models.user import User
@ -46,81 +47,80 @@ class Setup(Handler):
SUDOERS_DIR_PATH = Path("/etc") / "sudoers.d"
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
Setup.configuration_create_ahriman(args, architecture, args.repository, configuration)
Setup.configuration_create_ahriman(args, repository_id, configuration)
configuration.reload()
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
Setup.configuration_create_makepkg(args.packager, args.makeflags_jobs, application.repository.paths)
Setup.executable_create(application.repository.paths, args.build_command, architecture)
Setup.executable_create(application.repository.paths, repository_id)
repository_server = f"file://{application.repository.paths.repository}" if args.server is None else args.server
Setup.configuration_create_devtools(args.build_command, architecture, args.from_configuration, args.mirror,
args.multilib, args.repository, repository_server)
Setup.configuration_create_sudo(application.repository.paths, args.build_command, architecture)
Setup.configuration_create_devtools(repository_id, args.from_configuration, args.mirror, args.multilib,
repository_server)
Setup.configuration_create_sudo(application.repository.paths, repository_id)
application.repository.repo.init()
# lazy database sync
application.repository.pacman.handle # pylint: disable=pointless-statement
@staticmethod
def build_command(root: Path, prefix: str, architecture: str) -> Path:
def build_command(root: Path, repository_id: RepositoryId) -> Path:
"""
generate build command name
Args:
root(Path): root directory for the build command (must be root of the repository)
prefix(str): command prefix in {prefix}-{architecture}-build
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
Returns:
Path: valid devtools command name
"""
return root / f"{prefix}-{architecture}-build"
return root / f"{repository_id.name}-{repository_id.architecture}-build"
@staticmethod
def configuration_create_ahriman(args: argparse.Namespace, architecture: str, repository: str,
def configuration_create_ahriman(args: argparse.Namespace, repository_id: RepositoryId,
root: Configuration) -> None:
"""
create service specific configuration
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository(str): repository name
repository_id(RepositoryId): repository unique identifier
root(Configuration): root configuration instance
"""
configuration = Configuration()
section = Configuration.section_name("build", architecture)
build_command = Setup.build_command(root.repository_paths.root, args.build_command, architecture)
section = Configuration.section_name("build", repository_id.name, repository_id.architecture)
build_command = Setup.build_command(root.repository_paths.root, repository_id)
configuration.set_option(section, "build_command", str(build_command))
configuration.set_option("repository", "name", repository)
configuration.set_option("repository", "name", repository_id.name) # backward compatibility for docker
if args.build_as_user is not None:
configuration.set_option(section, "makechrootpkg_flags", f"-U {args.build_as_user}")
section = Configuration.section_name("alpm", architecture)
section = Configuration.section_name("alpm", repository_id.name, repository_id.architecture)
if args.mirror is not None:
configuration.set_option(section, "mirror", args.mirror)
if not args.multilib:
repositories = filter(lambda r: r != "multilib", root.getlist("alpm", "repositories"))
configuration.set_option(section, "repositories", " ".join(repositories))
section = Configuration.section_name("sign", architecture)
section = Configuration.section_name("sign", repository_id.name, repository_id.architecture)
if args.sign_key is not None:
configuration.set_option(section, "target", " ".join([target.name.lower() for target in args.sign_target]))
configuration.set_option(section, "key", args.sign_key)
section = Configuration.section_name("web", architecture)
section = Configuration.section_name("web", repository_id.name, repository_id.architecture)
if args.web_port is not None:
configuration.set_option(section, "port", str(args.web_port))
if args.web_unix_socket is not None:
@ -134,8 +134,8 @@ class Setup(Handler):
configuration.write(ahriman_configuration)
@staticmethod
def configuration_create_devtools(prefix: str, architecture: str, source: Path, mirror: str | None,
multilib: bool, repository: str, repository_server: str) -> None:
def configuration_create_devtools(repository_id: RepositoryId, source: Path, mirror: str | None,
multilib: bool, repository_server: str) -> None:
"""
create configuration for devtools based on ``source`` configuration
@ -143,12 +143,10 @@ class Setup(Handler):
devtools does not allow to specify the pacman configuration, thus we still have to use configuration in /usr
Args:
prefix(str): command prefix in {prefix}-{architecture}-build
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
source(Path): path to source configuration file
mirror(str | None): link to package server mirror
multilib(bool): add or do not multilib repository to the configuration
repository(str): repository name
repository_server(str): url of the repository
"""
# allow_no_value=True is required because pacman uses boolean configuration in which just keys present
@ -163,7 +161,7 @@ class Setup(Handler):
configuration.read(source)
# set our architecture now
configuration.set_option("options", "Architecture", architecture)
configuration.set_option("options", "Architecture", repository_id.architecture)
# add multilib
if multilib:
@ -178,10 +176,10 @@ class Setup(Handler):
configuration.set_option(section, "Server", mirror)
# add repository itself
configuration.set_option(repository, "SigLevel", "Never") # we don't care
configuration.set_option(repository, "Server", repository_server)
configuration.set_option(repository_id.name, "SigLevel", "Never") # we don't care
configuration.set_option(repository_id.name, "Server", repository_server)
target = source.parent / f"{prefix}-{architecture}.conf"
target = source.parent / f"{repository_id.name}-{repository_id.architecture}.conf"
with target.open("w") as devtools_configuration:
configuration.write(devtools_configuration)
@ -205,31 +203,29 @@ class Setup(Handler):
(home_dir / ".makepkg.conf").write_text(content, encoding="utf8")
@staticmethod
def configuration_create_sudo(paths: RepositoryPaths, prefix: str, architecture: str) -> None:
def configuration_create_sudo(paths: RepositoryPaths, repository_id: RepositoryId) -> None:
"""
create configuration to run build command with sudo without password
Args:
paths(RepositoryPaths): repository paths instance
prefix(str): command prefix in {prefix}-{architecture}-build
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
"""
command = Setup.build_command(paths.root, prefix, architecture)
sudoers_file = Setup.build_command(Setup.SUDOERS_DIR_PATH, prefix, architecture)
command = Setup.build_command(paths.root, repository_id)
sudoers_file = Setup.build_command(Setup.SUDOERS_DIR_PATH, repository_id)
sudoers_file.write_text(f"ahriman ALL=(ALL) NOPASSWD:SETENV: {command} *\n", encoding="utf8")
sudoers_file.chmod(0o400) # security!
@staticmethod
def executable_create(paths: RepositoryPaths, prefix: str, architecture: str) -> None:
def executable_create(paths: RepositoryPaths, repository_id: RepositoryId) -> None:
"""
create executable for the service
Args:
paths(RepositoryPaths): repository paths instance
prefix(str): command prefix in {prefix}-{architecture}-build
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
"""
command = Setup.build_command(paths.root, prefix, architecture)
command = Setup.build_command(paths.root, repository_id)
command.unlink(missing_ok=True)
command.symlink_to(Setup.ARCHBUILD_COMMAND_PATH)
paths.chown(command) # we would like to keep owner inside ahriman's home

14
src/ahriman/application/handlers/shell.py
View File

@ -26,6 +26,7 @@ from pathlib import Path
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import StringPrinter
from ahriman.models.repository_id import RepositoryId
class Shell(Handler):
@ -36,13 +37,14 @@ class Shell(Handler):
ALLOW_MULTI_ARCHITECTURE_RUN = False
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
@ -50,7 +52,13 @@ class Shell(Handler):
# licensed by https://creativecommons.org/licenses/by-sa/3.0
path = Path(sys.prefix) / "share" / "ahriman" / "templates" / "shell"
StringPrinter(path.read_text(encoding="utf8")).print(verbose=False)
local_variables = {"architecture": architecture, "configuration": configuration}
local_variables = {
"architecture": repository_id.architecture,
"configuration": configuration,
"repository_id": repository_id,
}
if args.code is None:
code.interact(local=local_variables)
else:

8
src/ahriman/application/handlers/sign.py
View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
class Sign(Handler):
@ -30,14 +31,15 @@ class Sign(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
Application(architecture, configuration, report=report).sign(args.package)
Application(repository_id, configuration, report=report).sign(args.package)

8
src/ahriman/application/handlers/status.py
View File

@ -27,6 +27,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.formatters import PackagePrinter, StatusPrinter
from ahriman.models.build_status import BuildStatus
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class Status(Handler):
@ -37,18 +38,19 @@ class Status(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
# we are using reporter here
client = Application(architecture, configuration, report=True).repository.reporter
client = Application(repository_id, configuration, report=True).repository.reporter
if args.ahriman:
service_status = client.status_get()
StatusPrinter(service_status.status).print(verbose=args.info)

8
src/ahriman/application/handlers/status_update.py
View File

@ -23,6 +23,7 @@ from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.action import Action
from ahriman.models.repository_id import RepositoryId
class StatusUpdate(Handler):
@ -33,18 +34,19 @@ class StatusUpdate(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
# we are using reporter here
client = Application(architecture, configuration, report=True).repository.reporter
client = Application(repository_id, configuration, report=True).repository.reporter
match args.action:
case Action.Update if args.package:

8
src/ahriman/application/handlers/structure.py
View File

@ -24,6 +24,7 @@ from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import StringPrinter, TreePrinter
from ahriman.core.tree import Tree
from ahriman.models.repository_id import RepositoryId
class Structure(Handler):
@ -34,17 +35,18 @@ class Structure(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
partitions = Tree.partition(application.repository.packages(), count=args.partitions)
for partition_id, partition in enumerate(partitions):

10
src/ahriman/application/handlers/triggers.py
View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -31,19 +32,20 @@ class Triggers(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report)
application = Application(repository_id, configuration, report=report)
if args.trigger:
loader = application.repository.triggers
loader.triggers = [loader.load_trigger(trigger, architecture, configuration) for trigger in args.trigger]
loader.triggers = [loader.load_trigger(trigger, repository_id, configuration) for trigger in args.trigger]
application.on_start()
application.on_result(Result())

6
src/ahriman/application/handlers/unsafe_commands.py
View File

@ -22,6 +22,7 @@ import argparse
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import StringPrinter
from ahriman.models.repository_id import RepositoryId
class UnsafeCommands(Handler):
@ -32,13 +33,14 @@ class UnsafeCommands(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

8
src/ahriman/application/handlers/update.py
View File

@ -25,6 +25,7 @@ from ahriman.application.application import Application
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.models.packagers import Packagers
from ahriman.models.repository_id import RepositoryId
class Update(Handler):
@ -33,17 +34,18 @@ class Update(Handler):
"""
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
application = Application(architecture, configuration, report=report, refresh_pacman_database=args.refresh)
application = Application(repository_id, configuration, report=report, refresh_pacman_database=args.refresh)
application.on_start()
packages = application.updates(args.package, aur=args.aur, local=args.local, manual=args.manual, vcs=args.vcs)
Update.check_if_empty(args.exit_code, not packages)

6
src/ahriman/application/handlers/users.py
View File

@ -26,6 +26,7 @@ from ahriman.core.database import SQLite
from ahriman.core.exceptions import PasswordError
from ahriman.core.formatters import UserPrinter
from ahriman.models.action import Action
from ahriman.models.repository_id import RepositoryId
from ahriman.models.user import User
@ -37,13 +38,14 @@ class Users(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture"
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

16
src/ahriman/application/handlers/validate.py
View File

@ -29,6 +29,7 @@ from ahriman.core.configuration.validator import Validator
from ahriman.core.exceptions import ExtensionError
from ahriman.core.formatters import ValidationPrinter
from ahriman.core.triggers import TriggerLoader
from ahriman.models.repository_id import RepositoryId
class Validate(Handler):
@ -39,17 +40,18 @@ class Validate(Handler):
ALLOW_AUTO_ARCHITECTURE_RUN = False
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
schema = Validate.schema(architecture, configuration)
schema = Validate.schema(repository_id, configuration)
validator = Validator(configuration=configuration, schema=schema)
if validator.validate(configuration.dump()):
@ -61,12 +63,12 @@ class Validate(Handler):
Validate.check_if_empty(args.exit_code, True)
@staticmethod
def schema(architecture: str, configuration: Configuration) -> ConfigurationSchema:
def schema(repository_id: RepositoryId, configuration: Configuration) -> ConfigurationSchema:
"""
get schema with triggers
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
Returns:
@ -85,12 +87,12 @@ class Validate(Handler):
continue
# default settings if any
for schema_name, schema in trigger_class.configuration_schema(architecture, None).items():
for schema_name, schema in trigger_class.configuration_schema(repository_id, None).items():
erased = Validate.schema_erase_required(copy.deepcopy(schema))
root[schema_name] = Validate.schema_merge(root.get(schema_name, {}), erased)
# settings according to enabled triggers
for schema_name, schema in trigger_class.configuration_schema(architecture, configuration).items():
for schema_name, schema in trigger_class.configuration_schema(repository_id, configuration).items():
root[schema_name] = Validate.schema_merge(root.get(schema_name, {}), copy.deepcopy(schema))
return root

6
src/ahriman/application/handlers/versions.py
View File

@ -28,6 +28,7 @@ from ahriman import __version__
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.formatters import VersionPrinter
from ahriman.models.repository_id import RepositoryId
class Versions(Handler):
@ -42,13 +43,14 @@ class Versions(Handler):
PEP423_PACKAGE_NAME = re.compile(r"^[A-Za-z0-9._-]+")
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""

20
src/ahriman/application/handlers/web.py
View File

@ -24,6 +24,7 @@ from collections.abc import Generator
from ahriman.application.handlers import Handler
from ahriman.core.configuration import Configuration
from ahriman.core.spawn import Spawn
from ahriman.models.repository_id import RepositoryId
class Web(Handler):
@ -35,24 +36,25 @@ class Web(Handler):
ALLOW_MULTI_ARCHITECTURE_RUN = False # required to be able to spawn external processes
@classmethod
def run(cls, args: argparse.Namespace, architecture: str, configuration: Configuration, *, report: bool) -> None:
def run(cls, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration, *,
report: bool) -> None:
"""
callback for command line
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
report(bool): force enable or disable reporting
"""
# we are using local import for optional dependencies
from ahriman.web.web import run_server, setup_service
spawner_args = Web.extract_arguments(args, architecture, configuration)
spawner = Spawn(args.parser(), architecture, list(spawner_args))
spawner_args = Web.extract_arguments(args, repository_id, configuration)
spawner = Spawn(args.parser(), repository_id, list(spawner_args))
spawner.start()
application = setup_service(architecture, configuration, spawner)
application = setup_service(repository_id, configuration, spawner)
run_server(application)
# terminate spawn process at the last
@ -60,21 +62,23 @@ class Web(Handler):
spawner.join()
@staticmethod
def extract_arguments(args: argparse.Namespace, architecture: str,
def extract_arguments(args: argparse.Namespace, repository_id: RepositoryId,
configuration: Configuration) -> Generator[str, None, None]:
"""
extract list of arguments used for current command, except for command specific ones
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
Returns:
Generator[str, None, None]: command line arguments which were used for this specific command
"""
# read architecture from the same argument list
yield from ["--architecture", architecture]
yield from ["--architecture", repository_id.architecture]
if repository_id.name is not None:
yield from ["--repository", repository_id.name]
# read configuration path from current settings
if (configuration_path := configuration.path) is not None:
yield from ["--configuration", str(configuration_path)]

12
src/ahriman/application/lock.py
View File

@ -30,6 +30,7 @@ from ahriman.core.log import LazyLogging
from ahriman.core.status.client import Client
from ahriman.core.util import check_user
from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.repository_id import RepositoryId
from ahriman.models.waiter import Waiter
@ -50,26 +51,29 @@ class Lock(LazyLogging):
The common flow is to create instance in ``with`` block and handle exceptions after all::
>>> from ahriman.core.configuration import Configuration
>>> from ahriman.models.repository_id import RepositoryId
>>>
>>> configuration = Configuration()
>>> try:
>>> with Lock(args, "x86_64", configuration):
>>> with Lock(args, RepositoryId("x86_64", "aur-clone"), configuration):
>>> perform_actions()
>>> except Exception as exception:
>>> handle_exceptions(exception)
"""
def __init__(self, args: argparse.Namespace, architecture: str, configuration: Configuration) -> None:
def __init__(self, args: argparse.Namespace, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
args(argparse.Namespace): command line args
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
lock_suffix = f"{repository_id.name}_{repository_id.architecture}" if repository_id.name is not None else repository_id.architecture
self.path: Path | None = \
args.lock.with_stem(f"{args.lock.stem}_{architecture}") if args.lock is not None else None
args.lock.with_stem(f"{args.lock.stem}_{lock_suffix}") if args.lock is not None else None
self.force: bool = args.force
self.unsafe: bool = args.unsafe
self.wait_timeout: int = args.wait_timeout

22
src/ahriman/core/alpm/pacman.py
View File

@ -23,11 +23,13 @@ from collections.abc import Callable, Generator
from functools import cached_property
from pathlib import Path
from pyalpm import DB, Handle, Package, SIG_PACKAGE, error as PyalpmError # type: ignore[import]
from string import Template
from ahriman.core.configuration import Configuration
from ahriman.core.log import LazyLogging
from ahriman.core.util import trim_package
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
@ -36,18 +38,18 @@ class Pacman(LazyLogging):
alpm wrapper
"""
def __init__(self, architecture: str, configuration: Configuration, *,
def __init__(self, repository_id: RepositoryId, configuration: Configuration, *,
refresh_database: PacmanSynchronization) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
refresh_database(PacmanSynchronization): synchronize local cache to remote
"""
self.__create_handle_fn: Callable[[], Handle] = lambda: self.__create_handle(
architecture, configuration, refresh_database=refresh_database)
repository_id, configuration, refresh_database=refresh_database)
@cached_property
def handle(self) -> Handle:
@ -59,13 +61,13 @@ class Pacman(LazyLogging):
"""
return self.__create_handle_fn()
def __create_handle(self, architecture: str, configuration: Configuration, *,
def __create_handle(self, repository_id: RepositoryId, configuration: Configuration, *,
refresh_database: PacmanSynchronization) -> Handle:
"""
create lazy handle function
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
refresh_database(PacmanSynchronization): synchronize local cache to remote
@ -81,7 +83,7 @@ class Pacman(LazyLogging):
handle = Handle(str(root), str(database_path))
for repository in configuration.getlist("alpm", "repositories"):
database = self.database_init(handle, repository, mirror, architecture)
database = self.database_init(handle, repository, mirror, repository_id.architecture)
self.database_copy(handle, database, pacman_root, paths, use_ahriman_cache=use_ahriman_cache)
if use_ahriman_cache and refresh_database:
@ -136,8 +138,14 @@ class Pacman(LazyLogging):
"""
self.logger.info("loading pacman database %s", repository)
database: DB = handle.register_syncdb(repository, SIG_PACKAGE)
# replace variables in mirror address
database.servers = [mirror.replace("$repo", repository).replace("$arch", architecture)]
variables = {
"arch": architecture,
"repo": repository,
}
database.servers = [Template(mirror).safe_substitute(variables)]
return database
def database_sync(self, handle: Handle, *, force: bool) -> None:

2
src/ahriman/core/build_tools/sources.py
View File

@ -154,7 +154,7 @@ class Sources(LazyLogging):
shutil.copytree(cache_dir, sources_dir, dirs_exist_ok=True)
instance.fetch(sources_dir, package.remote)
patches.extend(instance.extend_architectures(sources_dir, paths.architecture))
patches.extend(instance.extend_architectures(sources_dir, paths.repository_id.architecture))
for patch in patches:
instance.patch_apply(sources_dir, patch)

102
src/ahriman/core/configuration/configuration.py
View File

@ -27,6 +27,7 @@ from typing import Any, Self
from ahriman.core.configuration.shell_interpolator import ShellInterpolator
from ahriman.core.exceptions import InitializeError
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
@ -38,9 +39,9 @@ class Configuration(configparser.RawConfigParser):
ARCHITECTURE_SPECIFIC_SECTIONS(list[str]): (class attribute) known sections which can be architecture specific.
Required by dump and merging functions
SYSTEM_CONFIGURATION_PATH(Path): (class attribute) default system configuration path distributed by package
architecture(str | None): repository architecture
includes(list[Path]): list of includes which were read
path(Path | None): path to root configuration file
repository_id(RepositoryId | None): repository unique identifier
Examples:
Configuration class provides additional method in order to handle application configuration. Since this class is
@ -49,7 +50,7 @@ class Configuration(configparser.RawConfigParser):
>>> from pathlib import Path
>>>
>>> configuration = Configuration.from_path(Path("/etc/ahriman.ini"), "x86_64")
>>> configuration = Configuration.from_path(Path("/etc/ahriman.ini"), RepositoryId("x86_64", "aur-clone"))
>>> repository_name = configuration.get("repository", "name")
>>> makepkg_flags = configuration.getlist("build", "makepkg_flags")
@ -59,7 +60,7 @@ class Configuration(configparser.RawConfigParser):
In order to get current settings, the ``check_loaded`` method can be used. This method will raise an
``InitializeError`` in case if configuration was not yet loaded::
>>> path, architecture = configuration.check_loaded()
>>> path, repository_id = configuration.check_loaded()
"""
ARCHITECTURE_SPECIFIC_SECTIONS = ["alpm", "build", "sign", "web"]
@ -84,7 +85,7 @@ class Configuration(configparser.RawConfigParser):
}
)
self.architecture: str | None = None
self.repository_id: RepositoryId | None = None
self.path: Path | None = None
self.includes: list[Path] = []
@ -111,12 +112,13 @@ class Configuration(configparser.RawConfigParser):
@property
def repository_name(self) -> str:
"""
repository name as defined by configuration
repository name for backward compatibility
Returns:
str: repository name from configuration
str: repository name
"""
return self.get("repository", "name")
_, repository_id = self.check_loaded()
return repository_id.name
@property
def repository_paths(self) -> RepositoryPaths:
@ -126,39 +128,60 @@ class Configuration(configparser.RawConfigParser):
Returns:
RepositoryPaths: repository paths instance
"""
_, architecture = self.check_loaded()
return RepositoryPaths(self.getpath("repository", "root"), architecture)
_, repository_id = self.check_loaded()
return RepositoryPaths(self.getpath("repository", "root"), repository_id)
@classmethod
def from_path(cls, path: Path, architecture: str) -> Self:
def from_path(cls, path: Path, repository_id: RepositoryId) -> Self:
"""
constructor with full object initialization
Args:
path(Path): path to root configuration file
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
Returns:
Self: configuration instance
"""
configuration = cls()
configuration.load(path)
configuration.merge_sections(architecture)
configuration.merge_sections(repository_id)
return configuration
@staticmethod
def section_name(section: str, suffix: str) -> str:
def override_sections(section: str, repository_id: RepositoryId) -> list[str]:
"""
extract override sections
Args:
section(str): section name
repository_id(RepositoryId): repository unique identifier
Returns:
list[str]: architecture and repository specific sections in correct order
"""
# the valid order is global < per architecture < per repository < per repository and architecture
return [
Configuration.section_name(section, repository_id.architecture), # architecture specific override
Configuration.section_name(section, repository_id.name),
Configuration.section_name(section, repository_id.name, repository_id.architecture),
]
@staticmethod
def section_name(section: str, *suffixes: str) -> str:
"""
generate section name for sections which depends on context
Args:
section(str): section name
suffix(str): session suffix, e.g. repository architecture
*suffixes(str): session suffix, e.g. repository architecture
Returns:
str: correct section name for repository specific section
"""
return f"{section}:{suffix}"
for suffix in suffixes:
section = f"{section}:{suffix}"
return section
def _convert_path(self, value: str) -> Path:
"""
@ -175,19 +198,19 @@ class Configuration(configparser.RawConfigParser):
return path
return self.path.parent / path
def check_loaded(self) -> tuple[Path, str]:
def check_loaded(self) -> tuple[Path, RepositoryId]:
"""
check if service was actually loaded
Returns:
tuple[Path, str]: configuration root path and architecture if loaded
tuple[Path, RepositoryId]: configuration root path and architecture if loaded
Raises:
InitializeError: in case if architecture and/or path are not set
"""
if self.path is None or self.architecture is None:
raise InitializeError("Configuration path and/or architecture are not set")
return self.path, self.architecture
if self.path is None or self.repository_id is None:
raise InitializeError("Configuration path and/or repository id are not set")
return self.path, self.repository_id
def dump(self) -> dict[str, dict[str, str]]:
"""
@ -207,14 +230,14 @@ class Configuration(configparser.RawConfigParser):
def getpath(self, *args: Any, **kwargs: Any) -> Path: ... # type: ignore[empty-body]
def gettype(self, section: str, architecture: str, *, fallback: str | None = None) -> tuple[str, str]:
def gettype(self, section: str, repository_id: RepositoryId, *, fallback: str | None = None) -> tuple[str, str]:
"""
get type variable with fallback to old logic. Despite the fact that it has same semantics as other get* methods,
but it has different argument list
Args:
section(str): section name
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
fallback(str | None, optional): optional fallback type if any. If set, second element of the tuple will
be always set to this value (Default value = None)
@ -227,9 +250,9 @@ class Configuration(configparser.RawConfigParser):
if (group_type := self.get(section, "type", fallback=fallback)) is not None:
return section, group_type # new-style logic
# okay lets check for the section with architecture name
full_section = self.section_name(section, architecture)
if self.has_section(full_section):
return full_section, section
for specific in self.override_sections(section, repository_id):
if self.has_section(specific):
return specific, section
# okay lets just use section as type
if self.has_section(section):
return section, section
@ -262,23 +285,24 @@ class Configuration(configparser.RawConfigParser):
except (FileNotFoundError, configparser.NoOptionError, configparser.NoSectionError):
pass
def merge_sections(self, architecture: str) -> None:
def merge_sections(self, repository_id: RepositoryId) -> None:
"""
merge architecture specific sections into main configuration
merge architecture and repository specific sections into main configuration
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
"""
self.architecture = architecture
self.repository_id = repository_id
for section in self.ARCHITECTURE_SPECIFIC_SECTIONS:
# get overrides
specific = self.section_name(section, architecture)
if self.has_section(specific):
# if there is no such section it means that there is no overrides for this arch,
# but we anyway will have to delete sections for others architectures
for key, value in self[specific].items():
self.set_option(section, key, value)
# remove any arch specific section
for specific in self.override_sections(section, repository_id):
if self.has_section(specific):
# if there is no such section it means that there is no overrides for this arch,
# but we anyway will have to delete sections for others architectures
for key, value in self[specific].items():
self.set_option(section, key, value)
# remove any arch/repo specific section
for foreign in self.sections():
# we would like to use lambda filter here, but pylint is too dumb
if not foreign.startswith(f"{section}:"):
@ -289,11 +313,11 @@ class Configuration(configparser.RawConfigParser):
"""
reload configuration if possible or raise exception otherwise
"""
path, architecture = self.check_loaded()
path, repository_id = self.check_loaded()
for section in self.sections(): # clear current content
self.remove_section(section)
self.load(path)
self.merge_sections(architecture)
self.merge_sections(repository_id)
def set_option(self, section: str, option: str, value: str) -> None:
"""

1
src/ahriman/core/configuration/schema.py
View File

@ -187,7 +187,6 @@ CONFIGURATION_SCHEMA: ConfigurationSchema = {
"schema": {
"name": {
"type": "string",
"required": True,
},
"root": {
"type": "string",

16
src/ahriman/core/database/migrations/m001_package_source.py
View File

@ -71,7 +71,18 @@ def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> N
paths(RepositoryPaths): repository paths instance
"""
from ahriman.core.alpm.remote import AUR
from ahriman.core.database.operations import PackageOperations
from ahriman.models.package import Package
def get_packages() -> dict[str, Package]:
return {
row["package_base"]: Package(
base=row["package_base"],
version=row["version"],
remote=RemoteSource.from_json(row),
packages={},
packager=row["packager"] or None,
) for row in connection.execute("""select * from package_bases""")
}
def insert_remote(base: str, remote: RemoteSource) -> None:
connection.execute(
@ -88,8 +99,7 @@ def migrate_package_remotes(connection: Connection, paths: RepositoryPaths) -> N
}
)
packages = PackageOperations._packages_get_select_package_bases(connection)
for package_base, package in packages.items():
for package_base, package in get_packages().items():
local_cache = paths.cache_for(package_base)
if local_cache.exists() and not package.is_vcs:
continue # skip packages which are not VCS and with local cache

4
src/ahriman/core/database/migrations/m005_make_opt_depends.py
View File

@ -61,8 +61,8 @@ def migrate_package_depends(connection: Connection, configuration: Configuration
if not configuration.repository_paths.repository.is_dir():
return
_, architecture = configuration.check_loaded()
pacman = Pacman(architecture, configuration, refresh_database=PacmanSynchronization.Disabled)
_, repository_id = configuration.check_loaded()
pacman = Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled)
package_list = []
for full_path in filter(package_like, configuration.repository_paths.repository.iterdir()):

4
src/ahriman/core/database/migrations/m006_packages_architecture_required.py
View File

@ -45,9 +45,9 @@ steps = [
)
""",
"""
insert into packages select * from packages_ where architecture is not null;
insert into packages select * from packages_ where architecture is not null
""",
"""
drop table packages_;
drop table packages_
""",
]

4
src/ahriman/core/database/migrations/m007_check_depends.py
View File

@ -58,8 +58,8 @@ def migrate_package_check_depends(connection: Connection, configuration: Configu
if not configuration.repository_paths.repository.is_dir():
return
_, architecture = configuration.check_loaded()
pacman = Pacman(architecture, configuration, refresh_database=PacmanSynchronization.Disabled)
_, repository_id = configuration.check_loaded()
pacman = Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled)
package_list = []
for full_path in filter(package_like, configuration.repository_paths.repository.iterdir()):

4
src/ahriman/core/database/migrations/m008_packagers.py
View File

@ -64,8 +64,8 @@ def migrate_package_base_packager(connection: Connection, configuration: Configu
if not configuration.repository_paths.repository.is_dir():
return
_, architecture = configuration.check_loaded()
pacman = Pacman(architecture, configuration, refresh_database=PacmanSynchronization.Disabled)
_, repository_id = configuration.check_loaded()
pacman = Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled)
package_list = []
for full_path in filter(package_like, configuration.repository_paths.repository.iterdir()):

245
src/ahriman/core/database/migrations/m011_repository_name.py Normal file
View File

@ -0,0 +1,245 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from sqlite3 import Connection
from ahriman.core.configuration import Configuration
__all__ = ["migrate_data", "steps"]
steps = [
# set correct types for schema
"""
alter table users rename to users_
""",
"""
create table users (
username text not null unique,
access text not null,
password text,
packager_id text,
key_id text
)
""",
"""
insert into users select * from users_
""",
"""
drop table users_
""",
# update base tables
# build_queue
"""
alter table build_queue add column repository text not null default ''
""",
"""
alter table build_queue rename to build_queue_
""",
"""
create table build_queue (
package_base text not null,
properties json not null,
repository text not null,
primary key (package_base, repository)
)
""",
"""
insert into build_queue select * from build_queue_
""",
"""
drop table build_queue_
""",
# package_bases
"""
alter table package_bases add column repository text not null default ''
""",
"""
alter table package_bases rename to package_bases_
""",
"""
create table package_bases (
package_base text not null,
version text not null,
branch text,
git_url text,
path text,
web_url text,
source text,
packager text,
repository text not null,
primary key (package_base, repository)
)
""",
"""
insert into package_bases select * from package_bases_
""",
"""
drop table package_bases_
""",
# package_statuses
"""
alter table package_statuses add column repository text not null default ''
""",
"""
alter table package_statuses rename to package_statuses_
""",
"""
create table package_statuses (
package_base text not null,
status text not null,
last_updated integer,
repository text not null,
primary key (package_base, repository)
)
""",
"""
insert into package_statuses select * from package_statuses_
""",
"""
drop table package_statuses_
""",
# packages
"""
alter table packages add column repository text not null default ''
""",
"""
alter table packages rename to packages_
""",
"""
create table packages (
package text not null,
package_base text not null,
architecture text not null,
archive_size integer,
build_date integer,
depends json,
description text,
filename text,
"groups" json,
installed_size integer,
licenses json,
provides json,
url text,
make_depends json,
opt_depends json,
check_depends json,
repository text not null,
primary key (package, architecture, repository)
)
""",
"""
insert into packages select * from packages_
""",
"""
drop table packages_
""",
# patches
"""
alter table patches add column repository text not null default ''
""",
"""
drop index patches_package_base_variable
""",
"""
alter table patches rename to patches_
""",
"""
create table patches (
package_base text not null,
variable text,
patch blob not null,
repository text not null
)
""",
"""
create unique index patches_package_base_variable_repository
on patches (package_base, coalesce(variable, ''), repository)
""",
"""
insert into patches select * from patches_
""",
"""
drop table patches_
""",
# logs
"""
alter table logs add column repository text not null default ''
""",
"""
drop index logs_package_base_version
""",
"""
alter table logs rename to logs_
""",
"""
create table logs (
package_base text not null,
created real not null,
record text,
version text not null,
repository text not null
)
""",
"""
insert into logs select * from logs_
""",
"""
create index logs_package_base_version on logs (package_base, version)
""",
"""
drop table logs_
""",
]
def migrate_data(connection: Connection, configuration: Configuration) -> None:
"""
perform data migration
Args:
connection(Connection): database connection
configuration(Configuration): configuration instance
"""
migrate_package_repository(connection, configuration)
def migrate_package_repository(connection: Connection, configuration: Configuration) -> None:
"""
update repository name from current settings
Args:
connection(Connection): database connection
configuration(Configuration): configuration instance
"""
_, repository_id = configuration.check_loaded()
connection.execute("""update build_queue set repository = :repository""",
{"repository": repository_id.name, })
connection.execute("""update package_bases set repository = :repository""",
{"repository": repository_id.name, })
connection.execute("""update package_statuses set repository = :repository""",
{"repository": repository_id.name, })
connection.execute("""update packages set repository = :repository""",
{"repository": repository_id.name, })
connection.execute("""update patches set repository = :repository""",
{"repository": repository_id.name, })
connection.execute("""update logs set repository = :repository""",
{"repository": repository_id.name, })

17
src/ahriman/core/database/operations/build_operations.py
View File

@ -39,9 +39,9 @@ class BuildOperations(Operations):
connection.execute(
"""
delete from build_queue
where :package_base is null or package_base = :package_base
where (:package_base is null or package_base = :package_base) and repository = :repository
""",
{"package_base": package_base})
{"package_base": package_base, "repository": self.repository_id.name})
return self.with_connection(run, commit=True)
@ -55,7 +55,10 @@ class BuildOperations(Operations):
def run(connection: Connection) -> list[Package]:
return [
Package.from_json(row["properties"])
for row in connection.execute("""select * from build_queue""")
for row in connection.execute(
"""select properties from build_queue where repository = :repository""",
{"repository": self.repository_id.name}
)
]
return self.with_connection(run)
@ -71,12 +74,12 @@ class BuildOperations(Operations):
connection.execute(
"""
insert into build_queue
(package_base, properties)
(package_base, properties, repository)
values
(:package_base, :properties)
on conflict (package_base) do update set
(:package_base, :properties, :repository)
on conflict (package_base, repository) do update set
properties = :properties
""",
{"package_base": package.base, "properties": package.view()})
{"package_base": package.base, "properties": package.view(), "repository": self.repository_id.name})
return self.with_connection(run, commit=True)

19
src/ahriman/core/database/operations/logs_operations.py
View File

@ -45,14 +45,11 @@ class LogsOperations(Operations):
(row["created"], row["record"])
for row in connection.execute(
"""
select created, record from logs where package_base = :package_base
select created, record from logs
where package_base = :package_base and repository = :repository
order by created limit :limit offset :offset
""",
{
"package_base": package_base,
"limit": limit,
"offset": offset,
})
{"package_base": package_base, "repository": self.repository_id.name, "limit": limit, "offset": offset})
]
return self.with_connection(run)
@ -70,15 +67,16 @@ class LogsOperations(Operations):
connection.execute(
"""
insert into logs
(package_base, version, created, record)
(package_base, version, created, record, repository)
values
(:package_base, :version, :created, :record)
(:package_base, :version, :created, :record, :repository)
""",
{
"package_base": log_record_id.package_base,
"version": log_record_id.version,
"created": created,
"record": record,
"repository": self.repository_id.name,
}
)
@ -97,9 +95,10 @@ class LogsOperations(Operations):
connection.execute(
"""
delete from logs
where package_base = :package_base and (:version is null or version <> :version)
where package_base = :package_base and repository = :repository
and (:version is null or version <> :version)
""",
{"package_base": package_base, "version": version}
{"package_base": package_base, "version": version, "repository": self.repository_id.name}
)
return self.with_connection(run, commit=True)

6
src/ahriman/core/database/operations/operations.py
View File

@ -24,6 +24,7 @@ from pathlib import Path
from typing import Any, TypeVar
from ahriman.core.log import LazyLogging
from ahriman.models.repository_id import RepositoryId
T = TypeVar("T")
@ -35,16 +36,19 @@ class Operations(LazyLogging):
Attributes:
path(Path): path to the database file
repository_id(RepositoryId): repository unique identifier to perform implicit filtering
"""
def __init__(self, path: Path) -> None:
def __init__(self, path: Path, repository_id: RepositoryId) -> None:
"""
default constructor
Args:
path(Path): path to the database file
repository_id(RepositoryId): repository unique identifier
"""
self.path = path
self.repository_id = repository_id
@staticmethod
def factory(cursor: sqlite3.Cursor, row: tuple[Any, ...]) -> dict[str, Any]:

102
src/ahriman/core/database/operations/package_operations.py
View File

@ -32,8 +32,7 @@ class PackageOperations(Operations):
package operations
"""
@staticmethod
def _package_remove_package_base(connection: Connection, package_base: str) -> None:
def _package_remove_package_base(self, connection: Connection, package_base: str) -> None:
"""
remove package base information
@ -41,13 +40,20 @@ class PackageOperations(Operations):
connection(Connection): database connection
package_base(str): package base name
"""
connection.execute("""delete from package_statuses where package_base = :package_base""",
{"package_base": package_base})
connection.execute("""delete from package_bases where package_base = :package_base""",
{"package_base": package_base})
connection.execute(
"""
delete from package_statuses
where package_base = :package_base and repository = :repository
""",
{"package_base": package_base, "repository": self.repository_id.name})
connection.execute(
"""
delete from package_bases
where package_base = :package_base and repository = :repository""",
{"package_base": package_base, "repository": self.repository_id.name})
@staticmethod
def _package_remove_packages(connection: Connection, package_base: str, current_packages: Iterable[str]) -> None:
def _package_remove_packages(self, connection: Connection, package_base: str,
current_packages: Iterable[str]) -> None:
"""
remove packages belong to the package base
@ -59,13 +65,20 @@ class PackageOperations(Operations):
packages = [
package
for package in connection.execute(
"""select package from packages where package_base = :package_base""", {"package_base": package_base})
"""
select package, repository from packages
where package_base = :package_base and repository = :repository""",
{"package_base": package_base, "repository": self.repository_id.name})
if package["package"] not in current_packages
]
connection.executemany("""delete from packages where package = :package""", packages)
connection.executemany(
"""
delete from packages
where package = :package and repository = :repository
""",
packages)
@staticmethod
def _package_update_insert_base(connection: Connection, package: Package) -> None:
def _package_update_insert_base(self, connection: Connection, package: Package) -> None:
"""
insert base package into table
@ -76,10 +89,10 @@ class PackageOperations(Operations):
connection.execute(
"""
insert into package_bases
(package_base, version, source, branch, git_url, path, web_url, packager)
(package_base, version, source, branch, git_url, path, web_url, packager, repository)
values
(:package_base, :version, :source, :branch, :git_url, :path, :web_url, :packager)
on conflict (package_base) do update set
(:package_base, :version, :source, :branch, :git_url, :path, :web_url, :packager, :repository)
on conflict (package_base, repository) do update set
version = :version, branch = :branch, git_url = :git_url, path = :path, web_url = :web_url,
source = :source, packager = :packager
""",
@ -92,11 +105,11 @@ class PackageOperations(Operations):
"web_url": package.remote.web_url,
"source": package.remote.source.value,
"packager": package.packager,
"repository": self.repository_id.name,
}
)
@staticmethod
def _package_update_insert_packages(connection: Connection, package: Package) -> None:
def _package_update_insert_packages(self, connection: Connection, package: Package) -> None:
"""
insert packages into table
@ -108,20 +121,27 @@ class PackageOperations(Operations):
for name, description in package.packages.items():
if description.architecture is None:
continue # architecture is required
package_list.append({"package": name, "package_base": package.base, **description.view()})
package_list.append({
"package": name,
"package_base": package.base,
"repository": self.repository_id.name,
**description.view(),
})
connection.executemany(
"""
insert into packages
(package, package_base, architecture, archive_size,
build_date, depends, description, filename,
"groups", installed_size, licenses, provides,
url, make_depends, opt_depends, check_depends)
url, make_depends, opt_depends, check_depends,
repository)
values
(:package, :package_base, :architecture, :archive_size,
:build_date, :depends, :description, :filename,
:groups, :installed_size, :licenses, :provides,
:url, :make_depends, :opt_depends, :check_depends)
on conflict (package, architecture) do update set
:url, :make_depends, :opt_depends, :check_depends,
:repository)
on conflict (package, architecture, repository) do update set
package_base = :package_base, archive_size = :archive_size,
build_date = :build_date, depends = :depends, description = :description, filename = :filename,
"groups" = :groups, installed_size = :installed_size, licenses = :licenses, provides = :provides,
@ -129,8 +149,7 @@ class PackageOperations(Operations):
""",
package_list)
@staticmethod
def _package_update_insert_status(connection: Connection, package_base: str, status: BuildStatus) -> None:
def _package_update_insert_status(self, connection: Connection, package_base: str, status: BuildStatus) -> None:
"""
insert base package status into table
@ -141,16 +160,21 @@ class PackageOperations(Operations):
"""
connection.execute(
"""
insert into package_statuses (package_base, status, last_updated)
insert into package_statuses
(package_base, status, last_updated, repository)
values
(:package_base, :status, :last_updated)
on conflict (package_base) do update set
(:package_base, :status, :last_updated, :repository)
on conflict (package_base, repository) do update set
status = :status, last_updated = :last_updated
""",
{"package_base": package_base, "status": status.status.value, "last_updated": status.timestamp})
{
"package_base": package_base,
"status": status.status.value,
"last_updated": status.timestamp,
"repository": self.repository_id.name,
})
@staticmethod
def _packages_get_select_package_bases(connection: Connection) -> dict[str, Package]:
def _packages_get_select_package_bases(self, connection: Connection) -> dict[str, Package]:
"""
select package bases from the table
@ -167,11 +191,13 @@ class PackageOperations(Operations):
remote=RemoteSource.from_json(row),
packages={},
packager=row["packager"] or None,
) for row in connection.execute("""select * from package_bases""")
) for row in connection.execute(
"""select * from package_bases where repository = :repository""",
{"repository": self.repository_id.name}
)
}
@staticmethod
def _packages_get_select_packages(connection: Connection, packages: dict[str, Package]) -> dict[str, Package]:
def _packages_get_select_packages(self, connection: Connection, packages: dict[str, Package]) -> dict[str, Package]:
"""
select packages from the table
@ -182,14 +208,15 @@ class PackageOperations(Operations):
Returns:
dict[str, Package]: map of the package base to its descriptor including individual packages
"""
for row in connection.execute("""select * from packages"""):
for row in connection.execute(
"""select * from packages where repository = :repository""",
{"repository": self.repository_id.name}):
if row["package_base"] not in packages:
continue # normally must never happen though
packages[row["package_base"]].packages[row["package"]] = PackageDescription.from_json(row)
return packages
@staticmethod
def _packages_get_select_statuses(connection: Connection) -> dict[str, BuildStatus]:
def _packages_get_select_statuses(self, connection: Connection) -> dict[str, BuildStatus]:
"""
select package build statuses from the table
@ -201,7 +228,10 @@ class PackageOperations(Operations):
"""
return {
row["package_base"]: BuildStatus.from_json({"status": row["status"], "timestamp": row["last_updated"]})
for row in connection.execute("""select * from package_statuses""")
for row in connection.execute(
"""select * from package_statuses where repository = :repository""",
{"repository": self.repository_id.name}
)
}
def package_remove(self, package_base: str) -> None:

38
src/ahriman/core/database/operations/patch_operations.py
View File

@ -54,13 +54,19 @@ class PatchOperations(Operations):
connection.execute(
"""
insert into patches
(package_base, variable, patch)
(package_base, variable, patch, repository)
values
(:package_base, :variable, :patch)
on conflict (package_base, coalesce(variable, '')) do update set
(:package_base, :variable, :patch, :repository)
on conflict (package_base, coalesce(variable, ''), repository) do update set
patch = :patch
""",
{"package_base": package_base, "variable": patch.key, "patch": patch.value})
{
"package_base": package_base,
"variable": patch.key,
"patch": patch.value,
"repository": self.repository_id.name,
}
)
return self.with_connection(run, commit=True)
@ -79,8 +85,10 @@ class PatchOperations(Operations):
return [
(row["package_base"], PkgbuildPatch(row["variable"], row["patch"]))
for row in connection.execute(
"""select * from patches where :package_base is null or package_base = :package_base""",
{"package_base": package_base})
"""
select * from patches
where (:package_base is null or package_base = :package_base) and repository = :repository""",
{"package_base": package_base, "repository": self.repository_id.name})
]
# we could use itertools & operator but why?
@ -101,13 +109,23 @@ class PatchOperations(Operations):
"""
def run_many(connection: Connection) -> None:
connection.executemany(
"""delete from patches where package_base = :package_base and variable = :variable""",
[{"package_base": package_base, "variable": variable} for variable in variables])
"""
delete from patches
where package_base = :package_base and variable = :variable and repository = :repository
""",
[
{
"package_base": package_base,
"variable": variable,
"repository": self.repository_id.name,
} for variable in variables
]
)
def run(connection: Connection) -> None:
connection.execute(
"""delete from patches where package_base = :package_base""",
{"package_base": package_base})
"""delete from patches where package_base = :package_base and repository = :repository""",
{"package_base": package_base, "repository": self.repository_id.name})
if variables:
return self.with_connection(run_many, commit=True)

5
src/ahriman/core/database/sqlite.py
View File

@ -56,8 +56,11 @@ class SQLite(AuthOperations, BuildOperations, LogsOperations, PackageOperations,
Self: fully initialized instance of the database
"""
path = cls.database_path(configuration)
database = cls(path)
_, repository_id = configuration.check_loaded()
database = cls(path, repository_id)
database.init(configuration)
return database
@staticmethod

7
src/ahriman/core/gitremote/remote_pull.py
View File

@ -30,6 +30,7 @@ from ahriman.core.util import walk
from ahriman.models.package import Package
from ahriman.models.package_source import PackageSource
from ahriman.models.remote_source import RemoteSource
from ahriman.models.repository_id import RepositoryId
class RemotePull(LazyLogging):
@ -42,13 +43,13 @@ class RemotePull(LazyLogging):
repository_paths(RepositoryPaths): repository paths instance
"""
def __init__(self, configuration: Configuration, architecture: str, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
architecture(str): repository architecture
section(str): settings section name
"""
self.remote_source = RemoteSource(
@ -58,7 +59,7 @@ class RemotePull(LazyLogging):
branch=configuration.get(section, "pull_branch", fallback="master"),
source=PackageSource.Local,
)
self.architecture = architecture
self.architecture = repository_id.architecture
self.repository_paths = configuration.repository_paths
def package_copy(self, pkgbuild_path: Path) -> None:

11
src/ahriman/core/gitremote/remote_pull_trigger.py
View File

@ -20,6 +20,7 @@
from ahriman.core.configuration import Configuration
from ahriman.core.gitremote.remote_pull import RemotePull
from ahriman.core.triggers import Trigger
from ahriman.models.repository_id import RepositoryId
class RemotePullTrigger(Trigger):
@ -56,15 +57,15 @@ class RemotePullTrigger(Trigger):
}
CONFIGURATION_SCHEMA_FALLBACK = "gitremote"
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, architecture, configuration)
Trigger.__init__(self, repository_id, configuration)
self.targets = self.configuration_sections(configuration)
@classmethod
@ -86,6 +87,6 @@ class RemotePullTrigger(Trigger):
"""
for target in self.targets:
section, _ = self.configuration.gettype(
target, self.architecture, fallback=self.CONFIGURATION_SCHEMA_FALLBACK)
runner = RemotePull(self.configuration, self.architecture, section)
target, self.repository_id, fallback=self.CONFIGURATION_SCHEMA_FALLBACK)
runner = RemotePull(self.repository_id, self.configuration, section)
runner.run()

9
src/ahriman/core/gitremote/remote_push_trigger.py
View File

@ -24,6 +24,7 @@ from ahriman.core.gitremote.remote_push import RemotePush
from ahriman.core.triggers import Trigger
from ahriman.models.context_key import ContextKey
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -67,15 +68,15 @@ class RemotePushTrigger(Trigger):
}
CONFIGURATION_SCHEMA_FALLBACK = "gitremote"
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, architecture, configuration)
Trigger.__init__(self, repository_id, configuration)
self.targets = self.configuration_sections(configuration)
@classmethod
@ -107,6 +108,6 @@ class RemotePushTrigger(Trigger):
for target in self.targets:
section, _ = self.configuration.gettype(
target, self.architecture, fallback=self.CONFIGURATION_SCHEMA_FALLBACK)
target, self.repository_id, fallback=self.CONFIGURATION_SCHEMA_FALLBACK)
runner = RemotePush(database, self.configuration, section)
runner.run(result)

4
src/ahriman/core/http/sync_http_client.py
View File

@ -41,14 +41,14 @@ class SyncHttpClient(LazyLogging):
timeout(int): HTTP request timeout in seconds
"""
def __init__(self, section: str | None = None, configuration: Configuration | None = None, *,
def __init__(self, configuration: Configuration | None = None, section: str | None = None, *,
suppress_errors: bool = False) -> None:
"""
default constructor
Args:
section(str, optional): settings section name (Default value = None)
configuration(Configuration | None): configuration instance (Default value = None)
section(str, optional): settings section name (Default value = None)
suppress_errors(bool, optional): suppress logging of request errors (Default value = False)
"""
if configuration is None:

1
src/ahriman/core/log/__init__.py
View File

@ -18,4 +18,3 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from ahriman.core.log.lazy_logging import LazyLogging
from ahriman.core.log.log import Log

7
src/ahriman/core/log/log.py → src/ahriman/core/log/log_loader.py
View File

@ -27,7 +27,7 @@ from ahriman.core.log.http_log_handler import HttpLogHandler
from ahriman.models.log_handler import LogHandler
class Log:
class LogLoader:
"""
simple static method class which setups application loggers
@ -63,7 +63,7 @@ class Log:
del JournalHandler
return LogHandler.Journald # journald import was found
except ImportError:
if Log.DEFAULT_SYSLOG_DEVICE.exists():
if LogLoader.DEFAULT_SYSLOG_DEVICE.exists():
return LogHandler.Syslog
return LogHandler.Console
@ -94,8 +94,7 @@ class Log:
fileConfig(log_configuration, disable_existing_loggers=True)
logging.debug("using %s logger", default_handler)
except Exception:
logging.basicConfig(filename=None, format=Log.DEFAULT_LOG_FORMAT,
level=Log.DEFAULT_LOG_LEVEL)
logging.basicConfig(filename=None, format=LogLoader.DEFAULT_LOG_FORMAT, level=LogLoader.DEFAULT_LOG_LEVEL)
logging.exception("could not load logging from configuration, fallback to stderr")
HttpLogHandler.load(configuration, report=report)

7
src/ahriman/core/report/console.py
View File

@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.formatters import BuildPrinter
from ahriman.core.report.report import Report
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -32,16 +33,16 @@ class Console(Report):
use_utf(bool): print utf8 symbols instead of ASCII
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Report.__init__(self, architecture, configuration)
Report.__init__(self, repository_id, configuration)
self.use_utf = configuration.getboolean(section, "use_utf", fallback=True)
def generate(self, packages: list[Package], result: Result) -> None:

9
src/ahriman/core/report/email.py
View File

@ -27,6 +27,7 @@ from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.report.report import Report
from ahriman.core.util import pretty_datetime, utcnow
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
from ahriman.models.smtp_ssl_settings import SmtpSSLSettings
@ -48,17 +49,17 @@ class Email(Report, JinjaTemplate):
user(str | None): username to authenticate via SMTP
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Report.__init__(self, architecture, configuration)
JinjaTemplate.__init__(self, section, configuration)
Report.__init__(self, repository_id, configuration)
JinjaTemplate.__init__(self, repository_id, configuration, section)
self.full_template_path = configuration.getpath(section, "full_template_path", fallback=None)
self.template_path = configuration.getpath(section, "template_path")

9
src/ahriman/core/report/html.py
View File

@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.report.report import Report
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -33,17 +34,17 @@ class HTML(Report, JinjaTemplate):
template_path(Path): path to template for full package list
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Report.__init__(self, architecture, configuration)
JinjaTemplate.__init__(self, section, configuration)
Report.__init__(self, repository_id, configuration)
JinjaTemplate.__init__(self, repository_id, configuration, section)
self.report_path = configuration.getpath(section, "path")
self.template_path = configuration.getpath(section, "template_path")

8
src/ahriman/core/report/jinja_template.py
View File

@ -25,6 +25,7 @@ from pathlib import Path
from ahriman.core.configuration import Configuration
from ahriman.core.sign.gpg import GPG
from ahriman.core.util import pretty_datetime, pretty_size
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
from ahriman.models.sign_settings import SignSettings
@ -63,19 +64,20 @@ class JinjaTemplate:
sign_targets(set[SignSettings]): targets to sign enabled in configuration
"""
def __init__(self, section: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
section(str): settings section name
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
self.link_path = configuration.get(section, "link_path")
# base template vars
self.homepage = configuration.get(section, "homepage", fallback=None)
self.name = configuration.repository_name
self.name = repository_id.name
self.sign_targets, self.default_pgp_key = GPG.sign_options(configuration)

7
src/ahriman/core/report/remote_call.py
View File

@ -23,6 +23,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.report.report import Report
from ahriman.core.status.web_client import WebClient
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
from ahriman.models.waiter import Waiter
@ -39,16 +40,16 @@ class RemoteCall(Report):
wait_timeout(int): timeout to wait external process
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Report.__init__(self, architecture, configuration)
Report.__init__(self, repository_id, configuration)
self.client = WebClient(configuration)

31
src/ahriman/core/report/report.py
View File

@ -24,6 +24,7 @@ from ahriman.core.exceptions import ReportError
from ahriman.core.log import LazyLogging
from ahriman.models.package import Package
from ahriman.models.report_settings import ReportSettings
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -32,17 +33,15 @@ class Report(LazyLogging):
base report generator
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
repository_id(RepositoryId): repository unique identifier
Examples:
``Report`` classes provide several method in order to operate with the report generation and additional class
method ``load`` which can be used in order to determine right report instance::
>>> from ahriman.core.configuration import Configuration
>>>
>>> configuration = Configuration()
>>> report = Report.load("x86_64", configuration, "email")
>>> report = Report.load(RepositoryId("x86_64", "aur-clone"), configuration, "email")
The ``generate`` method can be used in order to perform the report itself, whereas ``run`` method handles
exception and raises ``ReportFailed`` instead::
@ -55,49 +54,49 @@ class Report(LazyLogging):
>>> report.run(Result(), [])
"""
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
self.architecture = architecture
self.repository_id = repository_id
self.configuration = configuration
@staticmethod
def load(architecture: str, configuration: Configuration, target: str) -> Report:
def load(repository_id: RepositoryId, configuration: Configuration, target: str) -> Report:
"""
load client from settings
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
target(str): target to generate report aka section name (e.g. html)
Returns:
Report: client according to current settings
"""
section, provider_name = configuration.gettype(target, architecture)
section, provider_name = configuration.gettype(target, repository_id)
match ReportSettings.from_option(provider_name):
case ReportSettings.HTML:
from ahriman.core.report.html import HTML
return HTML(architecture, configuration, section)
return HTML(repository_id, configuration, section)
case ReportSettings.Email:
from ahriman.core.report.email import Email
return Email(architecture, configuration, section)
return Email(repository_id, configuration, section)
case ReportSettings.Console:
from ahriman.core.report.console import Console
return Console(architecture, configuration, section)
return Console(repository_id, configuration, section)
case ReportSettings.Telegram:
from ahriman.core.report.telegram import Telegram
return Telegram(architecture, configuration, section)
return Telegram(repository_id, configuration, section)
case ReportSettings.RemoteCall:
from ahriman.core.report.remote_call import RemoteCall
return RemoteCall(architecture, configuration, section)
return RemoteCall(repository_id, configuration, section)
case _:
return Report(architecture, configuration) # should never happen
return Report(repository_id, configuration) # should never happen
def generate(self, packages: list[Package], result: Result) -> None:
"""

9
src/ahriman/core/report/report_trigger.py
View File

@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.triggers import Trigger
from ahriman.core.report.report import Report
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -218,15 +219,15 @@ class ReportTrigger(Trigger):
}
}
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, architecture, configuration)
Trigger.__init__(self, repository_id, configuration)
self.targets = self.configuration_sections(configuration)
@classmethod
@ -251,5 +252,5 @@ class ReportTrigger(Trigger):
packages(list[Package]): list of all available packages
"""
for target in self.targets:
runner = Report.load(self.architecture, self.configuration, target)
runner = Report.load(self.repository_id, self.configuration, target)
runner.run(result, packages)

11
src/ahriman/core/report/telegram.py
View File

@ -22,6 +22,7 @@ from ahriman.core.http import SyncHttpClient
from ahriman.core.report.jinja_template import JinjaTemplate
from ahriman.core.report.report import Report
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -41,18 +42,18 @@ class Telegram(Report, JinjaTemplate, SyncHttpClient):
TELEGRAM_API_URL = "https://api.telegram.org"
TELEGRAM_MAX_CONTENT_LENGTH = 4096
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Report.__init__(self, architecture, configuration)
JinjaTemplate.__init__(self, section, configuration)
SyncHttpClient.__init__(self, section, configuration)
Report.__init__(self, repository_id, configuration)
JinjaTemplate.__init__(self, repository_id, configuration, section)
SyncHttpClient.__init__(self, configuration, section)
self.api_key = configuration.get(section, "api_key")
self.chat_id = configuration.get(section, "chat_id")

9
src/ahriman/core/repository/repository.py
View File

@ -32,6 +32,7 @@ from ahriman.core.util import package_like
from ahriman.models.context_key import ContextKey
from ahriman.models.package import Package
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.repository_id import RepositoryId
class Repository(Executor, UpdateHandler):
@ -47,7 +48,7 @@ class Repository(Executor, UpdateHandler):
>>>
>>> configuration = Configuration()
>>> database = SQLite.load(configuration)
>>> repository = Repository.load("x86_64", configuration, database, report=True)
>>> repository = Repository.load(RepositoryId("x86_64", "x86_64"), configuration, database, report=True)
>>> known_packages = repository.packages()
>>>
>>> build_result = repository.process_build(known_packages)
@ -58,13 +59,13 @@ class Repository(Executor, UpdateHandler):
"""
@classmethod
def load(cls, architecture: str, configuration: Configuration, database: SQLite, *, report: bool,
def load(cls, repository_id: RepositoryId, configuration: Configuration, database: SQLite, *, report: bool,
refresh_pacman_database: PacmanSynchronization = PacmanSynchronization.Disabled) -> Self:
"""
load instance from argument list
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
database(SQLite): database instance
report(bool): force enable or disable reporting
@ -74,7 +75,7 @@ class Repository(Executor, UpdateHandler):
Returns:
Self: fully loaded repository class instance
"""
instance = cls(architecture, configuration, database,
instance = cls(repository_id, configuration, database,
report=report, refresh_pacman_database=refresh_pacman_database)
instance._set_context()
return instance

35
src/ahriman/core/repository/repository_properties.py
View File

@ -27,6 +27,7 @@ from ahriman.core.status.client import Client
from ahriman.core.triggers import TriggerLoader
from ahriman.models.packagers import Packagers
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
from ahriman.models.user import User
from ahriman.models.user_access import UserAccess
@ -37,47 +38,65 @@ class RepositoryProperties(LazyLogging):
repository internal objects holder
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
database(SQLite): database instance
ignore_list(list[str]): package bases which will be ignored during auto updates
name(str): repository name
pacman(Pacman): alpm wrapper instance
paths(RepositoryPaths): repository paths instance
repo(Repo): repo commands wrapper instance
reporter(Client): build status reporter instance
repository_id(RepositoryId): repository unique identifier
sign(GPG): GPG wrapper instance
triggers(TriggerLoader): triggers holder
vcs_allowed_age(int): maximal age of the VCS packages before they will be checked
"""
def __init__(self, architecture: str, configuration: Configuration, database: SQLite, *, report: bool,
def __init__(self, repository_id: RepositoryId, configuration: Configuration, database: SQLite, *, report: bool,
refresh_pacman_database: PacmanSynchronization) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
database(SQLite): database instance
report(bool): force enable or disable reporting
refresh_pacman_database(PacmanSynchronization): pacman database synchronization level
"""
self.architecture = architecture
self.repository_id = repository_id
self.configuration = configuration
self.database = database
self.name = configuration.repository_name
self.vcs_allowed_age = configuration.getint("build", "vcs_allowed_age", fallback=0)
self.paths: RepositoryPaths = configuration.repository_paths # additional workaround for pycharm typing
self.ignore_list = configuration.getlist("build", "ignore_packages", fallback=[])
self.pacman = Pacman(architecture, configuration, refresh_database=refresh_pacman_database)
self.pacman = Pacman(repository_id, configuration, refresh_database=refresh_pacman_database)
self.sign = GPG(configuration)
self.repo = Repo(self.name, self.paths, self.sign.repository_sign_args)
self.reporter = Client.load(configuration, report=report)
self.triggers = TriggerLoader.load(architecture, configuration)
self.triggers = TriggerLoader.load(repository_id, configuration)
@property
def architecture(self) -> str:
"""
repository architecture for backward compatibility
Returns:
str: repository architecture
"""
return self.repository_id.architecture
@property
def name(self) -> str:
"""
repository name for backward compatibility
Returns:
str: repository name
"""
return self.repository_id.name
def packager(self, packagers: Packagers, package_base: str) -> User:
"""

20
src/ahriman/core/spawn.py
View File

@ -28,6 +28,7 @@ from multiprocessing import Process, Queue
from threading import Lock, Thread
from ahriman.core.log import LazyLogging
from ahriman.models.repository_id import RepositoryId
class Spawn(Thread, LazyLogging):
@ -37,22 +38,23 @@ class Spawn(Thread, LazyLogging):
Attributes:
active(dict[str, Process]): map of active child processes required to avoid zombies
architecture(str): repository architecture
command_arguments(list[str]): base command line arguments
queue(Queue[tuple[str, bool, int]]): multiprocessing queue to read updates from processes
repository_id(RepositoryId): repository unique identifier
"""
def __init__(self, args_parser: argparse.ArgumentParser, architecture: str, command_arguments: list[str]) -> None:
def __init__(self, args_parser: argparse.ArgumentParser, repository_id: RepositoryId,
command_arguments: list[str]) -> None:
"""
default constructor
Args:
args_parser(argparse.ArgumentParser): command line parser for the application
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
command_arguments(list[str]): base command line arguments
"""
Thread.__init__(self, name="spawn")
self.architecture = architecture
self.repository_id = repository_id
self.args_parser = args_parser
self.command_arguments = command_arguments
@ -77,20 +79,20 @@ class Spawn(Thread, LazyLogging):
return name if value else f"no-{name}"
@staticmethod
def process(callback: Callable[[argparse.Namespace, str], bool], args: argparse.Namespace, architecture: str,
process_id: str, queue: Queue[tuple[str, bool, int]]) -> None: # pylint: disable=unsubscriptable-object
def process(callback: Callable[[argparse.Namespace, RepositoryId], bool], args: argparse.Namespace,
repository_id: RepositoryId, process_id: str, queue: Queue[tuple[str, bool, int]]) -> None: # pylint: disable=unsubscriptable-object
"""
helper to run external process
Args:
callback(Callable[[argparse.Namespace, str], bool]): application run function (i.e. Handler.run method)
args(argparse.Namespace): command line arguments
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
process_id(str): process unique identifier
queue(Queue[tuple[str, bool, int]]): output queue
"""
start_time = time.monotonic()
result = callback(args, architecture)
result = callback(args, repository_id)
stop_time = time.monotonic()
consumed_time = int(1000 * (stop_time - start_time))
@ -128,7 +130,7 @@ class Spawn(Thread, LazyLogging):
callback = parsed.handler.call
process = Process(target=self.process,
args=(callback, parsed, self.architecture, process_id, self.queue),
args=(callback, parsed, self.repository_id, process_id, self.queue),
daemon=True)
process.start()

11
src/ahriman/core/status/watcher.py
View File

@ -25,6 +25,7 @@ from ahriman.core.repository import Repository
from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.log_record_id import LogRecordId
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class Watcher(LazyLogging):
@ -32,26 +33,26 @@ class Watcher(LazyLogging):
package status watcher
Attributes:
architecture(str): repository architecture
database(SQLite): database instance
known(dict[str, tuple[Package, BuildStatus]]): list of known packages. For the most cases ``packages`` should
be used instead
repository(Repository): repository object
repository_id(RepositoryId): repository unique identifier
status(BuildStatus): daemon status
"""
def __init__(self, architecture: str, configuration: Configuration, database: SQLite) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, database: SQLite) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
database(SQLite): database instance
"""
self.architecture = architecture
self.repository_id = repository_id
self.database = database
self.repository = Repository.load(architecture, configuration, database, report=False)
self.repository = Repository.load(repository_id, configuration, database, report=False)
self.known: dict[str, tuple[Package, BuildStatus]] = {}
self.status = BuildStatus()

2
src/ahriman/core/status/web_client.py
View File

@ -51,7 +51,7 @@ class WebClient(Client, SyncHttpClient):
configuration(Configuration): configuration instance
"""
suppress_errors = configuration.getboolean("settings", "suppress_http_log_errors", fallback=False)
SyncHttpClient.__init__(self, "web", configuration, suppress_errors=suppress_errors)
SyncHttpClient.__init__(self, configuration, "web", suppress_errors=suppress_errors)
self.address, self.use_unix_socket = self.parse_address(configuration)

9
src/ahriman/core/support/keyring_trigger.py
View File

@ -25,6 +25,7 @@ from ahriman.core.support.package_creator import PackageCreator
from ahriman.core.support.pkgbuild.keyring_generator import KeyringGenerator
from ahriman.core.triggers import Trigger
from ahriman.models.context_key import ContextKey
from ahriman.models.repository_id import RepositoryId
class KeyringTrigger(Trigger):
@ -82,15 +83,15 @@ class KeyringTrigger(Trigger):
},
}
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, architecture, configuration)
Trigger.__init__(self, repository_id, configuration)
self.targets = self.configuration_sections(configuration)
@classmethod
@ -115,6 +116,6 @@ class KeyringTrigger(Trigger):
database = ctx.get(ContextKey("database", SQLite))
for target in self.targets:
generator = KeyringGenerator(database, sign, self.configuration, target)
generator = KeyringGenerator(database, sign, self.repository_id, self.configuration, target)
runner = PackageCreator(self.configuration, generator)
runner.run()

9
src/ahriman/core/support/mirrorlist_trigger.py
View File

@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.support.package_creator import PackageCreator
from ahriman.core.support.pkgbuild.mirrorlist_generator import MirrorlistGenerator
from ahriman.core.triggers import Trigger
from ahriman.models.repository_id import RepositoryId
class MirrorlistTrigger(Trigger):
@ -75,15 +76,15 @@ class MirrorlistTrigger(Trigger):
},
}
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, architecture, configuration)
Trigger.__init__(self, repository_id, configuration)
self.targets = self.configuration_sections(configuration)
@classmethod
@ -104,6 +105,6 @@ class MirrorlistTrigger(Trigger):
trigger action which will be called at the start of the application
"""
for target in self.targets:
generator = MirrorlistGenerator(self.configuration, target)
generator = MirrorlistGenerator(self.repository_id, self.configuration, target)
runner = PackageCreator(self.configuration, generator)
runner.run()

4
src/ahriman/core/support/package_creator.py
View File

@ -66,6 +66,6 @@ class PackageCreator:
# register package
ctx = context.get()
database: SQLite = ctx.get(ContextKey("database", SQLite))
_, architecture = self.configuration.check_loaded()
package = Package.from_build(local_path, architecture, None)
_, repository_id = self.configuration.check_loaded()
package = Package.from_build(local_path, repository_id.architecture, None)
database.package_update(package, BuildStatus())

7
src/ahriman/core/support/pkgbuild/keyring_generator.py
View File

@ -25,6 +25,7 @@ from ahriman.core.database import SQLite
from ahriman.core.exceptions import PkgbuildGeneratorError
from ahriman.core.sign.gpg import GPG
from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator
from ahriman.models.repository_id import RepositoryId
class KeyringGenerator(PkgbuildGenerator):
@ -43,18 +44,20 @@ class KeyringGenerator(PkgbuildGenerator):
trusted(list[str]): lif of trusted PGP keys
"""
def __init__(self, database: SQLite, sign: GPG, configuration: Configuration, section: str) -> None:
def __init__(self, database: SQLite, sign: GPG, repository_id: RepositoryId,
configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
database(SQLite): database instance
sign(GPG): GPG wrapper instance
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
self.sign = sign
self.name = configuration.repository_name
self.name = repository_id.name
# configuration fields
packager_keys = [packager.key for packager in database.user_list(None, None) if packager.key is not None]

13
src/ahriman/core/support/pkgbuild/mirrorlist_generator.py
View File

@ -23,6 +23,7 @@ from pathlib import Path
from ahriman.core.configuration import Configuration
from ahriman.core.support.pkgbuild.pkgbuild_generator import PkgbuildGenerator
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_id import RepositoryId
class MirrorlistGenerator(PkgbuildGenerator):
@ -38,24 +39,24 @@ class MirrorlistGenerator(PkgbuildGenerator):
servers(list[str]): list of mirror servers
"""
def __init__(self, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
name = configuration.repository_name
# configuration fields
self.servers = configuration.getlist(section, "servers")
self.path = configuration.getpath(section, "path", fallback=Path("/etc") / "pacman.d" / f"{name}-mirrorlist")
self.path = configuration.getpath(
section, "path", fallback=Path("/etc") / "pacman.d" / f"{repository_id.name}-mirrorlist")
self.path = self.path.relative_to("/") # in pkgbuild we are always operating with relative to / path
# pkgbuild description fields
self.pkgbuild_pkgname = configuration.get(section, "package", fallback=f"{name}-mirrorlist")
self.pkgbuild_pkgname = configuration.get(section, "package", fallback=f"{repository_id.name}-mirrorlist")
self.pkgbuild_pkgdesc = configuration.get(
section, "description", fallback=f"{name} mirror list for use by pacman")
section, "description", fallback=f"{repository_id.name} mirror list for use by pacman")
self.pkgbuild_license = configuration.getlist(section, "license", fallback=["Unlicense"])
self.pkgbuild_url = configuration.get(section, "homepage", fallback="")

3
src/ahriman/core/tree.py
View File

@ -102,10 +102,11 @@ class Tree:
>>> from ahriman.core.configuration import Configuration
>>> from ahriman.core.database import SQLite
>>> from ahriman.core.repository import Repository
>>> from ahriman.models.repository_id import RepositoryId
>>>
>>> configuration = Configuration()
>>> database = SQLite.load(configuration)
>>> repository = Repository.load("x86_64", configuration, database, report=True)
>>> repository = Repository.load(RepositoryId("x86_64", "aur-clone"), configuration, database, report=True)
>>> packages = repository.packages()
>>>
>>> tree = Tree.resolve(packages)

28
src/ahriman/core/triggers/trigger.py
View File

@ -23,6 +23,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.configuration.schema import ConfigurationSchema
from ahriman.core.log import LazyLogging
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -34,8 +35,8 @@ class Trigger(LazyLogging):
CONFIGURATION_SCHEMA(ConfigurationSchema): (class attribute) configuration schema template
CONFIGURATION_SCHEMA_FALLBACK(str | None): (class attribute) optional fallback option for defining
configuration schema type used
architecture(str): repository architecture
configuration(Configuration): configuration instance
repository_id(RepositoryId): repository unique identifier
Examples:
This class must be used in order to create own extension. Basically idea is the following::
@ -51,26 +52,37 @@ class Trigger(LazyLogging):
>>> configuration = Configuration()
>>> configuration.set_option("build", "triggers", "my.awesome.package.CustomTrigger")
>>>
>>> loader = TriggerLoader.load("x86_64", configuration)
>>> loader = TriggerLoader.load(RepositoryId("x86_64", "aur-clone"), configuration)
>>> loader.on_result(Result(), [])
"""
CONFIGURATION_SCHEMA: ConfigurationSchema = {}
CONFIGURATION_SCHEMA_FALLBACK: str | None = None
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
self.architecture = architecture
self.repository_id = repository_id
self.configuration = configuration
@property
def architecture(self) -> str:
"""
repository architecture for backward compatibility
Returns:
str: repository architecture
"""
return self.repository_id.architecture
@classmethod
def configuration_schema(cls, architecture: str, configuration: Configuration | None) -> ConfigurationSchema:
def configuration_schema(cls, repository_id: RepositoryId,
configuration: Configuration | None) -> ConfigurationSchema:
"""
configuration schema based on supplied service configuration
@ -78,7 +90,7 @@ class Trigger(LazyLogging):
Schema must be in cerberus format, for details and examples you can check built-in triggers.
Args:
architecture(str): repository architecture
repository_id(str): repository unique identifier
configuration(Configuration | None): configuration instance. If set to None, the default schema
should be returned
@ -93,7 +105,7 @@ class Trigger(LazyLogging):
if not configuration.has_section(target):
continue
section, schema_name = configuration.gettype(
target, architecture, fallback=cls.CONFIGURATION_SCHEMA_FALLBACK)
target, repository_id, fallback=cls.CONFIGURATION_SCHEMA_FALLBACK)
if schema_name not in cls.CONFIGURATION_SCHEMA:
continue
result[section] = cls.CONFIGURATION_SCHEMA[schema_name]

15
src/ahriman/core/triggers/trigger_loader.py
View File

@ -31,6 +31,7 @@ from ahriman.core.exceptions import ExtensionError
from ahriman.core.log import LazyLogging
from ahriman.core.triggers import Trigger
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -49,7 +50,7 @@ class TriggerLoader(LazyLogging):
Having such configuration you can create instance of the loader::
>>> loader = TriggerLoader.load("x86_64", configuration)
>>> loader = TriggerLoader.load(RepositoryId("x86_64", "aur-clone"), configuration)
>>> print(loader.triggers)
After that you are free to run triggers::
@ -65,12 +66,12 @@ class TriggerLoader(LazyLogging):
self.triggers: list[Trigger] = []
@classmethod
def load(cls, architecture: str, configuration: Configuration) -> Self:
def load(cls, repository_id: RepositoryId, configuration: Configuration) -> Self:
"""
create instance from configuration
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
Returns:
@ -78,7 +79,7 @@ class TriggerLoader(LazyLogging):
"""
instance = cls()
instance.triggers = [
instance.load_trigger(trigger, architecture, configuration)
instance.load_trigger(trigger, repository_id, configuration)
for trigger in instance.selected_triggers(configuration)
]
@ -166,13 +167,13 @@ class TriggerLoader(LazyLogging):
except ModuleNotFoundError:
raise ExtensionError(f"Module {package} not found") from None
def load_trigger(self, module_path: str, architecture: str, configuration: Configuration) -> Trigger:
def load_trigger(self, module_path: str, repository_id: RepositoryId, configuration: Configuration) -> Trigger:
"""
load trigger by module path
Args:
module_path(str): module import path to load
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
Returns:
@ -183,7 +184,7 @@ class TriggerLoader(LazyLogging):
"""
trigger_type = self.load_trigger_class(module_path)
try:
trigger = trigger_type(architecture, configuration)
trigger = trigger_type(repository_id, configuration)
except Exception:
raise ExtensionError(f"Could not load instance of trigger from {trigger_type} loaded from {module_path}")

27
src/ahriman/core/upload/github.py
View File

@ -25,32 +25,44 @@ from typing import Any
from ahriman.core.configuration import Configuration
from ahriman.core.upload.http_upload import HttpUpload
from ahriman.core.upload.upload import Upload
from ahriman.core.util import walk
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class Github(HttpUpload):
class GitHub(Upload, HttpUpload):
"""
upload files to GitHub releases
Attributes:
github_owner(str): GitHub repository owner
github_release_tag(str): GitHub release tag
github_release_tag_name(str): GitHub release tag name
github_repository(str): GitHub repository name
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
HttpUpload.__init__(self, architecture, configuration, section)
Upload.__init__(self, repository_id, configuration)
HttpUpload.__init__(self, configuration, section)
self.github_owner = configuration.get(section, "owner")
self.github_repository = configuration.get(section, "repository")
if configuration.getboolean(section, "use_full_release_name", fallback=False):
self.github_release_tag = f"{repository_id.name}-{repository_id.architecture}"
self.github_release_tag_name = f"{repository_id.name} {repository_id.architecture}"
else:
self.github_release_tag_name = self.github_release_tag = repository_id.architecture
def asset_remove(self, release: dict[str, Any], name: str) -> None:
"""
remove asset from the release by name
@ -136,7 +148,10 @@ class Github(HttpUpload):
dict[str, Any]: GitHub API release object for the new release
"""
url = f"https://api.github.com/repos/{self.github_owner}/{self.github_repository}/releases"
response = self.make_request("POST", url, json={"tag_name": self.architecture, "name": self.architecture})
response = self.make_request("POST", url, json={
"tag_name": self.github_release_tag,
"name": self.github_release_tag_name,
})
release: dict[str, Any] = response.json()
return release
@ -147,7 +162,7 @@ class Github(HttpUpload):
Returns:
dict[str, Any] | None: GitHub API release object if release found and None otherwise
"""
url = f"https://api.github.com/repos/{self.github_owner}/{self.github_repository}/releases/tags/{self.architecture}"
url = f"https://api.github.com/repos/{self.github_owner}/{self.github_repository}/releases/tags/{self.github_release_tag}"
try:
response = self.make_request("GET", url)
release: dict[str, Any] = response.json()

16
src/ahriman/core/upload/http_upload.py
View File

@ -21,28 +21,14 @@ import hashlib
from pathlib import Path
from ahriman.core.configuration import Configuration
from ahriman.core.http import SyncHttpClient
from ahriman.core.upload.upload import Upload
class HttpUpload(Upload, SyncHttpClient):
class HttpUpload(SyncHttpClient):
"""
helper for the http based uploads
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
configuration(Configuration): configuration instance
section(str): configuration section name
"""
Upload.__init__(self, architecture, configuration)
SyncHttpClient.__init__(self, section, configuration)
@staticmethod
def calculate_hash(path: Path) -> str:
"""

11
src/ahriman/core/upload/remote_service.py
View File

@ -27,10 +27,12 @@ from ahriman.core.http import MultipartType
from ahriman.core.sign.gpg import GPG
from ahriman.core.status.web_client import WebClient
from ahriman.core.upload.http_upload import HttpUpload
from ahriman.core.upload.upload import Upload
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class RemoteService(HttpUpload):
class RemoteService(Upload, HttpUpload):
"""
upload files to another server instance
@ -38,16 +40,17 @@ class RemoteService(HttpUpload):
client(WebClient): web client instance
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
HttpUpload.__init__(self, architecture, configuration, section)
Upload.__init__(self, repository_id, configuration)
HttpUpload.__init__(self, configuration, section)
self.client = WebClient(configuration)
@cached_property

7
src/ahriman/core/upload/rsync.py
View File

@ -23,6 +23,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.upload.upload import Upload
from ahriman.core.util import check_output
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class Rsync(Upload):
@ -36,16 +37,16 @@ class Rsync(Upload):
_check_output = check_output
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Upload.__init__(self, architecture, configuration)
Upload.__init__(self, repository_id, configuration)
self.command = configuration.getlist(section, "command")
self.remote = configuration.get(section, "remote")

17
src/ahriman/core/upload/s3.py
View File

@ -28,6 +28,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.upload.upload import Upload
from ahriman.core.util import walk
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
class S3(Upload):
@ -37,21 +38,25 @@ class S3(Upload):
Attributes
bucket(Any): boto3 S3 bucket object
chunk_size(int): chunk size for calculating checksums
remote_root(Path): relative path to which packages will be uploaded
"""
def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration, section: str) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
section(str): settings section name
"""
Upload.__init__(self, architecture, configuration)
Upload.__init__(self, repository_id, configuration)
self.bucket = self.get_bucket(configuration, section)
self.chunk_size = configuration.getint(section, "chunk_size", fallback=8 * 1024 * 1024)
paths = configuration.repository_paths
self.remote_root = paths.repository.relative_to(paths.root / "repository")
@staticmethod
def calculate_etag(path: Path, chunk_size: int) -> str:
"""
@ -127,7 +132,7 @@ class S3(Upload):
continue
local_path = path / local_file
remote_path = Path(self.architecture) / local_file
remote_path = self.remote_root / local_file
(mime, _) = mimetypes.guess_type(local_path)
extra_args = {"ContentType": mime} if mime is not None else None
@ -155,8 +160,8 @@ class S3(Upload):
Returns:
dict[Path, Any]: map of path object to the remote s3 object
"""
objects = self.bucket.objects.filter(Prefix=self.architecture)
return {Path(item.key).relative_to(self.architecture): item for item in objects}
objects = self.bucket.objects.filter(Prefix=str(self.remote_root))
return {Path(item.key).relative_to(self.remote_root): item for item in objects}
def sync(self, path: Path, built_packages: list[Package]) -> None:
"""

31
src/ahriman/core/upload/upload.py
View File

@ -25,6 +25,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.exceptions import SynchronizationError
from ahriman.core.log import LazyLogging
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.upload_settings import UploadSettings
@ -33,18 +34,16 @@ class Upload(LazyLogging):
base remote sync class
Attributes:
architecture(str): repository architecture
configuration(Configuration): configuration instance
repository_id(RepositoryId): repository unique identifier
Examples:
These classes provide the way to upload packages to remote sources as it is described in their implementations.
Basic flow includes class instantiating by using the ``load`` method and then calling the ``run`` method which
wraps any internal exceptions into the ``SyncFailed`` exception::
>>> from ahriman.core.configuration import Configuration
>>>
>>> configuration = Configuration()
>>> upload = Upload.load("x86_64", configuration, "s3")
>>> upload = Upload.load(RepositoryId("x86_64", "aur-clone"), configuration, "s3")
>>> upload.run(configuration.repository_paths.repository, [])
Or in case if direct access to exception is required, the ``sync`` method can be used::
@ -55,46 +54,46 @@ class Upload(LazyLogging):
>>> handle_exceptions(ex)
"""
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
self.architecture = architecture
self.repository_id = repository_id
self.configuration = configuration
@staticmethod
def load(architecture: str, configuration: Configuration, target: str) -> Upload:
def load(repository_id: RepositoryId, configuration: Configuration, target: str) -> Upload:
"""
load client from settings
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
target(str): target to run sync (e.g. s3)
Returns:
Upload: client according to current settings
"""
section, provider_name = configuration.gettype(target, architecture)
section, provider_name = configuration.gettype(target, repository_id)
match UploadSettings.from_option(provider_name):
case UploadSettings.Rsync:
from ahriman.core.upload.rsync import Rsync
return Rsync(architecture, configuration, section)
return Rsync(repository_id, configuration, section)
case UploadSettings.S3:
from ahriman.core.upload.s3 import S3
return S3(architecture, configuration, section)
return S3(repository_id, configuration, section)
case UploadSettings.GitHub:
from ahriman.core.upload.github import Github
return Github(architecture, configuration, section)
from ahriman.core.upload.github import GitHub
return GitHub(repository_id, configuration, section)
case UploadSettings.RemoteService:
from ahriman.core.upload.remote_service import RemoteService
return RemoteService(architecture, configuration, section)
return RemoteService(repository_id, configuration, section)
case _:
return Upload(architecture, configuration) # should never happen
return Upload(repository_id, configuration) # should never happen
def run(self, path: Path, built_packages: list[Package]) -> None:
"""

13
src/ahriman/core/upload/upload_trigger.py
View File

@ -21,6 +21,7 @@ from ahriman.core.configuration import Configuration
from ahriman.core.triggers import Trigger
from ahriman.core.upload.upload import Upload
from ahriman.models.package import Package
from ahriman.models.repository_id import RepositoryId
from ahriman.models.result import Result
@ -67,6 +68,10 @@ class UploadTrigger(Trigger):
"coerce": "integer",
"min": 0,
},
"use_full_release_name": {
"type": "boolean",
"coerce": "boolean",
},
"username": {
"type": "string",
},
@ -138,15 +143,15 @@ class UploadTrigger(Trigger):
},
}
def __init__(self, architecture: str, configuration: Configuration) -> None:
def __init__(self, repository_id: RepositoryId, configuration: Configuration) -> None:
"""
default constructor
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
"""
Trigger.__init__(self, architecture, configuration)
Trigger.__init__(self, repository_id, configuration)
self.targets = self.configuration_sections(configuration)
@classmethod
@ -171,5 +176,5 @@ class UploadTrigger(Trigger):
packages(list[Package]): list of all available packages
"""
for target in self.targets:
runner = Upload.load(self.architecture, self.configuration, target)
runner = Upload.load(self.repository_id, self.configuration, target)
runner.run(self.configuration.repository_paths.repository, result.success)

3
src/ahriman/models/aur_package.py
View File

@ -68,9 +68,10 @@ class AURPackage:
>>>
>>> from ahriman.core.alpm.pacman import Pacman
>>> from ahriman.core.configuration import Configuration
>>> from ahriman.models.repository_id import RepositoryId
>>>
>>> configuration = Configuration()
>>> pacman = Pacman("x86_64", configuration)
>>> pacman = Pacman(RepositoryId("x86_64", "aur-clone"), configuration)
>>> metadata = pacman.package_get("pacman")
>>> package = AURPackage.from_pacman(next(metadata)) # load package from pyalpm wrapper
"""

3
src/ahriman/models/package_description.py
View File

@ -56,9 +56,10 @@ class PackageDescription:
>>> from pathlib import Path
>>> from ahriman.core.alpm.pacman import Pacman
>>> from ahriman.core.configuration import Configuration
>>> from ahriman.models.repository_id import RepositoryId
>>>
>>> configuration = Configuration()
>>> pacman = Pacman("x86_64", configuration)
>>> pacman = Pacman(RepositoryId("x86_64", "aur-clone"), configuration)
>>> pyalpm_description = next(package for package in pacman.package_get("pacman"))
>>> description = PackageDescription.from_package(
>>> pyalpm_description, Path("/var/cache/pacman/pkg/pacman-6.0.1-4-x86_64.pkg.tar.zst"))

65
src/ahriman/models/repository_id.py Normal file
View File

@ -0,0 +1,65 @@
#
# Copyright (c) 2021-2023 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
from dataclasses import dataclass
from typing import Any
from ahriman.core.exceptions import InitializeError
@dataclass(frozen=True)
class RepositoryId:
"""
unique identifier of the repository
Attributes:
architecture(str): repository architecture
name(str): repository name
"""
architecture: str
name: str
def __post_init__(self) -> None:
"""
check that name is set
Raises:
InitializeError: in case if name is not set
"""
if not self.name:
raise InitializeError("Repository name is not set")
def __lt__(self, other: Any) -> bool:
"""
comparison operator for sorting
Args:
other(Any): other object to compare
Returns:
bool: True in case if this is less than other and False otherwise
Raises:
TypeError: if other is different from RepositoryId type
"""
if not isinstance(other, RepositoryId):
raise ValueError(f"'<' not supported between instances of '{type(self)}' and '{type(other)}'")
return self.name <= other.name and self.architecture < other.architecture

56
src/ahriman/models/repository_paths.py
View File

@ -20,25 +20,29 @@
import os
import shutil
from collections.abc import Generator
from dataclasses import dataclass
from functools import cached_property
from pathlib import Path
from ahriman.core.exceptions import PathError
from ahriman.core.log import LazyLogging
from ahriman.models.repository_id import RepositoryId
@dataclass(frozen=True)
class RepositoryPaths:
class RepositoryPaths(LazyLogging):
"""
repository paths holder. For the most operations with paths you want to use this object
Attributes:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
root(Path): repository root (i.e. ahriman home)
Examples:
This class can be used in order to access the repository tree structure::
>>> paths = RepositoryPaths(Path("/var/lib/ahriman"), "x86_64")
>>> paths = RepositoryPaths(Path("/var/lib/ahriman"), RepositoryId("x86_64", "aur-clone"))
Additional methods can be used in order to ensure that tree is created::
@ -51,7 +55,21 @@ class RepositoryPaths:
"""
root: Path
architecture: str
repository_id: RepositoryId
@cached_property
def _suffix(self) -> Path:
"""
suffix of the paths as defined by repository structure
Returns:
Path: relative path which contains only architecture segment in case if legacy tree is used and repository
name and architecture otherwise
"""
if (self.root / "repository" / self.repository_id.architecture).is_dir():
self.logger.warning("legacy single repository tree has been found")
return Path(self.repository_id.architecture)
return Path(self.repository_id.name) / self.repository_id.architecture
@property
def cache(self) -> Path:
@ -72,7 +90,7 @@ class RepositoryPaths:
Path: full patch to devtools chroot directory
"""
# for the chroot directory devtools will create own tree, and we don"t have to specify architecture here
return self.root / "chroot"
return self.root / "chroot" / self.repository_id.name
@property
def packages(self) -> Path:
@ -82,7 +100,7 @@ class RepositoryPaths:
Returns:
Path: full path to built packages directory
"""
return self.root / "packages" / self.architecture
return self.root / "packages" / self._suffix
@property
def pacman(self) -> Path:
@ -92,7 +110,7 @@ class RepositoryPaths:
Returns:
Path: full path to pacman local database cache
"""
return self.root / "pacman" / self.architecture
return self.root / "pacman" / self._suffix
@property
def repository(self) -> Path:
@ -102,7 +120,7 @@ class RepositoryPaths:
Returns:
Path: full path to the repository directory
"""
return self.root / "repository" / self.architecture
return self.root / "repository" / self._suffix
@property
def root_owner(self) -> tuple[int, int]:
@ -115,22 +133,28 @@ class RepositoryPaths:
return self.owner(self.root)
@classmethod
def known_architectures(cls, root: Path) -> set[str]:
def known_architectures(cls, root: Path, name: str) -> set[RepositoryId]:
"""
get known architectures
Args:
root(Path): repository root
name(str): repository name from configuration
Returns:
set[str]: list of architectures for which tree is created
set[RepositoryId]: list of tuple of repository name and architectures for which tree is created
"""
paths = cls(root, "")
return {
path.name
for path in paths.repository.iterdir()
if path.is_dir()
}
def walk(repository_path: Path, repository_name: str) -> Generator[RepositoryId, None, None]:
for architecture in filter(lambda path: path.is_dir(), repository_path.iterdir()):
yield RepositoryId(architecture.name, repository_name)
def walk_with_name(paths: RepositoryPaths) -> Generator[RepositoryId, None, None]:
for repository in filter(lambda path: path.is_dir(), paths.repository.iterdir()):
yield from walk(repository, repository.name)
instance = cls(root, RepositoryId("", ""))
# try to get list per repository first and then fallback to old schema if nothing found
return set(walk_with_name(instance)) or set(walk(instance.repository, name))
@staticmethod
def owner(path: Path) -> tuple[int, int]:

4
src/ahriman/web/views/v1/status/status.py
View File

@ -65,9 +65,9 @@ class StatusView(BaseView):
counters = Counters.from_packages(self.service.packages)
status = InternalStatus(
status=self.service.status,
architecture=self.service.architecture,
architecture=self.service.repository_id.architecture,
packages=counters,
repository=self.service.repository.name,
repository=self.service.repository_id.name,
version=__version__,
)

7
src/ahriman/web/web.py
View File

@ -31,6 +31,7 @@ from ahriman.core.exceptions import InitializeError
from ahriman.core.log.filtered_access_logger import FilteredAccessLogger
from ahriman.core.spawn import Spawn
from ahriman.core.status.watcher import Watcher
from ahriman.models.repository_id import RepositoryId
from ahriman.web.apispec import setup_apispec
from ahriman.web.cors import setup_cors
from ahriman.web.middlewares.exception_handler import exception_handler
@ -120,12 +121,12 @@ def run_server(application: Application) -> None:
access_log=logging.getLogger("http"), access_log_class=FilteredAccessLogger)
def setup_service(architecture: str, configuration: Configuration, spawner: Spawn) -> Application:
def setup_service(repository_id: RepositoryId, configuration: Configuration, spawner: Spawn) -> Application:
"""
create web application
Args:
architecture(str): repository architecture
repository_id(RepositoryId): repository unique identifier
configuration(Configuration): configuration instance
spawner(Spawn): spawner thread
@ -155,7 +156,7 @@ def setup_service(architecture: str, configuration: Configuration, spawner: Spaw
database = application["database"] = SQLite.load(configuration)
application.logger.info("setup watcher")
application["watcher"] = Watcher(architecture, configuration, database)
application["watcher"] = Watcher(repository_id, configuration, database)
application.logger.info("setup process spawner")
application["spawn"] = spawner

9
tests/ahriman/application/application/conftest.py
View File

@ -27,7 +27,8 @@ def application_packages(configuration: Configuration, database: SQLite, reposit
"""
mocker.patch("ahriman.core.repository.Repository.load", return_value=repository)
mocker.patch("ahriman.core.database.SQLite.load", return_value=database)
return ApplicationPackages("x86_64", configuration, report=False)
_, repository_id = configuration.check_loaded()
return ApplicationPackages(repository_id, configuration, report=False)
@pytest.fixture
@ -47,7 +48,8 @@ def application_properties(configuration: Configuration, database: SQLite, repos
"""
mocker.patch("ahriman.core.repository.Repository.load", return_value=repository)
mocker.patch("ahriman.core.database.SQLite.load", return_value=database)
return ApplicationProperties("x86_64", configuration, report=False)
_, repository_id = configuration.check_loaded()
return ApplicationProperties(repository_id, configuration, report=False)
@pytest.fixture
@ -67,4 +69,5 @@ def application_repository(configuration: Configuration, database: SQLite, repos
"""
mocker.patch("ahriman.core.repository.Repository.load", return_value=repository)
mocker.patch("ahriman.core.database.SQLite.load", return_value=database)
return ApplicationRepository("x86_64", configuration, report=False)
_, repository_id = configuration.check_loaded()
return ApplicationRepository(repository_id, configuration, report=False)

6
tests/ahriman/application/conftest.py
View File

@ -28,7 +28,8 @@ def application(configuration: Configuration, repository: Repository, database:
"""
mocker.patch("ahriman.core.database.SQLite.load", return_value=database)
mocker.patch("ahriman.core.repository.Repository.load", return_value=repository)
return Application("x86_64", configuration, report=False)
_, repository_id = configuration.check_loaded()
return Application(repository_id, configuration, report=False)
@pytest.fixture
@ -54,7 +55,8 @@ def lock(args: argparse.Namespace, configuration: Configuration) -> Lock:
Returns:
Lock: file lock test instance
"""
return Lock(args, "x86_64", configuration)
_, repository_id = configuration.check_loaded()
return Lock(args, repository_id, configuration)
@pytest.fixture

34
tests/ahriman/conftest.py
View File

@ -19,7 +19,9 @@ from ahriman.models.build_status import BuildStatus, BuildStatusEnum
from ahriman.models.package import Package
from ahriman.models.package_description import PackageDescription
from ahriman.models.package_source import PackageSource
from ahriman.models.pacman_synchronization import PacmanSynchronization
from ahriman.models.remote_source import RemoteSource
from ahriman.models.repository_id import RepositoryId
from ahriman.models.repository_paths import RepositoryPaths
from ahriman.models.result import Result
from ahriman.models.user import User
@ -243,18 +245,19 @@ def auth(configuration: Configuration) -> Auth:
@pytest.fixture
def configuration(resource_path_root: Path) -> Configuration:
def configuration(repository_id: RepositoryId, resource_path_root: Path) -> Configuration:
"""
configuration fixture
Args:
repository_id(RepositoryId): repository identifier fixture
resource_path_root(Path): resource path root directory
Returns:
Configuration: configuration test instance
"""
path = resource_path_root / "core" / "ahriman.ini"
return Configuration.from_path(path=path, architecture="x86_64")
return Configuration.from_path(path, repository_id)
@pytest.fixture
@ -434,7 +437,8 @@ def pacman(configuration: Configuration) -> Pacman:
Returns:
Pacman: pacman wrapper test instance
"""
return Pacman("x86_64", configuration, refresh_database=0)
_, repository_id = configuration.check_loaded()
return Pacman(repository_id, configuration, refresh_database=PacmanSynchronization.Disabled)
@pytest.fixture
@ -481,7 +485,19 @@ def repository(configuration: Configuration, database: SQLite, mocker: MockerFix
Repository: repository test instance
"""
mocker.patch("ahriman.core.repository.Repository._set_context")
return Repository.load("x86_64", configuration, database, report=False)
_, repository_id = configuration.check_loaded()
return Repository.load(repository_id, configuration, database, report=False)
@pytest.fixture
def repository_id() -> RepositoryId:
"""
fixture for repository identifier
Returns:
RepositoryId: repository identifier test instance
"""
return RepositoryId("x86_64", "aur-clone")
@pytest.fixture
@ -525,7 +541,12 @@ def spawner(configuration: Configuration) -> Spawn:
Returns:
Spawn: spawner fixture
"""
return Spawn(MagicMock(), "x86_64", ["--architecture", "x86_64", "--configuration", str(configuration.path)])
_, repository_id = configuration.check_loaded()
return Spawn(MagicMock(), repository_id, [
"--architecture", "x86_64",
"--repository", repository_id.name,
"--configuration", str(configuration.path),
])
@pytest.fixture
@ -554,4 +575,5 @@ def watcher(configuration: Configuration, database: SQLite, repository: Reposito
Watcher: package status watcher test instance
"""
mocker.patch("ahriman.core.repository.Repository.load", return_value=repository)
return Watcher("x86_64", configuration, database)
_, repository_id = configuration.check_loaded()
return Watcher(repository_id, configuration, database)

20
tests/ahriman/core/log/test_log.py → tests/ahriman/core/log/test_log_loader.py
View File

@ -7,7 +7,7 @@ from pytest_mock import MockerFixture
from systemd.journal import JournalHandler
from ahriman.core.configuration import Configuration
from ahriman.core.log import Log
from ahriman.core.log.log_loader import LogLoader
from ahriman.models.log_handler import LogHandler
@ -15,14 +15,14 @@ def test_handler() -> None:
"""
must extract journald handler if available
"""
assert Log.handler(None) == LogHandler.Journald
assert LogLoader.handler(None) == LogHandler.Journald
def test_handler_selected() -> None:
"""
must return selected log handler
"""
assert Log.handler(LogHandler.Console) == LogHandler.Console
assert LogLoader.handler(LogHandler.Console) == LogHandler.Console
def test_handler_syslog(mocker: MockerFixture) -> None:
@ -31,7 +31,7 @@ def test_handler_syslog(mocker: MockerFixture) -> None:
"""
mocker.patch("pathlib.Path.exists", return_value=True)
mocker.patch.dict(sys.modules, {"systemd.journal": None})
assert Log.handler(None) == LogHandler.Syslog
assert LogLoader.handler(None) == LogHandler.Syslog
def test_handler_console(mocker: MockerFixture) -> None:
@ -40,17 +40,17 @@ def test_handler_console(mocker: MockerFixture) -> None:
"""
mocker.patch("pathlib.Path.exists", return_value=False)
mocker.patch.dict(sys.modules, {"systemd.journal": None})
assert Log.handler(None) == LogHandler.Console
assert LogLoader.handler(None) == LogHandler.Console
def test_load(configuration: Configuration, mocker: MockerFixture) -> None:
"""
must load logging
"""
logging_mock = mocker.patch("ahriman.core.log.log.fileConfig", side_effect=fileConfig)
logging_mock = mocker.patch("ahriman.core.log.log_loader.fileConfig", side_effect=fileConfig)
http_log_mock = mocker.patch("ahriman.core.log.http_log_handler.HttpLogHandler.load")
Log.load(configuration, LogHandler.Journald, quiet=False, report=False)
LogLoader.load(configuration, LogHandler.Journald, quiet=False, report=False)
logging_mock.assert_called_once_with(pytest.helpers.anyvar(int), disable_existing_loggers=True)
http_log_mock.assert_called_once_with(configuration, report=False)
assert all(isinstance(handler, JournalHandler) for handler in logging.getLogger().handlers)
@ -60,8 +60,8 @@ def test_load_fallback(configuration: Configuration, mocker: MockerFixture) -> N
"""
must fall back to stderr without errors
"""
mocker.patch("ahriman.core.log.log.fileConfig", side_effect=PermissionError())
Log.load(configuration, LogHandler.Journald, quiet=False, report=False)
mocker.patch("ahriman.core.log.log_loader.fileConfig", side_effect=PermissionError())
LogLoader.load(configuration, LogHandler.Journald, quiet=False, report=False)
def test_load_quiet(configuration: Configuration, mocker: MockerFixture) -> None:
@ -69,5 +69,5 @@ def test_load_quiet(configuration: Configuration, mocker: MockerFixture) -> None
must disable logging in case if quiet flag set
"""
disable_mock = mocker.patch("logging.disable")
Log.load(configuration, LogHandler.Journald, quiet=True, report=False)
LogLoader.load(configuration, LogHandler.Journald, quiet=True, report=False)
disable_mock.assert_called_once_with(logging.WARNING)

3
tests/ahriman/core/report/conftest.py
View File

@ -17,4 +17,5 @@ def remote_call(configuration: Configuration) -> RemoteCall:
"""
configuration.set_option("web", "host", "localhost")
configuration.set_option("web", "port", "8080")
return RemoteCall("x86_64", configuration, "remote-call")
_, repository_id = configuration.check_loaded()
return RemoteCall(repository_id, configuration, "remote-call")

9
tests/ahriman/core/repository/conftest.py
View File

@ -22,7 +22,8 @@ def cleaner(configuration: Configuration, database: SQLite) -> Cleaner:
Returns:
Cleaner: cleaner test instance
"""
return Cleaner("x86_64", configuration, database, report=False,
_, repository_id = configuration.check_loaded()
return Cleaner(repository_id, configuration, database, report=False,
refresh_pacman_database=PacmanSynchronization.Disabled)
@ -43,7 +44,8 @@ def executor(configuration: Configuration, database: SQLite, mocker: MockerFixtu
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_chroot")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_packages")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_queue")
return Executor("x86_64", configuration, database, report=False,
_, repository_id = configuration.check_loaded()
return Executor(repository_id, configuration, database, report=False,
refresh_pacman_database=PacmanSynchronization.Disabled)
@ -64,5 +66,6 @@ def update_handler(configuration: Configuration, database: SQLite, mocker: Mocke
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_chroot")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_packages")
mocker.patch("ahriman.core.repository.cleaner.Cleaner.clear_queue")
return UpdateHandler("x86_64", configuration, database, report=False,
_, repository_id = configuration.check_loaded()
return UpdateHandler(repository_id, configuration, database, report=False,
refresh_pacman_database=PacmanSynchronization.Disabled)

6
tests/ahriman/core/triggers/conftest.py
View File

@ -15,7 +15,8 @@ def trigger(configuration: Configuration) -> Trigger:
Returns:
Trigger: trigger test instance
"""
return Trigger("x86_64", configuration)
_, repository_id = configuration.check_loaded()
return Trigger(repository_id, configuration)
@pytest.fixture
@ -28,4 +29,5 @@ def trigger_loader(configuration: Configuration) -> TriggerLoader:
Returns:
TriggerLoader: trigger loader test instance
"""
return TriggerLoader.load("x86_64", configuration)
_, repository_id = configuration.check_loaded()
return TriggerLoader.load(repository_id, configuration)

Some files were not shown because too many files have changed in this diff Show More