automatically bump pkgrel on version duplicates

The new --(no-)increment flag has been added to add, update and rebuild
subcommands. In case if it is true and package version is the same as in
repository, it will automatically bump pkgrel appending (increasing)
minor part of it (e.g. 1.0.0-1 -> 1.0.0-1.1).

Inn order to implement this, the shadow (e.g. it will not store it in
database) patch for pkgrel will be created
This commit is contained in:
Evgenii Alekseev 2023-08-07 00:04:08 +03:00
parent 368db86dde
commit 3b3ef43863
27 changed files with 288 additions and 51 deletions

View File

@ -21,4 +21,3 @@ python:
- docs
- s3
- web
system_packages: true

View File

@ -16,6 +16,7 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github
* VCS packages support.
* Official repository support.
* Ability to patch AUR packages and even create package from local PKGBUILDs.
* Various rebuild options with ability to automatically bump package version.
* Sign support with gpg (repository, package), multiple packagers support.
* Triggers for repository updates, e.g. synchronization to remote services (rsync, s3 and github) and report generation (email, html, telegram).
* Repository status interface with optional authorization and control options:

View File

@ -224,7 +224,7 @@ usage: ahriman help\-version [\-h]
print application and its dependencies versions
.SH COMMAND \fI\,'ahriman package\-add'\/\fR
usage: ahriman package\-add [\-h] [\-\-dependencies | \-\-no\-dependencies] [\-e] [\-n] [\-y]
usage: ahriman package\-add [\-h] [\-\-dependencies | \-\-no\-dependencies] [\-e] [\-\-increment | \-\-no\-increment] [\-n] [\-y]
[\-s {auto,archive,aur,directory,local,remote,repository}] [\-u USERNAME]
package [package ...]
@ -243,6 +243,10 @@ process missing package dependencies
\fB\-e\fR, \fB\-\-exit\-code\fR
return non\-zero exit status if result is empty
.TP
\fB\-\-increment\fR, \fB\-\-no\-increment\fR
increment package release (pkgrel) version on duplicate
.TP
\fB\-n\fR, \fB\-\-now\fR
run update function after
@ -459,8 +463,8 @@ fetch actual version of VCS packages
download fresh package databases from the mirror before actions, \-yy to force refresh even if up to date
.SH COMMAND \fI\,'ahriman repo\-rebuild'\/\fR
usage: ahriman repo\-rebuild [\-h] [\-\-depends\-on DEPENDS_ON] [\-\-dry\-run] [\-\-from\-database] [\-e]
[\-s {unknown,pending,building,failed,success}] [\-u USERNAME]
usage: ahriman repo\-rebuild [\-h] [\-\-depends\-on DEPENDS_ON] [\-\-dry\-run] [\-\-from\-database] [\-\-increment | \-\-no\-increment]
[\-e] [\-s {unknown,pending,building,failed,success}] [\-u USERNAME]
force rebuild whole repository
@ -479,6 +483,10 @@ read packages from database instead of filesystem. This feature in particular is
restore repository from another repository instance. Note, however, that in order to restore packages you need to have
original ahriman instance run with web service and have run repo\-update at least once.
.TP
\fB\-\-increment\fR, \fB\-\-no\-increment\fR
increment package release (pkgrel) on duplicate
.TP
\fB\-e\fR, \fB\-\-exit\-code\fR
return non\-zero exit status if result is empty
@ -560,7 +568,8 @@ instead of running all triggers as set by configuration, just process specified
.SH COMMAND \fI\,'ahriman repo\-update'\/\fR
usage: ahriman repo\-update [\-h] [\-\-aur | \-\-no\-aur] [\-\-dependencies | \-\-no\-dependencies] [\-\-dry\-run] [\-e]
[\-\-local | \-\-no\-local] [\-\-manual | \-\-no\-manual] [\-u USERNAME] [\-\-vcs | \-\-no\-vcs] [\-y]
[\-\-increment | \-\-no\-increment] [\-\-local | \-\-no\-local] [\-\-manual | \-\-no\-manual] [\-u USERNAME]
[\-\-vcs | \-\-no\-vcs] [\-y]
[package ...]
check for packages updates and run build process if requested
@ -586,6 +595,10 @@ just perform check for updates, same as check command
\fB\-e\fR, \fB\-\-exit\-code\fR
return non\-zero exit status if result is empty
.TP
\fB\-\-increment\fR, \fB\-\-no\-increment\fR
increment package release (pkgrel) on duplicate
.TP
\fB\-\-local\fR, \fB\-\-no\-local\fR
enable or disable checking of local packages for updates

View File

@ -168,6 +168,7 @@ This feature is divided into to stages: check AUR for updates and run rebuild fo
#. For each level of tree it does:
#. Download package data from AUR.
#. Bump ``pkgrel`` if there is duplicate version in the local repository (see explanation below).
#. Build every package in clean chroot.
#. Sign packages if required.
#. Add packages to database and sign database if required.
@ -175,6 +176,20 @@ This feature is divided into to stages: check AUR for updates and run rebuild fo
After any step any package data is being removed.
pkgrel bump rules
^^^^^^^^^^^^^^^^^
The application is able to automatically bump package release (``pkgrel``) during build process if there is duplicate version in repository. The version will be incremented as following:
#. Get version of the remote package.
#. Get version of the local package if any.
#. If local version is not set, proceed with remote one.
#. If local version is set and epoch or package version (``pkgver``) are different, proceed with remote version.
#. If local version is set and remote version is newer than local one, proceed with remote.
#. Extract ``pkgrel`` value.
#. If it has ``major.minor`` notation (e.g. ``1.1``), then increment last part by 1, e.g. ``1.1 -> 1.2``, ``1.0.1 -> 1.0.2``.
#. If ``pkgrel`` is a number (e.g. ``1``), then append 1 to the end of the string, e.g. ``1 -> 1.1``.
Core functions reference
------------------------

View File

@ -10,9 +10,9 @@ _shtab_ahriman_help_commands_unsafe_option_strings=('-h' '--help')
_shtab_ahriman_help_updates_option_strings=('-h' '--help' '-e' '--exit-code')
_shtab_ahriman_help_version_option_strings=('-h' '--help')
_shtab_ahriman_version_option_strings=('-h' '--help')
_shtab_ahriman_package_add_option_strings=('-h' '--help' '--dependencies' '--no-dependencies' '-e' '--exit-code' '-n' '--now' '-y' '--refresh' '-s' '--source' '-u' '--username')
_shtab_ahriman_add_option_strings=('-h' '--help' '--dependencies' '--no-dependencies' '-e' '--exit-code' '-n' '--now' '-y' '--refresh' '-s' '--source' '-u' '--username')
_shtab_ahriman_package_update_option_strings=('-h' '--help' '--dependencies' '--no-dependencies' '-e' '--exit-code' '-n' '--now' '-y' '--refresh' '-s' '--source' '-u' '--username')
_shtab_ahriman_package_add_option_strings=('-h' '--help' '--dependencies' '--no-dependencies' '-e' '--exit-code' '--increment' '--no-increment' '-n' '--now' '-y' '--refresh' '-s' '--source' '-u' '--username')
_shtab_ahriman_add_option_strings=('-h' '--help' '--dependencies' '--no-dependencies' '-e' '--exit-code' '--increment' '--no-increment' '-n' '--now' '-y' '--refresh' '-s' '--source' '-u' '--username')
_shtab_ahriman_package_update_option_strings=('-h' '--help' '--dependencies' '--no-dependencies' '-e' '--exit-code' '--increment' '--no-increment' '-n' '--now' '-y' '--refresh' '-s' '--source' '-u' '--username')
_shtab_ahriman_package_remove_option_strings=('-h' '--help')
_shtab_ahriman_remove_option_strings=('-h' '--help')
_shtab_ahriman_package_status_option_strings=('-h' '--help' '--ahriman' '-e' '--exit-code' '--info' '--no-info' '-s' '--status')
@ -31,8 +31,8 @@ _shtab_ahriman_repo_create_keyring_option_strings=('-h' '--help')
_shtab_ahriman_repo_create_mirrorlist_option_strings=('-h' '--help')
_shtab_ahriman_repo_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--dependencies' '--no-dependencies' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh')
_shtab_ahriman_daemon_option_strings=('-h' '--help' '-i' '--interval' '--aur' '--no-aur' '--dependencies' '--no-dependencies' '--local' '--no-local' '--manual' '--no-manual' '--vcs' '--no-vcs' '-y' '--refresh')
_shtab_ahriman_repo_rebuild_option_strings=('-h' '--help' '--depends-on' '--dry-run' '--from-database' '-e' '--exit-code' '-s' '--status' '-u' '--username')
_shtab_ahriman_rebuild_option_strings=('-h' '--help' '--depends-on' '--dry-run' '--from-database' '-e' '--exit-code' '-s' '--status' '-u' '--username')
_shtab_ahriman_repo_rebuild_option_strings=('-h' '--help' '--depends-on' '--dry-run' '--from-database' '--increment' '--no-increment' '-e' '--exit-code' '-s' '--status' '-u' '--username')
_shtab_ahriman_rebuild_option_strings=('-h' '--help' '--depends-on' '--dry-run' '--from-database' '--increment' '--no-increment' '-e' '--exit-code' '-s' '--status' '-u' '--username')
_shtab_ahriman_repo_remove_unknown_option_strings=('-h' '--help' '--dry-run')
_shtab_ahriman_remove_unknown_option_strings=('-h' '--help' '--dry-run')
_shtab_ahriman_repo_report_option_strings=('-h' '--help')
@ -45,8 +45,8 @@ _shtab_ahriman_repo_sync_option_strings=('-h' '--help')
_shtab_ahriman_sync_option_strings=('-h' '--help')
_shtab_ahriman_repo_tree_option_strings=('-h' '--help')
_shtab_ahriman_repo_triggers_option_strings=('-h' '--help')
_shtab_ahriman_repo_update_option_strings=('-h' '--help' '--aur' '--no-aur' '--dependencies' '--no-dependencies' '--dry-run' '-e' '--exit-code' '--local' '--no-local' '--manual' '--no-manual' '-u' '--username' '--vcs' '--no-vcs' '-y' '--refresh')
_shtab_ahriman_update_option_strings=('-h' '--help' '--aur' '--no-aur' '--dependencies' '--no-dependencies' '--dry-run' '-e' '--exit-code' '--local' '--no-local' '--manual' '--no-manual' '-u' '--username' '--vcs' '--no-vcs' '-y' '--refresh')
_shtab_ahriman_repo_update_option_strings=('-h' '--help' '--aur' '--no-aur' '--dependencies' '--no-dependencies' '--dry-run' '-e' '--exit-code' '--increment' '--no-increment' '--local' '--no-local' '--manual' '--no-manual' '-u' '--username' '--vcs' '--no-vcs' '-y' '--refresh')
_shtab_ahriman_update_option_strings=('-h' '--help' '--aur' '--no-aur' '--dependencies' '--no-dependencies' '--dry-run' '-e' '--exit-code' '--increment' '--no-increment' '--local' '--no-local' '--manual' '--no-manual' '-u' '--username' '--vcs' '--no-vcs' '-y' '--refresh')
_shtab_ahriman_service_clean_option_strings=('-h' '--help' '--cache' '--no-cache' '--chroot' '--no-chroot' '--manual' '--no-manual' '--packages' '--no-packages' '--pacman' '--no-pacman')
_shtab_ahriman_clean_option_strings=('-h' '--help' '--cache' '--no-cache' '--chroot' '--no-chroot' '--manual' '--no-manual' '--packages' '--no-packages' '--pacman' '--no-pacman')
_shtab_ahriman_repo_clean_option_strings=('-h' '--help' '--cache' '--no-cache' '--chroot' '--no-chroot' '--manual' '--no-manual' '--packages' '--no-packages' '--pacman' '--no-pacman')
@ -151,6 +151,8 @@ _shtab_ahriman_package_add___dependencies_nargs=0
_shtab_ahriman_package_add___no_dependencies_nargs=0
_shtab_ahriman_package_add__e_nargs=0
_shtab_ahriman_package_add___exit_code_nargs=0
_shtab_ahriman_package_add___increment_nargs=0
_shtab_ahriman_package_add___no_increment_nargs=0
_shtab_ahriman_package_add__n_nargs=0
_shtab_ahriman_package_add___now_nargs=0
_shtab_ahriman_package_add__y_nargs=0
@ -162,6 +164,8 @@ _shtab_ahriman_add___dependencies_nargs=0
_shtab_ahriman_add___no_dependencies_nargs=0
_shtab_ahriman_add__e_nargs=0
_shtab_ahriman_add___exit_code_nargs=0
_shtab_ahriman_add___increment_nargs=0
_shtab_ahriman_add___no_increment_nargs=0
_shtab_ahriman_add__n_nargs=0
_shtab_ahriman_add___now_nargs=0
_shtab_ahriman_add__y_nargs=0
@ -173,6 +177,8 @@ _shtab_ahriman_package_update___dependencies_nargs=0
_shtab_ahriman_package_update___no_dependencies_nargs=0
_shtab_ahriman_package_update__e_nargs=0
_shtab_ahriman_package_update___exit_code_nargs=0
_shtab_ahriman_package_update___increment_nargs=0
_shtab_ahriman_package_update___no_increment_nargs=0
_shtab_ahriman_package_update__n_nargs=0
_shtab_ahriman_package_update___now_nargs=0
_shtab_ahriman_package_update__y_nargs=0
@ -274,12 +280,16 @@ _shtab_ahriman_repo_rebuild__h_nargs=0
_shtab_ahriman_repo_rebuild___help_nargs=0
_shtab_ahriman_repo_rebuild___dry_run_nargs=0
_shtab_ahriman_repo_rebuild___from_database_nargs=0
_shtab_ahriman_repo_rebuild___increment_nargs=0
_shtab_ahriman_repo_rebuild___no_increment_nargs=0
_shtab_ahriman_repo_rebuild__e_nargs=0
_shtab_ahriman_repo_rebuild___exit_code_nargs=0
_shtab_ahriman_rebuild__h_nargs=0
_shtab_ahriman_rebuild___help_nargs=0
_shtab_ahriman_rebuild___dry_run_nargs=0
_shtab_ahriman_rebuild___from_database_nargs=0
_shtab_ahriman_rebuild___increment_nargs=0
_shtab_ahriman_rebuild___no_increment_nargs=0
_shtab_ahriman_rebuild__e_nargs=0
_shtab_ahriman_rebuild___exit_code_nargs=0
_shtab_ahriman_repo_remove_unknown__h_nargs=0
@ -321,6 +331,8 @@ _shtab_ahriman_repo_update___no_dependencies_nargs=0
_shtab_ahriman_repo_update___dry_run_nargs=0
_shtab_ahriman_repo_update__e_nargs=0
_shtab_ahriman_repo_update___exit_code_nargs=0
_shtab_ahriman_repo_update___increment_nargs=0
_shtab_ahriman_repo_update___no_increment_nargs=0
_shtab_ahriman_repo_update___local_nargs=0
_shtab_ahriman_repo_update___no_local_nargs=0
_shtab_ahriman_repo_update___manual_nargs=0
@ -339,6 +351,8 @@ _shtab_ahriman_update___no_dependencies_nargs=0
_shtab_ahriman_update___dry_run_nargs=0
_shtab_ahriman_update__e_nargs=0
_shtab_ahriman_update___exit_code_nargs=0
_shtab_ahriman_update___increment_nargs=0
_shtab_ahriman_update___no_increment_nargs=0
_shtab_ahriman_update___local_nargs=0
_shtab_ahriman_update___no_local_nargs=0
_shtab_ahriman_update___manual_nargs=0

View File

@ -92,6 +92,7 @@ _shtab_ahriman_add_options=(
"(- : *)"{-h,--help}"[show this help message and exit]"
{--dependencies,--no-dependencies}"[process missing package dependencies (default\: True)]:dependencies:"
{-e,--exit-code}"[return non-zero exit status if result is empty (default\: False)]"
{--increment,--no-increment}"[increment package release (pkgrel) version on duplicate (default\: True)]:increment:"
{-n,--now}"[run update function after (default\: False)]"
"*"{-y,--refresh}"[download fresh package databases from the mirror before actions, -yy to force refresh even if up to date (default\: False)]"
{-s,--source}"[explicitly specify the package source for this command (default\: PackageSource.Auto)]:source:(auto archive aur directory local remote repository)"
@ -191,6 +192,7 @@ _shtab_ahriman_package_add_options=(
"(- : *)"{-h,--help}"[show this help message and exit]"
{--dependencies,--no-dependencies}"[process missing package dependencies (default\: True)]:dependencies:"
{-e,--exit-code}"[return non-zero exit status if result is empty (default\: False)]"
{--increment,--no-increment}"[increment package release (pkgrel) version on duplicate (default\: True)]:increment:"
{-n,--now}"[run update function after (default\: False)]"
"*"{-y,--refresh}"[download fresh package databases from the mirror before actions, -yy to force refresh even if up to date (default\: False)]"
{-s,--source}"[explicitly specify the package source for this command (default\: PackageSource.Auto)]:source:(auto archive aur directory local remote repository)"
@ -227,6 +229,7 @@ _shtab_ahriman_package_update_options=(
"(- : *)"{-h,--help}"[show this help message and exit]"
{--dependencies,--no-dependencies}"[process missing package dependencies (default\: True)]:dependencies:"
{-e,--exit-code}"[return non-zero exit status if result is empty (default\: False)]"
{--increment,--no-increment}"[increment package release (pkgrel) version on duplicate (default\: True)]:increment:"
{-n,--now}"[run update function after (default\: False)]"
"*"{-y,--refresh}"[download fresh package databases from the mirror before actions, -yy to force refresh even if up to date (default\: False)]"
{-s,--source}"[explicitly specify the package source for this command (default\: PackageSource.Auto)]:source:(auto archive aur directory local remote repository)"
@ -265,6 +268,7 @@ _shtab_ahriman_rebuild_options=(
"*--depends-on[only rebuild packages that depend on specified packages (default\: None)]:depends_on:"
"--dry-run[just perform check for packages without rebuild process itself (default\: False)]"
"--from-database[read packages from database instead of filesystem. This feature in particular is required in case if you would like to restore repository from another repository instance. Note, however, that in order to restore packages you need to have original ahriman instance run with web service and have run repo-update at least once. (default\: False)]"
{--increment,--no-increment}"[increment package release (pkgrel) on duplicate (default\: True)]:increment:"
{-e,--exit-code}"[return non-zero exit status if result is empty (default\: False)]"
{-s,--status}"[filter packages by status. Requires --from-database to be set (default\: None)]:status:(unknown pending building failed success)"
{-u,--username}"[build as user (default\: None)]:username:"
@ -353,6 +357,7 @@ _shtab_ahriman_repo_rebuild_options=(
"*--depends-on[only rebuild packages that depend on specified packages (default\: None)]:depends_on:"
"--dry-run[just perform check for packages without rebuild process itself (default\: False)]"
"--from-database[read packages from database instead of filesystem. This feature in particular is required in case if you would like to restore repository from another repository instance. Note, however, that in order to restore packages you need to have original ahriman instance run with web service and have run repo-update at least once. (default\: False)]"
{--increment,--no-increment}"[increment package release (pkgrel) on duplicate (default\: True)]:increment:"
{-e,--exit-code}"[return non-zero exit status if result is empty (default\: False)]"
{-s,--status}"[filter packages by status. Requires --from-database to be set (default\: None)]:status:(unknown pending building failed success)"
{-u,--username}"[build as user (default\: None)]:username:"
@ -419,6 +424,7 @@ _shtab_ahriman_repo_update_options=(
{--dependencies,--no-dependencies}"[process missing package dependencies (default\: True)]:dependencies:"
"--dry-run[just perform check for updates, same as check command (default\: False)]"
{-e,--exit-code}"[return non-zero exit status if result is empty (default\: False)]"
{--increment,--no-increment}"[increment package release (pkgrel) on duplicate (default\: True)]:increment:"
{--local,--no-local}"[enable or disable checking of local packages for updates (default\: True)]:local:"
{--manual,--no-manual}"[include or exclude manual updates (default\: True)]:manual:"
{-u,--username}"[build as user (default\: None)]:username:"
@ -538,6 +544,7 @@ _shtab_ahriman_update_options=(
{--dependencies,--no-dependencies}"[process missing package dependencies (default\: True)]:dependencies:"
"--dry-run[just perform check for updates, same as check command (default\: False)]"
{-e,--exit-code}"[return non-zero exit status if result is empty (default\: False)]"
{--increment,--no-increment}"[increment package release (pkgrel) on duplicate (default\: True)]:increment:"
{--local,--no-local}"[enable or disable checking of local packages for updates (default\: True)]:local:"
{--manual,--no-manual}"[include or exclude manual updates (default\: True)]:manual:"
{-u,--username}"[build as user (default\: None)]:username:"

View File

@ -305,7 +305,7 @@ TL;DR
sudo -u ahriman ahriman repo-rebuild --depends-on python
You can even rebuild the whole repository (which is particular useful in case if you would like to change packager) if you do not supply ``--depends-on`` option.
You can even rebuild the whole repository (which is particular useful in case if you would like to change packager) if you do not supply ``--depends-on`` option. This action will automatically increment ``pkgrel`` value; in case if you don't want to, the ``--no-increment`` option has to be supplied.
However, note that you do not need to rebuild repository in case if you just changed signing option, just use ``repo-sign`` command instead.

View File

@ -12,6 +12,7 @@ Features
* VCS packages support.
* Official repository support.
* Ability to patch AUR packages and even create package from local PKGBUILDs.
* Various rebuild options with ability to automatically bump package version.
* Sign support with gpg (repository, package), multiple packagers support.
* Triggers for repository updates, e.g. synchronization to remote services (rsync, s3 and github) and report generation (email, html, telegram).
* Repository status interface with optional authorization and control options.

View File

@ -256,6 +256,8 @@ def _set_package_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
parser.add_argument("--dependencies", help="process missing package dependencies",
action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
parser.add_argument("--increment", help="increment package release (pkgrel) version on duplicate",
action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("-n", "--now", help="run update function after", action="store_true")
parser.add_argument("-y", "--refresh", help="download fresh package databases from the mirror before actions, "
"-yy to force refresh even if up to date",
@ -577,6 +579,8 @@ def _set_repo_rebuild_parser(root: SubParserAction) -> argparse.ArgumentParser:
"instance. Note, however, that in order to restore packages you need to have original "
"ahriman instance run with web service and have run repo-update at least once.",
action="store_true")
parser.add_argument("--increment", help="increment package release (pkgrel) on duplicate",
action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
parser.add_argument("-s", "--status", help="filter packages by status. Requires --from-database to be set",
type=BuildStatusEnum, choices=enum_values(BuildStatusEnum))
@ -751,6 +755,8 @@ def _set_repo_update_parser(root: SubParserAction) -> argparse.ArgumentParser:
action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("--dry-run", help="just perform check for updates, same as check command", action="store_true")
parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
parser.add_argument("--increment", help="increment package release (pkgrel) on duplicate",
action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("--local", help="enable or disable checking of local packages for updates",
action=argparse.BooleanOptionalAction, default=True)
parser.add_argument("--manual", help="include or exclude manual updates",

View File

@ -123,7 +123,8 @@ class ApplicationRepository(ApplicationProperties):
result.extend(unknown_aur(package)) # local package not found
return result
def update(self, updates: Iterable[Package], packagers: Packagers | None = None) -> Result:
def update(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
bump_pkgrel: bool = False) -> Result:
"""
run package updates
@ -131,6 +132,7 @@ class ApplicationRepository(ApplicationProperties):
updates(Iterable[Package]): list of packages to update
packagers(Packagers | None, optional): optional override of username for build process
(Default value = None)
bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False)
Returns:
Result: update result
@ -150,7 +152,7 @@ class ApplicationRepository(ApplicationProperties):
tree = Tree.resolve(updates)
for num, level in enumerate(tree):
self.logger.info("processing level #%i %s", num, [package.base for package in level])
build_result = self.repository.process_build(level, packagers)
build_result = self.repository.process_build(level, packagers, bump_pkgrel=bump_pkgrel)
packages = self.repository.packages_built()
process_update(packages, build_result)

View File

@ -52,5 +52,5 @@ class Add(Handler):
packagers = Packagers(args.username, {package.base: package.packager for package in packages})
application.print_updates(packages, log_fn=application.logger.info)
result = application.update(packages, packagers)
result = application.update(packages, packagers, bump_pkgrel=args.increment)
Add.check_if_empty(args.exit_code, result.is_empty)

View File

@ -53,7 +53,7 @@ class Rebuild(Handler):
application.print_updates(updates, log_fn=print)
return
result = application.update(updates, args.username)
result = application.update(updates, args.username, bump_pkgrel=args.increment)
Rebuild.check_if_empty(args.exit_code, result.is_empty)
@staticmethod

View File

@ -54,7 +54,7 @@ class Update(Handler):
packagers = Packagers(args.username, {package.base: package.packager for package in packages})
application.print_updates(packages, log_fn=application.logger.info)
result = application.update(packages, packagers)
result = application.update(packages, packagers, bump_pkgrel=args.increment)
Update.check_if_empty(args.exit_code, result.is_empty)
@staticmethod

View File

@ -26,6 +26,7 @@ from ahriman.core.exceptions import BuildError
from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output
from ahriman.models.package import Package
from ahriman.models.pkgbuild_patch import PkgbuildPatch
from ahriman.models.repository_paths import RepositoryPaths
@ -34,6 +35,11 @@ class Task(LazyLogging):
base package build task
Attributes:
archbuild_flags(list[str]): command flags for archbuild command
architecture(str): repository architecture
build_command(str): build command
makechroootpkg_flags(list[str]): command flags for makechrootpkg command
makepkg_flags(list[str]): command flags for makepkg command
package(Package): package definitions
paths(RepositoryPaths): repository paths instance
uid(int): uid of the repository owner user
@ -41,18 +47,21 @@ class Task(LazyLogging):
_check_output = check_output
def __init__(self, package: Package, configuration: Configuration, paths: RepositoryPaths) -> None:
def __init__(self, package: Package, configuration: Configuration, architecture: str,
paths: RepositoryPaths) -> None:
"""
default constructor
Args:
package(Package): package definitions
configuration(Configuration): configuration instance
architecture(str): repository architecture
paths(RepositoryPaths): repository paths instance
"""
self.package = package
self.paths = paths
self.uid, _ = paths.root_owner
self.architecture = architecture
self.archbuild_flags = configuration.getlist("build", "archbuild_flags", fallback=[])
self.build_command = configuration.get("build", "build_command")
@ -98,12 +107,23 @@ class Task(LazyLogging):
).splitlines()
return [Path(package) for package in packages]
def init(self, sources_dir: Path, database: SQLite) -> None:
def init(self, sources_dir: Path, database: SQLite, local_version: str | None) -> None:
"""
fetch package from git
Args:
sources_dir(Path): local path to fetch
database(SQLite): database instance
local_version(str | None): local version of the package. If set and equal to current version, it will
automatically bump pkgrel
"""
Sources.load(sources_dir, self.package, database.patches_get(self.package.base), self.paths)
if local_version is None:
return # there is no local package or pkgrel increment is disabled
# load fresh package
loaded_package = Package.from_build(sources_dir, self.architecture, None)
if (pkgrel := loaded_package.next_pkgrel(local_version)) is not None:
self.logger.info("package %s is the same as in repo, bumping pkgrel to %s", self.package.base, pkgrel)
patch = PkgbuildPatch("pkgrel", pkgrel)
patch.write(sources_dir / "PKGBUILD")

View File

@ -64,7 +64,8 @@ class Executor(Cleaner):
"""
raise NotImplementedError
def process_build(self, updates: Iterable[Package], packagers: Packagers | None = None) -> Result:
def process_build(self, updates: Iterable[Package], packagers: Packagers | None = None, *,
bump_pkgrel: bool = False) -> Result:
"""
build packages
@ -72,20 +73,23 @@ class Executor(Cleaner):
updates(Iterable[Package]): list of packages properties to build
packagers(Packagers | None, optional): optional override of username for build process
(Default value = None)
bump_pkgrel(bool, optional): bump pkgrel in case of local version conflict (Default value = False)
Returns:
Result: build result
"""
def build_single(package: Package, local_path: Path, packager_id: str | None) -> None:
self.reporter.set_building(package.base)
task = Task(package, self.configuration, self.paths)
task.init(local_path, self.database)
task = Task(package, self.configuration, self.architecture, self.paths)
local_version = local_versions.get(package.base) if bump_pkgrel else None
task.init(local_path, self.database, local_version)
built = task.build(local_path, packager_id)
for src in built:
dst = self.paths.packages / src.name
shutil.move(src, dst)
packagers = packagers or Packagers()
local_versions = {package.base: package.version for package in self.packages()}
result = Result()
for single in updates:

View File

@ -25,6 +25,7 @@ import logging
import os
import re
import requests
import selectors
import subprocess
from collections.abc import Callable, Generator, Iterable
@ -48,6 +49,7 @@ __all__ = [
"filter_json",
"full_version",
"package_like",
"parse_version",
"partition",
"pretty_datetime",
"pretty_size",
@ -107,15 +109,24 @@ def check_output(*args: str, exception: Exception | None = None, cwd: Path | Non
channel: IO[str] | None = getattr(proc, channel_name, None)
return channel if channel is not None else io.StringIO()
def log(single: str) -> None:
if logger is not None:
logger.debug(single)
# wrapper around selectors polling
def poll(sel: selectors.BaseSelector) -> Generator[str, None, None]:
for key, _ in sel.select(): # we don't need to check mask here because we have only subscribed on reading
line = key.fileobj.readline() # type: ignore[union-attr]
if not line: # in case of empty line we remove selector as there is no data here anymore
sel.unregister(key.fileobj)
continue
line = line.rstrip()
if logger is not None:
logger.debug(line)
if key.data == "stdout":
yield line # yield only stdout data
environment = environment or {}
if user is not None:
environment["HOME"] = getpwuid(user).pw_dir
# FIXME additional workaround for linter and type check which do not know that user arg is supported
# pylint: disable=unexpected-keyword-arg
with subprocess.Popen(args, cwd=cwd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
user=user, env=environment, text=True, encoding="utf8", bufsize=1) as process:
if input_data is not None:
@ -123,16 +134,13 @@ def check_output(*args: str, exception: Exception | None = None, cwd: Path | Non
input_channel.write(input_data)
input_channel.close()
# read stdout and append to output result
result: list[str] = []
for line in iter(get_io(process, "stdout").readline, ""):
line = line.strip()
result.append(line)
log(line)
selector = selectors.DefaultSelector()
selector.register(get_io(process, "stdout"), selectors.EVENT_READ, data="stdout")
selector.register(get_io(process, "stderr"), selectors.EVENT_READ, data="stderr")
# read stderr and write info to logs
for line in iter(get_io(process, "stderr").readline, ""):
log(line.strip())
result: list[str] = []
while selector.get_map(): # while there are unread selectors, keep reading
result.extend(poll(selector))
process.terminate() # make sure that process is terminated
status_code = process.wait()
@ -275,6 +283,25 @@ def package_like(filename: Path) -> bool:
return ".pkg." in name and not name.endswith(".sig")
def parse_version(version: str) -> tuple[str | None, str, str]:
"""
parse version and returns its components
Args:
version(str): full version string
Returns:
tuple[str | None, str, str]: epoch if any, pkgver and pkgrel variables
"""
if ":" in version:
epoch, version = version.split(":", maxsplit=1)
else:
epoch = None
pkgver, pkgrel = version.rsplit("-", maxsplit=1)
return epoch, pkgver, pkgrel
def partition(source: list[T], predicate: Callable[[T], bool]) -> tuple[list[T], list[T]]:
"""
partition list into two based on predicate, based on # https://docs.python.org/dev/library/itertools.html#itertools-recipes

View File

@ -34,7 +34,7 @@ from ahriman.core.alpm.pacman import Pacman
from ahriman.core.alpm.remote import AUR, Official, OfficialSyncdb
from ahriman.core.exceptions import PackageInfoError
from ahriman.core.log import LazyLogging
from ahriman.core.util import check_output, dataclass_view, full_version, srcinfo_property_list, utcnow
from ahriman.core.util import check_output, dataclass_view, full_version, parse_version, srcinfo_property_list, utcnow
from ahriman.models.package_description import PackageDescription
from ahriman.models.package_source import PackageSource
from ahriman.models.remote_source import RemoteSource
@ -507,6 +507,35 @@ class Package(LazyLogging):
result: int = vercmp(self.version, remote_version)
return result < 0
def next_pkgrel(self, local_version: str) -> str | None:
"""
generate next pkgrel variable. The package release will be incremented if ``local_version`` is more or equal to
the ``Package.version``; in this case the function will return new pkgrel value, otherwise ``None`` will be
returned
Args:
local_version(str): locally stored package version
Returns:
str | None: new generated package release version if any. In case if the release contains dot (e.g. 1.2),
the minor part will be incremented by 1. If the release does not contain major.minor notation, the minor version
equals to 1 will be appended
"""
epoch, pkgver, _ = parse_version(self.version)
local_epoch, local_pkgver, local_pkgrel = parse_version(local_version)
if epoch != local_epoch or pkgver != local_pkgver:
return None # epoch or pkgver are different, keep upstream pkgrel
if vercmp(self.version, local_version) > 0:
return None # upstream version is newer than local one, keep upstream pkgrel
if "." in local_pkgrel:
major, minor = local_pkgrel.rsplit(".", maxsplit=1)
else:
major, minor = local_pkgrel, "0"
return f"{major}.{int(minor) + 1}"
def pretty_print(self) -> str:
"""
generate pretty string representation

View File

@ -48,6 +48,7 @@ def _info() -> dict[str, Any]:
* VCS packages support.
* Official repository support.
* Ability to patch AUR packages and even create package from local PKGBUILDs.
* Various rebuild options with ability to automatically bump package version.
* Sign support with gpg (repository, package), multiple packagers support.
* Triggers for repository updates, e.g. synchronization to remote services (rsync, s3 and github) and report generation (email, html, telegram).
* Repository status interface with optional authorization and control options.

View File

@ -6,6 +6,7 @@ from unittest.mock import call as MockCall
from ahriman.application.application.application_repository import ApplicationRepository
from ahriman.core.tree import Leaf, Tree
from ahriman.models.package import Package
from ahriman.models.packagers import Packagers
from ahriman.models.result import Result
@ -170,9 +171,12 @@ def test_update(application_repository: ApplicationRepository, package_ahriman:
on_result_mock = mocker.patch(
"ahriman.application.application.application_repository.ApplicationRepository.on_result")
application_repository.update([package_ahriman], "username")
build_mock.assert_called_once_with([package_ahriman], "username")
update_mock.assert_has_calls([MockCall(paths, "username"), MockCall(paths, "username")])
application_repository.update([package_ahriman], Packagers("username"), bump_pkgrel=True)
build_mock.assert_called_once_with([package_ahriman], Packagers("username"), bump_pkgrel=True)
update_mock.assert_has_calls([
MockCall(paths, Packagers("username")),
MockCall(paths, Packagers("username")),
])
on_result_mock.assert_has_calls([MockCall(result), MockCall(result)])

View File

@ -24,6 +24,7 @@ def _default_args(args: argparse.Namespace) -> argparse.Namespace:
"""
args.package = []
args.exit_code = False
args.increment = True
args.now = False
args.refresh = 0
args.source = PackageSource.Auto
@ -70,7 +71,8 @@ def test_run_with_updates(args: argparse.Namespace, configuration: Configuration
Add.run(args, "x86_64", configuration, report=False)
updates_mock.assert_called_once_with(args.package, aur=False, local=False, manual=True, vcs=False)
application_mock.assert_called_once_with([package_ahriman],
Packagers(args.username, {package_ahriman.base: "packager"}))
Packagers(args.username, {package_ahriman.base: "packager"}),
bump_pkgrel=args.increment)
dependencies_mock.assert_called_once_with([package_ahriman], process_dependencies=args.dependencies)
check_mock.assert_called_once_with(False, False)
print_mock.assert_called_once_with([package_ahriman], log_fn=pytest.helpers.anyvar(int))

View File

@ -27,6 +27,7 @@ def _default_args(args: argparse.Namespace) -> argparse.Namespace:
args.dry_run = False
args.from_database = False
args.exit_code = False
args.increment = True
args.status = None
args.username = "username"
return args
@ -51,7 +52,7 @@ def test_run(args: argparse.Namespace, package_ahriman: Package, configuration:
Rebuild.run(args, "x86_64", configuration, report=False)
extract_mock.assert_called_once_with(pytest.helpers.anyvar(int), args.status, from_database=args.from_database)
application_packages_mock.assert_called_once_with([package_ahriman], None)
application_mock.assert_called_once_with([package_ahriman], args.username)
application_mock.assert_called_once_with([package_ahriman], args.username, bump_pkgrel=args.increment)
check_mock.assert_has_calls([MockCall(False, False), MockCall(False, False)])
on_start_mock.assert_called_once_with()

View File

@ -27,6 +27,7 @@ def _default_args(args: argparse.Namespace) -> argparse.Namespace:
args.dependencies = True
args.dry_run = False
args.exit_code = False
args.increment = True
args.aur = True
args.local = True
args.manual = True
@ -55,7 +56,8 @@ def test_run(args: argparse.Namespace, package_ahriman: Package, configuration:
Update.run(args, "x86_64", configuration, report=False)
application_mock.assert_called_once_with([package_ahriman],
Packagers(args.username, {package_ahriman.base: "packager"}))
Packagers(args.username, {package_ahriman.base: "packager"}),
bump_pkgrel=args.increment)
updates_mock.assert_called_once_with(args.package, aur=args.aur, local=args.local, manual=args.manual, vcs=args.vcs)
dependencies_mock.assert_called_once_with([package_ahriman], process_dependencies=args.dependencies)
check_mock.assert_has_calls([MockCall(False, False), MockCall(False, False)])

View File

@ -18,6 +18,32 @@ def test_init(task_ahriman: Task, database: SQLite, mocker: MockerFixture) -> No
"""
must copy tree instead of fetch
"""
mocker.patch("ahriman.models.package.Package.from_build", return_value=task_ahriman.package)
load_mock = mocker.patch("ahriman.core.build_tools.sources.Sources.load")
task_ahriman.init(Path("ahriman"), database)
task_ahriman.init(Path("ahriman"), database, None)
load_mock.assert_called_once_with(Path("ahriman"), task_ahriman.package, [], task_ahriman.paths)
def test_init_bump_pkgrel(task_ahriman: Task, database: SQLite, mocker: MockerFixture) -> None:
"""
must bump pkgrel if it is same as provided
"""
mocker.patch("ahriman.models.package.Package.from_build", return_value=task_ahriman.package)
mocker.patch("ahriman.core.build_tools.sources.Sources.load")
write_mock = mocker.patch("ahriman.models.pkgbuild_patch.PkgbuildPatch.write")
local = Path("ahriman")
task_ahriman.init(local, database, task_ahriman.package.version)
write_mock.assert_called_once_with(local / "PKGBUILD")
def test_init_bump_pkgrel_skip(task_ahriman: Task, database: SQLite, mocker: MockerFixture) -> None:
"""
must keep pkgrel if version is different from provided
"""
mocker.patch("ahriman.models.package.Package.from_build", return_value=task_ahriman.package)
mocker.patch("ahriman.core.build_tools.sources.Sources.load")
write_mock = mocker.patch("ahriman.models.pkgbuild_patch.PkgbuildPatch.write")
task_ahriman.init(Path("ahriman"), database, f"2:{task_ahriman.package.version}")
write_mock.assert_not_called()

View File

@ -91,4 +91,4 @@ def task_ahriman(package_ahriman: Package, configuration: Configuration, reposit
Returns:
Task: built task test instance
"""
return Task(package_ahriman, configuration, repository_paths)
return Task(package_ahriman, configuration, "x86_64", repository_paths)

View File

@ -30,22 +30,41 @@ def test_process_build(executor: Executor, package_ahriman: Package, mocker: Moc
"""
must run build process
"""
mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman])
mocker.patch("ahriman.core.build_tools.task.Task.build", return_value=[Path(package_ahriman.base)])
mocker.patch("ahriman.core.build_tools.task.Task.init")
init_mock = mocker.patch("ahriman.core.build_tools.task.Task.init")
move_mock = mocker.patch("shutil.move")
status_client_mock = mocker.patch("ahriman.core.status.client.Client.set_building")
executor.process_build([package_ahriman], Packagers("packager"))
executor.process_build([package_ahriman], Packagers("packager"), bump_pkgrel=False)
init_mock.assert_called_once_with(pytest.helpers.anyvar(int), pytest.helpers.anyvar(int), None)
# must move files (once)
move_mock.assert_called_once_with(Path(package_ahriman.base), executor.paths.packages / package_ahriman.base)
# must update status
status_client_mock.assert_called_once_with(package_ahriman.base)
def test_process_build_bump_pkgrel(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must run build process and pass current package version to build tools
"""
mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman])
mocker.patch("ahriman.core.build_tools.task.Task.build", return_value=[Path(package_ahriman.base)])
mocker.patch("shutil.move")
mocker.patch("ahriman.core.status.client.Client.set_building")
init_mock = mocker.patch("ahriman.core.build_tools.task.Task.init")
executor.process_build([package_ahriman], Packagers("packager"), bump_pkgrel=True)
init_mock.assert_called_once_with(pytest.helpers.anyvar(int),
pytest.helpers.anyvar(int),
package_ahriman.version)
def test_process_build_failure(executor: Executor, package_ahriman: Package, mocker: MockerFixture) -> None:
"""
must run correct process failed builds
"""
mocker.patch("ahriman.core.repository.executor.Executor.packages", return_value=[package_ahriman])
mocker.patch("ahriman.core.repository.executor.Executor.packages_built")
mocker.patch("ahriman.core.build_tools.task.Task.build", return_value=[Path(package_ahriman.base)])
mocker.patch("ahriman.core.build_tools.task.Task.init")

View File

@ -8,12 +8,12 @@ import subprocess
from pathlib import Path
from pytest_mock import MockerFixture
from typing import Any
from unittest.mock import MagicMock
from unittest.mock import MagicMock, call as MockCall
from ahriman.core.exceptions import BuildError, OptionError, UnsafeRunError
from ahriman.core.util import check_output, check_user, dataclass_view, enum_values, exception_response_text, \
extract_user, filter_json, full_version, package_like, partition, pretty_datetime, pretty_size, safe_filename, \
srcinfo_property, srcinfo_property_list, trim_package, utcnow, walk
extract_user, filter_json, full_version, package_like, parse_version, partition, pretty_datetime, pretty_size, \
safe_filename, srcinfo_property, srcinfo_property_list, trim_package, utcnow, walk
from ahriman.models.package import Package
from ahriman.models.package_source import PackageSource
from ahriman.models.repository_paths import RepositoryPaths
@ -128,6 +128,15 @@ def test_check_output_failure_exception(mocker: MockerFixture) -> None:
check_output("echo", "hello", exception=exception, logger=logging.getLogger(""))
def test_check_output_empty_line(mocker: MockerFixture) -> None:
"""
must correctly process empty lines in command output
"""
logger_mock = mocker.patch("logging.Logger.debug")
assert check_output("python", "-c", """print(); print("hello")""", logger=logging.getLogger("")) == "\nhello"
logger_mock.assert_has_calls([MockCall(""), MockCall("hello")])
def test_check_user(mocker: MockerFixture) -> None:
"""
must check user correctly
@ -273,6 +282,17 @@ def test_package_like_sig(package_ahriman: Package) -> None:
assert not package_like(sig_file)
def test_parse_version() -> None:
"""
must correctly parse version into components
"""
assert parse_version("1.2.3-4") == (None, "1.2.3", "4")
assert parse_version("5:1.2.3-4") == ("5", "1.2.3", "4")
assert parse_version("1.2.3-4.2") == (None, "1.2.3", "4.2")
assert parse_version("0:1.2.3-4.2") == ("0", "1.2.3", "4.2")
assert parse_version("0:1.2.3-4") == ("0", "1.2.3", "4")
def test_partition() -> None:
"""
must partition list based on predicate

View File

@ -501,6 +501,30 @@ def test_is_outdated_fresh_package(package_ahriman: Package, repository_paths: R
actual_version_mock.assert_not_called()
def test_next_pkgrel(package_ahriman: Package) -> None:
"""
must correctly bump pkgrel
"""
assert package_ahriman.next_pkgrel(package_ahriman.version) == "1.1"
package_ahriman.version = "1.0.0-1.1"
assert package_ahriman.next_pkgrel(package_ahriman.version) == "1.2"
package_ahriman.version = "1.0.0-1.2.1"
assert package_ahriman.next_pkgrel(package_ahriman.version) == "1.2.2"
package_ahriman.version = "1:1.0.0-1"
assert package_ahriman.next_pkgrel("1:1.0.1-1") is None
assert package_ahriman.next_pkgrel("2:1.0.0-1") is None
package_ahriman.version = "1.0.0-1.1"
assert package_ahriman.next_pkgrel("1.0.1-2") is None
assert package_ahriman.next_pkgrel("1.0.0-2") == "2.1"
package_ahriman.version = "1.0.0-2"
assert package_ahriman.next_pkgrel("1.0.0-1.1") is None
def test_build_status_pretty_print(package_ahriman: Package) -> None:
"""
must return string in pretty print function