Release 2.0.0rc7

basic templates require package info which is unavailable if package
wasn't built

.github/workflows/docker-image.yml

@@ -5,6 +5,7 @@ on:
     branches: [ master ]
     tags:
       - '*'
+      - '!*rc*'
 
 jobs:
   docker-image:

README.md

@@ -13,7 +13,7 @@ Wrapper for managing custom repository inspired by [repo-scripts](https://github
 * Multi-architecture support.
 * VCS packages support.
 * Sign support with gpg (repository, package, per package settings).
-* Synchronization to remote services (rsync, s3 and github) and report generation (html).
+* Synchronization to remote services (rsync, s3 and github) and report generation (email, html, telegram).
 * Dependency manager.
 * Ability to patch AUR packages and even create package from local PKGBUILDs.
 * Repository status interface with optional authorization and control options:

docs/ahriman.1

@@ -3,7 +3,7 @@
 ahriman
 .SH SYNOPSIS
 .B ahriman
-[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--no-report] [-q] [--unsafe] [-v] {aur-search,search,help,help-commands-unsafe,key-import,package-add,add,package-update,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,repo-check,check,repo-clean,clean,repo-config,config,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,restore,repo-setup,init,repo-init,setup,repo-sign,sign,repo-status-update,repo-sync,sync,repo-update,update,user-add,user-list,user-remove,web} ...
+[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--no-report] [-q] [--unsafe] [-v] {aur-search,search,help,help-commands-unsafe,key-import,package-add,add,package-update,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,repo-backup,repo-check,check,repo-clean,clean,repo-config,config,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-setup,init,repo-init,setup,repo-sign,sign,repo-status-update,repo-sync,sync,repo-update,update,user-add,user-list,user-remove,web} ...
 .SH DESCRIPTION
 ArcH Linux ReposItory MANager
 .SH OPTIONS
@@ -79,6 +79,9 @@ list patch sets
 \fBahriman\fR \fI\,patch-remove\/\fR
 remove patch set
 .TP
+\fBahriman\fR \fI\,repo-backup\/\fR
+backup repository data
+.TP
 \fBahriman\fR \fI\,repo-check\/\fR
 check for updates
 .TP
@@ -98,7 +101,7 @@ remove unknown packages
 generate report
 .TP
 \fBahriman\fR \fI\,repo-restore\/\fR
-restore repository
+restore repository data
 .TP
 \fBahriman\fR \fI\,repo-setup\/\fR
 initial service configuration
@@ -442,6 +445,16 @@ remove patches for the package
 package base
 
 
+.SH OPTIONS 'ahriman repo-backup'
+usage: ahriman repo-backup [-h] path
+
+backup settings and database
+
+.TP
+\fBpath\fR
+path of the output archive
+
+
 .SH OPTIONS 'ahriman repo-check'
 usage: ahriman repo-check [-h] [-e] [--no-vcs] [package ...]
 
@@ -535,7 +548,7 @@ dump configuration for the specified architecture
 
 
 .SH OPTIONS 'ahriman repo-rebuild'
-usage: ahriman repo-rebuild [-h] [--depends-on DEPENDS_ON] [--dry-run] [-e]
+usage: ahriman repo-rebuild [-h] [--depends-on DEPENDS_ON] [--dry-run] [--from-database] [-e]
 
 force rebuild whole repository
 
@@ -548,12 +561,18 @@ only rebuild packages that depend on specified package
 \fB\-\-dry\-run\fR
 just perform check for packages without rebuild process itself
 
+.TP
+\fB\-\-from\-database\fR
+read packages from database instead of filesystem. This feature in particular is required in case if you would like to
+restore repository from another repository instance. Note however that in order to restore packages you need to have
+original ahriman instance run with web service and have run repo\-update at least once.
+
 .TP
 \fB\-e\fR, \fB\-\-exit\-code\fR
 return non\-zero exit status if result is empty
 
 .SH OPTIONS 'ahriman rebuild'
-usage: ahriman repo-rebuild [-h] [--depends-on DEPENDS_ON] [--dry-run] [-e]
+usage: ahriman repo-rebuild [-h] [--depends-on DEPENDS_ON] [--dry-run] [--from-database] [-e]
 
 force rebuild whole repository
 
@@ -566,6 +585,12 @@ only rebuild packages that depend on specified package
 \fB\-\-dry\-run\fR
 just perform check for packages without rebuild process itself
 
+.TP
+\fB\-\-from\-database\fR
+read packages from database instead of filesystem. This feature in particular is required in case if you would like to
+restore repository from another repository instance. Note however that in order to restore packages you need to have
+original ahriman instance run with web service and have run repo\-update at least once.
+
 .TP
 \fB\-e\fR, \fB\-\-exit\-code\fR
 return non\-zero exit status if result is empty
@@ -619,40 +644,17 @@ target to generate report
 
 
 .SH OPTIONS 'ahriman repo-restore'
-usage: ahriman repo-restore [-h] [-e] [-n] [--without-dependencies]
-
-restore repository from database file
+usage: ahriman repo-restore [-h] [-o OUTPUT] path
 
+restore settings and database
 
 .TP
-\fB\-e\fR, \fB\-\-exit\-code\fR
-return non\-zero exit status if result is empty
+\fBpath\fR
+path of the input archive
 
 .TP
-\fB\-n\fR, \fB\-\-now\fR
-run update function after
-
-.TP
-\fB\-\-without\-dependencies\fR
-do not add dependencies
-
-.SH OPTIONS 'ahriman restore'
-usage: ahriman repo-restore [-h] [-e] [-n] [--without-dependencies]
-
-restore repository from database file
-
-
-.TP
-\fB\-e\fR, \fB\-\-exit\-code\fR
-return non\-zero exit status if result is empty
-
-.TP
-\fB\-n\fR, \fB\-\-now\fR
-run update function after
-
-.TP
-\fB\-\-without\-dependencies\fR
-do not add dependencies
+\fB\-o\fR \fI\,OUTPUT\/\fR, \fB\-\-output\fR \fI\,OUTPUT\/\fR
+root path of the extracted files
 
 .SH OPTIONS 'ahriman repo-setup'
 usage: ahriman repo-setup [-h] [--build-as-user BUILD_AS_USER] [--build-command BUILD_COMMAND]

docs/configuration.md

@@ -114,6 +114,18 @@ Section name must be either `html` (plus optional architecture name, e.g. `html:
 * `link_path` - prefix for HTML links, string, required.
 * `template_path` - path to Jinja2 template, string, required.
 
+### `telegram` type
+
+Section name must be either `telegram` (plus optional architecture name, e.g. `telegram:x86_64`) or random name with `type` set.
+
+* `type` - type of the report, string, optional, must be set to `telegram` if exists.
+* `api_key` - telegram bot API key, string, required. Please refer FAQ about how to create chat and bot
+* `chat_id` - telegram chat id, either string with `@` or integer value, required.
+* `homepage` - link to homepage, string, optional.
+* `link_path` - prefix for HTML links, string, required.
+* `template_path` - path to Jinja2 template, string, required.
+* `template_type` - `parse_mode` to be passed to telegram API, one of `MarkdownV2`, `HTML`, `Markdown`, string, optional, default `HTML`.
+
 ## `upload` group
 
 Remote synchronization settings.

docs/faq.md

@@ -199,7 +199,7 @@ server {
 
 ## Docker image
 
-We provide official images which can be found under `arcan1s/ahriman` repository. Docker image is being updated on each master commit as well as on each version. If you would like to use last (probably unstable build) you can use `latest` tag; otherwise you can use any version tag available. 
+We provide official images which can be found under `arcan1s/ahriman` repository. Docker image is being updated on each master commit as well as on each version. If you would like to use last (probably unstable) build you can use `edge` tag or `latest` for any tagged versions; otherwise you can use any version tag available. 
 
 The default action (in case if no arguments provided) is `repo-update`. Basically the idea is to run container, e.g.:
 
@@ -242,7 +242,7 @@ You can pass any of these variables by using `-e` argument, e.g.:
 docker run -e AHRIMAN_PORT=8080 arcan1s/ahriman:latest
 ```
 
-### Working with web service
+### Web service setup
 
 Well for that you would need to have web container instance running forever; it can be achieved by the following command:
 
@@ -402,6 +402,46 @@ There are several choices:
    
 After these steps `index.html` file will be automatically synced to S3
 
+### I would like to get messages to my telegram account/channel
+
+1. It still requires additional dependencies:
+
+   ```shell
+   yay -S python-jinja
+   ```
+
+2. Register bot in telegram. You can do it by talking with [@BotFather](https://t.me/botfather). For more details please refer to [official documentation](https://core.telegram.org/bots).
+
+3. Optionally (if you want to post message in chat):
+
+   1. Create telegram channel. 
+   2. Invite your bot into the channel.
+   3. Make your channel public
+
+4. Get chat id if you want to use by numerical id or just use id prefixed with `@` (e.g. `@ahriman`). If you are not using chat the chat id is your user id. If you don't want to make channel public you can use [this guide](https://stackoverflow.com/a/33862907).
+
+5. Configure the service:
+
+   ```ini
+   [report]
+   target = telegram
+   
+   [telegram]
+   api_key = aaAAbbBBccCC
+   chat_id = @ahriman
+   link_path = http://example.com/x86_64
+   ```
+   
+   `api_key` is the one sent by [@BotFather](https://t.me/botfather), `chat_id` is the value retrieved from previous step.
+
+If you did everything fine you should receive the message with the next update. Quick credentials check can be done by using the following command:
+
+```shell
+curl 'https://api.telegram.org/bot${CHAT_ID}/sendMessage?chat_id=${API_KEY}&text=hello'
+```
+
+(replace `${CHAT_ID}` and `${API_KEY}` with the values from configuration).
+
 ## Web service
 
 ### Readme mentions web interface, how do I use it?
@@ -479,6 +519,36 @@ After these steps `index.html` file will be automatically synced to S3
 5. Create end-user `sudo -u ahriman ahriman user-add -r write my-first-user`. When it will ask for the password leave it blank.
 6. Restart web service `systemctl restart ahriman-web@x86_64`.
 
+## Backup and restore
+
+The service provides several commands aim to do easy repository backup and restore. If you would like to move repository from the server `server1.example.com` to another `server2.example.com` you have to perform the following steps:
+
+1. On the source server `server1.example.com` run `repo-backup` command, e.g.:
+
+   ```shell
+   sudo ahriman repo-backup /tmp/repo.tar.gz
+   ```
+   
+   This command will pack all configuration files together with database file into the archive specified as command line argument (i.e. `/tmp/repo.tar.gz`). In addition it will also archive `cache` directory (the one which contains local clones used by e.g. local packages) and `.gnupg` of the `ahriman` user.
+
+2. Copy created archive from source server `server1.example.com` to target `server2.example.com`.
+
+3. Install ahriman as usual on the target server `server2.example.com` if you didn't yet.
+
+4. Extract archive e.g. by using subcommand:
+   
+   ```shell
+   sudo ahriman repo-restore /tmp/repo.tar.gz
+   ```
+   
+   An additional argument `-o`/`--output` can be used to specify extraction root (`/` by default).
+
+5. Rebuild repository:
+
+   ```shell
+   sudo -u ahriman ahriman repo-rebuild --from-database
+   ```
+
 ## Other topics
 
 ### How does it differ from %another-manager%?
@@ -518,6 +588,10 @@ Though originally I've created ahriman by trying to improve the project, it stil
 
 `repo-scripts` also have bad architecture and bad quality code and uses out-of-dated `yaourt` and `package-query`.
 
+#### [toolbox](https://github.com/chaotic-aur/toolbox)
+
+It is automation tools for `repoctl` mentioned above. Except for using shell it looks pretty cool and also offers some additional features like patches, remote synchronization (isn't it?) and reporting.
+
 ### I would like to check service logs
 
 By default, the service writes logs to `/dev/log` which can be accessed by using `journalctl` command (logs are written to the journal of the user under which command is run).
@@ -528,7 +602,7 @@ You can also edit configuration and forward logs to `stderr`, just change `handl
 sed -i 's/handlers = syslog_handler/handlers = console_handler/g' /etc/ahriman.ini.d/logging.ini
 ```
 
-You can even configure logging as you wish, but kindly refer to python `logging` module configuration.
+You can even configure logging as you wish, but kindly refer to python `logging` module [configuration](https://docs.python.org/3/library/logging.config.html).
 
 ### Html customization
 

package/archlinux/PKGBUILD

@@ -1,7 +1,7 @@
 # Maintainer: Evgeniy Alekseev
 
 pkgname='ahriman'
-pkgver=2.0.0rc4
+pkgver=2.0.0rc7
 pkgrel=1
 pkgdesc="ArcH Linux ReposItory MANager"
 arch=('any')
@@ -38,7 +38,7 @@ build() {
 package() {
   cd "$pkgname"
 
-  python -m installer --destdir="$pkgdir" dist/*.whl
+  python -m installer --destdir="$pkgdir" "dist/$pkgname-$pkgver-py3-none-any.whl"
 
   # python-installer actually thinks that you cannot just copy files to root
   # thus we need to copy them manually

package/share/ahriman/settings/ahriman.ini

@@ -45,6 +45,9 @@ ssl = disabled
 [html]
 template_path = /usr/share/ahriman/templates/repo-index.jinja2
 
+[telegram]
+template_path = /usr/share/ahriman/templates/telegram-index.jinja2
+
 [upload]
 target =
 

package/share/ahriman/templates/telegram-index.jinja2

@@ -0,0 +1,4 @@
+{#simplified version of full report#}
+<b>{{ repository }} update</b>
+{% for package in packages %}
+<a href="{{ link_path }}/{{ package.filename }}">{{ package.name }}</a> {{ package.version }}{% endfor %}

setup.py

@@ -67,6 +67,7 @@ setup(
             "package/share/ahriman/templates/build-status.jinja2",
             "package/share/ahriman/templates/email-index.jinja2",
             "package/share/ahriman/templates/repo-index.jinja2",
+            "package/share/ahriman/templates/telegram-index.jinja2",
         ]),
         ("share/ahriman/templates/build-status", [
             "package/share/ahriman/templates/build-status/login-modal.jinja2",

src/ahriman/application/ahriman.py

@@ -83,6 +83,7 @@ def _parser() -> argparse.ArgumentParser:
     _set_patch_add_parser(subparsers)
     _set_patch_list_parser(subparsers)
     _set_patch_remove_parser(subparsers)
+    _set_repo_backup_parser(subparsers)
     _set_repo_check_parser(subparsers)
     _set_repo_clean_parser(subparsers)
     _set_repo_config_parser(subparsers)
@@ -315,6 +316,19 @@ def _set_patch_remove_parser(root: SubParserAction) -> argparse.ArgumentParser:
     return parser
 
 
+def _set_repo_backup_parser(root: SubParserAction) -> argparse.ArgumentParser:
+    """
+    add parser for repository backup subcommand
+    :param root: subparsers for the commands
+    :return: created argument parser
+    """
+    parser = root.add_parser("repo-backup", help="backup repository data",
+                             description="backup settings and database", formatter_class=_formatter)
+    parser.add_argument("path", help="path of the output archive", type=Path)
+    parser.set_defaults(handler=handlers.Backup, architecture=[""], lock=None, no_report=True, unsafe=True)
+    return parser
+
+
 def _set_repo_check_parser(root: SubParserAction) -> argparse.ArgumentParser:
     """
     add parser for repository check subcommand
@@ -375,6 +389,12 @@ def _set_repo_rebuild_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser.add_argument("--depends-on", help="only rebuild packages that depend on specified package", action="append")
     parser.add_argument("--dry-run", help="just perform check for packages without rebuild process itself",
                         action="store_true")
+    parser.add_argument("--from-database",
+                        help="read packages from database instead of filesystem. This feature in particular is "
+                             "required in case if you would like to restore repository from another repository "
+                             "instance. Note however that in order to restore packages you need to have original "
+                             "ahriman instance run with web service and have run repo-update at least once.",
+                        action="store_true")
     parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
     parser.set_defaults(handler=handlers.Rebuild)
     return parser
@@ -412,16 +432,15 @@ def _set_repo_report_parser(root: SubParserAction) -> argparse.ArgumentParser:
 
 def _set_repo_restore_parser(root: SubParserAction) -> argparse.ArgumentParser:
     """
-    add parser for package addition subcommand
+    add parser for repository restore subcommand
     :param root: subparsers for the commands
     :return: created argument parser
     """
-    parser = root.add_parser("repo-restore", aliases=["restore"], help="restore repository",
-                             description="restore repository from database file", formatter_class=_formatter)
-    parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
-    parser.add_argument("-n", "--now", help="run update function after", action="store_true")
-    parser.add_argument("--without-dependencies", help="do not add dependencies", action="store_true")
-    parser.set_defaults(handler=handlers.Add, package=None, source=PackageSource.AUR)
+    parser = root.add_parser("repo-restore", help="restore repository data",
+                             description="restore settings and database", formatter_class=_formatter)
+    parser.add_argument("path", help="path of the input archive", type=Path)
+    parser.add_argument("-o", "--output", help="root path of the extracted files", type=Path, default=Path("/"))
+    parser.set_defaults(handler=handlers.Restore, architecture=[""], lock=None, no_report=True, unsafe=True)
     return parser
 
 

src/ahriman/application/handlers/__init__.py

@@ -20,6 +20,7 @@
 from ahriman.application.handlers.handler import Handler
 
 from ahriman.application.handlers.add import Add
+from ahriman.application.handlers.backup import Backup
 from ahriman.application.handlers.clean import Clean
 from ahriman.application.handlers.dump import Dump
 from ahriman.application.handlers.help import Help
@@ -29,6 +30,7 @@ from ahriman.application.handlers.rebuild import Rebuild
 from ahriman.application.handlers.remove import Remove
 from ahriman.application.handlers.remove_unknown import RemoveUnknown
 from ahriman.application.handlers.report import Report
+from ahriman.application.handlers.restore import Restore
 from ahriman.application.handlers.search import Search
 from ahriman.application.handlers.setup import Setup
 from ahriman.application.handlers.sign import Sign

src/ahriman/application/handlers/add.py

@@ -19,7 +19,7 @@
 #
 import argparse
 
-from typing import List, Type
+from typing import Type
 
 from ahriman.application.application import Application
 from ahriman.application.handlers.handler import Handler
@@ -43,20 +43,10 @@ class Add(Handler):
         :param unsafe: if set no user check will be performed before path creation
         """
         application = Application(architecture, configuration, no_report, unsafe)
-        packages = Add.extract_packages(application) if args.package is None else args.package
-        application.add(packages, args.source, args.without_dependencies)
+        application.add(args.package, args.source, args.without_dependencies)
         if not args.now:
             return
 
-        packages = application.updates(packages, True, True, False, True, application.logger.info)
+        packages = application.updates(args.package, True, True, False, True, application.logger.info)
         result = application.update(packages)
         Add.check_if_empty(args.exit_code, result.is_empty)
-
-    @staticmethod
-    def extract_packages(application: Application) -> List[str]:
-        """
-        extract packages from database file
-        :param application: application instance
-        :return: list of packages which were stored in database
-        """
-        return [package.base for (package, _) in application.database.packages_get()]

src/ahriman/application/handlers/backup.py

@@ -0,0 +1,80 @@
+#
+# Copyright (c) 2021-2022 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+import pwd
+
+from pathlib import Path
+from tarfile import TarFile
+from typing import Set, Type
+
+from ahriman.application.handlers.handler import Handler
+from ahriman.core.configuration import Configuration
+from ahriman.core.database.sqlite import SQLite
+
+
+class Backup(Handler):
+    """
+    backup packages handler
+    """
+
+    ALLOW_AUTO_ARCHITECTURE_RUN = False  # it should be called only as "no-architecture"
+
+    @classmethod
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool, unsafe: bool) -> None:
+        """
+        callback for command line
+        :param args: command line args
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
+        :param unsafe: if set no user check will be performed before path creation
+        """
+        backup_paths = Backup.get_paths(configuration)
+        with TarFile(args.path, mode="w") as archive:  # well we don't actually use compression
+            for backup_path in backup_paths:
+                archive.add(backup_path)
+
+    @staticmethod
+    def get_paths(configuration: Configuration) -> Set[Path]:
+        """
+        extract paths to backup
+        :param configuration: configuration instance
+        :return: map of the filesystem paths
+        """
+        paths = set(configuration.include.glob("*.ini"))
+
+        root, _ = configuration.check_loaded()
+        paths.add(root)  # the configuration itself
+        paths.add(SQLite.database_path(configuration))  # database
+
+        # local caches
+        repository_paths = configuration.repository_paths
+        if repository_paths.cache.is_dir():
+            paths.add(repository_paths.cache)
+
+        # gnupg home with imported keys
+        uid, _ = repository_paths.root_owner
+        system_user = pwd.getpwuid(uid)
+        gnupg_home = Path(system_user.pw_dir) / ".gnupg"
+        if gnupg_home.is_dir():
+            paths.add(gnupg_home)
+
+        return paths

src/ahriman/application/handlers/rebuild.py

@@ -19,12 +19,13 @@
 #
 import argparse
 
-from typing import Type
+from typing import List, Type
 
 from ahriman.application.application import Application
 from ahriman.application.handlers.handler import Handler
 from ahriman.core.configuration import Configuration
 from ahriman.core.formatters.update_printer import UpdatePrinter
+from ahriman.models.package import Package
 
 
 class Rebuild(Handler):
@@ -46,7 +47,10 @@ class Rebuild(Handler):
         depends_on = set(args.depends_on) if args.depends_on else None
 
         application = Application(architecture, configuration, no_report, unsafe)
-        updates = application.repository.packages_depends_on(depends_on)
+        if args.from_database:
+            updates = Rebuild.extract_packages(application)
+        else:
+            updates = application.repository.packages_depends_on(depends_on)
 
         Rebuild.check_if_empty(args.exit_code, not updates)
         if args.dry_run:
@@ -56,3 +60,12 @@ class Rebuild(Handler):
 
         result = application.update(updates)
         Rebuild.check_if_empty(args.exit_code, result.is_empty)
+
+    @staticmethod
+    def extract_packages(application: Application) -> List[Package]:
+        """
+        extract packages from database file
+        :param application: application instance
+        :return: list of packages which were stored in database
+        """
+        return [package for (package, _) in application.database.packages_get()]

src/ahriman/application/handlers/restore.py

@@ -0,0 +1,48 @@
+#
+# Copyright (c) 2021-2022 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+
+from typing import Type
+from tarfile import TarFile
+
+from ahriman.application.handlers.handler import Handler
+from ahriman.core.configuration import Configuration
+
+
+class Restore(Handler):
+    """
+    restore packages handler
+    """
+
+    ALLOW_AUTO_ARCHITECTURE_RUN = False  # it should be called only as "no-architecture"
+
+    @classmethod
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool, unsafe: bool) -> None:
+        """
+        callback for command line
+        :param args: command line args
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
+        :param unsafe: if set no user check will be performed before path creation
+        """
+        with TarFile(args.path) as archive:
+            archive.extractall(path=args.output)

src/ahriman/core/build_tools/sources.py

@@ -86,7 +86,8 @@ class Sources:
             Sources.logger.warning("%s is not initialized, but no remote provided", sources_dir)
         else:
             Sources.logger.info("clone remote %s to %s", remote, sources_dir)
-            Sources._check_output("git", "clone", remote, str(sources_dir), exception=None, logger=Sources.logger)
+            Sources._check_output("git", "clone", remote, str(sources_dir),
+                                  exception=None, cwd=sources_dir, logger=Sources.logger)
         # and now force reset to our branch
         Sources._check_output("git", "checkout", "--force", Sources._branch,
                               exception=None, cwd=sources_dir, logger=Sources.logger)

src/ahriman/core/database/data/users.py

@@ -32,9 +32,9 @@ def migrate_users_data(connection: Connection, configuration: Configuration) ->
         for option, value in configuration[section].items():
             if not section.startswith("auth:"):
                 continue
-            permission = section[5:]
+            access = section[5:]
             connection.execute(
-                """insert into users (username, permission, password) values (:username, :permission, :password)""",
-                {"username": option.lower(), "permission": permission, "password": value})
+                """insert into users (username, access, password) values (:username, :access, :password)""",
+                {"username": option.lower(), "access": access, "password": value})
 
     connection.commit()

src/ahriman/core/report/jinja_template.py

@@ -101,7 +101,7 @@ class JinjaTemplate:
                 "name": package,
                 "url": properties.url or "",
                 "version": base.version
-            } for base in result.updated for package, properties in base.packages.items()
+            } for base in result.success for package, properties in base.packages.items()
         ]
         comparator: Callable[[Dict[str, str]], str] = lambda item: item["filename"]
 

src/ahriman/core/report/report.py

@@ -68,6 +68,9 @@ class Report:
         if provider == ReportSettings.Console:
             from ahriman.core.report.console import Console
             return Console(architecture, configuration, section)
+        if provider == ReportSettings.Telegram:
+            from ahriman.core.report.telegram import Telegram
+            return Telegram(architecture, configuration, section)
         return cls(architecture, configuration)  # should never happen
 
     def generate(self, packages: Iterable[Package], result: Result) -> None:

src/ahriman/core/report/telegram.py

@@ -0,0 +1,95 @@
+#
+# Copyright (c) 2021-2022 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# technically we could use python-telegram-bot, but it is just a single request, cmon
+import requests
+
+from typing import Iterable
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.report.jinja_template import JinjaTemplate
+from ahriman.core.report.report import Report
+from ahriman.core.util import exception_response_text
+from ahriman.models.package import Package
+from ahriman.models.result import Result
+
+
+class Telegram(Report, JinjaTemplate):
+    """
+    telegram report generator
+    :cvar TELEGRAM_API_URL: telegram api base url
+    :cvar TELEGRAM_MAX_CONTENT_LENGTH: max content length of the message
+    :ivar api_key: bot api key
+    :ivar chat_id: chat id to post message, either string with @ or integer
+    :ivar template_path: path to template for built packages
+    :ivar template_type: template message type to be used in parse mode, one of MarkdownV2, HTML, Markdown
+    """
+
+    TELEGRAM_API_URL = "https://api.telegram.org"
+    TELEGRAM_MAX_CONTENT_LENGTH = 4096
+
+    def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
+        """
+        default constructor
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param section: settings section name
+        """
+        Report.__init__(self, architecture, configuration)
+        JinjaTemplate.__init__(self, section, configuration)
+
+        self.api_key = configuration.get(section, "api_key")
+        self.chat_id = configuration.get(section, "chat_id")
+        self.template_path = configuration.getpath(section, "template_path")
+        self.template_type = configuration.get(section, "template_type", fallback="HTML")
+
+    def _send(self, text: str) -> None:
+        """
+        send message to telegram channel
+        :param text: message body text
+        """
+        try:
+            response = requests.post(
+                f"{self.TELEGRAM_API_URL}/bot{self.api_key}/sendMessage",
+                data={"chat_id": self.chat_id, "text": text, "parse_mode": self.template_type})
+            response.raise_for_status()
+        except requests.HTTPError as e:
+            self.logger.exception("could not perform request: %s", exception_response_text(e))
+            raise
+        except Exception:
+            self.logger.exception("could not perform request")
+            raise
+
+    def generate(self, packages: Iterable[Package], result: Result) -> None:
+        """
+        generate report for the specified packages
+        :param packages: list of packages to generate report
+        :param result: build result
+        """
+        if not result.success:
+            return
+        text = self.make_html(result, self.template_path)
+        # telegram content is limited by 4096 symbols, so we are going to split the message by new lines
+        # to fit into this restriction
+        if len(text) > self.TELEGRAM_MAX_CONTENT_LENGTH:
+            position = text.rfind("\n", 0, self.TELEGRAM_MAX_CONTENT_LENGTH)
+            portion, text = text[:position], text[position + 1:]  # +1 to exclude newline we split
+            self._send(portion)
+        # send remaining (or full in case if size is less than max length) text
+        self._send(text)

src/ahriman/core/tree.py

@@ -19,14 +19,11 @@
 #
 from __future__ import annotations
 
-import shutil
-import tempfile
-
-from pathlib import Path
 from typing import Iterable, List, Set, Type
 
 from ahriman.core.build_tools.sources import Sources
 from ahriman.core.database.sqlite import SQLite
+from ahriman.core.util import tmpdir
 from ahriman.models.package import Package
 
 
@@ -61,12 +58,9 @@ class Leaf:
         :param database: database instance
         :return: loaded class
         """
-        clone_dir = Path(tempfile.mkdtemp())
-        try:
+        with tmpdir() as clone_dir:
             Sources.load(clone_dir, package.git_url, database.patches_get(package.base))
             dependencies = Package.dependencies(clone_dir)
-        finally:
-            shutil.rmtree(clone_dir, ignore_errors=True)
         return cls(package, dependencies)
 
     def is_root(self, packages: Iterable[Leaf]) -> bool:

src/ahriman/models/report_settings.py

@@ -32,12 +32,14 @@ class ReportSettings(Enum):
     :cvar HTML: html report generation
     :cvar Email: email report generation
     :cvar Console: print result to console
+    :cvar Telegram: markdown report to telegram channel
     """
 
     Disabled = "disabled"  # for testing purpose
     HTML = "html"
     Email = "email"
     Console = "console"
+    Telegram = "telegram"
 
     @classmethod
     def from_option(cls: Type[ReportSettings], value: str) -> ReportSettings:
@@ -52,4 +54,6 @@ class ReportSettings(Enum):
             return cls.Email
         if value.lower() in ("console",):
             return cls.Console
+        if value.lower() in ("telegram",):
+            return cls.Telegram
         raise InvalidOption(value)

src/ahriman/models/result.py

@@ -62,13 +62,6 @@ class Result:
         """
         return list(self._success.values())
 
-    @property
-    def updated(self) -> List[Package]:
-        """
-        :return: list of updated packages inclding both success and failed
-        """
-        return self.success + self.failed
-
     def add_failed(self, package: Package) -> None:
         """
         add new package to failed built

src/ahriman/version.py

@@ -17,4 +17,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-__version__ = "2.0.0rc4"
+__version__ = "2.0.0rc7"

tests/ahriman/application/handlers/test_handler_add.py

@@ -3,7 +3,6 @@ import pytest
 
 from pytest_mock import MockerFixture
 
-from ahriman.application.application import Application
 from ahriman.application.handlers import Add
 from ahriman.core.configuration import Configuration
 from ahriman.models.package import Package
@@ -37,20 +36,6 @@ def test_run(args: argparse.Namespace, configuration: Configuration, mocker: Moc
     application_mock.assert_called_once_with(args.package, args.source, args.without_dependencies)
 
 
-def test_run_extract_packages(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
-    """
-    must run command
-    """
-    args = _default_args(args)
-    args.package = None
-    mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
-    mocker.patch("ahriman.application.application.Application.add")
-    extract_mock = mocker.patch("ahriman.application.handlers.Add.extract_packages", return_value=[])
-
-    Add.run(args, "x86_64", configuration, True, False)
-    extract_mock.assert_called_once_with(pytest.helpers.anyvar(int))
-
-
 def test_run_with_updates(args: argparse.Namespace, configuration: Configuration,
                           package_ahriman: Package, mocker: MockerFixture) -> None:
     """
@@ -87,12 +72,3 @@ def test_run_empty_exception(args: argparse.Namespace, configuration: Configurat
 
     Add.run(args, "x86_64", configuration, True, False)
     check_mock.assert_called_once_with(True, True)
-
-
-def test_extract_packages(application: Application, mocker: MockerFixture) -> None:
-    """
-    must extract packages from database
-    """
-    packages_mock = mocker.patch("ahriman.core.database.sqlite.SQLite.packages_get")
-    Add.extract_packages(application)
-    packages_mock.assert_called_once_with()

tests/ahriman/application/handlers/test_handler_backup.py

@@ -0,0 +1,58 @@
+import argparse
+
+from pathlib import Path
+from pytest_mock import MockerFixture
+from unittest.mock import MagicMock
+
+from ahriman.application.handlers import Backup
+from ahriman.core.configuration import Configuration
+from ahriman.models.repository_paths import RepositoryPaths
+
+
+def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
+    args.path = Path("result.tar.gz")
+    return args
+
+
+def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must run command
+    """
+    args = _default_args(args)
+    mocker.patch("ahriman.application.handlers.Backup.get_paths", return_value=[Path("path")])
+    tarfile = MagicMock()
+    add_mock = tarfile.__enter__.return_value = MagicMock()
+    mocker.patch("tarfile.TarFile.__new__", return_value=tarfile)
+
+    Backup.run(args, "x86_64", configuration, True, False)
+    add_mock.add.assert_called_once_with(Path("path"))
+
+
+def test_get_paths(configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must get paths to be archived
+    """
+    # gnupg export mock
+    mocker.patch("pathlib.Path.is_dir", return_value=True)
+    mocker.patch.object(RepositoryPaths, "root_owner", (42, 42))
+    getpwuid_mock = mocker.patch("pwd.getpwuid", return_value=MagicMock())
+    # well database does not exist so we override it
+    database_mock = mocker.patch("ahriman.core.database.sqlite.SQLite.database_path", return_value=configuration.path)
+
+    paths = Backup.get_paths(configuration)
+    getpwuid_mock.assert_called_once_with(42)
+    database_mock.assert_called_once_with(configuration)
+    assert configuration.path in paths
+    assert all(path.exists() for path in paths if path.name not in (".gnupg", "cache"))
+
+
+def test_disallow_auto_architecture_run() -> None:
+    """
+    must not allow multi architecture run
+    """
+    assert not Backup.ALLOW_AUTO_ARCHITECTURE_RUN

tests/ahriman/application/handlers/test_handler_rebuild.py

@@ -4,6 +4,7 @@ import pytest
 from pytest_mock import MockerFixture
 from unittest import mock
 
+from ahriman.application.application import Application
 from ahriman.application.handlers import Rebuild
 from ahriman.core.configuration import Configuration
 from ahriman.models.package import Package
@@ -18,6 +19,7 @@ def _default_args(args: argparse.Namespace) -> argparse.Namespace:
     """
     args.depends_on = []
     args.dry_run = False
+    args.from_database = False
     args.exit_code = False
     return args
 
@@ -42,6 +44,21 @@ def test_run(args: argparse.Namespace, package_ahriman: Package,
     check_mock.assert_has_calls([mock.call(False, False), mock.call(False, False)])
 
 
+def test_run_extract_packages(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must run command
+    """
+    args = _default_args(args)
+    args.from_database = True
+    args.dry_run = True
+    mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
+    mocker.patch("ahriman.application.application.Application.add")
+    extract_mock = mocker.patch("ahriman.application.handlers.Rebuild.extract_packages", return_value=[])
+
+    Rebuild.run(args, "x86_64", configuration, True, False)
+    extract_mock.assert_called_once_with(pytest.helpers.anyvar(int))
+
+
 def test_run_dry_run(args: argparse.Namespace, configuration: Configuration,
                      package_ahriman: Package, mocker: MockerFixture) -> None:
     """
@@ -116,3 +133,12 @@ def test_run_build_empty_exception(args: argparse.Namespace, configuration: Conf
 
     Rebuild.run(args, "x86_64", configuration, True, False)
     check_mock.assert_has_calls([mock.call(True, False), mock.call(True, True)])
+
+
+def test_extract_packages(application: Application, mocker: MockerFixture) -> None:
+    """
+    must extract packages from database
+    """
+    packages_mock = mocker.patch("ahriman.core.database.sqlite.SQLite.packages_get")
+    Rebuild.extract_packages(application)
+    packages_mock.assert_called_once_with()

tests/ahriman/application/handlers/test_handler_restore.py

@@ -0,0 +1,39 @@
+import argparse
+
+from pathlib import Path
+from pytest_mock import MockerFixture
+from unittest.mock import MagicMock
+
+from ahriman.application.handlers import Restore
+from ahriman.core.configuration import Configuration
+
+
+def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
+    args.path = Path("result.tar.gz")
+    args.output = Path.cwd()
+    return args
+
+
+def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must run command
+    """
+    args = _default_args(args)
+    tarfile = MagicMock()
+    extract_mock = tarfile.__enter__.return_value = MagicMock()
+    mocker.patch("tarfile.TarFile.__new__", return_value=tarfile)
+
+    Restore.run(args, "x86_64", configuration, True, False)
+    extract_mock.extractall.assert_called_once_with(path=args.output)
+
+
+def test_disallow_auto_architecture_run() -> None:
+    """
+    must not allow multi architecture run
+    """
+    assert not Restore.ALLOW_AUTO_ARCHITECTURE_RUN

tests/ahriman/application/test_ahriman.py

@@ -6,7 +6,6 @@ from pytest_mock import MockerFixture
 from ahriman.application.handlers import Handler
 from ahriman.models.action import Action
 from ahriman.models.build_status import BuildStatusEnum
-from ahriman.models.package_source import PackageSource
 from ahriman.models.sign_settings import SignSettings
 from ahriman.models.user_access import UserAccess
 
@@ -259,6 +258,25 @@ def test_subparsers_patch_remove_architecture(parser: argparse.ArgumentParser) -
     assert args.architecture == [""]
 
 
+def test_subparsers_repo_backup(parser: argparse.ArgumentParser) -> None:
+    """
+    repo-backup command must imply architecture list, lock, no-report and unsafe
+    """
+    args = parser.parse_args(["repo-backup", "output.zip"])
+    assert args.architecture == [""]
+    assert args.lock is None
+    assert args.no_report
+    assert args.unsafe
+
+
+def test_subparsers_repo_backup_architecture(parser: argparse.ArgumentParser) -> None:
+    """
+    repo-backup command must correctly parse architecture list
+    """
+    args = parser.parse_args(["-a", "x86_64", "repo-backup", "output.zip"])
+    assert args.architecture == [""]
+
+
 def test_subparsers_repo_check(parser: argparse.ArgumentParser) -> None:
     """
     repo-check command must imply dry-run, no-aur and no-manual
@@ -342,21 +360,21 @@ def test_subparsers_repo_report_architecture(parser: argparse.ArgumentParser) ->
 
 def test_subparsers_repo_restore(parser: argparse.ArgumentParser) -> None:
     """
-    repo-restore command must imply package and source
+    repo-restore command must imply architecture list, lock, no-report and unsafe
     """
-    args = parser.parse_args(["repo-restore"])
-    assert args.package is None
-    assert args.source == PackageSource.AUR
+    args = parser.parse_args(["repo-restore", "output.zip"])
+    assert args.architecture == [""]
+    assert args.lock is None
+    assert args.no_report
+    assert args.unsafe
 
 
 def test_subparsers_repo_restore_architecture(parser: argparse.ArgumentParser) -> None:
     """
     repo-restore command must correctly parse architecture list
     """
-    args = parser.parse_args(["repo-restore"])
-    assert args.architecture is None
-    args = parser.parse_args(["-a", "x86_64", "repo-restore"])
-    assert args.architecture == ["x86_64"]
+    args = parser.parse_args(["-a", "x86_64", "repo-restore", "output.zip"])
+    assert args.architecture == [""]
 
 
 def test_subparsers_repo_setup(parser: argparse.ArgumentParser) -> None:

tests/ahriman/core/build_tools/test_sources.py

@@ -87,7 +87,7 @@ def test_fetch_new(mocker: MockerFixture) -> None:
     local = Path("local")
     Sources.fetch(local, "remote")
     check_output_mock.assert_has_calls([
-        mock.call("git", "clone", "remote", str(local), exception=None, logger=pytest.helpers.anyvar(int)),
+        mock.call("git", "clone", "remote", str(local), exception=None, cwd=local, logger=pytest.helpers.anyvar(int)),
         mock.call("git", "checkout", "--force", Sources._branch,
                   exception=None, cwd=local, logger=pytest.helpers.anyvar(int)),
         mock.call("git", "reset", "--hard", f"origin/{Sources._branch}",

tests/ahriman/core/report/test_report.py

@@ -53,3 +53,12 @@ def test_report_html(configuration: Configuration, result: Result, mocker: Mocke
     report_mock = mocker.patch("ahriman.core.report.html.HTML.generate")
     Report.load("x86_64", configuration, "html").run([], result)
     report_mock.assert_called_once_with([], result)
+
+
+def test_report_telegram(configuration: Configuration, result: Result, mocker: MockerFixture) -> None:
+    """
+    must generate telegram report
+    """
+    report_mock = mocker.patch("ahriman.core.report.telegram.Telegram.generate")
+    Report.load("x86_64", configuration, "telegram").run([], result)
+    report_mock.assert_called_once_with([], result)

tests/ahriman/core/report/test_telegram.py

@@ -0,0 +1,83 @@
+import pytest
+import requests
+
+from pytest_mock import MockerFixture
+from unittest import mock
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.report.telegram import Telegram
+from ahriman.models.package import Package
+from ahriman.models.result import Result
+
+
+def test_send(configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must send a message
+    """
+    request_mock = mocker.patch("requests.post")
+    report = Telegram("x86_64", configuration, "telegram")
+
+    report._send("a text")
+    request_mock.assert_called_once_with(
+        pytest.helpers.anyvar(str, strict=True),
+        data={"chat_id": pytest.helpers.anyvar(str, strict=True), "text": "a text", "parse_mode": "HTML"})
+
+
+def test_send_failed(configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must reraise generic exception
+    """
+    mocker.patch("requests.post", side_effect=Exception())
+    report = Telegram("x86_64", configuration, "telegram")
+
+    with pytest.raises(Exception):
+        report._send("a text")
+
+
+def test_make_request_failed_http_error(configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must reraise http exception
+    """
+    mocker.patch("requests.post", side_effect=requests.exceptions.HTTPError())
+    report = Telegram("x86_64", configuration, "telegram")
+
+    with pytest.raises(requests.exceptions.HTTPError):
+        report._send("a text")
+
+
+def test_generate(configuration: Configuration, package_ahriman: Package, result: Result,
+                  mocker: MockerFixture) -> None:
+    """
+    must generate report
+    """
+    send_mock = mocker.patch("ahriman.core.report.telegram.Telegram._send")
+
+    report = Telegram("x86_64", configuration, "telegram")
+    report.generate([package_ahriman], result)
+    send_mock.assert_called_once_with(pytest.helpers.anyvar(int))
+
+
+def test_generate_big_text(configuration: Configuration, package_ahriman: Package, result: Result,
+                           mocker: MockerFixture) -> None:
+    """
+    must generate report with big text
+    """
+    mocker.patch("ahriman.core.report.jinja_template.JinjaTemplate.make_html", return_value="a\n" * 4096)
+    send_mock = mocker.patch("ahriman.core.report.telegram.Telegram._send")
+
+    report = Telegram("x86_64", configuration, "telegram")
+    report.generate([package_ahriman], result)
+    send_mock.assert_has_calls([
+        mock.call(pytest.helpers.anyvar(str, strict=True)), mock.call(pytest.helpers.anyvar(str, strict=True))
+    ])
+
+
+def test_generate_no_empty(configuration: Configuration, package_ahriman: Package, mocker: MockerFixture) -> None:
+    """
+    must generate report
+    """
+    send_mock = mocker.patch("ahriman.core.report.telegram.Telegram._send")
+
+    report = Telegram("x86_64", configuration, "telegram")
+    report.generate([package_ahriman], Result())
+    send_mock.assert_not_called()

tests/ahriman/core/test_util.py

@@ -329,6 +329,7 @@ def test_walk(resource_path_root: Path) -> None:
         resource_path_root / "web" / "templates" / "build-status.jinja2",
         resource_path_root / "web" / "templates" / "email-index.jinja2",
         resource_path_root / "web" / "templates" / "repo-index.jinja2",
+        resource_path_root / "web" / "templates" / "telegram-index.jinja2",
     ])
     local_files = list(sorted(walk(resource_path_root)))
     assert local_files == expected

tests/ahriman/models/test_report_settings.py

@@ -24,3 +24,6 @@ def test_from_option_valid() -> None:
 
     assert ReportSettings.from_option("console") == ReportSettings.Console
     assert ReportSettings.from_option("conSOle") == ReportSettings.Console
+
+    assert ReportSettings.from_option("telegram") == ReportSettings.Telegram
+    assert ReportSettings.from_option("TElegraM") == ReportSettings.Telegram

tests/testresources/core/ahriman.ini

@@ -52,6 +52,13 @@ homepage =
 link_path =
 template_path = ../web/templates/repo-index.jinja2
 
+[telegram]
+api_key = apikey
+chat_id = @ahrimantestchat
+homepage =
+link_path =
+template_path = ../web/templates/telegram-index.jinja2
+
 [upload]
 target =