Release 2.0.0rc7

.github/workflows/docker-image.yml

@@ -5,6 +5,7 @@ on:
     branches: [ master ]
     tags:
       - '*'
+      - '!*rc*'
 
 jobs:
   docker-image:

docs/ahriman.1

@@ -3,7 +3,7 @@
 ahriman
 .SH SYNOPSIS
 .B ahriman
-[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--no-report] [-q] [--unsafe] [-v] {aur-search,search,help,help-commands-unsafe,key-import,package-add,add,package-update,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,repo-check,check,repo-clean,clean,repo-config,config,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,restore,repo-setup,init,repo-init,setup,repo-sign,sign,repo-status-update,repo-sync,sync,repo-update,update,user-add,user-list,user-remove,web} ...
+[-h] [-a ARCHITECTURE] [-c CONFIGURATION] [--force] [-l LOCK] [--no-report] [-q] [--unsafe] [-v] {aur-search,search,help,help-commands-unsafe,key-import,package-add,add,package-update,package-remove,remove,package-status,status,package-status-remove,package-status-update,status-update,patch-add,patch-list,patch-remove,repo-backup,repo-check,check,repo-clean,clean,repo-config,config,repo-rebuild,rebuild,repo-remove-unknown,remove-unknown,repo-report,report,repo-restore,repo-setup,init,repo-init,setup,repo-sign,sign,repo-status-update,repo-sync,sync,repo-update,update,user-add,user-list,user-remove,web} ...
 .SH DESCRIPTION
 ArcH Linux ReposItory MANager
 .SH OPTIONS
@@ -79,6 +79,9 @@ list patch sets
 \fBahriman\fR \fI\,patch-remove\/\fR
 remove patch set
 .TP
+\fBahriman\fR \fI\,repo-backup\/\fR
+backup repository data
+.TP
 \fBahriman\fR \fI\,repo-check\/\fR
 check for updates
 .TP
@@ -98,7 +101,7 @@ remove unknown packages
 generate report
 .TP
 \fBahriman\fR \fI\,repo-restore\/\fR
-restore repository
+restore repository data
 .TP
 \fBahriman\fR \fI\,repo-setup\/\fR
 initial service configuration
@@ -442,6 +445,16 @@ remove patches for the package
 package base
 
 
+.SH OPTIONS 'ahriman repo-backup'
+usage: ahriman repo-backup [-h] path
+
+backup settings and database
+
+.TP
+\fBpath\fR
+path of the output archive
+
+
 .SH OPTIONS 'ahriman repo-check'
 usage: ahriman repo-check [-h] [-e] [--no-vcs] [package ...]
 
@@ -535,7 +548,7 @@ dump configuration for the specified architecture
 
 
 .SH OPTIONS 'ahriman repo-rebuild'
-usage: ahriman repo-rebuild [-h] [--depends-on DEPENDS_ON] [--dry-run] [-e]
+usage: ahriman repo-rebuild [-h] [--depends-on DEPENDS_ON] [--dry-run] [--from-database] [-e]
 
 force rebuild whole repository
 
@@ -548,12 +561,18 @@ only rebuild packages that depend on specified package
 \fB\-\-dry\-run\fR
 just perform check for packages without rebuild process itself
 
+.TP
+\fB\-\-from\-database\fR
+read packages from database instead of filesystem. This feature in particular is required in case if you would like to
+restore repository from another repository instance. Note however that in order to restore packages you need to have
+original ahriman instance run with web service and have run repo\-update at least once.
+
 .TP
 \fB\-e\fR, \fB\-\-exit\-code\fR
 return non\-zero exit status if result is empty
 
 .SH OPTIONS 'ahriman rebuild'
-usage: ahriman repo-rebuild [-h] [--depends-on DEPENDS_ON] [--dry-run] [-e]
+usage: ahriman repo-rebuild [-h] [--depends-on DEPENDS_ON] [--dry-run] [--from-database] [-e]
 
 force rebuild whole repository
 
@@ -566,6 +585,12 @@ only rebuild packages that depend on specified package
 \fB\-\-dry\-run\fR
 just perform check for packages without rebuild process itself
 
+.TP
+\fB\-\-from\-database\fR
+read packages from database instead of filesystem. This feature in particular is required in case if you would like to
+restore repository from another repository instance. Note however that in order to restore packages you need to have
+original ahriman instance run with web service and have run repo\-update at least once.
+
 .TP
 \fB\-e\fR, \fB\-\-exit\-code\fR
 return non\-zero exit status if result is empty
@@ -619,40 +644,17 @@ target to generate report
 
 
 .SH OPTIONS 'ahriman repo-restore'
-usage: ahriman repo-restore [-h] [-e] [-n] [--without-dependencies]
-
-restore repository from database file
+usage: ahriman repo-restore [-h] [-o OUTPUT] path
 
+restore settings and database
 
 .TP
-\fB\-e\fR, \fB\-\-exit\-code\fR
-return non\-zero exit status if result is empty
+\fBpath\fR
+path of the input archive
 
 .TP
-\fB\-n\fR, \fB\-\-now\fR
-run update function after
-
-.TP
-\fB\-\-without\-dependencies\fR
-do not add dependencies
-
-.SH OPTIONS 'ahriman restore'
-usage: ahriman repo-restore [-h] [-e] [-n] [--without-dependencies]
-
-restore repository from database file
-
-
-.TP
-\fB\-e\fR, \fB\-\-exit\-code\fR
-return non\-zero exit status if result is empty
-
-.TP
-\fB\-n\fR, \fB\-\-now\fR
-run update function after
-
-.TP
-\fB\-\-without\-dependencies\fR
-do not add dependencies
+\fB\-o\fR \fI\,OUTPUT\/\fR, \fB\-\-output\fR \fI\,OUTPUT\/\fR
+root path of the extracted files
 
 .SH OPTIONS 'ahriman repo-setup'
 usage: ahriman repo-setup [-h] [--build-as-user BUILD_AS_USER] [--build-command BUILD_COMMAND]

docs/faq.md

@@ -199,7 +199,7 @@ server {
 
 ## Docker image
 
-We provide official images which can be found under `arcan1s/ahriman` repository. Docker image is being updated on each master commit as well as on each version. If you would like to use last (probably unstable build) you can use `latest` tag; otherwise you can use any version tag available. 
+We provide official images which can be found under `arcan1s/ahriman` repository. Docker image is being updated on each master commit as well as on each version. If you would like to use last (probably unstable) build you can use `edge` tag or `latest` for any tagged versions; otherwise you can use any version tag available. 
 
 The default action (in case if no arguments provided) is `repo-update`. Basically the idea is to run container, e.g.:
 
@@ -242,7 +242,7 @@ You can pass any of these variables by using `-e` argument, e.g.:
 docker run -e AHRIMAN_PORT=8080 arcan1s/ahriman:latest
 ```
 
-### Working with web service
+### Web service setup
 
 Well for that you would need to have web container instance running forever; it can be achieved by the following command:
 
@@ -410,7 +410,7 @@ After these steps `index.html` file will be automatically synced to S3
    yay -S python-jinja
    ```
 
-2. Register bot in telegram. You can do it by using by talking with [@BotFather](https://t.me/botfather). For more details please refer to [official documentation](https://core.telegram.org/bots).
+2. Register bot in telegram. You can do it by talking with [@BotFather](https://t.me/botfather). For more details please refer to [official documentation](https://core.telegram.org/bots).
 
 3. Optionally (if you want to post message in chat):
 
@@ -519,6 +519,36 @@ curl 'https://api.telegram.org/bot${CHAT_ID}/sendMessage?chat_id=${API_KEY}&text
 5. Create end-user `sudo -u ahriman ahriman user-add -r write my-first-user`. When it will ask for the password leave it blank.
 6. Restart web service `systemctl restart ahriman-web@x86_64`.
 
+## Backup and restore
+
+The service provides several commands aim to do easy repository backup and restore. If you would like to move repository from the server `server1.example.com` to another `server2.example.com` you have to perform the following steps:
+
+1. On the source server `server1.example.com` run `repo-backup` command, e.g.:
+
+   ```shell
+   sudo ahriman repo-backup /tmp/repo.tar.gz
+   ```
+   
+   This command will pack all configuration files together with database file into the archive specified as command line argument (i.e. `/tmp/repo.tar.gz`). In addition it will also archive `cache` directory (the one which contains local clones used by e.g. local packages) and `.gnupg` of the `ahriman` user.
+
+2. Copy created archive from source server `server1.example.com` to target `server2.example.com`.
+
+3. Install ahriman as usual on the target server `server2.example.com` if you didn't yet.
+
+4. Extract archive e.g. by using subcommand:
+   
+   ```shell
+   sudo ahriman repo-restore /tmp/repo.tar.gz
+   ```
+   
+   An additional argument `-o`/`--output` can be used to specify extraction root (`/` by default).
+
+5. Rebuild repository:
+
+   ```shell
+   sudo -u ahriman ahriman repo-rebuild --from-database
+   ```
+
 ## Other topics
 
 ### How does it differ from %another-manager%?
@@ -558,6 +588,10 @@ Though originally I've created ahriman by trying to improve the project, it stil
 
 `repo-scripts` also have bad architecture and bad quality code and uses out-of-dated `yaourt` and `package-query`.
 
+#### [toolbox](https://github.com/chaotic-aur/toolbox)
+
+It is automation tools for `repoctl` mentioned above. Except for using shell it looks pretty cool and also offers some additional features like patches, remote synchronization (isn't it?) and reporting.
+
 ### I would like to check service logs
 
 By default, the service writes logs to `/dev/log` which can be accessed by using `journalctl` command (logs are written to the journal of the user under which command is run).
@@ -568,7 +602,7 @@ You can also edit configuration and forward logs to `stderr`, just change `handl
 sed -i 's/handlers = syslog_handler/handlers = console_handler/g' /etc/ahriman.ini.d/logging.ini
 ```
 
-You can even configure logging as you wish, but kindly refer to python `logging` module configuration.
+You can even configure logging as you wish, but kindly refer to python `logging` module [configuration](https://docs.python.org/3/library/logging.config.html).
 
 ### Html customization
 

package/archlinux/PKGBUILD

@@ -1,7 +1,7 @@
 # Maintainer: Evgeniy Alekseev
 
 pkgname='ahriman'
-pkgver=2.0.0rc6
+pkgver=2.0.0rc7
 pkgrel=1
 pkgdesc="ArcH Linux ReposItory MANager"
 arch=('any')

src/ahriman/application/ahriman.py

@@ -83,12 +83,14 @@ def _parser() -> argparse.ArgumentParser:
     _set_patch_add_parser(subparsers)
     _set_patch_list_parser(subparsers)
     _set_patch_remove_parser(subparsers)
+    _set_repo_backup_parser(subparsers)
     _set_repo_check_parser(subparsers)
     _set_repo_clean_parser(subparsers)
     _set_repo_config_parser(subparsers)
     _set_repo_rebuild_parser(subparsers)
     _set_repo_remove_unknown_parser(subparsers)
     _set_repo_report_parser(subparsers)
+    _set_repo_restore_parser(subparsers)
     _set_repo_setup_parser(subparsers)
     _set_repo_sign_parser(subparsers)
     _set_repo_status_update_parser(subparsers)
@@ -314,6 +316,19 @@ def _set_patch_remove_parser(root: SubParserAction) -> argparse.ArgumentParser:
     return parser
 
 
+def _set_repo_backup_parser(root: SubParserAction) -> argparse.ArgumentParser:
+    """
+    add parser for repository backup subcommand
+    :param root: subparsers for the commands
+    :return: created argument parser
+    """
+    parser = root.add_parser("repo-backup", help="backup repository data",
+                             description="backup settings and database", formatter_class=_formatter)
+    parser.add_argument("path", help="path of the output archive", type=Path)
+    parser.set_defaults(handler=handlers.Backup, architecture=[""], lock=None, no_report=True, unsafe=True)
+    return parser
+
+
 def _set_repo_check_parser(root: SubParserAction) -> argparse.ArgumentParser:
     """
     add parser for repository check subcommand
@@ -415,6 +430,20 @@ def _set_repo_report_parser(root: SubParserAction) -> argparse.ArgumentParser:
     return parser
 
 
+def _set_repo_restore_parser(root: SubParserAction) -> argparse.ArgumentParser:
+    """
+    add parser for repository restore subcommand
+    :param root: subparsers for the commands
+    :return: created argument parser
+    """
+    parser = root.add_parser("repo-restore", help="restore repository data",
+                             description="restore settings and database", formatter_class=_formatter)
+    parser.add_argument("path", help="path of the input archive", type=Path)
+    parser.add_argument("-o", "--output", help="root path of the extracted files", type=Path, default=Path("/"))
+    parser.set_defaults(handler=handlers.Restore, architecture=[""], lock=None, no_report=True, unsafe=True)
+    return parser
+
+
 def _set_repo_setup_parser(root: SubParserAction) -> argparse.ArgumentParser:
     """
     add parser for setup subcommand

src/ahriman/application/handlers/__init__.py

@@ -20,6 +20,7 @@
 from ahriman.application.handlers.handler import Handler
 
 from ahriman.application.handlers.add import Add
+from ahriman.application.handlers.backup import Backup
 from ahriman.application.handlers.clean import Clean
 from ahriman.application.handlers.dump import Dump
 from ahriman.application.handlers.help import Help
@@ -29,6 +30,7 @@ from ahriman.application.handlers.rebuild import Rebuild
 from ahriman.application.handlers.remove import Remove
 from ahriman.application.handlers.remove_unknown import RemoveUnknown
 from ahriman.application.handlers.report import Report
+from ahriman.application.handlers.restore import Restore
 from ahriman.application.handlers.search import Search
 from ahriman.application.handlers.setup import Setup
 from ahriman.application.handlers.sign import Sign

src/ahriman/application/handlers/backup.py

@@ -0,0 +1,80 @@
+#
+# Copyright (c) 2021-2022 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+import pwd
+
+from pathlib import Path
+from tarfile import TarFile
+from typing import Set, Type
+
+from ahriman.application.handlers.handler import Handler
+from ahriman.core.configuration import Configuration
+from ahriman.core.database.sqlite import SQLite
+
+
+class Backup(Handler):
+    """
+    backup packages handler
+    """
+
+    ALLOW_AUTO_ARCHITECTURE_RUN = False  # it should be called only as "no-architecture"
+
+    @classmethod
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool, unsafe: bool) -> None:
+        """
+        callback for command line
+        :param args: command line args
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
+        :param unsafe: if set no user check will be performed before path creation
+        """
+        backup_paths = Backup.get_paths(configuration)
+        with TarFile(args.path, mode="w") as archive:  # well we don't actually use compression
+            for backup_path in backup_paths:
+                archive.add(backup_path)
+
+    @staticmethod
+    def get_paths(configuration: Configuration) -> Set[Path]:
+        """
+        extract paths to backup
+        :param configuration: configuration instance
+        :return: map of the filesystem paths
+        """
+        paths = set(configuration.include.glob("*.ini"))
+
+        root, _ = configuration.check_loaded()
+        paths.add(root)  # the configuration itself
+        paths.add(SQLite.database_path(configuration))  # database
+
+        # local caches
+        repository_paths = configuration.repository_paths
+        if repository_paths.cache.is_dir():
+            paths.add(repository_paths.cache)
+
+        # gnupg home with imported keys
+        uid, _ = repository_paths.root_owner
+        system_user = pwd.getpwuid(uid)
+        gnupg_home = Path(system_user.pw_dir) / ".gnupg"
+        if gnupg_home.is_dir():
+            paths.add(gnupg_home)
+
+        return paths

src/ahriman/application/handlers/restore.py

@@ -0,0 +1,48 @@
+#
+# Copyright (c) 2021-2022 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+
+from typing import Type
+from tarfile import TarFile
+
+from ahriman.application.handlers.handler import Handler
+from ahriman.core.configuration import Configuration
+
+
+class Restore(Handler):
+    """
+    restore packages handler
+    """
+
+    ALLOW_AUTO_ARCHITECTURE_RUN = False  # it should be called only as "no-architecture"
+
+    @classmethod
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool, unsafe: bool) -> None:
+        """
+        callback for command line
+        :param args: command line args
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
+        :param unsafe: if set no user check will be performed before path creation
+        """
+        with TarFile(args.path) as archive:
+            archive.extractall(path=args.output)

src/ahriman/version.py

@@ -17,4 +17,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-__version__ = "2.0.0rc6"
+__version__ = "2.0.0rc7"

tests/ahriman/application/handlers/test_handler_backup.py

@@ -0,0 +1,58 @@
+import argparse
+
+from pathlib import Path
+from pytest_mock import MockerFixture
+from unittest.mock import MagicMock
+
+from ahriman.application.handlers import Backup
+from ahriman.core.configuration import Configuration
+from ahriman.models.repository_paths import RepositoryPaths
+
+
+def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
+    args.path = Path("result.tar.gz")
+    return args
+
+
+def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must run command
+    """
+    args = _default_args(args)
+    mocker.patch("ahriman.application.handlers.Backup.get_paths", return_value=[Path("path")])
+    tarfile = MagicMock()
+    add_mock = tarfile.__enter__.return_value = MagicMock()
+    mocker.patch("tarfile.TarFile.__new__", return_value=tarfile)
+
+    Backup.run(args, "x86_64", configuration, True, False)
+    add_mock.add.assert_called_once_with(Path("path"))
+
+
+def test_get_paths(configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must get paths to be archived
+    """
+    # gnupg export mock
+    mocker.patch("pathlib.Path.is_dir", return_value=True)
+    mocker.patch.object(RepositoryPaths, "root_owner", (42, 42))
+    getpwuid_mock = mocker.patch("pwd.getpwuid", return_value=MagicMock())
+    # well database does not exist so we override it
+    database_mock = mocker.patch("ahriman.core.database.sqlite.SQLite.database_path", return_value=configuration.path)
+
+    paths = Backup.get_paths(configuration)
+    getpwuid_mock.assert_called_once_with(42)
+    database_mock.assert_called_once_with(configuration)
+    assert configuration.path in paths
+    assert all(path.exists() for path in paths if path.name not in (".gnupg", "cache"))
+
+
+def test_disallow_auto_architecture_run() -> None:
+    """
+    must not allow multi architecture run
+    """
+    assert not Backup.ALLOW_AUTO_ARCHITECTURE_RUN

tests/ahriman/application/handlers/test_handler_rebuild.py

@@ -50,6 +50,7 @@ def test_run_extract_packages(args: argparse.Namespace, configuration: Configura
     """
     args = _default_args(args)
     args.from_database = True
+    args.dry_run = True
     mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
     mocker.patch("ahriman.application.application.Application.add")
     extract_mock = mocker.patch("ahriman.application.handlers.Rebuild.extract_packages", return_value=[])

tests/ahriman/application/handlers/test_handler_restore.py

@@ -0,0 +1,39 @@
+import argparse
+
+from pathlib import Path
+from pytest_mock import MockerFixture
+from unittest.mock import MagicMock
+
+from ahriman.application.handlers import Restore
+from ahriman.core.configuration import Configuration
+
+
+def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
+    args.path = Path("result.tar.gz")
+    args.output = Path.cwd()
+    return args
+
+
+def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must run command
+    """
+    args = _default_args(args)
+    tarfile = MagicMock()
+    extract_mock = tarfile.__enter__.return_value = MagicMock()
+    mocker.patch("tarfile.TarFile.__new__", return_value=tarfile)
+
+    Restore.run(args, "x86_64", configuration, True, False)
+    extract_mock.extractall.assert_called_once_with(path=args.output)
+
+
+def test_disallow_auto_architecture_run() -> None:
+    """
+    must not allow multi architecture run
+    """
+    assert not Restore.ALLOW_AUTO_ARCHITECTURE_RUN

tests/ahriman/application/test_ahriman.py

@@ -258,6 +258,25 @@ def test_subparsers_patch_remove_architecture(parser: argparse.ArgumentParser) -
     assert args.architecture == [""]
 
 
+def test_subparsers_repo_backup(parser: argparse.ArgumentParser) -> None:
+    """
+    repo-backup command must imply architecture list, lock, no-report and unsafe
+    """
+    args = parser.parse_args(["repo-backup", "output.zip"])
+    assert args.architecture == [""]
+    assert args.lock is None
+    assert args.no_report
+    assert args.unsafe
+
+
+def test_subparsers_repo_backup_architecture(parser: argparse.ArgumentParser) -> None:
+    """
+    repo-backup command must correctly parse architecture list
+    """
+    args = parser.parse_args(["-a", "x86_64", "repo-backup", "output.zip"])
+    assert args.architecture == [""]
+
+
 def test_subparsers_repo_check(parser: argparse.ArgumentParser) -> None:
     """
     repo-check command must imply dry-run, no-aur and no-manual
@@ -339,6 +358,25 @@ def test_subparsers_repo_report_architecture(parser: argparse.ArgumentParser) ->
     assert args.architecture == ["x86_64"]
 
 
+def test_subparsers_repo_restore(parser: argparse.ArgumentParser) -> None:
+    """
+    repo-restore command must imply architecture list, lock, no-report and unsafe
+    """
+    args = parser.parse_args(["repo-restore", "output.zip"])
+    assert args.architecture == [""]
+    assert args.lock is None
+    assert args.no_report
+    assert args.unsafe
+
+
+def test_subparsers_repo_restore_architecture(parser: argparse.ArgumentParser) -> None:
+    """
+    repo-restore command must correctly parse architecture list
+    """
+    args = parser.parse_args(["-a", "x86_64", "repo-restore", "output.zip"])
+    assert args.architecture == [""]
+
+
 def test_subparsers_repo_setup(parser: argparse.ArgumentParser) -> None:
     """
     repo-setup command must imply lock, no-report, quiet and unsafe