add backup and restore subcommands

docs/faq.md

@@ -242,7 +242,7 @@ You can pass any of these variables by using `-e` argument, e.g.:
 docker run -e AHRIMAN_PORT=8080 arcan1s/ahriman:latest
 ```
 
-### Working with web service
+### Web service setup
 
 Well for that you would need to have web container instance running forever; it can be achieved by the following command:
 
@@ -519,6 +519,36 @@ curl 'https://api.telegram.org/bot${CHAT_ID}/sendMessage?chat_id=${API_KEY}&text
 5. Create end-user `sudo -u ahriman ahriman user-add -r write my-first-user`. When it will ask for the password leave it blank.
 6. Restart web service `systemctl restart ahriman-web@x86_64`.
 
+## Backup and restore
+
+The service provides several commands aim to do easy repository backup and restore. If you would like to move repository from the server `server1.example.com` to another `server2.example.com` you have to perform the following steps:
+
+1. On the source server `server1.example.com` run `repo-backup` command, e.g.:
+
+   ```shell
+   sudo ahriman repo-backup /tmp/repo.tar.gz
+   ```
+   
+   This command will pack all configuration files together with database file into the archive specified as command line argument (i.e. `/tmp/repo.tar.gz`). In addition it will also archive `cache` directory (the one which contains local clones used by e.g. local packages) and `.gnupg` of the `ahriman` user.
+
+2. Copy created archive from source server `server1.example.com` to target `server2.example.com`.
+
+3. Install ahriman as usual on the target server `server2.example.com` if you didn't yet.
+
+4. Extract archive e.g. by using subcommand:
+   
+   ```shell
+   sudo ahriman repo-restore /tmp/repo.tar.gz
+   ```
+   
+   An additional argument `-o`/`--output` can be used to specify extraction root (`/` by default).
+
+5. Rebuild repository:
+
+   ```shell
+   sudo -u ahriman ahriman repo-rebuild --from-database
+   ```
+
 ## Other topics
 
 ### How does it differ from %another-manager%?
@@ -558,6 +588,10 @@ Though originally I've created ahriman by trying to improve the project, it stil
 
 `repo-scripts` also have bad architecture and bad quality code and uses out-of-dated `yaourt` and `package-query`.
 
+#### [toolbox](https://github.com/chaotic-aur/toolbox)
+
+It is automation tools for `repoctl` mentioned above. Except for using shell it looks pretty cool and also offers some additional features like patches, remote synchronization (isn't it?) and reporting.
+
 ### I would like to check service logs
 
 By default, the service writes logs to `/dev/log` which can be accessed by using `journalctl` command (logs are written to the journal of the user under which command is run).

src/ahriman/application/ahriman.py

@@ -83,12 +83,14 @@ def _parser() -> argparse.ArgumentParser:
     _set_patch_add_parser(subparsers)
     _set_patch_list_parser(subparsers)
     _set_patch_remove_parser(subparsers)
+    _set_repo_backup_parser(subparsers)
     _set_repo_check_parser(subparsers)
     _set_repo_clean_parser(subparsers)
     _set_repo_config_parser(subparsers)
     _set_repo_rebuild_parser(subparsers)
     _set_repo_remove_unknown_parser(subparsers)
     _set_repo_report_parser(subparsers)
+    _set_repo_restore_parser(subparsers)
     _set_repo_setup_parser(subparsers)
     _set_repo_sign_parser(subparsers)
     _set_repo_status_update_parser(subparsers)
@@ -314,6 +316,19 @@ def _set_patch_remove_parser(root: SubParserAction) -> argparse.ArgumentParser:
     return parser
 
 
+def _set_repo_backup_parser(root: SubParserAction) -> argparse.ArgumentParser:
+    """
+    add parser for repository backup subcommand
+    :param root: subparsers for the commands
+    :return: created argument parser
+    """
+    parser = root.add_parser("repo-backup", help="backup repository data",
+                             description="backup settings and database", formatter_class=_formatter)
+    parser.add_argument("path", help="path of the output archive", type=Path)
+    parser.set_defaults(handler=handlers.Backup, architecture=[""], lock=None, no_report=True, unsafe=True)
+    return parser
+
+
 def _set_repo_check_parser(root: SubParserAction) -> argparse.ArgumentParser:
     """
     add parser for repository check subcommand
@@ -415,6 +430,20 @@ def _set_repo_report_parser(root: SubParserAction) -> argparse.ArgumentParser:
     return parser
 
 
+def _set_repo_restore_parser(root: SubParserAction) -> argparse.ArgumentParser:
+    """
+    add parser for repository restore subcommand
+    :param root: subparsers for the commands
+    :return: created argument parser
+    """
+    parser = root.add_parser("repo-restore", help="restore repository data",
+                             description="restore settings and database", formatter_class=_formatter)
+    parser.add_argument("path", help="path of the input archive", type=Path)
+    parser.add_argument("-o", "--output", help="root path of the extracted files", type=Path, default=Path("/"))
+    parser.set_defaults(handler=handlers.Restore, architecture=[""], lock=None, no_report=True, unsafe=True)
+    return parser
+
+
 def _set_repo_setup_parser(root: SubParserAction) -> argparse.ArgumentParser:
     """
     add parser for setup subcommand

src/ahriman/application/handlers/__init__.py

@@ -20,6 +20,7 @@
 from ahriman.application.handlers.handler import Handler
 
 from ahriman.application.handlers.add import Add
+from ahriman.application.handlers.backup import Backup
 from ahriman.application.handlers.clean import Clean
 from ahriman.application.handlers.dump import Dump
 from ahriman.application.handlers.help import Help
@@ -29,6 +30,7 @@ from ahriman.application.handlers.rebuild import Rebuild
 from ahriman.application.handlers.remove import Remove
 from ahriman.application.handlers.remove_unknown import RemoveUnknown
 from ahriman.application.handlers.report import Report
+from ahriman.application.handlers.restore import Restore
 from ahriman.application.handlers.search import Search
 from ahriman.application.handlers.setup import Setup
 from ahriman.application.handlers.sign import Sign

src/ahriman/application/handlers/backup.py

@@ -0,0 +1,80 @@
+#
+# Copyright (c) 2021-2022 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+import pwd
+
+from pathlib import Path
+from tarfile import TarFile
+from typing import Set, Type
+
+from ahriman.application.handlers.handler import Handler
+from ahriman.core.configuration import Configuration
+from ahriman.core.database.sqlite import SQLite
+
+
+class Backup(Handler):
+    """
+    backup packages handler
+    """
+
+    ALLOW_AUTO_ARCHITECTURE_RUN = False  # it should be called only as "no-architecture"
+
+    @classmethod
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool, unsafe: bool) -> None:
+        """
+        callback for command line
+        :param args: command line args
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
+        :param unsafe: if set no user check will be performed before path creation
+        """
+        backup_paths = Backup.get_paths(configuration)
+        with TarFile(args.path, mode="w") as archive:  # well we don't actually use compression
+            for backup_path in backup_paths:
+                archive.add(backup_path)
+
+    @staticmethod
+    def get_paths(configuration: Configuration) -> Set[Path]:
+        """
+        extract paths to backup
+        :param configuration: configuration instance
+        :return: map of the filesystem paths
+        """
+        paths = set(configuration.include.glob("*.ini"))
+
+        root, _ = configuration.check_loaded()
+        paths.add(root)  # the configuration itself
+        paths.add(SQLite.database_path(configuration))  # database
+
+        # local caches
+        repository_paths = configuration.repository_paths
+        if repository_paths.cache.is_dir():
+            paths.add(repository_paths.cache)
+
+        # gnupg home with imported keys
+        uid, _ = repository_paths.root_owner
+        system_user = pwd.getpwuid(uid)
+        gnupg_home = Path(system_user.pw_dir) / ".gnupg"
+        if gnupg_home.is_dir():
+            paths.add(gnupg_home)
+
+        return paths

src/ahriman/application/handlers/restore.py

@@ -0,0 +1,48 @@
+#
+# Copyright (c) 2021-2022 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import argparse
+
+from typing import Type
+from tarfile import TarFile
+
+from ahriman.application.handlers.handler import Handler
+from ahriman.core.configuration import Configuration
+
+
+class Restore(Handler):
+    """
+    restore packages handler
+    """
+
+    ALLOW_AUTO_ARCHITECTURE_RUN = False  # it should be called only as "no-architecture"
+
+    @classmethod
+    def run(cls: Type[Handler], args: argparse.Namespace, architecture: str,
+            configuration: Configuration, no_report: bool, unsafe: bool) -> None:
+        """
+        callback for command line
+        :param args: command line args
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param no_report: force disable reporting
+        :param unsafe: if set no user check will be performed before path creation
+        """
+        with TarFile(args.path) as archive:
+            archive.extractall(path=args.output)

tests/ahriman/application/handlers/test_handler_backup.py

@@ -0,0 +1,58 @@
+import argparse
+
+from pathlib import Path
+from pytest_mock import MockerFixture
+from unittest.mock import MagicMock
+
+from ahriman.application.handlers import Backup
+from ahriman.core.configuration import Configuration
+from ahriman.models.repository_paths import RepositoryPaths
+
+
+def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
+    args.path = Path("result.tar.gz")
+    return args
+
+
+def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must run command
+    """
+    args = _default_args(args)
+    mocker.patch("ahriman.application.handlers.Backup.get_paths", return_value=[Path("path")])
+    tarfile = MagicMock()
+    add_mock = tarfile.__enter__.return_value = MagicMock()
+    mocker.patch("tarfile.TarFile.__new__", return_value=tarfile)
+
+    Backup.run(args, "x86_64", configuration, True, False)
+    add_mock.add.assert_called_once_with(Path("path"))
+
+
+def test_get_paths(configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must get paths to be archived
+    """
+    # gnupg export mock
+    mocker.patch("pathlib.Path.is_dir", return_value=True)
+    mocker.patch.object(RepositoryPaths, "root_owner", (42, 42))
+    getpwuid_mock = mocker.patch("pwd.getpwuid", return_value=MagicMock())
+    # well database does not exist so we override it
+    database_mock = mocker.patch("ahriman.core.database.sqlite.SQLite.database_path", return_value=configuration.path)
+
+    paths = Backup.get_paths(configuration)
+    getpwuid_mock.assert_called_once_with(42)
+    database_mock.assert_called_once_with(configuration)
+    assert configuration.path in paths
+    assert all(path.exists() for path in paths if path.name not in (".gnupg", "cache"))
+
+
+def test_disallow_auto_architecture_run() -> None:
+    """
+    must not allow multi architecture run
+    """
+    assert not Backup.ALLOW_AUTO_ARCHITECTURE_RUN

tests/ahriman/application/handlers/test_handler_rebuild.py

@@ -50,6 +50,7 @@ def test_run_extract_packages(args: argparse.Namespace, configuration: Configura
     """
     args = _default_args(args)
     args.from_database = True
+    args.dry_run = True
     mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create")
     mocker.patch("ahriman.application.application.Application.add")
     extract_mock = mocker.patch("ahriman.application.handlers.Rebuild.extract_packages", return_value=[])

tests/ahriman/application/handlers/test_handler_restore.py

@@ -0,0 +1,39 @@
+import argparse
+
+from pathlib import Path
+from pytest_mock import MockerFixture
+from unittest.mock import MagicMock
+
+from ahriman.application.handlers import Restore
+from ahriman.core.configuration import Configuration
+
+
+def _default_args(args: argparse.Namespace) -> argparse.Namespace:
+    """
+    default arguments for these test cases
+    :param args: command line arguments fixture
+    :return: generated arguments for these test cases
+    """
+    args.path = Path("result.tar.gz")
+    args.output = Path.cwd()
+    return args
+
+
+def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must run command
+    """
+    args = _default_args(args)
+    tarfile = MagicMock()
+    extract_mock = tarfile.__enter__.return_value = MagicMock()
+    mocker.patch("tarfile.TarFile.__new__", return_value=tarfile)
+
+    Restore.run(args, "x86_64", configuration, True, False)
+    extract_mock.extractall.assert_called_once_with(path=args.output)
+
+
+def test_disallow_auto_architecture_run() -> None:
+    """
+    must not allow multi architecture run
+    """
+    assert not Restore.ALLOW_AUTO_ARCHITECTURE_RUN

tests/ahriman/application/test_ahriman.py

@@ -258,6 +258,25 @@ def test_subparsers_patch_remove_architecture(parser: argparse.ArgumentParser) -
     assert args.architecture == [""]
 
 
+def test_subparsers_repo_backup(parser: argparse.ArgumentParser) -> None:
+    """
+    repo-backup command must imply architecture list, lock, no-report and unsafe
+    """
+    args = parser.parse_args(["repo-backup", "output.zip"])
+    assert args.architecture == [""]
+    assert args.lock is None
+    assert args.no_report
+    assert args.unsafe
+
+
+def test_subparsers_repo_backup_architecture(parser: argparse.ArgumentParser) -> None:
+    """
+    repo-backup command must correctly parse architecture list
+    """
+    args = parser.parse_args(["-a", "x86_64", "repo-backup", "output.zip"])
+    assert args.architecture == [""]
+
+
 def test_subparsers_repo_check(parser: argparse.ArgumentParser) -> None:
     """
     repo-check command must imply dry-run, no-aur and no-manual
@@ -339,6 +358,25 @@ def test_subparsers_repo_report_architecture(parser: argparse.ArgumentParser) ->
     assert args.architecture == ["x86_64"]
 
 
+def test_subparsers_repo_restore(parser: argparse.ArgumentParser) -> None:
+    """
+    repo-restore command must imply architecture list, lock, no-report and unsafe
+    """
+    args = parser.parse_args(["repo-restore", "output.zip"])
+    assert args.architecture == [""]
+    assert args.lock is None
+    assert args.no_report
+    assert args.unsafe
+
+
+def test_subparsers_repo_restore_architecture(parser: argparse.ArgumentParser) -> None:
+    """
+    repo-restore command must correctly parse architecture list
+    """
+    args = parser.parse_args(["-a", "x86_64", "repo-restore", "output.zip"])
+    assert args.architecture == [""]
+
+
 def test_subparsers_repo_setup(parser: argparse.ArgumentParser) -> None:
     """
     repo-setup command must imply lock, no-report, quiet and unsafe