From cb63bc08ff49d90aab9aa834b199491dd3ac49ed Mon Sep 17 00:00:00 2001 From: Evgeniy Alekseev Date: Sun, 10 Apr 2022 21:34:34 +0300 Subject: [PATCH] add backup and restore subcommands --- docs/faq.md | 36 ++++++++- src/ahriman/application/ahriman.py | 29 +++++++ src/ahriman/application/handlers/__init__.py | 2 + src/ahriman/application/handlers/backup.py | 80 +++++++++++++++++++ src/ahriman/application/handlers/restore.py | 48 +++++++++++ .../handlers/test_handler_backup.py | 58 ++++++++++++++ .../handlers/test_handler_rebuild.py | 1 + .../handlers/test_handler_restore.py | 39 +++++++++ tests/ahriman/application/test_ahriman.py | 38 +++++++++ 9 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 src/ahriman/application/handlers/backup.py create mode 100644 src/ahriman/application/handlers/restore.py create mode 100644 tests/ahriman/application/handlers/test_handler_backup.py create mode 100644 tests/ahriman/application/handlers/test_handler_restore.py diff --git a/docs/faq.md b/docs/faq.md index 3a369163..a76e8287 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -242,7 +242,7 @@ You can pass any of these variables by using `-e` argument, e.g.: docker run -e AHRIMAN_PORT=8080 arcan1s/ahriman:latest ``` -### Working with web service +### Web service setup Well for that you would need to have web container instance running forever; it can be achieved by the following command: @@ -519,6 +519,36 @@ curl 'https://api.telegram.org/bot${CHAT_ID}/sendMessage?chat_id=${API_KEY}&text 5. Create end-user `sudo -u ahriman ahriman user-add -r write my-first-user`. When it will ask for the password leave it blank. 6. Restart web service `systemctl restart ahriman-web@x86_64`. +## Backup and restore + +The service provides several commands aim to do easy repository backup and restore. If you would like to move repository from the server `server1.example.com` to another `server2.example.com` you have to perform the following steps: + +1. On the source server `server1.example.com` run `repo-backup` command, e.g.: + + ```shell + sudo ahriman repo-backup /tmp/repo.tar.gz + ``` + + This command will pack all configuration files together with database file into the archive specified as command line argument (i.e. `/tmp/repo.tar.gz`). In addition it will also archive `cache` directory (the one which contains local clones used by e.g. local packages) and `.gnupg` of the `ahriman` user. + +2. Copy created archive from source server `server1.example.com` to target `server2.example.com`. + +3. Install ahriman as usual on the target server `server2.example.com` if you didn't yet. + +4. Extract archive e.g. by using subcommand: + + ```shell + sudo ahriman repo-restore /tmp/repo.tar.gz + ``` + + An additional argument `-o`/`--output` can be used to specify extraction root (`/` by default). + +5. Rebuild repository: + + ```shell + sudo -u ahriman ahriman repo-rebuild --from-database + ``` + ## Other topics ### How does it differ from %another-manager%? @@ -558,6 +588,10 @@ Though originally I've created ahriman by trying to improve the project, it stil `repo-scripts` also have bad architecture and bad quality code and uses out-of-dated `yaourt` and `package-query`. +#### [toolbox](https://github.com/chaotic-aur/toolbox) + +It is automation tools for `repoctl` mentioned above. Except for using shell it looks pretty cool and also offers some additional features like patches, remote synchronization (isn't it?) and reporting. + ### I would like to check service logs By default, the service writes logs to `/dev/log` which can be accessed by using `journalctl` command (logs are written to the journal of the user under which command is run). diff --git a/src/ahriman/application/ahriman.py b/src/ahriman/application/ahriman.py index 5532b250..f80ed1bf 100644 --- a/src/ahriman/application/ahriman.py +++ b/src/ahriman/application/ahriman.py @@ -83,12 +83,14 @@ def _parser() -> argparse.ArgumentParser: _set_patch_add_parser(subparsers) _set_patch_list_parser(subparsers) _set_patch_remove_parser(subparsers) + _set_repo_backup_parser(subparsers) _set_repo_check_parser(subparsers) _set_repo_clean_parser(subparsers) _set_repo_config_parser(subparsers) _set_repo_rebuild_parser(subparsers) _set_repo_remove_unknown_parser(subparsers) _set_repo_report_parser(subparsers) + _set_repo_restore_parser(subparsers) _set_repo_setup_parser(subparsers) _set_repo_sign_parser(subparsers) _set_repo_status_update_parser(subparsers) @@ -314,6 +316,19 @@ def _set_patch_remove_parser(root: SubParserAction) -> argparse.ArgumentParser: return parser +def _set_repo_backup_parser(root: SubParserAction) -> argparse.ArgumentParser: + """ + add parser for repository backup subcommand + :param root: subparsers for the commands + :return: created argument parser + """ + parser = root.add_parser("repo-backup", help="backup repository data", + description="backup settings and database", formatter_class=_formatter) + parser.add_argument("path", help="path of the output archive", type=Path) + parser.set_defaults(handler=handlers.Backup, architecture=[""], lock=None, no_report=True, unsafe=True) + return parser + + def _set_repo_check_parser(root: SubParserAction) -> argparse.ArgumentParser: """ add parser for repository check subcommand @@ -415,6 +430,20 @@ def _set_repo_report_parser(root: SubParserAction) -> argparse.ArgumentParser: return parser +def _set_repo_restore_parser(root: SubParserAction) -> argparse.ArgumentParser: + """ + add parser for repository restore subcommand + :param root: subparsers for the commands + :return: created argument parser + """ + parser = root.add_parser("repo-restore", help="restore repository data", + description="restore settings and database", formatter_class=_formatter) + parser.add_argument("path", help="path of the input archive", type=Path) + parser.add_argument("-o", "--output", help="root path of the extracted files", type=Path, default=Path("/")) + parser.set_defaults(handler=handlers.Restore, architecture=[""], lock=None, no_report=True, unsafe=True) + return parser + + def _set_repo_setup_parser(root: SubParserAction) -> argparse.ArgumentParser: """ add parser for setup subcommand diff --git a/src/ahriman/application/handlers/__init__.py b/src/ahriman/application/handlers/__init__.py index 2a478798..5ebd54e4 100644 --- a/src/ahriman/application/handlers/__init__.py +++ b/src/ahriman/application/handlers/__init__.py @@ -20,6 +20,7 @@ from ahriman.application.handlers.handler import Handler from ahriman.application.handlers.add import Add +from ahriman.application.handlers.backup import Backup from ahriman.application.handlers.clean import Clean from ahriman.application.handlers.dump import Dump from ahriman.application.handlers.help import Help @@ -29,6 +30,7 @@ from ahriman.application.handlers.rebuild import Rebuild from ahriman.application.handlers.remove import Remove from ahriman.application.handlers.remove_unknown import RemoveUnknown from ahriman.application.handlers.report import Report +from ahriman.application.handlers.restore import Restore from ahriman.application.handlers.search import Search from ahriman.application.handlers.setup import Setup from ahriman.application.handlers.sign import Sign diff --git a/src/ahriman/application/handlers/backup.py b/src/ahriman/application/handlers/backup.py new file mode 100644 index 00000000..3595f6a4 --- /dev/null +++ b/src/ahriman/application/handlers/backup.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2021-2022 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import argparse +import pwd + +from pathlib import Path +from tarfile import TarFile +from typing import Set, Type + +from ahriman.application.handlers.handler import Handler +from ahriman.core.configuration import Configuration +from ahriman.core.database.sqlite import SQLite + + +class Backup(Handler): + """ + backup packages handler + """ + + ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture" + + @classmethod + def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, + configuration: Configuration, no_report: bool, unsafe: bool) -> None: + """ + callback for command line + :param args: command line args + :param architecture: repository architecture + :param configuration: configuration instance + :param no_report: force disable reporting + :param unsafe: if set no user check will be performed before path creation + """ + backup_paths = Backup.get_paths(configuration) + with TarFile(args.path, mode="w") as archive: # well we don't actually use compression + for backup_path in backup_paths: + archive.add(backup_path) + + @staticmethod + def get_paths(configuration: Configuration) -> Set[Path]: + """ + extract paths to backup + :param configuration: configuration instance + :return: map of the filesystem paths + """ + paths = set(configuration.include.glob("*.ini")) + + root, _ = configuration.check_loaded() + paths.add(root) # the configuration itself + paths.add(SQLite.database_path(configuration)) # database + + # local caches + repository_paths = configuration.repository_paths + if repository_paths.cache.is_dir(): + paths.add(repository_paths.cache) + + # gnupg home with imported keys + uid, _ = repository_paths.root_owner + system_user = pwd.getpwuid(uid) + gnupg_home = Path(system_user.pw_dir) / ".gnupg" + if gnupg_home.is_dir(): + paths.add(gnupg_home) + + return paths diff --git a/src/ahriman/application/handlers/restore.py b/src/ahriman/application/handlers/restore.py new file mode 100644 index 00000000..1dd4da24 --- /dev/null +++ b/src/ahriman/application/handlers/restore.py @@ -0,0 +1,48 @@ +# +# Copyright (c) 2021-2022 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import argparse + +from typing import Type +from tarfile import TarFile + +from ahriman.application.handlers.handler import Handler +from ahriman.core.configuration import Configuration + + +class Restore(Handler): + """ + restore packages handler + """ + + ALLOW_AUTO_ARCHITECTURE_RUN = False # it should be called only as "no-architecture" + + @classmethod + def run(cls: Type[Handler], args: argparse.Namespace, architecture: str, + configuration: Configuration, no_report: bool, unsafe: bool) -> None: + """ + callback for command line + :param args: command line args + :param architecture: repository architecture + :param configuration: configuration instance + :param no_report: force disable reporting + :param unsafe: if set no user check will be performed before path creation + """ + with TarFile(args.path) as archive: + archive.extractall(path=args.output) diff --git a/tests/ahriman/application/handlers/test_handler_backup.py b/tests/ahriman/application/handlers/test_handler_backup.py new file mode 100644 index 00000000..637aa666 --- /dev/null +++ b/tests/ahriman/application/handlers/test_handler_backup.py @@ -0,0 +1,58 @@ +import argparse + +from pathlib import Path +from pytest_mock import MockerFixture +from unittest.mock import MagicMock + +from ahriman.application.handlers import Backup +from ahriman.core.configuration import Configuration +from ahriman.models.repository_paths import RepositoryPaths + + +def _default_args(args: argparse.Namespace) -> argparse.Namespace: + """ + default arguments for these test cases + :param args: command line arguments fixture + :return: generated arguments for these test cases + """ + args.path = Path("result.tar.gz") + return args + + +def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None: + """ + must run command + """ + args = _default_args(args) + mocker.patch("ahriman.application.handlers.Backup.get_paths", return_value=[Path("path")]) + tarfile = MagicMock() + add_mock = tarfile.__enter__.return_value = MagicMock() + mocker.patch("tarfile.TarFile.__new__", return_value=tarfile) + + Backup.run(args, "x86_64", configuration, True, False) + add_mock.add.assert_called_once_with(Path("path")) + + +def test_get_paths(configuration: Configuration, mocker: MockerFixture) -> None: + """ + must get paths to be archived + """ + # gnupg export mock + mocker.patch("pathlib.Path.is_dir", return_value=True) + mocker.patch.object(RepositoryPaths, "root_owner", (42, 42)) + getpwuid_mock = mocker.patch("pwd.getpwuid", return_value=MagicMock()) + # well database does not exist so we override it + database_mock = mocker.patch("ahriman.core.database.sqlite.SQLite.database_path", return_value=configuration.path) + + paths = Backup.get_paths(configuration) + getpwuid_mock.assert_called_once_with(42) + database_mock.assert_called_once_with(configuration) + assert configuration.path in paths + assert all(path.exists() for path in paths if path.name not in (".gnupg", "cache")) + + +def test_disallow_auto_architecture_run() -> None: + """ + must not allow multi architecture run + """ + assert not Backup.ALLOW_AUTO_ARCHITECTURE_RUN diff --git a/tests/ahriman/application/handlers/test_handler_rebuild.py b/tests/ahriman/application/handlers/test_handler_rebuild.py index 99abe028..9c450ce8 100644 --- a/tests/ahriman/application/handlers/test_handler_rebuild.py +++ b/tests/ahriman/application/handlers/test_handler_rebuild.py @@ -50,6 +50,7 @@ def test_run_extract_packages(args: argparse.Namespace, configuration: Configura """ args = _default_args(args) args.from_database = True + args.dry_run = True mocker.patch("ahriman.models.repository_paths.RepositoryPaths.tree_create") mocker.patch("ahriman.application.application.Application.add") extract_mock = mocker.patch("ahriman.application.handlers.Rebuild.extract_packages", return_value=[]) diff --git a/tests/ahriman/application/handlers/test_handler_restore.py b/tests/ahriman/application/handlers/test_handler_restore.py new file mode 100644 index 00000000..2539d888 --- /dev/null +++ b/tests/ahriman/application/handlers/test_handler_restore.py @@ -0,0 +1,39 @@ +import argparse + +from pathlib import Path +from pytest_mock import MockerFixture +from unittest.mock import MagicMock + +from ahriman.application.handlers import Restore +from ahriman.core.configuration import Configuration + + +def _default_args(args: argparse.Namespace) -> argparse.Namespace: + """ + default arguments for these test cases + :param args: command line arguments fixture + :return: generated arguments for these test cases + """ + args.path = Path("result.tar.gz") + args.output = Path.cwd() + return args + + +def test_run(args: argparse.Namespace, configuration: Configuration, mocker: MockerFixture) -> None: + """ + must run command + """ + args = _default_args(args) + tarfile = MagicMock() + extract_mock = tarfile.__enter__.return_value = MagicMock() + mocker.patch("tarfile.TarFile.__new__", return_value=tarfile) + + Restore.run(args, "x86_64", configuration, True, False) + extract_mock.extractall.assert_called_once_with(path=args.output) + + +def test_disallow_auto_architecture_run() -> None: + """ + must not allow multi architecture run + """ + assert not Restore.ALLOW_AUTO_ARCHITECTURE_RUN diff --git a/tests/ahriman/application/test_ahriman.py b/tests/ahriman/application/test_ahriman.py index 7708950f..4d85cedd 100644 --- a/tests/ahriman/application/test_ahriman.py +++ b/tests/ahriman/application/test_ahriman.py @@ -258,6 +258,25 @@ def test_subparsers_patch_remove_architecture(parser: argparse.ArgumentParser) - assert args.architecture == [""] +def test_subparsers_repo_backup(parser: argparse.ArgumentParser) -> None: + """ + repo-backup command must imply architecture list, lock, no-report and unsafe + """ + args = parser.parse_args(["repo-backup", "output.zip"]) + assert args.architecture == [""] + assert args.lock is None + assert args.no_report + assert args.unsafe + + +def test_subparsers_repo_backup_architecture(parser: argparse.ArgumentParser) -> None: + """ + repo-backup command must correctly parse architecture list + """ + args = parser.parse_args(["-a", "x86_64", "repo-backup", "output.zip"]) + assert args.architecture == [""] + + def test_subparsers_repo_check(parser: argparse.ArgumentParser) -> None: """ repo-check command must imply dry-run, no-aur and no-manual @@ -339,6 +358,25 @@ def test_subparsers_repo_report_architecture(parser: argparse.ArgumentParser) -> assert args.architecture == ["x86_64"] +def test_subparsers_repo_restore(parser: argparse.ArgumentParser) -> None: + """ + repo-restore command must imply architecture list, lock, no-report and unsafe + """ + args = parser.parse_args(["repo-restore", "output.zip"]) + assert args.architecture == [""] + assert args.lock is None + assert args.no_report + assert args.unsafe + + +def test_subparsers_repo_restore_architecture(parser: argparse.ArgumentParser) -> None: + """ + repo-restore command must correctly parse architecture list + """ + args = parser.parse_args(["-a", "x86_64", "repo-restore", "output.zip"]) + assert args.architecture == [""] + + def test_subparsers_repo_setup(parser: argparse.ArgumentParser) -> None: """ repo-setup command must imply lock, no-report, quiet and unsafe