Add gitremote triggers (#68)

* add gitremote pull trigger

* add push gitremote trigger

* docs update

src/ahriman/application/application/application.py

@@ -49,15 +49,6 @@ class Application(ApplicationPackages, ApplicationRepository):
         be used instead.
     """
 
-    def _finalize(self, result: Result) -> None:
-        """
-        generate report and sync to remote server
-
-        Args:
-            result(Result): build result
-        """
-        self.repository.process_triggers(result)
-
     def _known_packages(self) -> Set[str]:
         """
         load packages from repository and pacman repositories
@@ -73,3 +64,26 @@ class Application(ApplicationPackages, ApplicationRepository):
                 known_packages.update(properties.provides)
         known_packages.update(self.repository.pacman.all_packages())
         return known_packages
+
+    def on_result(self, result: Result) -> None:
+        """
+        generate report and sync to remote server
+
+        Args:
+            result(Result): build result
+        """
+        packages = self.repository.packages()
+        self.repository.triggers.on_result(result, packages)
+
+    def on_start(self) -> None:
+        """
+        run triggers on start of the application
+        """
+        self.repository.triggers.on_start()
+
+    def on_stop(self) -> None:
+        """
+        run triggers on stop of the application. Note, however, that in most cases this method should not be called
+        directly as it will be called after on_start action
+        """
+        self.repository.triggers.on_stop()

src/ahriman/application/application/application_packages.py

@@ -37,18 +37,6 @@ class ApplicationPackages(ApplicationProperties):
     package control class
     """
 
-    def _finalize(self, result: Result) -> None:
-        """
-        generate report and sync to remote server
-
-        Args:
-            result(Result): build result
-
-        Raises:
-            NotImplementedError: not implemented method
-        """
-        raise NotImplementedError
-
     def _known_packages(self) -> Set[str]:
         """
         load packages from repository and pacman repositories
@@ -61,6 +49,18 @@ class ApplicationPackages(ApplicationProperties):
         """
         raise NotImplementedError
 
+    def on_result(self, result: Result) -> None:
+        """
+        generate report and sync to remote server
+
+        Args:
+            result(Result): build result
+
+        Raises:
+            NotImplementedError: not implemented method
+        """
+        raise NotImplementedError
+
     def _add_archive(self, source: str, *_: Any) -> None:
         """
         add package from archive
@@ -86,8 +86,7 @@ class ApplicationPackages(ApplicationProperties):
         self.database.build_queue_insert(package)
         self.database.remote_update(package)
 
-        with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, \
-                (local_dir := Path(dir_name)):  # pylint: disable=confusing-with-statement
+        with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (local_dir := Path(dir_name)):
             Sources.load(local_dir, package, self.database.patches_get(package.base), self.repository.paths)
             self._process_dependencies(local_dir, known_packages, without_dependencies)
 
@@ -187,4 +186,4 @@ class ApplicationPackages(ApplicationProperties):
             names(Iterable[str]): list of packages (either base or name) to remove
         """
         self.repository.process_remove(names)
-        self._finalize(Result())
+        self.on_result(Result())

src/ahriman/application/application/application_repository.py

@@ -35,7 +35,7 @@ class ApplicationRepository(ApplicationProperties):
     repository control class
     """
 
-    def _finalize(self, result: Result) -> None:
+    def on_result(self, result: Result) -> None:
         """
         generate report and sync to remote server
 
@@ -89,7 +89,7 @@ class ApplicationRepository(ApplicationProperties):
         self.update([])
         # sign repository database if set
         self.repository.sign.process_sign_repository(self.repository.repo.repo_path)
-        self._finalize(Result())
+        self.on_result(Result())
 
     def unknown(self) -> List[str]:
         """
@@ -139,7 +139,7 @@ class ApplicationRepository(ApplicationProperties):
             if not paths:
                 return  # don't need to process if no update supplied
             update_result = self.repository.process_update(paths)
-            self._finalize(result.merge(update_result))
+            self.on_result(result.merge(update_result))
 
         # process built packages
         build_result = Result()

src/ahriman/application/handlers/add.py

@@ -45,6 +45,7 @@ class Add(Handler):
             unsafe(bool): if set no user check will be performed before path creation
         """
         application = Application(architecture, configuration, no_report, unsafe)
+        application.on_start()
         application.add(args.package, args.source, args.without_dependencies)
         if not args.now:
             return

src/ahriman/application/handlers/clean.py

@@ -44,5 +44,6 @@ class Clean(Handler):
             no_report(bool): force disable reporting
             unsafe(bool): if set no user check will be performed before path creation
         """
-        Application(architecture, configuration, no_report, unsafe).clean(
-            args.cache, args.chroot, args.manual, args.packages)
+        application = Application(architecture, configuration, no_report, unsafe)
+        application.on_start()
+        application.clean(args.cache, args.chroot, args.manual, args.packages)

src/ahriman/application/handlers/key_import.py

@@ -46,5 +46,5 @@ class KeyImport(Handler):
             no_report(bool): force disable reporting
             unsafe(bool): if set no user check will be performed before path creation
         """
-        Application(architecture, configuration, no_report, unsafe).repository.sign.key_import(
-            args.key_server, args.key)
+        application = Application(architecture, configuration, no_report, unsafe)
+        application.repository.sign.key_import(args.key_server, args.key)

src/ahriman/application/handlers/patch.py

@@ -50,6 +50,7 @@ class Patch(Handler):
             unsafe(bool): if set no user check will be performed before path creation
         """
         application = Application(architecture, configuration, no_report, unsafe)
+        application.on_start()
 
         if args.action == Action.List:
             Patch.patch_set_list(application, args.package, args.exit_code)

src/ahriman/application/handlers/rebuild.py

@@ -49,6 +49,8 @@ class Rebuild(Handler):
         depends_on = set(args.depends_on) if args.depends_on else None
 
         application = Application(architecture, configuration, no_report, unsafe)
+        application.on_start()
+
         if args.from_database:
             updates = Rebuild.extract_packages(application)
         else:

src/ahriman/application/handlers/remove.py

@@ -44,4 +44,6 @@ class Remove(Handler):
             no_report(bool): force disable reporting
             unsafe(bool): if set no user check will be performed before path creation
         """
-        Application(architecture, configuration, no_report, unsafe).remove(args.package)
+        application = Application(architecture, configuration, no_report, unsafe)
+        application.on_start()
+        application.remove(args.package)

src/ahriman/application/handlers/remove_unknown.py

@@ -46,6 +46,7 @@ class RemoveUnknown(Handler):
             unsafe(bool): if set no user check will be performed before path creation
         """
         application = Application(architecture, configuration, no_report, unsafe)
+        application.on_start()
         unknown_packages = application.unknown()
 
         if args.dry_run:

src/ahriman/application/handlers/triggers.py

@@ -49,4 +49,5 @@ class Triggers(Handler):
         if args.trigger:
             loader = application.repository.triggers
             loader.triggers = [loader.load_trigger(trigger) for trigger in args.trigger]
-        application.repository.process_triggers(Result())
+        application.on_start()
+        application.on_result(Result())

src/ahriman/application/handlers/update.py

@@ -45,6 +45,7 @@ class Update(Handler):
             unsafe(bool): if set no user check will be performed before path creation
         """
         application = Application(architecture, configuration, no_report, unsafe)
+        application.on_start()
         packages = application.updates(args.package, args.no_aur, args.no_local, args.no_manual, args.no_vcs,
                                        Update.log_fn(application, args.dry_run))
         Update.check_if_empty(args.exit_code, not packages)

src/ahriman/core/build_tools/sources.py

@@ -17,6 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
+import datetime
 import shutil
 
 from pathlib import Path
@@ -170,6 +171,22 @@ class Sources(LazyLogging):
         diff = instance.diff(sources_dir)
         return f"{diff}\n"  # otherwise, patch will be broken
 
+    @staticmethod
+    def push(sources_dir: Path, remote: RemoteSource, *pattern: str) -> None:
+        """
+        commit selected changes and push files to the remote repository
+
+        Args:
+            sources_dir(Path): local path to git repository
+            remote(RemoteSource): remote target, branch and url
+            *pattern(str): glob patterns
+        """
+        instance = Sources()
+        instance.add(sources_dir, *pattern)
+        instance.commit(sources_dir)
+        Sources._check_output("git", "push", remote.git_url, remote.branch,
+                              exception=None, cwd=sources_dir, logger=instance.logger)
+
     def add(self, sources_dir: Path, *pattern: str) -> None:
         """
         track found files via git
@@ -190,6 +207,20 @@ class Sources(LazyLogging):
                               *[str(fn.relative_to(sources_dir)) for fn in found_files],
                               exception=None, cwd=sources_dir, logger=self.logger)
 
+    def commit(self, sources_dir: Path, commit_message: Optional[str] = None) -> None:
+        """
+        commit changes
+
+        Args:
+            sources_dir(Path): local path to git repository
+            commit_message(Optional[str]): optional commit message if any. If none set, message will be generated
+                according to the current timestamp
+        """
+        if commit_message is None:
+            commit_message = f"Autogenerated commit at {datetime.datetime.utcnow()}"
+        Sources._check_output("git", "commit", "--all", "--message", commit_message,
+                              exception=None, cwd=sources_dir, logger=self.logger)
+
     def diff(self, sources_dir: Path) -> str:
         """
         generate diff from the current version and write it to the output file

src/ahriman/core/gitremote/__init__.py

@@ -0,0 +1,21 @@
+#
+# copyright (c) 2021-2022 ahriman team.
+#
+# this file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# this program is free software: you can redistribute it and/or modify
+# it under the terms of the gnu general public license as published by
+# the free software foundation, either version 3 of the license, or
+# (at your option) any later version.
+#
+# this program is distributed in the hope that it will be useful,
+# but without any warranty; without even the implied warranty of
+# merchantability or fitness for a particular purpose.  see the
+# gnu general public license for more details.
+#
+# you should have received a copy of the gnu general public license
+# along with this program. if not, see <http://www.gnu.org/licenses/>.
+#
+from ahriman.core.gitremote.remote_pull_trigger import RemotePullTrigger
+from ahriman.core.gitremote.remote_push_trigger import RemotePushTrigger

src/ahriman/core/gitremote/remote_pull_trigger.py

@@ -0,0 +1,86 @@
+#
+# copyright (c) 2021-2022 ahriman team.
+#
+# this file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# this program is free software: you can redistribute it and/or modify
+# it under the terms of the gnu general public license as published by
+# the free software foundation, either version 3 of the license, or
+# (at your option) any later version.
+#
+# this program is distributed in the hope that it will be useful,
+# but without any warranty; without even the implied warranty of
+# merchantability or fitness for a particular purpose.  see the
+# gnu general public license for more details.
+#
+# you should have received a copy of the gnu general public license
+# along with this program. if not, see <http://www.gnu.org/licenses/>.
+#
+import shutil
+
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from ahriman.core.build_tools.sources import Sources
+from ahriman.core.configuration import Configuration
+from ahriman.core.triggers import Trigger
+from ahriman.core.util import walk
+from ahriman.models.package_source import PackageSource
+from ahriman.models.remote_source import RemoteSource
+
+
+class RemotePullTrigger(Trigger):
+    """
+    trigger for fetching PKGBUILDs from remote repository
+
+    Attributes:
+        remote_source(RemoteSource): repository remote source (remote pull url and branch)
+        repository_paths(RepositoryPaths): repository paths instance
+    """
+
+    def __init__(self, architecture: str, configuration: Configuration) -> None:
+        """
+        default constructor
+
+        Args:
+            architecture(str): repository architecture
+            configuration(Configuration): configuration instance
+        """
+        Trigger.__init__(self, architecture, configuration)
+        self.remote_source = RemoteSource(
+            git_url=configuration.get("gitremote", "pull_url"),
+            web_url="",
+            path=".",
+            branch=configuration.get("gitremote", "pull_branch", fallback="master"),
+            source=PackageSource.Local,
+        )
+        self.repository_paths = configuration.repository_paths
+
+    def on_start(self) -> None:
+        """
+        trigger action which will be called at the start of the application
+        """
+        self.repo_clone()
+
+    def repo_clone(self) -> None:
+        """
+        clone repository from remote source
+        """
+        with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (clone_dir := Path(dir_name)):
+            Sources.fetch(clone_dir, self.remote_source)
+            self.repo_copy(clone_dir)
+
+    def repo_copy(self, clone_dir: Path) -> None:
+        """
+        copy directories from cloned remote source to local cache
+
+        Args:
+            clone_dir(Path): path to temporary cloned directory
+        """
+        for pkgbuild_path in filter(lambda path: path.name == "PKGBUILD", walk(clone_dir)):
+            cloned_pkgbuild_dir = pkgbuild_path.parent
+            package_base = cloned_pkgbuild_dir.name
+            local_pkgbuild_dir = self.repository_paths.cache_for(package_base)
+            shutil.copytree(cloned_pkgbuild_dir, local_pkgbuild_dir, dirs_exist_ok=True)
+            Sources.init(local_pkgbuild_dir)  # initialized git repository is required for local sources

src/ahriman/core/gitremote/remote_push_trigger.py

@@ -0,0 +1,110 @@
+#
+# copyright (c) 2021-2022 ahriman team.
+#
+# this file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# this program is free software: you can redistribute it and/or modify
+# it under the terms of the gnu general public license as published by
+# the free software foundation, either version 3 of the license, or
+# (at your option) any later version.
+#
+# this program is distributed in the hope that it will be useful,
+# but without any warranty; without even the implied warranty of
+# merchantability or fitness for a particular purpose.  see the
+# gnu general public license for more details.
+#
+# you should have received a copy of the gnu general public license
+# along with this program. if not, see <http://www.gnu.org/licenses/>.
+#
+import shutil
+
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Generator, Iterable
+
+from ahriman.core.build_tools.sources import Sources
+from ahriman.core.configuration import Configuration
+from ahriman.core.triggers import Trigger
+from ahriman.models.package import Package
+from ahriman.models.package_source import PackageSource
+from ahriman.models.remote_source import RemoteSource
+from ahriman.models.result import Result
+
+
+class RemotePushTrigger(Trigger):
+    """
+    trigger for syncing PKGBUILDs to remote repository
+
+    Attributes:
+        remote_source(RemoteSource): repository remote source (remote pull url and branch)
+    """
+
+    def __init__(self, architecture: str, configuration: Configuration) -> None:
+        """
+        default constructor
+
+        Args:
+            architecture(str): repository architecture
+            configuration(Configuration): configuration instance
+        """
+        Trigger.__init__(self, architecture, configuration)
+        self.remote_source = RemoteSource(
+            git_url=configuration.get("gitremote", "push_url"),
+            web_url="",
+            path=".",
+            branch=configuration.get("gitremote", "push_branch", fallback="master"),
+            source=PackageSource.Local,
+        )
+
+    @staticmethod
+    def package_update(package: Package, target_dir: Path) -> str:
+        """
+        clone specified package and update its content in cloned PKGBUILD repository
+
+        Args:
+            package(Package): built package to update pkgbuild repository
+            target_dir(Path): path to the cloned PKGBUILD repository
+
+        Returns:
+            str: relative path to be added as new file
+        """
+        # instead of iterating by directory we can simplify the process
+        # firstly, we need to remove old data to make sure that removed files are not tracked anymore...
+        package_target_dir = target_dir / package.base
+        shutil.rmtree(package_target_dir, ignore_errors=True)
+        # ...secondly, we copy whole tree...
+        with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (clone_dir := Path(dir_name)):
+            Sources.fetch(clone_dir, package.remote)
+            shutil.copytree(clone_dir, package_target_dir)
+        # ...and last, but not least, we remove the dot-git directory...
+        shutil.rmtree(package_target_dir / ".git", ignore_errors=True)
+        # ...and finally return path to the copied directory
+        return package.base
+
+    @staticmethod
+    def packages_update(result: Result, target_dir: Path) -> Generator[str, None, None]:
+        """
+        update all packages from the build result
+
+        Args:
+            result(Result): build result
+            target_dir(Path): path to the cloned PKGBUILD repository
+
+        Yields:
+            str: path to updated files
+        """
+        for package in result.success:
+            yield RemotePushTrigger.package_update(package, target_dir)
+
+    def on_result(self, result: Result, packages: Iterable[Package]) -> None:
+        """
+        trigger action which will be called after build process with process result
+
+        Args:
+            result(Result): build result
+            packages(Iterable[Package]): list of all available packages
+        """
+        with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (clone_dir := Path(dir_name)):
+            Sources.fetch(clone_dir, self.remote_source)
+            Sources.push(clone_dir, self.remote_source, *RemotePushTrigger.packages_update(result, clone_dir))

src/ahriman/core/report/__init__.py

@@ -17,5 +17,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from ahriman.core.report.report import Report
 from ahriman.core.report.report_trigger import ReportTrigger

src/ahriman/core/report/console.py

@@ -21,7 +21,7 @@ from typing import Iterable
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.formatters import BuildPrinter
-from ahriman.core.report import Report
+from ahriman.core.report.report import Report
 from ahriman.models.package import Package
 from ahriman.models.result import Result
 

src/ahriman/core/report/email.py

@@ -25,8 +25,8 @@ from email.mime.text import MIMEText
 from typing import Dict, Iterable
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.report import Report
 from ahriman.core.report.jinja_template import JinjaTemplate
+from ahriman.core.report.report import Report
 from ahriman.core.util import pretty_datetime
 from ahriman.models.package import Package
 from ahriman.models.result import Result

src/ahriman/core/report/html.py

@@ -20,8 +20,8 @@
 from typing import Iterable
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.report import Report
 from ahriman.core.report.jinja_template import JinjaTemplate
+from ahriman.core.report.report import Report
 from ahriman.models.package import Package
 from ahriman.models.result import Result
 

src/ahriman/core/report/report_trigger.py

@@ -21,7 +21,7 @@ from typing import Iterable
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.triggers import Trigger
-from ahriman.core.report import Report
+from ahriman.core.report.report import Report
 from ahriman.models.package import Package
 from ahriman.models.result import Result
 

src/ahriman/core/report/telegram.py

@@ -23,8 +23,8 @@ import requests
 from typing import Iterable
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.report import Report
 from ahriman.core.report.jinja_template import JinjaTemplate
+from ahriman.core.report.report import Report
 from ahriman.core.util import exception_response_text
 from ahriman.models.package import Package
 from ahriman.models.result import Result

src/ahriman/core/repository/executor.py

@@ -84,8 +84,7 @@ class Executor(Cleaner):
 
         result = Result()
         for single in updates:
-            with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, \
-                    (build_dir := Path(dir_name)):  # pylint: disable=confusing-with-statement
+            with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (build_dir := Path(dir_name)):
                 try:
                     build_single(single, build_dir)
                     result.add_success(single)
@@ -144,15 +143,6 @@ class Executor(Cleaner):
 
         return self.repo.repo_path
 
-    def process_triggers(self, result: Result) -> None:
-        """
-        process triggers setup by settings
-
-        Args:
-            result(Result): build result
-        """
-        self.triggers.on_result(result, self.packages())
-
     def process_update(self, packages: Iterable[Path]) -> Result:
         """
         sign packages, add them to repository and update repository database

src/ahriman/core/tree.py

@@ -72,8 +72,7 @@ class Leaf:
         Returns:
             Leaf: loaded class
         """
-        with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, \
-                (clone_dir := Path(dir_name)):  # pylint: disable=confusing-with-statement
+        with TemporaryDirectory(ignore_cleanup_errors=True) as dir_name, (clone_dir := Path(dir_name)):
             Sources.load(clone_dir, package, database.patches_get(package.base), paths)
             dependencies = Package.dependencies(clone_dir)
         return cls(package, dependencies)

src/ahriman/core/triggers/trigger_loader.py

@@ -20,7 +20,6 @@
 import contextlib
 import importlib
 import os
-import weakref
 
 from pathlib import Path
 from types import ModuleType
@@ -74,9 +73,15 @@ class TriggerLoader(LazyLogging):
             self.load_trigger(trigger)
             for trigger in configuration.getlist("build", "triggers")
         ]
+        self._on_stop_requested = False
 
-        self.on_start()
-        self._finalizer = weakref.finalize(self, self.on_stop)
+    def __del__(self) -> None:
+        """
+        custom destructor object which calls on_stop in case if it was requested
+        """
+        if not self._on_stop_requested:
+            return
+        self.on_stop()
 
     @contextlib.contextmanager
     def __execute_trigger(self, trigger: Trigger) -> Generator[None, None, None]:
@@ -178,6 +183,7 @@ class TriggerLoader(LazyLogging):
             result(Result): build result
             packages(Iterable[Package]): list of all available packages
         """
+        self.logger.debug("executing triggers on result")
         for trigger in self.triggers:
             with self.__execute_trigger(trigger):
                 trigger.on_result(result, packages)
@@ -186,6 +192,8 @@ class TriggerLoader(LazyLogging):
         """
         run triggers on load
         """
+        self.logger.debug("executing triggers on start")
+        self._on_stop_requested = True
         for trigger in self.triggers:
             with self.__execute_trigger(trigger):
                 trigger.on_start()
@@ -194,6 +202,7 @@ class TriggerLoader(LazyLogging):
         """
         run triggers before the application exit
         """
+        self.logger.debug("executing triggers on stop")
         for trigger in self.triggers:
             with self.__execute_trigger(trigger):
                 trigger.on_stop()

src/ahriman/core/upload/__init__.py

@@ -17,5 +17,4 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from ahriman.core.upload.upload import Upload
 from ahriman.core.upload.upload_trigger import UploadTrigger

src/ahriman/core/upload/http_upload.py

@@ -24,7 +24,7 @@ from pathlib import Path
 from typing import Any, Dict
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.upload import Upload
+from ahriman.core.upload.upload import Upload
 from ahriman.core.util import exception_response_text
 
 

src/ahriman/core/upload/rsync.py

@@ -21,7 +21,7 @@ from pathlib import Path
 from typing import Iterable
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.upload import Upload
+from ahriman.core.upload.upload import Upload
 from ahriman.core.util import check_output
 from ahriman.models.package import Package
 

src/ahriman/core/upload/s3.py

@@ -25,7 +25,7 @@ from pathlib import Path
 from typing import Any, Dict, Iterable
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.upload import Upload
+from ahriman.core.upload.upload import Upload
 from ahriman.core.util import walk
 from ahriman.models.package import Package
 

src/ahriman/core/upload/upload_trigger.py

@@ -21,7 +21,7 @@ from typing import Iterable
 
 from ahriman.core.configuration import Configuration
 from ahriman.core.triggers import Trigger
-from ahriman.core.upload import Upload
+from ahriman.core.upload.upload import Upload
 from ahriman.models.package import Package
 from ahriman.models.result import Result