update dependencies before build (#91)

Old implementation has used add step in order to fetch dependencies,
which could lead to build errors in case if dependency list was updated.

New solution uses dependencies which are declared at current version and
fetch them (if required and if enabled) before update process.

Closes #90

src/ahriman/application/ahriman.py

@@ -245,6 +245,8 @@ def _set_package_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
                                     "5) and finally you can add package from AUR.",
                              formatter_class=_formatter)
     parser.add_argument("package", help="package source (base name, path to local files, remote URL)", nargs="+")
+    parser.add_argument("--dependencies", help="process missing package dependencies",
+                        action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
     parser.add_argument("-n", "--now", help="run update function after", action="store_true")
     parser.add_argument("-y", "--refresh", help="download fresh package databases from the mirror before actions, "
@@ -252,7 +254,6 @@ def _set_package_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
                         action="count", default=False)
     parser.add_argument("-s", "--source", help="explicitly specify the package source for this command",
                         type=PackageSource, choices=enum_values(PackageSource), default=PackageSource.Auto)
-    parser.add_argument("--without-dependencies", help="do not add dependencies", action="store_true")
     parser.set_defaults(handler=handlers.Add)
     return parser
 
@@ -472,7 +473,7 @@ def _set_repo_check_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser.add_argument("-y", "--refresh", help="download fresh package databases from the mirror before actions, "
                                                 "-yy to force refresh even if up to date",
                         action="count", default=False)
-    parser.set_defaults(handler=handlers.Update, dry_run=True, aur=True, local=True, manual=False)
+    parser.set_defaults(handler=handlers.Update, dependencies=False, dry_run=True, aur=True, local=True, manual=False)
     return parser
 
 
@@ -492,6 +493,8 @@ def _set_repo_daemon_parser(root: SubParserAction) -> argparse.ArgumentParser:
     parser.add_argument("-i", "--interval", help="interval between runs in seconds", type=int, default=60 * 60 * 12)
     parser.add_argument("--aur", help="enable or disable checking for AUR updates",
                         action=argparse.BooleanOptionalAction, default=True)
+    parser.add_argument("--dependencies", help="process missing package dependencies",
+                        action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("--local", help="enable or disable checking of local packages for updates",
                         action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("--manual", help="include or exclude manual updates",
@@ -691,10 +694,12 @@ def _set_repo_update_parser(root: SubParserAction) -> argparse.ArgumentParser:
                              description="check for packages updates and run build process if requested",
                              formatter_class=_formatter)
     parser.add_argument("package", help="filter check by package base", nargs="*")
-    parser.add_argument("--dry-run", help="just perform check for updates, same as check command", action="store_true")
-    parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
     parser.add_argument("--aur", help="enable or disable checking for AUR updates",
                         action=argparse.BooleanOptionalAction, default=True)
+    parser.add_argument("--dependencies", help="process missing package dependencies",
+                        action=argparse.BooleanOptionalAction, default=True)
+    parser.add_argument("--dry-run", help="just perform check for updates, same as check command", action="store_true")
+    parser.add_argument("-e", "--exit-code", help="return non-zero exit status if result is empty", action="store_true")
     parser.add_argument("--local", help="enable or disable checking of local packages for updates",
                         action=argparse.BooleanOptionalAction, default=True)
     parser.add_argument("--manual", help="include or exclude manual updates",

src/ahriman/application/application/application.py

@@ -17,10 +17,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Set
+from typing import Iterable, List, Set
 
 from ahriman.application.application.application_packages import ApplicationPackages
 from ahriman.application.application.application_repository import ApplicationRepository
+from ahriman.models.package import Package
 from ahriman.models.result import Result
 
 
@@ -87,3 +88,39 @@ class Application(ApplicationPackages, ApplicationRepository):
         directly as it will be called after on_start action
         """
         self.repository.triggers.on_stop()
+
+    def with_dependencies(self, packages: List[Package], *, process_dependencies: bool) -> List[Package]:
+        """
+        add missing dependencies to list of packages
+
+        Args:
+            packages(List[Package]): list of source packages of which dependencies have to be processed
+            process_dependencies(bool): if no set, dependencies will not be processed
+        """
+        def missing_dependencies(source: Iterable[Package]) -> Set[str]:
+            # build initial list of dependencies
+            result = set()
+            for package in source:
+                result.update(package.depends_build)
+
+            # remove ones which are already well-known
+            result = result.difference(known_packages)
+
+            # remove ones which are in this list already
+            for package in source:
+                result = result.difference(package.packages_full)
+
+            return result
+
+        if not process_dependencies or not packages:
+            return packages
+
+        known_packages = self._known_packages()
+        with_dependencies = {package.base: package for package in packages}
+
+        while missing := missing_dependencies(with_dependencies.values()):
+            for package_name in missing:
+                package = Package.from_aur(package_name, self.repository.pacman)
+                with_dependencies[package.base] = package
+
+        return list(with_dependencies.values())

src/ahriman/application/application/application_packages.py

@@ -21,7 +21,7 @@ import requests
 import shutil
 
 from pathlib import Path
-from typing import Any, Iterable, Set
+from typing import Any, Iterable
 
 from ahriman.application.application.application_properties import ApplicationProperties
 from ahriman.core.build_tools.sources import Sources
@@ -47,22 +47,18 @@ class ApplicationPackages(ApplicationProperties):
         dst = self.repository.paths.packages / local_path.name
         shutil.copy(local_path, dst)
 
-    def _add_aur(self, source: str, known_packages: Set[str], without_dependencies: bool) -> None:
+    def _add_aur(self, source: str) -> None:
         """
         add package from AUR
 
         Args:
             source(str): package base name
-            known_packages(Set[str]): list of packages which are known by the service
-            without_dependencies(bool): if set, dependency check will be disabled
         """
         package = Package.from_aur(source, self.repository.pacman)
 
         self.database.build_queue_insert(package)
         self.database.remote_update(package)
 
-        self._process_dependencies(package, known_packages, without_dependencies)
-
     def _add_directory(self, source: str, *_: Any) -> None:
         """
         add packages from directory
@@ -74,14 +70,12 @@ class ApplicationPackages(ApplicationProperties):
         for full_path in filter(package_like, local_dir.iterdir()):
             self._add_archive(str(full_path))
 
-    def _add_local(self, source: str, known_packages: Set[str], without_dependencies: bool) -> None:
+    def _add_local(self, source: str) -> None:
         """
         add package from local PKGBUILDs
 
         Args:
             source(str): path to directory with local source files
-            known_packages(Set[str]): list of packages which are known by the service
-            without_dependencies(bool): if set, dependency check will be disabled
         """
         source_dir = Path(source)
         package = Package.from_build(source_dir, self.architecture)
@@ -91,8 +85,6 @@ class ApplicationPackages(ApplicationProperties):
 
         self.database.build_queue_insert(package)
 
-        self._process_dependencies(package, known_packages, without_dependencies)
-
     def _add_remote(self, source: str, *_: Any) -> None:
         """
         add package from remote sources (e.g. HTTP)
@@ -118,50 +110,19 @@ class ApplicationPackages(ApplicationProperties):
         package = Package.from_official(source, self.repository.pacman)
         self.database.build_queue_insert(package)
         self.database.remote_update(package)
-        # repository packages must not depend on unknown packages, thus we are not going to process dependencies
 
-    def _known_packages(self) -> Set[str]:
-        """
-        load packages from repository and pacman repositories
-
-        Returns:
-            Set[str]: list of known packages
-
-        Raises:
-            NotImplementedError: not implemented method
-        """
-        raise NotImplementedError
-
-    def _process_dependencies(self, package: Package, known_packages: Set[str], without_dependencies: bool) -> None:
-        """
-        process package dependencies
-
-        Args:
-            package(Package): source package of which dependencies have to be processed
-            known_packages(Set[str]): list of packages which are known by the service
-            without_dependencies(bool): if set, dependency check will be disabled
-        """
-        if without_dependencies:
-            return
-
-        dependencies = package.depends_build
-        self.add(dependencies.difference(known_packages), PackageSource.AUR, without_dependencies)
-
-    def add(self, names: Iterable[str], source: PackageSource, without_dependencies: bool) -> None:
+    def add(self, names: Iterable[str], source: PackageSource) -> None:
         """
         add packages for the next build
 
         Args:
             names(Iterable[str]): list of package bases to add
             source(PackageSource): package source to add
-            without_dependencies(bool): if set, dependency check will be disabled
         """
-        known_packages = self._known_packages()  # speedup dependencies processing
-
         for name in names:
             resolved_source = source.resolve(name)
             fn = getattr(self, f"_add_{resolved_source.value}")
-            fn(name, known_packages, without_dependencies)
+            fn(name)
 
     def on_result(self, result: Result) -> None:
         """

src/ahriman/application/handlers/add.py

@@ -47,11 +47,12 @@ class Add(Handler):
         application = Application(architecture, configuration,
                                   report=report, unsafe=unsafe, refresh_pacman_database=args.refresh)
         application.on_start()
-        application.add(args.package, args.source, args.without_dependencies)
+        application.add(args.package, args.source)
         if not args.now:
             return
 
         packages = application.updates(args.package, aur=False, local=False, manual=True, vcs=False,
                                        log_fn=application.logger.info)
+        packages = application.with_dependencies(packages, process_dependencies=args.dependencies)
         result = application.update(packages)
         Add.check_if_empty(args.exit_code, result.is_empty)

src/ahriman/application/handlers/update.py

@@ -53,6 +53,7 @@ class Update(Handler):
         if args.dry_run:
             return
 
+        packages = application.with_dependencies(packages, process_dependencies=args.dependencies)
         result = application.update(packages)
         Update.check_if_empty(args.exit_code, result.is_empty)
 

src/ahriman/models/package.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-# pylint: disable=too-many-lines
+# pylint: disable=too-many-lines,too-many-public-methods
 from __future__ import annotations
 
 import copy
@@ -96,7 +96,7 @@ class Package(LazyLogging):
         Returns:
             Set[str]: full dependencies list used by devtools
         """
-        return (set(self.depends) | set(self.depends_make)) - self.packages.keys()
+        return (set(self.depends) | set(self.depends_make)).difference(self.packages_full)
 
     @property
     def depends_make(self) -> List[str]:
@@ -163,6 +163,20 @@ class Package(LazyLogging):
         """
         return sorted(set(sum((package.licenses for package in self.packages.values()), start=[])))
 
+    @property
+    def packages_full(self) -> List[str]:
+        """
+        get full packages list including provides
+
+        Returns:
+            List[str]: full list of packages which this base contains
+        """
+        packages = set()
+        for package, properties in self.packages.items():
+            packages.add(package)
+            packages.update(properties.provides)
+        return sorted(packages)
+
     @classmethod
     def from_archive(cls: Type[Package], path: Path, pacman: Pacman, remote: Optional[RemoteSource]) -> Package:
         """