add ability to download package from external links (e.g. HTTP)

src/ahriman/application/ahriman.py

@@ -144,11 +144,12 @@ def _set_package_add_parser(root: SubParserAction) -> argparse.ArgumentParser:
                                     "from another repository source); "
                                     "3) it is also possible to add package from local PKGBUILD, but in this case it "
                                     "will be ignored during the next automatic updates; "
-                                    "4) and finally you can add package from AUR.",
+                                    "4) ahriman supports downloading archives from remote (e.g. HTTP) sources; "
+                                    "5) and finally you can add package from AUR.",
                              formatter_class=_formatter)
-    parser.add_argument("package", help="package base/name or path to local files", nargs="+")
+    parser.add_argument("package", help="package source (base name, path to local files, remote URL)", nargs="+")
     parser.add_argument("-n", "--now", help="run update function after", action="store_true")
-    parser.add_argument("-s", "--source", help="package source",
+    parser.add_argument("-s", "--source", help="explicitly specify the package source for this command",
                         type=PackageSource, choices=PackageSource, default=PackageSource.Auto)
     parser.add_argument("--without-dependencies", help="do not add dependencies", action="store_true")
     parser.set_defaults(handler=handlers.Add)

src/ahriman/application/application.py

@@ -18,6 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import logging
+import requests
 import shutil
 
 from pathlib import Path
@@ -111,6 +112,12 @@ class Application:
             dst = self.repository.paths.packages / src.name
             shutil.copy(src, dst)
 
+        def add_aur(src: str) -> Path:
+            package = Package.load(src, self.repository.pacman, aur_url)
+            Sources.load(self.repository.paths.manual_for(package.base), package.git_url,
+                         self.repository.paths.patches_for(package.base))
+            return self.repository.paths.manual_for(package.base)
+
         def add_directory(path: Path) -> None:
             for full_path in filter(package_like, path.iterdir()):
                 add_archive(full_path)
@@ -123,11 +130,13 @@ class Application:
             shutil.copytree(cache_dir, self.repository.paths.manual_for(package.base))  # copy package for the build
             return self.repository.paths.manual_for(package.base)
 
-        def add_remote(src: str) -> Path:
+        def add_remote(src: str) -> None:
-            package = Package.load(src, self.repository.pacman, aur_url)
+            dst = self.repository.paths.packages / Path(src).name  # URL is path, is not it?
-            Sources.load(self.repository.paths.manual_for(package.base), package.git_url,
+            response = requests.get(src, stream=True)
-                         self.repository.paths.patches_for(package.base))
+            response.raise_for_status()
-            return self.repository.paths.manual_for(package.base)
+            with dst.open("wb") as local_file:
+                for chunk in response.iter_content(chunk_size=1024):
+                    local_file.write(chunk)
 
         def process_dependencies(path: Path) -> None:
             if without_dependencies:
@@ -140,13 +149,15 @@ class Application:
             if resolved_source == PackageSource.Archive:
                 add_archive(Path(src))
             elif resolved_source == PackageSource.AUR:
-                path = add_remote(src)
+                path = add_aur(src)
                 process_dependencies(path)
             elif resolved_source == PackageSource.Directory:
                 add_directory(Path(src))
             elif resolved_source == PackageSource.Local:
                 path = add_local(Path(src))
                 process_dependencies(path)
+            elif resolved_source == PackageSource.Remote:
+                add_remote(src)
 
         for name in names:
             process_single(name)

src/ahriman/models/package_source.py

@@ -21,6 +21,7 @@ from __future__ import annotations
 
 from enum import Enum
 from pathlib import Path
+from urllib.parse import urlparse
 
 from ahriman.core.util import package_like
 
@@ -33,6 +34,7 @@ class PackageSource(Enum):
     :cvar AUR: source is an AUR package for which it should search
     :cvar Directory: source is a directory which contains packages
     :cvar Local: source is locally stored PKGBUILD
+    :cvar Remote: source is remote (http, ftp etc) link
     """
 
     Auto = "auto"
@@ -40,6 +42,7 @@ class PackageSource(Enum):
     AUR = "aur"
     Directory = "directory"
     Local = "local"
+    Remote = "remote"
 
     def resolve(self, source: str) -> PackageSource:
         """
@@ -50,11 +53,16 @@ class PackageSource(Enum):
         if self != PackageSource.Auto:
             return self
 
-        maybe_path = Path(source)
+        maybe_url = urlparse(source)  # handle file:// like paths
+        maybe_path = Path(maybe_url.path)
+
+        if maybe_url.scheme and maybe_url.scheme not in ("data", "file") and package_like(maybe_path):
+            return PackageSource.Remote
         if (maybe_path / "PKGBUILD").is_file():
             return PackageSource.Local
         if maybe_path.is_dir():
             return PackageSource.Directory
         if maybe_path.is_file() and package_like(maybe_path):
             return PackageSource.Archive
+
         return PackageSource.AUR

tests/ahriman/application/test_application.py

@@ -3,10 +3,12 @@ import pytest
 from pathlib import Path
 from pytest_mock import MockerFixture
 from unittest import mock
+from unittest.mock import MagicMock
 
 from ahriman.application.application import Application
 from ahriman.core.tree import Leaf, Tree
 from ahriman.models.package import Package
+from ahriman.models.package_description import PackageDescription
 from ahriman.models.package_source import PackageSource
 
 
@@ -116,7 +118,7 @@ def test_add_archive(application: Application, package_ahriman: Package, mocker:
     copy_mock.assert_called_once()
 
 
-def test_add_remote(application: Application, package_ahriman: Package, mocker: MockerFixture) -> None:
+def test_add_aur(application: Application, package_ahriman: Package, mocker: MockerFixture) -> None:
     """
     must add package from AUR
     """
@@ -128,8 +130,7 @@ def test_add_remote(application: Application, package_ahriman: Package, mocker:
     load_mock.assert_called_once()
 
 
-def test_add_remote_with_dependencies(application: Application, package_ahriman: Package,
+def test_add_aur_with_dependencies(application: Application, package_ahriman: Package, mocker: MockerFixture) -> None:
-                                      mocker: MockerFixture) -> None:
     """
     must add package from AUR with dependencies
     """
@@ -188,6 +189,24 @@ def test_add_local_with_dependencies(application: Application, package_ahriman:
     dependencies_mock.assert_called_once()
 
 
+def test_add_remote(application: Application, package_description_ahriman: PackageDescription,
+                    mocker: MockerFixture) -> None:
+    """
+    must add package from remote source
+    """
+    mocker.patch("ahriman.application.application.Application._known_packages", return_value=set())
+    response_mock = MagicMock()
+    response_mock.iter_content.return_value = ["chunk"]
+    open_mock = mocker.patch("pathlib.Path.open")
+    request_mock = mocker.patch("requests.get", return_value=response_mock)
+    url = f"https://host/{package_description_ahriman.filename}"
+
+    application.add([url], PackageSource.Remote, False)
+    open_mock.assert_called_once_with("wb")
+    request_mock.assert_called_once_with(url, stream=True)
+    response_mock.raise_for_status.assert_called_once()
+
+
 def test_clean_build(application: Application, mocker: MockerFixture) -> None:
     """
     must clean build directory

tests/ahriman/models/test_package_source.py

@@ -2,6 +2,7 @@ from pytest_mock import MockerFixture
 from pathlib import Path
 from typing import Callable
 
+from ahriman.models.package_description import PackageDescription
 from ahriman.models.package_source import PackageSource
 
 
@@ -24,13 +25,13 @@ def test_resolve_non_auto() -> None:
         assert source.resolve("") == source
 
 
-def test_resolve_archive(mocker: MockerFixture) -> None:
+def test_resolve_archive(package_description_ahriman: PackageDescription, mocker: MockerFixture) -> None:
     """
     must resolve auto type into the archive
     """
     mocker.patch("pathlib.Path.is_dir", return_value=False)
     mocker.patch("pathlib.Path.is_file", autospec=True, side_effect=_is_file_mock(True, False))
-    assert PackageSource.Auto.resolve("linux-5.14.2.arch1-2-x86_64.pkg.tar.zst") == PackageSource.Archive
+    assert PackageSource.Auto.resolve(package_description_ahriman.filename) == PackageSource.Archive
 
 
 def test_resolve_aur(mocker: MockerFixture) -> None:
@@ -56,14 +57,23 @@ def test_resolve_directory(mocker: MockerFixture) -> None:
     must resolve auto type into the directory
     """
     mocker.patch("pathlib.Path.is_dir", return_value=True)
-    mocker.patch("pathlib.Path.is_file", return_value=False)
+    mocker.patch("pathlib.Path.is_file", autospec=True, side_effect=_is_file_mock(False, False))
     assert PackageSource.Auto.resolve("path") == PackageSource.Directory
 
 
 def test_resolve_local(mocker: MockerFixture) -> None:
     """
-    must resolve auto type into the directory
+    must resolve auto type into the local sources
     """
     mocker.patch("pathlib.Path.is_dir", return_value=False)
     mocker.patch("pathlib.Path.is_file", autospec=True, side_effect=_is_file_mock(True, True))
     assert PackageSource.Auto.resolve("path") == PackageSource.Local
+
+
+def test_resolve_remote(package_description_ahriman: PackageDescription, mocker: MockerFixture) -> None:
+    """
+    must resolve auto type into the remote sources
+    """
+    mocker.patch("pathlib.Path.is_dir", return_value=False)
+    mocker.patch("pathlib.Path.is_file", autospec=True, side_effect=_is_file_mock(False, False))
+    assert PackageSource.Auto.resolve(f"https://host/{package_description_ahriman.filename}") == PackageSource.Remote