github upload support (#41)

tests/ahriman/application/test_application.py

@@ -110,10 +110,10 @@ def test_add_archive(application: Application, package_ahriman: Package, mocker:
     must add package from archive
     """
     mocker.patch("ahriman.application.application.Application._known_packages", return_value=set())
-    move_mock = mocker.patch("shutil.move")
+    copy_mock = mocker.patch("shutil.copy")
 
     application.add([package_ahriman.base], PackageSource.Archive, False)
-    move_mock.assert_called_once()
+    copy_mock.assert_called_once()
 
 
 def test_add_remote(application: Application, package_ahriman: Package, mocker: MockerFixture) -> None:
@@ -149,11 +149,11 @@ def test_add_directory(application: Application, package_ahriman: Package, mocke
     mocker.patch("ahriman.application.application.Application._known_packages", return_value=set())
     iterdir_mock = mocker.patch("pathlib.Path.iterdir",
                                 return_value=[package.filepath for package in package_ahriman.packages.values()])
-    move_mock = mocker.patch("shutil.move")
+    copy_mock = mocker.patch("shutil.copy")
 
     application.add([package_ahriman.base], PackageSource.Directory, False)
     iterdir_mock.assert_called_once()
-    move_mock.assert_called_once()
+    copy_mock.assert_called_once()
 
 
 def test_add_local(application: Application, package_ahriman: Package, mocker: MockerFixture) -> None:

tests/ahriman/core/test_util.py

@@ -2,10 +2,11 @@ import logging
 import pytest
 import subprocess
 
+from pathlib import Path
 from pytest_mock import MockerFixture
 
 from ahriman.core.exceptions import InvalidOption
-from ahriman.core.util import check_output, package_like, pretty_datetime, pretty_size
+from ahriman.core.util import check_output, package_like, pretty_datetime, pretty_size, walk
 from ahriman.models.package import Package
 
 
@@ -138,3 +139,29 @@ def test_pretty_size_empty() -> None:
     must generate empty string for None value
     """
     assert pretty_size(None) == ""
+
+
+def test_walk(resource_path_root: Path) -> None:
+    """
+    must traverse directory recursively
+    """
+    expected = sorted([
+        resource_path_root / "core/ahriman.ini",
+        resource_path_root / "core/logging.ini",
+        resource_path_root / "models/big_file_checksum",
+        resource_path_root / "models/empty_file_checksum",
+        resource_path_root / "models/package_ahriman_srcinfo",
+        resource_path_root / "models/package_tpacpi-bat-git_srcinfo",
+        resource_path_root / "models/package_yay_srcinfo",
+        resource_path_root / "web/templates/build-status/login-modal.jinja2",
+        resource_path_root / "web/templates/build-status/package-actions-modals.jinja2",
+        resource_path_root / "web/templates/build-status/package-actions-script.jinja2",
+        resource_path_root / "web/templates/static/favicon.ico",
+        resource_path_root / "web/templates/utils/bootstrap-scripts.jinja2",
+        resource_path_root / "web/templates/utils/style.jinja2",
+        resource_path_root / "web/templates/build-status.jinja2",
+        resource_path_root / "web/templates/email-index.jinja2",
+        resource_path_root / "web/templates/repo-index.jinja2",
+    ])
+    local_files = list(sorted(walk(resource_path_root)))
+    assert local_files == expected

tests/ahriman/core/upload/conftest.py

@@ -1,16 +1,58 @@
 import pytest
 
 from collections import namedtuple
-from typing import List
+from typing import Any, Dict, List
 from unittest.mock import MagicMock
 
 from ahriman.core.configuration import Configuration
+from ahriman.core.upload.github import Github
+from ahriman.core.upload.rsync import Rsync
 from ahriman.core.upload.s3 import S3
 
 
 _s3_object = namedtuple("s3_object", ["key", "e_tag", "delete"])
 
 
+@pytest.fixture
+def github(configuration: Configuration) -> Github:
+    """
+    fixture for github synchronization
+    :param configuration: configuration fixture
+    :return: github test instance
+    """
+    return Github("x86_64", configuration)
+
+
+@pytest.fixture
+def github_release() -> Dict[str, Any]:
+    """
+    fixture for the github release object
+    :return: github test release object
+    """
+    return {
+        "url": "release_url",
+        "assets_url": "assets_url",
+        "upload_url": "upload_url{?name,label}",
+        "tag_name": "x86_64",
+        "name": "x86_64",
+        "assets": [{
+            "url": "asset_url",
+            "name": "asset_name",
+        }],
+        "body": None,
+    }
+
+
+@pytest.fixture
+def rsync(configuration: Configuration) -> Rsync:
+    """
+    fixture for rsync synchronization
+    :param configuration: configuration fixture
+    :return: rsync test instance
+    """
+    return Rsync("x86_64", configuration)
+
+
 @pytest.fixture
 def s3(configuration: Configuration) -> S3:
     """

tests/ahriman/core/upload/test_github.py

@@ -0,0 +1,266 @@
+import pytest
+import requests
+
+from pathlib import Path
+from pytest_mock import MockerFixture
+from typing import Any, Dict
+from unittest import mock
+from unittest.mock import MagicMock
+
+from ahriman.core.upload.github import Github
+
+
+def test_calculate_hash_empty(resource_path_root: Path) -> None:
+    """
+    must calculate checksum for empty file correctly
+    """
+    path = resource_path_root / "models" / "empty_file_checksum"
+    assert Github.calculate_hash(path) == "d41d8cd98f00b204e9800998ecf8427e"
+
+
+def test_calculate_hash_small(resource_path_root: Path) -> None:
+    """
+    must calculate checksum for path which is single chunk
+    """
+    path = resource_path_root / "models" / "package_ahriman_srcinfo"
+    assert Github.calculate_hash(path) == "a55f82198e56061295d405aeb58f4062"
+
+
+def test_get_body_get_hashes() -> None:
+    """
+    must generate readable body
+    """
+    source = {Path("c"): "c_md5", Path("a"): "a_md5", Path("b"): "b_md5"}
+    body = Github.get_body(source)
+    parsed = Github.get_hashes({"body": body})
+    assert {fn.name: md5 for fn, md5 in source.items()} == parsed
+
+
+def test_get_hashes_empty() -> None:
+    """
+    must read empty body
+    """
+    assert Github.get_hashes({"body": None}) == {}
+
+
+def test_request(github: Github, mocker: MockerFixture) -> None:
+    """
+    must call request method
+    """
+    response_mock = MagicMock()
+    request_mock = mocker.patch("requests.request", return_value=response_mock)
+
+    github._request("GET", "url", arg="arg")
+    request_mock.assert_called_once_with("GET", "url", auth=github.auth, arg="arg")
+    response_mock.raise_for_status.assert_called_once()
+
+
+def test_request_exception(github: Github, mocker: MockerFixture) -> None:
+    """
+    must call request method and log HTTPError exception
+    """
+    mocker.patch("requests.request", side_effect=requests.HTTPError())
+    with pytest.raises(requests.HTTPError):
+        github._request("GET", "url", arg="arg")
+
+
+def test_asset_remove(github: Github, github_release: Dict[str, Any], mocker: MockerFixture) -> None:
+    """
+    must remove asset from the release
+    """
+    request_mock = mocker.patch("ahriman.core.upload.github.Github._request")
+    github.asset_remove(github_release, "asset_name")
+    request_mock.assert_called_with("DELETE", "asset_url")
+
+
+def test_asset_remove_unknown(github: Github, github_release: Dict[str, Any], mocker: MockerFixture) -> None:
+    """
+    must not fail if no asset found
+    """
+    request_mock = mocker.patch("ahriman.core.upload.github.Github._request")
+    github.asset_remove(github_release, "unknown_asset_name")
+    request_mock.assert_not_called()
+
+
+def test_asset_upload(github: Github, github_release: Dict[str, Any], mocker: MockerFixture) -> None:
+    """
+    must upload asset to the repository
+    """
+    mocker.patch("pathlib.Path.open", return_value=b"")
+    request_mock = mocker.patch("ahriman.core.upload.github.Github._request")
+    remove_mock = mocker.patch("ahriman.core.upload.github.Github.asset_remove")
+
+    github.asset_upload(github_release, Path("/root/new.tar.xz"))
+    request_mock.assert_called_with("POST", "upload_url", params={"name": "new.tar.xz"},
+                                    data=b"", headers={"Content-Type": "application/x-tar"})
+    remove_mock.assert_not_called()
+
+
+def test_asset_upload_with_removal(github: Github, github_release: Dict[str, Any], mocker: MockerFixture) -> None:
+    """
+    must remove existing file before upload
+    """
+    mocker.patch("pathlib.Path.open", return_value=b"")
+    mocker.patch("ahriman.core.upload.github.Github._request")
+    remove_mock = mocker.patch("ahriman.core.upload.github.Github.asset_remove")
+
+    github.asset_upload(github_release, Path("asset_name"))
+    remove_mock.assert_called_with(github_release, "asset_name")
+
+    github.asset_upload(github_release, Path("/root/asset_name"))
+    remove_mock.assert_called_with(github_release, "asset_name")
+
+
+def test_asset_upload_empty_mimetype(github: Github, github_release: Dict[str, Any], mocker: MockerFixture) -> None:
+    """
+    must upload asset to the repository with empty mime type if cannot guess it
+    """
+    mocker.patch("pathlib.Path.open", return_value=b"")
+    mocker.patch("ahriman.core.upload.github.Github.asset_remove")
+    mocker.patch("mimetypes.guess_type", return_value=(None, None))
+    request_mock = mocker.patch("ahriman.core.upload.github.Github._request")
+
+    github.asset_upload(github_release, Path("/root/new.tar.xz"))
+    request_mock.assert_called_with("POST", "upload_url", params={"name": "new.tar.xz"},
+                                    data=b"", headers={"Content-Type": "application/octet-stream"})
+
+
+def test_get_local_files(github: Github, resource_path_root: Path, mocker: MockerFixture) -> None:
+    """
+    must get all local files recursively
+    """
+    walk_mock = mocker.patch("ahriman.core.util.walk")
+    github.get_local_files(resource_path_root)
+    walk_mock.assert_called()
+
+
+def test_files_remove(github: Github, github_release: Dict[str, Any], mocker: MockerFixture) -> None:
+    """
+    must remove files from the remote
+    """
+    remove_mock = mocker.patch("ahriman.core.upload.github.Github.asset_remove")
+    github.files_remove(github_release, {Path("a"): "a"}, {"a": "a", "b": "b"})
+    remove_mock.assert_called_once_with(github_release, "b")
+
+
+def test_files_remove_empty(github: Github, github_release: Dict[str, Any], mocker: MockerFixture) -> None:
+    """
+    must remove nothing if nothing changed
+    """
+    remove_mock = mocker.patch("ahriman.core.upload.github.Github.asset_remove")
+    github.files_remove(github_release, {Path("a"): "a"}, {"a": "a"})
+    remove_mock.assert_not_called()
+
+
+def test_files_upload(github: Github, github_release: Dict[str, Any], mocker: MockerFixture) -> None:
+    """
+    must upload files to the remote
+    """
+    upload_mock = mocker.patch("ahriman.core.upload.github.Github.asset_upload")
+    github.files_upload(github_release, {Path("a"): "a", Path("b"): "c", Path("c"): "c"}, {"a": "a", "b": "b"})
+    upload_mock.assert_has_calls([
+        mock.call(github_release, Path("b")),
+        mock.call(github_release, Path("c")),
+    ])
+
+
+def test_files_upload_empty(github: Github, github_release: Dict[str, Any], mocker: MockerFixture) -> None:
+    """
+    must upload nothing if nothing changed
+    """
+    upload_mock = mocker.patch("ahriman.core.upload.github.Github.asset_upload")
+    github.files_upload(github_release, {Path("a"): "a"}, {"a": "a"})
+    upload_mock.assert_not_called()
+
+
+def test_release_create(github: Github, mocker: MockerFixture) -> None:
+    """
+    must create release
+    """
+    request_mock = mocker.patch("ahriman.core.upload.github.Github._request")
+    github.release_create()
+    request_mock.assert_called_once_with("POST", pytest.helpers.anyvar(str, True),
+                                         json={"tag_name": github.architecture, "name": github.architecture})
+
+
+def test_release_get(github: Github, mocker: MockerFixture) -> None:
+    """
+    must get release
+    """
+    request_mock = mocker.patch("ahriman.core.upload.github.Github._request")
+    github.release_get()
+    request_mock.assert_called_once_with("GET", pytest.helpers.anyvar(str, True))
+
+
+def test_release_get_empty(github: Github, mocker: MockerFixture) -> None:
+    """
+    must return nothing in case of 404 status code
+    """
+    response = requests.Response()
+    response.status_code = 404
+    mocker.patch("ahriman.core.upload.github.Github._request", side_effect=requests.HTTPError(response=response))
+    assert github.release_get() is None
+
+
+def test_release_get_exception(github: Github, mocker: MockerFixture) -> None:
+    """
+    must re-raise non HTTPError exception
+    """
+    mocker.patch("ahriman.core.upload.github.Github._request", side_effect=Exception())
+    with pytest.raises(Exception):
+        github.release_get()
+
+
+def test_release_get_exception_http_error(github: Github, mocker: MockerFixture) -> None:
+    """
+    must re-raise HTTPError exception with code differs from 404
+    """
+    exception = requests.HTTPError(response=requests.Response())
+    mocker.patch("ahriman.core.upload.github.Github._request", side_effect=exception)
+    with pytest.raises(requests.HTTPError):
+        github.release_get()
+
+
+def test_release_update(github: Github, github_release: Dict[str, Any], mocker: MockerFixture) -> None:
+    """
+    must update release
+    """
+    request_mock = mocker.patch("ahriman.core.upload.github.Github._request")
+    github.release_update(github_release, "body")
+    request_mock.assert_called_once_with("POST", "release_url", json={"body": "body"})
+
+
+def test_release_sync(github: Github, mocker: MockerFixture) -> None:
+    """
+    must run sync command
+    """
+    release_get_mock = mocker.patch("ahriman.core.upload.github.Github.release_get")
+    get_hashes_mock = mocker.patch("ahriman.core.upload.github.Github.get_hashes")
+    get_local_files_mock = mocker.patch("ahriman.core.upload.github.Github.get_local_files")
+    files_upload_mock = mocker.patch("ahriman.core.upload.github.Github.files_upload")
+    files_remove_mock = mocker.patch("ahriman.core.upload.github.Github.files_remove")
+    release_update_mock = mocker.patch("ahriman.core.upload.github.Github.release_update")
+
+    github.sync(Path("local"), [])
+    release_get_mock.assert_called_once()
+    get_hashes_mock.assert_called_once()
+    get_local_files_mock.assert_called_once()
+    files_upload_mock.assert_called_once()
+    files_remove_mock.assert_called_once()
+    release_update_mock.assert_called_once()
+
+
+def test_release_sync_create_release(github: Github, mocker: MockerFixture) -> None:
+    """
+    must create release in case if it does not exist
+    """
+    mocker.patch("ahriman.core.upload.github.Github.release_get", return_value=None)
+    mocker.patch("ahriman.core.upload.github.Github.get_hashes")
+    mocker.patch("ahriman.core.upload.github.Github.get_local_files")
+    mocker.patch("ahriman.core.upload.github.Github.files_upload")
+    mocker.patch("ahriman.core.upload.github.Github.files_remove")
+    mocker.patch("ahriman.core.upload.github.Github.release_update")
+    release_create_mock = mocker.patch("ahriman.core.upload.github.Github.release_create")
+
+    github.sync(Path("local"), [])
+    release_create_mock.assert_called_once()

tests/ahriman/core/upload/test_rsync.py

@@ -1,16 +1,13 @@
 from pathlib import Path
 from pytest_mock import MockerFixture
 
-from ahriman.core.configuration import Configuration
 from ahriman.core.upload.rsync import Rsync
 
 
-def test_sync(configuration: Configuration, mocker: MockerFixture) -> None:
+def test_sync(rsync: Rsync, mocker: MockerFixture) -> None:
     """
     must run sync command
     """
     check_output_mock = mocker.patch("ahriman.core.upload.rsync.Rsync._check_output")
-
-    upload = Rsync("x86_64", configuration)
-    upload.sync(Path("path"), [])
+    rsync.sync(Path("path"), [])
     check_output_mock.assert_called_once()

tests/ahriman/core/upload/test_s3.py

@@ -34,7 +34,7 @@ def test_calculate_etag_small(resource_path_root: Path) -> None:
     assert S3.calculate_etag(path, _chunk_size) == "a55f82198e56061295d405aeb58f4062"
 
 
-def test_remove_files(s3_remote_objects: List[Any]) -> None:
+def test_files_remove(s3_remote_objects: List[Any]) -> None:
     """
     must remove remote objects
     """
@@ -43,35 +43,49 @@ def test_remove_files(s3_remote_objects: List[Any]) -> None:
     }
     remote_objects = {Path(item.key): item for item in s3_remote_objects}
 
-    S3.remove_files(local_files, remote_objects)
+    S3.files_remove(local_files, remote_objects)
     remote_objects[Path("x86_64/a")].delete.assert_called_once()
 
 
-def test_get_local_files(s3: S3, resource_path_root: Path) -> None:
+def test_files_upload(s3: S3, s3_remote_objects: List[Any], mocker: MockerFixture) -> None:
+    """
+    must upload changed files
+    """
+    def mimetype(path: Path) -> Tuple[Optional[str], None]:
+        return ("text/html", None) if path.name == "b" else (None, None)
+
+    root = Path("path")
+    local_files = {
+        Path(item.key.replace("a", "d")): item.e_tag.replace("b", "d").replace("\"", "")
+        for item in s3_remote_objects
+    }
+    remote_objects = {Path(item.key): item for item in s3_remote_objects}
+
+    mocker.patch("mimetypes.guess_type", side_effect=mimetype)
+    upload_mock = s3.bucket = MagicMock()
+
+    s3.files_upload(root, local_files, remote_objects)
+    upload_mock.upload_file.assert_has_calls(
+        [
+            mock.call(
+                Filename=str(root / s3.architecture / "b"),
+                Key=f"{s3.architecture}/{s3.architecture}/b",
+                ExtraArgs={"ContentType": "text/html"}),
+            mock.call(
+                Filename=str(root / s3.architecture / "d"),
+                Key=f"{s3.architecture}/{s3.architecture}/d",
+                ExtraArgs=None),
+        ],
+        any_order=True)
+
+
+def test_get_local_files(s3: S3, resource_path_root: Path, mocker: MockerFixture) -> None:
     """
     must get all local files recursively
     """
-    expected = sorted([
-        Path("core/ahriman.ini"),
-        Path("core/logging.ini"),
-        Path("models/big_file_checksum"),
-        Path("models/empty_file_checksum"),
-        Path("models/package_ahriman_srcinfo"),
-        Path("models/package_tpacpi-bat-git_srcinfo"),
-        Path("models/package_yay_srcinfo"),
-        Path("web/templates/build-status/login-modal.jinja2"),
-        Path("web/templates/build-status/package-actions-modals.jinja2"),
-        Path("web/templates/build-status/package-actions-script.jinja2"),
-        Path("web/templates/static/favicon.ico"),
-        Path("web/templates/utils/bootstrap-scripts.jinja2"),
-        Path("web/templates/utils/style.jinja2"),
-        Path("web/templates/build-status.jinja2"),
-        Path("web/templates/email-index.jinja2"),
-        Path("web/templates/repo-index.jinja2"),
-    ])
-
-    local_files = list(sorted(s3.get_local_files(resource_path_root).keys()))
-    assert local_files == expected
+    walk_mock = mocker.patch("ahriman.core.util.walk")
+    s3.get_local_files(resource_path_root)
+    walk_mock.assert_called()
 
 
 def test_get_remote_objects(s3: S3, s3_remote_objects: List[Any]) -> None:
@@ -92,43 +106,11 @@ def test_sync(s3: S3, mocker: MockerFixture) -> None:
     """
     local_files_mock = mocker.patch("ahriman.core.upload.s3.S3.get_local_files")
     remote_objects_mock = mocker.patch("ahriman.core.upload.s3.S3.get_remote_objects")
-    remove_files_mock = mocker.patch("ahriman.core.upload.s3.S3.remove_files")
-    upload_files_mock = mocker.patch("ahriman.core.upload.s3.S3.upload_files")
+    remove_files_mock = mocker.patch("ahriman.core.upload.s3.S3.files_remove")
+    upload_files_mock = mocker.patch("ahriman.core.upload.s3.S3.files_upload")
 
     s3.sync(Path("root"), [])
     local_files_mock.assert_called_once()
     remote_objects_mock.assert_called_once()
     remove_files_mock.assert_called_once()
     upload_files_mock.assert_called_once()
-
-
-def test_upload_files(s3: S3, s3_remote_objects: List[Any], mocker: MockerFixture) -> None:
-    """
-    must upload changed files
-    """
-    def mimetype(path: Path) -> Tuple[Optional[str], None]:
-        return ("text/html", None) if path.name == "b" else (None, None)
-
-    root = Path("path")
-    local_files = {
-        Path(item.key.replace("a", "d")): item.e_tag.replace("b", "d").replace("\"", "")
-        for item in s3_remote_objects
-    }
-    remote_objects = {Path(item.key): item for item in s3_remote_objects}
-
-    mocker.patch("mimetypes.guess_type", side_effect=mimetype)
-    upload_mock = s3.bucket = MagicMock()
-
-    s3.upload_files(root, local_files, remote_objects)
-    upload_mock.upload_file.assert_has_calls(
-        [
-            mock.call(
-                Filename=str(root / s3.architecture / "b"),
-                Key=f"{s3.architecture}/{s3.architecture}/b",
-                ExtraArgs={"ContentType": "text/html"}),
-            mock.call(
-                Filename=str(root / s3.architecture / "d"),
-                Key=f"{s3.architecture}/{s3.architecture}/d",
-                ExtraArgs=None),
-        ],
-        any_order=True)

tests/ahriman/core/upload/test_upload.py

@@ -44,3 +44,12 @@ def test_upload_s3(configuration: Configuration, mocker: MockerFixture) -> None:
     upload_mock = mocker.patch("ahriman.core.upload.s3.S3.sync")
     Upload.load("x86_64", configuration, UploadSettings.S3.name).run(Path("path"), [])
     upload_mock.assert_called_once()
+
+
+def test_upload_github(configuration: Configuration, mocker: MockerFixture) -> None:
+    """
+    must upload via github
+    """
+    upload_mock = mocker.patch("ahriman.core.upload.github.Github.sync")
+    Upload.load("x86_64", configuration, UploadSettings.Github.name).run(Path("path"), [])
+    upload_mock.assert_called_once()

tests/ahriman/models/test_upload_settings.py

@@ -21,3 +21,6 @@ def test_from_option_valid() -> None:
 
     assert UploadSettings.from_option("s3") == UploadSettings.S3
     assert UploadSettings.from_option("S3") == UploadSettings.S3
+
+    assert UploadSettings.from_option("github") == UploadSettings.Github
+    assert UploadSettings.from_option("GitHub") == UploadSettings.Github

tests/testresources/core/ahriman.ini

@@ -61,6 +61,11 @@ bucket = bucket
 region = eu-central-1
 secret_key =
 
+[github]
+api_key =
+owner = arcan1s
+repository = ahriman
+
 [web]
 debug = no
 debug_check_host = no