split github upload into generic http method and github specific

We might use some features from the http upload for another parser

docs/configuration.md

@@ -107,14 +107,14 @@ Remote synchronization settings.
 
 Group name must refer to architecture, e.g. it should be `github:x86_64` for x86_64 architecture. This feature requires Github key creation (see below).
 
-* `api_key` - created Github API key. In order to create it do the following:
-    1. Go to [settings page](https://github.com/settings/profile).
-    2. Switch to [developers settings](https://github.com/settings/apps).
-    3. Switch to [personal access tokens](https://github.com/settings/tokens).
-    4. Generate new token. Required scope is `public_repo` (or `repo` for private repository support).
 * `owner` - Github repository owner, string, required.
+* `password` - created Github API key. In order to create it do the following:
+  1. Go to [settings page](https://github.com/settings/profile).
+  2. Switch to [developers settings](https://github.com/settings/apps).
+  3. Switch to [personal access tokens](https://github.com/settings/tokens).
+  4. Generate new token. Required scope is `public_repo` (or `repo` for private repository support).
 * `repository` - Github repository name, string, required. Repository must be created before any action and must have active branch (e.g. with readme).
-* `username` - Github authorization user, string, optional, default is the same as `owner`.
+* `username` - Github authorization user, string, required. Basically the same as `owner`.
 
 ### `rsync:*` groups
 

docs/faq.md

@@ -194,9 +194,10 @@ There are several choices:
     target = github
 
     [github]
-    api_key = ...
     owner = ahriman
+    password = ...
     repository = repository
+    username = ahriman
     ```
 
 ## Reporting

src/ahriman/core/upload/github.py

@@ -17,7 +17,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-import hashlib
 import mimetypes
 import requests
 
@@ -25,15 +24,14 @@ from pathlib import Path
 from typing import Any, Dict, Iterable, Optional
 
 from ahriman.core.configuration import Configuration
-from ahriman.core.upload.upload import Upload
+from ahriman.core.upload.http_upload import HttpUpload
-from ahriman.core.util import exception_response_text, walk
+from ahriman.core.util import walk
 from ahriman.models.package import Package
 
 
-class Github(Upload):
+class Github(HttpUpload):
     """
     upload files to github releases
-    :ivar auth: requests authentication tuple
     :ivar gh_owner: github repository owner
     :ivar gh_repository: github repository name
     """
@@ -44,64 +42,10 @@ class Github(Upload):
         :param architecture: repository architecture
         :param configuration: configuration instance
         """
-        Upload.__init__(self, architecture, configuration)
+        HttpUpload.__init__(self, architecture, configuration, "github")
         self.gh_owner = configuration.get("github", "owner")
         self.gh_repository = configuration.get("github", "repository")
 
-        gh_api_key = configuration.get("github", "api_key")
-        gh_username = configuration.get("github", "username", fallback=self.gh_owner)
-        self.auth = (gh_username, gh_api_key)
-
-    @staticmethod
-    def calculate_hash(path: Path) -> str:
-        """
-        calculate file checksum. Github API does not provide hashes itself, so we have to handle it manually
-        :param path: path to local file
-        :return: calculated checksum of the file
-        """
-        with path.open("rb") as local_file:
-            md5 = hashlib.md5(local_file.read())  # nosec
-            return md5.hexdigest()
-
-    @staticmethod
-    def get_body(local_files: Dict[Path, str]) -> str:
-        """
-        generate release body from the checksums as returned from Github.get_hashes method
-        :param local_files: map of the paths to its checksum
-        :return: body to be inserted into release
-        """
-        return "\n".join(f"{file.name} {md5}" for file, md5 in sorted(local_files.items()))
-
-    @staticmethod
-    def get_hashes(release: Dict[str, Any]) -> Dict[str, str]:
-        """
-        get checksums of the content from the repository
-        :param release: release object
-        :return: map of the filename to its checksum as it is written in body
-        """
-        body: str = release["body"] or ""
-        files = {}
-        for line in body.splitlines():
-            file, md5 = line.split()
-            files[file] = md5
-        return files
-
-    def _request(self, method: str, url: str, **kwargs: Any) -> requests.Response:
-        """
-        github request wrapper
-        :param method: request method
-        :param url: request url
-        :param kwargs: request parameters to be passed as is
-        :return: request response object
-        """
-        try:
-            response = requests.request(method, url, auth=self.auth, **kwargs)
-            response.raise_for_status()
-        except requests.HTTPError as e:
-            self.logger.exception("could not perform %s request to %s: %s", method, url, exception_response_text(e))
-            raise
-        return response
-
     def asset_remove(self, release: Dict[str, Any], name: str) -> None:
         """
         remove asset from the release by name
@@ -210,7 +154,8 @@ class Github(Upload):
         if release is None:
             release = self.release_create()
 
-        remote_files = self.get_hashes(release)
+        body: str = release.get("body") or ""
+        remote_files = self.get_hashes(body)
         local_files = self.get_local_files(path)
 
         self.files_upload(release, local_files, remote_files)

src/ahriman/core/upload/http_upload.py

@@ -0,0 +1,96 @@
+#
+# Copyright (c) 2021 ahriman team.
+#
+# This file is part of ahriman
+# (see https://github.com/arcan1s/ahriman).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+import hashlib
+import requests
+
+from pathlib import Path
+from typing import Any, Dict
+
+from ahriman.core.configuration import Configuration
+from ahriman.core.upload.upload import Upload
+from ahriman.core.util import exception_response_text
+
+
+class HttpUpload(Upload):
+    """
+    helper for the http based uploads
+    :ivar auth: HTTP auth object
+    """
+
+    def __init__(self, architecture: str, configuration: Configuration, section: str) -> None:
+        """
+        default constructor
+        :param architecture: repository architecture
+        :param configuration: configuration instance
+        :param section: configuration section name
+        """
+        Upload.__init__(self, architecture, configuration)
+        password = configuration.get(section, "password")
+        username = configuration.get(section, "username")
+        self.auth = (password, username)
+
+    @staticmethod
+    def calculate_hash(path: Path) -> str:
+        """
+        calculate file checksum
+        :param path: path to local file
+        :return: calculated checksum of the file
+        """
+        with path.open("rb") as local_file:
+            md5 = hashlib.md5(local_file.read())  # nosec
+            return md5.hexdigest()
+
+    @staticmethod
+    def get_body(local_files: Dict[Path, str]) -> str:
+        """
+        generate release body from the checksums as returned from HttpUpload.get_hashes method
+        :param local_files: map of the paths to its checksum
+        :return: body to be inserted into release
+        """
+        return "\n".join(f"{file.name} {md5}" for file, md5 in sorted(local_files.items()))
+
+    @staticmethod
+    def get_hashes(body: str) -> Dict[str, str]:
+        """
+        get checksums of the content from the repository
+        :param body: release string body object
+        :return: map of the filename to its checksum as it is written in body
+        """
+        files = {}
+        for line in body.splitlines():
+            file, md5 = line.split()
+            files[file] = md5
+        return files
+
+    def _request(self, method: str, url: str, **kwargs: Any) -> requests.Response:
+        """
+        request wrapper
+        :param method: request method
+        :param url: request url
+        :param kwargs: request parameters to be passed as is
+        :return: request response object
+        """
+        try:
+            response = requests.request(method, url, auth=self.auth, **kwargs)
+            response.raise_for_status()
+        except requests.HTTPError as e:
+            self.logger.exception("could not perform %s request to %s: %s", method, url, exception_response_text(e))
+            raise
+        return response

tests/ahriman/core/upload/test_github.py

@@ -5,65 +5,10 @@ from pathlib import Path
 from pytest_mock import MockerFixture
 from typing import Any, Dict
 from unittest import mock
-from unittest.mock import MagicMock
 
 from ahriman.core.upload.github import Github
 
 
-def test_calculate_hash_empty(resource_path_root: Path) -> None:
-    """
-    must calculate checksum for empty file correctly
-    """
-    path = resource_path_root / "models" / "empty_file_checksum"
-    assert Github.calculate_hash(path) == "d41d8cd98f00b204e9800998ecf8427e"
-
-
-def test_calculate_hash_small(resource_path_root: Path) -> None:
-    """
-    must calculate checksum for path which is single chunk
-    """
-    path = resource_path_root / "models" / "package_ahriman_srcinfo"
-    assert Github.calculate_hash(path) == "a55f82198e56061295d405aeb58f4062"
-
-
-def test_get_body_get_hashes() -> None:
-    """
-    must generate readable body
-    """
-    source = {Path("c"): "c_md5", Path("a"): "a_md5", Path("b"): "b_md5"}
-    body = Github.get_body(source)
-    parsed = Github.get_hashes({"body": body})
-    assert {fn.name: md5 for fn, md5 in source.items()} == parsed
-
-
-def test_get_hashes_empty() -> None:
-    """
-    must read empty body
-    """
-    assert Github.get_hashes({"body": None}) == {}
-
-
-def test_request(github: Github, mocker: MockerFixture) -> None:
-    """
-    must call request method
-    """
-    response_mock = MagicMock()
-    request_mock = mocker.patch("requests.request", return_value=response_mock)
-
-    github._request("GET", "url", arg="arg")
-    request_mock.assert_called_once_with("GET", "url", auth=github.auth, arg="arg")
-    response_mock.raise_for_status.assert_called_once()
-
-
-def test_request_exception(github: Github, mocker: MockerFixture) -> None:
-    """
-    must call request method and log HTTPError exception
-    """
-    mocker.patch("requests.request", side_effect=requests.HTTPError())
-    with pytest.raises(requests.HTTPError):
-        github._request("GET", "url", arg="arg")
-
-
 def test_asset_remove(github: Github, github_release: Dict[str, Any], mocker: MockerFixture) -> None:
     """
     must remove asset from the release

tests/ahriman/core/upload/test_http_upload.py

@@ -0,0 +1,63 @@
+import pytest
+import requests
+
+from pathlib import Path
+from pytest_mock import MockerFixture
+from unittest.mock import MagicMock
+
+from ahriman.core.upload.github import Github
+from ahriman.core.upload.http_upload import HttpUpload
+
+
+def test_calculate_hash_empty(resource_path_root: Path) -> None:
+    """
+    must calculate checksum for empty file correctly
+    """
+    path = resource_path_root / "models" / "empty_file_checksum"
+    assert HttpUpload.calculate_hash(path) == "d41d8cd98f00b204e9800998ecf8427e"
+
+
+def test_calculate_hash_small(resource_path_root: Path) -> None:
+    """
+    must calculate checksum for path which is single chunk
+    """
+    path = resource_path_root / "models" / "package_ahriman_srcinfo"
+    assert HttpUpload.calculate_hash(path) == "a55f82198e56061295d405aeb58f4062"
+
+
+def test_get_body_get_hashes() -> None:
+    """
+    must generate readable body
+    """
+    source = {Path("c"): "c_md5", Path("a"): "a_md5", Path("b"): "b_md5"}
+    body = HttpUpload.get_body(source)
+    parsed = HttpUpload.get_hashes(body)
+    assert {fn.name: md5 for fn, md5 in source.items()} == parsed
+
+
+def test_get_hashes_empty() -> None:
+    """
+    must read empty body
+    """
+    assert HttpUpload.get_hashes("") == {}
+
+
+def test_request(github: Github, mocker: MockerFixture) -> None:
+    """
+    must call request method
+    """
+    response_mock = MagicMock()
+    request_mock = mocker.patch("requests.request", return_value=response_mock)
+
+    github._request("GET", "url", arg="arg")
+    request_mock.assert_called_once_with("GET", "url", auth=github.auth, arg="arg")
+    response_mock.raise_for_status.assert_called_once()
+
+
+def test_request_exception(github: Github, mocker: MockerFixture) -> None:
+    """
+    must call request method and log HTTPError exception
+    """
+    mocker.patch("requests.request", side_effect=requests.HTTPError())
+    with pytest.raises(requests.HTTPError):
+        github._request("GET", "url", arg="arg")

tests/testresources/core/ahriman.ini

@@ -62,9 +62,10 @@ region = eu-central-1
 secret_key =
 
 [github]
-api_key =
 owner = arcan1s
+password =
 repository = ahriman
+username = arcan1s
 
 [web]
 debug = no