diff --git a/docs/configuration.md b/docs/configuration.md index 86f0878b..d1366e9b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -107,14 +107,14 @@ Remote synchronization settings. Group name must refer to architecture, e.g. it should be `github:x86_64` for x86_64 architecture. This feature requires Github key creation (see below). -* `api_key` - created Github API key. In order to create it do the following: - 1. Go to [settings page](https://github.com/settings/profile). - 2. Switch to [developers settings](https://github.com/settings/apps). - 3. Switch to [personal access tokens](https://github.com/settings/tokens). - 4. Generate new token. Required scope is `public_repo` (or `repo` for private repository support). * `owner` - Github repository owner, string, required. +* `password` - created Github API key. In order to create it do the following: + 1. Go to [settings page](https://github.com/settings/profile). + 2. Switch to [developers settings](https://github.com/settings/apps). + 3. Switch to [personal access tokens](https://github.com/settings/tokens). + 4. Generate new token. Required scope is `public_repo` (or `repo` for private repository support). * `repository` - Github repository name, string, required. Repository must be created before any action and must have active branch (e.g. with readme). -* `username` - Github authorization user, string, optional, default is the same as `owner`. +* `username` - Github authorization user, string, required. Basically the same as `owner`. ### `rsync:*` groups diff --git a/docs/faq.md b/docs/faq.md index 473f8458..15212dee 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -194,9 +194,10 @@ There are several choices: target = github [github] - api_key = ... owner = ahriman + password = ... repository = repository + username = ahriman ``` ## Reporting diff --git a/src/ahriman/core/upload/github.py b/src/ahriman/core/upload/github.py index e968958f..0b84aeab 100644 --- a/src/ahriman/core/upload/github.py +++ b/src/ahriman/core/upload/github.py @@ -17,7 +17,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -import hashlib import mimetypes import requests @@ -25,15 +24,14 @@ from pathlib import Path from typing import Any, Dict, Iterable, Optional from ahriman.core.configuration import Configuration -from ahriman.core.upload.upload import Upload -from ahriman.core.util import exception_response_text, walk +from ahriman.core.upload.http_upload import HttpUpload +from ahriman.core.util import walk from ahriman.models.package import Package -class Github(Upload): +class Github(HttpUpload): """ upload files to github releases - :ivar auth: requests authentication tuple :ivar gh_owner: github repository owner :ivar gh_repository: github repository name """ @@ -44,64 +42,10 @@ class Github(Upload): :param architecture: repository architecture :param configuration: configuration instance """ - Upload.__init__(self, architecture, configuration) + HttpUpload.__init__(self, architecture, configuration, "github") self.gh_owner = configuration.get("github", "owner") self.gh_repository = configuration.get("github", "repository") - gh_api_key = configuration.get("github", "api_key") - gh_username = configuration.get("github", "username", fallback=self.gh_owner) - self.auth = (gh_username, gh_api_key) - - @staticmethod - def calculate_hash(path: Path) -> str: - """ - calculate file checksum. Github API does not provide hashes itself, so we have to handle it manually - :param path: path to local file - :return: calculated checksum of the file - """ - with path.open("rb") as local_file: - md5 = hashlib.md5(local_file.read()) # nosec - return md5.hexdigest() - - @staticmethod - def get_body(local_files: Dict[Path, str]) -> str: - """ - generate release body from the checksums as returned from Github.get_hashes method - :param local_files: map of the paths to its checksum - :return: body to be inserted into release - """ - return "\n".join(f"{file.name} {md5}" for file, md5 in sorted(local_files.items())) - - @staticmethod - def get_hashes(release: Dict[str, Any]) -> Dict[str, str]: - """ - get checksums of the content from the repository - :param release: release object - :return: map of the filename to its checksum as it is written in body - """ - body: str = release["body"] or "" - files = {} - for line in body.splitlines(): - file, md5 = line.split() - files[file] = md5 - return files - - def _request(self, method: str, url: str, **kwargs: Any) -> requests.Response: - """ - github request wrapper - :param method: request method - :param url: request url - :param kwargs: request parameters to be passed as is - :return: request response object - """ - try: - response = requests.request(method, url, auth=self.auth, **kwargs) - response.raise_for_status() - except requests.HTTPError as e: - self.logger.exception("could not perform %s request to %s: %s", method, url, exception_response_text(e)) - raise - return response - def asset_remove(self, release: Dict[str, Any], name: str) -> None: """ remove asset from the release by name @@ -210,7 +154,8 @@ class Github(Upload): if release is None: release = self.release_create() - remote_files = self.get_hashes(release) + body: str = release.get("body") or "" + remote_files = self.get_hashes(body) local_files = self.get_local_files(path) self.files_upload(release, local_files, remote_files) diff --git a/src/ahriman/core/upload/http_upload.py b/src/ahriman/core/upload/http_upload.py new file mode 100644 index 00000000..e23cefc4 --- /dev/null +++ b/src/ahriman/core/upload/http_upload.py @@ -0,0 +1,96 @@ +# +# Copyright (c) 2021 ahriman team. +# +# This file is part of ahriman +# (see https://github.com/arcan1s/ahriman). +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +import hashlib +import requests + +from pathlib import Path +from typing import Any, Dict + +from ahriman.core.configuration import Configuration +from ahriman.core.upload.upload import Upload +from ahriman.core.util import exception_response_text + + +class HttpUpload(Upload): + """ + helper for the http based uploads + :ivar auth: HTTP auth object + """ + + def __init__(self, architecture: str, configuration: Configuration, section: str) -> None: + """ + default constructor + :param architecture: repository architecture + :param configuration: configuration instance + :param section: configuration section name + """ + Upload.__init__(self, architecture, configuration) + password = configuration.get(section, "password") + username = configuration.get(section, "username") + self.auth = (password, username) + + @staticmethod + def calculate_hash(path: Path) -> str: + """ + calculate file checksum + :param path: path to local file + :return: calculated checksum of the file + """ + with path.open("rb") as local_file: + md5 = hashlib.md5(local_file.read()) # nosec + return md5.hexdigest() + + @staticmethod + def get_body(local_files: Dict[Path, str]) -> str: + """ + generate release body from the checksums as returned from HttpUpload.get_hashes method + :param local_files: map of the paths to its checksum + :return: body to be inserted into release + """ + return "\n".join(f"{file.name} {md5}" for file, md5 in sorted(local_files.items())) + + @staticmethod + def get_hashes(body: str) -> Dict[str, str]: + """ + get checksums of the content from the repository + :param body: release string body object + :return: map of the filename to its checksum as it is written in body + """ + files = {} + for line in body.splitlines(): + file, md5 = line.split() + files[file] = md5 + return files + + def _request(self, method: str, url: str, **kwargs: Any) -> requests.Response: + """ + request wrapper + :param method: request method + :param url: request url + :param kwargs: request parameters to be passed as is + :return: request response object + """ + try: + response = requests.request(method, url, auth=self.auth, **kwargs) + response.raise_for_status() + except requests.HTTPError as e: + self.logger.exception("could not perform %s request to %s: %s", method, url, exception_response_text(e)) + raise + return response diff --git a/tests/ahriman/core/upload/test_github.py b/tests/ahriman/core/upload/test_github.py index e30b27e3..e11c2420 100644 --- a/tests/ahriman/core/upload/test_github.py +++ b/tests/ahriman/core/upload/test_github.py @@ -5,65 +5,10 @@ from pathlib import Path from pytest_mock import MockerFixture from typing import Any, Dict from unittest import mock -from unittest.mock import MagicMock from ahriman.core.upload.github import Github -def test_calculate_hash_empty(resource_path_root: Path) -> None: - """ - must calculate checksum for empty file correctly - """ - path = resource_path_root / "models" / "empty_file_checksum" - assert Github.calculate_hash(path) == "d41d8cd98f00b204e9800998ecf8427e" - - -def test_calculate_hash_small(resource_path_root: Path) -> None: - """ - must calculate checksum for path which is single chunk - """ - path = resource_path_root / "models" / "package_ahriman_srcinfo" - assert Github.calculate_hash(path) == "a55f82198e56061295d405aeb58f4062" - - -def test_get_body_get_hashes() -> None: - """ - must generate readable body - """ - source = {Path("c"): "c_md5", Path("a"): "a_md5", Path("b"): "b_md5"} - body = Github.get_body(source) - parsed = Github.get_hashes({"body": body}) - assert {fn.name: md5 for fn, md5 in source.items()} == parsed - - -def test_get_hashes_empty() -> None: - """ - must read empty body - """ - assert Github.get_hashes({"body": None}) == {} - - -def test_request(github: Github, mocker: MockerFixture) -> None: - """ - must call request method - """ - response_mock = MagicMock() - request_mock = mocker.patch("requests.request", return_value=response_mock) - - github._request("GET", "url", arg="arg") - request_mock.assert_called_once_with("GET", "url", auth=github.auth, arg="arg") - response_mock.raise_for_status.assert_called_once() - - -def test_request_exception(github: Github, mocker: MockerFixture) -> None: - """ - must call request method and log HTTPError exception - """ - mocker.patch("requests.request", side_effect=requests.HTTPError()) - with pytest.raises(requests.HTTPError): - github._request("GET", "url", arg="arg") - - def test_asset_remove(github: Github, github_release: Dict[str, Any], mocker: MockerFixture) -> None: """ must remove asset from the release diff --git a/tests/ahriman/core/upload/test_http_upload.py b/tests/ahriman/core/upload/test_http_upload.py new file mode 100644 index 00000000..97846527 --- /dev/null +++ b/tests/ahriman/core/upload/test_http_upload.py @@ -0,0 +1,63 @@ +import pytest +import requests + +from pathlib import Path +from pytest_mock import MockerFixture +from unittest.mock import MagicMock + +from ahriman.core.upload.github import Github +from ahriman.core.upload.http_upload import HttpUpload + + +def test_calculate_hash_empty(resource_path_root: Path) -> None: + """ + must calculate checksum for empty file correctly + """ + path = resource_path_root / "models" / "empty_file_checksum" + assert HttpUpload.calculate_hash(path) == "d41d8cd98f00b204e9800998ecf8427e" + + +def test_calculate_hash_small(resource_path_root: Path) -> None: + """ + must calculate checksum for path which is single chunk + """ + path = resource_path_root / "models" / "package_ahriman_srcinfo" + assert HttpUpload.calculate_hash(path) == "a55f82198e56061295d405aeb58f4062" + + +def test_get_body_get_hashes() -> None: + """ + must generate readable body + """ + source = {Path("c"): "c_md5", Path("a"): "a_md5", Path("b"): "b_md5"} + body = HttpUpload.get_body(source) + parsed = HttpUpload.get_hashes(body) + assert {fn.name: md5 for fn, md5 in source.items()} == parsed + + +def test_get_hashes_empty() -> None: + """ + must read empty body + """ + assert HttpUpload.get_hashes("") == {} + + +def test_request(github: Github, mocker: MockerFixture) -> None: + """ + must call request method + """ + response_mock = MagicMock() + request_mock = mocker.patch("requests.request", return_value=response_mock) + + github._request("GET", "url", arg="arg") + request_mock.assert_called_once_with("GET", "url", auth=github.auth, arg="arg") + response_mock.raise_for_status.assert_called_once() + + +def test_request_exception(github: Github, mocker: MockerFixture) -> None: + """ + must call request method and log HTTPError exception + """ + mocker.patch("requests.request", side_effect=requests.HTTPError()) + with pytest.raises(requests.HTTPError): + github._request("GET", "url", arg="arg") diff --git a/tests/testresources/core/ahriman.ini b/tests/testresources/core/ahriman.ini index 11f17573..af4c20e2 100644 --- a/tests/testresources/core/ahriman.ini +++ b/tests/testresources/core/ahriman.ini @@ -62,9 +62,10 @@ region = eu-central-1 secret_key = [github] -api_key = owner = arcan1s +password = repository = ahriman +username = arcan1s [web] debug = no