arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-07-16 15:29:56 +00:00
Code Issues Projects Releases Wiki Activity

github upload support (#41)

Browse Source
This commit is contained in:
Evgenii Alekseev
2021-10-14 02:30:13 +03:00
committed by GitHub
parent 72b26603bf
commit fcb167b1a3
17 changed files with 690 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/ahriman/application/application.py
View File

@ -109,7 +109,7 @@ class Application:
def add_archive(src: Path) -> None:
dst = self.repository.paths.packages / src.name
shutil.move(src, dst)
shutil.copy(src, dst)
def add_directory(path: Path) -> None:
for full_path in filter(package_like, path.iterdir()):

18
src/ahriman/core/status/web_client.py
View File

@ -111,7 +111,7 @@ class WebClient(Client):
try:
response = self.__session.post(self._login_url, json=payload)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
except requests.HTTPError as e:
self.logger.exception("could not login as %s: %s", self.user, exception_response_text(e))
except Exception:
self.logger.exception("could not login as %s", self.user)
@ -138,7 +138,7 @@ class WebClient(Client):
try:
response = self.__session.post(self._package_url(package.base), json=payload)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
except requests.HTTPError as e:
self.logger.exception("could not add %s: %s", package.base, exception_response_text(e))
except Exception:
self.logger.exception("could not add %s", package.base)
@ -158,7 +158,7 @@ class WebClient(Client):
(Package.from_json(package["package"]), BuildStatus.from_json(package["status"]))
for package in status_json
]
except requests.exceptions.HTTPError as e:
except requests.HTTPError as e:
self.logger.exception("could not get %s: %s", base, exception_response_text(e))
except Exception:
self.logger.exception("could not get %s", base)
@ -175,7 +175,7 @@ class WebClient(Client):
status_json = response.json()
return InternalStatus.from_json(status_json)
except requests.exceptions.HTTPError as e:
except requests.HTTPError as e:
self.logger.exception("could not get web service status: %s", exception_response_text(e))
except Exception:
self.logger.exception("could not get web service status")
@ -192,7 +192,7 @@ class WebClient(Client):
status_json = response.json()
return BuildStatus.from_json(status_json)
except requests.exceptions.HTTPError as e:
except requests.HTTPError as e:
self.logger.exception("could not get service status: %s", exception_response_text(e))
except Exception:
self.logger.exception("could not get service status")
@ -205,7 +205,7 @@ class WebClient(Client):
try:
response = self.__session.post(self._reload_auth_url)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
except requests.HTTPError as e:
self.logger.exception("could not reload auth module: %s", exception_response_text(e))
except Exception:
self.logger.exception("could not reload auth module")
@ -218,7 +218,7 @@ class WebClient(Client):
try:
response = self.__session.delete(self._package_url(base))
response.raise_for_status()
except requests.exceptions.HTTPError as e:
except requests.HTTPError as e:
self.logger.exception("could not delete %s: %s", base, exception_response_text(e))
except Exception:
self.logger.exception("could not delete %s", base)
@ -234,7 +234,7 @@ class WebClient(Client):
try:
response = self.__session.post(self._package_url(base), json=payload)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
except requests.HTTPError as e:
self.logger.exception("could not update %s: %s", base, exception_response_text(e))
except Exception:
self.logger.exception("could not update %s", base)
@ -249,7 +249,7 @@ class WebClient(Client):
try:
response = self.__session.post(self._ahriman_url, json=payload)
response.raise_for_status()
except requests.exceptions.HTTPError as e:
except requests.HTTPError as e:
self.logger.exception("could not update service status: %s", exception_response_text(e))
except Exception:
self.logger.exception("could not update service status")

218
src/ahriman/core/upload/github.py Normal file
View File

@ -0,0 +1,218 @@
#
# Copyright (c) 2021 ahriman team.
#
# This file is part of ahriman
# (see https://github.com/arcan1s/ahriman).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import hashlib
import mimetypes
import requests
from pathlib import Path
from typing import Any, Dict, Iterable, Optional
from ahriman.core.configuration import Configuration
from ahriman.core.upload.upload import Upload
from ahriman.core.util import exception_response_text, walk
from ahriman.models.package import Package
class Github(Upload):
"""
upload files to github releases
:ivar auth: requests authentication tuple
:ivar gh_owner: github repository owner
:ivar gh_repository: github repository name
"""
def __init__(self, architecture: str, configuration: Configuration) -> None:
"""
default constructor
:param architecture: repository architecture
:param configuration: configuration instance
"""
Upload.__init__(self, architecture, configuration)
self.gh_owner = configuration.get("github", "owner")
self.gh_repository = configuration.get("github", "repository")
gh_api_key = configuration.get("github", "api_key")
gh_username = configuration.get("github", "username", fallback=self.gh_owner)
self.auth = (gh_username, gh_api_key)
@staticmethod
def calculate_hash(path: Path) -> str:
"""
calculate file checksum. Github API does not provide hashes itself, so we have to handle it manually
:param path: path to local file
:return: calculated checksum of the file
"""
with path.open("rb") as local_file:
md5 = hashlib.md5(local_file.read()) # nosec
return md5.hexdigest()
@staticmethod
def get_body(local_files: Dict[Path, str]) -> str:
"""
generate release body from the checksums as returned from Github.get_hashes method
:param local_files: map of the paths to its checksum
:return: body to be inserted into release
"""
return "\n".join(f"{file.name} {md5}" for file, md5 in sorted(local_files.items()))
@staticmethod
def get_hashes(release: Dict[str, Any]) -> Dict[str, str]:
"""
get checksums of the content from the repository
:param release: release object
:return: map of the filename to its checksum as it is written in body
"""
body: str = release["body"] or ""
files = {}
for line in body.splitlines():
file, md5 = line.split()
files[file] = md5
return files
def _request(self, method: str, url: str, **kwargs: Any) -> requests.Response:
"""
github request wrapper
:param method: request method
:param url: request url
:param kwargs: request parameters to be passed as is
:return: request response object
"""
try:
response = requests.request(method, url, auth=self.auth, **kwargs)
response.raise_for_status()
except requests.HTTPError as e:
self.logger.exception("could not perform %s request to %s: %s", method, url, exception_response_text(e))
raise
return response
def asset_remove(self, release: Dict[str, Any], name: str) -> None:
"""
remove asset from the release by name
:param release: release object
:param name: asset name
"""
try:
asset = next(asset for asset in release["assets"] if asset["name"] == name)
self._request("DELETE", asset["url"])
except StopIteration:
self.logger.info("no asset %s found in release %s", name, release["name"])
def asset_upload(self, release: Dict[str, Any], path: Path) -> None:
"""
upload asset to the release
:param release: release object
:param path: path to local file
"""
exists = any(path.name == asset["name"] for asset in release["assets"])
if exists:
self.asset_remove(release, path.name)
(url, _) = release["upload_url"].split("{") # it is parametrized url
(mime, _) = mimetypes.guess_type(path)
headers = {"Content-Type": mime} if mime is not None else {"Content-Type": "application/octet-stream"}
self._request("POST", url, params={"name": path.name}, data=path.open("rb"), headers=headers)
def get_local_files(self, path: Path) -> Dict[Path, str]:
"""
get all local files and their calculated checksums
:param path: local path to sync
:return: map of path objects to its checksum
"""
return {
local_file: self.calculate_hash(local_file)
for local_file in walk(path)
}
def files_remove(self, release: Dict[str, Any], local_files: Dict[Path, str], remote_files: Dict[str, str]) -> None:
"""
remove files from github
:param release: release object
:param local_files: map of local file paths to its checksum
:param remote_files: map of the remote files and its checksum
"""
local_filenames = {local_file.name for local_file in local_files}
for remote_file in remote_files:
if remote_file in local_filenames:
continue
self.asset_remove(release, remote_file)
def files_upload(self, release: Dict[str, Any], local_files: Dict[Path, str], remote_files: Dict[str, str]) -> None:
"""
upload files to github
:param release: release object
:param local_files: map of local file paths to its checksum
:param remote_files: map of the remote files and its checksum
"""
for local_file, checksum in local_files.items():
remote_checksum = remote_files.get(local_file.name)
if remote_checksum == checksum:
continue
self.asset_upload(release, local_file)
def release_create(self) -> Dict[str, Any]:
"""
create empty release
:return: github API release object for the new release
"""
response = self._request("POST", f"https://api.github.com/repos/{self.gh_owner}/{self.gh_repository}/releases",
json={"tag_name": self.architecture, "name": self.architecture})
release: Dict[str, Any] = response.json()
return release
def release_get(self) -> Optional[Dict[str, Any]]:
"""
get release object if any
:return: github API release object if release found and None otherwise
"""
try:
response = self._request(
"GET",
f"https://api.github.com/repos/{self.gh_owner}/{self.gh_repository}/releases/tags/{self.architecture}")
release: Dict[str, Any] = response.json()
return release
except requests.HTTPError as e:
status_code = e.response.status_code if e.response is not None else None
if status_code == 404:
return None
raise
def release_update(self, release: Dict[str, Any], body: str) -> None:
"""
update release
:param release: release object
:param body: new release body
"""
self._request("POST", release["url"], json={"body": body})
def sync(self, path: Path, built_packages: Iterable[Package]) -> None:
"""
sync data to remote server
:param path: local path to sync
:param built_packages: list of packages which has just been built
"""
release = self.release_get()
if release is None:
release = self.release_create()
remote_files = self.get_hashes(release)
local_files = self.get_local_files(path)
self.files_upload(release, local_files, remote_files)
self.files_remove(release, local_files, remote_files)
self.release_update(release, self.get_body(local_files))

58
src/ahriman/core/upload/s3.py
View File

@ -22,10 +22,11 @@ import hashlib
import mimetypes
from pathlib import Path
from typing import Any, Dict, Generator, Iterable
from typing import Any, Dict, Iterable
from ahriman.core.configuration import Configuration
from ahriman.core.upload.upload import Upload
from ahriman.core.util import walk
from ahriman.models.package import Package
@ -82,7 +83,7 @@ class S3(Upload):
return client.Bucket(configuration.get("s3", "bucket"))
@staticmethod
def remove_files(local_files: Dict[Path, str], remote_objects: Dict[Path, Any]) -> None:
def files_remove(local_files: Dict[Path, str], remote_objects: Dict[Path, Any]) -> None:
"""
remove files which have been removed locally
:param local_files: map of local path object to its checksum
@ -93,19 +94,33 @@ class S3(Upload):
continue
remote_object.delete()
def files_upload(self, path: Path, local_files: Dict[Path, str], remote_objects: Dict[Path, Any]) -> None:
"""
upload changed files to s3
:param path: local path to sync
:param local_files: map of local path object to its checksum
:param remote_objects: map of remote path object to the remote s3 object
"""
for local_file, checksum in local_files.items():
remote_object = remote_objects.get(local_file)
# 0 and -1 elements are " (double quote)
remote_checksum = remote_object.e_tag[1:-1] if remote_object is not None else None
if remote_checksum == checksum:
continue
local_path = path / local_file
remote_path = Path(self.architecture) / local_file
(mime, _) = mimetypes.guess_type(local_path)
extra_args = {"ContentType": mime} if mime is not None else None
self.bucket.upload_file(Filename=str(local_path), Key=str(remote_path), ExtraArgs=extra_args)
def get_local_files(self, path: Path) -> Dict[Path, str]:
"""
get all local files and their calculated checksums
:param path: local path to sync
:return: map of path object to its checksum
"""
# credits to https://stackoverflow.com/a/64915960
def walk(directory_path: Path) -> Generator[Path, None, None]:
for element in directory_path.iterdir():
if element.is_dir():
yield from walk(element)
continue
yield element
return {
local_file.relative_to(path): self.calculate_etag(local_file, self.chunk_size)
for local_file in walk(path)
@ -128,26 +143,5 @@ class S3(Upload):
remote_objects = self.get_remote_objects()
local_files = self.get_local_files(path)
self.upload_files(path, local_files, remote_objects)
self.remove_files(local_files, remote_objects)
def upload_files(self, path: Path, local_files: Dict[Path, str], remote_objects: Dict[Path, Any]) -> None:
"""
upload changed files to s3
:param path: local path to sync
:param local_files: map of local path object to its checksum
:param remote_objects: map of remote path object to the remote s3 object
"""
for local_file, checksum in local_files.items():
remote_object = remote_objects.get(local_file)
# 0 and -1 elements are " (double quote)
remote_checksum = remote_object.e_tag[1:-1] if remote_object is not None else None
if remote_checksum == checksum:
continue
local_path = path / local_file
remote_path = Path(self.architecture) / local_file
(mime, _) = mimetypes.guess_type(local_path)
extra_args = {"ContentType": mime} if mime is not None else None
self.bucket.upload_file(Filename=str(local_path), Key=str(remote_path), ExtraArgs=extra_args)
self.files_upload(path, local_files, remote_objects)
self.files_remove(local_files, remote_objects)

3
src/ahriman/core/upload/upload.py
View File

@ -64,6 +64,9 @@ class Upload:
if provider == UploadSettings.S3:
from ahriman.core.upload.s3 import S3
return S3(architecture, configuration)
if provider == UploadSettings.Github:
from ahriman.core.upload.github import Github
return Github(architecture, configuration)
return cls(architecture, configuration) # should never happen
def run(self, path: Path, built_packages: Iterable[Package]) -> None:

16
src/ahriman/core/util.py
View File

@ -23,7 +23,7 @@ import requests
from logging import Logger
from pathlib import Path
from typing import Optional, Union
from typing import Generator, Optional, Union
from ahriman.core.exceptions import InvalidOption
@ -106,3 +106,17 @@ def pretty_size(size: Optional[float], level: int = 0) -> str:
if size < 1024 or level >= 3:
return f"{size:.1f} {str_level()}"
return pretty_size(size / 1024, level + 1)
def walk(directory_path: Path) -> Generator[Path, None, None]:
"""
list all file paths in given directory
Credits to https://stackoverflow.com/a/64915960
:param directory_path: root directory path
:return: all found files in given directory with full path
"""
for element in directory_path.iterdir():
if element.is_dir():
yield from walk(element)
continue
yield element

4
src/ahriman/models/upload_settings.py
View File

@ -31,11 +31,13 @@ class UploadSettings(Enum):
:cvar Disabled: no sync will be performed, required for testing purpose
:cvar Rsync: sync via rsync
:cvar S3: sync to Amazon S3
:cvar Github: sync to github releases page
"""
Disabled = "disabled" # for testing purpose
Rsync = "rsync"
S3 = "s3"
Github = "github"
@classmethod
def from_option(cls: Type[UploadSettings], value: str) -> UploadSettings:
@ -48,4 +50,6 @@ class UploadSettings(Enum):
return cls.Rsync
if value.lower() in ("s3",):
return cls.S3
if value.lower() in ("github",):
return cls.Github
raise InvalidOption(value)