arcanis/ahriman
1
0
Fork 0
mirror of https://github.com/arcan1s/ahriman.git synced 2025-07-03 00:55:49 +00:00
Code Issues Projects Releases Wiki Activity

split S3.sync to different methods

Browse Source
This commit is contained in:
Evgenii Alekseev
2021-08-18 12:13:11 +03:00
parent 343768e015
commit 2842e40bf7
2 changed files with 53 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
src/ahriman/core/upload/s3.py
View File

@ -81,6 +81,18 @@ class S3(Upload):
aws_secret_access_key=configuration.get("s3", "secret_key"))
return client.Bucket(configuration.get("s3", "bucket"))
@staticmethod
def remove_files(local_files: Dict[Path, str], remote_objects: Dict[Path, Any]) -> None:
"""
remove files which have been removed locally
:param local_files: map of local path object to its checksum
:param remote_objects: map of remote path object to the remote s3 object
"""
for local_file, remote_object in remote_objects.items():
if local_file in local_files:
continue
remote_object.delete()
def get_local_files(self, path: Path) -> Dict[Path, str]:
"""
get all local files and their calculated checksums
@ -116,7 +128,16 @@ class S3(Upload):
remote_objects = self.get_remote_objects()
local_files = self.get_local_files(path)
# sync to remotes first
self.upload_files(path, local_files, remote_objects)
self.remove_files(local_files, remote_objects)
def upload_files(self, path: Path, local_files: Dict[Path, str], remote_objects: Dict[Path, Any]) -> None:
"""
upload changed files to s3
:param path: local path to sync
:param local_files: map of local path object to its checksum
:param remote_objects: map of remote path object to the remote s3 object
"""
for local_file, checksum in local_files.items():
remote_object = remote_objects.get(local_file)
# 0 and -1 elements are " (double quote)
@ -130,9 +151,3 @@ class S3(Upload):
extra_args = {"Content-Type": mime} if mime is not None else None
self.bucket.upload_file(Filename=str(local_path), Key=str(remote_path), ExtraArgs=extra_args)
# remove files which were removed locally
for local_file, remote_object in remote_objects.items():
if local_file in local_files:
continue
remote_object.delete()