azaion/ai-training
1
0
Fork 0
mirror of https://github.com/azaion/ai-training.git synced 2026-04-22 11:06:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

update cdn manager WIP

Browse Source
This commit is contained in:
Dennis Popov
2025-04-18 10:39:05 +02:00
parent 3985d424c0
commit adcbdb4915
3 changed files with 52 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cdn_manager.py
+45 -9
View File
@@ -1,25 +1,34 @@
import io
import sys
import boto3
import yaml
import os
class CDNCredentials:
def __init__(self, host, access_key, secret_key):
def __init__(self, host, downloader_access_key, downloader_access_secret, uploader_access_key, uploader_access_secret):
self.host = host
self.access_key = access_key
self.secret_key = secret_key
self.downloader_access_key = downloader_access_key
self.downloader_access_secret = downloader_access_secret
self.uploader_access_key = uploader_access_key
self.uploader_access_secret = uploader_access_secret
class CDNManager:
def __init__(self, credentials: CDNCredentials):
self.creds = credentials
self.minio_client = boto3.client('s3', endpoint_url=self.creds.host,
aws_access_key_id=self.creds.access_key,
aws_secret_access_key=self.creds.secret_key)
self.download_client = boto3.client('s3', endpoint_url=self.creds.host,
aws_access_key_id=self.creds.downloader_access_key,
aws_secret_access_key=self.creds.downloader_access_secret)
self.upload_client = boto3.client('s3', endpoint_url=self.creds.host,
aws_access_key_id=self.creds.uploader_access_key,
aws_secret_access_key=self.creds.uploader_access_secret)
def upload(self, bucket: str, filename: str, file_bytes: bytearray):
try:
self.minio_client.upload_fileobj(io.BytesIO(file_bytes), bucket, filename)
print(f'Uploaded {len(file_bytes)} bytes to {self.creds.host}/{bucket}/{filename}')
self.upload_client.upload_fileobj(io.BytesIO(file_bytes), bucket, filename)
print(f'uploaded {filename} ({len(file_bytes)} bytes) to the {bucket}')
return True
except Exception as e:
print(e)
@@ -27,8 +36,35 @@ class CDNManager:
def download(self, bucket: str, filename: str):
try:
self.minio_client.download_file(bucket, filename, filename)
self.download_client.download_file(bucket, filename, filename)
print(f'downloaded {filename} from the {bucket} to current folder')
return True
except Exception as e:
print(e)
return False
config_file = os.path.join('downloader_config.yaml')
with open(config_file, 'r') as f:
config = yaml.safe_load(f)
cdn_manager = CDNManager(CDNCredentials(
config['host'],
config['downloader_access_key'],
config['downloader_access_secret'],
config['uploader_access_key'],
config['uploader_access_secret']
))
input_action = sys.argv[1]
input_bucket = sys.argv[2]
input_filename = sys.argv[3]
if len(sys.argv) > 4: # 0 is this script's path, hence 5 args is max
input_path = sys.argv[4]
if input_action == 'upload':
with open(input_path, 'rb') as f:
cdn_manager.upload(input_bucket, input_filename, bytearray(f.read()))
elif input_action == 'download':
cdn_manager.download(input_bucket, input_filename)