implemented real s3 mode on worker

Signed-off-by: Duncan Ragsdale <[email protected]>

workers/pvinsight-validation-runner.py

@@ -35,6 +35,7 @@
 import zipfile
 import subprocess
 import logging
+import boto3
 
 # Basic logging configuration
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
@@ -53,6 +54,8 @@ def is_local():
 
 is_s3_emulation = is_local()
 
+S3_BUCKET_NAME = "pv-validation-hub-bucket"
+
 def pull_from_s3(s3_file_path):
     if s3_file_path.startswith('/'):
         s3_file_path = s3_file_path[1:]
@@ -71,7 +74,8 @@ def pull_from_s3(s3_file_path):
         with open(target_file_path, "wb") as f:
             f.write(r.content)
     else:
-        raise NotImplementedError("real s3 mode not implemented")
+        s3 = boto3.client('s3')
+        s3.download_file(S3_BUCKET_NAME, s3_file_path, target_file_path)
 
     return target_file_path
 
@@ -92,7 +96,8 @@ def push_to_s3(local_file_path, s3_file_path):
             if r.status_code != 204:
                 print(f"error put file {s3_file_path} to s3, status code {r.status_code} {r.content}", file=sys.stderr)
     else:
-        raise NotImplementedError("real s3 mode not implemented")
+        s3 = boto3.client('s3')
+        s3.upload_file(local_file_path, S3_BUCKET_NAME, s3_file_path)
 
 
 def convert_compressed_file_path_to_directory(compressed_file_path):

workers/submission_worker.py

@@ -27,6 +27,7 @@ def is_local():
 
 is_s3_emulation = is_local()
 
+S3_BUCKET_NAME = "pv-validation-hub-bucket"
 
 
 SUBMITTING = "submitting"
@@ -55,7 +56,12 @@ def pull_from_s3(s3_file_path):
         with open(target_file_path, "wb") as f:
             f.write(r.content)
     else:
-        raise NotImplementedError("real s3 mode not implemented")
+        s3 = boto3.client('s3')
+
+        try:
+            s3.download_file(S3_BUCKET_NAME, s3_file_path, target_file_path)
+        except:
+            return None
 
     return target_file_path
 
@@ -76,9 +82,14 @@ def push_to_s3(local_file_path, s3_file_path):
             if r.status_code != 204:
                 print(f"error put file {s3_file_path} to s3, status code {r.status_code} {r.content}", file=sys.stderr)
     else:
-        raise NotImplementedError("real s3 mode not implemented")
-
 
+        s3 = boto3.client('s3')
+
+        try:
+            s3.upload_file(local_file_path, S3_BUCKET_NAME, s3_file_path)
+        except:
+            return None
+
 def list_s3_bucket(s3_dir):
     if s3_dir.startswith('/'):
         s3_dir = s3_dir[1:]