Merge pull request #122 from kbase/develop

Develop

source/custom_scripts/get_copy_info_for_narratives.py

@@ -26,7 +26,7 @@
 
 mongoDB_metrics_connection = os.environ["MONGO_PATH"]
 
-ws_url = os.environ["WS_URL"]
+#ws_url = os.environ["WS_URL"]
 ws_user_token = os.environ["METRICS_WS_USER_TOKEN"]
 to_workspace = os.environ["WRK_SUFFIX"]
 

source/custom_scripts/get_duplicate_narrative_object_ids.py

@@ -27,7 +27,7 @@
 
 mongoDB_metrics_connection = os.environ["MONGO_PATH"]
 
-ws_url = os.environ["WS_URL"]
+#ws_url = os.environ["WS_URL"]
 ws_user_token = os.environ["METRICS_WS_USER_TOKEN"]
 to_workspace = os.environ["WRK_SUFFIX"]
 

source/custom_scripts/populate_orphaned_blobstore_nodes_handles.py

@@ -1,6 +1,6 @@
 from pymongo import MongoClient
 from pymongo import ReadPreference
-from biokbase.workspace.client import Workspace
+#from biokbase.workspace.client import Workspace
 #from installed_clients.AbstractHandleClient import AbstractHandle as HandleService
 from biokbase.service.Client import Client as ServiceClient
 import json as _json
@@ -22,7 +22,7 @@
 
 mongoDB_metrics_connection = os.environ["MONGO_PATH"]
 
-ws_url = os.environ["WS_URL"]
+#ws_url = os.environ["WS_URL"]
 ws_user_token = os.environ["METRICS_WS_USER_TOKEN"]
 to_workspace = os.environ["WRK_SUFFIX"]
 
@@ -159,7 +159,8 @@ def get_workspace_handles ():
     "values(%s, %s, %s, %s, %s, %s)"
 )
 
-
+wsov_handle_ids_not_in_handle_insert_count = 0
+wsov_handle_ids_not_in_handle_skipped_insert_count = 0
 for handle_id in workspace_handles_not_in_handles_set:
     for full_obj_id in workspaces_dict[handle_id]:
         ws_id = workspaces_dict[handle_id][full_obj_id]["ws"]
@@ -175,7 +176,14 @@ def get_workspace_handles ():
             user,
             obj_type,
         )
-        wsov_handle_ids_not_in_handle_insert_cursor.execute(wsov_handle_ids_not_in_handle_insert_statement, input_vals)
+
+        try:
+            wsov_handle_ids_not_in_handle_insert_cursor.execute(wsov_handle_ids_not_in_handle_insert_statement, input_vals)
+            wsov_handle_ids_not_in_handle_insert_count += 1
+        except mysql.Error as err:
+            wsov_handle_ids_not_in_handle_skipped_insert_count += 1
+
+#####
 
 handle_ids_not_in_ws_obj_ver_insert_cursor = db_connection.cursor(prepared=True)
 handle_ids_not_in_ws_obj_ver_insert_statement = (
@@ -184,6 +192,8 @@ def get_workspace_handles ():
     "values(%s, %s, %s, %s) "
 )
 
+handle_ids_not_in_ws_obj_ver_insert_count = 0
+handle_ids_not_in_ws_obj_ver_skipped_insert_count = 0
 for handle_id in handles_not_in_worspace_handles_set:
     bsid = handles_by_hid_dict[handle_id]["bsid"]
     user = handles_by_hid_dict[handle_id]["user"]
@@ -197,15 +207,23 @@ def get_workspace_handles ():
         user,
         save_date,
         )
-    handle_ids_not_in_ws_obj_ver_insert_cursor.execute(handle_ids_not_in_ws_obj_ver_insert_statement, input_vals)
-
+    try:
+        handle_ids_not_in_ws_obj_ver_insert_cursor.execute(handle_ids_not_in_ws_obj_ver_insert_statement, input_vals)
+        handle_ids_not_in_ws_obj_ver_insert_count += 1
+    except mysql.Error as err:
+        handle_ids_not_in_ws_obj_ver_skipped_insert_count += 1
+
+#####
+
 handles_blobstore_ids_not_in_nodes_insert_cursor = db_connection.cursor(prepared=True)
 handles_blobstore_ids_not_in_nodes_insert_statement = (
     "insert into metrics.handles_blobstore_ids_not_in_nodes "
     "(blobstore_id, handle_id, username, save_date) "
     "values(%s, %s, %s, %s) "
 )
 
+handles_blobstore_ids_not_in_nodes_insert_count = 0
+handles_blobstore_ids_not_in_nodes_skipped_insert_count = 0
 for bsid in handles_blobstores_not_in_blobstore_nodes:
     handle_id = handles_by_bsid_dict[bsid]["handle"]
     user = handles_by_bsid_dict[bsid]["user"]
@@ -219,16 +237,23 @@ def get_workspace_handles ():
         user,
         save_date,
         )
-    handles_blobstore_ids_not_in_nodes_insert_cursor.execute(handles_blobstore_ids_not_in_nodes_insert_statement, input_vals)
-
-
+    try:
+        handles_blobstore_ids_not_in_nodes_insert_cursor.execute(handles_blobstore_ids_not_in_nodes_insert_statement, input_vals)
+        handles_blobstore_ids_not_in_nodes_insert_count += 1
+    except mysql.Error as err:
+        handles_blobstore_ids_not_in_nodes_skipped_insert_count += 1
+
+#####
+
 blobstore_ids_not_in_handle_insert_cursor = db_connection.cursor(prepared=True)
 blobstore_ids_not_in_handle_insert_statement = (
         "insert into metrics.blobstore_ids_not_in_handle "
         "(blobstore_id, username, save_date) "
         "values(%s, %s, %s) "
     )
 
+blobstore_ids_not_in_handle_insert_count = 0
+blobstore_ids_not_in_handle_skipped_insert_count = 0
 for blobstore_id in blobstore_nodes_not_in_handles_set:
     user = blobstore_dict[blobstore_id]["user"]
     if user is None:
@@ -240,8 +265,11 @@ def get_workspace_handles ():
         user,
         save_date,
         )
-    blobstore_ids_not_in_handle_insert_cursor.execute(blobstore_ids_not_in_handle_insert_statement, input_vals)
-
+    try:
+        blobstore_ids_not_in_handle_insert_cursor.execute(blobstore_ids_not_in_handle_insert_statement, input_vals)
+        blobstore_ids_not_in_handle_insert_count += 1
+    except mysql.Error as err:
+        blobstore_ids_not_in_handle_skipped_insert_count += 1
 
 i = 0
 print("Blobstore_dict :")
@@ -285,6 +313,16 @@ def get_workspace_handles ():
 print("handles_not_in_worspace_handles_set length : " + str(len(handles_not_in_worspace_handles_set)))
 print("workspace_handles_not_in_handles_set : " + str(len(workspace_handles_not_in_handles_set)))
 
+print("wsov_handle_ids_not_in_handle_insert_count : " + str(wsov_handle_ids_not_in_handle_insert_count))
+print("wsov_handle_ids_not_in_handle_skipped_insert_count : " + str(wsov_handle_ids_not_in_handle_skipped_insert_count))
+print("handle_ids_not_in_ws_obj_ver_insert_count : " + str(handle_ids_not_in_ws_obj_ver_insert_count))
+print("handle_ids_not_in_ws_obj_ver_skipped_insert_count : " + str(handle_ids_not_in_ws_obj_ver_skipped_insert_count))
+
+print("handles_blobstore_ids_not_in_nodes_insert_count : " + str(handles_blobstore_ids_not_in_nodes_insert_count))
+print("handles_blobstore_ids_not_in_nodes_skipped_insert_count : " + str(handles_blobstore_ids_not_in_nodes_skipped_insert_count))
+print("blobstore_ids_not_in_handle_insert_count : " + str(blobstore_ids_not_in_handle_insert_count))
+print("blobstore_ids_not_in_handle_skipped_insert_count : " + str(blobstore_ids_not_in_handle_skipped_insert_count))
+
 print("--- total seconds %s seconds ---" % (time.time() - start_time))
 
 db_connection.commit()

source/custom_scripts/upload_get_copy_info_for_narratives.py

@@ -27,7 +27,7 @@
 
 mongoDB_metrics_connection = os.environ["MONGO_PATH"]
 
-ws_url = os.environ["WS_URL"]
+#ws_url = os.environ["WS_URL"]
 ws_user_token = os.environ["METRICS_WS_USER_TOKEN"]
 to_workspace = os.environ["WRK_SUFFIX"]
 

source/daily_cron_jobs/methods_upload_blobstore_details.py

@@ -1,6 +1,6 @@
 from pymongo import MongoClient
 from pymongo import ReadPreference
-from biokbase.workspace.client import Workspace
+#from biokbase.workspace.client import Workspace
 #from installed_clients.AbstractHandleClient import AbstractHandle as HandleService
 from biokbase.service.Client import Client as ServiceClient
 import json as _json
@@ -18,7 +18,7 @@
 
 mongoDB_metrics_connection = os.environ["MONGO_PATH"]
 
-ws_url = os.environ["WS_URL"]
+#ws_url = os.environ["WS_URL"]
 ws_user_token = os.environ["METRICS_WS_USER_TOKEN"]
 to_workspace = os.environ["WRK_SUFFIX"]