diff --git a/source/custom_scripts/get_copy_info_for_narratives.py b/source/custom_scripts/get_copy_info_for_narratives.py index 225df03..51c7374 100644 --- a/source/custom_scripts/get_copy_info_for_narratives.py +++ b/source/custom_scripts/get_copy_info_for_narratives.py @@ -26,7 +26,7 @@ mongoDB_metrics_connection = os.environ["MONGO_PATH"] -ws_url = os.environ["WS_URL"] +#ws_url = os.environ["WS_URL"] ws_user_token = os.environ["METRICS_WS_USER_TOKEN"] to_workspace = os.environ["WRK_SUFFIX"] diff --git a/source/custom_scripts/get_duplicate_narrative_object_ids.py b/source/custom_scripts/get_duplicate_narrative_object_ids.py index b1ce05c..43a15a5 100644 --- a/source/custom_scripts/get_duplicate_narrative_object_ids.py +++ b/source/custom_scripts/get_duplicate_narrative_object_ids.py @@ -27,7 +27,7 @@ mongoDB_metrics_connection = os.environ["MONGO_PATH"] -ws_url = os.environ["WS_URL"] +#ws_url = os.environ["WS_URL"] ws_user_token = os.environ["METRICS_WS_USER_TOKEN"] to_workspace = os.environ["WRK_SUFFIX"] diff --git a/source/custom_scripts/populate_orphaned_blobstore_nodes_handles.py b/source/custom_scripts/populate_orphaned_blobstore_nodes_handles.py index 1776779..3d06eab 100644 --- a/source/custom_scripts/populate_orphaned_blobstore_nodes_handles.py +++ b/source/custom_scripts/populate_orphaned_blobstore_nodes_handles.py @@ -1,6 +1,6 @@ from pymongo import MongoClient from pymongo import ReadPreference -from biokbase.workspace.client import Workspace +#from biokbase.workspace.client import Workspace #from installed_clients.AbstractHandleClient import AbstractHandle as HandleService from biokbase.service.Client import Client as ServiceClient import json as _json @@ -22,7 +22,7 @@ mongoDB_metrics_connection = os.environ["MONGO_PATH"] -ws_url = os.environ["WS_URL"] +#ws_url = os.environ["WS_URL"] ws_user_token = os.environ["METRICS_WS_USER_TOKEN"] to_workspace = os.environ["WRK_SUFFIX"] @@ -159,7 +159,8 @@ def get_workspace_handles (): "values(%s, %s, %s, %s, %s, %s)" ) - +wsov_handle_ids_not_in_handle_insert_count = 0 +wsov_handle_ids_not_in_handle_skipped_insert_count = 0 for handle_id in workspace_handles_not_in_handles_set: for full_obj_id in workspaces_dict[handle_id]: ws_id = workspaces_dict[handle_id][full_obj_id]["ws"] @@ -175,7 +176,14 @@ def get_workspace_handles (): user, obj_type, ) - wsov_handle_ids_not_in_handle_insert_cursor.execute(wsov_handle_ids_not_in_handle_insert_statement, input_vals) + + try: + wsov_handle_ids_not_in_handle_insert_cursor.execute(wsov_handle_ids_not_in_handle_insert_statement, input_vals) + wsov_handle_ids_not_in_handle_insert_count += 1 + except mysql.Error as err: + wsov_handle_ids_not_in_handle_skipped_insert_count += 1 + +##### handle_ids_not_in_ws_obj_ver_insert_cursor = db_connection.cursor(prepared=True) handle_ids_not_in_ws_obj_ver_insert_statement = ( @@ -184,6 +192,8 @@ def get_workspace_handles (): "values(%s, %s, %s, %s) " ) +handle_ids_not_in_ws_obj_ver_insert_count = 0 +handle_ids_not_in_ws_obj_ver_skipped_insert_count = 0 for handle_id in handles_not_in_worspace_handles_set: bsid = handles_by_hid_dict[handle_id]["bsid"] user = handles_by_hid_dict[handle_id]["user"] @@ -197,8 +207,14 @@ def get_workspace_handles (): user, save_date, ) - handle_ids_not_in_ws_obj_ver_insert_cursor.execute(handle_ids_not_in_ws_obj_ver_insert_statement, input_vals) - + try: + handle_ids_not_in_ws_obj_ver_insert_cursor.execute(handle_ids_not_in_ws_obj_ver_insert_statement, input_vals) + handle_ids_not_in_ws_obj_ver_insert_count += 1 + except mysql.Error as err: + handle_ids_not_in_ws_obj_ver_skipped_insert_count += 1 + +##### + handles_blobstore_ids_not_in_nodes_insert_cursor = db_connection.cursor(prepared=True) handles_blobstore_ids_not_in_nodes_insert_statement = ( "insert into metrics.handles_blobstore_ids_not_in_nodes " @@ -206,6 +222,8 @@ def get_workspace_handles (): "values(%s, %s, %s, %s) " ) +handles_blobstore_ids_not_in_nodes_insert_count = 0 +handles_blobstore_ids_not_in_nodes_skipped_insert_count = 0 for bsid in handles_blobstores_not_in_blobstore_nodes: handle_id = handles_by_bsid_dict[bsid]["handle"] user = handles_by_bsid_dict[bsid]["user"] @@ -219,9 +237,14 @@ def get_workspace_handles (): user, save_date, ) - handles_blobstore_ids_not_in_nodes_insert_cursor.execute(handles_blobstore_ids_not_in_nodes_insert_statement, input_vals) - - + try: + handles_blobstore_ids_not_in_nodes_insert_cursor.execute(handles_blobstore_ids_not_in_nodes_insert_statement, input_vals) + handles_blobstore_ids_not_in_nodes_insert_count += 1 + except mysql.Error as err: + handles_blobstore_ids_not_in_nodes_skipped_insert_count += 1 + +##### + blobstore_ids_not_in_handle_insert_cursor = db_connection.cursor(prepared=True) blobstore_ids_not_in_handle_insert_statement = ( "insert into metrics.blobstore_ids_not_in_handle " @@ -229,6 +252,8 @@ def get_workspace_handles (): "values(%s, %s, %s) " ) +blobstore_ids_not_in_handle_insert_count = 0 +blobstore_ids_not_in_handle_skipped_insert_count = 0 for blobstore_id in blobstore_nodes_not_in_handles_set: user = blobstore_dict[blobstore_id]["user"] if user is None: @@ -240,8 +265,11 @@ def get_workspace_handles (): user, save_date, ) - blobstore_ids_not_in_handle_insert_cursor.execute(blobstore_ids_not_in_handle_insert_statement, input_vals) - + try: + blobstore_ids_not_in_handle_insert_cursor.execute(blobstore_ids_not_in_handle_insert_statement, input_vals) + blobstore_ids_not_in_handle_insert_count += 1 + except mysql.Error as err: + blobstore_ids_not_in_handle_skipped_insert_count += 1 i = 0 print("Blobstore_dict :") @@ -285,6 +313,16 @@ def get_workspace_handles (): print("handles_not_in_worspace_handles_set length : " + str(len(handles_not_in_worspace_handles_set))) print("workspace_handles_not_in_handles_set : " + str(len(workspace_handles_not_in_handles_set))) +print("wsov_handle_ids_not_in_handle_insert_count : " + str(wsov_handle_ids_not_in_handle_insert_count)) +print("wsov_handle_ids_not_in_handle_skipped_insert_count : " + str(wsov_handle_ids_not_in_handle_skipped_insert_count)) +print("handle_ids_not_in_ws_obj_ver_insert_count : " + str(handle_ids_not_in_ws_obj_ver_insert_count)) +print("handle_ids_not_in_ws_obj_ver_skipped_insert_count : " + str(handle_ids_not_in_ws_obj_ver_skipped_insert_count)) + +print("handles_blobstore_ids_not_in_nodes_insert_count : " + str(handles_blobstore_ids_not_in_nodes_insert_count)) +print("handles_blobstore_ids_not_in_nodes_skipped_insert_count : " + str(handles_blobstore_ids_not_in_nodes_skipped_insert_count)) +print("blobstore_ids_not_in_handle_insert_count : " + str(blobstore_ids_not_in_handle_insert_count)) +print("blobstore_ids_not_in_handle_skipped_insert_count : " + str(blobstore_ids_not_in_handle_skipped_insert_count)) + print("--- total seconds %s seconds ---" % (time.time() - start_time)) db_connection.commit() diff --git a/source/custom_scripts/upload_get_copy_info_for_narratives.py b/source/custom_scripts/upload_get_copy_info_for_narratives.py index 251a830..350fcbf 100644 --- a/source/custom_scripts/upload_get_copy_info_for_narratives.py +++ b/source/custom_scripts/upload_get_copy_info_for_narratives.py @@ -27,7 +27,7 @@ mongoDB_metrics_connection = os.environ["MONGO_PATH"] -ws_url = os.environ["WS_URL"] +#ws_url = os.environ["WS_URL"] ws_user_token = os.environ["METRICS_WS_USER_TOKEN"] to_workspace = os.environ["WRK_SUFFIX"] diff --git a/source/daily_cron_jobs/methods_upload_blobstore_details.py b/source/daily_cron_jobs/methods_upload_blobstore_details.py index 0ea3f5a..b7919e9 100644 --- a/source/daily_cron_jobs/methods_upload_blobstore_details.py +++ b/source/daily_cron_jobs/methods_upload_blobstore_details.py @@ -1,6 +1,6 @@ from pymongo import MongoClient from pymongo import ReadPreference -from biokbase.workspace.client import Workspace +#from biokbase.workspace.client import Workspace #from installed_clients.AbstractHandleClient import AbstractHandle as HandleService from biokbase.service.Client import Client as ServiceClient import json as _json @@ -18,7 +18,7 @@ mongoDB_metrics_connection = os.environ["MONGO_PATH"] -ws_url = os.environ["WS_URL"] +#ws_url = os.environ["WS_URL"] ws_user_token = os.environ["METRICS_WS_USER_TOKEN"] to_workspace = os.environ["WRK_SUFFIX"]