added support for http endpoints and OAuth2 PAM configuration - secon…

…d part

rulebase/eudat.re

@@ -7,12 +7,14 @@
 # List of the functions:
 #
 # ---- logging ---
+# logVerbose(*msg)
 # logInfo(*msg)
 # logDebug(*msg)
 # logError(*msg)
 # logWithLevel(*level, *msg)
 #---- authorization ---
 # EUDATAuthZ(*user, *action, *target, *response)
+# EUDATGetPAMusers(*json_map)
 #---- utility ---
 # EUDATObjExist(*path, *response)
 # EUDATPushMetadata(*path, *queue)
@@ -747,6 +749,21 @@ EUDATcountMetaKeys( *Path, *Key, *Value ) {
     logVerbose("[EUDATcountMetaKeys] got count = *Value");     
 }
 
+# get the content of the user map file for OAuth2 authentication in json format
+# 
+# Parameters:
+#     *json_map     [OUT] a string representing the user mapping
+#
+# Author: Claudio Cacciari, CINECA
+# -----------------------------------------------------------------------------
+EUDATGetPAMusers(*json_map) {
+    logVerbose("[EUDATGetPAMusers] checking authorization for $userNameClient to read users");
+    EUDATAuthZ($userNameClient, "read", "users", *response);
+    msiExecCmd("pam_user_reader.py", "null", "null", "null", "null", *outUsersJson);
+    msiGetStdoutInExecCmdOut(*outUsersJson, *json_map);    
+    logVerbose("[EUDATGetPAMusers] json user map = *json_map");
+}
+
 ################################################################################
 #                                                                              #
 # Repository Packages                                                          #

rulebase/pid-service.re

@@ -23,7 +23,7 @@
 # EUDATePIDsearch(*field, *value, *PID)
 # EUDATeCHECKSUMupdate(*PID, *path)
 # EUDATeURLupdate(*PID, *newURL)
-# EUDATeURLsearch(*PID, *URL)
+# EUDATeURLupdateColl(*PID, *newURL)
 # EUDATePIDremove(*path, *force)
 # EUDATiRORupdate(*source, *pid)
 # EUDATeRORupdate(*pid,*newRor)
@@ -167,13 +167,13 @@ EUDATSearchPID(*path, *existing_pid) {
 #-------------------------------------------------------------------------------
 EUDATSearchPIDchecksum(*path, *existing_pid, *existing_url) {
 
-    logDebug("[EUDATSearchPIDchecksum] search pid for *path");
+    logDebug("[EUDATSearchPIDchecksum] searching checksum for *path");
     getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
 
     *resource = "";
     EUDATiCHECKSUMget(*path, *checksum, *modtime, *resource)
     EUDATePIDsearch("EUDAT/CHECKSUM", *checksum, *existing_pid);
-    EUDATeURLsearch(*existing_pid, *existing_url);
+    *existing_url = EUDATGeteValPid(*existing_pid, "URL");
     logDebug("[EUDATSearchPIDchecksum] PID = *existing_pid, URL = *existing_url");
 }
 
@@ -412,8 +412,8 @@ EUDATeURLupdate(*PID, *newURL) {
 # and all its sub-collections and objects.
 #
 # Arguments:
-#   *PID                [IN] The PID associated to $collName
-#   *newURL             [IN] The new URL to be associated to the PID of $collName
+#   *PID                [IN] The PID associated to the collection
+#   *newURL             [IN] The new URL to be associated to the PID of the collection
 #
 # Author: Claudio Cacciari, CINECA
 #-------------------------------------------------------------------------------
@@ -422,14 +422,13 @@ EUDATeURLupdateColl(*PID, *newURL) {
     logDebug("[EUDATeURLupdateColl] updating collection *PID with URL *newURL and all its content");
     getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
     getHttpApiParameters(*serverApireg, *serverApipub);
+    *oldURL = EUDATGeteValPid(*PID, "URL");
     if (*oldURL like "*serverApireg\*" ) {
         *serverApi = *serverApireg
     }
     else {
        *serverApi = *serverApipub
     }
-
-    EUDATeURLsearch(*PID, *oldURL);
     msiStrlen(*serverApi,*serverApiLength);
     msiSubstr(*oldURL, str(int(*serverApiLength)), "null", *sourcePath);
     msiSubstr(*newURL, str(int(*serverApiLength)), "null", *targetPath);
@@ -459,23 +458,6 @@ EUDATeURLupdateColl(*PID, *newURL) {
     }
 }
 
-# This function search the URL field of the PID
-#
-# Arguments:
-#   *PID                [IN] The PID associated to $objPath
-#   *newURL             [IN] The new URL to be associated to the PID
-#
-# Author: Giacomo Mariani, CINECA
-#-------------------------------------------------------------------------------
-EUDATeURLsearch(*PID, *URL) {
-    getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
-    logDebug("[EUDATeURLsearch] search URL in PID *PID");
-    msiExecCmd("epicclient.py","*credStoreType *credStorePath read *PID --key URL ",
-               "null", "null", "null", *outEUS);
-    msiGetStdoutInExecCmdOut(*outEUS, *URL);
-    logDebug("[EUDATeURLsearch] response = *URL");
-}
-
 # This function remove an ePID... even if its EUDAT/REPLICA field is not empty!
 # To be improved.       
 #

rulebase/replication.re

@@ -10,15 +10,20 @@
 # List of the functions:
 #
 # EUDATUpdateLogging(*status_transfer_success, *source, *destination, *cause)
-# EUDATCheckIntegrity(*source, *source_res, *destination, *dest_res, *logEnabled,*notification,*response)
-# EUDATReplication(*source, *destination, *dest_res, *registered, *recursive)
+# EUDATCheckIntegrity(*source, *source_res, *destination, *dest_res, *logEnabled, *notification, *response)
+# EUDATCheckIntegrity(*source, *destination, *logEnabled, *notification, *response)
+# EUDATReplication(*source, *destination, *dest_res, *registered, *recursive, *response)
+# EUDATReplication(*source, *destination, *registered, *recursive, *response)
 # EUDATTransferUsingFailLog(*buffer_length, *stats)
 # EUDATRegDataRepl(*source, *destination, *dest_res, *recursive, *response)
+# EUDATRegDataRepl(*source, *destination, *recursive, *response)
 # EUDATPIDRegistration(*source, *destination, *notification, *registration_response)
 # EUDATSearchAndCreatePID(*path, *pid)
 # EUDATSearchAndDefineField(*path, *pid, *key)
-# EUDATCheckIntegrityColl(*sCollPesult*source_res, *dCollPath, *dest_res, *logEnabled, *response) 
+# EUDATCheckIntegrityColl(*sCollPath, *source_res, *dCollPath, *dest_res, *logEnabled, *response) 
+# EUDATCheckIntegrityColl(*sCollPath, *dCollPath, *logEnabled, *check_response)
 # EUDATCheckIntegrityDO(*source, *source_res, *destination, *dest_res, *logEnabled, *response)
+# EUDATCheckIntegrityDO(*source, *destination, *logEnabled, *response)
 
 
 # Update the logging files specific for EUDAT B2SAFE

scripts/tests/test_suite.sh

@@ -21,6 +21,8 @@ fi
 # Define test file name
 testFileName="test_data.txt"
 testFileNameRemote="test_data2.txt"
+testCollName="test_coll_root"
+testCollNameSub="test_coll_sub"
 
 # Get user name and zone name from ienv
 irods_user_name=`ienv | grep irods_user_name | cut -d '-' -f 2 | tr -d '[[:space:]]'`
@@ -72,6 +74,35 @@ echo "############ Data Object ############"
 ils -l ${testFileName}
 echo ""
 
+# Define collection
+collPath="${irods_home}/${testCollName}"
+# If exists, ask whether replace or exit
+exists=`ils ${irods_home} | grep ${testCollName}`
+if [ ! -z $exists ]
+then
+    echo "The collection $collPath already exists. Remove it before continuing (y or n)? Otherwise, script will exit."
+    read shouldRemove
+    echo "You entered $shouldRemove"
+    if [ $shouldRemove == "y" ]
+    then
+        echo "Will be removed"
+        irm -rf $collPath
+    else
+        echo "Exiting..."
+        exit 1
+    fi
+fi
+
+imkdir ${testCollName}
+imkdir "${testCollName}/${testCollNameSub}"
+echo "Hello World!" > ${testFileName}
+iput ${testFileName} ${testCollName}
+iput ${testFileName} "${testCollName}/${testCollNameSub}"
+rm ${testFileName}
+echo "############ Collection ############"
+ils -rl ${testCollName}
+echo ""
+
 
 createPID () {
   rule="{EUDATCreatePID(*parent_pid, *path, *ror, *fio, *fixed, *newPID)}"
@@ -105,8 +136,12 @@ createPID () {
 replication () {
   echo ""
   echo "############ REPLICATION ############"
-#  destPath="${irods_home}/test_data2.txt"
-  destPath="/${REMOTE_ZONE}/home/${irods_user_name}#${irods_zone_name}/${testFileNameRemote}"
+  if [ "${REMOTE_ZONE}" == "${irods_zone_name}" ]
+  then
+    destPath="${irods_home}/${testFileNameRemote}"
+  else
+    destPath="/${REMOTE_ZONE}/home/${irods_user_name}#${irods_zone_name}/${testFileNameRemote}"
+  fi
   echo "Replica path: ${destPath}"
   rule="{*status = EUDATReplication(*source, *destination, *dest_res, *registered, *recursive, *response); 
         if (*status) {
@@ -161,10 +196,53 @@ replication () {
   irm ${destPath}
 }
 
+moveCollection () {
+  rule="{EUDATPidsForColl(*collPath, *fixed)}"
+  input="*collPath=${collPath}%*fixed=true"
+
+  echo ""
+  echo "############ Collection PID creation ############"
+  echo "Rule: irule ${rule} ${input} null"
+
+  irule "${rule}" "${input}" null
+  rule="{EUDATSearchPID(*path, *existing_pid)}"
+  input="*path=${collPath}"
+  output="*existing_pid"
+  pid_raw=`irule "${rule}" "${input}" "${output}"`
+  pid=`echo ${pid_raw} | cut -d '=' -f 2 | tr -d '[[:space:]]'`
+  echo "ROOT Collection PID: ${pid}"
+  root_coll_pid=${pid}
+  root_coll_url_raw=`irule "{EUDATGeteValPid(*pid, *key)}" "*pid=${root_coll_pid}%*key='URL'" ruleExecOut`
+  root_coll_url=`echo ${root_coll_url_raw} | cut -d '=' -f 2 | tr -d '[[:space:]]'`
+  echo "ROOT Collection URL: ${root_coll_url}"
+  new_collPath="${irods_home}/${testCollName}_new"
+
+  echo ""
+  echo "############ Moving collection ############"
+  imv ${collPath} ${new_collPath}
+
+  echo ""
+  echo "############ Updating collection PID ############"
+  http_url_raw=`irule "{getHttpApiParameters(*serverApireg, *serverApipub)}" null "*serverApireg"`
+  http_url=`echo ${http_url_raw} | cut -d '=' -f 2 | tr -d '[[:space:]]'`
+  newURL="${http_url}${new_collPath}"
+  irule "{EUDATeURLupdateColl(*pid, *newURL)}" "*pid=${pid}%*newURL=${newURL}" null
+  new_root_coll_url_raw=`irule "{EUDATGeteValPid(*pid, *key)}" "*pid=${root_coll_pid}%*key='URL'" ruleExecOut`
+  new_root_coll_url=`echo ${new_root_coll_url_raw} | cut -d '=' -f 2 | tr -d '[[:space:]]'`
+  echo "NEW ROOT Collection URL: ${new_root_coll_url}"
+  irule "{EUDATePIDremove(*path, *force)}" "*path=${new_collPath}%*force=true" null
+  irule "{EUDATePIDremove(*path, *force)}" "*path=${new_collPath}/${testFileName}%*force=true" null
+  irule "{EUDATePIDremove(*path, *force)}" "*path=${new_collPath}/${testCollNameSub}%*force=true" null
+  irule "{EUDATePIDremove(*path, *force)}" "*path=${new_collPath}/${testCollNameSub}/${testFileName}%*force=true" null
+  irm -rf ${new_collPath}
+}
+
 createPID
 if [ -n "$REMOTE_ZONE" ]; then
   replication
 fi
+moveCollection
 
 irule "{EUDATePIDremove(*path, *force)}" "*path=${sourcePath}%*force=true" null
 irm ${sourcePath}
+#irm -rf ${collPath}