Merge remote-tracking branch 'upstream/devel' into devel

cmd/pam_user_reader.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+#
+#   * use 4 spaces!!! not tabs
+#   * See PEP-8 Python style guide http://www.python.org/dev/peps/pep-0008/
+#   * use pylint
+#
+
+import json
+
+user_map_file="/etc/irods/user_map.json"
+
+def parseUserMap():
+    """Parse the user map file"""
+
+    with open(user_map_file, 'r') as f:
+        user_map = json.load(f)
+        jstr = json.dumps(user_map, ensure_ascii=True, indent=4)
+        print(jstr)
+
+
+if __name__ == "__main__":
+
+    parseUserMap()

rulebase/eudat.re

@@ -7,12 +7,14 @@
 # List of the functions:
 #
 # ---- logging ---
+# logVerbose(*msg)
 # logInfo(*msg)
 # logDebug(*msg)
 # logError(*msg)
 # logWithLevel(*level, *msg)
 #---- authorization ---
 # EUDATAuthZ(*user, *action, *target, *response)
+# EUDATGetPAMusers(*json_map)
 #---- utility ---
 # EUDATObjExist(*path, *response)
 # EUDATPushMetadata(*path, *queue)
@@ -747,6 +749,21 @@ EUDATcountMetaKeys( *Path, *Key, *Value ) {
     logVerbose("[EUDATcountMetaKeys] got count = *Value");     
 }
 
+# get the content of the user map file for OAuth2 authentication in json format
+# 
+# Parameters:
+#     *json_map     [OUT] a string representing the user mapping
+#
+# Author: Claudio Cacciari, CINECA
+# -----------------------------------------------------------------------------
+EUDATGetPAMusers(*json_map) {
+    logVerbose("[EUDATGetPAMusers] checking authorization for $userNameClient to read users");
+    EUDATAuthZ($userNameClient, "read", "users", *response);
+    msiExecCmd("pam_user_reader.py", "null", "null", "null", "null", *outUsersJson);
+    msiGetStdoutInExecCmdOut(*outUsersJson, *json_map);    
+    logVerbose("[EUDATGetPAMusers] json user map = *json_map");
+}
+
 ################################################################################
 #                                                                              #
 # Repository Packages                                                          #

rulebase/local.re

@@ -39,6 +39,20 @@ getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicD
     }
 }
 
+# Provides parameters for the connection with the EPIC service
+#  
+# Arguments:
+# *serverApireg     [OUT] the endpoint of the HTTP API interface for data with PIDs
+# *serverApipub     [OUT] the endpoint of the HTTP API interface for public data
+#
+# Author: Claudio Cacciari (Cineca)
+#-------------------------------------------------------------------------------
+getHttpApiParameters(*serverApireg, *serverApipub) {
+
+    *serverApireg="https://<hostnameWithFullDomain>/api/registered";
+    *serverApipub="https://<hostnameWithFullDomain>/api/public";
+}
+
 # Parse the credentials to connect to an EPIC server. A file called
 # "credentials" MUST contain all the connection details in the home folder of
 # the user running this rule.
@@ -124,8 +138,8 @@ getConfParameters(*authzEnabled) {
 #-------------------------------------------------------------------------------
 getB2SAFEVersion(*version) {
     *major_version = "4";
-    *minor_version = "1";
-    *sub_version = "1";
+    *minor_version = "2";
+    *sub_version = "0";
     *version = *major_version ++ "." ++ *minor_version ++ "-" ++ *sub_version;
 }
 

rulebase/pid-service.re

@@ -23,7 +23,7 @@
 # EUDATePIDsearch(*field, *value, *PID)
 # EUDATeCHECKSUMupdate(*PID, *path)
 # EUDATeURLupdate(*PID, *newURL)
-# EUDATeURLsearch(*PID, *URL)
+# EUDATeURLupdateColl(*PID, *newURL)
 # EUDATePIDremove(*path, *force)
 # EUDATiRORupdate(*source, *pid)
 # EUDATeRORupdate(*pid,*newRor)
@@ -148,8 +148,12 @@ EUDATCreatePID(*parent_pid, *path, *ror, *fio, *fixed, *newPID) {
 EUDATSearchPID(*path, *existing_pid) {
     logDebug("[EUDATSearchPID] search pid for *path");
     getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
+    getHttpApiParameters(*serverApireg, *serverApipub);
     EUDATReplaceHash(*path, *path1);
-    *status = EUDATePIDsearch("URL", "*serverID*path1", *existing_pid);
+    *status = EUDATePIDsearch("URL", "*serverApireg*path1", *existing_pid);
+    if (!(*status)) {
+        *status = EUDATePIDsearch("URL", "*serverApipub*path1", *existing_pid);
+    }
     logDebug("[EUDATSearchPID] pid = *existing_pid");
     *status;
 }
@@ -163,13 +167,13 @@ EUDATSearchPID(*path, *existing_pid) {
 #-------------------------------------------------------------------------------
 EUDATSearchPIDchecksum(*path, *existing_pid, *existing_url) {
 
-    logDebug("[EUDATSearchPIDchecksum] search pid for *path");
+    logDebug("[EUDATSearchPIDchecksum] searching checksum for *path");
     getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
 
     *resource = "";
     EUDATiCHECKSUMget(*path, *checksum, *modtime, *resource)
     EUDATePIDsearch("EUDAT/CHECKSUM", *checksum, *existing_pid);
-    EUDATeURLsearch(*existing_pid, *existing_url);
+    *existing_url = EUDATGeteValPid(*existing_pid, "URL");
     logDebug("[EUDATSearchPIDchecksum] PID = *existing_pid, URL = *existing_url");
 }
 
@@ -264,8 +268,36 @@ EUDATiPIDcreate(*path, *PID) {
 EUDATePIDcreate(*path, *extraType, *PID) {
 
     getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug) ;
+    getHttpApiParameters(*serverApireg, *serverApipub);
 
-    *url="*serverID*path";
+    *access = "private";
+    # Verify that source input path is a collection
+    msiGetObjType(*path, *type);
+    if (*type != '-c') {
+        msiSplitPath(*path, *parent, *child); 
+        foreach ( *R in SELECT USER_NAME WHERE DATA_NAME = '*child' AND COLL_NAME = '*parent') {
+            if (*R.USER_NAME == "anonymous") {
+                *access = "public";
+                break;
+            }
+        }
+    }
+    else {
+        foreach ( *R in SELECT COLL_OWNER_NAME WHERE COLL_NAME = '*path' ) {
+            if (*R.COLL_OWNER_NAME == "anonymous") {
+                *access = "public";
+                break;
+            }
+        }
+    }
+
+    if (*access == "public") {
+        *url = "*serverApipub*path";
+    }
+    else {
+        *url = "*serverApireg*path";
+    }
+
     *resource = "";
     EUDATiCHECKSUMget(*path, *checksum, *modtime, *resource);
     if (*checksum != "") {
@@ -380,19 +412,26 @@ EUDATeURLupdate(*PID, *newURL) {
 # and all its sub-collections and objects.
 #
 # Arguments:
-#   *PID                [IN] The PID associated to $collName
-#   *newURL             [IN] The new URL to be associated to the PID of $collName
+#   *PID                [IN] The PID associated to the collection
+#   *newURL             [IN] The new URL to be associated to the PID of the collection
 #
 # Author: Claudio Cacciari, CINECA
 #-------------------------------------------------------------------------------
 EUDATeURLupdateColl(*PID, *newURL) {
 
     logDebug("[EUDATeURLupdateColl] updating collection *PID with URL *newURL and all its content");
     getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
-    EUDATeURLsearch(*PID, *oldURL);
-    msiStrlen(*serverID,*serverIDLength);
-    msiSubstr(*oldURL, str(int(*serverIDLength)), "null", *sourcePath);
-    msiSubstr(*newURL, str(int(*serverIDLength)), "null", *targetPath);
+    getHttpApiParameters(*serverApireg, *serverApipub);
+    *oldURL = EUDATGeteValPid(*PID, "URL");
+    if (*oldURL like "*serverApireg\*" ) {
+        *serverApi = *serverApireg
+    }
+    else {
+       *serverApi = *serverApipub
+    }
+    msiStrlen(*serverApi,*serverApiLength);
+    msiSubstr(*oldURL, str(int(*serverApiLength)), "null", *sourcePath);
+    msiSubstr(*newURL, str(int(*serverApiLength)), "null", *targetPath);
     msiStrlen(*targetPath, *pathLength);
 
     logDebug("[EUDATeURLupdateColl] loop over the sub-collections of *targetPath"); 
@@ -401,7 +440,7 @@ EUDATeURLupdateColl(*PID, *newURL) {
         msiSubstr(*Row.COLL_NAME, str(int(*pathLength)), "null", *subCollection);
         *sourceColl = "*sourcePath*subCollection"
         if (EUDATSearchPID(*sourceColl, *existing_pid)) {
-            *targetColl = *serverID ++ *Row.COLL_NAME;
+            *targetColl = *serverApi ++ *Row.COLL_NAME;
             EUDATeURLupdate(*existing_pid, *targetColl);
         }
     }
@@ -413,29 +452,12 @@ EUDATeURLupdateColl(*PID, *newURL) {
         msiSubstr(*objPath, str(int(*pathLength)), "null", *subObjPath);
         *sourceObj = "*sourcePath*subObjPath";
         if (EUDATSearchPID(*sourceObj, *existing_pid)) {
-            *targetObj = *serverID ++ *objPath;           
+            *targetObj = *serverApi ++ *objPath;           
             EUDATeURLupdate(*existing_pid, *targetObj);
         }
     }
 }
 
-# This function search the URL field of the PID
-#
-# Arguments:
-#   *PID                [IN] The PID associated to $objPath
-#   *newURL             [IN] The new URL to be associated to the PID
-#
-# Author: Giacomo Mariani, CINECA
-#-------------------------------------------------------------------------------
-EUDATeURLsearch(*PID, *URL) {
-    getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
-    logDebug("[EUDATeURLsearch] search URL in PID *PID");
-    msiExecCmd("epicclient.py","*credStoreType *credStorePath read *PID --key URL ",
-               "null", "null", "null", *outEUS);
-    msiGetStdoutInExecCmdOut(*outEUS, *URL);
-    logDebug("[EUDATeURLsearch] response = *URL");
-}
-
 # This function remove an ePID... even if its EUDAT/REPLICA field is not empty!
 # To be improved.       
 #
@@ -445,7 +467,7 @@ EUDATeURLsearch(*PID, *URL) {
 # Author: Giacomo Mariani, CINECA
 #-------------------------------------------------------------------------------
 EUDATePIDremove(*path, *force) {
-    getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug) 
+    getEpicApiParameters(*credStoreType, *credStorePath, *epicApi, *serverID, *epicDebug);
     logDebug("[EUDATePIDremove] removing PID associated to: $userNameClient, *path");
 
     if (EUDATSearchPID(*path, *pid)) {
@@ -547,7 +569,7 @@ EUDATeFIOupdate(*pid, *newFio) {
 #-----------------------------------------------------------------------------
 EUDATPidsForColl(*collPath, *fixed) {
 
-    logInfo("[EUDATPidsForColl] Creating PIDs for collection *collPath");
+    logInfo("[EUDATPidsForColl] Creating PIDs for collection *collPath and fixed = *fixed");
 
     # Verify that source input path is a collection
     msiGetObjType(*collPath, *type);

rulebase/replication.re

@@ -10,15 +10,20 @@
 # List of the functions:
 #
 # EUDATUpdateLogging(*status_transfer_success, *source, *destination, *cause)
-# EUDATCheckIntegrity(*source, *source_res, *destination, *dest_res, *logEnabled,*notification,*response)
-# EUDATReplication(*source, *destination, *dest_res, *registered, *recursive)
+# EUDATCheckIntegrity(*source, *source_res, *destination, *dest_res, *logEnabled, *notification, *response)
+# EUDATCheckIntegrity(*source, *destination, *logEnabled, *notification, *response)
+# EUDATReplication(*source, *destination, *dest_res, *registered, *recursive, *response)
+# EUDATReplication(*source, *destination, *registered, *recursive, *response)
 # EUDATTransferUsingFailLog(*buffer_length, *stats)
 # EUDATRegDataRepl(*source, *destination, *dest_res, *recursive, *response)
+# EUDATRegDataRepl(*source, *destination, *recursive, *response)
 # EUDATPIDRegistration(*source, *destination, *notification, *registration_response)
 # EUDATSearchAndCreatePID(*path, *pid)
 # EUDATSearchAndDefineField(*path, *pid, *key)
-# EUDATCheckIntegrityColl(*sCollPesult*source_res, *dCollPath, *dest_res, *logEnabled, *response) 
+# EUDATCheckIntegrityColl(*sCollPath, *source_res, *dCollPath, *dest_res, *logEnabled, *response) 
+# EUDATCheckIntegrityColl(*sCollPath, *dCollPath, *logEnabled, *check_response)
 # EUDATCheckIntegrityDO(*source, *source_res, *destination, *dest_res, *logEnabled, *response)
+# EUDATCheckIntegrityDO(*source, *destination, *logEnabled, *response)
 
 
 # Update the logging files specific for EUDAT B2SAFE

scripts/tests/test_suite.sh

@@ -21,6 +21,8 @@ fi
 # Define test file name
 testFileName="test_data.txt"
 testFileNameRemote="test_data2.txt"
+testCollName="test_coll_root"
+testCollNameSub="test_coll_sub"
 
 # Get user name and zone name from ienv
 irods_user_name=`ienv | grep irods_user_name | cut -d '-' -f 2 | tr -d '[[:space:]]'`
@@ -72,6 +74,35 @@ echo "############ Data Object ############"
 ils -l ${testFileName}
 echo ""
 
+# Define collection
+collPath="${irods_home}/${testCollName}"
+# If exists, ask whether replace or exit
+exists=`ils ${irods_home} | grep ${testCollName}`
+if [ ! -z $exists ]
+then
+    echo "The collection $collPath already exists. Remove it before continuing (y or n)? Otherwise, script will exit."
+    read shouldRemove
+    echo "You entered $shouldRemove"
+    if [ $shouldRemove == "y" ]
+    then
+        echo "Will be removed"
+        irm -rf $collPath
+    else
+        echo "Exiting..."
+        exit 1
+    fi
+fi
+
+imkdir ${testCollName}
+imkdir "${testCollName}/${testCollNameSub}"
+echo "Hello World!" > ${testFileName}
+iput ${testFileName} ${testCollName}
+iput ${testFileName} "${testCollName}/${testCollNameSub}"
+rm ${testFileName}
+echo "############ Collection ############"
+ils -rl ${testCollName}
+echo ""
+
 
 createPID () {
   rule="{EUDATCreatePID(*parent_pid, *path, *ror, *fio, *fixed, *newPID)}"
@@ -105,8 +136,12 @@ createPID () {
 replication () {
   echo ""
   echo "############ REPLICATION ############"
-#  destPath="${irods_home}/test_data2.txt"
-  destPath="/${REMOTE_ZONE}/home/${irods_user_name}#${irods_zone_name}/${testFileNameRemote}"
+  if [ "${REMOTE_ZONE}" == "${irods_zone_name}" ]
+  then
+    destPath="${irods_home}/${testFileNameRemote}"
+  else
+    destPath="/${REMOTE_ZONE}/home/${irods_user_name}#${irods_zone_name}/${testFileNameRemote}"
+  fi
   echo "Replica path: ${destPath}"
   rule="{*status = EUDATReplication(*source, *destination, *dest_res, *registered, *recursive, *response); 
         if (*status) {
@@ -161,10 +196,53 @@ replication () {
   irm ${destPath}
 }
 
+moveCollection () {
+  rule="{EUDATPidsForColl(*collPath, *fixed)}"
+  input="*collPath=${collPath}%*fixed=true"
+
+  echo ""
+  echo "############ Collection PID creation ############"
+  echo "Rule: irule ${rule} ${input} null"
+
+  irule "${rule}" "${input}" null
+  rule="{EUDATSearchPID(*path, *existing_pid)}"
+  input="*path=${collPath}"
+  output="*existing_pid"
+  pid_raw=`irule "${rule}" "${input}" "${output}"`
+  pid=`echo ${pid_raw} | cut -d '=' -f 2 | tr -d '[[:space:]]'`
+  echo "ROOT Collection PID: ${pid}"
+  root_coll_pid=${pid}
+  root_coll_url_raw=`irule "{EUDATGeteValPid(*pid, *key)}" "*pid=${root_coll_pid}%*key='URL'" ruleExecOut`
+  root_coll_url=`echo ${root_coll_url_raw} | cut -d '=' -f 2 | tr -d '[[:space:]]'`
+  echo "ROOT Collection URL: ${root_coll_url}"
+  new_collPath="${irods_home}/${testCollName}_new"
+
+  echo ""
+  echo "############ Moving collection ############"
+  imv ${collPath} ${new_collPath}
+
+  echo ""
+  echo "############ Updating collection PID ############"
+  http_url_raw=`irule "{getHttpApiParameters(*serverApireg, *serverApipub)}" null "*serverApireg"`
+  http_url=`echo ${http_url_raw} | cut -d '=' -f 2 | tr -d '[[:space:]]'`
+  newURL="${http_url}${new_collPath}"
+  irule "{EUDATeURLupdateColl(*pid, *newURL)}" "*pid=${pid}%*newURL=${newURL}" null
+  new_root_coll_url_raw=`irule "{EUDATGeteValPid(*pid, *key)}" "*pid=${root_coll_pid}%*key='URL'" ruleExecOut`
+  new_root_coll_url=`echo ${new_root_coll_url_raw} | cut -d '=' -f 2 | tr -d '[[:space:]]'`
+  echo "NEW ROOT Collection URL: ${new_root_coll_url}"
+  irule "{EUDATePIDremove(*path, *force)}" "*path=${new_collPath}%*force=true" null
+  irule "{EUDATePIDremove(*path, *force)}" "*path=${new_collPath}/${testFileName}%*force=true" null
+  irule "{EUDATePIDremove(*path, *force)}" "*path=${new_collPath}/${testCollNameSub}%*force=true" null
+  irule "{EUDATePIDremove(*path, *force)}" "*path=${new_collPath}/${testCollNameSub}/${testFileName}%*force=true" null
+  irm -rf ${new_collPath}
+}
+
 createPID
 if [ -n "$REMOTE_ZONE" ]; then
   replication
 fi
+moveCollection
 
 irule "{EUDATePIDremove(*path, *force)}" "*path=${sourcePath}%*force=true" null
 irm ${sourcePath}
+#irm -rf ${collPath}