Script and instructions to access ACES data using the Globus Command Line Interface (CLI)

# Basic setup for Globus
# One needs two pieces of software: 
# 1) Install https://www.globus.org/globus-connect-personal
# In my case there was a problem with tcllib missing but that's only needed for GUI
# 2) # Install the Globus Command Line Interface (CLI), which is a Python package
# https://docs.globus.org/cli/
# Also, you need a globus account and crate your personal endpoint (pointing toward your machine)
# That can be done logging into the Globus webpage or in the shell setup: 
# > globusconnectpersonal -setup --no-gui


# Once the setup is done: 
# 1) globus-connect-personal needs to be running in your terminal 
# In this case the executable is already in my path
# > globusconnectpersonal -start &

# Issue 1: Why do I get "Permission Denied" or "Path not allowed" errors on my endpoint?
# Globus by default only writes in your inside your /home, to override, see:  
# https://docs.globus.org/faq/globus-connect-endpoints/#why_do_i_get_permission_denied_or_path_not_allowed_errors_on_my_endpoint
# https://github.com/Supercomputing/DailyTasks/wiki/Using-Globus-Online-from-the-command-line
# In my case, to be able to write in my institutional massive space, I added the line: 
# /path,0,1
# to .globusonline/lta/config-paths


# Activate personal (destination) endpoint 
# dest_ep can be obtained from the Globus webpage in: Endpoints -> Administered by you
dest_ep=aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee
globus endpoint activate $dest_ep

#ALMA-IMF Parent Directory
source_ep=uuuuuuuu-vvvv-wwww-xxxx-yyyyyyyyyyyy

# Save path
save_path="/home/user/ALMA_IMF/lines/data/"

# File with download list (see comments at the end)
# In this case download all ms files for: G338, B3, spw0, contsub (to image N2H+)
downl_files=$save_path"contsub_list_G338.txt"


# List of files to download
#echo $downl_files

# Loop to download 
while var= read -r line; do
  case "$line" in \#*) continue ;; esac
  echo "Downloading:"
  echo $source_ep":scigoals/"$line
  echo "To:"
  echo $dest_ep":"$save_path$line
  globus transfer --recursive $source_ep":scigoals/"$line $dest_ep":"$save_path$line
done < $downl_files
 
# Below are a few examples to search within an endpoint. It can be slow or even crash if 
# too many files or levels exist 

#globus ls $source_ep':scigoals/science_goal.uid___A001_X1296_X105/group.uid___A001_X1296_X106/member.uid___A001_X1296_X107/calibrated/' --filter '~*spw*.split.contsub'

#globus ls $source_ep':scigoals/' --recursive --recursive-depth-limit=5 --filter '~*spw*.split.contsub'

#globus ls $source_ep':scigoals/science_goal.uid___A001_X1296_X105/group.uid___A001_X1296_X106/member.uid___A001_X1296_X107/calibrated/uid___A002_Xc75eba_X4f72_W51-E_B3_spw0.split.contsub/' --recursive

#globus ls $source_ep':scigoals/' --filter '~*.txt'

#Example of a transfer of a single file (non recursive)
#globus transfer $source_ep":scigoals/contsub_list.txt" $dest_ep":"$save_path"contsub_list.txt"