Individual docker scripts ci changes

.gitlab/end_to_end.yml

@@ -96,28 +96,13 @@ end-to-end-core-setup:
     - cat $CI_DATAFED_CORE_PUB_KEY > /shared/keys/datafed-core-key.pub
     - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/keys/datafed-core-key.priv
     - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
-    - echo "#!/bin/bash" > run_core.sh
-    - echo "docker run -d \\" >> run_core.sh
-    - echo "--name \"core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_core.sh
-    - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_core.sh
-    - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_core.sh
-    - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_core.sh
-    - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_core.sh
-    - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_core.sh
-    - echo "-e DATAFED_DATABASE_PASSWORD=\"$CI_DATAFED_DATABASE_PASSWORD\" \\" >> run_core.sh
-    - echo "-e DATAFED_DATABASE_IP_ADDRESS_PORT=\"$CI_DATAFED_DATABASE_IP_ADDRESS_PORT\" \\" >> run_core.sh
-    - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_core.sh
-    - echo "-e UID=\"$USER_ID\" \\" >> run_core.sh
-    - echo "-p 7513:7513 \\" >> run_core.sh
-    - echo "-p 7512:7512 \\" >> run_core.sh
-    - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_core.sh
-    - echo "-v \"/shared/keys/datafed-core-key.pub\":/opt/datafed/keys/datafed-core-key.pub \\" >> run_core.sh
-    - echo "-v \"/shared/keys/datafed-core-key.priv\":/opt/datafed/keys/datafed-core-key.priv \\" >> run_core.sh
-    - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest\"" >> run_core.sh
-    - chmod +x run_core.sh
-    - ./run_core.sh
+    - ./scripts/generate_metadata_container_scripts.sh --core-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --core-name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --install-path "$(pwd)"
+    - ./scripts/create_datafed_network.sh
+    - ./scripts/run_core_container.sh
     - sleep 10
     - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" 
+    - ./scripts/remove_core_container.sh
+    - ./scripts/remove_datafed_network.sh
 
 end-to-end-ws-setup:
   variables:
@@ -153,29 +138,13 @@ end-to-end-ws-setup:
     - env
     - ./scripts/container_stop.sh -n "ws-" -p 
     - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
-    - echo "#!/bin/bash" > run_web.sh
-    - echo "docker run -d \\" >> run_web.sh
-    - echo "--name \"ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_web.sh
-    - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_web.sh
-    - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_web.sh
-    - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_web.sh
-    - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_web.sh
-    - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_web.sh
-    - echo "-e DATAFED_WEB_CERT_PATH=\"/opt/datafed/keys/${DATAFED_WEB_CERT_NAME}\" \\" >> run_web.sh
-    - echo "-e DATAFED_WEB_KEY_PATH=\"/opt/datafed/keys/${DATAFED_WEB_KEY_NAME}\" \\" >> run_web.sh
-    - echo "-e DATAFED_DEFAULT_LOG_PATH=\"${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh
-    - echo "-e DATAFED_CORE_ADDRESS_PORT_INTERNAL=\"$CORE_ADDRESS:7513\" \\" >> run_web.sh
-    - echo "-e UID=\"$USER_ID\" \\" >> run_web.sh
-    - echo "-p 443:443 \\" >> run_web.sh
-    - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh
-    - echo "-v \"/shared/keys/datafed-core-key.pub:/opt/datafed/keys/datafed-core-key.pub\" \\" >> run_web.sh
-    - echo "-v \"${DATAFED_WEB_CERT_PATH}:/opt/datafed/keys/${DATAFED_WEB_CERT_NAME}\" \\" >> run_web.sh
-    - echo "-v \"${DATAFED_WEB_KEY_PATH}:/opt/datafed/keys/${DATAFED_WEB_KEY_NAME}\" \\" >> run_web.sh
-    - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest\" " >> run_web.sh
-    - chmod +x run_web.sh
-    - ./run_web.sh
+    - ./scripts/generate_metadata_container_scripts.sh --web-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --web-name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --install-path "$(pwd)"
+    - ./scripts/create_datafed_network.sh
+    - ./scripts/run_web_container.sh
     - sleep 30
     - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" 
+    - ./scripts/remove_core_container.sh
+    - ./scripts/remove_datafed_network.sh
 
 # Repo server currently will crash on startup if it cannot connect to the core
 # server.
@@ -208,28 +177,8 @@ end-to-end-repo-setup:
     - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
     - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub
     - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv
-    - echo "#!/bin/bash" > "${RUN_FILE}"
-    - echo "docker run -d \\" >> "${RUN_FILE}"
-    - echo "--name \"repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> "${RUN_FILE}"
-    - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> "${RUN_FILE}"
-    - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> "${RUN_FILE}"
-    - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> "${RUN_FILE}"
-    - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> "${RUN_FILE}"
-    - echo "-e DATAFED_HTTPS_SERVER_PORT=\"443\" \\" >> "${RUN_FILE}"
-    - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> "${RUN_FILE}"
-    - echo "-e DATAFED_CORE_ADDRESS_PORT_INTERNAL=\"${CI_DATAFED_DOMAIN}:7513\" \\" >> "${RUN_FILE}"
-    - echo "-e DATAFED_REPO_ID_AND_DIR=\"${CI_DATAFED_REPO_ID_AND_DIR}\" \\" >> "${RUN_FILE}"
-    - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> "${RUN_FILE}"
-    - echo "-e DATAFED_GCS_COLLECTION_BASE_PATH=\"/mnt\" \\" >> "${RUN_FILE}"
-    - echo "-e UID=\"$USER_ID\" \\" >> "${RUN_FILE}"
-    - echo "-p 9000:9000 \\" >> "${RUN_FILE}"
-    - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> "${RUN_FILE}"
-    - echo "-v \"${DATAFED_HOST_COLLECTION_MOUNT}:/mnt/datafed\" \\" >> "${RUN_FILE}"
-    - echo "-v \"/shared/datafed-repo-key.pub\":/opt/datafed/keys/datafed-repo-key.pub \\" >> "${RUN_FILE}"
-    - echo "-v \"/shared/datafed-repo-key.priv\":/opt/datafed/keys/datafed-repo-key.priv \\" >> "${RUN_FILE}"
-    - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest\"" >> "${RUN_FILE}"
-    - chmod +x "${RUN_FILE}"
-    - ./${RUN_FILE}
+    - ./scripts/generate_repo_container_scripts.sh --repo-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --repo-name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" --install-path "$(pwd)"
+    - ./scripts/run_repo_container.sh
     - sleep 10
     - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" 
 
@@ -270,39 +219,8 @@ end-to-end-gcs-authz-setup:
     - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1")
     - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub
     - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv
-    - echo "#!/bin/bash" > run_globus.sh
-    - echo "docker run -d \\" >> run_globus.sh
-    - echo "--name \"gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_globus.sh
-    - echo "--network host \\" >> run_globus.sh
-    - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_globus.sh
-    - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_globus.sh
-    - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_globus.sh
-    - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_globus.sh
-    - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_globus.sh
-    - echo "-e DATAFED_HTTPS_SERVER_PORT=\"443\" \\" >> run_globus.sh
-    - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_globus.sh
-    - echo "-e DATAFED_CORE_ADDRESS_PORT_INTERNAL=\"${CI_DATAFED_DOMAIN}:7513\" \\" >> run_globus.sh
-    - echo "-e DATAFED_GCS_ROOT_NAME=\"${CI_DATAFED_GCS_ROOT_NAME}\" \\" >> run_globus.sh
-    - echo "-e DATAFED_GCS_COLLECTION_BASE_PATH=\"/mnt\" \\" >> run_globus.sh
-    - echo "-e DATAFED_GCS_COLLECTION_ROOT_PATH=\"/mnt/datafed\" \\" >> run_globus.sh
-    - echo "-e DATAFED_GLOBUS_SUBSCRIPTION=\"${CI_DATAFED_GLOBUS_SUBSCRIPTION}\" \\" >> run_globus.sh
-    - echo "-e DATAFED_GLOBUS_CONTROL_PORT=\"443\" \\" >> run_globus.sh
-    - echo "-e DATAFED_REPO_USER=\"datafed\" \\" >> run_globus.sh
-    - echo "-e DATAFED_AUTHZ_USER=\"datafed\" \\" >> run_globus.sh
-    - echo "-e UID=\"$USER_ID\" \\" >> run_globus.sh
-    - echo "-e BUILD_WITH_METADATA_SERVICES=\"FALSE\" \\" >> run_globus.sh
-    - echo "-e DATAFED_REPO_ID_AND_DIR=\"${CI_DATAFED_REPO_ID_AND_DIR}\" \\" >> run_globus.sh
-    - echo "-e DATAFED_GCS_IP=\"${CI_DATAFED_GCS_IP}\" \\" >> run_globus.sh
-    - echo "-e DATAFED_REPO_DOMAIN=\"${CI_DATAFED_REPO_DOMAIN}\" \\" >> run_globus.sh
-    - echo "-v \"${DATAFED_GLOBUS_DIR}:/opt/datafed/globus\" \\" >> run_globus.sh
-    - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_globus.sh
-    - echo "-v \"${DATAFED_HOST_COLLECTION_MOUNT}:/mnt/datafed\" \\" >> run_globus.sh
-    - echo "-v \"/shared/datafed-repo-key.pub\":/opt/datafed/keys/datafed-repo-key.pub \\" >> run_globus.sh
-    - echo "-v \"/shared/datafed-repo-key.priv\":/opt/datafed/keys/datafed-repo-key.priv \\" >> run_globus.sh
-    - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest\"" >> run_globus.sh
-    - if [ -f "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" ]; then rm "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"; fi
-    - chmod +x run_globus.sh
-    - ./run_globus.sh
+    - ./scripts/generate_repo_container_scripts.sh --gcs-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --gcs-name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" --ip-address "$CI_DATAFED_GCS_IP" --repo-domain "$CI_DATAFED_REPO_DOMAIN"--install-path "$(pwd)"
+    - ./scripts/run_gcs_container.sh
     - while [ ! -f "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" ]; do echo "Waiting for ${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"; sleep 10; done
     - cat "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"
     - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest"

scripts/generate_datafed.sh

@@ -58,6 +58,14 @@ else
   local_DATAFED_DATABASE_HOST=$(printenv DATAFED_DATABASE_HOST)
 fi
 
+local_DATAFED_DATABASE_PORT=""
+if [[ -z "$DATAFED_DATABASE_PORT" ]]
+then
+  # Empty
+  local_DATAFED_DATABASE_PORT="8529"
+else
+  local_DATAFED_DATABASE_PORT=$(printenv DATAFED_DATABASE_PORT)
+fi
 
 local_DATAFED_ZEROMQ_SESSION_SECRET=""
 if [ -z "${DATAFED_ZEROMQ_SESSION_SECRET}" ]
@@ -275,6 +283,8 @@ export DATAFED_DATABASE_PASSWORD="$local_DATAFED_DATABASE_PASSWORD"
 # Host of the metadata database, can be a domain name
 # or an IP address.
 export DATAFED_DATABASE_HOST="$local_DATAFED_DATABASE_HOST"
+# i.e. 8529 - Arango port
+export DATAFED_DATABASE_PORT="$local_DATAFED_DATABASE_PORT"
 # The user account the datafed core application will run under
 export DATAFED_CORE_USER="$local_DATAFED_CORE_USER"