From 7b7fed8c0f94b73d0834fd26e5e193b138064fa9 Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Thu, 1 Aug 2024 09:44:36 -0400 Subject: [PATCH 01/15] added initial versions of scripts to support generating docker scripts --- scripts/generate_datafed.sh | 10 + .../generate_metadata_container_scripts.sh | 213 +++++++++++++++++ scripts/generate_repo_container_scripts.sh | 223 ++++++++++++++++++ 3 files changed, 446 insertions(+) create mode 100755 scripts/generate_metadata_container_scripts.sh create mode 100755 scripts/generate_repo_container_scripts.sh diff --git a/scripts/generate_datafed.sh b/scripts/generate_datafed.sh index ce0e398d2..fdd26b1e5 100755 --- a/scripts/generate_datafed.sh +++ b/scripts/generate_datafed.sh @@ -58,6 +58,14 @@ else local_DATAFED_DATABASE_HOST=$(printenv DATAFED_DATABASE_HOST) fi +local_DATAFED_DATABASE_PORT="" +if [[ -z "$DATAFED_DATABASE_PORT" ]] +then + # Empty + local_DATAFED_DATABASE_PORT="8529" +else + local_DATAFED_DATABASE_PORT=$(printenv DATAFED_DATABASE_PORT) +fi local_DATAFED_ZEROMQ_SESSION_SECRET="" if [ -z "${DATAFED_ZEROMQ_SESSION_SECRET}" ] @@ -275,6 +283,8 @@ export DATAFED_DATABASE_PASSWORD="$local_DATAFED_DATABASE_PASSWORD" # Host of the metadata database, can be a domain name # or an IP address. export DATAFED_DATABASE_HOST="$local_DATAFED_DATABASE_HOST" +# i.e. 8529 - Arango port +export DATAFED_DATABASE_PORT="$local_DATAFED_DATABASE_PORT" # The user account the datafed core application will run under export DATAFED_CORE_USER="$local_DATAFED_CORE_USER" diff --git a/scripts/generate_metadata_container_scripts.sh b/scripts/generate_metadata_container_scripts.sh new file mode 100755 index 000000000..cffe66574 --- /dev/null +++ b/scripts/generate_metadata_container_scripts.sh @@ -0,0 +1,213 @@ +#!/bin/bash + +# Cannot run with -u because we check for unbound variables +# and the script will exit prematurely if '-u' is set +set -ef -o pipefail + +SCRIPT=$(realpath "$0") +FILE_NAME=$(basename "${SCRIPT}") +SOURCE=$(dirname "$SCRIPT") +PROJECT_ROOT=$(realpath "${SOURCE}/..") +source "${PROJECT_ROOT}/config/datafed.sh" + +VERSION="1.0.0" +echo "$FILE_NAME $VERSION" + +Help() +{ + echo "$(basename $0) Will set up the docker container scripts for the metadata server" + echo + echo "Syntax: $(basename $0) [-h|u|p|o]" + echo "options:" + echo "-h, --help Print this help message." + echo "-t, --docker-tag The tag on Savannah that the currently released containers are under" + echo +} + +local_DOCKER_TAG="" + +if [ -z "${DATAFED_DOCKER_TAG}" ] +then + local_DOCKER_TAG="" +else + local_DOCKER_TAG=$(printenv DATAFED_DOCKER_TAG) +fi + +VALID_ARGS=$(getopt -o ht: --long 'help',docker-tag: -- "$@") +if [[ $? -ne 0 ]]; then + exit 1; +fi +eval set -- "$VALID_ARGS" +while [ : ]; do + case "$1" in + -h | --help) + Help + exit 0 + ;; + -t | --docker-tag) + local_DOCKER_TAG=$2 + shift 2 + ;; + --) shift; + break + ;; + \?) # incorrect option + echo "Error: Invalid option" + exit;; + esac +done + +ERROR_DETECTED=0 +if [ -z "$local_DOCKER_TAG" ] +then + echo "Error DOCKER_TAG is not defined, this is a required argument" + echo " This variable can be set using the command line option -t, --docker-tag" + echo " or with the environment variable DATAFED_DOCKER_TAG." + ERROR_DETECTED=1 +fi + +if [ "$ERROR_DETECTED" == "1" ] +then + exit 1 +fi + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh" +#!/bin/bash + +docker network create datafed-network +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_core_container.sh" +#!/bin/bash + +CONFIG_FILE_PATH="\$DATAFED_INSTALL_PATH/config/datafed.sh" +source "\${CONFIG_FILE_PATH}" + +USER_ID=$(id -u) + +docker run -d \\ + --restart=always \\ + --name datafed-core_$local_DOCKER_TAG \\ + -e DATAFED_GLOBUS_APP_SECRET="\$DATAFED_GLOBUS_APP_SECRET" \\ + -e DATAFED_GLOBUS_APP_ID="\$DATAFED_GLOBUS_APP_ID" \\ + -e DATAFED_ZEROMQ_SESSION_SECRET="\$DATAFED_ZEROMQ_SESSION_SECRET" \\ + -e DATAFED_ZEROMQ_SYSTEM_SECRET="\$DATAFED_ZEROMQ_SYSTEM_SECRET" \\ + -e DATAFED_DOMAIN="\$DATAFED_DOMAIN" \\ + -e DATAFED_DATABASE_PASSWORD="\$DATAFED_DATABASE_PASSWORD" \\ + -e DATAFED_DATABASE_IP_ADDRESS_PORT="http://\$DATAFED_DATABASE_HOST:\$DATAFED_DATABASE_PORT" \\ + -e DATAFED_DEFAULT_LOG_PATH="/datafed/logs" \\ + -e DATAFED_CORE_ADDRESS_PORT_INTERNAL="\$DATAFED_CORE_ADDRESS_PORT_INTERNAL" \\ + -e UID="\$USER_ID" \\ + --network datafed-network \\ + -p 7513:7513 \\ + -p 7512:7512 \\ + -v "\$DATAFED_INSTALL_PATH/logs:/datafed/logs" \\ + -v "\$DATAFED_INSTALL_PATH/keys/datafed-core-key.pub:/opt/datafed/keys/datafed-core-key.pub" \\ + -v "\$DATAFED_INSTALL_PATH/keys/datafed-core-key.priv:/opt/datafed/keys/datafed-core-key.priv" \\ + -t "datafed-core-prod:$local_DOCKER_TAG" +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_core_container.sh" +#!/bin/bash + +docker container stop datafed-core_$local_DOCKER_TAG +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_core_container.sh" +#!/bin/bash + +docker container stop datafed-core_$local_DOCKER_TAG +docker container rm datafed-core_$local_DOCKER_TAG +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_web_container.sh" +#!/bin/bash + +CONFIG_FILE_PATH="\$DATAFED_INSTALL_PATH/config/datafed.sh" +source "\$CONFIG_FILE_PATH" + +USER_ID=$(id -u) + +docker run -d \\ + --restart=always \\ + --name datafed-web_$local_DOCKER_TAG \\ + -e DATAFED_GLOBUS_APP_SECRET="\$DATAFED_GLOBUS_APP_SECRET" \\ + -e DATAFED_GLOBUS_APP_ID="\$DATAFED_GLOBUS_APP_ID" \\ + -e DATAFED_ZEROMQ_SESSION_SECRET="\$DATAFED_ZEROMQ_SESSION_SECRET" \\ + -e DATAFED_ZEROMQ_SYSTEM_SECRET="\$DATAFED_ZEROMQ_SYSTEM_SECRET" \\ + -e DATAFED_DOMAIN="\$DATAFED_DOMAIN" \\ + -e DATAFED_WEB_CERT_PATH="\$DATAFED_WEB_CERT_PATH" \\ + -e DATAFED_WEB_KEY_PATH="\$DATAFED_WEB_KEY_PATH" \\ + -e DATAFED_DEFAULT_LOG_PATH="/datafed/logs" \\ + -e DATAFED_CORE_ADDRESS_PORT_INTERNAL="\$DATAFED_CORE_ADDRESS_PORT_INTERNAL" \\ + -e DATAFED_GOOGLE_ANALYTICS_TAG="\$DATAFED_GOOGLE_ANALYTICS_TAG" \\ + -e UID="\$USER_ID" \\ + --network datafed-network \\ + -v "\$DATAFED_INSTALL_PATH/logs:/datafed/logs" \\ + -v "\$DATAFED_INSTALL_PATH/keys/datafed-core-key.pub:/opt/datafed/keys/datafed-core-key.pub" \\ + -v "\$DATAFED_WEB_CERT_PATH:\$DATAFED_WEB_CERT_PATH" \\ + -v "\$DATAFED_WEB_KEY_PATH:\$DATAFED_WEB_KEY_PATH" \\ + -t "datafed-web-prod:$local_DOCKER_TAG" +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_web_container.sh" +#!/bin/bash + +docker container stop datafed-web_$local_DOCKER_TAG +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_web_container.sh" +#!/bin/bash + +docker container stop datafed-web_$local_DOCKER_TAG +docker container rm datafed-web_$local_DOCKER_TAG +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_nginx_container.sh" +#!/bin/bash + +CONFIG_FILE_PATH="\$DATAFED_INSTALL_PATH/config/datafed.sh" +source "\$CONFIG_FILE_PATH" + +USER_ID=$(id -u) + +docker run -d \ + --restart=always \ + --name datafed-nginx \ + --network datafed-network \ + -p 443:443 \ + -p 80:80 \ + -v "\$DATAFED_INSTALL_PATH/nginx/nginx.conf:/etc/nginx/conf.d/default.conf" \ + -v "\$DATAFED_INSTALL_PATH/nginx/sites-enabled:/etc/nginx/sites-enabled" \ + -v "\$DATAFED_INSTALL_PATH/nginx/www:/www" \ + -v "\$DATAFED_INSTALL_PATH/keys/datafed.ornl.gov.crt:/etc/nginx/certs/datafed.ornl.gov.crt" \ + -v "\$DATAFED_INSTALL_PATH/keys/datafed.ornl.gov.key:/etc/nginx/certs/datafed.ornl.gov.key" \ + nginx:latest +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_nginx_container.sh" +#!/bin/bash + +docker container stop datafed-nginx +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_nginx_container.sh" +#!/bin/bash + +docker container stop datafed-nginx +docker container rm datafed-nginx +EOF + +chmod +x "$DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh" + +chmod +x "$DATAFED_INSTALL_PATH/scripts/run_core_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_core_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_core_container.sh" + +chmod +x "$DATAFED_INSTALL_PATH/scripts/run_web_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_web_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_web_container.sh" + +chmod +x "$DATAFED_INSTALL_PATH/scripts/run_nginx_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_nginx_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_nginx_container.sh" diff --git a/scripts/generate_repo_container_scripts.sh b/scripts/generate_repo_container_scripts.sh new file mode 100755 index 000000000..733b58920 --- /dev/null +++ b/scripts/generate_repo_container_scripts.sh @@ -0,0 +1,223 @@ +#!/bin/bash + +# Cannot run with -u because we check for unbound variables +# and the script will exit prematurely if '-u' is set +set -ef -o pipefail + +SCRIPT=$(realpath "$0") +FILE_NAME=$(basename "${SCRIPT}") +SOURCE=$(dirname "$SCRIPT") +PROJECT_ROOT=$(realpath "${SOURCE}/..") +source "${PROJECT_ROOT}/config/datafed.sh" + +VERSION="1.0.0" +echo "$FILE_NAME $VERSION" + +Help() +{ + echo "$(basename $0) Will set up the docker container scripts for the repository server" + echo + echo "Syntax: $(basename $0) [-h|t|m|i|d]" + echo "options:" + echo "-h, --help Print this help message." + echo "-t, --docker-tag The tag on Savannah that the currently released containers are under" + echo "-m, --host-collection-mount The path on the host that the container volume will mount to" + echo "-i, --ip-address The public ip address of the host that the GCS container is running on" + echo "-d, --repo-domain The publicly accessible domain of the server that the host is running on" + echo +} + +local_DOCKER_TAG="" +local_HOST_COLLECTION_MOUNT="" +local_IP_ADDRESS="" +local_REPO_DOMAIN="" + +if [ -z "${DATAFED_DOCKER_TAG}" ] +then + local_DOCKER_TAG="" +else + local_DOCKER_TAG=$(printenv DATAFED_DOCKER_TAG) +fi + +if [ -z "${DATAFED_HOST_COLLECTION_MOUNT}" ] +then + local_HOST_COLLECTION_MOUNT="" +else + local_HOST_COLLECTION_MOUNT=$(printenv DATAFED_HOST_COLLECTION_MOUNT) +fi + +VALID_ARGS=$(getopt -o ht:m:i:d: --long 'help',docker-tag:,host-collection-mount:,ip-address:,repo-domain: -- "$@") +if [[ $? -ne 0 ]]; then + exit 1; +fi +eval set -- "$VALID_ARGS" +while [ : ]; do + case "$1" in + -h | --help) + Help + exit 0 + ;; + -t | --docker-tag) + local_DOCKER_TAG=$2 + shift 2 + ;; + -m | --host-collection-mount) + local_HOST_COLLECTION_MOUNT=$2 + shift 2 + ;; + -i | --ip-address) + local_IP_ADDRESS=$2 + shift 2 + ;; + -d | --repo-domain) + local_REPO_DOMAIN=$2 + shift 2 + ;; + --) shift; + break + ;; + \?) # incorrect option + echo "Error: Invalid option" + exit;; + esac +done + +ERROR_DETECTED=0 +if [ -z "$local_DOCKER_TAG" ] +then + echo "Error DOCKER_TAG is not defined, this is a required argument" + echo " This variable can be set using the command line option -t, --docker-tag" + echo " or with the environment variable DATAFED_DOCKER_TAG." + ERROR_DETECTED=1 +fi + +if [ -z "$local_HOST_COLLECTION_MOUNT" ] +then + echo "Error HOST_COLLECTION_MOUNT is not defined, this is a required argument" + echo " This variable can be set using the command line option -m, --host-collection-mount" + echo " or with the environment variable DATAFED_HOST_COLLECTION_MOUNT." + ERROR_DETECTED=1 +fi + +if [ -z "$local_IP_ADDRESS" ] +then + echo "Error IP_ADDRESS is not defined, this is a required argument" + echo " This variable can be set using the command line option -i, --ip-address" + ERROR_DETECTED=1 +fi + +if [ -z "$local_REPO_DOMAIN" ] +then + echo "Error REPO_DOMAIN is not defined, this is a required argument" + echo " This variable can be set using the command line option -d, --repo-domain" + ERROR_DETECTED=1 +fi + +if [ "$ERROR_DETECTED" == "1" ] +then + exit 1 +fi + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_repo_container.sh" +#!/bin/bash + +CONFIG_FILE_PATH="\$DATAFED_INSTALL_PATH/config/datafed.sh" +source "\${CONFIG_FILE_PATH}" + +USER_ID=$(id -u) + +docker run -d \\ + --restart=always \\ + --name "datafed-repo_$local_DOCKER_TAG" \\ + -e DATAFED_GLOBUS_APP_SECRET="\$DATAFED_GLOBUS_APP_SECRET" \\ + -e DATAFED_GLOBUS_APP_ID="\$DATAFED_GLOBUS_APP_ID" \\ + -e DATAFED_ZEROMQ_SESSION_SECRET="\$DATAFED_ZEROMQ_SESSION_SECRET" \\ + -e DATAFED_ZEROMQ_SYSTEM_SECRET="\$DATAFED_ZEROMQ_SYSTEM_SECRET" \\ + -e DATAFED_HTTPS_SERVER_PORT="443" \\ + -e DATAFED_DOMAIN="\$DATAFED_DOMAIN" \\ + -e DATAFED_CORE_ADDRESS_PORT_INTERNAL="\$DATAFED_DOMAIN:7513" \\ + -e DATAFED_DEFAULT_LOG_PATH="\$DATAFED_DEFAULT_LOG_PATH" \\ + -e DATAFED_GCS_BASE_PATH="/mnt" \\ + -e DATAFED_GCS_COLLECTION_ROOT_PATH="\$DATAFED_GCS_COLLECTION_ROOT_PATH" \\ + -e UID="\$USER_ID" \\ + -p 9000:9000 \\ + -v "\$DATAFED_INSTALL_PATH/logs:/datafed/logs" \\ + -v "\$DATAFED_INSTALL_PATH/keys/datafed-repo-key.pub:/opt/datafed/keys/datafed-repo-key.pub" \\ + -v "\$DATAFED_INSTALL_PATH/keys/datafed-repo-key.priv:/opt/datafed/keys/datafed-repo-key.priv" \\ + -v $local_HOST_COLLECTION_MOUNT:\$DATAFED_GCS_COLLECTION_ROOT_PATH/\$DATAFED_REPO_ID_AND_DIR + -t "datafed-repo-prod:$local_DOCKER_TAG" +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_repo_container.sh" +#!/bin/bash + +docker container stop datafed-repo_$local_DOCKER_TAG +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_repo_container.sh" +#!/bin/bash + +docker container stop datafed-repo_$local_DOCKER_TAG +docker container rm datafed-repo_$local_DOCKER_TAG +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_gcs_container.sh" +#!/bin/bash + +CONFIG_FILE_PATH="\$DATAFED_INSTALL_PATH/config/datafed.sh" +source "\${CONFIG_FILE_PATH}" + +USER_ID=$(id -u) + +docker run -d \\ + --restart=always \\ + --name "datafed-gcs_$local_DOCKER_TAG" \\ + -e DATAFED_GLOBUS_APP_SECRET="\$DATAFED_GLOBUS_APP_SECRET" \\ + -e DATAFED_GLOBUS_APP_ID="\$DATAFED_GLOBUS_APP_ID" \\ + -e DATAFED_ZEROMQ_SESSION_SECRET="\$DATAFED_ZEROMQ_SESSION_SECRET" \\ + -e DATAFED_ZEROMQ_SYSTEM_SECRET="\$DATAFED_ZEROMQ_SYSTEM_SECRET" \\ + -e DATAFED_HTTPS_SERVER_PORT="443" \\ + -e DATAFED_DOMAIN="\$DATAFED_DOMAIN" \\ + -e DATAFED_CORE_ADDRESS_PORT_INTERNAL="\$DATAFED_DOMAIN:7513" \\ + -e DATAFED_DEFAULT_LOG_PATH="\$DATAFED_DEFAULT_LOG_PATH" \\ + -e DATAFED_GCS_BASE_PATH="/mnt" \\ + -e DATAFED_GCS_COLLECTION_ROOT_PATH="\$DATAFED_GCS_COLLECTION_ROOT_PATH" \\ + -e DATAFED_GCS_ROOT_NAME="\$DATAFED_GCS_ROOT_NAME" \\ + -e DATAFED_GLOBUS_SUBSCRIPTION="\$DATAFED_GLOBUS_SUBSCRIPTION" \\ + -e DATAFED_GLOBUS_CONTROL_PORT="\$DATAFED_GLOBUS_CONTROL_PORT" \\ + -e DATAFED_REPO_USER="datafed" \\ + -e DATAFED_AUTHZ_USER="datafed" \\ + -e BUILD_WITH_METADATA_SERVICES="FALSE" \\ + -e DATAFED_REPO_ID_AND_DIR="\$DATAFED_REPO_ID_AND_DIR" \\ + -e DATAFED_GCS_IP="$local_IP_ADDRESS" \\ + -e DATAFED_REPO_DOMAIN="$local_REPO_DOMAIN" \\ + -e UID="\$USER_ID" \\ + --network=host \\ + -v "\$DATAFED_INSTALL_PATH/logs:/datafed/logs" \\ + -v "\$DATAFED_INSTALL_PATH/globus:/opt/datafed/globus" \\ + -v "\$DATAFED_INSTALL_PATH/keys/datafed-repo-key.pub:/opt/datafed/keys/datafed-repo-key.pub" \\ + -v "\$DATAFED_INSTALL_PATH/keys/datafed-repo-key.priv:/opt/datafed/keys/datafed-repo-key.priv" \\ + -v $local_HOST_COLLECTION_MOUNT:\$DATAFED_GCS_COLLECTION_ROOT_PATH/\$DATAFED_REPO_ID_AND_DIR + -t "datafed-gcs-prod:$local_DOCKER_TAG" +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_gcs_container.sh" +#!/bin/bash + +docker container stop datafed-gcs_$local_DOCKER_TAG +EOF + +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_gcs_container.sh" +#!/bin/bash + +docker container stop datafed-gcs_$local_DOCKER_TAG +docker container rm datafed-gcs_$local_DOCKER_TAG +EOF + +chmod +x "$DATAFED_INSTALL_PATH/scripts/run_repo_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_repo_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_repo_container.sh" + +chmod +x "$DATAFED_INSTALL_PATH/scripts/run_gcs_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_gcs_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_gcs_container.sh" From 3a7e525a9b474688d5c230b170537990f562509b Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Tue, 3 Sep 2024 03:23:49 -0400 Subject: [PATCH 02/15] included changes to container scripts --- .gitlab/end_to_end.yml | 104 ++---------------- .../generate_metadata_container_scripts.sh | 58 ++++++++-- scripts/generate_repo_container_scripts.sh | 56 ++++++++-- 3 files changed, 99 insertions(+), 119 deletions(-) diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml index 95df3ca26..a53857abb 100644 --- a/.gitlab/end_to_end.yml +++ b/.gitlab/end_to_end.yml @@ -96,26 +96,8 @@ end-to-end-core-setup: - cat $CI_DATAFED_CORE_PUB_KEY > /shared/keys/datafed-core-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/keys/datafed-core-key.priv - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - echo "#!/bin/bash" > run_core.sh - - echo "docker run -d \\" >> run_core.sh - - echo "--name \"core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_core.sh - - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_core.sh - - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_core.sh - - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_core.sh - - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_core.sh - - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_core.sh - - echo "-e DATAFED_DATABASE_PASSWORD=\"$CI_DATAFED_DATABASE_PASSWORD\" \\" >> run_core.sh - - echo "-e DATAFED_DATABASE_IP_ADDRESS_PORT=\"$CI_DATAFED_DATABASE_IP_ADDRESS_PORT\" \\" >> run_core.sh - - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_core.sh - - echo "-e UID=\"$USER_ID\" \\" >> run_core.sh - - echo "-p 7513:7513 \\" >> run_core.sh - - echo "-p 7512:7512 \\" >> run_core.sh - - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_core.sh - - echo "-v \"/shared/keys/datafed-core-key.pub\":/opt/datafed/keys/datafed-core-key.pub \\" >> run_core.sh - - echo "-v \"/shared/keys/datafed-core-key.priv\":/opt/datafed/keys/datafed-core-key.priv \\" >> run_core.sh - - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest\"" >> run_core.sh - - chmod +x run_core.sh - - ./run_core.sh + - ./scripts/generate_metadata_container_scripts.sh --core-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --core-name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" + - $DATAFED_INSTALL_PATH/scripts/run_core_container.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" @@ -153,27 +135,8 @@ end-to-end-ws-setup: - env - ./scripts/container_stop.sh -n "ws-" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - echo "#!/bin/bash" > run_web.sh - - echo "docker run -d \\" >> run_web.sh - - echo "--name \"ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_web.sh - - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_web.sh - - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_web.sh - - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_web.sh - - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_web.sh - - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_web.sh - - echo "-e DATAFED_WEB_CERT_PATH=\"/opt/datafed/keys/${DATAFED_WEB_CERT_NAME}\" \\" >> run_web.sh - - echo "-e DATAFED_WEB_KEY_PATH=\"/opt/datafed/keys/${DATAFED_WEB_KEY_NAME}\" \\" >> run_web.sh - - echo "-e DATAFED_DEFAULT_LOG_PATH=\"${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh - - echo "-e DATAFED_CORE_ADDRESS_PORT_INTERNAL=\"$CORE_ADDRESS:7513\" \\" >> run_web.sh - - echo "-e UID=\"$USER_ID\" \\" >> run_web.sh - - echo "-p 443:443 \\" >> run_web.sh - - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_web.sh - - echo "-v \"/shared/keys/datafed-core-key.pub:/opt/datafed/keys/datafed-core-key.pub\" \\" >> run_web.sh - - echo "-v \"${DATAFED_WEB_CERT_PATH}:/opt/datafed/keys/${DATAFED_WEB_CERT_NAME}\" \\" >> run_web.sh - - echo "-v \"${DATAFED_WEB_KEY_PATH}:/opt/datafed/keys/${DATAFED_WEB_KEY_NAME}\" \\" >> run_web.sh - - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest\" " >> run_web.sh - - chmod +x run_web.sh - - ./run_web.sh + - ./scripts/generate_metadata_container_scripts.sh --web-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --web-name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" + - $DATAFED_INSTALL_PATH/scripts/run_web_container.sh - sleep 30 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" @@ -208,28 +171,8 @@ end-to-end-repo-setup: - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv - - echo "#!/bin/bash" > "${RUN_FILE}" - - echo "docker run -d \\" >> "${RUN_FILE}" - - echo "--name \"repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> "${RUN_FILE}" - - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> "${RUN_FILE}" - - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> "${RUN_FILE}" - - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> "${RUN_FILE}" - - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> "${RUN_FILE}" - - echo "-e DATAFED_HTTPS_SERVER_PORT=\"443\" \\" >> "${RUN_FILE}" - - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> "${RUN_FILE}" - - echo "-e DATAFED_CORE_ADDRESS_PORT_INTERNAL=\"${CI_DATAFED_DOMAIN}:7513\" \\" >> "${RUN_FILE}" - - echo "-e DATAFED_REPO_ID_AND_DIR=\"${CI_DATAFED_REPO_ID_AND_DIR}\" \\" >> "${RUN_FILE}" - - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> "${RUN_FILE}" - - echo "-e DATAFED_GCS_COLLECTION_BASE_PATH=\"/mnt\" \\" >> "${RUN_FILE}" - - echo "-e UID=\"$USER_ID\" \\" >> "${RUN_FILE}" - - echo "-p 9000:9000 \\" >> "${RUN_FILE}" - - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> "${RUN_FILE}" - - echo "-v \"${DATAFED_HOST_COLLECTION_MOUNT}:/mnt/datafed\" \\" >> "${RUN_FILE}" - - echo "-v \"/shared/datafed-repo-key.pub\":/opt/datafed/keys/datafed-repo-key.pub \\" >> "${RUN_FILE}" - - echo "-v \"/shared/datafed-repo-key.priv\":/opt/datafed/keys/datafed-repo-key.priv \\" >> "${RUN_FILE}" - - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest\"" >> "${RUN_FILE}" - - chmod +x "${RUN_FILE}" - - ./${RUN_FILE} + - ./scripts/generate_repo_container_scripts.sh --repo-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --repo-name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" + - $DATAFED_INSTALL_PATH/scripts/run_repo_container.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" @@ -270,39 +213,8 @@ end-to-end-gcs-authz-setup: - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv - - echo "#!/bin/bash" > run_globus.sh - - echo "docker run -d \\" >> run_globus.sh - - echo "--name \"gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}\" \\" >> run_globus.sh - - echo "--network host \\" >> run_globus.sh - - echo "-e DATAFED_GLOBUS_APP_SECRET=\"$CI_DATAFED_GLOBUS_APP_SECRET\" \\" >> run_globus.sh - - echo "-e DATAFED_GLOBUS_APP_ID=\"$CI_DATAFED_GLOBUS_APP_ID\" \\" >> run_globus.sh - - echo "-e DATAFED_ZEROMQ_SESSION_SECRET=\"$CI_DATAFED_ZEROMQ_SESSION_SECRET\" \\" >> run_globus.sh - - echo "-e DATAFED_ZEROMQ_SYSTEM_SECRET=\"$CI_DATAFED_ZEROMQ_SYSTEM_SECRET\" \\" >> run_globus.sh - - echo "-e DATAFED_DOMAIN=\"$CI_DATAFED_DOMAIN\" \\" >> run_globus.sh - - echo "-e DATAFED_HTTPS_SERVER_PORT=\"443\" \\" >> run_globus.sh - - echo "-e DATAFED_DEFAULT_LOG_PATH=\"$CONTAINER_LOG_FILE_PATH\" \\" >> run_globus.sh - - echo "-e DATAFED_CORE_ADDRESS_PORT_INTERNAL=\"${CI_DATAFED_DOMAIN}:7513\" \\" >> run_globus.sh - - echo "-e DATAFED_GCS_ROOT_NAME=\"${CI_DATAFED_GCS_ROOT_NAME}\" \\" >> run_globus.sh - - echo "-e DATAFED_GCS_COLLECTION_BASE_PATH=\"/mnt\" \\" >> run_globus.sh - - echo "-e DATAFED_GCS_COLLECTION_ROOT_PATH=\"/mnt/datafed\" \\" >> run_globus.sh - - echo "-e DATAFED_GLOBUS_SUBSCRIPTION=\"${CI_DATAFED_GLOBUS_SUBSCRIPTION}\" \\" >> run_globus.sh - - echo "-e DATAFED_GLOBUS_CONTROL_PORT=\"443\" \\" >> run_globus.sh - - echo "-e DATAFED_REPO_USER=\"datafed\" \\" >> run_globus.sh - - echo "-e DATAFED_AUTHZ_USER=\"datafed\" \\" >> run_globus.sh - - echo "-e UID=\"$USER_ID\" \\" >> run_globus.sh - - echo "-e BUILD_WITH_METADATA_SERVICES=\"FALSE\" \\" >> run_globus.sh - - echo "-e DATAFED_REPO_ID_AND_DIR=\"${CI_DATAFED_REPO_ID_AND_DIR}\" \\" >> run_globus.sh - - echo "-e DATAFED_GCS_IP=\"${CI_DATAFED_GCS_IP}\" \\" >> run_globus.sh - - echo "-e DATAFED_REPO_DOMAIN=\"${CI_DATAFED_REPO_DOMAIN}\" \\" >> run_globus.sh - - echo "-v \"${DATAFED_GLOBUS_DIR}:/opt/datafed/globus\" \\" >> run_globus.sh - - echo "-v \"${HOST_LOG_FILE_PATH}:${CONTAINER_LOG_FILE_PATH}\" \\" >> run_globus.sh - - echo "-v \"${DATAFED_HOST_COLLECTION_MOUNT}:/mnt/datafed\" \\" >> run_globus.sh - - echo "-v \"/shared/datafed-repo-key.pub\":/opt/datafed/keys/datafed-repo-key.pub \\" >> run_globus.sh - - echo "-v \"/shared/datafed-repo-key.priv\":/opt/datafed/keys/datafed-repo-key.priv \\" >> run_globus.sh - - echo "-t \"${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest\"" >> run_globus.sh - - if [ -f "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" ]; then rm "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"; fi - - chmod +x run_globus.sh - - ./run_globus.sh + - ./scripts/generate_repo_container_scripts.sh --gcs-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --gcs-name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" --ip-address "$CI_DATAFED_GCS_IP" --repo-domain "$CI_DATAFED_REPO_DOMAIN" + - $DATAFED_INSTALL_PATH/scripts/run_gcs_container.sh - while [ ! -f "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" ]; do echo "Waiting for ${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"; sleep 10; done - cat "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" diff --git a/scripts/generate_metadata_container_scripts.sh b/scripts/generate_metadata_container_scripts.sh index cffe66574..eee6c9ae5 100755 --- a/scripts/generate_metadata_container_scripts.sh +++ b/scripts/generate_metadata_container_scripts.sh @@ -20,11 +20,19 @@ Help() echo "Syntax: $(basename $0) [-h|u|p|o]" echo "options:" echo "-h, --help Print this help message." - echo "-t, --docker-tag The tag on Savannah that the currently released containers are under" + echo "-t, --docker-tag The tag on the selected registry that the currently released containers are under" + echo "-c, --core-name The name of the core container" + echo "-o, --core-image The name of the core image" + echo "-w, --web-name The name of the web container" + echo "-e, --web-image The name of the web image" echo } local_DOCKER_TAG="" +local_CORE_NAME="" +local_WEB_NAME="" +local_CORE_IMAGE="datafed-core-prod" +local_WEB_IMAGE="datafed-web-prod" if [ -z "${DATAFED_DOCKER_TAG}" ] then @@ -33,7 +41,7 @@ else local_DOCKER_TAG=$(printenv DATAFED_DOCKER_TAG) fi -VALID_ARGS=$(getopt -o ht: --long 'help',docker-tag: -- "$@") +VALID_ARGS=$(getopt -o ht:c:n:o:e: --long 'help',docker-tag:,core-name:,web-name:,core-image:,web-image: -- "$@") if [[ $? -ne 0 ]]; then exit 1; fi @@ -48,6 +56,22 @@ while [ : ]; do local_DOCKER_TAG=$2 shift 2 ;; + -c | --core-name) + local_CORE_NAME=$2 + shift 2 + ;; + -o | --core-image) + local_CORE_IMAGE=$2 + shift 2 + ;; + -w | --web-name) + local_WEB_NAME=$2 + shift 2 + ;; + -e | --web-image) + local_WEB_IMAGE=$2 + shift 2 + ;; --) shift; break ;; @@ -71,6 +95,16 @@ then exit 1 fi +if [ -z "$local_CORE_NAME" ] +then + local_CORE_NAME="datafed-core_$local_DOCKER_TAG" +fi + +if [ -z "$local_WEB_NAME" ] +then + local_WEB_NAME="datafed-web_$local_DOCKER_TAG" +fi + cat << EOF > "$DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh" #!/bin/bash @@ -87,7 +121,7 @@ USER_ID=$(id -u) docker run -d \\ --restart=always \\ - --name datafed-core_$local_DOCKER_TAG \\ + --name $local_CORE_NAME \\ -e DATAFED_GLOBUS_APP_SECRET="\$DATAFED_GLOBUS_APP_SECRET" \\ -e DATAFED_GLOBUS_APP_ID="\$DATAFED_GLOBUS_APP_ID" \\ -e DATAFED_ZEROMQ_SESSION_SECRET="\$DATAFED_ZEROMQ_SESSION_SECRET" \\ @@ -104,20 +138,20 @@ docker run -d \\ -v "\$DATAFED_INSTALL_PATH/logs:/datafed/logs" \\ -v "\$DATAFED_INSTALL_PATH/keys/datafed-core-key.pub:/opt/datafed/keys/datafed-core-key.pub" \\ -v "\$DATAFED_INSTALL_PATH/keys/datafed-core-key.priv:/opt/datafed/keys/datafed-core-key.priv" \\ - -t "datafed-core-prod:$local_DOCKER_TAG" + -t "$local_CORE_IMAGE:$local_DOCKER_TAG" EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_core_container.sh" #!/bin/bash -docker container stop datafed-core_$local_DOCKER_TAG +docker container stop $local_CORE_NAME EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_core_container.sh" #!/bin/bash -docker container stop datafed-core_$local_DOCKER_TAG -docker container rm datafed-core_$local_DOCKER_TAG +docker container stop $local_CORE_NAME +docker container rm $local_CORE_NAME EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_web_container.sh" @@ -130,7 +164,7 @@ USER_ID=$(id -u) docker run -d \\ --restart=always \\ - --name datafed-web_$local_DOCKER_TAG \\ + --name $local_WEB_NAME \\ -e DATAFED_GLOBUS_APP_SECRET="\$DATAFED_GLOBUS_APP_SECRET" \\ -e DATAFED_GLOBUS_APP_ID="\$DATAFED_GLOBUS_APP_ID" \\ -e DATAFED_ZEROMQ_SESSION_SECRET="\$DATAFED_ZEROMQ_SESSION_SECRET" \\ @@ -147,20 +181,20 @@ docker run -d \\ -v "\$DATAFED_INSTALL_PATH/keys/datafed-core-key.pub:/opt/datafed/keys/datafed-core-key.pub" \\ -v "\$DATAFED_WEB_CERT_PATH:\$DATAFED_WEB_CERT_PATH" \\ -v "\$DATAFED_WEB_KEY_PATH:\$DATAFED_WEB_KEY_PATH" \\ - -t "datafed-web-prod:$local_DOCKER_TAG" + -t "$local_WEB_IMAGE:$local_DOCKER_TAG" EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_web_container.sh" #!/bin/bash -docker container stop datafed-web_$local_DOCKER_TAG +docker container stop $local_WEB_NAME EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_web_container.sh" #!/bin/bash -docker container stop datafed-web_$local_DOCKER_TAG -docker container rm datafed-web_$local_DOCKER_TAG +docker container stop $local_WEB_NAME +docker container rm $local_WEB_NAME EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_nginx_container.sh" diff --git a/scripts/generate_repo_container_scripts.sh b/scripts/generate_repo_container_scripts.sh index 733b58920..8e569e170 100755 --- a/scripts/generate_repo_container_scripts.sh +++ b/scripts/generate_repo_container_scripts.sh @@ -24,6 +24,10 @@ Help() echo "-m, --host-collection-mount The path on the host that the container volume will mount to" echo "-i, --ip-address The public ip address of the host that the GCS container is running on" echo "-d, --repo-domain The publicly accessible domain of the server that the host is running on" + echo "-r, --repo-name The name of the repository container" + echo "-e, --repo-image The name of the repository image" + echo "-g, --gcs-name The name of the gcs container" + echo "-c, --gcs-image The name of the gcs image" echo } @@ -31,6 +35,10 @@ local_DOCKER_TAG="" local_HOST_COLLECTION_MOUNT="" local_IP_ADDRESS="" local_REPO_DOMAIN="" +local_REPO_NAME="" +local_GCS_NAME="" +local_REPO_IMAGE="datafed-repo-prod" +local_GCS_IMAGE="datafed-gcs-prod" if [ -z "${DATAFED_DOCKER_TAG}" ] then @@ -46,7 +54,7 @@ else local_HOST_COLLECTION_MOUNT=$(printenv DATAFED_HOST_COLLECTION_MOUNT) fi -VALID_ARGS=$(getopt -o ht:m:i:d: --long 'help',docker-tag:,host-collection-mount:,ip-address:,repo-domain: -- "$@") +VALID_ARGS=$(getopt -o ht:m:i:d:r:e:g:c: --long 'help',docker-tag:,host-collection-mount:,ip-address:,repo-domain:,repo-name:,repo-image:,gcs-name:,gcs-image: -- "$@") if [[ $? -ne 0 ]]; then exit 1; fi @@ -73,6 +81,22 @@ while [ : ]; do local_REPO_DOMAIN=$2 shift 2 ;; + -r | --repo-name) + local_REPO_NAME=$2 + shift 2 + ;; + -e | --repo-image) + local_REPO_IMAGE=$2 + shift 2 + ;; + -g | --gcs-name) + local_GCS_NAME=$2 + shift 2 + ;; + -c | --gcs-image) + local_GCS_IMAGE=$2 + shift 2 + ;; --) shift; break ;; @@ -118,6 +142,16 @@ then exit 1 fi +if [ -z "$local_REPO_NAME" ] +then + local_REPO_NAME="datafed-repo_$local_DOCKER_TAG" +fi + +if [ -z "$local_GCS_NAME" ] +then + local_GCS_NAME="datafed-gcs_$local_DOCKER_TAG" +fi + cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_repo_container.sh" #!/bin/bash @@ -128,7 +162,7 @@ USER_ID=$(id -u) docker run -d \\ --restart=always \\ - --name "datafed-repo_$local_DOCKER_TAG" \\ + --name "$local_REPO_NAME" \\ -e DATAFED_GLOBUS_APP_SECRET="\$DATAFED_GLOBUS_APP_SECRET" \\ -e DATAFED_GLOBUS_APP_ID="\$DATAFED_GLOBUS_APP_ID" \\ -e DATAFED_ZEROMQ_SESSION_SECRET="\$DATAFED_ZEROMQ_SESSION_SECRET" \\ @@ -145,20 +179,20 @@ docker run -d \\ -v "\$DATAFED_INSTALL_PATH/keys/datafed-repo-key.pub:/opt/datafed/keys/datafed-repo-key.pub" \\ -v "\$DATAFED_INSTALL_PATH/keys/datafed-repo-key.priv:/opt/datafed/keys/datafed-repo-key.priv" \\ -v $local_HOST_COLLECTION_MOUNT:\$DATAFED_GCS_COLLECTION_ROOT_PATH/\$DATAFED_REPO_ID_AND_DIR - -t "datafed-repo-prod:$local_DOCKER_TAG" + -t "$local_REPO_IMAGE:$local_DOCKER_TAG" EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_repo_container.sh" #!/bin/bash -docker container stop datafed-repo_$local_DOCKER_TAG +docker container stop $local_REPO_NAME EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_repo_container.sh" #!/bin/bash -docker container stop datafed-repo_$local_DOCKER_TAG -docker container rm datafed-repo_$local_DOCKER_TAG +docker container stop $local_REPO_NAME +docker container rm $local_REPO_NAME EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_gcs_container.sh" @@ -171,7 +205,7 @@ USER_ID=$(id -u) docker run -d \\ --restart=always \\ - --name "datafed-gcs_$local_DOCKER_TAG" \\ + --name "$local_GCS_NAME" \\ -e DATAFED_GLOBUS_APP_SECRET="\$DATAFED_GLOBUS_APP_SECRET" \\ -e DATAFED_GLOBUS_APP_ID="\$DATAFED_GLOBUS_APP_ID" \\ -e DATAFED_ZEROMQ_SESSION_SECRET="\$DATAFED_ZEROMQ_SESSION_SECRET" \\ @@ -198,20 +232,20 @@ docker run -d \\ -v "\$DATAFED_INSTALL_PATH/keys/datafed-repo-key.pub:/opt/datafed/keys/datafed-repo-key.pub" \\ -v "\$DATAFED_INSTALL_PATH/keys/datafed-repo-key.priv:/opt/datafed/keys/datafed-repo-key.priv" \\ -v $local_HOST_COLLECTION_MOUNT:\$DATAFED_GCS_COLLECTION_ROOT_PATH/\$DATAFED_REPO_ID_AND_DIR - -t "datafed-gcs-prod:$local_DOCKER_TAG" + -t "$local_GCS_IMAGE:$local_DOCKER_TAG" EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_gcs_container.sh" #!/bin/bash -docker container stop datafed-gcs_$local_DOCKER_TAG +docker container stop $local_GCS_NAME EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_gcs_container.sh" #!/bin/bash -docker container stop datafed-gcs_$local_DOCKER_TAG -docker container rm datafed-gcs_$local_DOCKER_TAG +docker container stop $local_GCS_NAME +docker container rm $local_GCS_NAME EOF chmod +x "$DATAFED_INSTALL_PATH/scripts/run_repo_container.sh" From 1a67869c49aa041afc80d3ae664d04e4b28701ae Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Tue, 3 Sep 2024 04:15:41 -0400 Subject: [PATCH 03/15] touched files --- .gitlab/end_to_end.yml | 6 +++++ .../generate_metadata_container_scripts.sh | 22 +++++++++++++++++++ scripts/generate_repo_container_scripts.sh | 8 +++++++ 3 files changed, 36 insertions(+) diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml index a53857abb..bb5dad44e 100644 --- a/.gitlab/end_to_end.yml +++ b/.gitlab/end_to_end.yml @@ -97,9 +97,12 @@ end-to-end-core-setup: - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/keys/datafed-core-key.priv - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - ./scripts/generate_metadata_container_scripts.sh --core-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --core-name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" + - $DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh - $DATAFED_INSTALL_PATH/scripts/run_core_container.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" + - $DATAFED_INSTALL_PATH/scripts/remove_core_container.sh + - $DATAFED_INSTALL_PATH/scripts/remove_datafed_network.sh end-to-end-ws-setup: variables: @@ -136,9 +139,12 @@ end-to-end-ws-setup: - ./scripts/container_stop.sh -n "ws-" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - ./scripts/generate_metadata_container_scripts.sh --web-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --web-name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" + - $DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh - $DATAFED_INSTALL_PATH/scripts/run_web_container.sh - sleep 30 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" + - $DATAFED_INSTALL_PATH/scripts/remove_core_container.sh + - $DATAFED_INSTALL_PATH/scripts/remove_datafed_network.sh # Repo server currently will crash on startup if it cannot connect to the core # server. diff --git a/scripts/generate_metadata_container_scripts.sh b/scripts/generate_metadata_container_scripts.sh index eee6c9ae5..ef6136296 100755 --- a/scripts/generate_metadata_container_scripts.sh +++ b/scripts/generate_metadata_container_scripts.sh @@ -105,12 +105,33 @@ then local_WEB_NAME="datafed-web_$local_DOCKER_TAG" fi +touch "$DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh" +touch "$DATAFED_INSTALL_PATH/scripts/remove_datafed_network.sh" + +touch "$DATAFED_INSTALL_PATH/scripts/run_core_container.sh" +touch "$DATAFED_INSTALL_PATH/scripts/stop_core_container.sh" +touch "$DATAFED_INSTALL_PATH/scripts/remove_core_container.sh" + +touch "$DATAFED_INSTALL_PATH/scripts/run_web_container.sh" +touch "$DATAFED_INSTALL_PATH/scripts/stop_web_container.sh" +touch "$DATAFED_INSTALL_PATH/scripts/remove_web_container.sh" + +touch "$DATAFED_INSTALL_PATH/scripts/run_nginx_container.sh" +touch "$DATAFED_INSTALL_PATH/scripts/stop_nginx_container.sh" +touch "$DATAFED_INSTALL_PATH/scripts/remove_nginx_container.sh" + cat << EOF > "$DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh" #!/bin/bash docker network create datafed-network EOF +cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_datafed_network.sh" +#!/bin/bash + +docker network rm datafed-network +EOF + cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_core_container.sh" #!/bin/bash @@ -233,6 +254,7 @@ docker container rm datafed-nginx EOF chmod +x "$DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_datafed_network.sh" chmod +x "$DATAFED_INSTALL_PATH/scripts/run_core_container.sh" chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_core_container.sh" diff --git a/scripts/generate_repo_container_scripts.sh b/scripts/generate_repo_container_scripts.sh index 8e569e170..8675a8500 100755 --- a/scripts/generate_repo_container_scripts.sh +++ b/scripts/generate_repo_container_scripts.sh @@ -152,6 +152,14 @@ then local_GCS_NAME="datafed-gcs_$local_DOCKER_TAG" fi +chmod +x "$DATAFED_INSTALL_PATH/scripts/run_repo_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_repo_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_repo_container.sh" + +chmod +x "$DATAFED_INSTALL_PATH/scripts/run_gcs_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_gcs_container.sh" +chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_gcs_container.sh" + cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_repo_container.sh" #!/bin/bash From 92bcec1921c43918a4125f28f723db490bd24d07 Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Tue, 3 Sep 2024 04:31:34 -0400 Subject: [PATCH 04/15] ensure directory is created --- .gitlab/end_to_end.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml index bb5dad44e..af84a3ce4 100644 --- a/.gitlab/end_to_end.yml +++ b/.gitlab/end_to_end.yml @@ -96,6 +96,7 @@ end-to-end-core-setup: - cat $CI_DATAFED_CORE_PUB_KEY > /shared/keys/datafed-core-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/keys/datafed-core-key.priv - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") + - mkdir -p $DATAFED_INSTALL_PATH - ./scripts/generate_metadata_container_scripts.sh --core-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --core-name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - $DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh - $DATAFED_INSTALL_PATH/scripts/run_core_container.sh @@ -138,6 +139,7 @@ end-to-end-ws-setup: - env - ./scripts/container_stop.sh -n "ws-" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") + - mkdir -p $DATAFED_INSTALL_PATH - ./scripts/generate_metadata_container_scripts.sh --web-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --web-name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - $DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh - $DATAFED_INSTALL_PATH/scripts/run_web_container.sh @@ -177,6 +179,7 @@ end-to-end-repo-setup: - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv + - mkdir -p $DATAFED_INSTALL_PATH - ./scripts/generate_repo_container_scripts.sh --repo-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --repo-name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" - $DATAFED_INSTALL_PATH/scripts/run_repo_container.sh - sleep 10 @@ -219,6 +222,7 @@ end-to-end-gcs-authz-setup: - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv + - mkdir -p $DATAFED_INSTALL_PATH - ./scripts/generate_repo_container_scripts.sh --gcs-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --gcs-name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" --ip-address "$CI_DATAFED_GCS_IP" --repo-domain "$CI_DATAFED_REPO_DOMAIN" - $DATAFED_INSTALL_PATH/scripts/run_gcs_container.sh - while [ ! -f "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" ]; do echo "Waiting for ${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"; sleep 10; done From e28b239d9d15524321221cfb954d72ad1eae13a8 Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Tue, 3 Sep 2024 09:35:29 -0400 Subject: [PATCH 05/15] moved mkdir command --- .gitlab/end_to_end.yml | 8 ++++---- scripts/generate_metadata_container_scripts.sh | 15 +-------------- scripts/generate_repo_container_scripts.sh | 8 +------- 3 files changed, 6 insertions(+), 25 deletions(-) diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml index af84a3ce4..7d2960e88 100644 --- a/.gitlab/end_to_end.yml +++ b/.gitlab/end_to_end.yml @@ -96,8 +96,8 @@ end-to-end-core-setup: - cat $CI_DATAFED_CORE_PUB_KEY > /shared/keys/datafed-core-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/keys/datafed-core-key.priv - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - mkdir -p $DATAFED_INSTALL_PATH - ./scripts/generate_metadata_container_scripts.sh --core-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --core-name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" + - source ../config/datafed.sh - $DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh - $DATAFED_INSTALL_PATH/scripts/run_core_container.sh - sleep 10 @@ -139,8 +139,8 @@ end-to-end-ws-setup: - env - ./scripts/container_stop.sh -n "ws-" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - mkdir -p $DATAFED_INSTALL_PATH - ./scripts/generate_metadata_container_scripts.sh --web-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --web-name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" + - source ../config/datafed.sh - $DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh - $DATAFED_INSTALL_PATH/scripts/run_web_container.sh - sleep 30 @@ -179,8 +179,8 @@ end-to-end-repo-setup: - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv - - mkdir -p $DATAFED_INSTALL_PATH - ./scripts/generate_repo_container_scripts.sh --repo-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --repo-name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" + - source ../config/datafed.sh - $DATAFED_INSTALL_PATH/scripts/run_repo_container.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" @@ -222,8 +222,8 @@ end-to-end-gcs-authz-setup: - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv - - mkdir -p $DATAFED_INSTALL_PATH - ./scripts/generate_repo_container_scripts.sh --gcs-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --gcs-name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" --ip-address "$CI_DATAFED_GCS_IP" --repo-domain "$CI_DATAFED_REPO_DOMAIN" + - source ../config/datafed.sh - $DATAFED_INSTALL_PATH/scripts/run_gcs_container.sh - while [ ! -f "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" ]; do echo "Waiting for ${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"; sleep 10; done - cat "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" diff --git a/scripts/generate_metadata_container_scripts.sh b/scripts/generate_metadata_container_scripts.sh index ef6136296..bbe60b231 100755 --- a/scripts/generate_metadata_container_scripts.sh +++ b/scripts/generate_metadata_container_scripts.sh @@ -105,20 +105,7 @@ then local_WEB_NAME="datafed-web_$local_DOCKER_TAG" fi -touch "$DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh" -touch "$DATAFED_INSTALL_PATH/scripts/remove_datafed_network.sh" - -touch "$DATAFED_INSTALL_PATH/scripts/run_core_container.sh" -touch "$DATAFED_INSTALL_PATH/scripts/stop_core_container.sh" -touch "$DATAFED_INSTALL_PATH/scripts/remove_core_container.sh" - -touch "$DATAFED_INSTALL_PATH/scripts/run_web_container.sh" -touch "$DATAFED_INSTALL_PATH/scripts/stop_web_container.sh" -touch "$DATAFED_INSTALL_PATH/scripts/remove_web_container.sh" - -touch "$DATAFED_INSTALL_PATH/scripts/run_nginx_container.sh" -touch "$DATAFED_INSTALL_PATH/scripts/stop_nginx_container.sh" -touch "$DATAFED_INSTALL_PATH/scripts/remove_nginx_container.sh" +mkdir -p "$DATAFED_INSTALL_PATH" cat << EOF > "$DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh" #!/bin/bash diff --git a/scripts/generate_repo_container_scripts.sh b/scripts/generate_repo_container_scripts.sh index 8675a8500..98dc25375 100755 --- a/scripts/generate_repo_container_scripts.sh +++ b/scripts/generate_repo_container_scripts.sh @@ -152,13 +152,7 @@ then local_GCS_NAME="datafed-gcs_$local_DOCKER_TAG" fi -chmod +x "$DATAFED_INSTALL_PATH/scripts/run_repo_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_repo_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_repo_container.sh" - -chmod +x "$DATAFED_INSTALL_PATH/scripts/run_gcs_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_gcs_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_gcs_container.sh" +mkdir -p "$DATAFED_INSTALL_PATH" cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_repo_container.sh" #!/bin/bash From 40cad7b71c288249fe38e6a4d484d0f1c1a54a2d Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Tue, 3 Sep 2024 13:32:37 -0400 Subject: [PATCH 06/15] created directory with adequate permissions --- .gitlab/end_to_end.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml index 7d2960e88..1ee9d92e8 100644 --- a/.gitlab/end_to_end.yml +++ b/.gitlab/end_to_end.yml @@ -85,6 +85,8 @@ end-to-end-core-setup: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - mkdir -p "$HOST_LOG_FILE_PATH" - chmod o+w "${HOST_LOG_FILE_PATH}" + - mkdir -p "/opt/datafed" + - chmod o+w "/opt/datafed" - USER_ID=$(id -u) - chown gitlab-runner "$HOST_LOG_FILE_PATH" - echo "$BRANCH_LOWER" @@ -127,6 +129,8 @@ end-to-end-ws-setup: - echo "$BRANCH_LOWER" - mkdir -p "$HOST_LOG_FILE_PATH" - mkdir -p "${DATAFED_WEB_KEY_DIR}" + - mkdir -p "/opt/datafed" + - chmod o+w "/opt/datafed" - ./scripts/ci_setup_web_certs.sh - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" @@ -171,6 +175,8 @@ end-to-end-repo-setup: - if [ -d "${DATAFED_HOST_COLLECTION_MOUNT}" ]; then rm -rf "${DATAFED_HOST_COLLECTION_MOUNT}"; fi - mkdir -p "${DATAFED_HOST_COLLECTION_MOUNT}" - chmod o+w "${HOST_LOG_FILE_PATH}" + - mkdir -p "/opt/datafed" + - chmod o+w "/opt/datafed" - USER_ID=$(id -u) - chown gitlab-runner "$HOST_LOG_FILE_PATH" - ./scripts/generate_datafed.sh @@ -213,6 +219,8 @@ end-to-end-gcs-authz-setup: - mkdir -p "${DATAFED_GLOBUS_DIR}" - cp "${CI_DATAFED_GCS_DEPLOYMENT_KEY}" "${DATAFED_GLOBUS_DIR}/deployment-key.json" - cp "${CI_DATAFED_GCS_CLIENT_CRED}" "${DATAFED_GLOBUS_DIR}/client_cred.json" + - mkdir -p "/opt/datafed" + - chmod o+w "/opt/datafed" - USER_ID=$(id -u) - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" From 97bb7e1a8550f4e24d391117fd53441b674a3ab8 Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Wed, 4 Sep 2024 05:01:12 -0400 Subject: [PATCH 07/15] changed value of install path --- .gitlab/end_to_end.yml | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml index 1ee9d92e8..3f4022bca 100644 --- a/.gitlab/end_to_end.yml +++ b/.gitlab/end_to_end.yml @@ -85,12 +85,10 @@ end-to-end-core-setup: - BRANCH_LOWER=$(echo "$CI_COMMIT_REF_NAME" | tr '[:upper:]' '[:lower:]') - mkdir -p "$HOST_LOG_FILE_PATH" - chmod o+w "${HOST_LOG_FILE_PATH}" - - mkdir -p "/opt/datafed" - - chmod o+w "/opt/datafed" - USER_ID=$(id -u) - chown gitlab-runner "$HOST_LOG_FILE_PATH" - echo "$BRANCH_LOWER" - - ./scripts/generate_datafed.sh + - DATAFED_INSTALL_PATH=$(pwd) ./scripts/generate_datafed.sh - env > env_file - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" - ./scripts/container_stop.sh -n "core-" -p @@ -100,12 +98,12 @@ end-to-end-core-setup: - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - ./scripts/generate_metadata_container_scripts.sh --core-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --core-name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - source ../config/datafed.sh - - $DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh - - $DATAFED_INSTALL_PATH/scripts/run_core_container.sh + - ./scripts/create_datafed_network.sh + - ./scripts/run_core_container.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" - - $DATAFED_INSTALL_PATH/scripts/remove_core_container.sh - - $DATAFED_INSTALL_PATH/scripts/remove_datafed_network.sh + - ./scripts/remove_core_container.sh + - ./scripts/remove_datafed_network.sh end-to-end-ws-setup: variables: @@ -129,12 +127,10 @@ end-to-end-ws-setup: - echo "$BRANCH_LOWER" - mkdir -p "$HOST_LOG_FILE_PATH" - mkdir -p "${DATAFED_WEB_KEY_DIR}" - - mkdir -p "/opt/datafed" - - chmod o+w "/opt/datafed" - ./scripts/ci_setup_web_certs.sh - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" - - ./scripts/generate_datafed.sh + - DATAFED_INSTALL_PATH=$(pwd) ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" - USER_ID=$(id -u) - GROUP_ID=$(id -g) @@ -145,12 +141,12 @@ end-to-end-ws-setup: - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - ./scripts/generate_metadata_container_scripts.sh --web-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --web-name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - source ../config/datafed.sh - - $DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh - - $DATAFED_INSTALL_PATH/scripts/run_web_container.sh + - ./scripts/create_datafed_network.sh + - ./scripts/run_web_container.sh - sleep 30 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" - - $DATAFED_INSTALL_PATH/scripts/remove_core_container.sh - - $DATAFED_INSTALL_PATH/scripts/remove_datafed_network.sh + - ./scripts/remove_core_container.sh + - ./scripts/remove_datafed_network.sh # Repo server currently will crash on startup if it cannot connect to the core # server. @@ -175,11 +171,9 @@ end-to-end-repo-setup: - if [ -d "${DATAFED_HOST_COLLECTION_MOUNT}" ]; then rm -rf "${DATAFED_HOST_COLLECTION_MOUNT}"; fi - mkdir -p "${DATAFED_HOST_COLLECTION_MOUNT}" - chmod o+w "${HOST_LOG_FILE_PATH}" - - mkdir -p "/opt/datafed" - - chmod o+w "/opt/datafed" - USER_ID=$(id -u) - chown gitlab-runner "$HOST_LOG_FILE_PATH" - - ./scripts/generate_datafed.sh + - DATAFED_INSTALL_PATH=$(pwd) ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" - ./scripts/container_stop.sh -n "repo-" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") @@ -187,7 +181,7 @@ end-to-end-repo-setup: - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv - ./scripts/generate_repo_container_scripts.sh --repo-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --repo-name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" - source ../config/datafed.sh - - $DATAFED_INSTALL_PATH/scripts/run_repo_container.sh + - ./scripts/run_repo_container.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" @@ -219,12 +213,10 @@ end-to-end-gcs-authz-setup: - mkdir -p "${DATAFED_GLOBUS_DIR}" - cp "${CI_DATAFED_GCS_DEPLOYMENT_KEY}" "${DATAFED_GLOBUS_DIR}/deployment-key.json" - cp "${CI_DATAFED_GCS_CLIENT_CRED}" "${DATAFED_GLOBUS_DIR}/client_cred.json" - - mkdir -p "/opt/datafed" - - chmod o+w "/opt/datafed" - USER_ID=$(id -u) - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" - - ./scripts/generate_datafed.sh + - DATAFED_INSTALL_PATH=$(pwd) ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" - ./scripts/container_stop.sh -n "gcs-authz" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") @@ -232,7 +224,7 @@ end-to-end-gcs-authz-setup: - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv - ./scripts/generate_repo_container_scripts.sh --gcs-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --gcs-name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" --ip-address "$CI_DATAFED_GCS_IP" --repo-domain "$CI_DATAFED_REPO_DOMAIN" - source ../config/datafed.sh - - $DATAFED_INSTALL_PATH/scripts/run_gcs_container.sh + - ./scripts/run_gcs_container.sh - while [ ! -f "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" ]; do echo "Waiting for ${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"; sleep 10; done - cat "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" From a1a3131a8e02efd44bbc200b69f97f580c8b86fc Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Wed, 4 Sep 2024 05:35:15 -0400 Subject: [PATCH 08/15] removed erroneous lines --- .gitlab/end_to_end.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml index 3f4022bca..365dc15f6 100644 --- a/.gitlab/end_to_end.yml +++ b/.gitlab/end_to_end.yml @@ -97,7 +97,6 @@ end-to-end-core-setup: - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/keys/datafed-core-key.priv - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - ./scripts/generate_metadata_container_scripts.sh --core-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --core-name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - - source ../config/datafed.sh - ./scripts/create_datafed_network.sh - ./scripts/run_core_container.sh - sleep 10 @@ -140,7 +139,6 @@ end-to-end-ws-setup: - ./scripts/container_stop.sh -n "ws-" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - ./scripts/generate_metadata_container_scripts.sh --web-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --web-name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" - - source ../config/datafed.sh - ./scripts/create_datafed_network.sh - ./scripts/run_web_container.sh - sleep 30 @@ -180,7 +178,6 @@ end-to-end-repo-setup: - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv - ./scripts/generate_repo_container_scripts.sh --repo-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --repo-name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" - - source ../config/datafed.sh - ./scripts/run_repo_container.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" @@ -223,7 +220,6 @@ end-to-end-gcs-authz-setup: - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv - ./scripts/generate_repo_container_scripts.sh --gcs-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --gcs-name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" --ip-address "$CI_DATAFED_GCS_IP" --repo-domain "$CI_DATAFED_REPO_DOMAIN" - - source ../config/datafed.sh - ./scripts/run_gcs_container.sh - while [ ! -f "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" ]; do echo "Waiting for ${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"; sleep 10; done - cat "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" From 2ac62b2103d251853aac6c7e75eff6db9ad1e71f Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Wed, 4 Sep 2024 06:00:38 -0400 Subject: [PATCH 09/15] made scripts snapshot install path at creation time --- scripts/generate_metadata_container_scripts.sh | 6 +++--- scripts/generate_repo_container_scripts.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/generate_metadata_container_scripts.sh b/scripts/generate_metadata_container_scripts.sh index bbe60b231..097b38383 100755 --- a/scripts/generate_metadata_container_scripts.sh +++ b/scripts/generate_metadata_container_scripts.sh @@ -122,7 +122,7 @@ EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_core_container.sh" #!/bin/bash -CONFIG_FILE_PATH="\$DATAFED_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$DATAFED_INSTALL_PATH/config/datafed.sh" source "\${CONFIG_FILE_PATH}" USER_ID=$(id -u) @@ -165,7 +165,7 @@ EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_web_container.sh" #!/bin/bash -CONFIG_FILE_PATH="\$DATAFED_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$DATAFED_INSTALL_PATH/config/datafed.sh" source "\$CONFIG_FILE_PATH" USER_ID=$(id -u) @@ -208,7 +208,7 @@ EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_nginx_container.sh" #!/bin/bash -CONFIG_FILE_PATH="\$DATAFED_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$DATAFED_INSTALL_PATH/config/datafed.sh" source "\$CONFIG_FILE_PATH" USER_ID=$(id -u) diff --git a/scripts/generate_repo_container_scripts.sh b/scripts/generate_repo_container_scripts.sh index 98dc25375..7c7a3686e 100755 --- a/scripts/generate_repo_container_scripts.sh +++ b/scripts/generate_repo_container_scripts.sh @@ -157,7 +157,7 @@ mkdir -p "$DATAFED_INSTALL_PATH" cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_repo_container.sh" #!/bin/bash -CONFIG_FILE_PATH="\$DATAFED_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$DATAFED_INSTALL_PATH/config/datafed.sh" source "\${CONFIG_FILE_PATH}" USER_ID=$(id -u) @@ -200,7 +200,7 @@ EOF cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_gcs_container.sh" #!/bin/bash -CONFIG_FILE_PATH="\$DATAFED_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$DATAFED_INSTALL_PATH/config/datafed.sh" source "\${CONFIG_FILE_PATH}" USER_ID=$(id -u) From 73dedbcf36e50326b0ee688ef99712ef09ed1fe0 Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Wed, 11 Sep 2024 14:14:57 -0400 Subject: [PATCH 10/15] added configurable install path and volume mounts --- .gitlab/end_to_end.yml | 12 +-- .../generate_metadata_container_scripts.sh | 102 +++++++++++------- scripts/generate_repo_container_scripts.sh | 62 ++++++----- 3 files changed, 104 insertions(+), 72 deletions(-) diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml index 365dc15f6..0d789cf58 100644 --- a/.gitlab/end_to_end.yml +++ b/.gitlab/end_to_end.yml @@ -88,7 +88,6 @@ end-to-end-core-setup: - USER_ID=$(id -u) - chown gitlab-runner "$HOST_LOG_FILE_PATH" - echo "$BRANCH_LOWER" - - DATAFED_INSTALL_PATH=$(pwd) ./scripts/generate_datafed.sh - env > env_file - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" - ./scripts/container_stop.sh -n "core-" -p @@ -96,7 +95,7 @@ end-to-end-core-setup: - cat $CI_DATAFED_CORE_PUB_KEY > /shared/keys/datafed-core-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/keys/datafed-core-key.priv - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - ./scripts/generate_metadata_container_scripts.sh --core-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --core-name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" + - ./scripts/generate_metadata_container_scripts.sh --core-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --core-name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --install-path "$(pwd)" - ./scripts/create_datafed_network.sh - ./scripts/run_core_container.sh - sleep 10 @@ -129,7 +128,6 @@ end-to-end-ws-setup: - ./scripts/ci_setup_web_certs.sh - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" - - DATAFED_INSTALL_PATH=$(pwd) ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" - USER_ID=$(id -u) - GROUP_ID=$(id -g) @@ -138,7 +136,7 @@ end-to-end-ws-setup: - env - ./scripts/container_stop.sh -n "ws-" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - - ./scripts/generate_metadata_container_scripts.sh --web-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --web-name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" + - ./scripts/generate_metadata_container_scripts.sh --web-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --web-name "ws-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --install-path "$(pwd)" - ./scripts/create_datafed_network.sh - ./scripts/run_web_container.sh - sleep 30 @@ -171,13 +169,12 @@ end-to-end-repo-setup: - chmod o+w "${HOST_LOG_FILE_PATH}" - USER_ID=$(id -u) - chown gitlab-runner "$HOST_LOG_FILE_PATH" - - DATAFED_INSTALL_PATH=$(pwd) ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" - ./scripts/container_stop.sh -n "repo-" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv - - ./scripts/generate_repo_container_scripts.sh --repo-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --repo-name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" + - ./scripts/generate_repo_container_scripts.sh --repo-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --repo-name "repo-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" --install-path "$(pwd)" - ./scripts/run_repo_container.sh - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" @@ -213,13 +210,12 @@ end-to-end-gcs-authz-setup: - USER_ID=$(id -u) - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" - - DATAFED_INSTALL_PATH=$(pwd) ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" - ./scripts/container_stop.sh -n "gcs-authz" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - cat $CI_DATAFED_CORE_PUB_KEY > /shared/datafed-repo-key.pub - cat $CI_DATAFED_CORE_PRIV_KEY > /shared/datafed-repo-key.priv - - ./scripts/generate_repo_container_scripts.sh --gcs-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --gcs-name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" --ip-address "$CI_DATAFED_GCS_IP" --repo-domain "$CI_DATAFED_REPO_DOMAIN" + - ./scripts/generate_repo_container_scripts.sh --gcs-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --gcs-name "gcs-authz-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --host-collection-mount "$DATAFED_HOST_COLLECTION_MOUNT" --ip-address "$CI_DATAFED_GCS_IP" --repo-domain "$CI_DATAFED_REPO_DOMAIN"--install-path "$(pwd)" - ./scripts/run_gcs_container.sh - while [ ! -f "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" ]; do echo "Waiting for ${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh"; sleep 10; done - cat "${DATAFED_GLOBUS_DIR}/${CI_DATAFED_REPO_ID_AND_DIR}-repo-form.sh" diff --git a/scripts/generate_metadata_container_scripts.sh b/scripts/generate_metadata_container_scripts.sh index 097b38383..fc564b55c 100755 --- a/scripts/generate_metadata_container_scripts.sh +++ b/scripts/generate_metadata_container_scripts.sh @@ -25,6 +25,10 @@ Help() echo "-o, --core-image The name of the core image" echo "-w, --web-name The name of the web container" echo "-e, --web-image The name of the web image" + echo "-v, --volume-base-path The base path for the majority of the volume mounts (Default: \$DATAFED_INSTALL_PATH)" + echo "-r, --volume-cert-path The path for the web certificate file (Default: \$DATAFED_WEB_CERT_PATH)" + echo "-k, --volume-key-path The path for the web key file (Default: \$DATAFED_WEB_KEY_PATH)" + echo "-i, --install-path The path to install the generated scripts to (Default: \$DATAFED_INSTALL_PATH)" echo } @@ -33,6 +37,10 @@ local_CORE_NAME="" local_WEB_NAME="" local_CORE_IMAGE="datafed-core-prod" local_WEB_IMAGE="datafed-web-prod" +local_VOLUME_BASE_PATH="\$DATAFED_INSTALL_PATH" +local_VOLUME_CERT_PATH="\$DATAFED_WEB_CERT_PATH" +local_VOLUME_KEY_PATH="\$DATAFED_WEB_KEY_PATH" +local_INSTALL_PATH="$DATAFED_INSTALL_PATH" if [ -z "${DATAFED_DOCKER_TAG}" ] then @@ -41,7 +49,7 @@ else local_DOCKER_TAG=$(printenv DATAFED_DOCKER_TAG) fi -VALID_ARGS=$(getopt -o ht:c:n:o:e: --long 'help',docker-tag:,core-name:,web-name:,core-image:,web-image: -- "$@") +VALID_ARGS=$(getopt -o ht:c:n:o:e:v:r:k:i: --long 'help',docker-tag:,core-name:,web-name:,core-image:,web-image:,volume-base-path:,volume-cert-path:,volume-key-path:,install-path: -- "$@") if [[ $? -ne 0 ]]; then exit 1; fi @@ -72,6 +80,22 @@ while [ : ]; do local_WEB_IMAGE=$2 shift 2 ;; + -v | --volume-base-path) + local_VOLUME_BASE_PATH=$2 + shift 2 + ;; + -r | --volume-cert-path) + local_VOLUME_CERT_PATH=$2 + shift 2 + ;; + -k | --volume-key-path) + local_VOLUME_KEY_PATH=$2 + shift 2 + ;; + -i | --install-path) + local_INSTALL_PATH=$2 + shift 2 + ;; --) shift; break ;; @@ -105,24 +129,24 @@ then local_WEB_NAME="datafed-web_$local_DOCKER_TAG" fi -mkdir -p "$DATAFED_INSTALL_PATH" +mkdir -p "$local_INSTALL_PATH" -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/create_datafed_network.sh" #!/bin/bash docker network create datafed-network EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_datafed_network.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/remove_datafed_network.sh" #!/bin/bash docker network rm datafed-network EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_core_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/run_core_container.sh" #!/bin/bash -CONFIG_FILE_PATH="$DATAFED_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$local_INSTALL_PATH/config/datafed.sh" source "\${CONFIG_FILE_PATH}" USER_ID=$(id -u) @@ -143,29 +167,29 @@ docker run -d \\ --network datafed-network \\ -p 7513:7513 \\ -p 7512:7512 \\ - -v "\$DATAFED_INSTALL_PATH/logs:/datafed/logs" \\ - -v "\$DATAFED_INSTALL_PATH/keys/datafed-core-key.pub:/opt/datafed/keys/datafed-core-key.pub" \\ - -v "\$DATAFED_INSTALL_PATH/keys/datafed-core-key.priv:/opt/datafed/keys/datafed-core-key.priv" \\ + -v "$local_VOLUME_BASE_PATH/logs:/datafed/logs" \\ + -v "$local_VOLUME_BASE_PATH/keys/datafed-core-key.pub:/opt/datafed/keys/datafed-core-key.pub" \\ + -v "$local_VOLUME_BASE_PATH/keys/datafed-core-key.priv:/opt/datafed/keys/datafed-core-key.priv" \\ -t "$local_CORE_IMAGE:$local_DOCKER_TAG" EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_core_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/stop_core_container.sh" #!/bin/bash docker container stop $local_CORE_NAME EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_core_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/remove_core_container.sh" #!/bin/bash docker container stop $local_CORE_NAME docker container rm $local_CORE_NAME EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_web_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/run_web_container.sh" #!/bin/bash -CONFIG_FILE_PATH="$DATAFED_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$local_INSTALL_PATH/config/datafed.sh" source "\$CONFIG_FILE_PATH" USER_ID=$(id -u) @@ -185,30 +209,30 @@ docker run -d \\ -e DATAFED_GOOGLE_ANALYTICS_TAG="\$DATAFED_GOOGLE_ANALYTICS_TAG" \\ -e UID="\$USER_ID" \\ --network datafed-network \\ - -v "\$DATAFED_INSTALL_PATH/logs:/datafed/logs" \\ - -v "\$DATAFED_INSTALL_PATH/keys/datafed-core-key.pub:/opt/datafed/keys/datafed-core-key.pub" \\ - -v "\$DATAFED_WEB_CERT_PATH:\$DATAFED_WEB_CERT_PATH" \\ - -v "\$DATAFED_WEB_KEY_PATH:\$DATAFED_WEB_KEY_PATH" \\ + -v "$local_VOLUME_BASE_PATH/logs:/datafed/logs" \\ + -v "$local_VOLUME_BASE_PATH/keys/datafed-core-key.pub:/opt/datafed/keys/datafed-core-key.pub" \\ + -v "$local_VOLUME_CERT_PATH:$local_VOLUME_CERT_PATH" \\ + -v "$local_VOLUME_KEY_PATH:$local_VOLUME_KEY_PATH" \\ -t "$local_WEB_IMAGE:$local_DOCKER_TAG" EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_web_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/stop_web_container.sh" #!/bin/bash docker container stop $local_WEB_NAME EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_web_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/remove_web_container.sh" #!/bin/bash docker container stop $local_WEB_NAME docker container rm $local_WEB_NAME EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_nginx_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/run_nginx_container.sh" #!/bin/bash -CONFIG_FILE_PATH="$DATAFED_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$local_INSTALL_PATH/config/datafed.sh" source "\$CONFIG_FILE_PATH" USER_ID=$(id -u) @@ -219,38 +243,38 @@ docker run -d \ --network datafed-network \ -p 443:443 \ -p 80:80 \ - -v "\$DATAFED_INSTALL_PATH/nginx/nginx.conf:/etc/nginx/conf.d/default.conf" \ - -v "\$DATAFED_INSTALL_PATH/nginx/sites-enabled:/etc/nginx/sites-enabled" \ - -v "\$DATAFED_INSTALL_PATH/nginx/www:/www" \ - -v "\$DATAFED_INSTALL_PATH/keys/datafed.ornl.gov.crt:/etc/nginx/certs/datafed.ornl.gov.crt" \ - -v "\$DATAFED_INSTALL_PATH/keys/datafed.ornl.gov.key:/etc/nginx/certs/datafed.ornl.gov.key" \ + -v "$local_VOLUME_BASE_PATH/nginx/nginx.conf:/etc/nginx/conf.d/default.conf" \ + -v "$local_VOLUME_BASE_PATH/nginx/sites-enabled:/etc/nginx/sites-enabled" \ + -v "$local_VOLUME_BASE_PATH/nginx/www:/www" \ + -v "$local_VOLUME_BASE_PATH/keys/datafed.ornl.gov.crt:/etc/nginx/certs/datafed.ornl.gov.crt" \ + -v "$local_VOLUME_BASE_PATH/keys/datafed.ornl.gov.key:/etc/nginx/certs/datafed.ornl.gov.key" \ nginx:latest EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_nginx_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/stop_nginx_container.sh" #!/bin/bash docker container stop datafed-nginx EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_nginx_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/remove_nginx_container.sh" #!/bin/bash docker container stop datafed-nginx docker container rm datafed-nginx EOF -chmod +x "$DATAFED_INSTALL_PATH/scripts/create_datafed_network.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_datafed_network.sh" +chmod +x "$local_INSTALL_PATH/scripts/create_datafed_network.sh" +chmod +x "$local_INSTALL_PATH/scripts/remove_datafed_network.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/run_core_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_core_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_core_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/run_core_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/stop_core_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/remove_core_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/run_web_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_web_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_web_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/run_web_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/stop_web_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/remove_web_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/run_nginx_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_nginx_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_nginx_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/run_nginx_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/stop_nginx_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/remove_nginx_container.sh" diff --git a/scripts/generate_repo_container_scripts.sh b/scripts/generate_repo_container_scripts.sh index 7c7a3686e..a6774f4ac 100755 --- a/scripts/generate_repo_container_scripts.sh +++ b/scripts/generate_repo_container_scripts.sh @@ -22,12 +22,14 @@ Help() echo "-h, --help Print this help message." echo "-t, --docker-tag The tag on Savannah that the currently released containers are under" echo "-m, --host-collection-mount The path on the host that the container volume will mount to" - echo "-i, --ip-address The public ip address of the host that the GCS container is running on" + echo "-p, --ip-address The public ip address of the host that the GCS container is running on" echo "-d, --repo-domain The publicly accessible domain of the server that the host is running on" echo "-r, --repo-name The name of the repository container" echo "-e, --repo-image The name of the repository image" echo "-g, --gcs-name The name of the gcs container" echo "-c, --gcs-image The name of the gcs image" + echo "-v, --volume-base-path The base path for the majority of the volume mounts (Default: \$DATAFED_INSTALL_PATH)" + echo "-i, --install-path The path to install the generated scripts to (Default: \$DATAFED_INSTALL_PATH)" echo } @@ -39,6 +41,8 @@ local_REPO_NAME="" local_GCS_NAME="" local_REPO_IMAGE="datafed-repo-prod" local_GCS_IMAGE="datafed-gcs-prod" +local_VOLUME_BASE_PATH="\$DATAFED_INSTALL_PATH" +local_INSTALL_PATH="$DATAFED_INSTALL_PATH" if [ -z "${DATAFED_DOCKER_TAG}" ] then @@ -54,7 +58,7 @@ else local_HOST_COLLECTION_MOUNT=$(printenv DATAFED_HOST_COLLECTION_MOUNT) fi -VALID_ARGS=$(getopt -o ht:m:i:d:r:e:g:c: --long 'help',docker-tag:,host-collection-mount:,ip-address:,repo-domain:,repo-name:,repo-image:,gcs-name:,gcs-image: -- "$@") +VALID_ARGS=$(getopt -o ht:m:p:d:r:e:g:c:v:i: --long 'help',docker-tag:,host-collection-mount:,ip-address:,repo-domain:,repo-name:,repo-image:,gcs-name:,gcs-image:,volume-base-path:,install-path: -- "$@") if [[ $? -ne 0 ]]; then exit 1; fi @@ -73,7 +77,7 @@ while [ : ]; do local_HOST_COLLECTION_MOUNT=$2 shift 2 ;; - -i | --ip-address) + -p | --ip-address) local_IP_ADDRESS=$2 shift 2 ;; @@ -97,6 +101,14 @@ while [ : ]; do local_GCS_IMAGE=$2 shift 2 ;; + -v | --volume-base-path) + local_VOLUME_BASE_PATH=$2 + shift 2 + ;; + -i | --install-path) + local_INSTALL_PATH=$2 + shift 2 + ;; --) shift; break ;; @@ -152,12 +164,12 @@ then local_GCS_NAME="datafed-gcs_$local_DOCKER_TAG" fi -mkdir -p "$DATAFED_INSTALL_PATH" +mkdir -p "$local_INSTALL_PATH" -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_repo_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/run_repo_container.sh" #!/bin/bash -CONFIG_FILE_PATH="$DATAFED_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$local_INSTALL_PATH/config/datafed.sh" source "\${CONFIG_FILE_PATH}" USER_ID=$(id -u) @@ -177,30 +189,30 @@ docker run -d \\ -e DATAFED_GCS_COLLECTION_ROOT_PATH="\$DATAFED_GCS_COLLECTION_ROOT_PATH" \\ -e UID="\$USER_ID" \\ -p 9000:9000 \\ - -v "\$DATAFED_INSTALL_PATH/logs:/datafed/logs" \\ - -v "\$DATAFED_INSTALL_PATH/keys/datafed-repo-key.pub:/opt/datafed/keys/datafed-repo-key.pub" \\ - -v "\$DATAFED_INSTALL_PATH/keys/datafed-repo-key.priv:/opt/datafed/keys/datafed-repo-key.priv" \\ + -v "$local_VOLUME_BASE_PATH/logs:/datafed/logs" \\ + -v "$local_VOLUME_BASE_PATH/keys/datafed-repo-key.pub:/opt/datafed/keys/datafed-repo-key.pub" \\ + -v "$local_VOLUME_BASE_PATH/keys/datafed-repo-key.priv:/opt/datafed/keys/datafed-repo-key.priv" \\ -v $local_HOST_COLLECTION_MOUNT:\$DATAFED_GCS_COLLECTION_ROOT_PATH/\$DATAFED_REPO_ID_AND_DIR -t "$local_REPO_IMAGE:$local_DOCKER_TAG" EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_repo_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/stop_repo_container.sh" #!/bin/bash docker container stop $local_REPO_NAME EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_repo_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/remove_repo_container.sh" #!/bin/bash docker container stop $local_REPO_NAME docker container rm $local_REPO_NAME EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/run_gcs_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/run_gcs_container.sh" #!/bin/bash -CONFIG_FILE_PATH="$DATAFED_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$local_INSTALL_PATH/config/datafed.sh" source "\${CONFIG_FILE_PATH}" USER_ID=$(id -u) @@ -229,31 +241,31 @@ docker run -d \\ -e DATAFED_REPO_DOMAIN="$local_REPO_DOMAIN" \\ -e UID="\$USER_ID" \\ --network=host \\ - -v "\$DATAFED_INSTALL_PATH/logs:/datafed/logs" \\ - -v "\$DATAFED_INSTALL_PATH/globus:/opt/datafed/globus" \\ - -v "\$DATAFED_INSTALL_PATH/keys/datafed-repo-key.pub:/opt/datafed/keys/datafed-repo-key.pub" \\ - -v "\$DATAFED_INSTALL_PATH/keys/datafed-repo-key.priv:/opt/datafed/keys/datafed-repo-key.priv" \\ + -v "$local_VOLUME_BASE_PATH/logs:/datafed/logs" \\ + -v "$local_VOLUME_BASE_PATH/globus:/opt/datafed/globus" \\ + -v "$local_VOLUME_BASE_PATH/keys/datafed-repo-key.pub:/opt/datafed/keys/datafed-repo-key.pub" \\ + -v "$local_VOLUME_BASE_PATH/keys/datafed-repo-key.priv:/opt/datafed/keys/datafed-repo-key.priv" \\ -v $local_HOST_COLLECTION_MOUNT:\$DATAFED_GCS_COLLECTION_ROOT_PATH/\$DATAFED_REPO_ID_AND_DIR -t "$local_GCS_IMAGE:$local_DOCKER_TAG" EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/stop_gcs_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/stop_gcs_container.sh" #!/bin/bash docker container stop $local_GCS_NAME EOF -cat << EOF > "$DATAFED_INSTALL_PATH/scripts/remove_gcs_container.sh" +cat << EOF > "$local_INSTALL_PATH/scripts/remove_gcs_container.sh" #!/bin/bash docker container stop $local_GCS_NAME docker container rm $local_GCS_NAME EOF -chmod +x "$DATAFED_INSTALL_PATH/scripts/run_repo_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_repo_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_repo_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/run_repo_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/stop_repo_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/remove_repo_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/run_gcs_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/stop_gcs_container.sh" -chmod +x "$DATAFED_INSTALL_PATH/scripts/remove_gcs_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/run_gcs_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/stop_gcs_container.sh" +chmod +x "$local_INSTALL_PATH/scripts/remove_gcs_container.sh" From d6813a01688b714e9d76d456853598b62ce61e5c Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Wed, 11 Sep 2024 14:45:27 -0400 Subject: [PATCH 11/15] add back missing config generation --- .gitlab/end_to_end.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml index 0d789cf58..cf316c1f8 100644 --- a/.gitlab/end_to_end.yml +++ b/.gitlab/end_to_end.yml @@ -88,6 +88,7 @@ end-to-end-core-setup: - USER_ID=$(id -u) - chown gitlab-runner "$HOST_LOG_FILE_PATH" - echo "$BRANCH_LOWER" + - ./scripts/generate_datafed.sh - env > env_file - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" - ./scripts/container_stop.sh -n "core-" -p @@ -128,6 +129,7 @@ end-to-end-ws-setup: - ./scripts/ci_setup_web_certs.sh - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" + - ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" - USER_ID=$(id -u) - GROUP_ID=$(id -g) @@ -169,6 +171,7 @@ end-to-end-repo-setup: - chmod o+w "${HOST_LOG_FILE_PATH}" - USER_ID=$(id -u) - chown gitlab-runner "$HOST_LOG_FILE_PATH" + - ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" - ./scripts/container_stop.sh -n "repo-" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") @@ -210,6 +213,7 @@ end-to-end-gcs-authz-setup: - USER_ID=$(id -u) - chmod o+w "${HOST_LOG_FILE_PATH}" - chown gitlab-runner "$HOST_LOG_FILE_PATH" + - ./scripts/generate_datafed.sh - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" - ./scripts/container_stop.sh -n "gcs-authz" -p - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") From 44d540c624a50800b6372ddee8c0b61284148fbd Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Tue, 17 Sep 2024 04:50:21 -0400 Subject: [PATCH 12/15] added debug prints --- .gitlab/end_to_end.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml index cf316c1f8..f01dc0a69 100644 --- a/.gitlab/end_to_end.yml +++ b/.gitlab/end_to_end.yml @@ -99,7 +99,10 @@ end-to-end-core-setup: - ./scripts/generate_metadata_container_scripts.sh --core-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --core-name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --install-path "$(pwd)" - ./scripts/create_datafed_network.sh - ./scripts/run_core_container.sh - - sleep 10 + # - sleep 10 + - sleep 60 + - echo "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" + - docker container ls - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" - ./scripts/remove_core_container.sh - ./scripts/remove_datafed_network.sh From 892aa20f0fbb88003126fee99d9a3ae502dcc37e Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Tue, 17 Sep 2024 05:49:08 -0400 Subject: [PATCH 13/15] added more debug information --- .gitlab/end_to_end.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml index f01dc0a69..5cfdcb832 100644 --- a/.gitlab/end_to_end.yml +++ b/.gitlab/end_to_end.yml @@ -98,11 +98,12 @@ end-to-end-core-setup: - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - ./scripts/generate_metadata_container_scripts.sh --core-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --core-name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --install-path "$(pwd)" - ./scripts/create_datafed_network.sh - - ./scripts/run_core_container.sh + - CONTAINER_ID=$(./scripts/run_core_container.sh) # - sleep 10 - sleep 60 - echo "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" - docker container ls + - docker container logs $(CONTAINER_ID) - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" - ./scripts/remove_core_container.sh - ./scripts/remove_datafed_network.sh From d5996231d20678a5cf2d8d4fc6d331231a0c8625 Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Tue, 17 Sep 2024 05:56:52 -0400 Subject: [PATCH 14/15] fixed env reference --- .gitlab/end_to_end.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml index 5cfdcb832..ff8d13635 100644 --- a/.gitlab/end_to_end.yml +++ b/.gitlab/end_to_end.yml @@ -103,7 +103,7 @@ end-to-end-core-setup: - sleep 60 - echo "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" - docker container ls - - docker container logs $(CONTAINER_ID) + - docker container logs $CONTAINER_ID - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" - ./scripts/remove_core_container.sh - ./scripts/remove_datafed_network.sh From 2bd00b9664d76525f380544eb6be9c7c75fc3bfb Mon Sep 17 00:00:00 2001 From: Blake Nedved Date: Thu, 19 Sep 2024 06:03:38 -0400 Subject: [PATCH 15/15] fixed reference to config files --- .gitlab/end_to_end.yml | 8 ++------ scripts/generate_metadata_container_scripts.sh | 10 ++++++---- scripts/generate_repo_container_scripts.sh | 8 +++++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.gitlab/end_to_end.yml b/.gitlab/end_to_end.yml index ff8d13635..cf316c1f8 100644 --- a/.gitlab/end_to_end.yml +++ b/.gitlab/end_to_end.yml @@ -98,12 +98,8 @@ end-to-end-core-setup: - random_string=$(bash -c "cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w "10" | head -n 1") - ./scripts/generate_metadata_container_scripts.sh --core-image "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}" --docker-tag latest --core-name "core-${BRANCH_LOWER}-${CI_COMMIT_SHORT_SHA}-${random_string}" --install-path "$(pwd)" - ./scripts/create_datafed_network.sh - - CONTAINER_ID=$(./scripts/run_core_container.sh) - # - sleep 10 - - sleep 60 - - echo "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" - - docker container ls - - docker container logs $CONTAINER_ID + - ./scripts/run_core_container.sh + - sleep 10 - ./scripts/container_run_test.sh -e -c "1" -t "${REGISTRY}/${IMAGE_TAG}${BRANCH_LOWER}:latest" - ./scripts/remove_core_container.sh - ./scripts/remove_datafed_network.sh diff --git a/scripts/generate_metadata_container_scripts.sh b/scripts/generate_metadata_container_scripts.sh index fc564b55c..531404ef2 100755 --- a/scripts/generate_metadata_container_scripts.sh +++ b/scripts/generate_metadata_container_scripts.sh @@ -8,7 +8,8 @@ SCRIPT=$(realpath "$0") FILE_NAME=$(basename "${SCRIPT}") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath "${SOURCE}/..") -source "${PROJECT_ROOT}/config/datafed.sh" + +$PROJECT_ROOT/config/datafed.sh VERSION="1.0.0" echo "$FILE_NAME $VERSION" @@ -41,6 +42,7 @@ local_VOLUME_BASE_PATH="\$DATAFED_INSTALL_PATH" local_VOLUME_CERT_PATH="\$DATAFED_WEB_CERT_PATH" local_VOLUME_KEY_PATH="\$DATAFED_WEB_KEY_PATH" local_INSTALL_PATH="$DATAFED_INSTALL_PATH" +local_CONFIG_PATH="$PROJECT_ROOT" if [ -z "${DATAFED_DOCKER_TAG}" ] then @@ -146,7 +148,7 @@ EOF cat << EOF > "$local_INSTALL_PATH/scripts/run_core_container.sh" #!/bin/bash -CONFIG_FILE_PATH="$local_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$local_CONFIG_PATH/config/datafed.sh" source "\${CONFIG_FILE_PATH}" USER_ID=$(id -u) @@ -189,7 +191,7 @@ EOF cat << EOF > "$local_INSTALL_PATH/scripts/run_web_container.sh" #!/bin/bash -CONFIG_FILE_PATH="$local_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$local_CONFIG_PATH/config/datafed.sh" source "\$CONFIG_FILE_PATH" USER_ID=$(id -u) @@ -232,7 +234,7 @@ EOF cat << EOF > "$local_INSTALL_PATH/scripts/run_nginx_container.sh" #!/bin/bash -CONFIG_FILE_PATH="$local_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$local_CONFIG_PATH/config/datafed.sh" source "\$CONFIG_FILE_PATH" USER_ID=$(id -u) diff --git a/scripts/generate_repo_container_scripts.sh b/scripts/generate_repo_container_scripts.sh index a6774f4ac..f23091ae3 100755 --- a/scripts/generate_repo_container_scripts.sh +++ b/scripts/generate_repo_container_scripts.sh @@ -8,7 +8,8 @@ SCRIPT=$(realpath "$0") FILE_NAME=$(basename "${SCRIPT}") SOURCE=$(dirname "$SCRIPT") PROJECT_ROOT=$(realpath "${SOURCE}/..") -source "${PROJECT_ROOT}/config/datafed.sh" + +$PROJECT_ROOT/config/datafed.sh VERSION="1.0.0" echo "$FILE_NAME $VERSION" @@ -43,6 +44,7 @@ local_REPO_IMAGE="datafed-repo-prod" local_GCS_IMAGE="datafed-gcs-prod" local_VOLUME_BASE_PATH="\$DATAFED_INSTALL_PATH" local_INSTALL_PATH="$DATAFED_INSTALL_PATH" +local_CONFIG_PATH="$PROJECT_ROOT" if [ -z "${DATAFED_DOCKER_TAG}" ] then @@ -169,7 +171,7 @@ mkdir -p "$local_INSTALL_PATH" cat << EOF > "$local_INSTALL_PATH/scripts/run_repo_container.sh" #!/bin/bash -CONFIG_FILE_PATH="$local_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$local_CONFIG_PATH/config/datafed.sh" source "\${CONFIG_FILE_PATH}" USER_ID=$(id -u) @@ -212,7 +214,7 @@ EOF cat << EOF > "$local_INSTALL_PATH/scripts/run_gcs_container.sh" #!/bin/bash -CONFIG_FILE_PATH="$local_INSTALL_PATH/config/datafed.sh" +CONFIG_FILE_PATH="$local_CONFIG_PATH/config/datafed.sh" source "\${CONFIG_FILE_PATH}" USER_ID=$(id -u)