From f8bb5038ee88506c848165dadf04432cfe9c9337 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Thu, 19 Dec 2024 21:54:47 +0200 Subject: [PATCH 01/10] Remove obsolete `version` property from `docker-compose.custom.yml`. --- docker/docker-compose.custom.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/docker-compose.custom.yml b/docker/docker-compose.custom.yml index a2adcdd..29cb012 100644 --- a/docker/docker-compose.custom.yml +++ b/docker/docker-compose.custom.yml @@ -1,5 +1,4 @@ --- -version: "3.8" services: # build and run the local image instead # note: it's currently expose on port 3009 From edc022a9b60a4c373f50bf7af93c6455f4983dd8 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Thu, 2 Jan 2025 13:51:32 +0200 Subject: [PATCH 02/10] Fix an error in `development.sh` instructions. --- development.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/development.sh b/development.sh index 22deb64..831b831 100755 --- a/development.sh +++ b/development.sh @@ -14,7 +14,7 @@ instruct_and_exit() { echo "Available commands:" echo "start Start the dependencies and the dockerized application" echo "start:deps Start the dependencies only" - echo "generate:jooq Start the dependencies and generate jOOQ classes" + echo "generate:jooq Generate jOOQ classes" echo "build:data-inserter Installs required dependencies and builds the timetables data inserter" echo "stop Stop the dependencies and the dockerized application" exit 1 From 3700c863556fa1f353eecba2b328afbc4ce847ef Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Thu, 19 Dec 2024 22:11:31 +0200 Subject: [PATCH 03/10] From now on, download the JORE4 Docker Compose bundle from the "jore4-docker-compose-bundle" repository. The CLI commands used work on both Linux (GNU) and macOS (BSD) and are not dependent on shell built-ins. --- development.sh | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/development.sh b/development.sh index 831b831..5f5c7b0 100755 --- a/development.sh +++ b/development.sh @@ -20,11 +20,25 @@ instruct_and_exit() { exit 1 } +# Download Docker Compose bundle from the "jore4-docker-compose-bundle" +# repository. download_docker_bundle() { - # based on https://github.com/HSLdevcom/jore4-tools#download-docker-bundlesh - - echo "Downloading latest version of E2E docker-compose package..." - curl https://raw.githubusercontent.com/HSLdevcom/jore4-tools/main/docker/download-docker-bundle.sh | bash + # First, clean untracked files from `docker` directory even if they are + # git-ignored. + git clean -fx ./docker + + echo "Downloading the latest version of the JORE4 Docker Compose bundle..." + + # Download the latest Docker Compose bundle from the jore4-docker-compose-bundle + # repository as a ZIP file. Extracts the files from the `docker-compose` + # directory (of the ZIP file) to your local `docker` directory. + curl -sL https://github.com/HSLdevcom/jore4-docker-compose-bundle/archive/refs/heads/main.zip \ + -o /tmp/jore4-docker-compose-bundle.zip \ + && unzip -q /tmp/jore4-docker-compose-bundle.zip -d /tmp \ + && mv /tmp/jore4-docker-compose-bundle-main/docker-compose/* ./docker \ + && rm -fr \ + /tmp/jore4-docker-compose-bundle.zip \ + /tmp/jore4-docker-compose-bundle-main/ } start_all() { From 7ed68563ed52639725a426f27904d01a31ca2176 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Fri, 20 Dec 2024 15:29:31 +0200 Subject: [PATCH 04/10] Fetch the commit ID of the Docker Compose bundle to ease debugging. --- development.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/development.sh b/development.sh index 5f5c7b0..85f84fe 100755 --- a/development.sh +++ b/development.sh @@ -39,6 +39,17 @@ download_docker_bundle() { && rm -fr \ /tmp/jore4-docker-compose-bundle.zip \ /tmp/jore4-docker-compose-bundle-main/ + + echo "Generating a release version file for the downloaded bundle..." + + # Create a release version file containing the SHA digest of the last commit + # in the main branch of the jore4-docker-compose-bundle repository. + curl -s -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/HSLdevcom/jore4-docker-compose-bundle/branches/main \ + | grep '"sha"' \ + | head -1 \ + | sed -E 's/.*"sha": "(.*)",/\1/' \ + > ./docker/RELEASE_VERSION.txt } start_all() { From e937667b855cad1ead6a1f15dac54fa15f48c7cb Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Fri, 3 Jan 2025 17:42:19 +0200 Subject: [PATCH 05/10] Add commit SHA parameter for downloading a specific version of Docker Compose bundle. Refactor `development.sh` to use GitHub CLI. --- development.sh | 114 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 87 insertions(+), 27 deletions(-) diff --git a/development.sh b/development.sh index 85f84fe..e743f50 100755 --- a/development.sh +++ b/development.sh @@ -5,62 +5,122 @@ set -eu cd "$(dirname "$0")" # Setting the working directory as the script directory COMMAND=${1:-} +ARG1=${2:-} DOCKER_COMPOSE_CMD="docker compose -f ./docker/docker-compose.yml -f ./docker/docker-compose.custom.yml" instruct_and_exit() { - echo "Usage: ${0} " + echo "Usage: ${0} []" echo "" echo "Available commands:" - echo "start Start the dependencies and the dockerized application" - echo "start:deps Start the dependencies only" - echo "generate:jooq Generate jOOQ classes" - echo "build:data-inserter Installs required dependencies and builds the timetables data inserter" - echo "stop Stop the dependencies and the dockerized application" + echo "" + echo "start [] Start the dependencies and the dockerized application. A commit SHA can" + echo " be given as an argument to fetch a specific version of Docker Compose" + echo " bundle. Without argument, the latest commit in the main branch of the" + echo " jore4-docker-compose-bundle repository is used." + echo "" + echo "start:deps [] Start the dependencies only. A commit SHA can be given as an argument to" + echo " fetch a specific version of Docker Compose bundle. Without argument, the" + echo " latest commit in the main branch of the jore4-docker-compose-bundle" + echo " repository is used." + echo "" + echo "generate:jooq Generate jOOQ classes" + echo "" + echo "build:data-inserter Installs required dependencies and builds the timetables data inserter" + echo "" + echo "stop Stop the dependencies and the dockerized application" + echo "" exit 1 } # Download Docker Compose bundle from the "jore4-docker-compose-bundle" -# repository. +# repository. GitHub CLI is required to be installed. +# +# A commit SHA can be given as an argument. download_docker_bundle() { + local commit_sha="${1:-}" + + local repo_name="jore4-docker-compose-bundle" + local repo_owner="HSLdevcom" + local gh_common_path="/repos/${repo_owner}/${repo_name}" + + if [[ -n "$commit_sha" ]]; then + # Verify that a commit with SHA actually exists in the repository. + echo "Verifying that a commit with SHA '${commit_sha}' exists in the ${repo_owner}/${repo_name} repository..." + + # First, query GitHub API using the commit SHA argument. + local http_response=$( + gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${gh_common_path}/commits/${commit_sha}" \ + --silent \ + --include + ) + local http_status=$(echo "$http_response" | awk '/^HTTP/{print $2}') + + # Then, check if the status is not 200. + if [[ $http_status != "200" ]]; then + echo "Error: Querying GitHub API with the commit ID '${commit_sha}' failed." >&2 + exit 1 + else + echo "SHA digest OK." + fi + else # when no argument given + # Resolve the SHA digest of the last commit in the main branch. + commit_sha=$( + gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${gh_common_path}/branches/main" \ + --jq '.commit.sha' + ) + + echo "Resolved the SHA digest of the last commit in the main branch: ${commit_sha}" + fi + # First, clean untracked files from `docker` directory even if they are # git-ignored. git clean -fx ./docker - echo "Downloading the latest version of the JORE4 Docker Compose bundle..." + local zip_file="/tmp/${repo_name}.zip" + local unzip_target_dir_prefix="/tmp/${repo_owner}-${repo_name}" + + # Remove old temporary directories if any remain. + rm -fr "$unzip_target_dir_prefix"-* + + echo "Downloading the JORE4 Docker Compose bundle..." # Download the latest Docker Compose bundle from the jore4-docker-compose-bundle # repository as a ZIP file. Extracts the files from the `docker-compose` # directory (of the ZIP file) to your local `docker` directory. - curl -sL https://github.com/HSLdevcom/jore4-docker-compose-bundle/archive/refs/heads/main.zip \ - -o /tmp/jore4-docker-compose-bundle.zip \ - && unzip -q /tmp/jore4-docker-compose-bundle.zip -d /tmp \ - && mv /tmp/jore4-docker-compose-bundle-main/docker-compose/* ./docker \ - && rm -fr \ - /tmp/jore4-docker-compose-bundle.zip \ - /tmp/jore4-docker-compose-bundle-main/ + gh api "${gh_common_path}/zipball/${commit_sha}" > "$zip_file" \ + && unzip -q "$zip_file" -d /tmp \ + && mv "$unzip_target_dir_prefix"-*/docker-compose/* ./docker + + # Remove the temporary files and directories created above. + rm -fr "$zip_file" "$unzip_target_dir_prefix"-* echo "Generating a release version file for the downloaded bundle..." - # Create a release version file containing the SHA digest of the last commit - # in the main branch of the jore4-docker-compose-bundle repository. - curl -s -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/HSLdevcom/jore4-docker-compose-bundle/branches/main \ - | grep '"sha"' \ - | head -1 \ - | sed -E 's/.*"sha": "(.*)",/\1/' \ - > ./docker/RELEASE_VERSION.txt + # Create a release version file containing the SHA digest of the referenced + # commit. + echo "$commit_sha" > ./docker/RELEASE_VERSION.txt } start_all() { - download_docker_bundle + local commit_sha="${1:-}" + + download_docker_bundle ${commit_sha} $DOCKER_COMPOSE_CMD up -d jore4-hasura jore4-testdb $DOCKER_COMPOSE_CMD up --build -d jore4-timetables-api prepare_timetables_data_inserter } start_deps() { - download_docker_bundle + local commit_sha="${1:-}" + + download_docker_bundle ${commit_sha} # Runs the following services: # jore4-hasura - Hasura. We have to start Hasura because it ensures that db migrations are run to the Jore 4 database. # jore4-testdb - Jore 4 database. This is the database used by the API. @@ -108,12 +168,12 @@ if [[ -z ${COMMAND} ]]; then fi if [[ ${COMMAND} == "start" ]]; then - start_all + start_all ${ARG1} exit 0 fi if [[ ${COMMAND} == "start:deps" ]]; then - start_deps + start_deps ${ARG1} exit 0 fi From 431501a304003590c78f5ddf1a658fdffd5d956a Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Fri, 3 Jan 2025 20:10:57 +0200 Subject: [PATCH 06/10] Order some operations a bit differently in `download_docker_bundle` function. --- development.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/development.sh b/development.sh index e743f50..8932a34 100755 --- a/development.sh +++ b/development.sh @@ -79,10 +79,6 @@ download_docker_bundle() { echo "Resolved the SHA digest of the last commit in the main branch: ${commit_sha}" fi - # First, clean untracked files from `docker` directory even if they are - # git-ignored. - git clean -fx ./docker - local zip_file="/tmp/${repo_name}.zip" local unzip_target_dir_prefix="/tmp/${repo_owner}-${repo_name}" @@ -91,12 +87,18 @@ download_docker_bundle() { echo "Downloading the JORE4 Docker Compose bundle..." - # Download the latest Docker Compose bundle from the jore4-docker-compose-bundle - # repository as a ZIP file. Extracts the files from the `docker-compose` - # directory (of the ZIP file) to your local `docker` directory. + # Download the latest Docker Compose bundle from the + # jore4-docker-compose-bundle repository as a ZIP file and extract its + # contents to a temporary directory. gh api "${gh_common_path}/zipball/${commit_sha}" > "$zip_file" \ - && unzip -q "$zip_file" -d /tmp \ - && mv "$unzip_target_dir_prefix"-*/docker-compose/* ./docker + && unzip -q "$zip_file" -d /tmp + + # Clean untracked files from `docker` directory even if they are git-ignored. + git clean -fx ./docker + + # Copy files from the `docker-compose` directory of the ZIP file to your + # local `docker` directory. + mv "$unzip_target_dir_prefix"-*/docker-compose/* ./docker # Remove the temporary files and directories created above. rm -fr "$zip_file" "$unzip_target_dir_prefix"-* From 3a7c885ae47bc268443b7fa474310f2c61341637 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Fri, 3 Jan 2025 18:20:21 +0200 Subject: [PATCH 07/10] Fix downloading Docker Compose bundle to work with partial SHA arguments. --- development.sh | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/development.sh b/development.sh index 8932a34..d64347e 100755 --- a/development.sh +++ b/development.sh @@ -36,7 +36,8 @@ instruct_and_exit() { # Download Docker Compose bundle from the "jore4-docker-compose-bundle" # repository. GitHub CLI is required to be installed. # -# A commit SHA can be given as an argument. +# A commit SHA can be given as an argument. The given SHA may contain only a +# substring of the actual value. download_docker_bundle() { local commit_sha="${1:-}" @@ -48,23 +49,24 @@ download_docker_bundle() { # Verify that a commit with SHA actually exists in the repository. echo "Verifying that a commit with SHA '${commit_sha}' exists in the ${repo_owner}/${repo_name} repository..." - # First, query GitHub API using the commit SHA argument. - local http_response=$( + # First, try to find matching commit SHA from GitHub. + local gh_commit_sha=$( gh api \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ "${gh_common_path}/commits/${commit_sha}" \ - --silent \ - --include + --jq '.sha' ) - local http_status=$(echo "$http_response" | awk '/^HTTP/{print $2}') - # Then, check if the status is not 200. - if [[ $http_status != "200" ]]; then + # Then, compare if a substring match is found. + if [[ "$gh_commit_sha" == "$commit_sha"* ]]; then + echo "Commit with the following SHA digest is found: ${gh_commit_sha}" + + # Replace the commit SHA given as argument with a full SHA digest. + commit_sha="$gh_commit_sha" + else echo "Error: Querying GitHub API with the commit ID '${commit_sha}' failed." >&2 exit 1 - else - echo "SHA digest OK." fi else # when no argument given # Resolve the SHA digest of the last commit in the main branch. From 910f3f2dc174561bcb4d6691c450f70c70217e98 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Wed, 8 Jan 2025 13:14:21 +0200 Subject: [PATCH 08/10] Simplify downloading Docker Compose bundle based on the review comments. --- development.sh | 91 ++++++++++++++++++++------------------------------ 1 file changed, 37 insertions(+), 54 deletions(-) diff --git a/development.sh b/development.sh index d64347e..25f3dcb 100755 --- a/development.sh +++ b/development.sh @@ -5,7 +5,6 @@ set -eu cd "$(dirname "$0")" # Setting the working directory as the script directory COMMAND=${1:-} -ARG1=${2:-} DOCKER_COMPOSE_CMD="docker compose -f ./docker/docker-compose.yml -f ./docker/docker-compose.custom.yml" @@ -14,15 +13,15 @@ instruct_and_exit() { echo "" echo "Available commands:" echo "" - echo "start [] Start the dependencies and the dockerized application. A commit SHA can" - echo " be given as an argument to fetch a specific version of Docker Compose" - echo " bundle. Without argument, the latest commit in the main branch of the" - echo " jore4-docker-compose-bundle repository is used." + echo "start [] Start the dependencies and the dockerized application. A commit" + echo " reference can be given as an argument to fetch a specific version of" + echo " Docker Compose bundle. Without argument, the latest commit in the main" + echo " branch of the jore4-docker-compose-bundle repository is used." echo "" - echo "start:deps [] Start the dependencies only. A commit SHA can be given as an argument to" - echo " fetch a specific version of Docker Compose bundle. Without argument, the" - echo " latest commit in the main branch of the jore4-docker-compose-bundle" - echo " repository is used." + echo "start:deps [] Start the dependencies only. A commit reference can be given as an" + echo " argument to fetch a specific version of Docker Compose bundle. Without" + echo " argument, the latest commit in the main branch of the" + echo " jore4-docker-compose-bundle repository is used." echo "" echo "generate:jooq Generate jOOQ classes" echo "" @@ -36,51 +35,35 @@ instruct_and_exit() { # Download Docker Compose bundle from the "jore4-docker-compose-bundle" # repository. GitHub CLI is required to be installed. # -# A commit SHA can be given as an argument. The given SHA may contain only a -# substring of the actual value. +# A commit reference can be given as an argument. It can contain, for example, +# only a substring of an actual SHA digest. download_docker_bundle() { - local commit_sha="${1:-}" + local commit_ref="${1:-main}" local repo_name="jore4-docker-compose-bundle" local repo_owner="HSLdevcom" local gh_common_path="/repos/${repo_owner}/${repo_name}" - if [[ -n "$commit_sha" ]]; then - # Verify that a commit with SHA actually exists in the repository. - echo "Verifying that a commit with SHA '${commit_sha}' exists in the ${repo_owner}/${repo_name} repository..." - - # First, try to find matching commit SHA from GitHub. - local gh_commit_sha=$( - gh api \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "${gh_common_path}/commits/${commit_sha}" \ - --jq '.sha' - ) - - # Then, compare if a substring match is found. - if [[ "$gh_commit_sha" == "$commit_sha"* ]]; then - echo "Commit with the following SHA digest is found: ${gh_commit_sha}" - - # Replace the commit SHA given as argument with a full SHA digest. - commit_sha="$gh_commit_sha" - else - echo "Error: Querying GitHub API with the commit ID '${commit_sha}' failed." >&2 - exit 1 - fi - else # when no argument given - # Resolve the SHA digest of the last commit in the main branch. - commit_sha=$( - gh api \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "${gh_common_path}/branches/main" \ - --jq '.commit.sha' - ) - - echo "Resolved the SHA digest of the last commit in the main branch: ${commit_sha}" + echo "Using the commit reference '${commit_ref}' to fetch a Docker Compose bundle..." + + # First, try to find a commit on GitHub that matches the given reference. + local commit_sha=$( + gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${gh_common_path}/commits/${commit_ref}" \ + --jq '.sha' + ) + + # Then, check if a match wasn't found, meaning the previous GitHub CLI command + # failed. + if [[ $? -ne 0 ]]; then + echo "Error: Querying GitHub API with the commit ref '${commit_ref}' failed." >&2 + exit 1 fi + echo "Commit with the following SHA digest was found: ${commit_sha}" + local zip_file="/tmp/${repo_name}.zip" local unzip_target_dir_prefix="/tmp/${repo_owner}-${repo_name}" @@ -113,18 +96,14 @@ download_docker_bundle() { } start_all() { - local commit_sha="${1:-}" - - download_docker_bundle ${commit_sha} + download_docker_bundle "$@" $DOCKER_COMPOSE_CMD up -d jore4-hasura jore4-testdb $DOCKER_COMPOSE_CMD up --build -d jore4-timetables-api prepare_timetables_data_inserter } start_deps() { - local commit_sha="${1:-}" - - download_docker_bundle ${commit_sha} + download_docker_bundle "$@" # Runs the following services: # jore4-hasura - Hasura. We have to start Hasura because it ensures that db migrations are run to the Jore 4 database. # jore4-testdb - Jore 4 database. This is the database used by the API. @@ -171,13 +150,17 @@ if [[ -z ${COMMAND} ]]; then instruct_and_exit fi +# Shift other arguments after the command so that we can refer to them later +# with "$@". +shift + if [[ ${COMMAND} == "start" ]]; then - start_all ${ARG1} + start_all "$@" exit 0 fi if [[ ${COMMAND} == "start:deps" ]]; then - start_deps ${ARG1} + start_deps "$@" exit 0 fi From bcf609c25df2c674e8f57274a28e04e03c2c7b66 Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Wed, 8 Jan 2025 16:06:03 +0200 Subject: [PATCH 09/10] Add some check to shell script code downloading Docker Compose bundle. --- development.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/development.sh b/development.sh index 25f3dcb..76b4346 100755 --- a/development.sh +++ b/development.sh @@ -44,6 +44,15 @@ download_docker_bundle() { local repo_owner="HSLdevcom" local gh_common_path="/repos/${repo_owner}/${repo_name}" + # Check GitHub CLI availability. + if ! command -v gh &> /dev/null; then + echo "Please install the GitHub CLI (gh) on your machine." + exit 1 + fi + + # Make sure the user is authenticated to GitHub. + gh auth status || gh auth login + echo "Using the commit reference '${commit_ref}' to fetch a Docker Compose bundle..." # First, try to find a commit on GitHub that matches the given reference. From 58ddfd53bf62b1e5bd05485e550e13e415bdebbe Mon Sep 17 00:00:00 2001 From: Jarkko Kaura Date: Wed, 8 Jan 2025 17:26:32 +0200 Subject: [PATCH 10/10] Refactor function call flow in `development.sh`. --- development.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/development.sh b/development.sh index 76b4346..e3691d5 100755 --- a/development.sh +++ b/development.sh @@ -37,7 +37,7 @@ instruct_and_exit() { # # A commit reference can be given as an argument. It can contain, for example, # only a substring of an actual SHA digest. -download_docker_bundle() { +download_docker_compose_bundle() { local commit_ref="${1:-main}" local repo_name="jore4-docker-compose-bundle" @@ -105,19 +105,15 @@ download_docker_bundle() { } start_all() { - download_docker_bundle "$@" $DOCKER_COMPOSE_CMD up -d jore4-hasura jore4-testdb $DOCKER_COMPOSE_CMD up --build -d jore4-timetables-api - prepare_timetables_data_inserter } start_deps() { - download_docker_bundle "$@" # Runs the following services: # jore4-hasura - Hasura. We have to start Hasura because it ensures that db migrations are run to the Jore 4 database. # jore4-testdb - Jore 4 database. This is the database used by the API. $DOCKER_COMPOSE_CMD -f ./docker/docker-compose.test.yml up --build -d jore4-hasura jore4-testdb jore4-hasura-test jore4-testdb-test - prepare_timetables_data_inserter } generate_jooq() { @@ -164,12 +160,16 @@ fi shift if [[ ${COMMAND} == "start" ]]; then - start_all "$@" + download_docker_compose_bundle "$@" + start_all + prepare_timetables_data_inserter exit 0 fi if [[ ${COMMAND} == "start:deps" ]]; then - start_deps "$@" + download_docker_compose_bundle "$@" + start_deps + prepare_timetables_data_inserter exit 0 fi