chore(ci): migrate ci to new aws account (#15753)

.github/aws-config/Dockerfile

@@ -4,8 +4,8 @@ CMD bash
 # Commands:
 #   aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/x5u3w5h
 #   docker build -t risingwave-build-env .
-#   docker tag risingwave-build-env:latest public.ecr.aws/x5u3w5h6/risingwave-build-env:latest
-#   docker push public.ecr.aws/x5u3w5h6/risingwave-build-env:latest
+#   docker tag risingwave-build-env:latest public.ecr.aws/w1p7b4n3/risingwave-build-env:latest
+#   docker push public.ecr.aws/w1p7b4n3/risingwave-build-env:latest
 
 # Install Ubuntu packages.
 # Please add packages in alphabetical order.

ci/build-ci-image.sh

@@ -10,9 +10,9 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20240229
+export BUILD_ENV_VERSION=v20240313
 
-export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
+export BUILD_TAG="public.ecr.aws/w1p7b4n3/rw-build-env:${BUILD_ENV_VERSION}"
 
 echo "+++ Arch"
 arch
@@ -26,7 +26,7 @@ fi
 set -e
 
 echo "--- Docker login"
-aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/x5u3w5h6
+aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/w1p7b4n3
 
 echo "--- Check image existence"
 set +e

ci/docker-compose.yml

@@ -71,7 +71,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240229
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240313
     depends_on:
       - mysql
       - db
@@ -81,7 +81,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240229
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240313
     depends_on:
       - mysql
       - db
@@ -98,12 +98,12 @@ services:
 
 
   rw-build-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240229
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240313
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240229
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240313
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -114,7 +114,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240229
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240313
     depends_on:
       db:
         condition: service_healthy

ci/scripts/common.sh

@@ -5,14 +5,14 @@ export RISINGWAVE_CI=true
 export RUST_BACKTRACE=1
 export ENABLE_TELEMETRY=false
 export RUSTC_WRAPPER=sccache
-export SCCACHE_BUCKET=ci-sccache-bucket
+export SCCACHE_BUCKET=rw-ci-sccache-bucket
 export SCCACHE_REGION=us-east-2
 export SCCACHE_IDLE_TIMEOUT=0
 export CARGO_INCREMENTAL=0
 export CARGO_MAKE_PRINT_TIME_SUMMARY=true
-export MINIO_DOWNLOAD_BIN=https://ci-deps-dist.s3.amazonaws.com/minio
-export MCLI_DOWNLOAD_BIN=https://ci-deps-dist.s3.amazonaws.com/mc
-export GCLOUD_DOWNLOAD_TGZ=https://ci-deps-dist.s3.amazonaws.com/google-cloud-cli-406.0.0-linux-x86_64.tar.gz
+export MINIO_DOWNLOAD_BIN=https://rw-ci-deps-dist.s3.amazonaws.com/minio
+export MCLI_DOWNLOAD_BIN=https://rw-ci-deps-dist.s3.amazonaws.com/mc
+export GCLOUD_DOWNLOAD_TGZ=https://rw-ci-deps-dist.s3.amazonaws.com/google-cloud-cli-406.0.0-linux-x86_64.tar.gz
 export NEXTEST_HIDE_PROGRESS_BAR=true
 unset LANG
 if [ -n "${BUILDKITE_COMMIT:-}" ]; then

ci/scripts/e2e-deltalake-sink-rust-test.sh

@@ -38,7 +38,7 @@ sleep 1
 # prepare minio deltalake sink
 echo "--- preparing deltalake"
 .risingwave/bin/mcli -C .risingwave/config/mcli mb hummock-minio/deltalake
-wget https://ci-deps-dist.s3.amazonaws.com/spark-3.3.1-bin-hadoop3.tgz
+wget https://rw-ci-deps-dist.s3.amazonaws.com/spark-3.3.1-bin-hadoop3.tgz
 tar -xf spark-3.3.1-bin-hadoop3.tgz --no-same-owner
 DEPENDENCIES=io.delta:delta-core_2.12:2.2.0,org.apache.hadoop:hadoop-aws:3.3.2
 spark-3.3.1-bin-hadoop3/bin/spark-sql --packages $DEPENDENCIES \

ci/scripts/e2e-iceberg-sink-test.sh

@@ -39,7 +39,7 @@ sleep 1
 # prepare minio iceberg sink
 echo "--- preparing iceberg"
 .risingwave/bin/mcli -C .risingwave/config/mcli mb hummock-minio/iceberg
-wget https://ci-deps-dist.s3.amazonaws.com/spark-3.3.1-bin-hadoop3.tgz
+wget https://rw-ci-deps-dist.s3.amazonaws.com/spark-3.3.1-bin-hadoop3.tgz
 tar -xf spark-3.3.1-bin-hadoop3.tgz --no-same-owner
 DEPENDENCIES=org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.3.1,org.apache.hadoop:hadoop-aws:3.3.2
 spark-3.3.1-bin-hadoop3/bin/spark-sql --packages $DEPENDENCIES \

ci/scripts/release.sh

@@ -19,7 +19,7 @@ echo "--- Install lld"
 # The lld in the CentOS 7 repository is too old and contains a bug that causes a linker error.
 # So we install a newer version here. (17.0.6, latest version at the time of writing)
 # It is manually built in the same environent and uploaded to S3.
-aws s3 cp s3://ci-deps-dist/llvm-lld-manylinux2014_${ARCH}.tar.gz .
+aws s3 cp s3://rw-ci-deps-dist/llvm-lld-manylinux2014_${ARCH}.tar.gz .
 tar -zxvf llvm-lld-manylinux2014_${ARCH}.tar.gz --directory=/usr/local
 ld.lld --version
 
@@ -29,7 +29,7 @@ yum install -y perl-core
 echo "--- Install java and maven"
 yum install -y java-11-openjdk java-11-openjdk-devel wget python3 python3-devel cyrus-sasl-devel
 pip3 install toml-cli
-wget https://ci-deps-dist.s3.amazonaws.com/apache-maven-3.9.3-bin.tar.gz && tar -zxvf apache-maven-3.9.3-bin.tar.gz
+wget https://rw-ci-deps-dist.s3.amazonaws.com/apache-maven-3.9.3-bin.tar.gz && tar -zxvf apache-maven-3.9.3-bin.tar.gz
 export PATH="${REPO_ROOT}/apache-maven-3.9.3/bin:$PATH"
 mvn -v
 
@@ -73,10 +73,10 @@ cd target/release && chmod +x risingwave risectl
 echo "--- Upload nightly binary to s3"
 if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then
   tar -czvf risingwave-"$(date '+%Y%m%d')"-${ARCH}-unknown-linux.tar.gz risingwave
-  aws s3 cp risingwave-"$(date '+%Y%m%d')"-${ARCH}-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary
+  aws s3 cp risingwave-"$(date '+%Y%m%d')"-${ARCH}-unknown-linux.tar.gz s3://rw-nightly-pre-built-binary
 elif [[ -n "${BINARY_NAME+x}" ]]; then
     tar -czvf risingwave-${BINARY_NAME}-${ARCH}-unknown-linux.tar.gz risingwave
-    aws s3 cp risingwave-${BINARY_NAME}-${ARCH}-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary
+    aws s3 cp risingwave-${BINARY_NAME}-${ARCH}-unknown-linux.tar.gz s3://rw-nightly-pre-built-binary
 fi
 
 echo "--- Build connector node"

ci/workflows/main-cron.yml

@@ -1040,6 +1040,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
           environment:
+            - BINARY_NAME
             - GITHUB_TOKEN
             - BUILDKITE_TAG
             - BUILDKITE_SOURCE
@@ -1058,6 +1059,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
           environment:
+            - BINARY_NAME
             - GITHUB_TOKEN
             - BUILDKITE_TAG
             - BUILDKITE_SOURCE

ci/workflows/pull-request.yml

@@ -430,7 +430,7 @@ steps:
             - "v1-cache-{{ id }}-"
           backend: s3
           s3:
-            bucket: ci-cache-bucket
+            bucket: rw-ci-cache-bucket
             args: "--no-progress"
           paths:
             - ".cargo/advisory-db"
@@ -455,7 +455,7 @@ steps:
             - "v1-cache-{{ id }}-"
           backend: s3
           s3:
-            bucket: ci-cache-bucket
+            bucket: rw-ci-cache-bucket
             args: "--no-progress"
           paths:
             - ".cargo/advisory-db"