Merge branch 'main' into siyuan/cdc-auto-schema-change-followup

Cargo.toml

@@ -298,6 +298,15 @@ incremental = true
 split-debuginfo = "packed"
 lto = "off"
 
+# Patch profile for production clusters.
+# It will trade-off lto for faster build time.
+[profile.patch-production]
+inherits = "production"
+debug = "full"
+incremental = false
+split-debuginfo = "packed"
+lto = "off"
+
 [profile.production]
 inherits = "release"
 incremental = false

ci/scripts/docker.sh

@@ -6,6 +6,7 @@ set -euo pipefail
 ghcraddr="ghcr.io/risingwavelabs/risingwave"
 dockerhubaddr="risingwavelabs/risingwave"
 arch="$(uname -m)"
+CARGO_PROFILE=${CARGO_PROFILE:-production}
 
 echo "--- ghcr login"
 echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
@@ -23,12 +24,15 @@ fi
 
 # Build RisingWave docker image ${BUILDKITE_COMMIT}-${arch}
 echo "--- docker build and tag"
+echo "CARGO_PROFILE is set to ${CARGO_PROFILE}"
 docker buildx create \
   --name container \
   --driver=docker-container
 
 docker buildx build -f docker/Dockerfile \
-  --build-arg "GIT_SHA=${BUILDKITE_COMMIT}" -t "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" \
+  --build-arg "GIT_SHA=${BUILDKITE_COMMIT}" \
+  --build-arg "CARGO_PROFILE=${CARGO_PROFILE}" \
+  -t "${ghcraddr}:${BUILDKITE_COMMIT}-${arch}" \
   --progress plain \
   --builder=container \
   --load \

ci/scripts/multi-arch-docker.sh

@@ -25,6 +25,8 @@ if [ "${SKIP_TARGET_AARCH64:-false}" != "true" ]; then
   arches+=("aarch64")
 fi
 
+echo "--- arches: ${arches[*]}"
+
 # push images to gchr
 function pushGchr() {
   GHCRTAG="${ghcraddr}:$1"

ci/workflows/docker-arm-fast.yml

@@ -0,0 +1,33 @@
+auto-retry: &auto-retry
+  automatic:
+    # Agent terminated because the AWS EC2 spot instance killed by AWS.
+    - signal_reason: agent_stop
+      limit: 3
+
+steps:
+  - label: "docker-build-push: aarch64"
+    if: build.env("SKIP_TARGET_AARCH64") != "true"
+    command: "CARGO_PROFILE=patch-production ci/scripts/docker.sh"
+    key: "build-aarch64"
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            GHCR_USERNAME: ghcr-username
+            GHCR_TOKEN: ghcr-token
+            DOCKER_TOKEN: docker-token
+            GITHUB_TOKEN: github-token
+    agents:
+      queue: "linux-arm64"
+    retry: *auto-retry
+
+  - label: "multi-arch-image-create-push"
+    command: "SKIP_TARGET_AMD64=true ci/scripts/multi-arch-docker.sh"
+    depends_on:
+      - "build-aarch64"
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            GHCR_USERNAME: ghcr-username
+            GHCR_TOKEN: ghcr-token
+            DOCKER_TOKEN: docker-token
+    retry: *auto-retry

docker/Dockerfile

@@ -63,18 +63,21 @@ FROM rust-base AS rust-builder
 ARG GIT_SHA
 ENV GIT_SHA=$GIT_SHA
 
+ARG CARGO_PROFILE
+ENV CARGO_PROFILE=$CARGO_PROFILE
+
 COPY ./ /risingwave
 WORKDIR /risingwave
 
 ENV ENABLE_BUILD_DASHBOARD=1
 ENV OPENSSL_STATIC=1
 
 RUN cargo fetch && \
-  cargo build -p risingwave_cmd_all --profile production --features "rw-static-link" --features all-udf && \
+  cargo build -p risingwave_cmd_all --profile ${CARGO_PROFILE} --features "rw-static-link" --features all-udf && \
   mkdir -p /risingwave/bin && \
-    mv /risingwave/target/production/risingwave /risingwave/bin/ && \
-    mv /risingwave/target/production/risingwave.dwp /risingwave/bin/ && \
-  cp ./target/production/build/tikv-jemalloc-sys-*/out/build/bin/jeprof /risingwave/bin/ && \
+    mv /risingwave/target/${CARGO_PROFILE}/risingwave /risingwave/bin/ && \
+    mv /risingwave/target/${CARGO_PROFILE}/risingwave.dwp /risingwave/bin/ && \
+  cp ./target/${CARGO_PROFILE}/build/tikv-jemalloc-sys-*/out/build/bin/jeprof /risingwave/bin/ && \
   chmod +x /risingwave/bin/jeprof && \
   mkdir -p /risingwave/lib && cargo clean
 

grafana/risingwave-dev-dashboard.dashboard.py

@@ -570,14 +570,15 @@ def section_compaction(outer_panels):
 
 def section_object_storage(outer_panels):
     panels = outer_panels.sub_panel()
-    operation_rate_blacklist = (
-        "type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read'"
-    )
-    operation_duration_blacklist = "type!~'streaming_upload_write_bytes|streaming_read'"
-    write_op_filter = "type=~'upload|delete'"
-    read_op_filter = "type=~'read|readv|list|metadata'"
-    s3_request_cost_op1 = "type=~'read|streaming_read_start|streaming_read_init'"
-    s3_request_cost_op2 = "type=~'upload|streaming_upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|list'"
+    operation_rate_blacklist = "type!~'streaming_read'"
+    operation_duration_blacklist = "type!~'streaming_read'"
+    put_op_types = ["upload", "streaming_upload"]
+    post_op_types = ["streaming_upload_init", "streaming_upload_finish", "delete_objects"]
+    list_op_types = ["list"]
+    delete_op_types = ["delete"]
+    read_op_types = ["read", "streaming_read_init", "metadata"]
+    write_op_filter = f"type=~'({'|'.join(put_op_types + post_op_types + list_op_types + delete_op_types)})'"
+    read_op_filter = f"type=~'({'|'.join(read_op_types)})'"
     return [
         outer_panels.row_collapsed(
             "Object Storage",
@@ -685,11 +686,11 @@ def section_object_storage(outer_panels):
                             True,
                         ),
                         panels.target(
-                            f"sum({metric('object_store_operation_latency_count', s3_request_cost_op1)}) * 0.0004 / 1000",
+                            f"sum({metric('object_store_operation_latency_count', read_op_filter)}) * 0.0004 / 1000",
                             "GET, SELECT, and all other Requests Cost",
                         ),
                         panels.target(
-                            f"sum({metric('object_store_operation_latency_count', s3_request_cost_op2)}) * 0.005 / 1000",
+                            f"sum({metric('object_store_operation_latency_count', write_op_filter)}) * 0.005 / 1000",
                             "PUT, COPY, POST, LIST Requests Cost",
                         ),
                     ],