Skip to content

Commit

Permalink
chore(ci): bring back chaos tests (#4456)
Browse files Browse the repository at this point in the history 
* Revert "chore: temporarily disable fuzz chaos tests (#4457)"

This reverts commit f0c953f.

* chore: update config

* Update .github/actions/setup-greptimedb-cluster/with-remote-wal.yaml

Co-authored-by: Ruihang Xia <[email protected]>

---------

Co-authored-by: Ruihang Xia <[email protected]>
  • Loading branch information
@WenyXu @waynexia
WenyXu and waynexia authored Jul 31, 2024
1 parent 80aaa77 commit dd23d47
Show file tree
Hide file tree
Showing 2 changed files with 132 additions and 127 deletions.
5 changes: 5 additions & 0 deletions .github/actions/setup-greptimedb-cluster/with-remote-wal.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,8 @@ objectStorage:
credentials:
accessKeyId: rootuser
secretAccessKey: rootpass123
remoteWal:
enabled: true
kafka:
brokerEndpoints:
- "kafka.kafka-cluster.svc.cluster.local:9092"
254 changes: 127 additions & 127 deletions .github/workflows/develop.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -412,133 +412,133 @@ jobs:
docker stop $(docker ps -a -q)
docker rm $(docker ps -a -q)
docker system prune -f
# FIXME(weny): wait for the chart to release a new version.
# distributed-fuzztest-with-chaos:
# name: Fuzz Test with Chaos (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
# runs-on: ubuntu-latest
# needs: build-greptime-ci
# timeout-minutes: 60
# strategy:
# matrix:
# target: ["fuzz_migrate_mito_regions", "fuzz_failover_mito_regions", "fuzz_failover_metric_regions"]
# mode:
# - name: "Remote WAL"
# minio: true
# kafka: true
# values: "with-remote-wal.yaml"
# steps:
# - uses: actions/checkout@v4
# - name: Setup Kind
# uses: ./.github/actions/setup-kind
# - name: Setup Chaos Mesh
# uses: ./.github/actions/setup-chaos
# - if: matrix.mode.minio
# name: Setup Minio
# uses: ./.github/actions/setup-minio
# - if: matrix.mode.kafka
# name: Setup Kafka cluser
# uses: ./.github/actions/setup-kafka-cluster
# - name: Setup Etcd cluser
# uses: ./.github/actions/setup-etcd-cluster
# # Prepares for fuzz tests
# - uses: arduino/setup-protoc@v3
# with:
# repo-token: ${{ secrets.GITHUB_TOKEN }}
# - uses: dtolnay/rust-toolchain@master
# with:
# toolchain: ${{ env.RUST_TOOLCHAIN }}
# - name: Rust Cache
# uses: Swatinem/rust-cache@v2
# with:
# # Shares across multiple jobs
# shared-key: "fuzz-test-targets"
# - name: Set Rust Fuzz
# shell: bash
# run: |
# sudo apt-get install -y libfuzzer-14-dev
# rustup install nightly
# cargo +nightly install cargo-fuzz cargo-gc-bin
# # Downloads ci image
# - name: Download pre-built binariy
# uses: actions/download-artifact@v4
# with:
# name: bin
# path: .
# - name: Unzip binary
# run: |
# tar -xvf ./bin.tar.gz
# rm ./bin.tar.gz
# - name: Build and push GreptimeDB image
# uses: ./.github/actions/build-and-push-ci-image
# - name: Wait for etcd
# run: |
# kubectl wait \
# --for=condition=Ready \
# pod -l app.kubernetes.io/instance=etcd \
# --timeout=120s \
# -n etcd-cluster
# - if: matrix.mode.minio
# name: Wait for minio
# run: |
# kubectl wait \
# --for=condition=Ready \
# pod -l app=minio \
# --timeout=120s \
# -n minio
# - if: matrix.mode.kafka
# name: Wait for kafka
# run: |
# kubectl wait \
# --for=condition=Ready \
# pod -l app.kubernetes.io/instance=kafka \
# --timeout=120s \
# -n kafka-cluster
# - name: Print etcd info
# shell: bash
# run: kubectl get all --show-labels -n etcd-cluster
# # Setup cluster for test
# - name: Setup GreptimeDB cluster
# uses: ./.github/actions/setup-greptimedb-cluster
# with:
# image-registry: localhost:5001
# values-filename: ${{ matrix.mode.values }}
# enable-region-failover: true
# - name: Port forward (mysql)
# run: |
# kubectl port-forward service/my-greptimedb-frontend 4002:4002 -n my-greptimedb&
# - name: Fuzz Test
# uses: ./.github/actions/fuzz-test
# env:
# CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
# GT_MYSQL_ADDR: 127.0.0.1:4002
# with:
# target: ${{ matrix.target }}
# max-total-time: 120
# - name: Describe Nodes
# if: failure()
# shell: bash
# run: |
# kubectl describe nodes
# - name: Export kind logs
# if: failure()
# shell: bash
# run: |
# kind export logs /tmp/kind
# - name: Upload logs
# if: failure()
# uses: actions/upload-artifact@v4
# with:
# name: fuzz-tests-kind-logs-${{ matrix.mode.name }}-${{ matrix.target }}
# path: /tmp/kind
# retention-days: 3
# - name: Delete cluster
# if: success()
# shell: bash
# run: |
# kind delete cluster
# docker stop $(docker ps -a -q)
# docker rm $(docker ps -a -q)
# docker system prune -f
distributed-fuzztest-with-chaos:
name: Fuzz Test with Chaos (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
runs-on: ubuntu-latest
needs: build-greptime-ci
timeout-minutes: 60
strategy:
matrix:
target: ["fuzz_migrate_mito_regions", "fuzz_failover_mito_regions", "fuzz_failover_metric_regions"]
mode:
- name: "Remote WAL"
minio: true
kafka: true
values: "with-remote-wal.yaml"
steps:
- uses: actions/checkout@v4
- name: Setup Kind
uses: ./.github/actions/setup-kind
- name: Setup Chaos Mesh
uses: ./.github/actions/setup-chaos
- if: matrix.mode.minio
name: Setup Minio
uses: ./.github/actions/setup-minio
- if: matrix.mode.kafka
name: Setup Kafka cluser
uses: ./.github/actions/setup-kafka-cluster
- name: Setup Etcd cluser
uses: ./.github/actions/setup-etcd-cluster
# Prepares for fuzz tests
- uses: arduino/setup-protoc@v3
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.RUST_TOOLCHAIN }}
- name: Rust Cache
uses: Swatinem/rust-cache@v2
with:
# Shares across multiple jobs
shared-key: "fuzz-test-targets"
- name: Set Rust Fuzz
shell: bash
run: |
sudo apt-get install -y libfuzzer-14-dev
rustup install nightly
cargo +nightly install cargo-fuzz cargo-gc-bin
# Downloads ci image
- name: Download pre-built binariy
uses: actions/download-artifact@v4
with:
name: bin
path: .
- name: Unzip binary
run: |
tar -xvf ./bin.tar.gz
rm ./bin.tar.gz
- name: Build and push GreptimeDB image
uses: ./.github/actions/build-and-push-ci-image
- name: Wait for etcd
run: |
kubectl wait \
--for=condition=Ready \
pod -l app.kubernetes.io/instance=etcd \
--timeout=120s \
-n etcd-cluster
- if: matrix.mode.minio
name: Wait for minio
run: |
kubectl wait \
--for=condition=Ready \
pod -l app=minio \
--timeout=120s \
-n minio
- if: matrix.mode.kafka
name: Wait for kafka
run: |
kubectl wait \
--for=condition=Ready \
pod -l app.kubernetes.io/instance=kafka \
--timeout=120s \
-n kafka-cluster
- name: Print etcd info
shell: bash
run: kubectl get all --show-labels -n etcd-cluster
# Setup cluster for test
- name: Setup GreptimeDB cluster
uses: ./.github/actions/setup-greptimedb-cluster
with:
image-registry: localhost:5001
values-filename: ${{ matrix.mode.values }}
enable-region-failover: true
- name: Port forward (mysql)
run: |
kubectl port-forward service/my-greptimedb-frontend 4002:4002 -n my-greptimedb&
- name: Fuzz Test
uses: ./.github/actions/fuzz-test
env:
CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
GT_MYSQL_ADDR: 127.0.0.1:4002
with:
target: ${{ matrix.target }}
max-total-time: 120
- name: Describe Nodes
if: failure()
shell: bash
run: |
kubectl describe nodes
- name: Export kind logs
if: failure()
shell: bash
run: |
kind export logs /tmp/kind
- name: Upload logs
if: failure()
uses: actions/upload-artifact@v4
with:
name: fuzz-tests-kind-logs-${{ matrix.mode.name }}-${{ matrix.target }}
path: /tmp/kind
retention-days: 3
- name: Delete cluster
if: success()
shell: bash
run: |
kind delete cluster
docker stop $(docker ps -a -q)
docker rm $(docker ps -a -q)
docker system prune -f
sqlness:
name: Sqlness Test (${{ matrix.mode.name }})
Expand Down

0 comments on commit dd23d47

Please sign in to comment.