Skip to content

Commit

Permalink
feat: collect infrastructure cluster logs on failure
Browse files Browse the repository at this point in the history
  • Loading branch information
mrekucci committed Jul 4, 2024
1 parent dbe2d62 commit dc10f41
Show file tree
Hide file tree
Showing 2 changed files with 199 additions and 166 deletions.
332 changes: 166 additions & 166 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,175 +53,175 @@ jobs:
if: github.event_name == 'pull_request'
run: npx commitlint --from ${{ github.event.pull_request.head.sha }}~${{ github.event.pull_request.commits }} --to ${{ github.event.pull_request.head.sha }} --verbose

go-modules:
name: Test and Build Go Modules
runs-on: ubuntu-22.04
timeout-minutes: 60

steps:
- name: Checkout Code
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: recursive

- name: Setup Cache
uses: actions/cache@v4
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: ${{ runner.os }}-go-${{ hashFiles('**/go.work.sum') }}
restore-keys: ${{ runner.os }}-go-

- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: 1.22
check-latest: true
cache-dependency-path: go.work.sum

- name: Determine Workspace Modules
run: |
echo $(go list -f '{{.Dir}}' -m)
echo "GO_LINT_DIRS=$(go list -f '{{.Dir}}/...' -m | grep -v '/external/geth' | tr '\n' ' ')" >> ${GITHUB_ENV}
- name: Run Gofmt
run: go list -f '{{.Dir}}' -m | xargs gofmt -d -e -l

- name: Run Tidy & Workspace Sync
run: |
go list -f '{{.Dir}}' -m | xargs -L1 go mod tidy -C
go work sync
git checkout ${{ github.event.pull_request.head.ref }}
git diff --name-only --exit-code . || (echo "Golang modules/workspace not in sync with go.mod/go.sum/go.work/go.work.sum files" && exit 1)
git reset --hard HEAD
- name: Run Lint
uses: golangci/golangci-lint-action@v4
with:
version: v1.57.2
args: --timeout 15m --verbose ${{ env.GO_LINT_DIRS }}
skip-cache: true # TODO(mrekucci): remove when the following issue is solved https://github.com/golangci/golangci-lint-action/issues/135#issuecomment-2039548548

- name: Run Build
run: go list -f '{{.Dir}}/...' -m | xargs go build

# TODO(mrekucci): Re-enable /external/geth module when tests are passing.
- name: Run Test
run: go list -f '{{.Dir}}/...' -m | grep -v '/external/geth' | xargs go test -short -race

- name: Setup Protobuf
uses: bufbuild/[email protected]

- name: Protobuf Version
run: buf --version

- name: Check Protobuf Parity
run: |
make bufgen
git checkout ${{ github.event.pull_request.head.ref }}
git diff --name-only --exit-code . || (echo "Generated files not in parity with the source files." && exit 1)
git reset --hard HEAD
working-directory: p2p

foundry:
name: Foundry Checks and Reports
runs-on: ubuntu-22.04
timeout-minutes: 30
defaults:
run:
working-directory: contracts

steps:
- name: Checkout Code
uses: actions/checkout@v4
with:
submodules: recursive

- name: Install Foundry
uses: foundry-rs/foundry-toolchain@v1

- name: Print Versions
run: |
git --version
node --version
npm --version
forge --version
- name: Run Tests
run: forge clean && forge test -vvv --via-ir

- name: Run Snapshot
run: forge clean && forge snapshot --via-ir

- name: Run Coverage
run: forge clean && forge coverage --ir-minimum

contracts:
name: Test and Build Contracts Scripts
runs-on: ubuntu-22.04
timeout-minutes: 30
defaults:
run:
working-directory: contracts

strategy:
matrix:
node-version: [ 18.x, 20.x ] # See supported Node.js release schedule at https://nodejs.org/en/about/releases/

steps:
- name: Checkout Code
uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
cache-dependency-path: ./contracts/package-lock.json
cache: npm

- name: Install Foundry
uses: foundry-rs/foundry-toolchain@v1

- name: Install Hardhat
run: npm install -g hardhat

- name: Install Dependencies
run: npm install

- name: Print Versions
run: |
git --version
node --version
npm --version
forge --version
- name: Build
run: npm run build --if-present

- name: Install abigen
run: |
sudo add-apt-repository -y ppa:ethereum/ethereum
sudo apt-get update
sudo apt-get install -y ethereum
abigen --version
- name: Check ABI Parity
run: |
bash script.sh
git checkout ${{ github.event.pull_request.head.ref }}
git diff --name-only --exit-code . || (echo "Generated files not in parity with the source files." && exit 1)
git reset --hard HEAD
working-directory: contracts-abi
# go-modules:
# name: Test and Build Go Modules
# runs-on: ubuntu-22.04
# timeout-minutes: 60
#
# steps:
# - name: Checkout Code
# uses: actions/checkout@v4
# with:
# fetch-depth: 0
# submodules: recursive
#
# - name: Setup Cache
# uses: actions/cache@v4
# with:
# path: |
# ~/go/pkg/mod
# ~/.cache/go-build
# key: ${{ runner.os }}-go-${{ hashFiles('**/go.work.sum') }}
# restore-keys: ${{ runner.os }}-go-
#
# - name: Setup Go
# uses: actions/setup-go@v5
# with:
# go-version: 1.22
# check-latest: true
# cache-dependency-path: go.work.sum
#
# - name: Determine Workspace Modules
# run: |
# echo $(go list -f '{{.Dir}}' -m)
# echo "GO_LINT_DIRS=$(go list -f '{{.Dir}}/...' -m | grep -v '/external/geth' | tr '\n' ' ')" >> ${GITHUB_ENV}
#
# - name: Run Gofmt
# run: go list -f '{{.Dir}}' -m | xargs gofmt -d -e -l
#
# - name: Run Tidy & Workspace Sync
# run: |
# go list -f '{{.Dir}}' -m | xargs -L1 go mod tidy -C
# go work sync
# git checkout ${{ github.event.pull_request.head.ref }}
# git diff --name-only --exit-code . || (echo "Golang modules/workspace not in sync with go.mod/go.sum/go.work/go.work.sum files" && exit 1)
# git reset --hard HEAD
#
# - name: Run Lint
# uses: golangci/golangci-lint-action@v4
# with:
# version: v1.57.2
# args: --timeout 15m --verbose ${{ env.GO_LINT_DIRS }}
# skip-cache: true # TODO(mrekucci): remove when the following issue is solved https://github.com/golangci/golangci-lint-action/issues/135#issuecomment-2039548548
#
# - name: Run Build
# run: go list -f '{{.Dir}}/...' -m | xargs go build
#
# # TODO(mrekucci): Re-enable /external/geth module when tests are passing.
# - name: Run Test
# run: go list -f '{{.Dir}}/...' -m | grep -v '/external/geth' | xargs go test -short -race
#
# - name: Setup Protobuf
# uses: bufbuild/[email protected]
#
# - name: Protobuf Version
# run: buf --version
#
# - name: Check Protobuf Parity
# run: |
# make bufgen
# git checkout ${{ github.event.pull_request.head.ref }}
# git diff --name-only --exit-code . || (echo "Generated files not in parity with the source files." && exit 1)
# git reset --hard HEAD
# working-directory: p2p
#
# foundry:
# name: Foundry Checks and Reports
# runs-on: ubuntu-22.04
# timeout-minutes: 30
# defaults:
# run:
# working-directory: contracts
#
# steps:
# - name: Checkout Code
# uses: actions/checkout@v4
# with:
# submodules: recursive
#
# - name: Install Foundry
# uses: foundry-rs/foundry-toolchain@v1
#
# - name: Print Versions
# run: |
# git --version
# node --version
# npm --version
# forge --version
#
# - name: Run Tests
# run: forge clean && forge test -vvv --via-ir
#
# - name: Run Snapshot
# run: forge clean && forge snapshot --via-ir
#
# - name: Run Coverage
# run: forge clean && forge coverage --ir-minimum
#
# contracts:
# name: Test and Build Contracts Scripts
# runs-on: ubuntu-22.04
# timeout-minutes: 30
# defaults:
# run:
# working-directory: contracts
#
# strategy:
# matrix:
# node-version: [ 18.x, 20.x ] # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
#
# steps:
# - name: Checkout Code
# uses: actions/checkout@v4
# with:
# fetch-depth: 0
#
# - name: Use Node.js ${{ matrix.node-version }}
# uses: actions/setup-node@v4
# with:
# node-version: ${{ matrix.node-version }}
# cache-dependency-path: ./contracts/package-lock.json
# cache: npm
#
# - name: Install Foundry
# uses: foundry-rs/foundry-toolchain@v1
#
# - name: Install Hardhat
# run: npm install -g hardhat
#
# - name: Install Dependencies
# run: npm install
#
# - name: Print Versions
# run: |
# git --version
# node --version
# npm --version
# forge --version
#
# - name: Build
# run: npm run build --if-present
#
# - name: Install abigen
# run: |
# sudo add-apt-repository -y ppa:ethereum/ethereum
# sudo apt-get update
# sudo apt-get install -y ethereum
# abigen --version
#
# - name: Check ABI Parity
# run: |
# bash script.sh
# git checkout ${{ github.event.pull_request.head.ref }}
# git diff --name-only --exit-code . || (echo "Generated files not in parity with the source files." && exit 1)
# git reset --hard HEAD
# working-directory: contracts-abi

infrastructure:
uses: ./.github/workflows/infrastructure.yml
secrets: inherit
needs:
- commitlint
- go-modules
- foundry
- contracts
# - go-modules
# - foundry
# - contracts
33 changes: 33 additions & 0 deletions .github/workflows/infrastructure.yml
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,39 @@ jobs:
)
curl -X POST -H 'Content-type: application/json' --data "${PAYLOAD}" "${{ secrets.SLACK_CI_CHANNEL_WEBHOOK_URL }}"
- name: Collect Cluster Logs
if: ${{ env.IS_MANUAL_DEPLOYMENT == 'false' && failure() }}
run: |
set -x
NOMAD_SERVER="http://${TARGET_MACHINE_IP}:4646"
ALLOC_IDS=$(curl -s $NOMAD_SERVER/v1/allocations | jq -r '.[].ID')
for ALLOC_ID in ${ALLOC_IDS}; do
JOB=$(curl -s ${NOMAD_SERVER}/v1/allocation/${ALLOC_ID} | jq -r '.JobID')
TASKS=$(curl -s ${NOMAD_SERVER}/v1/allocation/${ALLOC_ID} | jq -r '.TaskStates | keys[]')
for TASK in ${TASKS}; do
STDOUT=$(curl -s "${NOMAD_SERVER}/v1/client/fs/logs/${ALLOC_ID}?task=${TASK}&type=stdout")
echo ${STDOUT}
if [ "$(jq -r '.Data' <<< $STDOUT)" != "null" ]; then
echo ${STDOUT} | jq -r '.Data' | base64 -d > "${JOB}_${TASK}_stdout.log"
fi
STDERR=$(curl -s "${NOMAD_SERVER}/v1/client/fs/logs/${ALLOC_ID}?task=${TASK}&type=stderr")
echo ${STDERR}
if [ "$(jq -r '.Data' <<< $STDERR)" != "null" ]; then
echo ${STDERR} | jq -r '.Data' | base64 -d > "${JOB}_${TASK}_stderr.log"
fi
done
done
- name: Upload Cluster Logs
if: ${{ env.IS_MANUAL_DEPLOYMENT == 'false' && failure() }}
uses: actions/upload-artifact@v4
with:
name: cluster-logs
if-no-files-found: error
path: |
*_stdout.log
*_stderr.log
- name: Initialize Debug Shell
if: ${{ env.IS_MANUAL_DEPLOYMENT == 'false' && failure() }}
run: |
Expand Down

0 comments on commit dc10f41

Please sign in to comment.