Merge pull request #1 from canonical/add_perf_testing_tools

Initial commit: Created a ROCK with YCSB

.github/workflows/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,30 @@
+---
+name: Bug report
+about: File a bug report
+labels: bug
+
+---
+
+<!-- Thank you for submitting a bug report! All fields are required unless marked optional. -->
+
+## Steps to reproduce
+1. 
+
+## Expected behavior
+
+
+## Actual behavior
+<!-- If applicable, add screenshots -->
+
+
+## Versions
+
+<!-- Run `lsb_release -sd` -->
+Operating system:
+
+## Log output
+<!-- (Optional) Copy the logs that are relevant to the bug & paste inside triple backticks below -->
+
+
+## Additional context
+<!-- (Optional) Add any additional information here -->

.github/workflows/ci.yaml

@@ -0,0 +1,67 @@
+name: Build 
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  workflow_call:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build Rock
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    outputs:
+      rock-file: ${{ steps.build-snap.outputs.rock }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup LXD
+        uses: canonical/setup-lxd@main
+
+      - name: Install required dependencies
+        run: |
+          # docker
+          sudo snap install docker
+          sudo addgroup --system docker; sudo adduser $USER docker
+          newgrp docker
+          sudo snap disable docker; sudo snap enable docker
+          
+          # skopeo
+          sudo snap install --devmode --channel edge skopeo
+          
+          # rockcraft
+          sudo snap install rockcraft --classic --edge
+          
+          # jq and yq
+          sudo snap install jq yq
+
+      - name: Upgrade linux deps with security updates
+        run: |
+          sudo apt-get update
+          
+          # install security updates
+          sudo apt-get -s dist-upgrade \
+            | grep "^Inst" \
+            | grep -i securi \
+            | awk -F " " {'print $2'} \
+            | xargs sudo apt-get install -y
+          
+          sudo apt-get autoremove -y
+
+      - name: Build rock
+        run: |
+          rockcraft pack --verbose
+
+      - name: Upload built rock job artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: mongodb_benchmarking_oci_rock_amd64
+          path: "mongodb-benchmarking-oci*.rock"
+

.github/workflows/on_push.yaml

@@ -0,0 +1,10 @@
+name: Build
+on:
+  push:
+    branches:
+      - feature/**
+
+jobs:
+  build-run-tests:
+    name: Build and Run Tests
+    uses: ./.github/workflows/ci.yaml

.github/workflows/release.yaml

@@ -0,0 +1,77 @@
+name: Publish to GHCR
+
+env:
+  RELEASE: edge
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  ci-tests:
+    name: Build
+    uses: ./.github/workflows/ci.yaml
+
+  publish:
+    name: publish
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    needs:
+      - ci-tests
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Install required dependencies
+        run: |
+          # docker
+          sudo snap install docker
+          sudo addgroup --system docker; sudo adduser $USER docker
+          newgrp docker
+          sudo snap disable docker; sudo snap enable docker
+
+          # skopeo
+          sudo snap install --devmode --channel edge skopeo
+          
+          # yq
+          sudo snap install yq
+
+      - name: Download rock file
+        uses: actions/download-artifact@v3
+        with:
+          name: mongodb_benchmarking_oci_rock_amd64
+          path: .
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.GHCR_USER }}
+          password: ${{ secrets.GHCR_TOKEN }}
+
+      - name: Publish rock to Store
+        run: |
+          version="$(cat rockcraft.yaml | yq .version)"
+          
+          base="$(cat rockcraft.yaml | yq .base)"
+          base="${base#*:}"
+          
+          # push major version to edge
+          major_tag_version="${version%%.*}-${{ env.RELEASE }}"
+          sudo skopeo \
+              --insecure-policy \
+              copy \
+              oci-archive:mongodb-benchmarking_${version}_amd64.rock \
+              docker-daemon:ghcr.io/canonical/mongodb-benchmarking:${major_tag_version}
+          docker push ghcr.io/canonical/mongodb-benchmarking:${major_tag_version}
+          
+          ### push full version to edge
+          tag_version="${version}-${base}_${{ env.RELEASE }}"
+          echo "Publishing mongodb-benchmarking:${tag_version}"
+          sudo skopeo \
+              --insecure-policy \
+              copy \
+              oci-archive:mongodb-benchmarking_${version}_amd64.rock \
+              docker-daemon:ghcr.io/canonical/mongodb-benchmarking:${tag_version}
+          docker push ghcr.io/canonical/mongodb-benchmarking:${tag_version}

.github/workflows/sync_issues_to_jira.yaml

@@ -0,0 +1,21 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.
+name: Sync issue to Jira
+
+on:
+  issues:
+    types: [opened, reopened, closed]
+
+jobs:
+  sync:
+    name: Sync GitHub issue to Jira
+    uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v2
+    with:
+      jira-base-url: https://warthogs.atlassian.net
+      jira-project-key: DPE
+      jira-component-names: opensearch-k8s,rock
+    secrets:
+      jira-api-token: ${{ secrets.JIRA_API_TOKEN }}
+      jira-user-email: ${{ secrets.JIRA_USER_EMAIL }}
+    permissions:
+      issues: write  # Needed to create GitHub issue comment

README.md

@@ -1,2 +1,7 @@
 # mongodb-benchmarking-oci
 Contains OCI image for perf testing of Charmed MongoDB K8s 
+
+
+# Usage
+
+TBD

mongodb-benchmarking.yaml

@@ -0,0 +1,23 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mongodb-benchmarking 
+  namespace: perf-test
+  labels:
+    app: mongodb-benchmarking
+spec:
+  selector:
+    matchLabels:
+      app: mongodb-benchmarking
+  template:
+    metadata:
+      labels:
+        app: mongodb-benchmarking
+    spec:
+      containers:
+        - name: mongodb-benchmarking
+          image: mongodb-benchmarking:1.0
+

rockcraft.yaml

@@ -0,0 +1,46 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+name: mongodb-benchmarking-oci  # the name of your ROCK
+base: [email protected]  # the base environment for this ROCK
+license: Apache-2.0
+
+version: '1.0.0' # just for humans. Semantic versioning is recommended
+
+summary: 'Performacnce testing for Charmed MongoDB'
+description: |
+  This ROCK contains tools for performace\load testing of
+  Charmed MongoDB. The image contains YCSB and NoSQLBench tools.
+
+platforms: # The platforms this ROCK should be built on and run on
+  amd64:
+
+parts:
+  entry:
+    plugin: dump
+    source: scripts
+    organize:
+      ycsb_load_data.sh: bin/ycsb_load_data.sh
+      ycsb_run_operations.sh: bin/ycsb_run_operations.sh
+
+    stage:
+      - bin/ycsb_load_data.sh
+      - bin/ycsb_run_operations.sh
+
+  setup_ycsb:
+    plugin: nil
+    after: [entry]
+    stage-packages:
+      - openjdk-21-jdk
+      - maven
+      - python2
+    override-prime: |
+      craftctl default
+      mkdir -p $CRAFT_PRIME/var/load-testing/
+    
+      chown -R 584788:584788 $CRAFT_PRIME/var/load-testing/
+      
+      cd $CRAFT_PRIME/var/load-testing
+      curl -O --location https://github.com/brianfrankcooper/YCSB/releases/download/0.17.0/ycsb-0.17.0.tar.gz
+      tar xfvz ycsb-0.17.0.tar.gz
+      chmod +x $CRAFT_PRIME/bin/ycsb_load_data.sh
+      chmod +x $CRAFT_PRIME/bin/ycsb_run_operations.sh

scripts/ycsb_load_data.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# preparation steps
+ln -sf /usr/bin/python2.7 /usr/bin/python
+export PATH="/usr/lib/jvm/java-21-openjdk-amd64/bin/:/usr/share/maven/bin/:$PATH"
+
+# Default values
+RECORD_COUNT_DEFAULT=500000
+LOAD_THREADS_COUNT_DEFAULT=16
+MONGODB_URI_DEFAULT="mongodb://localhost:21017/admin?replicaSet=mongodb"
+WORKLOAD_DEFAULT="workloada"
+
+# Read parameters or use defaults
+WORKLOAD=${1:-$WORKLOAD_DEFAULT}
+RECORD_COUNT=${2:-$RECORD_COUNT_DEFAULT}
+LOAD_THREADS_COUNT=${3:-$LOAD_THREADS_COUNT_DEFAULT}
+MONGODB_URI=${4:-$MONGODB_URI_DEFAULT}
+
+/var/load-testing/ycsb-0.17.0/bin/ycsb \
+    load mongodb -s \
+    -P "/var/load-testing/ycsb-0.17.0/workloads/$WORKLOAD" \
+    -p recordcount=$RECORD_COUNT \
+    -threads $LOAD_THREADS_COUNT \
+    -p mongodb.url="$MONGODB_URI"
+

scripts/ycsb_run_operations.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# preparation steps
+ln -sf /usr/bin/python2.7 /usr/bin/python
+export PATH="/usr/lib/jvm/java-21-openjdk-amd64/bin/:/usr/share/maven/bin/:$PATH"
+
+# Default values
+OPERATIONS_COUNT=1000000
+OPERATIONS_THREADS_COUNT=2
+MONGODB_URI_DEFAULT="mongodb://localhost:21017/admin?replicaSet=mongodb"
+WORKLOAD_DEFAULT="workloada"
+
+# Read parameters or use defaults
+WORKLOAD=${1:-$WORKLOAD_DEFAULT}
+RECORD_COUNT=${2:-$RECORD_COUNT_DEFAULT}
+LOAD_THREADS_COUNT=${3:-$LOAD_THREADS_COUNT_DEFAULT}
+MONGODB_URI=${4:-$MONGODB_URI_DEFAULT}
+
+
+
+/var/load-testing/ycsb-0.17.0/bin/ycsb \
+    run mongodb -s \
+    -P "/var/load-testing/ycsb-0.17.0/workloads/$WORKLOAD" \
+    -p operationcount=$OPERATIONS_COUNT -threads $OPERATIONS_THREADS_COUNT \
+    -p  mongodb.url="$MONGODB_URI"
+