From 5fa12a24c1fe2ad5750009ca01c343366b476bfa Mon Sep 17 00:00:00 2001
From: punkryn <gns816715@gmail.com>
Date: Mon, 3 Jun 2024 19:28:24 +0900
Subject: [PATCH 1/2] =?UTF-8?q?=EA=B0=80=EC=8B=9C=EC=84=B1=20=ED=99=95?=
 =?UTF-8?q?=EB=B3=B4=EB=A5=BC=20=EC=9C=84=ED=95=9C=20=EC=9E=91=EC=97=85?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 Dockerfile                                    |   11 +-
 build.gradle                                  |    5 +
 docker-compose.observe.yml                    |  154 +++
 etc/dashboards.yaml                           |   10 +
 etc/dashboards/spring-boot-observability.json | 1063 +++++++++++++++++
 etc/grafana/datasource.yml                    |   77 ++
 etc/prometheus/prometheus.yml                 |   46 +
 nginx.observe.conf                            |   38 +
 .../Tiketeer/auth/RequestMatcherHolder.java   |    4 +-
 .../Tiketeer/configuration/AppConfig.java     |    7 +
 10 files changed, 1413 insertions(+), 2 deletions(-)
 create mode 100644 docker-compose.observe.yml
 create mode 100644 etc/dashboards.yaml
 create mode 100644 etc/dashboards/spring-boot-observability.json
 create mode 100644 etc/grafana/datasource.yml
 create mode 100644 etc/prometheus/prometheus.yml
 create mode 100644 nginx.observe.conf

diff --git a/Dockerfile b/Dockerfile
index ae6e17cb..7eace575 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,3 +1,8 @@
+FROM curlimages/curl:8.2.1 AS download
+ARG OTEL_AGENT_VERSION="1.33.2"
+RUN curl --silent --fail -L "https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v${OTEL_AGENT_VERSION}/opentelemetry-javaagent.jar" \
+    -o "$HOME/opentelemetry-javaagent.jar"
+
 FROM eclipse-temurin:21 AS build
 COPY gradlew .
 COPY gradle gradle
@@ -10,4 +15,8 @@ RUN ./gradlew bootJar
 FROM eclipse-temurin:21
 COPY --from=build build/libs/*.jar app.jar
 
-ENTRYPOINT ["java", "-jar", "/app.jar"]
+COPY --from=download /home/curl_user/opentelemetry-javaagent.jar /opentelemetry-javaagent.jar
+ENTRYPOINT ["java", \
+  "-javaagent:/opentelemetry-javaagent.jar", \
+  "-jar", "/app.jar" \
+  ]
\ No newline at end of file
diff --git a/build.gradle b/build.gradle
index 4a697dcd..af2eccbb 100644
--- a/build.gradle
+++ b/build.gradle
@@ -66,6 +66,11 @@ dependencies {
 	// ETC
 	implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.3.0'
 
+	// OBSERVABILITY
+	implementation 'io.micrometer:micrometer-registry-prometheus:1.11.3'
+	implementation 'io.opentelemetry:opentelemetry-api'
+	implementation 'org.springframework.boot:spring-boot-starter-actuator'
+
 	// MAIL
 	implementation 'org.springframework.boot:spring-boot-starter-mail'
 
diff --git a/docker-compose.observe.yml b/docker-compose.observe.yml
new file mode 100644
index 00000000..5ca120cc
--- /dev/null
+++ b/docker-compose.observe.yml
@@ -0,0 +1,154 @@
+x-logging: &default-logging
+  driver: loki
+  options:
+    loki-url: 'http://loki:3100/loki/api/v1/push'
+    loki-pipeline-stages: |
+      - multiline:
+          firstline: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}'
+          max_wait_time: 3s
+      - regex:
+          expression: '^(?P<date>\d{4}-\d{2}-\d{2}) (?P<time>\d{2}:\d{2}:\d{2}.\d{3}) (?P<message>(?s:.*))$$'
+
+version: '3.8'
+
+services:
+  was-b:
+    build: .
+    expose:
+      - 4000
+    depends_on:
+      - mysql
+      - redis
+    restart: unless-stopped
+    networks:
+      - common-network
+    environment:
+      - SPRING_CONFIG_LOCATION=file:///etc/additional.yml
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317
+      - OTEL_SERVICE_NAME=was-b # service name for Span
+      - OTEL_RESOURCE_ATTRIBUTES=compose_service=was-b # add custom resource attribute to Span for Grafana Trace to Logs feature
+      - OTEL_METRICS_EXPORTER=none # disable metrics exporter
+      - MANAGEMENT_METRICS_TAGS_APPLICATION=was-b # override default value of management.metrics.tags.application in app/src/main/resources/application.properties
+    volumes:
+      - ./src/main/resources/application-prod.yml:/etc/additional.yml
+    logging: *default-logging
+
+  nginx:
+    image: nginx:latest
+    restart: unless-stopped
+    ports:
+      - 8888:8888
+    volumes:
+      - ./nginx.observe.conf:/etc/nginx/nginx.conf
+    depends_on:
+      - was-b
+      - was-waiting
+    networks:
+      - common-network
+    logging: *default-logging
+
+  mysql:
+    image: mysql:8.0.33
+    expose:
+      - 3306
+    volumes:
+      - ./db/mysql/data:/var/lib/mysql
+      - ./db/mysql/init:/docker-entrypoint-initdb.d
+    command:
+      - '--character-set-server=utf8mb4'
+      - '--collation-server=utf8mb4_unicode_ci'
+    environment:
+      MYSQL_ROOT_PASSWORD: ${DB_PASSWORD}
+      MYSQL_DATABASE: ${DB_DATABASE}
+    restart: unless-stopped
+    networks:
+      - common-network
+    logging: *default-logging
+
+  redis:
+    image: redis:6.2-alpine
+    expose:
+      - 6379
+    environment:
+      REDIS_PASSWORD: '${REDIS_PASSWORD}'
+    command: redis-server /usr/local/etc/redis/redis.conf --save 20 1 --loglevel warning --requirepass $REDIS_PASSWORD
+    volumes:
+      - ./db/redis/data:/data
+      - ./redis.conf:/usr/local/etc/redis/redis.conf
+    restart: unless-stopped
+    networks:
+      - common-network
+    logging: *default-logging
+
+  was-waiting:
+    image: tiketeer-waiting
+    expose:
+      - 8080
+    depends_on:
+      - mysql
+      - redis
+    restart: unless-stopped
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://tempo:4317 # send traces to Tempo
+      - OTEL_SERVICE_NAME=was-waiting # service name for Span
+      - OTEL_RESOURCE_ATTRIBUTES=compose_service=was-waiting # add custom resource attribute to Span for Grafana Trace to Logs feature
+      - OTEL_METRICS_EXPORTER=none # disable metrics exporter
+      - MANAGEMENT_METRICS_TAGS_APPLICATION=was-waiting # override default value of management.metrics.tags.application in app/src/main/resources/application.properties
+    networks:
+      - common-network
+    logging: *default-logging
+
+  loki:
+    image: grafana/loki:3.0.0
+    command: -config.file=/etc/loki/local-config.yaml
+    ports:
+      - "3100:3100"
+    networks:
+      - common-network
+
+  prometheus:
+    image: prom/prometheus:v2.51.2
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./etc/prometheus:/workspace
+    command:
+      - --config.file=/workspace/prometheus.yml
+      - --enable-feature=exemplar-storage
+    depends_on:
+      - loki
+    logging: *default-logging
+    networks:
+      - common-network
+
+  tempo:
+    image: grafana/tempo:2.4.1
+    command: [ "--target=all", "--storage.trace.backend=local", "--storage.trace.local.path=/var/tempo", "--auth.enabled=false" ]
+    ports:
+      - "14250:14250"
+      - "4317:4317"
+    depends_on:
+      - loki
+    logging: *default-logging
+    networks:
+      - common-network
+
+  grafana:
+    image: grafana/grafana:10.4.2
+    ports:
+      - "5000:3000"
+    volumes:
+      - ./etc/grafana/:/etc/grafana/provisioning/datasources
+      - ./etc/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml
+      - ./etc/dashboards:/etc/grafana/dashboards
+    depends_on:
+      - loki
+      - prometheus
+    logging: *default-logging
+    networks:
+      - common-network
+
+networks:
+  common-network:
+    driver: bridge
+
diff --git a/etc/dashboards.yaml b/etc/dashboards.yaml
new file mode 100644
index 00000000..e27221a8
--- /dev/null
+++ b/etc/dashboards.yaml
@@ -0,0 +1,10 @@
+apiVersion: 1
+providers:
+- name: 'Spring Boot Observability Test'
+  orgId: 1
+  folder: ''
+  type: 'file'
+  disableDeletion: true
+  editable: true
+  options:
+    path: '/etc/grafana/dashboards'
diff --git a/etc/dashboards/spring-boot-observability.json b/etc/dashboards/spring-boot-observability.json
new file mode 100644
index 00000000..64a59655
--- /dev/null
+++ b/etc/dashboards/spring-boot-observability.json
@@ -0,0 +1,1063 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 4,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "sum(http_server_requests_seconds_count{application=\"$app_name\", uri!=\"/actuator/prometheus\"})",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "timeFrom": "24h",
+      "title": "Total Requests",
+      "transformations": [
+        {
+          "id": "seriesToRows",
+          "options": {}
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "field": "Time"
+              }
+            ]
+          }
+        }
+      ],
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 8,
+        "x": 4,
+        "y": 0
+      },
+      "id": 16,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "value_and_name"
+      },
+      "pluginVersion": "10.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "sum by(uri) (http_server_requests_seconds_count{application=\"$app_name\", uri!=\"/actuator/prometheus\"})",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "{{method}} {{uri}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "timeFrom": "24h",
+      "title": "Requests Count",
+      "transformations": [
+        {
+          "id": "seriesToRows",
+          "options": {}
+        },
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": {},
+            "sort": [
+              {
+                "field": "Time"
+              }
+            ]
+          }
+        },
+        {
+          "id": "partitionByValues",
+          "options": {
+            "fields": [
+              "Metric"
+            ]
+          }
+        }
+      ],
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-GrYlRd"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 6,
+      "options": {
+        "displayMode": "lcd",
+        "minVizHeight": 10,
+        "minVizWidth": 0,
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "showUnfilled": true,
+        "valueMode": "color"
+      },
+      "pluginVersion": "10.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "sum by(uri)(http_server_requests_seconds_sum{application=\"$app_name\"}) / sum by(uri)(http_server_requests_seconds_count{application=\"$app_name\"})",
+          "interval": "",
+          "legendFormat": "{{method}} {{uri}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Requests Average Duration",
+      "type": "bargauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 4,
+        "x": 0,
+        "y": 6
+      },
+      "id": 22,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "10.1.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "sum(http_server_requests_seconds_count{application=\"$app_name\", outcome=\"SERVER_ERROR\"})",
+          "interval": "",
+          "legendFormat": "",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "timeFrom": "24h",
+      "title": "Total Exceptions",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMax": 1,
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "area"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 0.8
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 10,
+        "x": 4,
+        "y": 6
+      },
+      "id": 18,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "sum by(uri) (http_server_requests_seconds_count{application=\"$app_name\", status=~\"2.*\", uri!=\"/actuator/prometheus\"}) / sum by(uri) (http_server_requests_seconds_count{application=\"$app_name\", uri!=\"/actuator/prometheus\"})",
+          "interval": "",
+          "legendFormat": "{{path}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Percent of 2xx Requests",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "axisSoftMax": 1,
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "area"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 0.1
+              }
+            ]
+          },
+          "unit": "percentunit"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 10,
+        "x": 14,
+        "y": 6
+      },
+      "id": 20,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "sum by(uri) (http_server_requests_seconds_count{application=\"$app_name\", status=~\"5.*\", uri!=\"/actuator/prometheus\"}) / sum by(uri) (http_server_requests_seconds_count{application=\"$app_name\", uri!=\"/actuator/prometheus\"})",
+          "interval": "",
+          "legendFormat": "{{uri}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Percent of 5xx Requests",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 0,
+        "y": 13
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "histogram_quantile(.99,sum(rate(http_server_requests_seconds_bucket{application=\"$app_name\", uri!=\"/actuator/prometheus\"}[1m])) by(uri, le))",
+          "interval": "",
+          "legendFormat": "{{uri}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "PR 99 Requests Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 13
+      },
+      "id": 23,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "histogram_quantile(.95,sum(rate(http_server_requests_seconds_bucket{application=\"$app_name\", uri!=\"/actuator/prometheus\"}[1m])) by(uri, le))",
+          "interval": "",
+          "legendFormat": "{{uri}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "PR 95 Requests Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 16,
+        "y": 13
+      },
+      "id": 12,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "sum by(uri)(rate(http_server_requests_seconds_count{application=\"$app_name\"}[1m]))",
+          "interval": "",
+          "legendFormat": "{{uri}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Request Per Sec",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "ERROR"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "red",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "INFO"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "green",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "CRITICAL"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "purple",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 6,
+        "x": 0,
+        "y": 21
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.4.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "editorMode": "code",
+          "expr": "sum by(type) (rate({compose_service=~\"was-.*\"} | pattern `<date> <time> <_>=<trace_id> <_>=<span_id> <_>=<trace_flags> <type> <_> --- <msg>` | type != \"\" |= \"$log_keyword\" [1m]))",
+          "legendFormat": "{{type}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Log Type Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 18,
+        "x": 6,
+        "y": 21
+      },
+      "id": 2,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": true,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "pluginVersion": "8.4.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "editorMode": "code",
+          "expr": "{compose_service=~\"was-.*\"} | pattern `<date> <time> <_>=<trace_id> <_>=<span_id> <_>=<trace_flags> <type> <_> --- <msg>` | line_format \"{{.compose_service}}\\t{{.type}}\\ttrace_id={{.trace_id}}\\t{{.msg}}\" |= \"$log_keyword\"",
+          "hide": false,
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Log of All Spring Boot Apps",
+      "type": "logs"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 38,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "was",
+          "value": "was"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "definition": "label_values(application)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Application Name",
+        "multi": false,
+        "name": "app_name",
+        "options": [],
+        "query": {
+          "query": "label_values(application)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "",
+          "value": ""
+        },
+        "description": "query with keyword",
+        "hide": 0,
+        "label": "Log Query",
+        "name": "log_keyword",
+        "options": [
+          {
+            "selected": true,
+            "text": "",
+            "value": ""
+          }
+        ],
+        "query": "",
+        "skipUrlSync": false,
+        "type": "textbox"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Spring Boot Observability Test",
+  "uid": "dLsDQIUnzb",
+  "version": 1,
+  "weekStart": ""
+}
diff --git a/etc/grafana/datasource.yml b/etc/grafana/datasource.yml
new file mode 100644
index 00000000..42e32947
--- /dev/null
+++ b/etc/grafana/datasource.yml
@@ -0,0 +1,77 @@
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+  - name: Prometheus
+    orgId: 1
+
+# list of datasources to insert/update depending
+# what's available in the database
+datasources:
+  # <string, required> name of the datasource. Required
+  - uid: prometheus
+    orgId: 1
+    name: Prometheus
+    type: prometheus
+    typeName: Prometheus
+    access: proxy
+    url: http://prometheus:9090
+    password: ''
+    user: ''
+    database: ''
+    basicAuth: false
+    isDefault: true
+    jsonData:
+      exemplarTraceIdDestinations:
+        - datasourceUid: tempo
+          name: trace_id
+      httpMethod: POST
+    readOnly: false
+    editable: true
+  - uid: tempo
+    orgId: 1
+    name: Tempo
+    type: tempo
+    typeName: Tempo
+    access: proxy
+    url: http://tempo
+    password: ''
+    user: ''
+    database: ''
+    basicAuth: false
+    isDefault: false
+    jsonData:
+      nodeGraph:
+        enabled: true
+      search:
+        hide: false
+      tracesToLogs:
+        datasourceUid: loki
+        filterBySpanID: false
+        filterByTraceID: true
+        mapTagNamesEnabled: false
+        tags:
+          - compose_service
+    readOnly: false
+    editable: true
+  - uid: loki
+    orgId: 1
+    name: Loki
+    type: loki
+    typeName: Loki
+    access: proxy
+    url: http://loki:3100
+    password: ''
+    user: ''
+    database: ''
+    basicAuth: false
+    isDefault: false
+    jsonData:
+      derivedFields:
+        - datasourceUid: tempo
+          matcherRegex: (?:trace_id)=(\w+)
+          name: TraceID
+          url: $${__value.raw}
+    readOnly: false
+    editable: true
diff --git a/etc/prometheus/prometheus.yml b/etc/prometheus/prometheus.yml
new file mode 100644
index 00000000..9b2fc6e8
--- /dev/null
+++ b/etc/prometheus/prometheus.yml
@@ -0,0 +1,46 @@
+global:
+  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+          # - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: "prometheus"
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ["localhost:9090"]
+
+  - job_name: 'was-b'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 5s
+    metrics_path: "/api/actuator/prometheus"
+
+    static_configs:
+      - targets: ['was-b:4000']
+
+  - job_name: 'was-waiting'
+
+    # Override the global default and scrape targets from this job every 5 seconds.
+    scrape_interval: 5s
+    metrics_path: "/actuator/prometheus"
+
+    static_configs:
+      - targets: ['was-waiting:8080']
diff --git a/nginx.observe.conf b/nginx.observe.conf
new file mode 100644
index 00000000..4c8d8367
--- /dev/null
+++ b/nginx.observe.conf
@@ -0,0 +1,38 @@
+events {
+    worker_connections  1024;
+}
+
+http {
+    upstream was {
+        zone upstreams 64K;
+        server was-b:4000 fail_timeout=5s;
+        keepalive 10;
+    }
+
+    upstream waiting {
+        zone upstreams 64K;
+        server was-waiting:8080 fail_timeout=5s;
+        keepalive 10;
+    }
+
+    server {
+        listen 8888;
+        server_name tiketeer;
+
+        location / {
+            return 200 "NginX Health Check";
+        }
+
+        location /api/waiting {
+            proxy_pass http://waiting/waiting;
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+        }
+
+        location /api {
+            proxy_pass http://was/api;
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+        }
+    }
+}
diff --git a/src/main/java/com/tiketeer/Tiketeer/auth/RequestMatcherHolder.java b/src/main/java/com/tiketeer/Tiketeer/auth/RequestMatcherHolder.java
index 6daf69bf..8c97c079 100644
--- a/src/main/java/com/tiketeer/Tiketeer/auth/RequestMatcherHolder.java
+++ b/src/main/java/com/tiketeer/Tiketeer/auth/RequestMatcherHolder.java
@@ -56,7 +56,9 @@ public class RequestMatcherHolder {
 
 		// files
 		new RequestInfo(POST, "/files/**", RoleEnum.SELLER),
-		new RequestInfo(GET, "/files/**", null)
+		new RequestInfo(GET, "/files/**", null),
+
+		new RequestInfo(GET, "/actuator/**", null)
 	);
 	private final ConcurrentHashMap<String, RequestMatcher> reqMatcherCacheMap = new ConcurrentHashMap<>();
 
diff --git a/src/main/java/com/tiketeer/Tiketeer/configuration/AppConfig.java b/src/main/java/com/tiketeer/Tiketeer/configuration/AppConfig.java
index 004b8085..35554e93 100644
--- a/src/main/java/com/tiketeer/Tiketeer/configuration/AppConfig.java
+++ b/src/main/java/com/tiketeer/Tiketeer/configuration/AppConfig.java
@@ -14,6 +14,8 @@
 import com.tiketeer.Tiketeer.domain.purchase.usecase.decorator.CreatePurchaseUseCasePLock;
 import com.tiketeer.Tiketeer.domain.ticket.repository.TicketRepository;
 
+import io.prometheus.client.exemplars.tracer.otel_agent.OpenTelemetryAgentSpanContextSupplier;
+
 @Configuration
 @Profile("!test")
 public class AppConfig {
@@ -36,4 +38,9 @@ public CreatePurchaseUseCase createPurchaseUseCase(CreatePurchaseUseCase createP
 		};
 	}
 
+	@Bean
+	@Profile("prod")
+	public OpenTelemetryAgentSpanContextSupplier openTelemetryAgentSpanContextSupplier() {
+		return new OpenTelemetryAgentSpanContextSupplier();
+	}
 }

From 6b3a44d36a92fc66ef8f9d8910dc953d3a52865a Mon Sep 17 00:00:00 2001
From: punkryn <gns816715@gmail.com>
Date: Thu, 6 Jun 2024 19:04:53 +0900
Subject: [PATCH 2/2] =?UTF-8?q?loki=20plugin=20=EC=A0=9C=EA=B1=B0=20concur?=
 =?UTF-8?q?rency=20policy=20log=20=EC=B6=94=EA=B0=80=20simulate=20script?=
 =?UTF-8?q?=20100=EB=AA=85=EC=94=A9=20=EB=82=98=EB=88=A0=EC=84=9C=20?=
 =?UTF-8?q?=EC=9C=A0=EC=A0=80=20=EC=83=9D=EC=84=B1=ED=95=98=EB=8F=84?=
 =?UTF-8?q?=EB=A1=9D=20=EB=B3=80=EA=B2=BD?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docker-compose.observe.yml                    | 19 ------
 .../Tiketeer/configuration/AppConfig.java     |  4 ++
 stress-test/k6-scripts/simulate.js            | 66 +++++++++++--------
 3 files changed, 44 insertions(+), 45 deletions(-)

diff --git a/docker-compose.observe.yml b/docker-compose.observe.yml
index 5ca120cc..8ff01a04 100644
--- a/docker-compose.observe.yml
+++ b/docker-compose.observe.yml
@@ -1,14 +1,3 @@
-x-logging: &default-logging
-  driver: loki
-  options:
-    loki-url: 'http://loki:3100/loki/api/v1/push'
-    loki-pipeline-stages: |
-      - multiline:
-          firstline: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}'
-          max_wait_time: 3s
-      - regex:
-          expression: '^(?P<date>\d{4}-\d{2}-\d{2}) (?P<time>\d{2}:\d{2}:\d{2}.\d{3}) (?P<message>(?s:.*))$$'
-
 version: '3.8'
 
 services:
@@ -31,7 +20,6 @@ services:
       - MANAGEMENT_METRICS_TAGS_APPLICATION=was-b # override default value of management.metrics.tags.application in app/src/main/resources/application.properties
     volumes:
       - ./src/main/resources/application-prod.yml:/etc/additional.yml
-    logging: *default-logging
 
   nginx:
     image: nginx:latest
@@ -45,7 +33,6 @@ services:
       - was-waiting
     networks:
       - common-network
-    logging: *default-logging
 
   mysql:
     image: mysql:8.0.33
@@ -63,7 +50,6 @@ services:
     restart: unless-stopped
     networks:
       - common-network
-    logging: *default-logging
 
   redis:
     image: redis:6.2-alpine
@@ -78,7 +64,6 @@ services:
     restart: unless-stopped
     networks:
       - common-network
-    logging: *default-logging
 
   was-waiting:
     image: tiketeer-waiting
@@ -96,7 +81,6 @@ services:
       - MANAGEMENT_METRICS_TAGS_APPLICATION=was-waiting # override default value of management.metrics.tags.application in app/src/main/resources/application.properties
     networks:
       - common-network
-    logging: *default-logging
 
   loki:
     image: grafana/loki:3.0.0
@@ -117,7 +101,6 @@ services:
       - --enable-feature=exemplar-storage
     depends_on:
       - loki
-    logging: *default-logging
     networks:
       - common-network
 
@@ -129,7 +112,6 @@ services:
       - "4317:4317"
     depends_on:
       - loki
-    logging: *default-logging
     networks:
       - common-network
 
@@ -144,7 +126,6 @@ services:
     depends_on:
       - loki
       - prometheus
-    logging: *default-logging
     networks:
       - common-network
 
diff --git a/src/main/java/com/tiketeer/Tiketeer/configuration/AppConfig.java b/src/main/java/com/tiketeer/Tiketeer/configuration/AppConfig.java
index 35554e93..79504ca5 100644
--- a/src/main/java/com/tiketeer/Tiketeer/configuration/AppConfig.java
+++ b/src/main/java/com/tiketeer/Tiketeer/configuration/AppConfig.java
@@ -15,9 +15,11 @@
 import com.tiketeer.Tiketeer.domain.ticket.repository.TicketRepository;
 
 import io.prometheus.client.exemplars.tracer.otel_agent.OpenTelemetryAgentSpanContextSupplier;
+import lombok.extern.slf4j.Slf4j;
 
 @Configuration
 @Profile("!test")
+@Slf4j
 public class AppConfig {
 	@Autowired
 	private Environment env;
@@ -31,6 +33,8 @@ public CreatePurchaseUseCase createPurchaseUseCase(CreatePurchaseUseCase createP
 		var concurrencyPolicy = ConcurrencyEnum.fromValue(
 			env.getProperty("custom.policy.ticket-concurrency", String.class, ""));
 
+		log.info("Current concurrency policy is:" + concurrencyPolicy.getValue());
+
 		return switch (concurrencyPolicy) {
 			case OPTIMISTIC_LOCK -> new CreatePurchaseUseCaseOLock(createPurchaseUseCase, ticketRepository);
 			case PESSIMISTIC_LOCK -> new CreatePurchaseUseCasePLock(createPurchaseUseCase, ticketRepository);
diff --git a/stress-test/k6-scripts/simulate.js b/stress-test/k6-scripts/simulate.js
index 627503ab..ce9cafaa 100644
--- a/stress-test/k6-scripts/simulate.js
+++ b/stress-test/k6-scripts/simulate.js
@@ -1,11 +1,11 @@
-import {uuidv4} from "https://jslib.k6.io/k6-utils/1.4.0/index.js";
-import {check, sleep} from "k6";
+import { uuidv4 } from "https://jslib.k6.io/k6-utils/1.4.0/index.js";
+import { check, sleep } from "k6";
 import http from "k6/http";
-import {FormData} from "https://jslib.k6.io/formdata/0.0.2/index.js";
+import { FormData } from "https://jslib.k6.io/formdata/0.0.2/index.js";
 
 const setupVar = {
-    userNum: 35, // 구매자 수 (vUser),
-    ticketStock: 50, // 티켓 재고
+    userNum: 500, // 구매자 수 (vUser),
+    ticketStock: 500, // 티켓 재고
     ticketPrice: 1000,
     setupTimeout: "120s",
     tid: __ENV.TID || "",
@@ -20,7 +20,7 @@ export const options = {
         },
     },
 };
-const commonHeader = {"Content-Type": "application/json"};
+const commonHeader = { "Content-Type": "application/json" };
 
 const domain = "http://host.docker.internal:8888";
 
@@ -41,15 +41,28 @@ export function setup() {
 
     const sellerEmail = `test-seller-${uuidv4()}@test.com`;
 
-    wrapWithTimeLogging("유저 생성", () => {
+    wrapWithTimeLogging("seller 유저 생성", () => {
         createUsers({
             members: [
-                ...userMetadata,
-                {email: sellerEmail, password: commonPwd, role: "SELLER"},
+                { email: sellerEmail, password: commonPwd, role: "SELLER" },
             ],
         });
     });
 
+    for (let i = 0; i < setupVar.userNum / 100; i++) {
+        wrapWithTimeLogging("buyer 유저 생성", () => {
+            createUsers({
+                members: [
+                    ...userMetadata.slice(
+                        100 * i,
+                        Math.min(100 * (i + 1), setupVar.userNum)
+                    ),
+                ],
+            });
+        });
+
+        sleep(3);
+    }
 
     // 티케팅 생성
     const now = new Date();
@@ -57,7 +70,7 @@ export function setup() {
         now.getFullYear(),
         now.getMonth(),
         now.getDate(),
-        now.getHours() + 9,
+        now.getHours(),
         now.getMinutes(),
         now.getSeconds() + 5
     );
@@ -86,19 +99,17 @@ export function setup() {
 
     console.log(`생성완료된 티켓팅 ID: ${ticketingId}`);
 
-
-    return {userMetadata, ticketingId};
+    return { userMetadata, ticketingId };
 }
 
-export default function ({userMetadata, ticketingId}) {
+export default function ({ userMetadata, ticketingId }) {
     const userCounter = __VU - 1;
 
     const singleUserMetadata = userMetadata[userCounter];
-    const {email, password} = singleUserMetadata;
+    const { email, password } = singleUserMetadata;
 
     const accessToken = login(email, password);
 
-
     let token = undefined;
     do {
         const data = getWaiting(ticketingId, accessToken);
@@ -106,19 +117,17 @@ export default function ({userMetadata, ticketingId}) {
         sleep(1);
     } while (token === undefined || token === null);
 
-
-    const sleepDuration = getRandomFloat(15, 20);
+    const sleepDuration = getRandomFloat(3, 5);
     console.log(`VU${userCounter} ${sleepDuration}초 후 구매 시도`);
     sleep(sleepDuration);
     purchase(ticketingId, 1, accessToken, token);
 }
 
-
 function getWaiting(ticketingId, accessToken) {
     const response = http.get(
         domain + "/api/waiting" + `?ticketingId=${ticketingId}`,
         {
-            cookies: {accessToken},
+            cookies: { accessToken },
         }
     );
 
@@ -135,7 +144,7 @@ function getWaiting(ticketingId, accessToken) {
 // returns JSON object for ticketing
 function getTicketingById(ticketingId, accessToken) {
     const response = http.get(domain + "/api/ticketings/" + ticketingId, {
-        cookies: {accessToken},
+        cookies: { accessToken },
     });
 
     check(response, {
@@ -165,7 +174,7 @@ function createUsers(users) {
 function login(email, password) {
     const response = http.post(
         domain + "/api/auth/login",
-        JSON.stringify({email, password}),
+        JSON.stringify({ email, password }),
         {
             headers: commonHeader,
         }
@@ -186,8 +195,10 @@ function purchase(ticketingId, count, accessToken, purchaseToken) {
             count,
         }),
         {
-            headers: Object.assign({}, commonHeader, {"Purchase-Token": purchaseToken}),
-            cookies: {accessToken},
+            headers: Object.assign({}, commonHeader, {
+                "Purchase-Token": purchaseToken,
+            }),
+            cookies: { accessToken },
         }
     );
 
@@ -206,12 +217,15 @@ function jsonToFormData(jsonObject) {
 }
 
 function createTicketing(ticketingMetadata, sellerEmail, sellerPwd) {
+    console.log("sellerEmail: ", sellerEmail);
     const accessToken = login(sellerEmail, sellerPwd);
     const fd = jsonToFormData(ticketingMetadata);
 
     const response = http.post(domain + "/api/ticketings", fd.body(), {
-        headers: {"Content-Type": "multipart/form-data; boundary=" + fd.boundary},
-        cookies: {accessToken},
+        headers: {
+            "Content-Type": "multipart/form-data; boundary=" + fd.boundary,
+        },
+        cookies: { accessToken },
     });
 
     check(response, {
@@ -235,4 +249,4 @@ function wrapWithTimeLogging(tag, callback) {
 
 function getRandomFloat(min, max) {
     return Math.random() * (max - min) + min;
-}
\ No newline at end of file
+}