Skip to content

Commit

Permalink
fix(chart): connection in script of Node startup probe and preStop li…
Browse files Browse the repository at this point in the history
…fecycle

Fixed #2141
Fixed #2157

Signed-off-by: Viet Nguyen Duc <[email protected]>
  • Loading branch information
VietND96 committed Mar 4, 2024
1 parent d56f01c commit ea2ebb0
Show file tree
Hide file tree
Showing 14 changed files with 253 additions and 133 deletions.
6 changes: 5 additions & 1 deletion .github/workflows/nightly.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,11 @@ jobs:
- name: Update tag in docs and files
run: ./update_tag_in_docs_and_files.sh ${LATEST_TAG} ${NEXT_TAG}
- name: Setup environment to build chart
run: make chart_setup_env
uses: nick-invision/retry@master
with:
timeout_minutes: 10
max_attempts: 3
command: CLUSTER=${CLUSTER} HELM_VERSION=${HELM_VERSION} make chart_setup_env
- name: Build and lint charts
run: |
make chart_build_nightly
Expand Down
8 changes: 5 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -461,16 +461,18 @@ chart_test_edge:

chart_test_autoscaling_deployment_https:
CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_INGRESS_HOSTNAME=true CHART_ENABLE_BASIC_AUTH=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 \
SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 \
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
./tests/charts/make/chart_test.sh DeploymentAutoscaling

chart_test_autoscaling_deployment:
CHART_ENABLE_TRACING=true SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_HOST=$$(hostname -i) \
CHART_ENABLE_TRACING=true SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_HOST=$$(hostname -i) RELEASE_NAME=selenium \
SELENIUM_GRID_AUTOSCALING_MIN_REPLICA=1 \
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
./tests/charts/make/chart_test.sh DeploymentAutoscaling

chart_test_autoscaling_job_https:
SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https CHART_ENABLE_BASIC_AUTH=true RELEASE_NAME=selenium SELENIUM_GRID_PORT=443 \
SELENIUM_GRID_TEST_HEADLESS=true SELENIUM_GRID_PROTOCOL=https CHART_ENABLE_BASIC_AUTH=true RELEASE_NAME=selenium SELENIUM_GRID_PORT=443 SUB_PATH=/ \
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
./tests/charts/make/chart_test.sh JobAutoscaling

Expand All @@ -480,7 +482,7 @@ chart_test_autoscaling_job_hostname:
./tests/charts/make/chart_test.sh JobAutoscaling

chart_test_autoscaling_job:
CHART_ENABLE_TRACING=true CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_HOST=selenium-grid.local RELEASE_NAME=selenium \
CHART_ENABLE_TRACING=true CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_INGRESS_HOSTNAME=true SELENIUM_GRID_HOST=selenium-grid.local RELEASE_NAME=selenium SUB_PATH=/ \
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) NAMESPACE=$(NAMESPACE) \
./tests/charts/make/chart_test.sh JobAutoscaling

Expand Down
100 changes: 63 additions & 37 deletions charts/selenium-grid/configs/node/nodePreStop.sh
Original file line number Diff line number Diff line change
@@ -1,13 +1,24 @@
#!/bin/bash

probe_name="lifecycle.${1:-"preStop"}"

max_time=3

ID=$(echo $RANDOM)
tmp_node_file="/tmp/nodeProbe${ID}"

function on_exit() {
rm -rf /tmp/preStopOutput
rm -rf ${tmp_node_file}
}
trap on_exit EXIT

function init_file() {
echo "{}" > ${tmp_node_file}
}
init_file

# Set headers if Node Registration Secret is set
if [ ! -z "${SE_REGISTRATION_SECRET}" ];
then
if [ ! -z "${SE_REGISTRATION_SECRET}" ]; then
HEADERS="X-REGISTRATION-SECRET: ${SE_REGISTRATION_SECRET}"
else
HEADERS="X-REGISTRATION-SECRET;"
Expand All @@ -16,66 +27,81 @@ fi
function is_full_distributed_mode() {
if [ -n "${SE_DISTRIBUTOR_HOST}" ] && [ -n "${SE_DISTRIBUTOR_PORT}" ]; then
DISTRIBUTED_MODE=true
echo "Detected full distributed mode: ${DISTRIBUTED_MODE}. Since SE_DISTRIBUTOR_HOST and SE_DISTRIBUTOR_PORT are set in Node ConfigMap"
echo "$(date +%FT%T%Z) [${probe_name}] - Detected full distributed mode: ${DISTRIBUTED_MODE}. Since SE_DISTRIBUTOR_HOST and SE_DISTRIBUTOR_PORT are set in Node ConfigMap"
else
DISTRIBUTED_MODE=false
echo "Detected full distributed mode: ${DISTRIBUTED_MODE}"
echo "$(date +%FT%T%Z) [${probe_name}] - Detected full distributed mode: ${DISTRIBUTED_MODE}"
fi
}
is_full_distributed_mode

function get_grid_url() {
if [ -z "${SE_HUB_HOST:-$SE_ROUTER_HOST}" ] || [ -z "${SE_HUB_PORT:-$SE_ROUTER_PORT}" ]; then
echo "$(date +%FT%T%Z) [${probe_name}] - There is no configured HUB or ROUTER host. preStop ignores to send drain request to upstream."
grid_url=""
fi
if [ -n "${SE_BASIC_AUTH}" ] && [ "${SE_BASIC_AUTH}" != "*@" ]; then
SE_BASIC_AUTH="${SE_BASIC_AUTH}@"
fi
if [ "${SE_SUB_PATH}" = "/" ]; then
SE_SUB_PATH=""
fi
grid_url=${SE_SERVER_PROTOCOL}://${SE_BASIC_AUTH}${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}${SE_SUB_PATH}
grid_url_checks=$(curl -m ${max_time} -s -o /dev/null -w "%{http_code}" ${grid_url})
if [ "${grid_url_checks}" = "401" ]; then
echo "$(date +%FT%T%Z) [${probe_name}] - Host requires Basic Auth. Please add the credentials to the SE_BASIC_AUTH variable (e.g: user:password). preStop ignores to send drain request to upstream."
grid_url=""
fi
if [ "${grid_url_checks}" = "404" ]; then
echo "$(date +%FT%T%Z) [${probe_name}] - The Grid is not available or it might have /subPath configured. Please wait a moment or check the SE_SUB_PATH variable if needed."
fi
}

function signal_distributor_to_drain_node() {
if [ "${DISTRIBUTED_MODE}" = true ]; then
echo "Signaling Distributor to drain node"
set -x
curl -k -X POST ${SE_SERVER_PROTOCOL}://${SE_DISTRIBUTOR_HOST}:${SE_DISTRIBUTOR_PORT}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}"
set +x
echo "$(date +%FT%T%Z) [${probe_name}] - Signaling Distributor to drain node"
curl -m ${max_time} -k -X POST ${SE_SERVER_PROTOCOL}://${SE_DISTRIBUTOR_HOST}:${SE_DISTRIBUTOR_PORT}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}"
fi
}

function signal_hub_to_drain_node() {
if [ "${DISTRIBUTED_MODE}" = false ]; then
echo "Signaling Hub to drain node"
curl -k -X POST ${SE_GRID_URL}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}"
get_grid_url
if [ -n "${grid_url}" ]; then
echo "$(date +%FT%T%Z) [${probe_name}] - Signaling Hub to drain node"
curl -m ${max_time} -k -X POST ${grid_url}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}"
fi
fi
}

function signal_node_to_drain() {
echo "Signaling Node to drain itself"
curl -k -X POST ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/se/grid/node/drain --header "${HEADERS}"
echo "$(date +%FT%T%Z) [${probe_name}] - Signaling Node to drain itself"
curl -m ${max_time} -k -X POST ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/se/grid/node/drain --header "${HEADERS}"
}

function replace_localhost_by_service_name() {
internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}"
echo "SE_NODE_GRID_URL: ${SE_NODE_GRID_URL}"
if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then
SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}}
elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then
SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}}
elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then
SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}}
else
SE_GRID_URL=${SE_NODE_GRID_URL}
fi
echo "Set SE_GRID_URL internally: ${SE_GRID_URL}"
}
replace_localhost_by_service_name

if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status > /tmp/preStopOutput; then
NODE_ID=$(jq -r '.value.node.nodeId' /tmp/preStopOutput)
if curl -m ${max_time} -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status > ${tmp_node_file}; then
NODE_ID=$(jq -r '.value.node.nodeId' ${tmp_node_file} || "")
if [ -n "${NODE_ID}" ]; then
echo "Current Node ID is: ${NODE_ID}"
signal_hub_to_drain_node
echo "$(date +%FT%T%Z) [${probe_name}] - Current Node ID is: ${NODE_ID}"
signal_distributor_to_drain_node
signal_hub_to_drain_node
echo
fi
signal_node_to_drain
# Wait for the current session to be finished if any
while curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/preStopOutput;
while curl -m ${max_time} -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o ${tmp_node_file};
do
echo "Node preStop is waiting for current session to be finished if any. Node details: message: $(jq -r '.value.message' /tmp/preStopOutput || "unknown"), availability: $(jq -r '.value.node.availability' /tmp/preStopOutput || "unknown")"
sleep 1;
SLOT_HAS_SESSION=$(jq -e ".value.node.slots[]|select(.session != null).id.id" ${tmp_node_file} | tr -d '"' || "")
if [ -z "${SLOT_HAS_SESSION}" ]; then
echo "$(date +%FT%T%Z) [${probe_name}] - There is no session running. Node is ready to be terminated."
echo "$(date +%FT%T%Z) [${probe_name}] - $(cat ${tmp_node_file} || "")"
echo
exit 0
else
echo "$(date +%FT%T%Z) [${probe_name}] - Node preStop is waiting for current session on slot ${SLOT_HAS_SESSION} to be finished. Node details: message: $(jq -r '.value.message' ${tmp_node_file} || "unknown"), availability: $(jq -r '.value.node.availability' ${tmp_node_file} || "unknown")"
sleep 1;
fi
done
else
echo "Node is already drained. Shutting down gracefully!"
echo "$(date +%FT%T%Z) [${probe_name}] - Node is already drained. Shutting down gracefully!"
fi
82 changes: 54 additions & 28 deletions charts/selenium-grid/configs/node/nodeProbe.sh
Original file line number Diff line number Diff line change
@@ -1,53 +1,79 @@
#!/bin/bash

max_time=3
probe_name="Probe.${1:-"Startup"}"

ID=$(echo $RANDOM)
tmp_node_file="/tmp/nodeProbe${ID}"
tmp_grid_file="/tmp/gridProbe${ID}"

function on_exit() {
rm -rf /tmp/nodeProbe${ID}
rm -rf /tmp/gridProbe${ID}
rm -rf ${tmp_node_file}
rm -rf ${tmp_grid_file}
}
trap on_exit EXIT

ID=$(echo $RANDOM)
function init_file() {
echo "{}" > ${tmp_node_file}
echo "{}" > ${tmp_grid_file}
}
init_file

function replace_localhost_by_service_name() {
internal="${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}"
echo "SE_NODE_GRID_URL: ${SE_NODE_GRID_URL}"
if [[ "${SE_NODE_GRID_URL}" == *"/localhost"* ]]; then
SE_GRID_URL=${SE_NODE_GRID_URL//localhost/${internal}}
elif [[ "${SE_NODE_GRID_URL}" == *"/127.0.0.1"* ]]; then
SE_GRID_URL=${SE_NODE_GRID_URL//127.0.0.1/${internal}}
elif [[ "${SE_NODE_GRID_URL}" == *"/0.0.0.0"* ]]; then
SE_GRID_URL=${SE_NODE_GRID_URL//0.0.0.0/${internal}}
else
SE_GRID_URL=${SE_NODE_GRID_URL}
function help_message() {
echo "$(date +%FT%T%Z) [${probe_name}] - If you believe Node is registered successfully but probe still report this message and fail for a long time. Workaround by set 'global.seleniumGrid.defaultNodeStartupProbe' to 'httpGet' and report us an issue for Chart improvement with your scenario."
}

function get_grid_url() {
if [ -z "${SE_HUB_HOST:-$SE_ROUTER_HOST}" ] || [ -z "${SE_HUB_PORT:-$SE_ROUTER_PORT}" ]; then
echo "$(date +%FT%T%Z) [${probe_name}] - There is no configured HUB or ROUTER host. Probe ignores the registration checks on upstream."
exit 0
fi
if [[ -n "${SE_BASIC_AUTH}" && "${SE_BASIC_AUTH}" != *@ ]]; then
SE_BASIC_AUTH="${SE_BASIC_AUTH}@"
fi
if [ "${SE_SUB_PATH}" = "/" ]; then
SE_SUB_PATH=""
fi
grid_url=${SE_SERVER_PROTOCOL}://${SE_BASIC_AUTH}${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}${SE_SUB_PATH}
grid_url_checks=$(curl -m ${max_time} -skf -o /dev/null -w "%{http_code}" ${grid_url})
if [ "${grid_url_checks}" = "401" ]; then
echo "$(date +%FT%T%Z) [${probe_name}] - Host requires Basic Auth. Please add the credentials to the SE_BASIC_AUTH variable (e.g: user:password)."
help_message
exit 1
fi
if [ "${grid_url_checks}" = "404" ]; then
echo "$(date +%FT%T%Z) [${probe_name}] - The Grid is not available or it might have /subPath configured. Please wait a moment or check the SE_SUB_PATH variable if needed."
help_message
exit 1
fi
echo "Set SE_GRID_URL internally: ${SE_GRID_URL}"
}
replace_localhost_by_service_name

if curl -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o /tmp/nodeProbe${ID}; then
NODE_ID=$(jq -r '.value.node.nodeId' /tmp/nodeProbe${ID})
NODE_STATUS=$(jq -r '.value.node.availability' /tmp/nodeProbe${ID})
if curl -m ${max_time} -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE_NODE_PORT}/status -o ${tmp_node_file}; then
NODE_ID=$(jq -r '.value.node.nodeId' ${tmp_node_file} || "")
NODE_STATUS=$(jq -r '.value.node.availability' ${tmp_node_file} || "")
if [ -n "${NODE_ID}" ]; then
echo "Node responds the ID: ${NODE_ID} with status: ${NODE_STATUS}"
echo "$(date +%FT%T%Z) [${probe_name}] - Node responds the ID: ${NODE_ID} with status: ${NODE_STATUS}"
else
echo "Wait for the Node to report its status"
echo "$(date +%FT%T%Z) [${probe_name}] - Wait for the Node to report its status"
exit 1
fi

curl -sfk "${SE_GRID_URL}/status" -o /tmp/gridProbe${ID}
GRID_NODE_ID=$(jq -e ".value.nodes[].id|select(. == \"${NODE_ID}\")" /tmp/gridProbe${ID} | tr -d '"' || true)
get_grid_url

curl -m ${max_time} -sfk "${grid_url}/status" -o ${tmp_grid_file}
GRID_NODE_ID=$(jq -e ".value.nodes[].id|select(. == \"${NODE_ID}\")" ${tmp_grid_file} | tr -d '"' || "")
if [ -n "${GRID_NODE_ID}" ]; then
echo "Grid responds a matched Node ID: ${GRID_NODE_ID}"
echo "$(date +%FT%T%Z) [${probe_name}] - Grid responds a matched Node ID: ${GRID_NODE_ID}"
fi

if [ "${NODE_STATUS}" = "UP" ] && [ -n "${NODE_ID}" ] && [ -n "${GRID_NODE_ID}" ] && [ "${NODE_ID}" = "${GRID_NODE_ID}" ]; then
echo "Node ID: ${NODE_ID} is found in the Grid. The registration is successful."
if [ -n "${NODE_ID}" ] && [ -n "${GRID_NODE_ID}" ] && [ "${NODE_ID}" = "${GRID_NODE_ID}" ]; then
echo "$(date +%FT%T%Z) [${probe_name}] - Node ID: ${NODE_ID} is found in the Grid. Node is ready."
exit 0
else
echo "Node ID: ${NODE_ID} is not found in the Grid. The registration could be in progress."
echo "$(date +%FT%T%Z) [${probe_name}] - Node ID: ${NODE_ID} is not found in the Grid. Node is not ready."
exit 1
fi
else
echo "Wait for the Node to report its status"
echo "$(date +%FT%T%Z) [${probe_name}] - Wait for the Node to report its status"
exit 1
fi
27 changes: 24 additions & 3 deletions charts/selenium-grid/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,14 @@ Check user define custom probe method
{{- $overrideProbe | toYaml -}}
{{- end -}}

{{- define "seleniumGrid.probe.stdout" -}}
{{- $stdout := "" -}}
{{- if .Values.global.seleniumGrid.stdoutProbeLog -}}
{{- $stdout = ">> /proc/1/fd/1" -}}
{{- end -}}
{{- $stdout -}}
{{- end -}}

{{/*
Get probe settings
*/}}
Expand Down Expand Up @@ -155,8 +163,10 @@ Common autoscaling spec template
{{- if not .Values.autoscaling.scaledOptions.triggers }}
triggers:
- type: selenium-grid
metadata:
triggerIndex: '{{ default $.Values.autoscaling.scaledOptions.minReplicaCount (.node.scaledOptions).minReplicaCount }}'
{{- with .node.hpa }}
metadata: {{- tpl (toYaml .) $ | nindent 6 }}
{{- tpl (toYaml .) $ | nindent 6 }}
{{- end }}
{{- end }}
{{- end -}}
Expand Down Expand Up @@ -207,6 +217,14 @@ template:
value: {{ .name | quote }}
- name: SE_NODE_PORT
value: {{ .node.port | quote }}
{{- with .node.startupProbe.timeoutSeconds }}
- name: SE_NODE_REGISTER_PERIOD
value: {{ . | quote }}
{{- end }}
{{- with .node.startupProbe.periodSeconds }}
- name: SE_NODE_REGISTER_CYCLE
value: {{ . | quote }}
{{- end }}
{{- with .node.extraEnvironmentVariables }}
{{- tpl (toYaml .) $ | nindent 10 }}
{{- end }}
Expand Down Expand Up @@ -268,7 +286,7 @@ template:
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
{{- else if eq $.Values.global.seleniumGrid.defaultNodeStartupProbe "exec" }}
exec:
command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodeProbe.sh >> /proc/1/fd/1"]
command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodeProbe.sh Startup {{ include "seleniumGrid.probe.stdout" $ }}"]
{{- else }}
httpGet:
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
Expand Down Expand Up @@ -301,6 +319,9 @@ template:
livenessProbe:
{{- if (ne (include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $)) "{}") }}
{{- include "seleniumGrid.probe.fromUserDefine" (dict "values" . "root" $) | nindent 10 }}
{{- else if eq $.Values.global.seleniumGrid.defaultNodeLivenessProbe "exec" }}
exec:
command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodeProbe.sh Liveness {{ include "seleniumGrid.probe.stdout" $ }}"]
{{- else }}
httpGet:
scheme: {{ default (include "seleniumGrid.probe.httpGet.schema" $) .schema }}
Expand Down Expand Up @@ -547,7 +568,7 @@ Define preStop hook for the node pod. Node preStop script is stored in a ConfigM
{{- define "seleniumGrid.node.deregisterLifecycle" -}}
preStop:
exec:
command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodePreStop.sh >> /proc/1/fd/1"]
command: ["bash", "-c", "{{ $.Values.nodeConfigMap.extraScriptsDirectory }}/nodePreStop.sh {{ include "seleniumGrid.probe.stdout" $ }}"]
{{- end -}}

{{/*
Expand Down
2 changes: 2 additions & 0 deletions charts/selenium-grid/templates/node-configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ data:
SE_HUB_HOST: '{{ include "seleniumGrid.hub.fullname" . }}.{{ .Release.Namespace }}'
SE_HUB_PORT: '{{ .Values.hub.port }}'
{{- end }}
SE_BASIC_AUTH: '{{ template "seleniumGrid.url.basicAuth" $ }}'
SE_SUB_PATH: '{{ template "seleniumGrid.url.subPath" $ }}'
SE_DRAIN_AFTER_SESSION_COUNT: '{{- and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "1" "0" -}}'
SE_NODE_GRID_URL: '{{ include "seleniumGrid.url" .}}'
SE_NODE_GRID_GRAPHQL_URL: '{{ include "seleniumGrid.graphqlURL" . }}'
Expand Down
Loading

0 comments on commit ea2ebb0

Please sign in to comment.