From 579dcf0c05369a191daa232855863bdddf3bbdfa Mon Sep 17 00:00:00 2001 From: Fabrice Jammes Date: Thu, 24 Oct 2024 14:22:24 +0200 Subject: [PATCH] Add option to install prometheus stack --- .ciux | 2 +- doc/monitor.md | 16 ++++++ doc/prod.md | 126 ------------------------------------------ e2e/fink-ci.sh | 9 +-- e2e/prereq-install.sh | 8 ++- e2e/run.sh | 19 ++++++- 6 files changed, 45 insertions(+), 135 deletions(-) create mode 100644 doc/monitor.md delete mode 100644 doc/prod.md diff --git a/.ciux b/.ciux index 995be4ad..1e7af2ce 100644 --- a/.ciux +++ b/.ciux @@ -31,7 +31,7 @@ dependencies: - image: gitlab-registry.in2p3.fr/astrolabsoftware/fink/spark-py:k8s-3.4.1 labels: build: "true" - - package: github.com/k8s-school/ktbx@v1.1.3-rc9 + - package: github.com/k8s-school/ktbx@v1.1.4-rc1 labels: itest: "optional" - package: github.com/astrolabsoftware/finkctl/v3@v3.1.3-rc1 diff --git a/doc/monitor.md b/doc/monitor.md new file mode 100644 index 00000000..bc3774fd --- /dev/null +++ b/doc/monitor.md @@ -0,0 +1,16 @@ +# Monitor fink-broker platform + +```bash +# Run the fink-ci script to install fink-broker from scratch +fink-ci.sh -m -s -b + +# Port forward to grafana +kubectl port-forward $(kubectl get pods --selector=app.kubernetes.io/name=grafana -n monitoring --output=jsonpath="{.items..metadata.name}") -n monitoring 3001:3000 + +# Run ssh tunnel on workstation for remote access +ssh fink_lpc -L 3001:localhost:3001 -N +# Open URL below in a browser +# login: admin +# password: prom-operator +curl http://localhost:3001 +``` diff --git a/doc/prod.md b/doc/prod.md deleted file mode 100644 index 4f2cf874..00000000 --- a/doc/prod.md +++ /dev/null @@ -1,126 +0,0 @@ -# Install fink-broker platform in production - -## Pre-requisites - -Access to a bootstrap server which as access to the production Kubernetes server. -All below command are on launched on the bootstrap server, which is based on a Fedora distribution. - -## Install golang -```shell -GO_VERSION="1.21.5" -sudo rm -rf /usr/local/go -curl -sSL "https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz" | sudo tar -C /usr/local -xz -echo 'export PATH=/usr/local/go/bin:$PATH' >> ~/.bashrc -echo 'export GOPATH="$HOME/go"' >> ~/.bashrc -echo 'export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin' >> ~/.bashrc -``` - -## Install ciux - -`go install github.com/k8s-school/ciux@v0.0.3-rc2` - -## Clone source code -```shell -mkdir -p src/astrolabsoftware -cd src/astrolabsoftware/ -git clone https://github.com/astrolabsoftware/fink-broker.git -cd src/astrolabsoftware/fink-broker -``` - -## Prepare run - -```shell -ciux ignite $PWD -ktbx install kubectl -# Add kubeconfig for production cluster -mkdir .kube -cat > ~/.kube/config -``` - -### Create configuration files - -Configuration file is in `fink-broker/deploy/finkctl.yaml` - - -```shell -ln -s fink-broker/deploy/finkctl.yaml finkctl.yaml -cat finkctl.secret.yaml -s3: - id: "" - secret: "" -distribution: - kafka: - username: "" - # If empty, password is set to "kubectl get -n kafka secrets/fink-producer --template={{.data.password}} | base64 --decode" - # this is used for integration tests and CI which use a local kafka cluster - password: "" -``` - -### PodSecurityPolicies support - -This part apply only for Kubernetes versions 1.24-. - -Check PSP is enabled, but not for `spark` serviceaccount: -```shell -kubectl get psp -Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+ -NAME PRIV CAPS SELINUX RUNASUSER FSGROUP SUPGROUP READONLYROOTFS VOLUMES -magnum.privileged true * RunAsAny RunAsAny RunAsAny RunAsAny false * -kubectl --as=system:serviceaccount:spark:spark -n spark auth can-i use podsecuritypolicy/magnum.privileged -Warning: resource 'podsecuritypolicies' is not namespace scoped in group 'policy' -no -``` - -Grant PSP usage to `spark` serviceaccount: - -```shell -kubectl create ns spark -kubectl create sa -n spark spark -kubectl create -n spark role psp:magnum.privileged --verb=use --resource=podsecuritypolicy --resource-name=magnum.privileged -kubectl create -n spark rolebinding -n spark psp:magnum.privileged --role=psp:magnum.privileged --serviceaccount=spark:spark -``` - -Check PSP is enabled for `spark` serviceaccount: - -```shell -[fedora@fink-int fink-broker]$ kubectl --as=system:serviceaccount:spark:spark -n spark auth can-i use podsecuritypolicy/magnum.privileged -Warning: resource 'podsecuritypolicies' is not namespace scoped in group 'policy' - -yes -``` - -### Run fink-broker - -```shell -# -t parameter: fink will process alert for the current night -cd $HOME/src/astrolabsoftware/fink-broker -./e2e/fink-start.sh -f $PWD -t -``` - -### Install S3 - -```shell -sudo dnf install docker -sudo systemctl enable docker -sudo systemctl start docker -sudo usermod -a -G docker $USER -newgrp docker -docker run --rm -v ~/.aws:/root/.aws -e AWS_ACCESS_KEY_ID="" -e AWS_SECRET_ACCESS_KEY="" -e S3_ENDPOINT_URL=http://ijc-rgw-cloud-3.ijclab.in2p3.fr:8081 peakcom/s5cmd ls s3://finkk8s -``` - -### Web UI access - -```shell -kubectl port-forward -n spark $(kubectl get pods --selector=spark-app-name=stream2raw-py,spark-role=driver -n spark --output=jsonpath="{.items..metadata.name}") 4040 -curl http://localhost:4040 -# Eventually set up a tunnel to workstation -ssh fedora@ -L 4040:localhost:4040 -N -``` - -## Install kafka external services - -This is useful if output kafka servers are not declared inside DNS. - -```shell -kubectl apply -f ./deploy/fink_int/manifests/ -``` \ No newline at end of file diff --git a/e2e/fink-ci.sh b/e2e/fink-ci.sh index a3d4580f..eaf490a4 100755 --- a/e2e/fink-ci.sh +++ b/e2e/fink-ci.sh @@ -27,18 +27,19 @@ Available options: -s Use the science algorithms during the tests -b Branch name to clone (default: master) -Clone Fink-broker source code in a tempory directory and run e2e test -designed to be run as a cron job. +Clone Fink-broker source code in a tempory directory and run e2e test. +Designed to be run as a cron job. EOD } # Get the options -while getopts hcsb: c ; do +while getopts chmsb: c ; do case $c in h) usage ; exit 0 ;; b) branchname=$OPTARG ;; c) cleanup_opt="-c" ;; + m) monitoring_opt="-m" ;; s) science_opt="-s" ;; \?) usage ; exit 2 ;; esac @@ -49,4 +50,4 @@ shift "$((OPTIND-1))" git clone --single-branch $repo_url $repo --branch $branchname # Run fink ci in science mode -$repo/e2e/run.sh $cleanup_opt $science_opt +$repo/e2e/run.sh $cleanup_opt $science_opt $monitoring_opt diff --git a/e2e/prereq-install.sh b/e2e/prereq-install.sh index 81d7ac72..16f5d075 100755 --- a/e2e/prereq-install.sh +++ b/e2e/prereq-install.sh @@ -10,12 +10,14 @@ DIR=$(cd "$(dirname "$0")"; pwd -P) kind_version_opt="" cluster_name=$(ciux get clustername $DIR/..) +monitoring=false # Get kind version from option -k -while getopts k: flag +while getopts mk: flag do case "${flag}" in k) kind_version_opt=--kind-version=${OPTARG};; + m) monitoring=true;; esac done @@ -28,4 +30,8 @@ ink "Install OLM" ktbx install olm ink "Install ArgoCD operator" ktbx install argocd +if [ "$monitoring" = true ]; then + ink "Install prometheus monitoring stack" + ktbx install prometheus +fi diff --git a/e2e/run.sh b/e2e/run.sh index ee58d233..ab7c7cda 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -9,9 +9,13 @@ set -euxo pipefail DIR=$(cd "$(dirname "$0")"; pwd -P) usage () { - echo "Usage: $0 [-s]" + echo "Usage: $0 [-c] [-h] [-m] [-s]" echo " -s: Use the science algorithms during the tests" echo " -c: Cleanup the cluster after the tests" + echo " -m: Install monitoring stack" + echo " -h: Display this help" + echo "" + echo " Run fink-broker e2e tests, using source code from the parent directory." exit 1 } @@ -24,12 +28,13 @@ src_dir=$DIR/.. cleanup=false build=false e2e=false +monitoring=false push=false token="${TOKEN:-}" # Get options for suffix -while getopts hcs opt; do +while getopts hcms opt; do case ${opt} in s ) SUFFIX="" @@ -37,6 +42,9 @@ while getopts hcs opt; do c ) cleanup=true ;; + m ) + monitoring=true + ;; h ) usage exit 0 @@ -105,7 +113,12 @@ echo "Delete the cluster $cluster if it already exists" ktbx delete --name "$cluster" || true echo "Create a Kubernetes cluster (Kind), Install OLM and ArgoCD operators." -$DIR/prereq-install.sh +monitoring_opt="" +if [ $monitoring = true ] +then + monitoring_opt="-m" +fi +$DIR/prereq-install.sh $monitoring_opt . $CIUXCONFIG if [ $CIUX_BUILD = true ]; then