Skip to content

Commit

Permalink
Merge pull request #20 from arista-netdevops-community/5.9.4.1
Browse files Browse the repository at this point in the history
Release 5.9.4.1
  • Loading branch information
guillaumeVilar authored Mar 14, 2024
2 parents 4582577 + d6b90cc commit 26e5beb
Show file tree
Hide file tree
Showing 12 changed files with 288 additions and 42 deletions.
46 changes: 46 additions & 0 deletions ARISTA-KUBERNETES-MIB.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
ARISTA-KUBERNETES-MIB DEFINITIONS ::= BEGIN

IMPORTS
MODULE-IDENTITY, OBJECT-TYPE, Integer32, experimental FROM SNMPv2-SMI;

k8sEntry MODULE-IDENTITY
LAST-UPDATED "202402270000Z"
ORGANIZATION "Arista Networks"
CONTACT-INFO "[email protected]"

DESCRIPTION "SNMP KUBERNETES MIB"
REVISION "202402270000Z"
DESCRIPTION "SNMP KUBERNETES MIB"
::= { experimental 53 }

kubernetesInfo OBJECT IDENTIFIER ::= { k8sEntry 8 }

nbPodsInRunningState OBJECT-TYPE
SYNTAX Integer32
MAX-ACCESS read-only
STATUS current
DESCRIPTION "Number of Kubernetes pods in Running state."
::= { kubernetesInfo 0 }

nbNodesInReadyState OBJECT-TYPE
SYNTAX Integer32
MAX-ACCESS read-only
STATUS current
DESCRIPTION "Number of Kubernetes nodes in Ready state."
::= { kubernetesInfo 1 }

k8sNodesInfo OBJECT-TYPE
SYNTAX OCTET STRING
MAX-ACCESS read-only
STATUS current
DESCRIPTION "Kubernetes nodes information."
::= { kubernetesInfo 2 }

k8sPodsInfo OBJECT-TYPE
SYNTAX OCTET STRING
MAX-ACCESS read-only
STATUS current
DESCRIPTION "Kubernetes pods information."
::= { kubernetesInfo 3 }

END
44 changes: 44 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
SHELL := /bin/sh
.PHONY: build redeploy-dev redeploy sh

VERSION := 5.9.4.1

build: ## Build the docker image
docker image prune -a -f ;\
docker build -f docker/Dockerfile --no-cache -t net_snmp_image:latest -t net_snmp_image:$(VERSION) . ;\
docker save net_snmp_image -o docker/net_snmp_image ;\
cd docker ;\
tar -czvf ../net_snmp_image.tar.gz net_snmp_image


redeploy-dev: ## Redeploy the snmpd-monitor pod in dev environment (within k8s.io namespace)
kubectl delete -f snmpd-monitor.yaml ;\
kubectl wait --for=delete pod -l app=snmpd-monitor --timeout=60s ;\
nerdctl image --namespace=k8s.io rm net_snmp_image:latest ;\
tar -xf net_snmp_image.tar.gz ;\
nerdctl image --namespace=k8s.io load -i net_snmp_image ;\
nerdctl image ls --namespace=k8s.io | grep net_snmp_image ;\
kubectl apply -f snmpd-monitor.yaml ;\
kubectl wait --for=condition=ready pod -l app=snmpd-monitor --timeout=60s
kubectl get pod -l app=snmpd-monitor

redeploy: ## Redeploy the snmpd-monitor pod in default namespace (works only on single-node CVP cluster)
kubectl delete -f snmpd-monitor.yaml ;\
kubectl wait --for=delete pod -l app=snmpd-monitor --timeout=60s ;\
nerdctl image rm net_snmp_image:latest ;\
tar -xf net_snmp_image.tar.gz ;\
nerdctl image load -i net_snmp_image ;\
nerdctl image ls | grep net_snmp_image ;\
kubectl apply -f snmpd-monitor.yaml ;\
kubectl wait --for=condition=ready pod -l app=snmpd-monitor --timeout=60s
kubectl get pod -l app=snmpd-monitor

test: ## Test the snmpd-monitor pod. This only works with default SNMP credentials
kubectl exec -it $(shell kubectl get pod -l app=snmpd-monitor -o jsonpath='{.items[0].metadata.name}') -- snmpwalk -v2c -c testing 127.0.0.1:161 1.3.6.1.2.1.1.5.0 ;\
kubectl exec -it $(shell kubectl get pod -l app=snmpd-monitor -o jsonpath='{.items[0].metadata.name}') -- snmpwalk -v2c -c testing localhost .1.3.6.1.3.53.8.0 ;\
kubectl exec -it $(shell kubectl get pod -l app=snmpd-monitor -o jsonpath='{.items[0].metadata.name}') -- snmpwalk -v2c -c testing localhost ARISTA-KUBERNETES-MIB::nbNodesInReadyState ;\
kubectl exec -it $(shell kubectl get pod -l app=snmpd-monitor -o jsonpath='{.items[0].metadata.name}') -- snmpwalk -v2c -c testing localhost ARISTA-KUBERNETES-MIB::k8sNodesInfo ;\
kubectl exec -it $(shell kubectl get pod -l app=snmpd-monitor -o jsonpath='{.items[0].metadata.name}') -- snmpwalk -v3 -l authPriv -u arista -a SHA-512 -A 'arista1234' -x AES-256 -X 'arista1234' localhost ARISTA-KUBERNETES-MIB::k8sPodsInfo

sh: ## Open a shell in the snmpd-monitor pod
kubectl exec -it $(shell kubectl get pod -l app=snmpd-monitor -o jsonpath='{.items[0].metadata.name}') -- sh
26 changes: 23 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
The goal of this project is to find a cleaner way to install snmpd packages on CVP: this will allow a _remote management SNMP system to monitor basic CVP server information_ (CPU, memory, disk space, ...).

The following project will install **snmpd** version `5.9` in a Kubernetes pod to make a cleaner and easier-to-maintain solution.
The following project will install **snmpd** version `5.9.4` in a Kubernetes pod to make a cleaner and easier-to-maintain solution.

This snmpd package does support modern cryptographic algorithms (such as `SHA-512` or `AES-256`).

Expand Down Expand Up @@ -55,6 +55,9 @@ rocommunity testing
# For SNMPv3:
createUser arista SHA-512 'arista1234' AES-256 'arista1234'
rouser arista
# pass_persist for kubernetes monitoring
pass_persist .1.3.6.1.3.53.8 /kubernetes.py
```

A complete list of examples is available in command [`man 5 snmpd.examples`](https://linux.die.net/man/5/snmpd.examples)
Expand All @@ -65,7 +68,7 @@ A complete list of examples is available in command [`man 5 snmpd.examples`](htt
- For CVP version `>= 2022.3.0` :

```shell
tar -xf net_snmp_image-v5.9.tar.gz && nerdctl load -i net_snmp_image
tar -xf net_snmp_image.tar.gz && nerdctl load -i net_snmp_image

# Verification:
nerdctl image ls | grep snmp
Expand All @@ -74,7 +77,7 @@ nerdctl image ls | grep snmp
- For older CVP versions, use the following command:

```shell
tar -xf net_snmp_image-v5.9.tar.gz && docker load -i net_snmp_image
tar -xf net_snmp_image.tar.gz && docker load -i net_snmp_image

# Verification:
docker image ls | grep snmp
Expand Down Expand Up @@ -173,6 +176,23 @@ kubectl get pods -l app=snmpd-monitor -o wide
kubectl get daemonset -l app=snmpd-monitor
```

# Kubernetes monitoring

Some OIDs and one MIB file are also available to monitor Kubernetes resources.
The MIB file is available here `ARISTA-KUBERNETES-MIB.txt`.
The 4 following OIDs are available:
* `nbPodsInRunningState`: Number of Kubernetes pods in Running state
* `nbNodesInReadyState`: Number of Kubernetes nodes in Ready state
* `k8sNodesInfo`: Kubernetes nodes info (output of `kubectl get nodes -o wide`)
* `k8sPodsInfo`: Kubernetes pods info (output of `kubectl get pods --all-namespaces`)

Example:
```
# Where 'mibs' is a local directory containing the file (ARISTA-KUBERNETES-MIB.txt)
$ snmpwalk -v2c -M+mibs -c testing 10.83.13.33 ARISTA-KUBERNETES-MIB::nbNodesInReadyState
ARISTA-KUBERNETES-MIB::nbNodesInReadyState = INTEGER: 3
```

# SNMP for CVA appliance

If you would need to monitor the CVA appliance, the above steps will not work as the CVAs are not part of a Kubernetes cluster.
Expand Down
26 changes: 20 additions & 6 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,30 +1,44 @@
# Docker image based on alpine.
# This file is used in order to re-create the image in case we want to upgrade either the base image or the snmp package version.

FROM alpine:3.19.0
FROM alpine:3.19.1

RUN apk --update --no-cache add net-snmp=5.9.4-r0 net-snmp-tools=5.9.4-r0
RUN apk --update --no-cache add \
net-snmp=5.9.4-r0 \
net-snmp-tools=5.9.4-r0 \
kubectl=1.28.4-r1 \
python3=3.11.8-r0 \
py3-pip=23.3.1-r0 \
&& pip3 install snmp_passpersist==2.1.0 --break-system-packages

COPY container-files /


COPY --chmod=0777 docker/container-files/kubernetes.py /kubernetes.py
COPY ARISTA-KUBERNETES-MIB.txt /usr/share/snmp/mibs/ARISTA-KUBERNETES-MIB.txt
COPY docker/container-files/kubeconfig.yaml /root/.kube/config


# Expose the snmp port
EXPOSE 161

# Run snmpd
CMD ["/bootstrap.sh"]

# CMD sleep infinity
CMD snmpd -a -c /etc/snmp/snmpd.conf -f -Lo -C -p /var/run/snmpd.pid

# Commands to build and save the image

# 1. Building image
# docker build --no-cache -t net_snmp_image:latest -t net_snmp_image:5.9.4 .
# docker build --no-cache -t net_snmp_image:latest -t net_snmp_image:5.9.4.1 .

# 2. Tests:
# docker run --name docker-snmp -v $(pwd)/../snmpd.conf:/etc/snmp/snmpd.conf -it -d net_snmp_image:latest
# docker exec -it docker-snmp snmpd --version
# docker exec -it docker-snmp snmpwalk -v2c -c testing 127.0.0.1:161 1.3.6.1.2.1.1.5.0
# docker exec -it docker-snmp snmpwalk -v2c -c testing 127.0.0.1:161 1.3.6.1.2.1.25.1.1
# docker exec -it docker-snmp snmpwalk -v3 -l authPriv -u arista -a SHA-512 -A 'arista1234' -x AES-256 -X 'arista1234' localhost .1.3.6.1.3.53.8.0.2
# docker exec -it docker-snmp snmpwalk -v2c -c testing localhost .1.3.6.1.3.53.8.0
# docker exec -it docker-snmp snmpwalk -v2c -c testing localhost ARISTA-KUBERNETES-MIB::k8sPodsInfo
# docker exec -it docker-snmp snmptranslate -On ARISTA-KUBERNETES-MIB::nbPodsInRunningState

# 3. Saving image as a file
# docker save net_snmp_image -o net_snmp_image
Expand Down
27 changes: 0 additions & 27 deletions docker/container-files/bootstrap.sh

This file was deleted.

19 changes: 19 additions & 0 deletions docker/container-files/kubeconfig.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
apiVersion: v1
clusters:
- cluster:
certificate-authority: /host-root/cvpi/tls/certs/kube-cert.pem
server: https://127.0.0.1:6443
name: local
contexts:
- context:
cluster: local
user: kubelet
name: kubelet-context
current-context: kubelet-context
kind: Config
preferences: {}
users:
- name: kubelet
user:
client-certificate: /host-root/cvpi/tls/certs/kube-cert.pem
client-key: /host-root/cvpi/tls/certs/kube-cert.pem
93 changes: 93 additions & 0 deletions docker/container-files/kubernetes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
#!/usr/bin/python3 -u

import snmp_passpersist as snmp

import subprocess

# Enable logging to a file for debugging purposes
# import logging
# logging.basicConfig(filename='/script_snmp.log', encoding='utf-8', level=logging.DEBUG)
# logging.debug('Init logging for script_snmp.py')

# The position in the tree where the data will be added - SNMPv2-SMI::experimental network-objects
TREE_POSITION_START = ".1.3.6.1.3.53.8"

UPDATE_INTERVAL = 60 # Interval in seconds at which the script will be called


def call_command(command):
"""
call_command Execute the command as a subprocess and return the output decoded in ascii.
Parameters
----------
command : list
The command to call as a list (Ex: ['kubectl', 'get', 'nodes', '-o', 'wide'])
Returns
-------
str
The output of the command decoded in ascii
"""
# logging.debug(f"Calling command: '{command}'")
output = subprocess.check_output(command, stderr=subprocess.DEVNULL)
return output.decode("ascii")


def count_string_in_cmd_output_and_add_at_position(position_in_tree, commands, string_to_count):
"""
count_string_in_cmd_output_and_add_at_position Execute the command and count the number of times the string_to_count is present in the output and add it to the tree.
Parameters
----------
position_in_tree : str
Position in the tree to add the data (Ex: '0.' for the first level of the tree)
commands : list
The command to call as a list (Ex: ['kubectl', 'get', 'nodes', '-o', 'wide']).
string_to_count : str
The string to count in the output of the command
Returns
-------
None
"""
info_from_subprocess = call_command(commands)
# logging.debug(f"Info from subprocess: {info_from_subprocess}")
count = info_from_subprocess.count(string_to_count)
# logging.debug(f"Count: {count}")
position = position_in_tree[:-1] # Remove the last dot in position string
pp.add_int(position, count)


def add_command_in_tree_at_position(position_in_tree, commands):
"""
add_command_in_tree_at_position Execute the command and add it to the pass persist object at position_in_tree.
Parameters
----------
position_in_tree : str
Position in the tree to add the data (Ex: '0.' for the first level of the tree)
commands : list
The command to call as a list (Ex: ['kubectl', 'get', 'nodes', '-o', 'wide']).
Returns
-------
None
"""
info_from_subprocess = call_command(commands)
# Split the string into lines
lines = info_from_subprocess.strip().split('\n')
# logging.debug(f"Lines: {lines}")

# Process each line (excluding the first line of headers)
for index_line, line in enumerate(lines[1:]):
position = position_in_tree + str(index_line)
pp.add_str(position, line)

def update():
# try:

count_string_in_cmd_output_and_add_at_position("0.", ["kubectl", "get", "pods", "--all-namespaces", "--field-selector=status.phase=Running"], "Running")
count_string_in_cmd_output_and_add_at_position("1.", ["kubectl", "get", "nodes"], "Ready")

add_command_in_tree_at_position("2.", ["kubectl", "get", "nodes", "-o", "wide"])
add_command_in_tree_at_position("3.", ["kubectl", "get", "pods", "-o", "wide", "--all-namespaces"])
# except Exception as e:
# logging.error(f"Error: {e}")

pp=snmp.PassPersist(TREE_POSITION_START)
pp.start(update, UPDATE_INTERVAL)
10 changes: 5 additions & 5 deletions load_image_on_boot.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

# This script will load the image in nerdctl and delete / apply the Kubernetes daemonset and service.
# This needs to be configured as a cronjob like the following:
# @reboot /cvpi/cvp-snmp-monitor-with-kubernetes-main/load_image_on_boot.sh >> /cvpi/cvp-snmp-monitor-with-kubernetes-main/cron.log 2>&1
# @reboot /cvpi/cvp-snmp-container-main/load_image_on_boot.sh >> /cvpi/cvp-snmp-container-main/cron.log 2>&1
# The reason we need to do that, is that during a CVP upgrade, all the nerdctl images are removed.
# Thus, at the next boot of the server, post-upgrade, the SNMP pod will fail to start.
# This script will re-load the images at each boot to make sure the container image is present.
Expand All @@ -12,10 +12,10 @@ source /root/.bashrc
echo $(date) - sleep - $(sleep 300)

# Load image with nerdctl (CVP version >= 2022.3.0)
echo $(date) - load image nerdctl - $(nerdctl load -i /cvpi/cvp-snmp-monitor-with-kubernetes-main/net_snmp_image)
echo $(date) - load image nerdctl - $(nerdctl load -i /cvpi/cvp-snmp-container-main/net_snmp_image)
# Load image with docker (CVP version < 2022.3.0)
echo $(date) - load image docker - $(docker load -i /cvpi/cvp-snmp-monitor-with-kubernetes-main/net_snmp_image)
echo $(date) - load image docker - $(docker load -i /cvpi/cvp-snmp-container-main/net_snmp_image)

# Delete and re-apply Kubernetes file
echo $(date) - delete deployment - $(kubectl delete -f /cvpi/cvp-snmp-monitor-with-kubernetes-main/snmpd-monitor.yaml)
echo $(date) - apply deployment - $(kubectl apply -f /cvpi/cvp-snmp-monitor-with-kubernetes-main/snmpd-monitor.yaml)
echo $(date) - delete deployment - $(kubectl delete -f /cvpi/cvp-snmp-container-main/snmpd-monitor.yaml)
echo $(date) - apply deployment - $(kubectl apply -f /cvpi/cvp-snmp-container-main/snmpd-monitor.yaml)
Binary file removed net_snmp_image-v5.9.tar.gz
Binary file not shown.
Binary file added net_snmp_image.tar.gz
Binary file not shown.
Loading

0 comments on commit 26e5beb

Please sign in to comment.