diff --git a/Dockerfile b/Dockerfile index defe028..252faf6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,7 +29,7 @@ RUN \ apk add --no-cache --virtual=build-dependencies \ npm && \ groupmod -g 1000 users && \ - useradd -u 911 -U -d /config -s /bin/false nbxyz && \ + useradd -u 1011 -U -d /config -s /bin/false nbxyz && \ usermod -G users nbxyz && \ mkdir /app \ /config \ @@ -44,7 +44,9 @@ RUN \ /app/ --strip-components=1 && \ npm install --prefix /app && \ apk del --purge build-dependencies && \ - rm -rf /tmp/* + rm -rf /tmp/* && \ + chown -R nbxyz:nbxyz /var/lib/nginx && \ + chown -R nbxyz:nbxyz /var/log/nginx ENV TFTPD_OPTS='' ENV NGINX_PORT='80' @@ -56,5 +58,7 @@ EXPOSE 3000 COPY root/ / -# default command -CMD ["sh","/start.sh"] +# ENTRYPOINT is not overwriteable +ENTRYPOINT ["sh", "/start.sh"] +# default command can be replaced in container-start +#CMD ["sh","/start.sh"] diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml new file mode 100644 index 0000000..bb9d737 --- /dev/null +++ b/kubernetes/deployment.yaml @@ -0,0 +1,99 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pxe-bootserver-ds + namespace: network +spec: + selector: + matchLabels: + app: pxe-bootserver + replicas: 1 + strategy: + #This is not the default + type: Recreate + template: + metadata: + labels: + app: pxe-bootserver + spec: + #securityContext: + #runAsUser: 1011 + #runAsGroup: 1000 + hostname: tftp + subdomain: svc-pxeboot + containers: + - name: pxe-bootserver + image: ghcr.io/netbootxyz/netbootxyz + imagePullPolicy: IfNotPresent + env: + - name: MENU_VERSION + value: "2.0.47" + - name: NGINX_PORT + value: "80" + - name: WEB_APP_PORT + value: "3000" + ports: + - name: tftp + containerPort: 69 + protocol: UDP + - name: nginx-port + containerPort: 80 + - name: web-app-port + containerPort: 3000 + volumeMounts: + - name: pxe-bootserver-conf + mountPath: /etc/supervisor.conf + subPath: supervisor.conf + - name: pxe-bootserver-conf + mountPath: /etc/nginx.conf + subPath: nginx.conf + - name: pxe-bootserver-conf + mountPath: /config/nginx/nginx.conf + subPath: nginx.conf + - mountPath: /config/ + name: config-store + - mountPath: /assets/ + name: data-store + - mountPath: /tmp/ + name: tmp-store + resources: + limits: + memory: 1Gi + requests: + cpu: 100m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - ALL + privileged: true + readOnlyRootFilesystem: true + runAsNonRoot: false + dnsPolicy: ClusterFirst + restartPolicy: Always + securityContext: + capabilities: + add: ["CHOWN", "NET_BIND_SERVICE"] + seccompProfile: + type: RuntimeDefault + serviceAccount: pxebootserver-sa + serviceAccountName: pxebootserver-sa + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + name: pxe-bootserver-conf + items: + - key: supervisor.conf + path: supervisor.conf + - key: nginx.conf + path: nginx.conf + name: pxe-bootserver-conf + - name: config-store + persistentVolumeClaim: + claimName: pxe-bootserver-config-pvc + - name: data-store + persistentVolumeClaim: + claimName: pxe-bootserver-data-pvc + - name: tmp-store + emptyDir: {} diff --git a/kubernetes/pvc-config.yaml b/kubernetes/pvc-config.yaml new file mode 100644 index 0000000..5e3aa46 --- /dev/null +++ b/kubernetes/pvc-config.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pxe-bootserver-config-pvc + namespace: network +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Mi + storageClassName: rook-cephfs + volumeMode: Filesystem diff --git a/kubernetes/pvc.yaml b/kubernetes/pvc.yaml new file mode 100644 index 0000000..4664267 --- /dev/null +++ b/kubernetes/pvc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pxe-bootserver-data-pvc + namespace: network +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + storageClassName: rook-cephfs + volumeMode: Filesystem diff --git a/kubernetes/pxe-bootserver-conf.yaml b/kubernetes/pxe-bootserver-conf.yaml new file mode 100644 index 0000000..0f53e13 --- /dev/null +++ b/kubernetes/pxe-bootserver-conf.yaml @@ -0,0 +1,73 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: pxe-bootserver-conf + namespace: network +data: + supervisor.conf: |+ + [supervisord] + nodaemon=true + user=root + logfile=/config/supervisord.log + + [program:syslog-ng] + command=/usr/sbin/syslog-ng --foreground --no-caps + stdout_syslog=true + stdout_capture_maxbytes=1MB + priority = 1 + + [program:nginx] + command = /usr/sbin/nginx -c /config/nginx/nginx.conf -e /config/log/nginx/error.log + startretries = 2 + daemon=off + priority = 2 + + [program:webapp] + environment=NODE_ENV="production",PORT=%(ENV_WEB_APP_PORT)s + command=/usr/bin/node app.js + user=nbxyz + directory=/app + priority = 3 + + [program:dnsmasq] + command=/usr/sbin/dnsmasq --port=0 --keep-in-foreground --pid-file=/config/tmp/dnsmasq.pid --enable-tftp --user=nbxyz --tftp-secure --tftp-max=15 --tftp-port-range=1025,1039 --tftp-root=/config/menus %(ENV_TFTPD_OPTS)s + stdout_logfile=/config/tftpd.log + redirect_stderr=true + priority = 4 + + [program:messages-log] + command=tail -f /var/log/messages + stdout_logfile=/dev/stdout + stdout_logfile_maxbytes=0 + nginx.conf: |+ + user nbxyz; + worker_processes 4; + pid /config/tmp/nginx.pid; + include /etc/nginx/modules/*.conf; + + events { + worker_connections 1024; + } + + http { + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + client_max_body_size 1m; + include /etc/nginx/mime.types; + default_type application/octet-stream; + access_log /config/log/nginx/access.log; + error_log /config/log/nginx/error.log; + # Temporary directories for kubernetes "readonlyfilesystem" + client_body_temp_path /config/tmp/nginx/client-body; + proxy_temp_path /config/tmp/nginx/proxy; + fastcgi_temp_path /config/tmp/nginx/fastcgi; + uwsgi_temp_path /config/tmp/nginx/uwsgi; + scgi_temp_path /config/tmp/nginx/scgi; + gzip on; + gzip_disable "msie6"; + include /config/nginx/site-confs/*; + } + daemon off; diff --git a/kubernetes/readme.md b/kubernetes/readme.md new file mode 100644 index 0000000..8daeced --- /dev/null +++ b/kubernetes/readme.md @@ -0,0 +1,134 @@ +# Kubernetes Deployment example + +This is an example of one possibilty how to deploy netboot.xyz to a OKD/OpenShift cluster. +You can also use it for a Kubernetes cluster, but in this case you will add your ingress configuration instead of the route. + +In this example we use: + **cluster.local** as the kubernetes cluster domain. + **Please change it to your needs.** + +## Edit PVC config + +First edit both pvc (pvc.yaml and pvc-config.yaml) configs, so that they reference your storage-class. + +## Edit Route config (for OKD/OpenShift) + +Edit the host to your needs. +``` +spec: + host: pxeboot.apps.cluster.local +``` + +## Optional edit PXE-Bootserver-Conf + +Optionaly you can edit the pxe-bootserver-conf.yaml for your needs. + +## Deploy to Kubernetes or OKD/OpenShift + +### create a namespace + +``` +kubectl create ns network +``` + +### deploy + +``` +kubectl -n network apply -f pvc.yaml +kubectl -n network apply -f pvc-config.yaml +kubectl -n network apply -f pxe-bootserver-conf.yaml +kubectl -n network apply -f serviceaccount.yaml +kubectl -n network apply -f deployment.yaml +kubectl -n network apply -f route.yaml +kubectl -n network apply -f service.yaml +``` + +### CNI settings for the TFTP-Server + +Please notice the service configuration for the service: +- svc-pxboot +this is from **type: NodePort**. +This is important, because after a TFTP-client requests a file, +the TFTP-Server will initiate a new connection and send data back to the client over this new connection. +So you must configure your **CNI** **do not** use source nat (**SNAT**) for this connections ! + +#### CNI configs: + +- calico [SNAT-Config](https://docs.tigera.io/calico/latest/networking/configuring/workloads-outside-cluster) +- cilium [SNAT-Config](https://docs.cilium.io/en/stable/network/concepts/masquerading/) + +## Check if netboot.xyz is running + +### Check Deployment, Service and Pod + +``` +kubectl -n network get all + +NAME READY STATUS RESTARTS AGE +pod/pxe-bootserver-ds-5559fd7-4ncjb 1/1 Running 0 2d23h + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +service/svc-pxe-bootserver ClusterIP 10.20.255.66 80/TCP,3000/TCP 15d +service/svc-pxeboot NodePort 10.20.255.51 69:30936/UDP 15d + +NAME READY UP-TO-DATE AVAILABLE AGE +deployment.apps/pxe-bootserver-ds 1/1 1 1 3d7h + +NAME DESIRED CURRENT READY AGE +replicaset.apps/pxe-bootserver-ds-5559fd7 1 1 1 2d23h + +NAME HOST/PORT PATH SERVICES PORT TERMINATION WILDCARD +route.route.openshift.io/pxe-bootserver-route pxeboot.apps.cluster.local svc-pxe-bootserver 3000 edge None +``` + +### Check the logs of the Pod + +``` +kubectl -n network logs pxe-bootserver-ds-5559fd7-4ncjb + + _ _ _ + _ __ ___| |_| |__ ___ ___ | |_ __ ___ _ ____ +| '_ \ / _ \ __| '_ \ / _ \ / _ \| __| \ \/ / | | |_ / +| | | | __/ |_| |_) | (_) | (_) | |_ _ > <| |_| |/ / +|_| |_|\___|\__|_.__/ \___/ \___/ \__(_)_/\_\__, /___| + |___/ +2024-12-03 16:40:33,309 INFO Set uid to user 0 succeeded +2024-12-03 16:40:33,322 CRIT could not write pidfile /supervisord.pid +2024-12-03 16:40:34,326 INFO spawned: 'syslog-ng' with pid 13 +2024-12-03 16:40:34,330 INFO spawned: 'nginx' with pid 14 +2024-12-03 16:40:34,333 INFO spawned: 'webapp' with pid 15 +2024-12-03 16:40:34,336 INFO spawned: 'dnsmasq' with pid 16 +2024-12-03 16:40:34,340 INFO spawned: 'messages-log' with pid 17 +2024-12-03 16:40:34,348 WARN exited: messages-log (exit status 1; not expected) +2024-12-03 16:40:34,367 WARN exited: syslog-ng (exit status 2; not expected) +2024-12-03 16:40:35,714 INFO spawned: 'syslog-ng' with pid 28 +2024-12-03 16:40:35,715 INFO success: nginx entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) +2024-12-03 16:40:35,715 INFO success: webapp entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) +2024-12-03 16:40:35,715 INFO success: dnsmasq entered RUNNING state, process has stayed up for > than 1 seconds (startsecs) +2024-12-03 16:40:35,718 INFO spawned: 'messages-log' with pid 29 +2024-12-03 16:40:35,726 WARN exited: messages-log (exit status 1; not expected) +2024-12-03 16:40:35,754 WARN exited: syslog-ng (exit status 2; not expected) +2024-12-03 16:40:37,760 INFO spawned: 'syslog-ng' with pid 30 +2024-12-03 16:40:37,763 INFO spawned: 'messages-log' with pid 31 +2024-12-03 16:40:37,771 WARN exited: messages-log (exit status 1; not expected) +2024-12-03 16:40:37,802 WARN exited: syslog-ng (exit status 2; not expected) +2024-12-03 16:40:40,938 INFO spawned: 'syslog-ng' with pid 36 +2024-12-03 16:40:40,941 INFO spawned: 'messages-log' with pid 37 +2024-12-03 16:40:40,950 WARN exited: messages-log (exit status 1; not expected) +2024-12-03 16:40:40,950 INFO gave up: messages-log entered FATAL state, too many start retries too quickly +2024-12-03 16:40:40,981 WARN exited: syslog-ng (exit status 2; not expected) +2024-12-03 16:40:41,983 INFO gave up: syslog-ng entered FATAL state, too many start retries too quickly +``` +In the current version we have 2 bugs, which have no impact for the netboot.xyz server. +``` +syslog-ng must be reconfigured or have to be removed +messages-log must be reconfigured or have to be removed +``` + +## DHCP-Options for TFTP + +This deployment will need the following DHCP-Options: +- Option 66 Boot Server Hostname: tftp.svc-pxeboot.network.cluster.local +- Option 67 Bootfile Name: netboot.xyz.efi (for UEFI-Boot) +- Option 67 Bootfile Name: netboot.xyz-undionly.kpxe (for BIOS-Boot) + diff --git a/kubernetes/route.yaml b/kubernetes/route.yaml new file mode 100644 index 0000000..dfa87ce --- /dev/null +++ b/kubernetes/route.yaml @@ -0,0 +1,16 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: pxe-bootserver-route + namespace: network +spec: + host: pxeboot.apps.cluster.local + port: + targetPort: 3000 + tls: + termination: edge + to: + kind: Service + name: svc-pxe-bootserver + weight: 100 + wildcardPolicy: None diff --git a/kubernetes/service.yaml b/kubernetes/service.yaml new file mode 100644 index 0000000..0f4b487 --- /dev/null +++ b/kubernetes/service.yaml @@ -0,0 +1,48 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + name: svc-pxe-bootserver + namespace: network +spec: + #externalTrafficPolicy: Local + #internalTrafficPolicy: Local + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: nginx-port + port: 80 + protocol: TCP + targetPort: 80 + - name: http-admin-port + port: 3000 + protocol: TCP + targetPort: 3000 + selector: + app: pxe-bootserver + sessionAffinity: ClientIP + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: svc-pxeboot + namespace: network +spec: + #externalTrafficPolicy: Local + internalTrafficPolicy: Local + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: tftp-port + port: 69 + protocol: UDP + targetPort: 69 + selector: + app: pxe-bootserver + sessionAffinity: ClientIP + type: NodePort diff --git a/kubernetes/serviceaccount.yaml b/kubernetes/serviceaccount.yaml new file mode 100644 index 0000000..b5f2e6b --- /dev/null +++ b/kubernetes/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pxebootserver-sa + namespace: network diff --git a/root/defaults/nginx.conf b/root/defaults/nginx.conf index ae1d6f0..4ee431e 100644 --- a/root/defaults/nginx.conf +++ b/root/defaults/nginx.conf @@ -4,7 +4,7 @@ pid /run/nginx.pid; include /etc/nginx/modules/*.conf; events { - worker_connections 768; + worker_connections 1024; } http { @@ -18,6 +18,12 @@ http { default_type application/octet-stream; access_log /config/log/nginx/access.log; error_log /config/log/nginx/error.log; + # Temporary directories for kubernetes changed to writeable folders + client_body_temp_path /config/tmp/nginx/client-body; + proxy_temp_path /config/tmp/nginx/proxy; + fastcgi_temp_path /config/tmp/nginx/fastcgi; + uwsgi_temp_path /config/tmp/nginx/uwsgi; + scgi_temp_path /config/tmp/nginx/scgi; gzip on; gzip_disable "msie6"; include /config/nginx/site-confs/*; diff --git a/root/etc/supervisor.conf b/root/etc/supervisor.conf index 142e043..67dec6b 100644 --- a/root/etc/supervisor.conf +++ b/root/etc/supervisor.conf @@ -9,7 +9,7 @@ stdout_capture_maxbytes=1MB priority = 1 [program:nginx] -command = /usr/sbin/nginx -c /config/nginx/nginx.conf +command = /usr/sbin/nginx -c /config/nginx/nginx.conf -e /config/log/nginx/error.log startretries = 2 daemon=off priority = 2 @@ -22,7 +22,7 @@ directory=/app priority = 3 [program:dnsmasq] -command=/usr/sbin/dnsmasq --port=0 --keep-in-foreground --enable-tftp --user=nbxyz --tftp-secure --tftp-root=/config/menus %(ENV_TFTPD_OPTS)s +command=/usr/sbin/dnsmasq --port=0 --keep-in-foreground --pid-file=/config/tmp/dnsmasq.pid --enable-tftp --user=nbxyz --tftp-secure --tftp-max=15 --tftp-port-range=1025,1039 --tftp-root=/config/menus %(ENV_TFTPD_OPTS)s stdout_logfile=/config/tftpd.log redirect_stderr=true priority = 4 diff --git a/root/init.sh b/root/init.sh index 2eec0d7..97dbd9e 100755 --- a/root/init.sh +++ b/root/init.sh @@ -6,8 +6,11 @@ mkdir -p \ /config/nginx/site-confs \ /config/log/nginx \ /run \ - /var/lib/nginx/tmp/client_body \ - /var/tmp/nginx + /config/tmp/nginx/client-body \ + /config/tmp/nginx/proxy \ + /config/tmp/nginx/fastcgi \ + /config/tmp/nginx/uwsgi \ + /config/tmp/nginx/scgi # copy config files [[ ! -f /config/nginx/nginx.conf ]] && \ @@ -17,8 +20,8 @@ mkdir -p \ # Ownership chown -R nbxyz:nbxyz /assets -chown -R nbxyz:nbxyz /var/lib/nginx -chown -R nbxyz:nbxyz /var/log/nginx +#chown -R nbxyz:nbxyz /var/lib/nginx +#chown -R nbxyz:nbxyz /var/log/nginx # create local logs dir mkdir -p \