Merge pull request #137 from netboxlabs/develop

🚚 release

.github/workflows/server-release.yaml

@@ -95,7 +95,6 @@ jobs:
       fail-fast: false
       matrix:
         app: ${{ fromJSON(needs.setup.outputs.apps) }}
-    if: needs.setup.outputs.app != ''
     with:
       app_name: diode-${{ matrix.app }}
       app_dir: diode-server

README.md

@@ -1,8 +1,21 @@
 # Diode
 
-Diode is a service designed to streamline the process of data ingestion and reconciliation for NetBox users. It aims to lower the barriers to entry for integrating existing network infrastructure data into NetBox and reduce the maintenance efforts required to keep the data up-to-date.
+Diode is a NetBox data ingestion service that greatly simplifies and enhances the process to add and update network data in NetBox, ensuring your network source of truth is always accurate and can be trusted to power your network automation pipelines. Our guiding principle in designing Diode has been to make it as easy as possible to get data into NetBox, removing as much burden as possible from the user while shifting that effort to technology. 
 
-## Usage
+To achieve this, Diode sits in front of NetBox and provides an API purpose built for ingestion of complex network data. Diode eliminates the need to preprocess data to make it conform to the strict object hierarchy imposed by the NetBox data model. This allows data to be sent to NetBox in a more freeform manner, in blocks that are intuitive for network engineers (such as by device or by interface) with much of the related information treated as attributes or properties of these components of interest. Then, Diode takes care of the heavy lifting, automatically transforming the data to align it with NetBox’s structured and comprehensive data model. Diode can even create placeholder objects to compensate for missing information, which means even fragmented information about the network can be captured in NetBox. 
+
+## Project status
+
+The Diode project is currently in the _Public Preview_ stage. Please see [NetBox Labs Product and Feature Lifecycle](https://docs.netboxlabs.com/product_feature_lifecycle/) for more details. We actively welcome feedback to help identify and prioritize bugs, new features and areas of improvement. 
+
+## Get started
+
+Diode runs as a sidecar service to NetBox and can run anywhere with network connectivity to NetBox, whether on the same host or elsewhere. The overall Diode service is delivered through three main components (and a fourth optional component):
+
+1. Diode plugin - see how to [install the Diode plugin](https://github.com/netboxlabs/diode-netbox-plugin)
+2. Diode server - see how to [run the Diode server](https://github.com/netboxlabs/diode/tree/develop/diode-server#readme)
+3. Diode SDK - see how to [install the Diode client SDK](https://github.com/netboxlabs/diode-sdk-python) and [download Diode Python script examples](https://github.com/netboxlabs/netbox-learning/tree/develop/diode)
+4. Diode agent (optional) - see how to [install and run the Diode NAPALM discovery agent](https://github.com/netboxlabs/diode-agent/tree/develop/diode-napalm-agent)
 
 ## Related Projects
 
@@ -15,3 +28,8 @@ Diode is a service designed to streamline the process of data ingestion and reco
 Distributed under the PolyForm Shield License 1.0.0 License. See [LICENSE.md](./LICENSE.md) for more information.
 
 Diode protocol buffers are distributed under the Apache 2.0 License. See [LICENSE.txt](./diode-proto/LICENSE.txt) for more information.
+
+## Required Notice
+
+Copyright NetBox Labs, Inc.
+

diode-server/Makefile

@@ -71,7 +71,7 @@ docker-compose-down:
 	@DIODE_VERSION=$(DIODE_VERSION) COMMIT_SHA=$(COMMIT_SHA) \
 	$(DOCKER_COMPOSE) --env-file docker/sample.env -f docker/docker-compose.yaml down --remove-orphans
 
-docker-compose-netbox-up: docker-all
+docker-compose-netbox-up:
 	$(DOCKER_COMPOSE) -f docker/docker-compose.netbox.yaml up -d --build
 
 docker-compose-netbox-down:

diode-server/README.md

@@ -1,6 +1,17 @@
-# Diode servers
+# Diode server
 
-Diode server is splited into two services:
+The Diode server is a required component of the [Diode](https://github.com/netboxlabs/diode) ingestion service.
+
+Diode is a NetBox ingestion service that greatly simplifies and enhances the process to add and update network data
+in NetBox, ensuring your network source of truth is always accurate and can be trusted to power your network automation
+pipelines.
+
+More information about Diode can be found
+at [https://netboxlabs.com/blog/introducing-diode-streamlining-data-ingestion-in-netbox/](https://netboxlabs.com/blog/introducing-diode-streamlining-data-ingestion-in-netbox/).
+
+## Diode services
+
+Diode server is comprised of two services:
 
 ### Ingester Service
 
@@ -14,3 +25,52 @@ Diode server is splited into two services:
 - Processes data from Redis streams and converts it for storage.
 - Manages data sources and their API keys.
 - Implements a reconciliation engine to detect and store deltas between ingested data and the current NetBox object state.
+
+## Compatibility
+
+The Diode server has been tested with NetBox versions 3.7.2 and above. The Diode server also requires the [Diode NetBox Plugin](https://github.com/netboxlabs/diode-netbox-plugin).
+
+## Running the Diode server
+
+### Requirements
+
+Diode server requires Docker version 27.0.3 or above.
+
+### Installation
+
+Diode requires a configuration file and an environment file to execute successfully:
+
+* `docker-compose.yml` - to configure and run the Diode server containers
+* `.env` - to store the specific environmental settings
+
+We recommend placing both files in a clean directory:
+
+```bash
+mkdir /opt/diode
+cd /opt/diode
+```
+
+Download the default `docker-compose.yml` and `.env` files from this repository:
+
+```bash
+curl -o docker-compose.yml https://raw.githubusercontent.com/netboxlabs/diode/develop/diode-server/docker/docker-compose.yaml
+curl -o .env https://raw.githubusercontent.com/netboxlabs/diode/develop/diode-server/docker/sample.env
+```
+
+Edit the `.env` to match your environment:
+* `NETBOX_DIODE_PLUGIN_API_BASE_URL`: URL for the Diode NetBox plugin API
+* `DIODE_TO_NETBOX_API_KEY`: API key generated with the Diode NetBox plugin installation
+* `INGESTION_API_KEY`: API key generated with the Diode NetBox plugin installation
+* `NETBOX_TO_DIODE_API_KEY`: API key generated with the Diode NetBox plugin installation
+
+### Running the Diode server
+
+Start the Diode server:
+
+```bash
+docker compose -f docker-compose.yaml up -d
+```
+
+## License
+
+Distributed under the PolyForm Shield License 1.0.0 License. See [LICENSE.md](./LICENSE.md) for more information.

diode-server/docker/docker-compose.yaml

@@ -7,46 +7,36 @@ services:
         server diode-ingester:8081;
       }
 
-      upstream netbox {
-        server netbox:8080;
-      }
-
       server {
-        listen 80;
+        listen ${DIODE_NGINX_PORT};
         http2 on;
         server_name localhost;
+        client_max_body_size 25m;
         location /diode {
-          rewrite /diode/(.*) /$1 break;
+          rewrite /diode/(.*) /$$1 break;
           grpc_pass grpc://diode;
         }
-        location /netbox/static/ {
-          proxy_pass http://netbox/static/;
-        }
-        location /netbox/ {
-          proxy_pass http://netbox;
-          proxy_set_header X-Forwarded-Host $$http_host;
-          proxy_set_header X-Real-IP $$remote_addr;
-          proxy_set_header X-Forwarded-Proto $$scheme;
-        }
       }'
       > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"
     restart: always
+    environment:
+      - DIODE_NGINX_PORT=${DIODE_NGINX_PORT}
     ports:
-      - "80:80"
+      - ${DIODE_NGINX_PORT}:80
     depends_on:
       - diode-ingester
       - diode-reconciler
 
   diode-ingester:
-    image: netboxlabs/diode-ingester:${DIODE_VERSION}-${COMMIT_SHA}
+    image: netboxlabs/diode-ingester:latest
     environment:
-    - API_KEY=${RECONCILER_API_KEY}
-    - REDIS_PASSWORD=${REDIS_PASSWORD}
-    - REDIS_HOST=${REDIS_HOST}
-    - REDIS_PORT=${REDIS_PORT}
-    - RECONCILER_GRPC_HOST=${RECONCILER_GRPC_HOST}
-    - RECONCILER_GRPC_PORT=${RECONCILER_GRPC_PORT}
-    - SENTRY_DSN=${SENTRY_DSN}
+      - API_KEY=${RECONCILER_API_KEY}
+      - REDIS_PASSWORD=${REDIS_PASSWORD}
+      - REDIS_HOST=${REDIS_HOST}
+      - REDIS_PORT=${REDIS_PORT}
+      - RECONCILER_GRPC_HOST=${RECONCILER_GRPC_HOST}
+      - RECONCILER_GRPC_PORT=${RECONCILER_GRPC_PORT}
+      - SENTRY_DSN=${SENTRY_DSN}
     restart: always
     ports:
       - "8081:8081"
@@ -55,7 +45,7 @@ services:
       - diode-reconciler
 
   diode-reconciler:
-    image: netboxlabs/diode-reconciler:${DIODE_VERSION}-${COMMIT_SHA}
+    image: netboxlabs/diode-reconciler:latest
     environment:
       - REDIS_PASSWORD=${REDIS_PASSWORD}
       - REDIS_HOST=${REDIS_HOST}
@@ -64,36 +54,29 @@ services:
       - DIODE_TO_NETBOX_API_KEY=${DIODE_TO_NETBOX_API_KEY}
       - NETBOX_TO_DIODE_API_KEY=${NETBOX_TO_DIODE_API_KEY}
       - INGESTION_API_KEY=${INGESTION_API_KEY}
-      - NETBOX_API_URL=${NETBOX_API_URL}
       - LOGGING_LEVEL=${LOGGING_LEVEL}
       - SENTRY_DSN=${SENTRY_DSN}
     restart: always
-    ports:
-      - "8082:8081"
+    ports: [ ]
     depends_on:
       - diode-redis
   diode-redis:
     image: redis/redis-stack-server:latest
     command:
       - sh
       - -c
-      - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD --loadmodule /opt/redis-stack/lib/rejson.so --loadmodule /opt/redis-stack/lib/redisearch.so
+      - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD --loadmodule /opt/redis-stack/lib/rejson.so --loadmodule /opt/redis-stack/lib/redisearch.so --port $$REDIS_PORT
     environment:
       - REDIS_PASSWORD=${REDIS_PASSWORD}
-    ports:
-      - "6379:6379"
+      - REDIS_PORT=${REDIS_PORT}
+    ports: [ ]
     volumes:
       - diode-redis-data:/data
   diode-redis-cli:
     image: redis/redis-stack-server:latest
     links:
       - diode-redis
-    command: redis-cli -h "$REDIS_HOST" -p 6379 -a "$REDIS_PASSWORD" FT.CREATE ingest-entity ON JSON PREFIX 1 "ingest-entity:" SCHEMA $.data_type AS data_type TEXT $.state AS state NUMERIC
-    environment:
-      - REDIS_HOST=${REDIS_HOST}
-      - REDIS_PASSWORD=${REDIS_PASSWORD}
-    volumes:
-      - ./diode/redis:/home/redis
+    command: redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" -a "$REDIS_PASSWORD" FT.CREATE ingest-entity ON JSON PREFIX 1 "ingest-entity:" SCHEMA $$.data_type AS data_type TEXT $$.state AS state NUMERIC
 volumes:
   diode-redis-data:
     driver: local

diode-server/docker/sample.env

@@ -1,13 +1,13 @@
+DIODE_NGINX_PORT=8080
 RECONCILER_API_KEY=CHANGE_.ME
 REDIS_PASSWORD=@FmnLoA*VnebyVnZoL.!-.6z
 REDIS_HOST=diode-redis
-REDIS_PORT=6379
+REDIS_PORT=6378
 RECONCILER_GRPC_HOST=diode-reconciler
 RECONCILER_GRPC_PORT=8081
-NETBOX_DIODE_PLUGIN_API_BASE_URL=http://netbox:8080/netbox/api/plugins/diode
+NETBOX_DIODE_PLUGIN_API_BASE_URL=http://NETBOX_HOST/api/plugins/diode
 DIODE_TO_NETBOX_API_KEY=1368dbad13e418d5a443d93cf255edde03a2a754
 NETBOX_TO_DIODE_API_KEY=1e99338b8cab5fc637bc55f390bda1446f619c42
 INGESTION_API_KEY=5a52c45ee8231156cb620d193b0291912dd15433
-NETBOX_API_URL=http://netbox:8000/netbox/api
 LOGGING_LEVEL=DEBUG
 SENTRY_DSN=

diode-server/netboxdiodeplugin/client.go

@@ -3,11 +3,13 @@ package netboxdiodeplugin
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"log/slog"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -30,6 +32,9 @@ const (
 	// BaseURLEnvVarName is the environment variable name for the NetBox Diode plugin HTTP base URL
 	BaseURLEnvVarName = "NETBOX_DIODE_PLUGIN_API_BASE_URL"
 
+	// TLSSkipVerifyEnvVarName is the environment variable name for Netbox Diode plugin TLS verification
+	TLSSkipVerifyEnvVarName = "NETBOX_DIODE_PLUGIN_SKIP_TLS_VERIFY"
+
 	// TimeoutSecondsEnvVarName is the environment variable name for the NetBox Diode plugin HTTP timeout
 	TimeoutSecondsEnvVarName = "NETBOX_DIODE_PLUGIN_API_TIMEOUT_SECONDS"
 
@@ -94,9 +99,30 @@ type Client struct {
 	baseURL    *url.URL
 }
 
+// NewHTTPTransport creates a http Transport Layer
+func NewHTTPTransport() *http.Transport {
+	return &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).DialContext,
+		ForceAttemptHTTP2:     true,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: skipTLS(),
+		},
+	}
+}
+
 // NewClient creates a new NetBox Diode plugin client
 func NewClient(logger *slog.Logger, apiKey string) (*Client, error) {
-	rt, err := newAPIRoundTripper(apiKey, http.DefaultTransport)
+	transport := NewHTTPTransport()
+
+	rt, err := newAPIRoundTripper(apiKey, transport)
 	if err != nil {
 		return nil, err
 	}
@@ -137,6 +163,18 @@ func baseURL() string {
 	return u
 }
 
+func skipTLS() bool {
+	skipTLS, ok := os.LookupEnv(TLSSkipVerifyEnvVarName)
+	if !ok {
+		return false
+	}
+	skip, err := strconv.ParseBool(skipTLS)
+	if err != nil {
+		return false
+	}
+	return skip
+}
+
 func httpTimeout() (time.Duration, error) {
 	timeoutSecondsStr, ok := os.LookupEnv(TimeoutSecondsEnvVarName)
 	if !ok || len(timeoutSecondsStr) == 0 {