diff --git a/.github/workflows/server-release.yaml b/.github/workflows/server-release.yaml index e53585c0..dc4936dc 100644 --- a/.github/workflows/server-release.yaml +++ b/.github/workflows/server-release.yaml @@ -95,7 +95,6 @@ jobs: fail-fast: false matrix: app: ${{ fromJSON(needs.setup.outputs.apps) }} - if: needs.setup.outputs.app != '' with: app_name: diode-${{ matrix.app }} app_dir: diode-server diff --git a/README.md b/README.md index 16e3e817..0dbe3142 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,21 @@ # Diode -Diode is a service designed to streamline the process of data ingestion and reconciliation for NetBox users. It aims to lower the barriers to entry for integrating existing network infrastructure data into NetBox and reduce the maintenance efforts required to keep the data up-to-date. +Diode is a NetBox data ingestion service that greatly simplifies and enhances the process to add and update network data in NetBox, ensuring your network source of truth is always accurate and can be trusted to power your network automation pipelines. Our guiding principle in designing Diode has been to make it as easy as possible to get data into NetBox, removing as much burden as possible from the user while shifting that effort to technology. -## Usage +To achieve this, Diode sits in front of NetBox and provides an API purpose built for ingestion of complex network data. Diode eliminates the need to preprocess data to make it conform to the strict object hierarchy imposed by the NetBox data model. This allows data to be sent to NetBox in a more freeform manner, in blocks that are intuitive for network engineers (such as by device or by interface) with much of the related information treated as attributes or properties of these components of interest. Then, Diode takes care of the heavy lifting, automatically transforming the data to align it with NetBox’s structured and comprehensive data model. Diode can even create placeholder objects to compensate for missing information, which means even fragmented information about the network can be captured in NetBox. + +## Project status + +The Diode project is currently in the _Public Preview_ stage. Please see [NetBox Labs Product and Feature Lifecycle](https://docs.netboxlabs.com/product_feature_lifecycle/) for more details. We actively welcome feedback to help identify and prioritize bugs, new features and areas of improvement. + +## Get started + +Diode runs as a sidecar service to NetBox and can run anywhere with network connectivity to NetBox, whether on the same host or elsewhere. The overall Diode service is delivered through three main components (and a fourth optional component): + +1. Diode plugin - see how to [install the Diode plugin](https://github.com/netboxlabs/diode-netbox-plugin) +2. Diode server - see how to [run the Diode server](https://github.com/netboxlabs/diode/tree/develop/diode-server#readme) +3. Diode SDK - see how to [install the Diode client SDK](https://github.com/netboxlabs/diode-sdk-python) and [download Diode Python script examples](https://github.com/netboxlabs/netbox-learning/tree/develop/diode) +4. Diode agent (optional) - see how to [install and run the Diode NAPALM discovery agent](https://github.com/netboxlabs/diode-agent/tree/develop/diode-napalm-agent) ## Related Projects @@ -15,3 +28,8 @@ Diode is a service designed to streamline the process of data ingestion and reco Distributed under the PolyForm Shield License 1.0.0 License. See [LICENSE.md](./LICENSE.md) for more information. Diode protocol buffers are distributed under the Apache 2.0 License. See [LICENSE.txt](./diode-proto/LICENSE.txt) for more information. + +## Required Notice + +Copyright NetBox Labs, Inc. + diff --git a/diode-server/Makefile b/diode-server/Makefile index 79d3a583..910a33e4 100644 --- a/diode-server/Makefile +++ b/diode-server/Makefile @@ -71,7 +71,7 @@ docker-compose-down: @DIODE_VERSION=$(DIODE_VERSION) COMMIT_SHA=$(COMMIT_SHA) \ $(DOCKER_COMPOSE) --env-file docker/sample.env -f docker/docker-compose.yaml down --remove-orphans -docker-compose-netbox-up: docker-all +docker-compose-netbox-up: $(DOCKER_COMPOSE) -f docker/docker-compose.netbox.yaml up -d --build docker-compose-netbox-down: diff --git a/diode-server/README.md b/diode-server/README.md index 5145e14d..b49434fc 100644 --- a/diode-server/README.md +++ b/diode-server/README.md @@ -1,6 +1,17 @@ -# Diode servers +# Diode server -Diode server is splited into two services: +The Diode server is a required component of the [Diode](https://github.com/netboxlabs/diode) ingestion service. + +Diode is a NetBox ingestion service that greatly simplifies and enhances the process to add and update network data +in NetBox, ensuring your network source of truth is always accurate and can be trusted to power your network automation +pipelines. + +More information about Diode can be found +at [https://netboxlabs.com/blog/introducing-diode-streamlining-data-ingestion-in-netbox/](https://netboxlabs.com/blog/introducing-diode-streamlining-data-ingestion-in-netbox/). + +## Diode services + +Diode server is comprised of two services: ### Ingester Service @@ -14,3 +25,52 @@ Diode server is splited into two services: - Processes data from Redis streams and converts it for storage. - Manages data sources and their API keys. - Implements a reconciliation engine to detect and store deltas between ingested data and the current NetBox object state. + +## Compatibility + +The Diode server has been tested with NetBox versions 3.7.2 and above. The Diode server also requires the [Diode NetBox Plugin](https://github.com/netboxlabs/diode-netbox-plugin). + +## Running the Diode server + +### Requirements + +Diode server requires Docker version 27.0.3 or above. + +### Installation + +Diode requires a configuration file and an environment file to execute successfully: + +* `docker-compose.yml` - to configure and run the Diode server containers +* `.env` - to store the specific environmental settings + +We recommend placing both files in a clean directory: + +```bash +mkdir /opt/diode +cd /opt/diode +``` + +Download the default `docker-compose.yml` and `.env` files from this repository: + +```bash +curl -o docker-compose.yml https://raw.githubusercontent.com/netboxlabs/diode/develop/diode-server/docker/docker-compose.yaml +curl -o .env https://raw.githubusercontent.com/netboxlabs/diode/develop/diode-server/docker/sample.env +``` + +Edit the `.env` to match your environment: +* `NETBOX_DIODE_PLUGIN_API_BASE_URL`: URL for the Diode NetBox plugin API +* `DIODE_TO_NETBOX_API_KEY`: API key generated with the Diode NetBox plugin installation +* `INGESTION_API_KEY`: API key generated with the Diode NetBox plugin installation +* `NETBOX_TO_DIODE_API_KEY`: API key generated with the Diode NetBox plugin installation + +### Running the Diode server + +Start the Diode server: + +```bash +docker compose -f docker-compose.yaml up -d +``` + +## License + +Distributed under the PolyForm Shield License 1.0.0 License. See [LICENSE.md](./LICENSE.md) for more information. diff --git a/diode-server/docker/docker-compose.yaml b/diode-server/docker/docker-compose.yaml index 967d7203..cc856146 100644 --- a/diode-server/docker/docker-compose.yaml +++ b/diode-server/docker/docker-compose.yaml @@ -7,46 +7,36 @@ services: server diode-ingester:8081; } - upstream netbox { - server netbox:8080; - } - server { - listen 80; + listen ${DIODE_NGINX_PORT}; http2 on; server_name localhost; + client_max_body_size 25m; location /diode { - rewrite /diode/(.*) /$1 break; + rewrite /diode/(.*) /$$1 break; grpc_pass grpc://diode; } - location /netbox/static/ { - proxy_pass http://netbox/static/; - } - location /netbox/ { - proxy_pass http://netbox; - proxy_set_header X-Forwarded-Host $$http_host; - proxy_set_header X-Real-IP $$remote_addr; - proxy_set_header X-Forwarded-Proto $$scheme; - } }' > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'" restart: always + environment: + - DIODE_NGINX_PORT=${DIODE_NGINX_PORT} ports: - - "80:80" + - ${DIODE_NGINX_PORT}:80 depends_on: - diode-ingester - diode-reconciler diode-ingester: - image: netboxlabs/diode-ingester:${DIODE_VERSION}-${COMMIT_SHA} + image: netboxlabs/diode-ingester:latest environment: - - API_KEY=${RECONCILER_API_KEY} - - REDIS_PASSWORD=${REDIS_PASSWORD} - - REDIS_HOST=${REDIS_HOST} - - REDIS_PORT=${REDIS_PORT} - - RECONCILER_GRPC_HOST=${RECONCILER_GRPC_HOST} - - RECONCILER_GRPC_PORT=${RECONCILER_GRPC_PORT} - - SENTRY_DSN=${SENTRY_DSN} + - API_KEY=${RECONCILER_API_KEY} + - REDIS_PASSWORD=${REDIS_PASSWORD} + - REDIS_HOST=${REDIS_HOST} + - REDIS_PORT=${REDIS_PORT} + - RECONCILER_GRPC_HOST=${RECONCILER_GRPC_HOST} + - RECONCILER_GRPC_PORT=${RECONCILER_GRPC_PORT} + - SENTRY_DSN=${SENTRY_DSN} restart: always ports: - "8081:8081" @@ -55,7 +45,7 @@ services: - diode-reconciler diode-reconciler: - image: netboxlabs/diode-reconciler:${DIODE_VERSION}-${COMMIT_SHA} + image: netboxlabs/diode-reconciler:latest environment: - REDIS_PASSWORD=${REDIS_PASSWORD} - REDIS_HOST=${REDIS_HOST} @@ -64,12 +54,10 @@ services: - DIODE_TO_NETBOX_API_KEY=${DIODE_TO_NETBOX_API_KEY} - NETBOX_TO_DIODE_API_KEY=${NETBOX_TO_DIODE_API_KEY} - INGESTION_API_KEY=${INGESTION_API_KEY} - - NETBOX_API_URL=${NETBOX_API_URL} - LOGGING_LEVEL=${LOGGING_LEVEL} - SENTRY_DSN=${SENTRY_DSN} restart: always - ports: - - "8082:8081" + ports: [ ] depends_on: - diode-redis diode-redis: @@ -77,23 +65,18 @@ services: command: - sh - -c - - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD --loadmodule /opt/redis-stack/lib/rejson.so --loadmodule /opt/redis-stack/lib/redisearch.so + - redis-server --appendonly yes --requirepass $$REDIS_PASSWORD --loadmodule /opt/redis-stack/lib/rejson.so --loadmodule /opt/redis-stack/lib/redisearch.so --port $$REDIS_PORT environment: - REDIS_PASSWORD=${REDIS_PASSWORD} - ports: - - "6379:6379" + - REDIS_PORT=${REDIS_PORT} + ports: [ ] volumes: - diode-redis-data:/data diode-redis-cli: image: redis/redis-stack-server:latest links: - diode-redis - command: redis-cli -h "$REDIS_HOST" -p 6379 -a "$REDIS_PASSWORD" FT.CREATE ingest-entity ON JSON PREFIX 1 "ingest-entity:" SCHEMA $.data_type AS data_type TEXT $.state AS state NUMERIC - environment: - - REDIS_HOST=${REDIS_HOST} - - REDIS_PASSWORD=${REDIS_PASSWORD} - volumes: - - ./diode/redis:/home/redis + command: redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" -a "$REDIS_PASSWORD" FT.CREATE ingest-entity ON JSON PREFIX 1 "ingest-entity:" SCHEMA $$.data_type AS data_type TEXT $$.state AS state NUMERIC volumes: diode-redis-data: driver: local \ No newline at end of file diff --git a/diode-server/docker/sample.env b/diode-server/docker/sample.env index 058a843d..98644e8d 100644 --- a/diode-server/docker/sample.env +++ b/diode-server/docker/sample.env @@ -1,13 +1,13 @@ +DIODE_NGINX_PORT=8080 RECONCILER_API_KEY=CHANGE_.ME REDIS_PASSWORD=@FmnLoA*VnebyVnZoL.!-.6z REDIS_HOST=diode-redis -REDIS_PORT=6379 +REDIS_PORT=6378 RECONCILER_GRPC_HOST=diode-reconciler RECONCILER_GRPC_PORT=8081 -NETBOX_DIODE_PLUGIN_API_BASE_URL=http://netbox:8080/netbox/api/plugins/diode +NETBOX_DIODE_PLUGIN_API_BASE_URL=http://NETBOX_HOST/api/plugins/diode DIODE_TO_NETBOX_API_KEY=1368dbad13e418d5a443d93cf255edde03a2a754 NETBOX_TO_DIODE_API_KEY=1e99338b8cab5fc637bc55f390bda1446f619c42 INGESTION_API_KEY=5a52c45ee8231156cb620d193b0291912dd15433 -NETBOX_API_URL=http://netbox:8000/netbox/api LOGGING_LEVEL=DEBUG SENTRY_DSN= diff --git a/diode-server/netboxdiodeplugin/client.go b/diode-server/netboxdiodeplugin/client.go index 697ed485..f5b158bc 100644 --- a/diode-server/netboxdiodeplugin/client.go +++ b/diode-server/netboxdiodeplugin/client.go @@ -3,11 +3,13 @@ package netboxdiodeplugin import ( "bytes" "context" + "crypto/tls" "encoding/json" "errors" "fmt" "io" "log/slog" + "net" "net/http" "net/url" "os" @@ -30,6 +32,9 @@ const ( // BaseURLEnvVarName is the environment variable name for the NetBox Diode plugin HTTP base URL BaseURLEnvVarName = "NETBOX_DIODE_PLUGIN_API_BASE_URL" + // TLSSkipVerifyEnvVarName is the environment variable name for Netbox Diode plugin TLS verification + TLSSkipVerifyEnvVarName = "NETBOX_DIODE_PLUGIN_SKIP_TLS_VERIFY" + // TimeoutSecondsEnvVarName is the environment variable name for the NetBox Diode plugin HTTP timeout TimeoutSecondsEnvVarName = "NETBOX_DIODE_PLUGIN_API_TIMEOUT_SECONDS" @@ -94,9 +99,30 @@ type Client struct { baseURL *url.URL } +// NewHTTPTransport creates a http Transport Layer +func NewHTTPTransport() *http.Transport { + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: skipTLS(), + }, + } +} + // NewClient creates a new NetBox Diode plugin client func NewClient(logger *slog.Logger, apiKey string) (*Client, error) { - rt, err := newAPIRoundTripper(apiKey, http.DefaultTransport) + transport := NewHTTPTransport() + + rt, err := newAPIRoundTripper(apiKey, transport) if err != nil { return nil, err } @@ -137,6 +163,18 @@ func baseURL() string { return u } +func skipTLS() bool { + skipTLS, ok := os.LookupEnv(TLSSkipVerifyEnvVarName) + if !ok { + return false + } + skip, err := strconv.ParseBool(skipTLS) + if err != nil { + return false + } + return skip +} + func httpTimeout() (time.Duration, error) { timeoutSecondsStr, ok := os.LookupEnv(TimeoutSecondsEnvVarName) if !ok || len(timeoutSecondsStr) == 0 { diff --git a/diode-server/netboxdiodeplugin/client_test.go b/diode-server/netboxdiodeplugin/client_test.go index ffa876f1..492975f7 100644 --- a/diode-server/netboxdiodeplugin/client_test.go +++ b/diode-server/netboxdiodeplugin/client_test.go @@ -17,6 +17,34 @@ import ( "github.com/netboxlabs/diode/diode-server/netboxdiodeplugin" ) +func TestTransportSecurity(t *testing.T) { + tests := []struct { + name string + expectedInsecure bool + }{ + { + name: "enable insecure mode", + expectedInsecure: true, + }, + { + name: "default secure TLS config", + expectedInsecure: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cleanUpEnvVars() + + if tt.expectedInsecure { + _ = os.Setenv(netboxdiodeplugin.TLSSkipVerifyEnvVarName, "true") + } + + httpTransport := netboxdiodeplugin.NewHTTPTransport() + assert.Equal(t, tt.expectedInsecure, httpTransport.TLSClientConfig.InsecureSkipVerify) + }) + } +} + func TestNewClient(t *testing.T) { tests := []struct { name string @@ -25,6 +53,7 @@ func TestNewClient(t *testing.T) { timeout string setBaseURLEnvVar bool setTimeoutEnvVar bool + setTLSSkipEnvVar bool shouldError bool }{ { @@ -34,6 +63,7 @@ func TestNewClient(t *testing.T) { timeout: "5", setBaseURLEnvVar: true, setTimeoutEnvVar: true, + setTLSSkipEnvVar: false, shouldError: false, }, { @@ -52,6 +82,7 @@ func TestNewClient(t *testing.T) { timeout: "5", setBaseURLEnvVar: true, setTimeoutEnvVar: true, + setTLSSkipEnvVar: false, shouldError: true, }, { @@ -61,6 +92,7 @@ func TestNewClient(t *testing.T) { timeout: "", setBaseURLEnvVar: true, setTimeoutEnvVar: false, + setTLSSkipEnvVar: false, shouldError: false, }, { @@ -70,6 +102,7 @@ func TestNewClient(t *testing.T) { timeout: "-1", setBaseURLEnvVar: true, setTimeoutEnvVar: true, + setTLSSkipEnvVar: false, shouldError: true, }, { @@ -79,8 +112,19 @@ func TestNewClient(t *testing.T) { timeout: "5", setBaseURLEnvVar: true, setTimeoutEnvVar: true, + setTLSSkipEnvVar: false, shouldError: true, }, + { + name: "set TLS skip verify", + apiKey: "test", + baseURL: "", + timeout: "5", + setBaseURLEnvVar: false, + setTimeoutEnvVar: true, + setTLSSkipEnvVar: true, + shouldError: false, + }, } logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug, AddSource: false})) @@ -95,6 +139,9 @@ func TestNewClient(t *testing.T) { if tt.setTimeoutEnvVar { _ = os.Setenv(netboxdiodeplugin.TimeoutSecondsEnvVarName, tt.timeout) } + if tt.setTLSSkipEnvVar { + _ = os.Setenv(netboxdiodeplugin.TLSSkipVerifyEnvVarName, "true") + } client, err := netboxdiodeplugin.NewClient(logger, tt.apiKey) if tt.shouldError { @@ -115,6 +162,7 @@ func TestRetrieveObjectState(t *testing.T) { apiKey string mockServerResponse string response any + tlsSkipVerify bool shouldError bool }{ { @@ -132,7 +180,8 @@ func TestRetrieveObjectState(t *testing.T) { }, }, }, - shouldError: false, + tlsSkipVerify: true, + shouldError: false, }, { name: "valid response for DCIM site with query", @@ -150,7 +199,8 @@ func TestRetrieveObjectState(t *testing.T) { }, }, }, - shouldError: false, + tlsSkipVerify: true, + shouldError: false, }, { name: "valid response for DCIM device with query and additional attributes", @@ -173,7 +223,8 @@ func TestRetrieveObjectState(t *testing.T) { }, }, }, - shouldError: false, + tlsSkipVerify: true, + shouldError: false, }, { name: "response for invalid object - empty object", @@ -187,13 +238,23 @@ func TestRetrieveObjectState(t *testing.T) { Device: &netbox.DcimDevice{}, }, }, - shouldError: false, + tlsSkipVerify: true, + shouldError: false, }, { name: "invalid server response", params: netboxdiodeplugin.RetrieveObjectStateQueryParams{ObjectType: netbox.DcimDeviceObjectType, ObjectID: 1}, apiKey: "barfoo", mockServerResponse: ``, + tlsSkipVerify: true, + shouldError: true, + }, + { + name: "tls bad certificate", + params: netboxdiodeplugin.RetrieveObjectStateQueryParams{ObjectType: netbox.DcimDeviceObjectType, ObjectID: 1}, + apiKey: "barfoo", + mockServerResponse: ``, + tlsSkipVerify: false, shouldError: true, }, } @@ -220,12 +281,16 @@ func TestRetrieveObjectState(t *testing.T) { assert.Equal(t, r.Header.Get("User-Agent"), fmt.Sprintf("%s/%s", netboxdiodeplugin.SDKName, netboxdiodeplugin.SDKVersion)) _, _ = w.Write([]byte(tt.mockServerResponse)) } + mux := http.NewServeMux() mux.HandleFunc("/api/diode/object-state/", handler) - ts := httptest.NewServer(mux) + ts := httptest.NewTLSServer(mux) defer ts.Close() _ = os.Setenv(netboxdiodeplugin.BaseURLEnvVarName, fmt.Sprintf("%s/api/diode", ts.URL)) + if tt.tlsSkipVerify { + _ = os.Setenv(netboxdiodeplugin.TLSSkipVerifyEnvVarName, "true") + } client, err := netboxdiodeplugin.NewClient(logger, tt.apiKey) require.NoError(t, err) @@ -347,6 +412,7 @@ func TestApplyChangeSet(t *testing.T) { func cleanUpEnvVars() { _ = os.Unsetenv(netboxdiodeplugin.BaseURLEnvVarName) _ = os.Unsetenv(netboxdiodeplugin.TimeoutSecondsEnvVarName) + _ = os.Unsetenv(netboxdiodeplugin.TLSSkipVerifyEnvVarName) } func ptrInt(i int) *int { diff --git a/diode-server/reconciler/config.go b/diode-server/reconciler/config.go index 7bac55b5..7899ecb6 100644 --- a/diode-server/reconciler/config.go +++ b/diode-server/reconciler/config.go @@ -8,7 +8,6 @@ type Config struct { RedisPassword string `envconfig:"REDIS_PASSWORD" required:"true"` RedisDB int `envconfig:"REDIS_DB" default:"0"` RedisStreamDB int `envconfig:"REDIS_STREAM_DB" default:"1"` - NetBoxAPIURL string `envconfig:"NETBOX_API_URL" required:"true"` // API keys DiodeToNetBoxAPIKey string `envconfig:"DIODE_TO_NETBOX_API_KEY" required:"true"` diff --git a/diode-server/server/server_test.go b/diode-server/server/server_test.go index f363b070..8a269071 100644 --- a/diode-server/server/server_test.go +++ b/diode-server/server/server_test.go @@ -19,51 +19,65 @@ func TestNewServer(t *testing.T) { serverName string loggingLevel string loggingFormat string + sentryDSN string }{ { desc: "diode-test-server with debug level and json format", serverName: "diode-test-server", loggingLevel: "debug", loggingFormat: "json", + sentryDSN: "", }, { desc: "diode-test-server2 with debug level and text format", serverName: "diode-test-server2", loggingLevel: "debug", loggingFormat: "text", + sentryDSN: "", }, { desc: "diode-test-server with info level and json format", serverName: "diode-test-server", loggingLevel: "info", loggingFormat: "json", + sentryDSN: "", }, { desc: "diode-test-server with info level and text format", serverName: "diode-test-server", loggingLevel: "warn", loggingFormat: "json", + sentryDSN: "", }, { desc: "diode-test-server with error level and text format", serverName: "diode-test-server", loggingLevel: "error", loggingFormat: "text", + sentryDSN: "", }, { desc: "diode-test-server with error level and empty format", serverName: "diode-test-server", loggingLevel: "error", loggingFormat: "", + sentryDSN: "", }, { desc: "diode-test-server with empty level and text format", serverName: "diode-test-server", loggingLevel: "", loggingFormat: "text", + sentryDSN: "", + }, + { + desc: "diode-test-server with sentry DSN", + serverName: "diode-test-server", + loggingLevel: "error", + loggingFormat: "text", + sentryDSN: "https://public@sentry.example.com/1", }, } - for _, tt := range tests { t.Run(tt.desc, func(t *testing.T) { ctx := context.Background() @@ -71,6 +85,8 @@ func TestNewServer(t *testing.T) { require.NoError(t, err) err = os.Setenv("LOGGING_FORMAT", tt.loggingFormat) require.NoError(t, err) + err = os.Setenv("SENTRY_DSN", tt.sentryDSN) + require.NoError(t, err) s := server.New(ctx, tt.serverName)