feat: Reverse proxy lifecycle management and connectivity on Docker (#…

…1906) ## Description: In #1871 we added the Traefik labels to the user services so Traefik can discover them and route to them. This PR implements a reverse proxy using Traefik on Docker. The engine starts and stops Traefik. The Traefik container is automatically connected to the enclave networks. The implementation leverages what was done for the logs aggregator. ## Is this change user facing? YES ## References (if applicable): #1871
kurtosis-tech · Dec 8, 2023 · 69c5b27 · 69c5b27
1 parent 8f2427b
commit 69c5b27
Show file tree

Hide file tree

Showing 31 changed files with 1,289 additions and 36 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -80,14 +80,19 @@ steps_prepare_testing_k8s_k3s: &steps_prepare_testing_k8s_k3s
         command: "${KURTOSIS_BINPATH} gateway"
         background: true
 
-# Steps to prepare a job for Docker testing
-steps_prepare_testing_docker: &steps_prepare_testing_docker
-  steps:
-    - run: |
+run_prepare_testing_docker: &run_prepare_testing_docker
+  - run:
+      name: Load the engine, apic and file artifacts images and start the engine
+      command: |
         docker load -i  "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.core-server-image-filename >>"
         docker load -i  "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.engine-server-image-filename >>"
         docker load -i  "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.file-artifacts-expander-image-filename >>"
-    - run: "${KURTOSIS_BINPATH} engine start --cli-log-level trace"
+        ${KURTOSIS_BINPATH} engine start --cli-log-level trace
+
+# Steps to prepare a job for Docker testing
+steps_prepare_testing_docker: &steps_prepare_testing_docker
+  steps:
+    - <<: *run_prepare_testing_docker 
 
 # Run steps to dump kurtosis enclaves from docker
 run_dump_kurtosis_enclaves: &run_dump_kurtosis_enclaves
@@ -116,6 +121,17 @@ abort_job_if_only_docs_changes: &abort_job_if_only_docs_changes
       circleci-agent step halt
     fi 
 
+abort_job_if_kubernetes_backend: &abort_job_if_kubernetes_backend
+  when:
+    condition:
+      and:
+        - equal: [ "kubernetes", << parameters.cli-cluster-backend >> ]
+    steps:
+      - run: circleci-agent step halt
+
+abort_job: &abort_job
+  run: circleci-agent step halt
+
 ##############
 # CircleCI
 ##############
@@ -216,6 +232,9 @@ parameters:
   kurtosis-cluster-setting-abs-filepath:
     type: string
     default: "/home/circleci/.local/share/kurtosis/cluster-setting"
+  reverse-proxy-entrypoint-web-port:
+    type: string
+    default: "9730"
 
 
 
@@ -639,6 +658,9 @@ jobs:
   test_old_enclave_continuity:
     executor: ubuntu_vm
     steps:
+      # TODO: Re-enable once Traefik has been released.
+      - <<: *abort_job
+
       - checkout
 
       - <<: *abort_job_if_only_docs_changes
@@ -878,9 +900,9 @@ jobs:
             equal: [ "docker", << parameters.cli-cluster-backend >> ]
           steps:
             - run:
-                name: "Verify only the engine container and logs aggregator remains after the clean"
+                name: "Verify only the engine, logs aggregator and reverse proxy remain after the clean"
                 command: |
-                  if ! [ "$(docker container ls -a | tail -n+2 | wc -l)" -eq 2 ]; then
+                  if ! [ "$(docker container ls -a | tail -n+2 | wc -l)" -eq 3 ]; then
                     docker container ls -a
                     false
                   fi
@@ -904,6 +926,61 @@ jobs:
                     false
                   fi
 
+  test_reverse_proxy:
+    executor: ubuntu_vm
+    parameters:
+      <<: *param_cli_cluster_backend
+    steps:
+      - <<: *abort_job_if_kubernetes_backend
+
+      - checkout
+
+      - <<: *abort_job_if_only_docs_changes
+
+      # Set up Kurtosis
+      - attach_workspace:
+          at: "<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>"
+
+      - <<: *steps_install_go
+
+      - run: |
+          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt update
+          sudo apt install kurtosis-cli curl
+      # We don't send metrics to avoid polluting our logs
+      - run: |
+          echo 'export KURTOSIS_BINPATH="<< pipeline.parameters.workspace-with-cli-binary-and-images-mountpoint >>/<< pipeline.parameters.cli-dist-home-relative-dirpath >>/<< pipeline.parameters.cli-linux-amd-64-binary-relative-filepath >>"' >> "${BASH_ENV}"
+      - run: "${KURTOSIS_BINPATH} analytics disable"
+
+      - <<: *run_prepare_testing_docker
+
+      # Start a service and send an http request to it via the reverse proxy
+      - run: |
+          ${KURTOSIS_BINPATH} enclave add --name test-enclave
+          ${KURTOSIS_BINPATH} service add test-enclave test1 httpd --ports http=http:80/tcp
+          enclave_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}')
+          service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}')
+          # Give the reverse proxy enough time to discover the httpd user service
+          sleep 10
+          status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "X-Kurtosis-Enclave-Short-UUID: $(echo $enclave_uuid)" -H "X-Kurtosis-Service-Short-UUID: $(echo $service_uuid)" -H "X-Kurtosis-Service-Port-Number: 80"| head -1 | awk '{print $2}')
+          if ! [ "${status_code}" -eq "200" ]; then
+            echo 'HTTP request status code returned is '${status_code}' instead of 200'
+            false
+          fi
+
+      # Restart the engine and make sure Traefik restarted and reconfigured properly 
+      - run: |
+          ${KURTOSIS_BINPATH} engine restart
+          enclave_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | grep "^UUID:" | awk '{print $2}')
+          service_uuid=$(${KURTOSIS_BINPATH} enclave inspect test-enclave | tail -2 | awk '{print $1}')
+          # Give the reverse proxy enough time to discover the httpd user service
+          sleep 10
+          status_code=$(curl -I http://localhost:<< pipeline.parameters.reverse-proxy-entrypoint-web-port >> -H "X-Kurtosis-Enclave-Short-UUID: $(echo $enclave_uuid)" -H "X-Kurtosis-Service-Short-UUID: $(echo $service_uuid)" -H "X-Kurtosis-Service-Port-Number: 80"| head -1 | awk '{print $2}')
+          if ! [ "${status_code}" -eq "200" ]; then
+            echo 'HTTP request status code returned is '${status_code}' instead of 200'
+            false
+          fi
+
   test_ci_for_failure:
     executor: ubuntu_vm
     parameters:
@@ -1352,6 +1429,18 @@ workflows:
             - build_files_artifacts_expander
           <<: *filters_ignore_main
 
+      - test_reverse_proxy:
+          name: "Test reverse proxy against Docker"
+          cli-cluster-backend: "docker"
+          context:
+            - docker-user
+          requires:
+            - build_cli
+            - build_api_container_server
+            - build_engine_server
+            - build_files_artifacts_expander
+          <<: *filters_ignore_main
+
       # -- Artifact-publishing jobs --------------------------------
       - publish_kurtosis_sdk_rust:
           context:

diff --git a/...e-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go b/...e-lib/lib/backend_impls/docker/docker_kurtosis_backend/backend_creator/backend_creator.go
@@ -3,9 +3,14 @@ package backend_creator
 import (
 	"context"
 	"fmt"
+	"net"
+	"os"
+	"path"
+
 	"github.com/docker/docker/client"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/label_value_consts"
@@ -18,9 +23,6 @@ import (
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db/service_registration"
 	"github.com/kurtosis-tech/stacktrace"
 	"github.com/sirupsen/logrus"
-	"net"
-	"os"
-	"path"
 )
 
 const (
@@ -238,11 +240,21 @@ func getDockerKurtosisBackend(
 			return nil, stacktrace.Propagate(err, "An error occurred while getting the logs collector object for enclave '%v'; This is a bug in Kurtosis", enclaveUuid)
 		}
 
+		reverseProxy, err := reverse_proxy_functions.GetReverseProxy(ctx, dockerManager)
+		if err != nil {
+			return nil, stacktrace.Propagate(err, "An error occurred while getting the reverse proxy, This is a bug in Kurtosis")
+		}
+		reverseProxyEnclaveNetworkIpAddress, found := reverseProxy.GetEnclaveNetworksIpAddress()[network.GetId()]
+		if !found {
+			return nil, stacktrace.NewError("An error occured while getting the reverse proxy enclave network IP address for enclave '%v', This is a bug in Kurtosis", enclaveUuid)
+		}
+
 		alreadyTakenIps := map[string]bool{
 			networkIp.String():      true,
 			network.GetGatewayIp():  true,
 			apiContainerIp.String(): true,
 			logsCollectorObj.GetEnclaveNetworkIpAddress().String(): true,
+			reverseProxyEnclaveNetworkIpAddress.String():           true,
 		}
 
 		freeIpAddrProvider, err := free_ip_addr_tracker.GetOrCreateNewFreeIpAddrTracker(

diff --git a/...er-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go b/...er-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend.go
@@ -13,6 +13,8 @@ import (
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_aggregator_functions/implementations/vector"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/logs_collector_functions/implementations/fluentbit"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/reverse_proxy_functions/implementations/traefik"
 	user_service_functions "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/user_services_functions"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types"
@@ -28,6 +30,7 @@ import (
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/image_download_mode"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_aggregator"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/logs_collector"
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/reverse_proxy"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db/free_ip_addr_tracker"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/database_accessors/enclave_db/service_registration"
@@ -472,6 +475,57 @@ func (backend *DockerKurtosisBackend) DestroyLogsCollectorForEnclave(ctx context
 	return nil
 }
 
+func (backend *DockerKurtosisBackend) CreateReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) {
+	reverseProxyContainer := traefik.NewTraefikReverseProxyContainer()
+
+	reverseProxy, _, err := reverse_proxy_functions.CreateReverseProxy(
+		ctx,
+		reverseProxyContainer,
+		backend.dockerManager,
+		backend.objAttrsProvider,
+	)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred creating the reverse proxy using the reverse proxy container '%+v'.", reverseProxyContainer)
+	}
+	return reverseProxy, nil
+}
+
+func (backend *DockerKurtosisBackend) GetReverseProxy(ctx context.Context) (*reverse_proxy.ReverseProxy, error) {
+	maybeReverseProxy, err := reverse_proxy_functions.GetReverseProxy(
+		ctx,
+		backend.dockerManager,
+	)
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred getting the reverse proxy")
+	}
+
+	return maybeReverseProxy, nil
+}
+
+func (backend *DockerKurtosisBackend) DestroyReverseProxy(ctx context.Context) error {
+	if err := reverse_proxy_functions.DestroyReverseProxy(ctx, backend.dockerManager); err != nil {
+		return stacktrace.Propagate(err, "An error occurred destroying the reverse proxy")
+	}
+
+	return nil
+}
+
+func (backend *DockerKurtosisBackend) ConnectReverseProxyToNetwork(ctx context.Context, networkId string) error {
+	if err := reverse_proxy_functions.ConnectReverseProxyToNetwork(ctx, backend.dockerManager, networkId); err != nil {
+		return stacktrace.Propagate(err, "An error occurred connecting the reverse proxy to the network with ID '%v'", networkId)
+	}
+
+	return nil
+}
+
+func (backend *DockerKurtosisBackend) DisconnectReverseProxyFromNetwork(ctx context.Context, networkId string) error {
+	if err := reverse_proxy_functions.DisconnectReverseProxyFromNetwork(ctx, backend.dockerManager, networkId); err != nil {
+		return stacktrace.Propagate(err, "An error occurred disconnecting the reverse proxy from the network with ID '%v'", networkId)
+	}
+
+	return nil
+}
+
 func (backend *DockerKurtosisBackend) GetAvailableCPUAndMemory(ctx context.Context) (compute_resources.MemoryInMegaBytes, compute_resources.CpuMilliCores, bool, error) {
 	availableMemory, availableCpu, err := backend.dockerManager.GetAvailableCPUAndMemory(ctx)
 	if err != nil {

diff --git a/...d_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go b/...d_impls/docker/docker_kurtosis_backend/docker_kurtosis_backend_api_container_functions.go
@@ -2,10 +2,11 @@ package docker_kurtosis_backend
 
 import (
 	"context"
-	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key"
 	"net"
 	"time"
 
+	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/object_attributes_provider/docker_label_key"
+
 	"github.com/docker/go-connections/nat"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/consts"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_kurtosis_backend/shared_helpers"
@@ -73,14 +74,27 @@ func (backend *DockerKurtosisBackend) CreateAPIContainer(
 
 	enclaveLogsCollector, err := backend.GetLogsCollectorForEnclave(ctx, enclaveUuid)
 	if err != nil {
-		return nil, stacktrace.Propagate(err, "An error occurred while getting the logs collector for enclave '%v; This is a bug in Kurtosis'", enclaveUuid)
+		return nil, stacktrace.Propagate(err, "An error occurred while getting the logs collector for enclave '%v'; This is a bug in Kurtosis", enclaveUuid)
+	}
+
+	reverseProxy, err := backend.GetReverseProxy(ctx)
+	if reverseProxy == nil {
+		return nil, stacktrace.Propagate(err, "The reverse proxy is not running, This is a bug in Kurtosis")
+	}
+	if err != nil {
+		return nil, stacktrace.Propagate(err, "An error occurred while getting the reverse proxy, This is a bug in Kurtosis")
+	}
+	reverseProxyEnclaveNetworkIpAddress, found := reverseProxy.GetEnclaveNetworksIpAddress()[enclaveNetwork.GetId()]
+	if !found {
+		return nil, stacktrace.NewError("An error occured while getting the reverse proxy enclave network IP address for enclave '%v', This is a bug in Kurtosis", enclaveUuid)
 	}
 
 	networkCidr := enclaveNetwork.GetIpAndMask()
 	alreadyTakenIps := map[string]bool{
 		networkCidr.IP.String():                                    true,
 		enclaveNetwork.GetGatewayIp():                              true,
 		enclaveLogsCollector.GetEnclaveNetworkIpAddress().String(): true,
+		reverseProxyEnclaveNetworkIpAddress.String():               true,
 	}
 
 	ipAddr, err := network_helpers.GetFreeIpAddrFromSubnet(alreadyTakenIps, networkCidr)