From f13b4223eea3e8c5a74cdfbf08cf246c9ca8d515 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Tue, 24 Dec 2024 10:48:01 +0700 Subject: [PATCH] Refactor configuration builder for caddy --- lib/uplink/caddy/config/hosts.ex | 18 ++ lib/uplink/caddy/config/port.ex | 79 +++++++++ lib/uplink/caddy/config/upstreams.ex | 29 ++++ lib/uplink/clients/caddy/config/builder.ex | 106 +---------- .../clients/caddy/config/builder_test.exs | 164 ++++++++++++++++++ test/uplink/metrics/pipeline_test.exs | 4 +- 6 files changed, 299 insertions(+), 101 deletions(-) create mode 100644 lib/uplink/caddy/config/hosts.ex create mode 100644 lib/uplink/caddy/config/port.ex create mode 100644 lib/uplink/caddy/config/upstreams.ex diff --git a/lib/uplink/caddy/config/hosts.ex b/lib/uplink/caddy/config/hosts.ex new file mode 100644 index 0000000..8e8f458 --- /dev/null +++ b/lib/uplink/caddy/config/hosts.ex @@ -0,0 +1,18 @@ +defmodule Uplink.Caddy.Config.Hosts do + alias Uplink.Packages.Metadata + alias Uplink.Packages.Metadata.Port + + def routable?(%{metadata: %Metadata{main_port: nil}}), do: false + + def routable?(%{ + metadata: %Metadata{ + main_port: %{routing: %Port.Routing{}} + } + }), + do: true + + def routable?(%{metadata: %Metadata{hosts: hosts}}) when length(hosts) > 0, + do: true + + def routable?(%{metadata: %Metadata{hosts: []}}), do: false +end diff --git a/lib/uplink/caddy/config/port.ex b/lib/uplink/caddy/config/port.ex new file mode 100644 index 0000000..41661c7 --- /dev/null +++ b/lib/uplink/caddy/config/port.ex @@ -0,0 +1,79 @@ +defmodule Uplink.Caddy.Config.Port do + alias Uplink.Packages.Metadata + + alias Uplink.Caddy.Config.Upstreams + + def build(%Metadata{ports: ports} = metadata, install_id) do + ports + |> Enum.map(&build(&1, metadata, install_id)) + |> Enum.reject(&is_nil/1) + end + + def build(%Metadata.Port{} = port, metadata, install_id) do + hosts = Enum.map(metadata.hosts, &merge_slug_and_host(&1, port)) + + routing = Map.get(port, :routing) + + routing_hosts = + if routing do + Enum.map(routing.hosts, &merge_slug_and_host(&1, port)) + else + [] + end + + hosts = + hosts + |> Enum.concat(routing_hosts) + |> Enum.uniq() + |> Enum.sort() + + paths = + if routing && routing.paths != [] do + routing.paths + else + ["*"] + end + + group = + if routing, + do: "router_#{routing.router_id}", + else: "installation_#{metadata.id}" + + if hosts == [] do + nil + else + %{ + group: group, + match: [ + %{ + host: hosts, + path: paths + } + ], + handle: [ + %{ + handler: "reverse_proxy", + load_balancing: %{ + selection_policy: %{ + policy: "least_conn" + } + }, + health_checks: %{ + passive: %{ + fail_duration: "10s", + max_fails: 3, + unhealthy_request_count: 80, + unhealthy_status: [500, 501, 502, 503, 504], + unhealthy_latency: "30s" + } + }, + upstreams: Upstreams.build(metadata, port, install_id) + } + ] + } + end + end + + defp merge_slug_and_host(host, %Metadata.Port{slug: slug}), + do: slug <> "." <> host +end diff --git a/lib/uplink/caddy/config/upstreams.ex b/lib/uplink/caddy/config/upstreams.ex new file mode 100644 index 0000000..29b912a --- /dev/null +++ b/lib/uplink/caddy/config/upstreams.ex @@ -0,0 +1,29 @@ +defmodule Uplink.Caddy.Config.Upstreams do + alias Uplink.Cache + alias Uplink.Packages.Metadata + alias Uplink.Packages.Metadata.Port + + def build(%Metadata{instances: instances}, %Port{} = port, install_id) do + instances + |> filter_valid(install_id) + |> Enum.map(fn instance -> + %{ + dial: "#{instance.slug}:#{port.target}", + max_requests: 100 + } + end) + end + + def filter_valid(instances, install_id) do + completed_instances = Cache.get({:install, install_id, "completed"}) + + if is_list(completed_instances) and Enum.count(completed_instances) > 0 do + instances + |> Enum.filter(fn instance -> + instance.slug in completed_instances + end) + else + instances + end + end +end diff --git a/lib/uplink/clients/caddy/config/builder.ex b/lib/uplink/clients/caddy/config/builder.ex index 455c363..714ac0d 100644 --- a/lib/uplink/clients/caddy/config/builder.ex +++ b/lib/uplink/clients/caddy/config/builder.ex @@ -1,10 +1,13 @@ defmodule Uplink.Clients.Caddy.Config.Builder do alias Uplink.Repo - alias Uplink.Cache alias Uplink.Packages alias Uplink.Routings + alias Uplink.Caddy.Config.Hosts + alias Uplink.Caddy.Config.Port + alias Uplink.Caddy.Config.Upstreams + alias Uplink.Clients.Caddy alias Uplink.Clients.Caddy.Admin @@ -17,9 +20,7 @@ defmodule Uplink.Clients.Caddy.Config.Builder do |> Repo.all() |> Repo.preload(deployment: [:app]) |> Enum.map(&Packages.build_install_state/1) - |> Enum.reject(fn %{metadata: metadata} -> - metadata.hosts == [] || is_nil(metadata.main_port) - end) + |> Enum.filter(&Hosts.routable?/1) %{"organization" => %{"storage" => storage_params}} = uplink = Uplink.Clients.Instellar.get_self() @@ -157,8 +158,6 @@ defmodule Uplink.Clients.Caddy.Config.Builder do [] end - valid_instances = find_valid_instances(metadata.instances, install_id) - proxy_routes = proxies |> Enum.map(fn proxy -> @@ -225,90 +224,12 @@ defmodule Uplink.Clients.Caddy.Config.Builder do unhealthy_latency: "30s" } }, - upstreams: - Enum.map(valid_instances, fn instance -> - %{ - dial: "#{instance.slug}:#{metadata.main_port.target}", - max_requests: 100 - } - end) + upstreams: Upstreams.build(metadata, metadata.main_port, install_id) } ] } - sub_routes = - metadata.ports - |> Enum.map(fn port -> - hosts = - Enum.map(metadata.hosts, fn host -> - port.slug <> "." <> host - end) - - routing = Map.get(port, :routing) - - routing_hosts = - if routing do - Enum.map(routing.hosts, fn host -> - port.slug <> "." <> host - end) - else - [] - end - - hosts = - hosts - |> Enum.concat(routing_hosts) - |> Enum.uniq() - |> Enum.sort() - - paths = - if routing && routing.paths != [] do - routing.paths - else - ["*"] - end - - group = - if routing, - do: "router_#{routing.router_id}", - else: "installation_#{metadata.id}" - - %{ - group: group, - match: [ - %{ - host: hosts, - path: paths - } - ], - handle: [ - %{ - handler: "reverse_proxy", - load_balancing: %{ - selection_policy: %{ - policy: "least_conn" - } - }, - health_checks: %{ - passive: %{ - fail_duration: "10s", - max_fails: 3, - unhealthy_request_count: 80, - unhealthy_status: [500, 501, 502, 503, 504], - unhealthy_latency: "30s" - } - }, - upstreams: - Enum.map(valid_instances, fn instance -> - %{ - dial: "#{instance.slug}:#{port.target}", - max_requests: 100 - } - end) - } - ] - } - end) + sub_routes = Port.build(metadata, install_id) sub_routes_and_proxies = Enum.concat(sub_routes, proxy_routes) @@ -328,19 +249,6 @@ defmodule Uplink.Clients.Caddy.Config.Builder do ] end - defp find_valid_instances(instances, install_id) do - completed_instances = Cache.get({:install, install_id, "completed"}) - - if is_list(completed_instances) and Enum.count(completed_instances) > 0 do - instances - |> Enum.filter(fn instance -> - instance.slug in completed_instances - end) - else - instances - end - end - defp maybe_merge_tls(params, %{tls: true}) do Map.put(params, :transport, %{ protocol: "http", diff --git a/test/uplink/clients/caddy/config/builder_test.exs b/test/uplink/clients/caddy/config/builder_test.exs index 9e2968e..9707fe2 100644 --- a/test/uplink/clients/caddy/config/builder_test.exs +++ b/test/uplink/clients/caddy/config/builder_test.exs @@ -242,4 +242,168 @@ defmodule Uplink.Clients.Caddy.Config.BuilderTest do assert %{host: _hosts} = match end end + + describe "when metadata.hosts is empty" do + setup do + deployment_params = %{ + "hash" => "a-different-hash-234", + "archive_url" => "http://localhost/archives/packages.zip", + "stack" => "alpine/3.14", + "channel" => "develop", + "metadata" => %{ + "id" => 1, + "slug" => "uplink-web", + "main_port" => %{ + "slug" => "web", + "source" => 49152, + "target" => 4000, + "routing" => %{ + "router_id" => 1, + "hosts" => ["another.com"], + "paths" => ["*"] + } + }, + "ports" => [ + %{ + "slug" => "grpc", + "source" => 49153, + "target" => 6000 + } + ], + "hosts" => [], + "variables" => [ + %{"key" => "SOMETHING", "value" => "blah"} + ], + "channel" => %{ + "slug" => "develop", + "package" => %{ + "slug" => "something-1640927800", + "credential" => %{ + "public_key" => "public_key" + }, + "organization" => %{ + "slug" => "upmaru" + } + } + }, + "instances" => [ + %{ + "id" => 1, + "slug" => "something-1", + "node" => %{ + "slug" => "some-node" + } + } + ] + } + } + + {:ok, actor} = + Members.get_or_create_actor(%{ + "identifier" => "zacksiri", + "provider" => "instellar", + "id" => "1" + }) + + metadata = Map.get(deployment_params, "metadata") + + {:ok, metadata} = Packages.parse_metadata(metadata) + + app = + metadata + |> Metadata.app_slug() + |> Packages.get_or_create_app() + + {:ok, deployment} = + Packages.get_or_create_deployment(app, deployment_params) + + {:ok, %{resource: preparing_deployment}} = + Packages.transition_deployment_with(deployment, actor, "prepare") + + {:ok, %{resource: deployment}} = + Packages.transition_deployment_with( + preparing_deployment, + actor, + "complete" + ) + + {:ok, install} = + Packages.create_install(deployment, %{ + "installation_id" => 1, + "deployment" => deployment_params + }) + + signature = Secret.Signature.compute_signature(deployment.hash) + + Cache.put( + {:deployment, signature, install.instellar_installation_id}, + metadata + ) + + {:ok, %{resource: validating_install}} = + Packages.transition_install_with(install, actor, "validate") + + {:ok, %{resource: _executing_install}} = + Packages.transition_install_with(validating_install, actor, "execute") + + :ok + end + + test "render port when routing correctly", %{bypass: bypass} do + Uplink.Cache.delete({:proxies, 1}) + + Bypass.expect_once( + bypass, + "GET", + "/uplink/self/routers/1/proxies", + fn conn -> + conn + |> Plug.Conn.put_resp_header("content-type", "application/json") + |> Plug.Conn.send_resp( + 200, + Jason.encode!(%{ + "data" => [ + %{ + "attributes" => %{ + "id" => 1, + "router_id" => 1, + "hosts" => ["opsmaru.com", "www.opsmaru.com"], + "paths" => ["/how-to*"], + "tls" => true, + "target" => "proxy.webflow.com", + "port" => 80 + } + } + ] + }) + ) + end + ) + + assert %{apps: apps} = Uplink.Clients.Caddy.build_new_config() + + assert %{http: %{servers: %{"uplink" => server}}} = apps + + assert %{routes: routes} = server + + routes = Enum.sort(routes) + + [first_route, second_route] = routes + + assert %{handle: [handle], match: [match]} = first_route + assert %{handle: [second_handle], match: [second_match]} = second_route + + assert match.host == ["another.com"] + assert match.path == ["*"] + + assert second_match.path == ["/how-to*"] + + [second_upstream] = second_handle.upstreams + + assert second_upstream.dial =~ "80" + + assert %{handler: "reverse_proxy"} = handle + assert %{host: _hosts} = match + end + end end diff --git a/test/uplink/metrics/pipeline_test.exs b/test/uplink/metrics/pipeline_test.exs index 1976443..b36be1d 100644 --- a/test/uplink/metrics/pipeline_test.exs +++ b/test/uplink/metrics/pipeline_test.exs @@ -26,7 +26,7 @@ defmodule Uplink.Metrics.PipelineTest do } do ref = Broadway.test_message(Uplink.Metrics.Pipeline, message) - assert_receive {:ack, ^ref, [%{data: data}], []} + assert_receive {:ack, ^ref, [%{data: data}], []}, 10_000 assert %{ memory: memory, @@ -65,7 +65,7 @@ defmodule Uplink.Metrics.PipelineTest do } do ref = Broadway.test_message(Uplink.Metrics.Pipeline, message) - assert_receive {:ack, ^ref, [%{data: data}], []} + assert_receive {:ack, ^ref, [%{data: data}], []}, 10_000 assert %{network: network} = data