From 6f1f871e627367f14357789c5a2a711b336e0bf2 Mon Sep 17 00:00:00 2001 From: Mateusz Kulewicz Date: Wed, 3 Apr 2024 15:23:52 +0200 Subject: [PATCH] Route traffic to all ports with dynamic upstream creation --- src/charm.py | 7 ++--- src/nginx.py | 76 +++++++++++++++------------------------------------- 2 files changed, 25 insertions(+), 58 deletions(-) diff --git a/src/charm.py b/src/charm.py index 7e51a8a9..6564f89d 100755 --- a/src/charm.py +++ b/src/charm.py @@ -63,14 +63,15 @@ def __init__(self, *args): # we need otlp_http receiver for charm_tracing enable_receivers=["otlp_http"], ) + self.receiver_ports = tempo.get_ports(self.app.name) # set up a nginx container for routing to ingestion protocols and API - self.nginx = Nginx(server_name=self.hostname) + self.nginx = Nginx(server_name=self.hostname, ports = self.tempo.receiver_ports) # configure this tempo as a datasource in grafana self.grafana_source_provider = GrafanaSourceProvider( self, source_type="tempo", source_port=str(tempo.tempo_port) ) # # Patch the juju-created Kubernetes service to contain the right ports - self._service_patcher = KubernetesServicePatch(self, tempo.get_ports(self.app.name)) + self._service_patcher = KubernetesServicePatch(self, self.receiver_ports) # Provide ability for Tempo to be scraped by Prometheus using prometheus_scrape self._scraping = MetricsEndpointProvider( self, @@ -90,7 +91,7 @@ def __init__(self, *args): # ) self._tracing = TracingEndpointProvider(self, host=tempo.host) - self._ingress = IngressPerAppRequirer(self, port=8080) + self._ingress = IngressPerAppRequirer(self, port=8080, strip_prefix = True) self.framework.observe(self.on.tempo_pebble_ready, self._on_tempo_pebble_ready) self.framework.observe( diff --git a/src/nginx.py b/src/nginx.py index 9b7a27d0..b3607352 100644 --- a/src/nginx.py +++ b/src/nginx.py @@ -3,11 +3,13 @@ """Nginx workload.""" import logging -from typing import Any, Dict, List, Optional, Set +from typing import Any, Dict, List, Optional, Set, Tuple import crossplane from ops.pebble import Layer +from charms.tempo_k8s.v2.tracing import ReceiverProtocol + logger = logging.getLogger(__name__) @@ -17,22 +19,6 @@ CERT_PATH = f"{NGINX_DIR}/certs/server.cert" CA_CERT_PATH = f"{NGINX_DIR}/certs/ca.cert" -_PORTS_BY_PROTOCOL = [ - { - "directive": "upstream", - "args": ["api"], - "block": [ - {"directive": "server", "args": [f"localhost:3200"]} - ], - }, - { - "directive": "upstream", - "args": ["otlp_http"], - "block": [ - {"directive": "server", "args": [f"localhost:4318"]} - ], - } -] class Nginx: @@ -40,8 +26,9 @@ class Nginx: config_path = NGINX_CONFIG - def __init__(self, server_name: str): + def __init__(self, server_name: str, ports: Dict[ReceiverProtocol, int] = None): self.server_name = server_name + self.ports = ports def config(self, tls: bool = False) -> str: """Build and return the Nginx configuration.""" @@ -63,21 +50,7 @@ def config(self, tls: bool = False) -> str: "args": [], "block": [ # upstreams (load balancing) - # TODO once Tempo is HA we'll need addresses_by_role similar to mimir - { - "directive": "upstream", - "args": ["api"], - "block": [ - {"directive": "server", "args": [f"localhost:3200"]} - ], - }, - { - "directive": "upstream", - "args": ["otlp_http"], - "block": [ - {"directive": "server", "args": [f"localhost:4318"]} - ], - }, + *self._upstreams(), # temp paths {"directive": "client_body_temp_path", "args": ["/tmp/client_temp"]}, {"directive": "proxy_temp_path", "args": ["/tmp/proxy_temp_path"]}, @@ -111,6 +84,8 @@ def config(self, tls: bool = False) -> str: }, ] + logger.info(f"Built configuration: {full_config}") + return crossplane.build(full_config) @property @@ -146,15 +121,16 @@ def _log_verbose(self, verbose: bool = True) -> List[Dict[str, Any]]: {"directive": "access_log", "args": ["/dev/stderr"]}, ] - def _upstreams(self, addresses_by_role: Dict[str, Set[str]]) -> List[Dict[str, Any]]: + def _upstreams(self) -> List[Dict[str, Any]]: nginx_upstreams = [] - for role, address_set in addresses_by_role.items(): + for protocol, port in self.ports.items(): nginx_upstreams.append( { "directive": "upstream", - "args": [role], + "args": [protocol], "block": [ - {"directive": "server", "args": [f"{addr}:8080"]} for addr in address_set + # TODO for HA Tempo we'll need to add redirects to worker nodes similar to mimir + {"directive": "server", "args": [f"localhost:{port}"]} ], } ) @@ -178,28 +154,19 @@ def _basic_auth(self, enabled: bool) -> List[Optional[Dict[str, Any]]]: return [] def _locations(self) -> List[Dict[str, Any]]: - return [ - { - "directive": "location", - "args": ["/api/"], - "block": [ - { - "directive": "proxy_pass", - "args": ["http://api/"], - }, - ], - }, - { + locations = [] + for protocol, _ in self.ports.items(): + locations.append({ "directive": "location", - "args": ["/otlp_http/"], + "args": [f"/{protocol}/"], "block": [ { - "directive": "proxy_pass", - "args": ["http://otlp_http/"], + "directive": "proxy_pass" if "grpc" not in protocol else "grpc_pass", + "args": [f"http://{protocol}/" if "grpc" not in protocol else f"grpc://{protocol}"], }, ], - }, - ] + }) + return locations def _server(self, tls: bool = False) -> Dict[str, Any]: auth_enabled = False @@ -230,7 +197,6 @@ def _server(self, tls: bool = False) -> Dict[str, Any]: {"directive": "ssl_certificate_key", "args": [KEY_PATH]}, {"directive": "ssl_protocols", "args": ["TLSv1", "TLSv1.1", "TLSv1.2"]}, {"directive": "ssl_ciphers", "args": ["HIGH:!aNULL:!MD5"]}, # pyright: ignore - # TODO what's tempo's equivalent of this one? *self._locations(), ], }