feat: add pyroscope.receive_http component for profile handling (#1886

) * feat: add `pyroscope.receive_http` component for profile ingestion and forwarding * fix: improve AppendIngest with TeeReader and update AppendableFunc interface * Apply docs suggestions from code review Co-authored-by: Clayton Cornell <[email protected]> * use defaults and set maxConnLimit * changed back to use adhoc defaults and set max conn limit to 100 * Avoid reading request body in memory for write component * override profile headers with endpoint ones --------- Co-authored-by: Clayton Cornell <[email protected]>
grafana · Oct 29, 2024 · 044b2da · 044b2da
1 parent b9632e6
commit 044b2da
Show file tree

Hide file tree

Showing 10 changed files with 834 additions and 19 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -18,6 +18,7 @@ Main (unreleased)
 ### Features
 
 - Add the function `path_join` to the stdlib. (@wildum)
+- Add `pyroscope.receive_http` component to receive and forward Pyroscope profiles (@marcsanmi)
 
 - Add support to `loki.source.syslog` for the RFC3164 format ("BSD syslog"). (@sushain97)
 

diff --git a/docs/sources/reference/compatibility/_index.md b/docs/sources/reference/compatibility/_index.md
@@ -395,6 +395,7 @@ The following components, grouped by namespace, _consume_ Pyroscope `ProfilesRec
 {{< collapse title="pyroscope" >}}
 - [pyroscope.ebpf](../components/pyroscope/pyroscope.ebpf)
 - [pyroscope.java](../components/pyroscope/pyroscope.java)
+- [pyroscope.receive_http](../components/pyroscope/pyroscope.receive_http)
 - [pyroscope.scrape](../components/pyroscope/pyroscope.scrape)
 {{< /collapse >}}
 

diff --git a/docs/sources/reference/components/pyroscope/pyroscope.receive_http.md b/docs/sources/reference/components/pyroscope/pyroscope.receive_http.md
@@ -0,0 +1,119 @@
+---
+canonical: https://grafana.com/docs/alloy/latest/reference/components/pyroscope/pyroscope.receive_http/
+description: Learn about pyroscope.receive_http
+title: pyroscope.receive_http
+---
+
+# pyroscope.receive_http
+
+`pyroscope.receive_http` receives profiles over HTTP and forwards them to `pyroscope.*` components capable of receiving profiles.
+
+The HTTP API exposed is compatible with the Pyroscope [HTTP ingest API](https://grafana.com/docs/pyroscope/latest/configure-server/about-server-api/).
+This allows `pyroscope.receive_http` to act as a proxy for Pyroscope profiles, enabling flexible routing and distribution of profile data.
+
+## Usage
+
+```alloy
+pyroscope.receive_http "LABEL" {
+  http {
+    listen_address = "LISTEN_ADDRESS"
+    listen_port = PORT
+  }
+  forward_to = RECEIVER_LIST
+}
+```
+
+The component will start an HTTP server supporting the following endpoint.
+
+* `POST /ingest` - send profiles to the component, which will be forwarded to the receivers as configured in the `forward_to argument`. The request format must match the format of the Pyroscope ingest API.
+
+## Arguments
+
+The following arguments are supported:
+
+Name              | Type          | Description                                     | Default | Required
+------------------|---------------|-------------------------------------------------|---------|---------
+`forward_to` | `list(ProfilesReceiver)` | List of receivers to send profiles to. |         | yes
+
+## Blocks
+
+The following blocks are supported inside the definition of `pyroscope.receive_http`:
+
+Hierarchy | Name | Description                                        | Required
+----------|------|----------------------------------------------------|---------
+`http`    | `http` | Configures the HTTP server that receives requests. | no
+
+### http
+
+The `http` block configures the HTTP server.
+
+You can use the following arguments to configure the `http` block. Any omitted fields take their default values.
+
+Name                   | Type       | Description                                                                                                      | Default  | Required
+-----------------------|------------|------------------------------------------------------------------------------------------------------------------|----------|---------
+`conn_limit`           | `int`      | Maximum number of simultaneous HTTP connections. Defaults to 100.                                           | `0`      | no
+`listen_address`       | `string`   | Network address on which the server listens for new connections. Defaults to accepting all incoming connections. | `""`     | no
+`listen_port`          | `int`      | Port number on which the server listens for new connections.                                                     | `8080`   | no
+`server_idle_timeout`  | `duration` | Idle timeout for the HTTP server.                                                                                    | `"120s"` | no
+`server_read_timeout`  | `duration` | Read timeout for the HTTP server.                                                                                    | `"30s"`  | no
+`server_write_timeout` | `duration` | Write timeout for the HTTP server.                                                                                   | `"30s"`  | no
+
+## Exported fields
+
+`pyroscope.receive_http` does not export any fields.
+
+## Component health
+
+`pyroscope.receive_http` is reported as unhealthy if it is given an invalid configuration.
+
+## Example
+
+This example creates a `pyroscope.receive_http` component, which starts an HTTP server listening on `0.0.0.0` and port `9999`.
+The server receives profiles and forwards them to multiple `pyroscope.write` components, which write these profiles to different HTTP endpoints.
+```alloy
+// Receives profiles over HTTP
+pyroscope.receive_http "default" {
+  http {
+    listen_address = "0.0.0.0"
+    listen_port = 9999
+  }
+  forward_to = [pyroscope.write.staging.receiver, pyroscope.write.production.receiver]
+}
+
+// Send profiles to a staging Pyroscope instance
+pyroscope.write "staging" {
+  endpoint {
+    url = "http://pyroscope-staging:4040"
+  }
+}
+
+// Send profiles to a production Pyroscope instance
+pyroscope.write "production" {
+  endpoint {
+    url = "http://pyroscope-production:4040"
+  }
+}
+```
+
+{{< admonition type="note" >}}
+This example demonstrates forwarding to multiple `pyroscope.write` components.
+This configuration will duplicate the received profiles and send a copy to each configured `pyroscope.write` component.
+{{< /admonition >}}
+
+You can also create multiple `pyroscope.receive_http` components with different configurations to listen on different addresses or ports as needed. This flexibility allows you to design a setup that best fits your infrastructure and profile routing requirements.
+
+<!-- START GENERATED COMPATIBLE COMPONENTS -->
+
+## Compatible components
+
+`pyroscope.receive_http` can accept arguments from the following components:
+
+- Components that export [Pyroscope `ProfilesReceiver`](../../../compatibility/#pyroscope-profilesreceiver-exporters)
+
+
+{{< admonition type="note" >}}
+Connecting some components may not be sensible or components may require further configuration to make the connection work correctly.
+Refer to the linked documentation for more details.
+{{< /admonition >}}
+
+<!-- END GENERATED COMPATIBLE COMPONENTS -->
diff --git a/internal/component/all/all.go b/internal/component/all/all.go
@@ -140,6 +140,7 @@ import (
 	_ "github.com/grafana/alloy/internal/component/prometheus/write/queue"                   // Import prometheus.write.queue
 	_ "github.com/grafana/alloy/internal/component/pyroscope/ebpf"                           // Import pyroscope.ebpf
 	_ "github.com/grafana/alloy/internal/component/pyroscope/java"                           // Import pyroscope.java
+	_ "github.com/grafana/alloy/internal/component/pyroscope/receive_http"                   // Import pyroscope.receive_http
 	_ "github.com/grafana/alloy/internal/component/pyroscope/scrape"                         // Import pyroscope.scrape
 	_ "github.com/grafana/alloy/internal/component/pyroscope/write"                          // Import pyroscope.write
 	_ "github.com/grafana/alloy/internal/component/remote/http"                              // Import remote.http

diff --git a/internal/component/pyroscope/appender.go b/internal/component/pyroscope/appender.go
@@ -2,6 +2,9 @@ package pyroscope
 
 import (
 	"context"
+	"io"
+	"net/http"
+	"net/url"
 	"sync"
 	"time"
 
@@ -22,13 +25,20 @@ type Appendable interface {
 
 type Appender interface {
 	Append(ctx context.Context, labels labels.Labels, samples []*RawSample) error
+	AppendIngest(ctx context.Context, profile *IncomingProfile) error
 }
 
 type RawSample struct {
 	// raw_profile is the set of bytes of the pprof profile
 	RawProfile []byte
 }
 
+type IncomingProfile struct {
+	Body    io.ReadCloser
+	Headers http.Header
+	URL     *url.URL
+}
+
 var _ Appendable = (*Fanout)(nil)
 
 // Fanout supports the default Alloy style of appendables since it can go to multiple outputs. It also allows the intercepting of appends.
@@ -112,12 +122,33 @@ func (a *appender) Append(ctx context.Context, labels labels.Labels, samples []*
 	return multiErr
 }
 
+// AppendIngest satisfies the AppenderIngest interface.
+func (a *appender) AppendIngest(ctx context.Context, profile *IncomingProfile) error {
+	now := time.Now()
+	defer func() {
+		a.writeLatency.Observe(time.Since(now).Seconds())
+	}()
+	var multiErr error
+	for _, x := range a.children {
+		err := x.AppendIngest(ctx, profile)
+		if err != nil {
+			multiErr = multierror.Append(multiErr, err)
+		}
+	}
+	return multiErr
+}
+
 type AppendableFunc func(ctx context.Context, labels labels.Labels, samples []*RawSample) error
 
+func (f AppendableFunc) Appender() Appender {
+	return f
+}
+
 func (f AppendableFunc) Append(ctx context.Context, labels labels.Labels, samples []*RawSample) error {
 	return f(ctx, labels, samples)
 }
 
-func (f AppendableFunc) Appender() Appender {
-	return f
+func (f AppendableFunc) AppendIngest(_ context.Context, _ *IncomingProfile) error {
+	// This is a no-op implementation
+	return nil
 }