feat: initial cloak implementation

Makefile

@@ -45,7 +45,12 @@ start.pink: __start__
 __start__:
 	@env $$(cat .dev.env | xargs) PORT=${PORT} LOGFLARE_GRPC_PORT=${LOGFLARE_GRPC_PORT} iex --sname ${ERL_NAME} --cookie ${ERL_COOKIE} -S mix phx.server
 
-.PHONY: __start__
+
+migrate:
+	@env $$(cat .dev.env | xargs) mix ecto.migrate
+
+
+.PHONY: __start__ migrate
 
 # Encryption and decryption of secrets
 # Usage:
@@ -87,7 +92,6 @@ $(addprefix decrypt.,${envs}): decrypt.%: \
 	.$$*.gcloud.json \
  	.$$*.env \
  	.$$*.cacert.pem \
- 	.$$*.cacert.key \
  	.$$*.cert.key \
  	.$$*.cert.pem \
  	.$$*.db-client-cert.pem \
@@ -98,7 +102,6 @@ $(addprefix encrypt.,${envs}): encrypt.%: \
 	.$$*.gcloud.json.enc \
 	.$$*.env.enc \
 	.$$*.cacert.pem.enc \
-	.$$*.cacert.key.enc \
 	.$$*.cert.key.enc \
 	.$$*.cert.pem.enc \
  	.$$*.db-client-cert.pem.enc \

VERSION

@@ -1 +1 @@
-1.7.15
+1.8.0

cloudbuild/startup.sh

@@ -1,4 +1,9 @@
 #! /bin/sh
+if [ -z "$LOGFLARE_DB_ENCRYPTION_KEY" ]; then
+    echo "LOGFLARE_DB_ENCRYPTION_KEY is not set!" 1>&2
+    exit 1
+fi
+echo $?
 
 # wait for networking to be ready before starting Erlang
 echo 'Sleeping for 15 seconds for GCE networking to be ready...'

config/config.exs

@@ -6,12 +6,17 @@
 import Config
 
 # General application configuration
+
+hardcoded_encryption_key = "Q+IS7ogkzRxsj+zAIB1u6jNFquxkFzSrBZXItN27K/Q="
+
 config :logflare,
   ecto_repos: [Logflare.Repo],
   # https://cloud.google.com/compute/docs/instances/deleting-instance#delete_timeout
   # preemtible is 30 seconds from shutdown to sigterm
   # normal instances can be more than 90 seconds
-  sigterm_shutdown_grace_period_ms: 15_000
+  sigterm_shutdown_grace_period_ms: 15_000,
+  encryption_key_fallback: hardcoded_encryption_key,
+  encryption_key_default: hardcoded_encryption_key
 
 config :logflare, Logflare.Alerting, min_cluster_size: 1, enabled: true
 
@@ -129,4 +134,6 @@ config :opentelemetry,
   span_processor: :batch,
   traces_exporter: :none
 
+config :logflare, Logflare.Vault, json_library: Jason
+
 import_config "#{Mix.env()}.exs"

config/runtime.exs

@@ -23,7 +23,9 @@ config :logflare,
          single_tenant: System.get_env("LOGFLARE_SINGLE_TENANT", "false") == "true",
          supabase_mode: System.get_env("LOGFLARE_SUPABASE_MODE", "false") == "true",
          api_key: System.get_env("LOGFLARE_API_KEY"),
-         cache_stats: System.get_env("LOGFLARE_CACHE_STATS", "false") == "true"
+         cache_stats: System.get_env("LOGFLARE_CACHE_STATS", "false") == "true",
+         encryption_key_default: System.get_env("LOGFLARE_DB_ENCRYPTION_KEY"),
+         encryption_key_retired: System.get_env("LOGFLARE_DB_ENCRYPTION_KEY_RETIRED")
        ]
        |> filter_nil_kv_pairs.()
 

config/test.exs

@@ -7,7 +7,8 @@ config :logflare, LogflareWeb.Endpoint,
   server: false
 
 config :logflare,
-  env: :test
+  env: :test,
+  encryption_key_default: "Q+IS7ogkzRxsj+zAIB1u6jNFquxkFzSrBZXItN27K/Q="
 
 config :logflare, Logflare.Cluster.Utils, min_cluster_size: 1
 

docs/docs.logflare.com/docs/self-hosting/index.md

@@ -19,24 +19,26 @@ All browser authentication will be disabled when in single-tenant mode.
 
 ### Common Configuration
 
-| Env Var                                | Type                                                                | Description                                                                                                                                                      |
-| -------------------------------------- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `LOGFLARE_SINGLE_TENANT`               | Boolean, defaults to `false`                                        | If enabled, a singular user will be seeded. All browser usage will default to the user.                                                                          |
-| `LOGFLARE_API_KEY`                     | string, defaults to `nil`                                           | If set, this API Key can be used for interacting with the Logflare API. API key will be automatically generated if not set.                                      |
-| `LOGFLARE_SUPABASE_MODE`               | Boolean, defaults to `false`                                        | A special mode for Logflare, where Supabase-specific resources will be seeded. Intended for Suapbase self-hosted usage.                                          |
-| `PHX_HTTP_PORT`                        | Integer, defaults to `4000`                                         | Allows configuration of the HTTP server port.                                                                                                                    |
-| `DB_SCHEMA`                            | String, defaults to `nil`                                           | Allows configuration of the database schema to scope Logflare operations.                                                                                        |
-| `LOGFLARE_LOG_LEVEL`                   | String, defaults to `info`. <br/>Options: `error`,`warning`, `info` | Allows runtime configuration of log level.                                                                                                                       |
-| `LOGFLARE_NODE_HOST`                   | string, defaults to `127.0.0.1`                                     | Sets node host on startup, which affects the node name `logflare@<host>`                                                                                         |
-| `LOGFLARE_LOGGER_METADATA_CLUSTER`     | string, defaults to `nil`                                           | Sets global logging metadata for the cluster name. Useful for filtering logs by cluster name.                                                                    |
-| `LOGFLARE_PUBSUB_POOL_SIZE`            | Integer, defaults to `10`                                           | Sets the number of `Phoenix.PubSub.PG2` partitions to be created. Should be configured to the number of cores of your server for optimal multi-node performance. |
-| `LOGFLARE_ALERTS_ENABLED`              | Boolean, defaults to `true`                                         | Flag for enabling and disabling query alerts.                                                                                                                    |
-| `LOGFLARE_ALERTS_MIN_CLUSTER_SIZE`     | Integer, defaults to `1`                                            | Sets the required cluster size for Query Alerts to be run. If cluster size is below the provided value, query alerts will not run.                               |
-| `LOGFLARE_MIN_CLUSTER_SIZE`            | Integer, defaults to `1`                                            | Sets the target cluster size, and emits a warning log periodically if the cluster is below the set number of nodes..                                             |
-| `LOGFLARE_OTEL_ENDPOINT`               | String, defaults to `nil`                                           | Sets the OpenTelemetry Endpoint to send traces to via gRPC. Port number can be included, such as `https://logflare.app:443`                                      |
-| `LOGFLARE_OTEL_SOURCE_UUID`            | String, defaults to `nil`, optionally required for OpenTelemetry.   | Sets the appropriate header for ingesting OpenTelemetry events into a Logflare source.                                                                           |
-| `LOGFLARE_OTEL_ACCESS_TOKEN`           | String, defaults to `nil`, optionally required for OpenTelemetry.   | Sets the appropriate authentication header for ingesting OpenTelemetry events into a Logflare source.                                                            |
-| `LOGFLARE_OPEN_TELEMETRY_SAMPLE_RATIO` | Float, defaults to `0.001`, optionally required for OpenTelemetry.  | Sets the sample ratio for server traces. Ingestion and Endpoint routes are dropped and are not included in tracing.                                              |
+| Env Var                                | Type                                                                | Description                                                                                                                                                                             |
+| -------------------------------------- | ------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `LOGFLARE_DB_ENCRYPTION_KEY`           | Base64 encryption key, **required**                                 | Encryption key used for encrypting sensitive data.                                                                                                                                      |
+| `LOGFLARE_DB_ENCRYPTION_KEY_RETIRED`       | Base64 encryption key, defaults to `nil`                            | The deprecated encryption key to migrate existing database secrets from. Data will be migrated to the key set under `LOGFLARE_DB_ENCRYPTION_KEY`. Used for encryption key rolling only. |
+| `LOGFLARE_SINGLE_TENANT`               | Boolean, defaults to `false`                                        | If enabled, a singular user will be seeded. All browser usage will default to the user.                                                                                                 |
+| `LOGFLARE_API_KEY`                     | string, defaults to `nil`                                           | If set, this API Key can be used for interacting with the Logflare API. API key will be automatically generated if not set.                                                             |
+| `LOGFLARE_SUPABASE_MODE`               | Boolean, defaults to `false`                                        | A special mode for Logflare, where Supabase-specific resources will be seeded. Intended for Suapbase self-hosted usage.                                                                 |
+| `PHX_HTTP_PORT`                        | Integer, defaults to `4000`                                         | Allows configuration of the HTTP server port.                                                                                                                                           |
+| `DB_SCHEMA`                            | String, defaults to `nil`                                           | Allows configuration of the database schema to scope Logflare operations.                                                                                                               |
+| `LOGFLARE_LOG_LEVEL`                   | String, defaults to `info`. <br/>Options: `error`,`warning`, `info` | Allows runtime configuration of log level.                                                                                                                                              |
+| `LOGFLARE_NODE_HOST`                   | string, defaults to `127.0.0.1`                                     | Sets node host on startup, which affects the node name `logflare@<host>`                                                                                                                |
+| `LOGFLARE_LOGGER_METADATA_CLUSTER`     | string, defaults to `nil`                                           | Sets global logging metadata for the cluster name. Useful for filtering logs by cluster name.                                                                                           |
+| `LOGFLARE_PUBSUB_POOL_SIZE`            | Integer, defaults to `10`                                           | Sets the number of `Phoenix.PubSub.PG2` partitions to be created. Should be configured to the number of cores of your server for optimal multi-node performance.                        |
+| `LOGFLARE_ALERTS_ENABLED`              | Boolean, defaults to `true`                                         | Flag for enabling and disabling query alerts.                                                                                                                                           |
+| `LOGFLARE_ALERTS_MIN_CLUSTER_SIZE`     | Integer, defaults to `1`                                            | Sets the required cluster size for Query Alerts to be run. If cluster size is below the provided value, query alerts will not run.                                                      |
+| `LOGFLARE_MIN_CLUSTER_SIZE`            | Integer, defaults to `1`                                            | Sets the target cluster size, and emits a warning log periodically if the cluster is below the set number of nodes..                                                                    |
+| `LOGFLARE_OTEL_ENDPOINT`               | String, defaults to `nil`                                           | Sets the OpenTelemetry Endpoint to send traces to via gRPC. Port number can be included, such as `https://logflare.app:443`                                                             |
+| `LOGFLARE_OTEL_SOURCE_UUID`            | String, defaults to `nil`, optionally required for OpenTelemetry.   | Sets the appropriate header for ingesting OpenTelemetry events into a Logflare source.                                                                                                  |
+| `LOGFLARE_OTEL_ACCESS_TOKEN`           | String, defaults to `nil`, optionally required for OpenTelemetry.   | Sets the appropriate authentication header for ingesting OpenTelemetry events into a Logflare source.                                                                                   |
+| `LOGFLARE_OPEN_TELEMETRY_SAMPLE_RATIO` | Float, defaults to `0.001`, optionally required for OpenTelemetry.  | Sets the sample ratio for server traces. Ingestion and Endpoint routes are dropped and are not included in tracing.                                                                     |
 
 LOGFLARE_OPEN_TELEMETRY_SAMPLE_RATIO
 Additional environment variable configurations for the OpenTelemetry libraries used can be found [here](https://hexdocs.pm/opentelemetry_exporter/readme.html).perf/bq-pipeline-sharding
@@ -56,6 +58,25 @@ Additional environment variable configurations for the OpenTelemetry libraries u
 | `POSTGRES_BACKEND_URL`    | string, required                       | PostgreSQL connection string, for connecting to the database. User must have sufficient permssions to manage the schema. |
 | `POSTGRES_BACKEND_SCHEMA` | string, optional, defaults to `public` | Specifies the database schema to scope all operations.                                                                   |
 
+## Database Encryption
+
+Certain database columns that store sensitive data are encrypted with the `LOGFLARE_DB_ENCRYPTION_KEY` key.
+Encryption keys must be Base64 encoded.
+
+Cipher used is AES with a 256-bit key in GCM mode.
+
+### Rolling Encryption Keys
+
+In order to roll encryption keys and migrate existing encrypted data, use the `LOGFLARE_DB_ENCRYPTION_KEY_RETIRED` environment variable.
+
+Steps to perform the migration are:
+
+1. Move the retired encryption key from `LOGFLARE_DB_ENCRYPTION_KEY` to `LOGFLARE_DB_ENCRYPTION_KEY_RETIRED`.
+2. Generate a new encryption key and set it to `LOGFLARE_DB_ENCRYPTION_KEY`.
+3. Restart or deploy the server with the new environment variables.
+4. Upon successful server startup, an `info` log will be emitted that says that an retired encryption key is detected, and the migration will be initiated to transition all data encrypted with the retired key to be encrypted with the new key.
+5. Once the migration is complete, the retired encryption key can be safely removed.
+
 ## BigQuery Setup
 
 ### Pre-requisites

lib/logflare/application.ex

@@ -45,6 +45,7 @@ defmodule Logflare.Application do
         PubSubRates,
         Logs.RejectedLogEvents,
         Logflare.Repo,
+        Logflare.Vault,
         Logflare.Backends,
         {Registry,
          name: Logflare.V1SourceRegistry,
@@ -77,6 +78,7 @@ defmodule Logflare.Application do
         {Task.Supervisor, name: Logflare.TaskSupervisor},
         {Cluster.Supervisor, [topologies, [name: Logflare.ClusterSupervisor]]},
         Logflare.Repo,
+        Logflare.Vault,
         {Phoenix.PubSub, name: Logflare.PubSub, pool_size: pool_size},
         Logs.LogEvents.Cache,
         PubSubRates,

lib/logflare/backends.ex

@@ -97,7 +97,6 @@ defmodule Logflare.Backends do
     backend =
       %Backend{}
       |> Backend.changeset(attrs)
-      |> validate_config()
       |> Repo.insert()
 
     with {:ok, updated} <- backend do
@@ -115,7 +114,6 @@ defmodule Logflare.Backends do
     backend_config =
       backend
       |> Backend.changeset(attrs)
-      |> validate_config()
       |> Repo.update()
 
     with {:ok, updated} <- backend_config do
@@ -156,32 +154,20 @@ defmodule Logflare.Backends do
     end
   end
 
-  # common config validation function
-  defp validate_config(%{valid?: true} = changeset) do
-    type = Ecto.Changeset.get_field(changeset, :type)
-    mod = Backend.adaptor_mapping()[type]
-
-    Ecto.Changeset.validate_change(changeset, :config, fn :config, config ->
-      case Adaptor.cast_and_validate_config(mod, config) do
-        %{valid?: true} -> []
-        %{valid?: false, errors: errors} -> for {key, err} <- errors, do: {:"config.#{key}", err}
-      end
-    end)
-  end
-
-  defp validate_config(changeset), do: changeset
-
   # common typecasting from string map to attom for config
   defp typecast_config_string_map_to_atom_map(nil), do: nil
 
   defp typecast_config_string_map_to_atom_map(%Backend{type: type} = backend) do
     mod = Backend.adaptor_mapping()[type]
 
-    Map.update!(backend, :config, fn config ->
-      (config || %{})
-      |> mod.cast_config()
-      |> Ecto.Changeset.apply_changes()
-    end)
+    updated =
+      Map.update!(backend, :config_encrypted, fn config ->
+        (config || %{})
+        |> mod.cast_config()
+        |> Ecto.Changeset.apply_changes()
+      end)
+
+    Map.put(updated, :config, updated.config_encrypted)
   end
 
   @doc """

lib/logflare/backends/backend.ex

@@ -23,8 +23,9 @@ defmodule Logflare.Backends.Backend do
     field(:description, :string)
     field(:token, Ecto.UUID, autogenerate: true)
     field(:type, Ecto.Enum, values: Map.keys(@adaptor_mapping))
-    # TODO: maybe use polymorphic embeds
+    # TODO(Ziinc): make virtual once cluster is using encrypted fields fully
     field(:config, :map)
+    field(:config_encrypted, Logflare.Ecto.EncryptedMap)
     many_to_many(:sources, Source, join_through: "sources_backends")
     belongs_to(:user, User)
     has_many(:rules, Rule)
@@ -40,8 +41,36 @@ defmodule Logflare.Backends.Backend do
     |> cast(attrs, [:type, :config, :user_id, :name, :description, :metadata])
     |> validate_required([:user_id, :type, :config, :name])
     |> validate_inclusion(:type, Map.keys(@adaptor_mapping))
+    |> do_config_change()
+    |> validate_config()
   end
 
+  # temp function
+  defp do_config_change(%Ecto.Changeset{changes: %{config: config}} = changeset) do
+    changeset
+    |> put_change(:config_encrypted, config)
+
+    # TODO(Ziinc): uncomment once cluster is using encrypted fields fully
+    # |> delete_change(:config)
+  end
+
+  defp do_config_change(changeset), do: changeset
+
+  # common config validation function
+  defp validate_config(%{valid?: true} = changeset) do
+    type = Ecto.Changeset.get_field(changeset, :type)
+    mod = adaptor_mapping()[type]
+
+    Ecto.Changeset.validate_change(changeset, :config, fn :config, config ->
+      case Adaptor.cast_and_validate_config(mod, config) do
+        %{valid?: true} -> []
+        %{valid?: false, errors: errors} -> for {key, err} <- errors, do: {:"config.#{key}", err}
+      end
+    end)
+  end
+
+  defp validate_config(changeset), do: changeset
+
   @spec child_spec(Source.t(), Backend.t()) :: map()
   defdelegate child_spec(source, backend), to: Adaptor
 

lib/logflare/ecto/encrypted_map.ex

@@ -0,0 +1,3 @@
+defmodule Logflare.Ecto.EncryptedMap do
+  use Cloak.Ecto.Map, vault: Logflare.Vault
+end