Skip to content

Commit

Permalink
feat: initial cloak implementation (#2163)
Browse files Browse the repository at this point in the history 
* feat: initial cloak implementation

* chore: version bump to v1.8.0

* feat: add migrator test, use retired phasing for key retiring.

* feat: move migration logic to a Task to prevent GenServer blocking

* chore: add inline comments

* chore: fix wrong docs function name

* chore: compile error for interpolation of migration value

* feat: hardcoded fallback encryption key

* chore: update secrets for all envs

* chore: fix test module name

* chore: fix run.sh sleep

* chore: fix failing backend tests

* feat: adjust code to be backwards compatible

* chore: formatting

* chore: remove dbg call

* chore: remove newline

* chore: add in process killing at end of migration

* chore: fix process exit

* chore: add a small timer, unlink vault and kill

* chore: reduce sleep

* chore: fix failing test
  • Loading branch information
@Ziinc
Ziinc authored Aug 7, 2024
1 parent c720745 commit b1b2613
Show file tree
Hide file tree
Showing 22 changed files with 391 additions and 63 deletions.
9 changes: 6 additions & 3 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,12 @@ start.pink: __start__
__start__:
@env $$(cat .dev.env | xargs) PORT=${PORT} LOGFLARE_GRPC_PORT=${LOGFLARE_GRPC_PORT} iex --sname ${ERL_NAME} --cookie ${ERL_COOKIE} -S mix phx.server

.PHONY: __start__

migrate:
@env $$(cat .dev.env | xargs) mix ecto.migrate


.PHONY: __start__ migrate

# Encryption and decryption of secrets
# Usage:
Expand Down Expand Up @@ -87,7 +92,6 @@ $(addprefix decrypt.,${envs}): decrypt.%: \
.$$*.gcloud.json \
.$$*.env \
.$$*.cacert.pem \
.$$*.cacert.key \
.$$*.cert.key \
.$$*.cert.pem \
.$$*.db-client-cert.pem \
Expand All @@ -98,7 +102,6 @@ $(addprefix encrypt.,${envs}): encrypt.%: \
.$$*.gcloud.json.enc \
.$$*.env.enc \
.$$*.cacert.pem.enc \
.$$*.cacert.key.enc \
.$$*.cert.key.enc \
.$$*.cert.pem.enc \
.$$*.db-client-cert.pem.enc \
Expand Down
2 changes: 1 addition & 1 deletion VERSION
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.7.15
1.8.0
Binary file modified cloudbuild/.dev.env.enc
View file
Binary file not shown.
Binary file modified cloudbuild/.prod.env.enc
View file
Binary file not shown.
Binary file modified cloudbuild/.staging.env.enc
View file
Binary file not shown.
5 changes: 5 additions & 0 deletions cloudbuild/startup.sh
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
#! /bin/sh
if [ -z "$LOGFLARE_DB_ENCRYPTION_KEY" ]; then
echo "LOGFLARE_DB_ENCRYPTION_KEY is not set!" 1>&2
exit 1
fi
echo $?

# wait for networking to be ready before starting Erlang
echo 'Sleeping for 15 seconds for GCE networking to be ready...'
Expand Down
9 changes: 8 additions & 1 deletion config/config.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,17 @@
import Config

# General application configuration

hardcoded_encryption_key = "Q+IS7ogkzRxsj+zAIB1u6jNFquxkFzSrBZXItN27K/Q="

config :logflare,
ecto_repos: [Logflare.Repo],
# https://cloud.google.com/compute/docs/instances/deleting-instance#delete_timeout
# preemtible is 30 seconds from shutdown to sigterm
# normal instances can be more than 90 seconds
sigterm_shutdown_grace_period_ms: 15_000
sigterm_shutdown_grace_period_ms: 15_000,
encryption_key_fallback: hardcoded_encryption_key,
encryption_key_default: hardcoded_encryption_key

config :logflare, Logflare.Alerting, min_cluster_size: 1, enabled: true

Expand Down Expand Up @@ -129,4 +134,6 @@ config :opentelemetry,
span_processor: :batch,
traces_exporter: :none

config :logflare, Logflare.Vault, json_library: Jason

import_config "#{Mix.env()}.exs"
4 changes: 3 additions & 1 deletion config/runtime.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@ config :logflare,
single_tenant: System.get_env("LOGFLARE_SINGLE_TENANT", "false") == "true",
supabase_mode: System.get_env("LOGFLARE_SUPABASE_MODE", "false") == "true",
api_key: System.get_env("LOGFLARE_API_KEY"),
cache_stats: System.get_env("LOGFLARE_CACHE_STATS", "false") == "true"
cache_stats: System.get_env("LOGFLARE_CACHE_STATS", "false") == "true",
encryption_key_default: System.get_env("LOGFLARE_DB_ENCRYPTION_KEY"),
encryption_key_retired: System.get_env("LOGFLARE_DB_ENCRYPTION_KEY_RETIRED")
]
|> filter_nil_kv_pairs.()

Expand Down
3 changes: 2 additions & 1 deletion config/test.exs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ config :logflare, LogflareWeb.Endpoint,
server: false

config :logflare,
env: :test
env: :test,
encryption_key_default: "Q+IS7ogkzRxsj+zAIB1u6jNFquxkFzSrBZXItN27K/Q="

config :logflare, Logflare.Cluster.Utils, min_cluster_size: 1

Expand Down
57 changes: 39 additions & 18 deletions docs/docs.logflare.com/docs/self-hosting/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,26 @@ All browser authentication will be disabled when in single-tenant mode.

### Common Configuration

| Env Var | Type | Description |
| -------------------------------------- | ------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `LOGFLARE_SINGLE_TENANT` | Boolean, defaults to `false` | If enabled, a singular user will be seeded. All browser usage will default to the user. |
| `LOGFLARE_API_KEY` | string, defaults to `nil` | If set, this API Key can be used for interacting with the Logflare API. API key will be automatically generated if not set. |
| `LOGFLARE_SUPABASE_MODE` | Boolean, defaults to `false` | A special mode for Logflare, where Supabase-specific resources will be seeded. Intended for Suapbase self-hosted usage. |
| `PHX_HTTP_PORT` | Integer, defaults to `4000` | Allows configuration of the HTTP server port. |
| `DB_SCHEMA` | String, defaults to `nil` | Allows configuration of the database schema to scope Logflare operations. |
| `LOGFLARE_LOG_LEVEL` | String, defaults to `info`. <br/>Options: `error`,`warning`, `info` | Allows runtime configuration of log level. |
| `LOGFLARE_NODE_HOST` | string, defaults to `127.0.0.1` | Sets node host on startup, which affects the node name `logflare@<host>` |
| `LOGFLARE_LOGGER_METADATA_CLUSTER` | string, defaults to `nil` | Sets global logging metadata for the cluster name. Useful for filtering logs by cluster name. |
| `LOGFLARE_PUBSUB_POOL_SIZE` | Integer, defaults to `10` | Sets the number of `Phoenix.PubSub.PG2` partitions to be created. Should be configured to the number of cores of your server for optimal multi-node performance. |
| `LOGFLARE_ALERTS_ENABLED` | Boolean, defaults to `true` | Flag for enabling and disabling query alerts. |
| `LOGFLARE_ALERTS_MIN_CLUSTER_SIZE` | Integer, defaults to `1` | Sets the required cluster size for Query Alerts to be run. If cluster size is below the provided value, query alerts will not run. |
| `LOGFLARE_MIN_CLUSTER_SIZE` | Integer, defaults to `1` | Sets the target cluster size, and emits a warning log periodically if the cluster is below the set number of nodes.. |
| `LOGFLARE_OTEL_ENDPOINT` | String, defaults to `nil` | Sets the OpenTelemetry Endpoint to send traces to via gRPC. Port number can be included, such as `https://logflare.app:443` |
| `LOGFLARE_OTEL_SOURCE_UUID` | String, defaults to `nil`, optionally required for OpenTelemetry. | Sets the appropriate header for ingesting OpenTelemetry events into a Logflare source. |
| `LOGFLARE_OTEL_ACCESS_TOKEN` | String, defaults to `nil`, optionally required for OpenTelemetry. | Sets the appropriate authentication header for ingesting OpenTelemetry events into a Logflare source. |
| `LOGFLARE_OPEN_TELEMETRY_SAMPLE_RATIO` | Float, defaults to `0.001`, optionally required for OpenTelemetry. | Sets the sample ratio for server traces. Ingestion and Endpoint routes are dropped and are not included in tracing. |
| Env Var | Type | Description |
| -------------------------------------- | ------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `LOGFLARE_DB_ENCRYPTION_KEY` | Base64 encryption key, **required** | Encryption key used for encrypting sensitive data. |
| `LOGFLARE_DB_ENCRYPTION_KEY_RETIRED` | Base64 encryption key, defaults to `nil` | The deprecated encryption key to migrate existing database secrets from. Data will be migrated to the key set under `LOGFLARE_DB_ENCRYPTION_KEY`. Used for encryption key rolling only. |
| `LOGFLARE_SINGLE_TENANT` | Boolean, defaults to `false` | If enabled, a singular user will be seeded. All browser usage will default to the user. |
| `LOGFLARE_API_KEY` | string, defaults to `nil` | If set, this API Key can be used for interacting with the Logflare API. API key will be automatically generated if not set. |
| `LOGFLARE_SUPABASE_MODE` | Boolean, defaults to `false` | A special mode for Logflare, where Supabase-specific resources will be seeded. Intended for Suapbase self-hosted usage. |
| `PHX_HTTP_PORT` | Integer, defaults to `4000` | Allows configuration of the HTTP server port. |
| `DB_SCHEMA` | String, defaults to `nil` | Allows configuration of the database schema to scope Logflare operations. |
| `LOGFLARE_LOG_LEVEL` | String, defaults to `info`. <br/>Options: `error`,`warning`, `info` | Allows runtime configuration of log level. |
| `LOGFLARE_NODE_HOST` | string, defaults to `127.0.0.1` | Sets node host on startup, which affects the node name `logflare@<host>` |
| `LOGFLARE_LOGGER_METADATA_CLUSTER` | string, defaults to `nil` | Sets global logging metadata for the cluster name. Useful for filtering logs by cluster name. |
| `LOGFLARE_PUBSUB_POOL_SIZE` | Integer, defaults to `10` | Sets the number of `Phoenix.PubSub.PG2` partitions to be created. Should be configured to the number of cores of your server for optimal multi-node performance. |
| `LOGFLARE_ALERTS_ENABLED` | Boolean, defaults to `true` | Flag for enabling and disabling query alerts. |
| `LOGFLARE_ALERTS_MIN_CLUSTER_SIZE` | Integer, defaults to `1` | Sets the required cluster size for Query Alerts to be run. If cluster size is below the provided value, query alerts will not run. |
| `LOGFLARE_MIN_CLUSTER_SIZE` | Integer, defaults to `1` | Sets the target cluster size, and emits a warning log periodically if the cluster is below the set number of nodes.. |
| `LOGFLARE_OTEL_ENDPOINT` | String, defaults to `nil` | Sets the OpenTelemetry Endpoint to send traces to via gRPC. Port number can be included, such as `https://logflare.app:443` |
| `LOGFLARE_OTEL_SOURCE_UUID` | String, defaults to `nil`, optionally required for OpenTelemetry. | Sets the appropriate header for ingesting OpenTelemetry events into a Logflare source. |
| `LOGFLARE_OTEL_ACCESS_TOKEN` | String, defaults to `nil`, optionally required for OpenTelemetry. | Sets the appropriate authentication header for ingesting OpenTelemetry events into a Logflare source. |
| `LOGFLARE_OPEN_TELEMETRY_SAMPLE_RATIO` | Float, defaults to `0.001`, optionally required for OpenTelemetry. | Sets the sample ratio for server traces. Ingestion and Endpoint routes are dropped and are not included in tracing. |

LOGFLARE_OPEN_TELEMETRY_SAMPLE_RATIO
Additional environment variable configurations for the OpenTelemetry libraries used can be found [here](https://hexdocs.pm/opentelemetry_exporter/readme.html).perf/bq-pipeline-sharding
Expand All @@ -56,6 +58,25 @@ Additional environment variable configurations for the OpenTelemetry libraries u
| `POSTGRES_BACKEND_URL` | string, required | PostgreSQL connection string, for connecting to the database. User must have sufficient permssions to manage the schema. |
| `POSTGRES_BACKEND_SCHEMA` | string, optional, defaults to `public` | Specifies the database schema to scope all operations. |

## Database Encryption

Certain database columns that store sensitive data are encrypted with the `LOGFLARE_DB_ENCRYPTION_KEY` key.
Encryption keys must be Base64 encoded.

Cipher used is AES with a 256-bit key in GCM mode.

### Rolling Encryption Keys

In order to roll encryption keys and migrate existing encrypted data, use the `LOGFLARE_DB_ENCRYPTION_KEY_RETIRED` environment variable.

Steps to perform the migration are:

1. Move the retired encryption key from `LOGFLARE_DB_ENCRYPTION_KEY` to `LOGFLARE_DB_ENCRYPTION_KEY_RETIRED`.
2. Generate a new encryption key and set it to `LOGFLARE_DB_ENCRYPTION_KEY`.
3. Restart or deploy the server with the new environment variables.
4. Upon successful server startup, an `info` log will be emitted that says that an retired encryption key is detected, and the migration will be initiated to transition all data encrypted with the retired key to be encrypted with the new key.
5. Once the migration is complete, the retired encryption key can be safely removed.

## BigQuery Setup

### Pre-requisites
Expand Down
2 changes: 2 additions & 0 deletions lib/logflare/application.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ defmodule Logflare.Application do
PubSubRates,
Logs.RejectedLogEvents,
Logflare.Repo,
Logflare.Vault,
Logflare.Backends,
{Registry,
name: Logflare.V1SourceRegistry,
Expand Down Expand Up @@ -77,6 +78,7 @@ defmodule Logflare.Application do
{Task.Supervisor, name: Logflare.TaskSupervisor},
{Cluster.Supervisor, [topologies, [name: Logflare.ClusterSupervisor]]},
Logflare.Repo,
Logflare.Vault,
{Phoenix.PubSub, name: Logflare.PubSub, pool_size: pool_size},
Logs.LogEvents.Cache,
PubSubRates,
Expand Down
30 changes: 8 additions & 22 deletions lib/logflare/backends.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,6 @@ defmodule Logflare.Backends do
backend =
%Backend{}
|> Backend.changeset(attrs)
|> validate_config()
|> Repo.insert()

with {:ok, updated} <- backend do
Expand All @@ -115,7 +114,6 @@ defmodule Logflare.Backends do
backend_config =
backend
|> Backend.changeset(attrs)
|> validate_config()
|> Repo.update()

with {:ok, updated} <- backend_config do
Expand Down Expand Up @@ -156,32 +154,20 @@ defmodule Logflare.Backends do
end
end

# common config validation function
defp validate_config(%{valid?: true} = changeset) do
type = Ecto.Changeset.get_field(changeset, :type)
mod = Backend.adaptor_mapping()[type]

Ecto.Changeset.validate_change(changeset, :config, fn :config, config ->
case Adaptor.cast_and_validate_config(mod, config) do
%{valid?: true} -> []
%{valid?: false, errors: errors} -> for {key, err} <- errors, do: {:"config.#{key}", err}
end
end)
end

defp validate_config(changeset), do: changeset

# common typecasting from string map to attom for config
defp typecast_config_string_map_to_atom_map(nil), do: nil

defp typecast_config_string_map_to_atom_map(%Backend{type: type} = backend) do
mod = Backend.adaptor_mapping()[type]

Map.update!(backend, :config, fn config ->
(config || %{})
|> mod.cast_config()
|> Ecto.Changeset.apply_changes()
end)
updated =
Map.update!(backend, :config_encrypted, fn config ->
(config || %{})
|> mod.cast_config()
|> Ecto.Changeset.apply_changes()
end)

Map.put(updated, :config, updated.config_encrypted)
end

@doc """
Expand Down
31 changes: 30 additions & 1 deletion lib/logflare/backends/backend.ex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,9 @@ defmodule Logflare.Backends.Backend do
field(:description, :string)
field(:token, Ecto.UUID, autogenerate: true)
field(:type, Ecto.Enum, values: Map.keys(@adaptor_mapping))
# TODO: maybe use polymorphic embeds
# TODO(Ziinc): make virtual once cluster is using encrypted fields fully
field(:config, :map)
field(:config_encrypted, Logflare.Ecto.EncryptedMap)
many_to_many(:sources, Source, join_through: "sources_backends")
belongs_to(:user, User)
has_many(:rules, Rule)
Expand All @@ -40,8 +41,36 @@ defmodule Logflare.Backends.Backend do
|> cast(attrs, [:type, :config, :user_id, :name, :description, :metadata])
|> validate_required([:user_id, :type, :config, :name])
|> validate_inclusion(:type, Map.keys(@adaptor_mapping))
|> do_config_change()
|> validate_config()
end

# temp function
defp do_config_change(%Ecto.Changeset{changes: %{config: config}} = changeset) do
changeset
|> put_change(:config_encrypted, config)

# TODO(Ziinc): uncomment once cluster is using encrypted fields fully
# |> delete_change(:config)
end

defp do_config_change(changeset), do: changeset

# common config validation function
defp validate_config(%{valid?: true} = changeset) do
type = Ecto.Changeset.get_field(changeset, :type)
mod = adaptor_mapping()[type]

Ecto.Changeset.validate_change(changeset, :config, fn :config, config ->
case Adaptor.cast_and_validate_config(mod, config) do
%{valid?: true} -> []
%{valid?: false, errors: errors} -> for {key, err} <- errors, do: {:"config.#{key}", err}
end
end)
end

defp validate_config(changeset), do: changeset

@spec child_spec(Source.t(), Backend.t()) :: map()
defdelegate child_spec(source, backend), to: Adaptor

Expand Down
3 changes: 3 additions & 0 deletions lib/logflare/ecto/encrypted_map.ex
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
defmodule Logflare.Ecto.EncryptedMap do
use Cloak.Ecto.Map, vault: Logflare.Vault
end
Loading

0 comments on commit b1b2613

Please sign in to comment.