From 0d58a12cb068b9925f0ea2910282fa66347014e3 Mon Sep 17 00:00:00 2001 From: Dillon Nys Date: Wed, 9 Oct 2024 19:24:07 -0700 Subject: [PATCH] chore(runtime): Load project config from environment Loads the resolved project AST from the platform environment/filesystem at startup. This will be used to configure variables, secrets, auth, and routing without managing many environment variables. --- packages/celest/lib/src/core/context.dart | 16 ++ .../celest/lib/src/runtime/configuration.dart | 75 +++++++++ .../lib/src/runtime/http/environment.dart | 67 -------- packages/celest/lib/src/runtime/serve.dart | 6 +- packages/celest/pubspec.yaml | 3 + .../test/runtime/configuration_test.dart | 148 ++++++++++++++++++ 6 files changed, 244 insertions(+), 71 deletions(-) create mode 100644 packages/celest/lib/src/runtime/configuration.dart delete mode 100644 packages/celest/lib/src/runtime/http/environment.dart create mode 100644 packages/celest/test/runtime/configuration_test.dart diff --git a/packages/celest/lib/src/core/context.dart b/packages/celest/lib/src/core/context.dart index da18343e..8eb776c8 100644 --- a/packages/celest/lib/src/core/context.dart +++ b/packages/celest/lib/src/core/context.dart @@ -4,10 +4,13 @@ import 'dart:io' show HandshakeException, HttpClient, SocketException; import 'package:celest/src/config/config_values.dart'; import 'package:celest/src/core/environment.dart'; import 'package:celest/src/runtime/gcp/gcp.dart'; +import 'package:celest_ast/celest_ast.dart'; import 'package:celest_core/_internal.dart'; // ignore: implementation_imports import 'package:celest_core/src/auth/user.dart'; import 'package:cloud_http/cloud_http.dart'; +import 'package:file/file.dart'; +import 'package:file/local.dart'; import 'package:http/http.dart' as http; import 'package:http/io_client.dart' as http; import 'package:http/retry.dart' as http; @@ -101,6 +104,10 @@ final class Context { /// The platform of the current context. Platform get platform => get(ContextKey.platform) ?? const LocalPlatform(); + /// The file system of the current context. + FileSystem get fileSystem => + get(ContextKey.fileSystem) ?? const LocalFileSystem(); + /// Whether Celest is running in the cloud. bool get isRunningInCloud => root.get(googleCloudProjectKey) != null; @@ -113,6 +120,9 @@ final class Context { /// The Celest [Environment] of the running service. Environment get environment => expect(env.environment) as Environment; + /// The resolved project configuration for the current context. + ResolvedProject get project => expect(ContextKey.project); + /// The HTTP client for the current context. http.Client get httpClient => get(ContextKey.httpClient) ?? _defaultHttpClient; @@ -235,9 +245,15 @@ abstract interface class ContextKey { /// The context key for the context [Logger]. static const ContextKey logger = ContextKey('logger'); + /// The context key for the context [FileSystem]. + static const ContextKey fileSystem = ContextKey('file system'); + /// The context key for the context [Platform]. static const ContextKey platform = ContextKey('platform'); + /// The context key for the context [ResolvedProject]. + static const ContextKey project = ContextKey('project'); + /// Reads the value for `this` from the given [context]. V? read(Context context); diff --git a/packages/celest/lib/src/runtime/configuration.dart b/packages/celest/lib/src/runtime/configuration.dart new file mode 100644 index 00000000..9b1815e4 --- /dev/null +++ b/packages/celest/lib/src/runtime/configuration.dart @@ -0,0 +1,75 @@ +import 'package:celest/src/config/config_values.dart'; +import 'package:celest/src/core/context.dart'; +import 'package:celest/src/runtime/http/logging.dart'; +import 'package:celest_ast/celest_ast.dart'; +// ignore: implementation_imports +import 'package:celest_ast/src/proto/celest/ast/v1/resolved_ast.pb.dart' as pb; +import 'package:celest_core/_internal.dart'; +import 'package:logging/logging.dart'; + +Map? _loadJsonFromEnv(Context rootContext) { + final configJson = rootContext.platform.environment['CELEST_CONFIG_JSON']; + if (configJson == null) { + return null; + } + try { + return JsonUtf8.decodeMap(configJson); + } on FormatException catch (e) { + throw StateError('Failed to parse CELEST_CONFIG_JSON: $e'); + } +} + +Future?> _loadJsonFromFileSystem( + Context rootContext, +) async { + var configPath = rootContext.platform.environment['CELEST_CONFIG']; + if (configPath == null) { + final script = rootContext.platform.script; + configPath = script.resolve('./celest.json').toFilePath(); + } + if (!rootContext.fileSystem.isFileSync(configPath)) { + return null; + } + + final configFile = rootContext.fileSystem.file(configPath); + final configData = await configFile + .readAsBytes() + .onError((e, _) => throw StateError('Failed to load celest.json: $e')); + + try { + return JsonUtf8.decodeMap(configData); + } on FormatException catch (e) { + throw StateError('Failed to parse celest.json from "$configPath": $e'); + } +} + +/// Configures the environment in which Celest is running. +Future configure() async { + configureLogging(); + + final rootContext = Context.root; + + final configJson = _loadJsonFromEnv(rootContext) ?? + await _loadJsonFromFileSystem(rootContext); + if (configJson == null) { + throw StateError( + 'No project configuration found. Create a celest.json file and set ' + 'CELEST_CONFIG with its path or CELEST_CONFIG_JSON with its contents.', + ); + } + + final configPb = pb.ResolvedProject()..mergeFromProto3Json(configJson); + final config = ResolvedProject.fromProto(configPb); + Logger.root + ..config('Loaded project configuration') + ..config(config); + + rootContext.put(ContextKey.project, config); + + for (final ResolvedVariable(:name, :value) in config.variables) { + rootContext.put(env(name), value); + } + for (final ResolvedSecret(:name, :value) in config.secrets) { + rootContext.put(secret(name), value); + } +} diff --git a/packages/celest/lib/src/runtime/http/environment.dart b/packages/celest/lib/src/runtime/http/environment.dart deleted file mode 100644 index 6bc0006c..00000000 --- a/packages/celest/lib/src/runtime/http/environment.dart +++ /dev/null @@ -1,67 +0,0 @@ -import 'dart:convert'; -import 'dart:io'; - -import 'package:celest/src/config/config_values.dart'; -import 'package:celest/src/core/context.dart'; -import 'package:celest_core/_internal.dart'; -import 'package:logging/logging.dart'; - -/// Configures the environment in which Celest is running. -Future configureEnvironment() async { - final script = Platform.script; - final configUri = script.resolve('./config.json'); - if (!FileSystemEntity.isFileSync(configUri.toFilePath())) { - if (kReleaseMode) { - Logger.root.finer( - 'No config.json file found. Skipping environment configuration.', - ); - } - return; - } - - final configFile = await File.fromUri(configUri) - .readAsString() - .onError((e, _) => throw StateError('Failed to load config.json: $e')); - - final configJson = jsonDecode(configFile); - if (configJson is! Map) { - throw FormatException( - 'Invalid config.json: $configJson. ' - 'Expected Map, got ${configJson.runtimeType}', - ); - } - final environmentVariables = configJson['environmentVariables']; - if (environmentVariables is! Map) { - throw FormatException( - 'Invalid "environmentVariables" in config.json: $environmentVariables. ' - 'Expected Map, got ${environmentVariables.runtimeType}', - ); - } - final secrets = configJson['secrets']; - if (secrets is! Map) { - throw FormatException( - 'Invalid "secrets" in config.json: $secrets. ' - 'Expected Map, got ${secrets.runtimeType}', - ); - } - - for (final MapEntry(key: name, :value) in environmentVariables.entries) { - if (value is! String) { - throw FormatException( - 'Invalid value for environment variable "$name" in config.json: $value. ' - 'Expected String, got ${value.runtimeType}', - ); - } - Context.root.put(env(name), value); - } - - for (final MapEntry(key: name, :value) in secrets.entries) { - if (value is! String) { - throw FormatException( - 'Invalid value for secret "$name" in config.json: $value. ' - 'Expected String, got ${value.runtimeType}', - ); - } - Context.root.put(secret(name), value); - } -} diff --git a/packages/celest/lib/src/runtime/serve.dart b/packages/celest/lib/src/runtime/serve.dart index f309fac7..6e1d093f 100644 --- a/packages/celest/lib/src/runtime/serve.dart +++ b/packages/celest/lib/src/runtime/serve.dart @@ -6,10 +6,9 @@ import 'dart:io'; import 'package:async/async.dart'; import 'package:celest/celest.dart'; +import 'package:celest/src/runtime/configuration.dart'; import 'package:celest/src/runtime/gcp/gcp.dart'; import 'package:celest/src/runtime/http/cloud_middleware.dart'; -import 'package:celest/src/runtime/http/environment.dart'; -import 'package:celest/src/runtime/http/logging.dart'; import 'package:celest/src/runtime/http/middleware.dart'; import 'package:celest/src/runtime/json_utils.dart'; import 'package:celest/src/runtime/sse/sse_handler.dart'; @@ -35,8 +34,7 @@ const int defaultCelestPort = 7777; Future serve({ required Map targets, }) async { - configureLogging(); - await configureEnvironment(); + await configure(); final projectId = await googleCloudProject(); if (projectId != null) { Context.root.put(googleCloudProjectKey, projectId); diff --git a/packages/celest/pubspec.yaml b/packages/celest/pubspec.yaml index a26967c6..017d8016 100644 --- a/packages/celest/pubspec.yaml +++ b/packages/celest/pubspec.yaml @@ -10,6 +10,7 @@ environment: dependencies: async: ^2.11.0 + celest_ast: ^0.1.0 celest_auth: 1.0.0-dev.0 celest_cloud: ^0.1.1 celest_core: 1.0.0-dev.0 @@ -18,6 +19,7 @@ dependencies: collection: ^1.18.0 convert: ^3.1.1 crypto_keys: ^0.3.0 + file: ^7.0.1 fixnum: ^1.1.0 google_cloud: ^0.2.0 http: ^1.0.0 @@ -37,6 +39,7 @@ dependencies: dev_dependencies: celest_lints: path: ../celest_lints + pub_semver: ^2.1.4 stream_transform: ^2.1.0 test: ^1.25.1 web: '>=0.5.0 <2.0.0' diff --git a/packages/celest/test/runtime/configuration_test.dart b/packages/celest/test/runtime/configuration_test.dart new file mode 100644 index 00000000..b610bdd6 --- /dev/null +++ b/packages/celest/test/runtime/configuration_test.dart @@ -0,0 +1,148 @@ +@TestOn('vm') +library; + +import 'dart:convert'; + +import 'package:celest/src/config/config_values.dart'; +import 'package:celest/src/core/context.dart'; +import 'package:celest/src/runtime/configuration.dart'; +import 'package:celest_ast/celest_ast.dart'; +import 'package:file/memory.dart'; +import 'package:platform/platform.dart'; +import 'package:pub_semver/pub_semver.dart'; +import 'package:test/test.dart'; + +final testProject = ResolvedProject( + projectId: 'test', + environmentId: 'local', + sdkConfig: SdkConfiguration( + celest: Version.parse('1.0.0'), + dart: Sdk( + type: SdkType.dart, + version: Version.parse('3.5.0'), + enabledExperiments: ['native-assets'], + ), + targetSdk: SdkType.dart, + featureFlags: [], + ), + apis: { + 'greeting': ResolvedApi( + apiId: 'greeting', + functions: { + 'hello-world': ResolvedCloudFunction( + apiId: 'greeting', + functionId: 'hello-world', + variables: ['HELLO_WORLD'], + secrets: ['HELLO_WORLD_SECRET'], + streamConfig: null, + httpConfig: ResolvedHttpConfig( + method: 'POST', + route: ResolvedHttpRoute( + path: '/greeting/hello-world', + ), + status: 200, + statusMappings: {}, + ), + ), + }, + ), + }, + variables: [ + ResolvedVariable( + name: 'HELLO_WORLD', + value: 'hello', + ), + ], + secrets: [ + ResolvedSecret( + name: 'HELLO_WORLD_SECRET', + value: 'world', + ), + ], +); + +void main() { + group('configure', () { + test('loads ./celest.json', () async { + final platform = FakePlatform( + environment: {}, + script: Uri.parse('file:///app/test.aot'), + ); + final fileSystem = MemoryFileSystem.test(); + final configFile = fileSystem.file('/app/celest.json'); + configFile.parent.createSync(); + configFile.writeAsStringSync( + jsonEncode(testProject.toProto().toProto3Json()), + ); + + context.put(ContextKey.platform, platform); + context.put(ContextKey.fileSystem, fileSystem); + Context.root = context; + + await configure(); + + expect(context.project, equals(testProject)); + expect(context.get(const env('HELLO_WORLD')), equals('hello')); + expect(context.get(const secret('HELLO_WORLD_SECRET')), equals('world')); + }); + + test('loads CELEST_CONFIG', () async { + final platform = FakePlatform( + environment: { + 'CELEST_CONFIG': '/config/celest.json', + }, + script: Uri.parse('file:///app/test.aot'), + ); + final fileSystem = MemoryFileSystem.test(); + final configFile = fileSystem.file('/config/celest.json'); + configFile.parent.createSync(); + configFile.writeAsStringSync( + jsonEncode(testProject.toProto().toProto3Json()), + ); + + context.put(ContextKey.platform, platform); + context.put(ContextKey.fileSystem, fileSystem); + Context.root = context; + + await configure(); + + expect(context.project, equals(testProject)); + expect(context.get(const env('HELLO_WORLD')), equals('hello')); + expect(context.get(const secret('HELLO_WORLD_SECRET')), equals('world')); + }); + + test('loads CELEST_CONFIG_JSON', () async { + final platform = FakePlatform( + environment: { + 'CELEST_CONFIG_JSON': jsonEncode( + testProject.toProto().toProto3Json(), + ), + }, + script: Uri.parse('file:///app/test.aot'), + ); + + context.put(ContextKey.platform, platform); + Context.root = context; + + await configure(); + + expect(context.project, equals(testProject)); + expect(context.get(const env('HELLO_WORLD')), equals('hello')); + expect(context.get(const secret('HELLO_WORLD_SECRET')), equals('world')); + }); + + test('fails hard when missing', () async { + final platform = FakePlatform( + environment: {}, + script: Uri.parse('file:///app/test.aot'), + ); + final fileSystem = MemoryFileSystem.test(); + + context.put(ContextKey.platform, platform); + context.put(ContextKey.fileSystem, fileSystem); + Context.root = context; + + await expectLater(configure(), throwsStateError); + }); + }); +}