more metadata

- allow CTA to load metadata from multiple files in addition to the env var
- add driver metadata
- add creation date and version metadata

packages/cli/src/commands/up.ts

@@ -72,6 +72,7 @@ export default class Up extends MachineCreationDriverCommand<typeof Up> {
       userSpecifiedServices: restArgs,
       debug: flags.debug,
       machineDriver: driver,
+      machineDriverName: this.driverName,
       machineCreationDriver,
       userModel,
       userSpecifiedProjectName: flags.project,
@@ -85,6 +86,7 @@ export default class Up extends MachineCreationDriverCommand<typeof Up> {
       allowedSshHostKeys: hostKey,
       cwd: process.cwd(),
       skipUnchangedFiles: flags['skip-unchanged-files'],
+      version: this.config.version,
     })
 
     this.log(`Preview environment ${envId} provisioned at: ${machine.locationDescription}`)

packages/common/index.ts

@@ -15,7 +15,7 @@ export {
   SimpleEmitter, StateEmitter, EmitterConsumer, StateEmitterConsumer,
 } from './src/emitter'
 export { hasPropertyDefined, RequiredProperties } from './src/ts-utils'
-export { tryParseJson } from './src/json'
+export { tryParseJson, dateReplacer } from './src/json'
 export { Logger } from './src/log'
 export { requiredEnv, numberFromEnv } from './src/env'
 export { tunnelNameResolver, TunnelNameResolver } from './src/tunnel-name'

packages/common/src/json.ts

@@ -5,3 +5,5 @@ export const tryParseJson = (...args: Parameters<typeof JSON.parse>) => {
     return undefined
   }
 }
+
+export const dateReplacer = (_key: string, value: unknown) => (value instanceof Date ? value.toISOString() : value)

packages/compose-tunnel-agent/index.ts

@@ -7,37 +7,29 @@ import { rimraf } from 'rimraf'
 import pino from 'pino'
 import pinoPretty from 'pino-pretty'
 import { EOL } from 'os'
-import { ConnectionCheckResult, requiredEnv, checkConnection, formatPublicKey, parseSshUrl, SshConnectionConfig, tunnelNameResolver, MachineStatusCommand } from '@preevy/common'
+import {
+  ConnectionCheckResult,
+  requiredEnv,
+  checkConnection,
+  formatPublicKey,
+  parseSshUrl,
+  SshConnectionConfig,
+  tunnelNameResolver,
+  MachineStatusCommand,
+} from '@preevy/common'
 import createDockerClient from './src/docker'
 import createApiServerHandler from './src/http/api-server'
 import { sshClient as createSshClient } from './src/ssh'
 import { createDockerProxyHandlers } from './src/http/docker-proxy'
 import { tryHandler, tryUpgradeHandler } from './src/http/http-server-helpers'
 import { httpServerHandlers } from './src/http'
 import { runMachineStatusCommand } from './src/machine-status'
+import { envMetadata } from './src/metadata'
+import { readAllFiles } from './src/files'
 
 const homeDir = process.env.HOME || '/root'
 const dockerSocket = '/var/run/docker.sock'
 
-const readDir = async (dir: string) => {
-  try {
-    return ((await fs.promises.readdir(dir, { withFileTypes: true })) ?? [])
-      .filter(d => d.isFile()).map(f => f.name)
-  } catch (e) {
-    if ((e as { code: string }).code === 'ENOENT') {
-      return []
-    }
-    throw e
-  }
-}
-
-const readAllFiles = async (dir: string) => {
-  const files = await readDir(dir)
-  return await Promise.all(
-    files.map(file => fs.promises.readFile(path.join(dir, file), { encoding: 'utf8' }))
-  )
-}
-
 const sshConnectionConfigFromEnv = async (): Promise<{ connectionConfig: SshConnectionConfig; sshUrl: string }> => {
   const sshUrl = requiredEnv('SSH_URL')
   const parsed = parseSshUrl(sshUrl)
@@ -80,10 +72,6 @@ const machineStatusCommand = process.env.MACHINE_STATUS_COMMAND
   ? JSON.parse(process.env.MACHINE_STATUS_COMMAND) as MachineStatusCommand
   : undefined
 
-const envMetadata = process.env.ENV_METADATA
-  ? JSON.parse(process.env.ENV_METADATA) as Record<string, unknown>
-  : undefined
-
 const log = pino({
   level: process.env.DEBUG || process.env.DOCKER_PROXY_DEBUG ? 'debug' : 'info',
 }, pinoPretty({ destination: pino.destination(process.stderr) }))
@@ -143,7 +131,7 @@ const main = async () => {
       machineStatus: machineStatusCommand
         ? async () => await runMachineStatusCommand({ log, docker })(machineStatusCommand)
         : undefined,
-      envMetadata,
+      envMetadata: await envMetadata({ env: process.env, log }),
     }),
     dockerProxyHandlers: createDockerProxyHandlers({
       log: log.child({ name: 'docker-proxy' }),

packages/compose-tunnel-agent/package.json

@@ -7,6 +7,9 @@
     "out",
     "Dockerfile"
   ],
+  "engines": {
+    "node": ">=18.0.0"
+  },
   "license": "Apache-2.0",
   "dependencies": {
     "@preevy/common": "0.0.48",

packages/compose-tunnel-agent/src/files.ts

@@ -0,0 +1,32 @@
+import fs from 'fs'
+import path from 'path'
+
+const isNotFoundError = (e: unknown) => (e as { code?: unknown })?.code === 'ENOENT'
+
+export const readOrUndefined = (
+  ...args: Parameters<typeof fs.promises.readFile>
+) => fs.promises.readFile(...args).catch(err => {
+  if (isNotFoundError(err)) {
+    return undefined
+  }
+  throw err
+})
+
+const readDir = async (dir: string) => {
+  try {
+    return ((await fs.promises.readdir(dir, { withFileTypes: true })) ?? [])
+      .filter(d => d.isFile()).map(f => f.name)
+  } catch (e) {
+    if ((e as { code: string }).code === 'ENOENT') {
+      return []
+    }
+    throw e
+  }
+}
+
+export const readAllFiles = async (dir: string) => {
+  const files = await readDir(dir)
+  return await Promise.all(
+    files.map(file => fs.promises.readFile(path.join(dir, file), { encoding: 'utf8' }))
+  )
+}

packages/compose-tunnel-agent/src/metadata.ts

@@ -0,0 +1,34 @@
+import fs from 'fs'
+import { Logger } from '@preevy/common'
+import { merge } from 'lodash'
+import { inspect } from 'util'
+
+export const envMetadata = async ({ env, log }: { env: NodeJS.ProcessEnv; log: Logger }) => {
+  const jsons = [
+    ...await Promise.all(
+      (env.ENV_METADATA_FILES || '')
+        .split(' ')
+        .map(async f => {
+          try {
+            return await fs.promises.readFile(f, { encoding: 'utf8' })
+          } catch (err) {
+            log.warn('error reading env metadata from file "%s": %j', f, inspect(err))
+            return undefined
+          }
+        })
+    ),
+    env.ENV_METADATA,
+  ]
+
+  const objects = jsons.map((s, i) => {
+    try {
+      if (s === undefined) return undefined
+      return JSON.parse(s) as Record<string, unknown>
+    } catch (err) {
+      log.warn('error reading env metadta from JSON %d %s: %j', i, s, inspect(err))
+      return undefined
+    }
+  })
+
+  return merge({}, ...objects.filter(Boolean))
+}

packages/compose-tunnel-agent/tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "lib": ["ES2022"],
     "module": "CommonJS",
-    "target": "ES2019",
+    "target": "ES2022",
     "moduleResolution": "node",
     "strict": true,
     "esModuleInterop": true,

packages/core/src/commands/up/index.ts

@@ -14,7 +14,7 @@ import { remoteProjectDir } from '../../remote-files'
 import { Logger } from '../../log'
 import { tunnelUrlsForEnv } from '../../tunneling'
 import { FileToCopy, uploadWithSpinner } from '../../upload-files'
-import { detectEnvMetadata } from '../../env-metadata'
+import { envMetadata } from '../../env-metadata'
 
 const createCopiedFileInDataDir = (
   { projectLocalDataDir, filesToCopy } : {
@@ -65,6 +65,7 @@ const up = async ({
   rootUrl,
   debug,
   machineDriver,
+  machineDriverName,
   machineCreationDriver,
   tunnelOpts,
   userModel,
@@ -79,11 +80,13 @@ const up = async ({
   sshTunnelPrivateKey,
   cwd,
   skipUnchangedFiles,
+  version,
 }: {
   clientId: string
   rootUrl: string
   debug: boolean
   machineDriver: MachineDriver
+  machineDriverName: string
   machineCreationDriver: MachineCreationDriver
   tunnelOpts: TunnelOpts
   userModel: ComposeModel
@@ -98,6 +101,7 @@ const up = async ({
   allowedSshHostKeys: Buffer
   cwd: string
   skipUnchangedFiles: boolean
+  version: string
 }): Promise<{ machine: MachineBase; envId: string }> => {
   const projectName = userSpecifiedProjectName ?? userModel.name
   const remoteDir = remoteProjectDir(projectName)
@@ -139,7 +143,7 @@ const up = async ({
   ])
 
   const { machine, connection } = await ensureCustomizedMachine({
-    machineDriver, machineCreationDriver, envId, log, debug,
+    machineDriver, machineCreationDriver, machineDriverName, envId, log, debug,
   })
 
   try {
@@ -158,7 +162,7 @@ const up = async ({
       knownServerPublicKeyPath: path.join(remoteDir, knownServerPublicKey.remote),
       user,
       machineStatusCommand: await machineDriver.machineStatusCommand(machine),
-      envMetadata: await detectEnvMetadata(),
+      envMetadata: await envMetadata({ version }),
     }, fixedModel)
 
     const modelStr = yaml.stringify(remoteModel)

packages/core/src/commands/up/machine.ts

@@ -1,14 +1,19 @@
 import { EOL } from 'os'
 import retry from 'p-retry'
+import { dateReplacer } from '@preevy/common'
 import { withSpinner } from '../../spinner'
 import { MachineCreationDriver, SpecDiffItem, MachineDriver, MachineConnection, MachineBase, isPartialMachine, machineResourceType } from '../../driver'
 import { telemetryEmitter } from '../../telemetry'
 import { Logger } from '../../log'
 import { scriptExecuter } from '../../remote-script-executer'
+import { EnvMetadata, driverMetadataFilename } from '../../env-metadata'
+import { REMOTE_DIR_BASE } from '../../remote-files'
 
 const machineDiffText = (diff: SpecDiffItem[]) => diff
   .map(({ name, old, new: n }) => `* ${name}: ${old} -> ${n}`).join(EOL)
 
+type Origin = 'existing' | 'new-from-snapshot' | 'new-from-scratch'
+
 const ensureMachine = async ({
   machineDriver,
   machineCreationDriver,
@@ -21,7 +26,7 @@ const ensureMachine = async ({
   envId: string
   log: Logger
   debug: boolean
-}) => {
+}): Promise<{ machine: MachineBase; origin: Origin; connection: Promise<MachineConnection> }> => {
   log.debug('checking for existing machine')
   const existingMachine = await machineCreationDriver.getMachineAndSpecDiff({ envId })
 
@@ -37,7 +42,7 @@ const ensureMachine = async ({
       } else {
         return {
           machine: existingMachine,
-          installed: true,
+          origin: 'existing',
           connection: machineDriver.connect(existingMachine, { log, debug }),
         }
       }
@@ -63,28 +68,49 @@ const ensureMachine = async ({
     return {
       machine,
       connection: Promise.resolve(connection),
-      installed: machineCreation.fromSnapshot,
+      origin: machineCreation.fromSnapshot ? 'new-from-snapshot' : 'new-from-scratch',
     }
   }, {
     opPrefix: `${recreating ? 'Recreating' : 'Creating'} ${machineDriver.friendlyName} machine`,
     successText: `${machineDriver.friendlyName} machine ${recreating ? 'recreated' : 'created'}`,
   })
 }
 
+const writeMetadata = async (
+  machine: MachineBase,
+  machineDriverName: string,
+  driverOpts: Record<string, unknown>,
+  connection: MachineConnection
+) => {
+  const metadata: Pick<EnvMetadata, 'driver'> = {
+    driver: {
+      creationTime: new Date(),
+      machineLocationDescription: machine.locationDescription,
+      driver: machineDriverName,
+      opts: driverOpts,
+    },
+  }
+  await connection.exec(`mkdir -p "${REMOTE_DIR_BASE}" && cat > "${REMOTE_DIR_BASE}/${driverMetadataFilename}"`, {
+    stdin: Buffer.from(JSON.stringify(metadata, dateReplacer)),
+  })
+}
+
 export const ensureCustomizedMachine = async ({
   machineDriver,
   machineCreationDriver,
+  machineDriverName,
   envId,
   log,
   debug,
 }: {
   machineDriver: MachineDriver
   machineCreationDriver: MachineCreationDriver
+  machineDriverName: string
   envId: string
   log: Logger
   debug: boolean
 }): Promise<{ machine: MachineBase; connection: MachineConnection }> => {
-  const { machine, connection: connectionPromise, installed } = await ensureMachine(
+  const { machine, connection: connectionPromise, origin } = await ensureMachine(
     { machineDriver, machineCreationDriver, envId, log, debug },
   )
 
@@ -93,7 +119,7 @@ export const ensureCustomizedMachine = async ({
     { text: `Connecting to machine at ${machine.locationDescription}` },
   )
 
-  if (installed) {
+  if (origin === 'existing') {
     return { machine, connection }
   }
 
@@ -123,11 +149,14 @@ export const ensureCustomizedMachine = async ({
         }
       )
 
-      await machineCreationDriver.ensureMachineSnapshot({
-        providerId: machine.providerId,
-        envId,
-        wait: false,
-      })
+      await Promise.all([
+        writeMetadata(machine, machineDriverName, machineCreationDriver.metadata, connection),
+        machineCreationDriver.ensureMachineSnapshot({
+          providerId: machine.providerId,
+          envId,
+          wait: false,
+        }),
+      ])
     }, { opPrefix: 'Configuring machine', successText: 'Machine configured' })
   } catch (e) {
     await connection.close()