From 0ec14d7dc338cb0ae545d301b7c0406591ebc8b2 Mon Sep 17 00:00:00 2001 From: Yaroslav Grishajev Date: Thu, 15 Aug 2024 14:50:51 +0200 Subject: [PATCH] feat(env): improve env loading logic for the api --- .dockerignore | 26 +++++++++++++++-- .env.sandbox.docker-compose-dev | 4 ++- .gitignore | 10 ++++--- apps/api/.gitignore | 3 +- apps/api/README.md | 31 +++++++++----------- apps/api/env/.env | 9 ++++++ apps/api/{ => env}/.env.functional.test | 0 apps/api/env/.env.mainnet | 2 ++ apps/api/env/.env.production | 8 +++++ apps/api/env/.env.sample | 39 +++++++++++++++++++++++++ apps/api/env/.env.sandbox | 2 ++ apps/api/env/.env.staging | 8 +++++ apps/api/package.json | 1 + apps/api/src/db/dbConnection.ts | 10 +++---- apps/api/src/dotenv.ts | 24 +++++++++++++-- apps/api/src/utils/constants.ts | 2 +- apps/api/src/utils/env.ts | 2 +- apps/api/test/setup-functional-tests.ts | 2 +- apps/api/test/setup.ts | 4 +-- docker-compose.prod.yml | 3 +- docker/Dockerfile.node | 1 + package-lock.json | 26 +++++++++++++++++ 22 files changed, 176 insertions(+), 41 deletions(-) create mode 100644 apps/api/env/.env rename apps/api/{ => env}/.env.functional.test (100%) create mode 100644 apps/api/env/.env.mainnet create mode 100644 apps/api/env/.env.production create mode 100644 apps/api/env/.env.sample create mode 100644 apps/api/env/.env.sandbox create mode 100644 apps/api/env/.env.staging diff --git a/.dockerignore b/.dockerignore index 0b8528c3e..222169129 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,8 +2,30 @@ **/data **/npm-debug.log **/.git -**/.env -**/.env.* + +**/.env.local +**/.env.test +**/.env.*.local +**/.env.*.test + +apps/deploy-web/.env +apps/deploy-web/.env.* + +apps/indexer/.env +apps/indexer/.env.* + +apps/landing/.env +apps/landing/.env.* + +apps/provider-console/.env +apps/provider-console/.env.* + +apps/provider-proxy/.env +apps/provider-proxy/.env.* + +apps/stats-web/.env +apps/stats-web/.env.* + **/.next *.md build.ps1 \ No newline at end of file diff --git a/.env.sandbox.docker-compose-dev b/.env.sandbox.docker-compose-dev index 994a66358..dc22ab376 100644 --- a/.env.sandbox.docker-compose-dev +++ b/.env.sandbox.docker-compose-dev @@ -2,7 +2,6 @@ AkashSandboxDatabaseCS=postgres://postgres:password@db:5432/console-akash-sandbox UserDatabaseCS=postgres://postgres:password@db:5432/console-users Network=sandbox -ActiveChain=akashSandbox POSTGRES_DB_URI: postgres://postgres:password@db:5432/console-users # Deploy Web @@ -15,6 +14,9 @@ API_MAINNET_BASE_URL: http://api:3000 API_TESTNET_BASE_URL: http://api:3000 API_SANDBOX_BASE_URL: http://api:3000 +# Indexer +ActiveChain=akashSandbox + # DB POSTGRES_USER: postgres POSTGRES_PASSWORD: password diff --git a/.gitignore b/.gitignore index bd74e3da6..63c8b4825 100644 --- a/.gitignore +++ b/.gitignore @@ -65,10 +65,12 @@ build.ps1 .yarn-integrity # dotenv environment variable files -.env -.env.development.local -.env.test.local -.env.production.local +apps/deploy-web/.env +apps/indexer/.env +apps/provider-console/.env +apps/provider-proxy/.env +apps/stats-web/.env +.env.*.local .env.local # IDE files diff --git a/apps/api/.gitignore b/apps/api/.gitignore index 6cf1b0337..20fe2662c 100644 --- a/apps/api/.gitignore +++ b/apps/api/.gitignore @@ -35,8 +35,7 @@ node_modules/ .eslintcache #dotenv -.env -.env.local +env/.env.local # akash /api/data diff --git a/apps/api/README.md b/apps/api/README.md index f0a934628..581a9ab00 100644 --- a/apps/api/README.md +++ b/apps/api/README.md @@ -16,25 +16,20 @@ You can make sure the api is working by accessing the status endpoint: `http://l ## Environment Variables -When running the api locally the following environment variables can be set in a `.env` file. - -|Name|Value|Note| -|-|-|- -Network|`mainnet` or `testnet`|Specify if the api should be in mainnet or testnet mode. Default: `mainnet`. -RestApiNodeUrl|ex: `"https://api.akashnet.net"`|Rest api to use. Will default to `"https://rest.cosmos.directory/akash"` for mainnet and `"https://api.testnet-02.aksh.pw:443"` for testnet. -ServerOrigin|ex: `http://localhost:3080`|Origin of the api server. Will be used to populate the swagger server list. -HealthchecksEnabled|`true` or `false`|Specify if the [Scheduler](./src/index.ts#L42) should send health check pings. -SentryDSN|ex: `"https://1234...789@z645.ingest.sentry.io/1234"`|[Sentry DSN](https://docs.sentry.io/product/sentry-basics/dsn-explainer/) used when [initializing](./src/index.ts#L29) Sentry -AkashDatabaseCS|ex: `postgres://user:password@localhost:5432/cloudmos-akash`|Akash Database Connection String -AkashTestnetDatabaseCS|ex: `postgres://user:password@localhost:5432/cloudmos-akash-testnet`|Akash Testnet Database Connection String -UserDatabaseCS|ex: `postgres://user:password@localhost:5432/cloudmos-users`|User Database Connection String -Auth0JWKSUri|ex: `'https://1a2b3c.us.auth0.com/.well-known/jwks.json'`| -Auth0Audience|ex: `'https://api.cloudmos.io'` -Auth0Issuer|ex: `'https://dev-5aprb0lr.us.auth0.com/'` -Auth0Issuer|ex: `'https://auth.cloudmos.io/'` -StripeSecretKey|ex: `sk_test_12aw315wdawd3...293d12d32df8jf` -WebsiteUrl|`http://localhost:3001` +This app utilizes `.env*` files to manage environment variables. The list of environment variables can be found in the `env/.env.sample` file. These files are included in version control and should only contain non-sensitive values. Sensitive values are provided by the deployment system. +### Important Notes: +- **Sensitive Values**: The only env file that's ignored by Git is `env/.env.local`, which is intended for sensitive values used in development. +- **Loading Order**: Environment files are loaded in a specific order, depending on two environment variables: `DEPLOYMENT_ENV` and `NETWORK`. + +### Loading Order: +1. `env/.env.local` - Contains sensitive values for development. +2. `env/.env` - Default values applicable to all environments. +3. `env/.env.${DEPLOYMENT_ENV}` - Values specific to the deployment environment. +4. `env/.env.${NETWORK}` - Values specific to the network. + +### Additional Details: +- **Variable Precedence**: If a variable is already set in the environment, it will not be overridden by values in the `.env*` files. This behavior is critical when adjusting the loading order of these files. ## Testing Project is configured to use [Jest](https://jestjs.io/) for testing. It is intended to be covered with unit and functional tests where applicable. diff --git a/apps/api/env/.env b/apps/api/env/.env new file mode 100644 index 000000000..2422d335b --- /dev/null +++ b/apps/api/env/.env @@ -0,0 +1,9 @@ +AKASHLYTICS_CORS_WEBSITE_URLS=https://cloudmos.io,https://www.cloudmos.io,http://debug.cloudmos.io,https://deploy.cloudmos.io,https://beta.cloudmos.io,https://stats.akash.network,https://console.akash.network,https://akash.network,https://akash.hooman.digital,http://localhost:3000,https://beta2.cloudmos.io,https://beta3.cloudmos.io,https://akashconsole.vercel.app +WebsiteUrl=https://cloudmos.io +TRIAL_DEPLOYMENT_ALLOWANCE_AMOUNT=6000000 +DEPLOYMENT_ALLOWANCE_REFILL_AMOUNT=6000000 +DEPLOYMENT_ALLOWANCE_REFILL_THRESHOLD=600000 +TRIAL_FEES_ALLOWANCE_AMOUNT=1000000 +FEE_ALLOWANCE_REFILL_AMOUNT=1000000 +FEE_ALLOWANCE_REFILL_THRESHOLD=100000 +LOG_LEVEL=debug diff --git a/apps/api/.env.functional.test b/apps/api/env/.env.functional.test similarity index 100% rename from apps/api/.env.functional.test rename to apps/api/env/.env.functional.test diff --git a/apps/api/env/.env.mainnet b/apps/api/env/.env.mainnet new file mode 100644 index 000000000..0e32879f9 --- /dev/null +++ b/apps/api/env/.env.mainnet @@ -0,0 +1,2 @@ +RPC_NODE_ENDPOINT=https://rpc.akashnet.net +DEPLOYMENT_GRANT_DENOM=ibc/170C677610AC31DF0904FFE09CD3B5C657492170E7E52372E48756B71E56F2F1 diff --git a/apps/api/env/.env.production b/apps/api/env/.env.production new file mode 100644 index 000000000..74cce8b4b --- /dev/null +++ b/apps/api/env/.env.production @@ -0,0 +1,8 @@ +Auth0JWKSUri=https://cloudmos-prod.us.auth0.com/.well-known/jwks.json +Auth0Audience=https://api.cloudmos.io +Auth0Issuer=https://auth.cloudmos.io/ +PORT=80 +ServerOrigin=https://api.cloudmos.io +DRIZZLE_MIGRATIONS_FOLDER=./dist/drizzle +SENTRY_ENABLED=true +BILLING_ENABLED=false \ No newline at end of file diff --git a/apps/api/env/.env.sample b/apps/api/env/.env.sample new file mode 100644 index 000000000..6f409fc99 --- /dev/null +++ b/apps/api/env/.env.sample @@ -0,0 +1,39 @@ +# Secrets +AkashDatabaseCS= +AkashlyticsGithubPAT= +ANONYMOUS_USER_TOKEN_SECRET= +HealthChecks_SyncAKTMarketData= +MASTER_WALLET_MNEMONIC= +POSTGRES_DB_URI= +SecretToken= +SentryDSN= +SENTRY_DSN= +StripeSecretKey= +UserDatabaseCS= + +# Configuration +AKASHLYTICS_CORS_WEBSITE_URLS= +Auth0Audience= +Auth0Issuer= +Auth0JWKSUri= +BILLING_ENABLED= +DEPLOYMENT_ALLOWANCE_REFILL_AMOUNT= +DEPLOYMENT_ALLOWANCE_REFILL_THRESHOLD= +DEPLOYMENT_ENV= +DEPLOYMENT_GRANT_DENOM= +DRIZZLE_MIGRATIONS_FOLDER= +FEE_ALLOWANCE_REFILL_AMOUNT= +FEE_ALLOWANCE_REFILL_THRESHOLD= +HealthchecksEnabled= +LOG_LEVEL= +NETWORK= +PORT= +RPC_NODE_ENDPOINT= +SENTRY_ENABLED= +SENTRY_SERVER_NAME= +SentryServerName= +SENTRY_TRACES_RATE= +ServerOrigin= +TRIAL_DEPLOYMENT_ALLOWANCE_AMOUNT= +TRIAL_FEES_ALLOWANCE_AMOUNT= +WebsiteUrl= \ No newline at end of file diff --git a/apps/api/env/.env.sandbox b/apps/api/env/.env.sandbox new file mode 100644 index 000000000..e56fa4aff --- /dev/null +++ b/apps/api/env/.env.sandbox @@ -0,0 +1,2 @@ +RPC_NODE_ENDPOINT=https://rpc.sandbox-01.aksh.pw:443 +DEPLOYMENT_GRANT_DENOM=ibc/12C6A0C374171B595A0A9E18B83FA09D295FB1F2D8C6DAA3AC28683471752D84 \ No newline at end of file diff --git a/apps/api/env/.env.staging b/apps/api/env/.env.staging new file mode 100644 index 000000000..8bbec5dca --- /dev/null +++ b/apps/api/env/.env.staging @@ -0,0 +1,8 @@ +ServerOrigin=https://api-sandbox.cloudmos.io +DRIZZLE_MIGRATIONS_FOLDER=./dist/drizzle +Auth0JWKSUri=https://dev-5aprb0lr.us.auth0.com/.well-known/jwks.json +Auth0Audience=https://api.cloudmos.io +Auth0Issuer=https://dev-5aprb0lr.us.auth0.com/ +SENTRY_TRACES_RATE=1.0 +SENTRY_ENABLED=true +BILLING_ENABLED=true diff --git a/apps/api/package.json b/apps/api/package.json index e98849a82..29f6a4888 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -60,6 +60,7 @@ "date-fns": "^2.29.2", "date-fns-tz": "^1.3.6", "dotenv": "^12.0.4", + "dotenv-expand": "^11.0.6", "drizzle-orm": "^0.31.2", "hono": "3.12.0", "http-assert": "^1.5.0", diff --git a/apps/api/src/db/dbConnection.ts b/apps/api/src/db/dbConnection.ts index 147972eab..dd82647c3 100644 --- a/apps/api/src/db/dbConnection.ts +++ b/apps/api/src/db/dbConnection.ts @@ -18,19 +18,19 @@ const csMap = { sandbox: env.AkashSandboxDatabaseCS }; -if (!isValidNetwork(env.Network)) { - throw new Error(`Invalid network: ${env.Network}`); +if (!isValidNetwork(env.NETWORK)) { + throw new Error(`Invalid network: ${env.NETWORK}`); } -if (!csMap[env.Network]) { - throw new Error(`Missing connection string for network: ${env.Network}`); +if (!csMap[env.NETWORK]) { + throw new Error(`Missing connection string for network: ${env.NETWORK}`); } const logger = new PostgresLoggerService({ orm: "sequelize" }); const logging = (msg: string) => logger.write(msg); pg.defaults.parseInt8 = true; -export const chainDb = new Sequelize(csMap[env.Network], { +export const chainDb = new Sequelize(csMap[env.NETWORK], { dialectModule: pg, logging, logQueryParameters: true, diff --git a/apps/api/src/dotenv.ts b/apps/api/src/dotenv.ts index e8ae1ea63..7160d35cb 100644 --- a/apps/api/src/dotenv.ts +++ b/apps/api/src/dotenv.ts @@ -1,4 +1,24 @@ import dotenv from "dotenv"; +import dotenvExpand from "dotenv-expand"; +import fs from "fs"; +import pino from "pino"; -dotenv.config({ path: ".env.local" }); -dotenv.config(); +const logger = pino().child({ context: "ENV" }); + +const config = (path: string) => { + if (fs.existsSync(path)) { + dotenvExpand.expand(dotenv.config({ path })); + logger.info(`Loaded ${path}`); + } +}; +config("env/.env.local"); +config("env/.env"); + +const deploymentEnv = process.env.DEPLOYMENT_ENV; + +if (deploymentEnv && deploymentEnv !== "local") { + config(`env/.env.${deploymentEnv}`); +} + +const network = process.env.NETWORK || "mainnet"; +config(`env/.env.${network}`); diff --git a/apps/api/src/utils/constants.ts b/apps/api/src/utils/constants.ts index d5c860cbd..24bc25835 100644 --- a/apps/api/src/utils/constants.ts +++ b/apps/api/src/utils/constants.ts @@ -20,6 +20,6 @@ export const defaultNodeUrlMapping: { [key: string]: string } = { testnet: "https://api.testnet-02.aksh.pw" }; -export const apiNodeUrl = env.RestApiNodeUrl ?? defaultNodeUrlMapping[env.Network] ?? defaultNodeUrlMapping.mainnet; +export const apiNodeUrl = env.RestApiNodeUrl ?? defaultNodeUrlMapping[env.NETWORK] ?? defaultNodeUrlMapping.mainnet; export const betaTypeVersion = "v1beta3"; export const betaTypeVersionMarket = "v1beta4"; diff --git a/apps/api/src/utils/env.ts b/apps/api/src/utils/env.ts index f4fedb1f8..9445584bb 100644 --- a/apps/api/src/utils/env.ts +++ b/apps/api/src/utils/env.ts @@ -9,7 +9,7 @@ export const env = z AkashTestnetDatabaseCS: z.string().optional(), AkashSandboxDatabaseCS: z.string().optional(), UserDatabaseCS: z.string().optional(), - Network: z.string().default("mainnet"), + NETWORK: z.string().default("mainnet"), RestApiNodeUrl: z.string().optional(), ServerOrigin: z.string().optional().default("http://localhost:3080"), AkashlyticsGithubPAT: z.string().optional(), diff --git a/apps/api/test/setup-functional-tests.ts b/apps/api/test/setup-functional-tests.ts index e219016d2..22a8cd193 100644 --- a/apps/api/test/setup-functional-tests.ts +++ b/apps/api/test/setup-functional-tests.ts @@ -4,7 +4,7 @@ import dotenv from "dotenv"; import { closeConnections, migratePG } from "@src/core"; -dotenv.config({ path: ".env.functional.test" }); +dotenv.config({ path: "env/.env.functional.test" }); beforeAll(async () => { await migratePG(); diff --git a/apps/api/test/setup.ts b/apps/api/test/setup.ts index e8ae1ea63..c2b2b8f7f 100644 --- a/apps/api/test/setup.ts +++ b/apps/api/test/setup.ts @@ -1,4 +1,4 @@ import dotenv from "dotenv"; -dotenv.config({ path: ".env.local" }); -dotenv.config(); +dotenv.config({ path: "env/.env.local" }); +dotenv.config({ path: "env/.env" }); diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 521ffe91a..4fe4225d3 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -1,10 +1,9 @@ services: api: restart: always - env_file: - - .env.sandbox.docker-compose-dev environment: PORT: 3000 + DB_HOST: db ports: - '3080:3000' diff --git a/docker/Dockerfile.node b/docker/Dockerfile.node index bfe4364cc..b3568128d 100644 --- a/docker/Dockerfile.node +++ b/docker/Dockerfile.node @@ -31,6 +31,7 @@ RUN addgroup --system --gid $APP_GROUP_ID $APP_GROUP \ && adduser --system --uid $APP_GROUP_ID --ingroup $APP_GROUP $APP_USER COPY --from=builder /app/$WORKSPACE/dist /app/$WORKSPACE/dist +COPY --from=builder /app/$WORKSPACE/.env* /app/$WORKSPACE COPY --from=builder /app/packages /app/packages COPY --from=builder /app/package.json /app/package.json COPY --from=builder /app/package-lock.json /app/package-lock.json diff --git a/package-lock.json b/package-lock.json index c5d73ebc4..520bca9da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -59,6 +59,7 @@ "date-fns": "^2.29.2", "date-fns-tz": "^1.3.6", "dotenv": "^12.0.4", + "dotenv-expand": "^11.0.6", "drizzle-orm": "^0.31.2", "hono": "3.12.0", "http-assert": "^1.5.0", @@ -26364,6 +26365,31 @@ "node": ">=12" } }, + "node_modules/dotenv-expand": { + "version": "11.0.6", + "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-11.0.6.tgz", + "integrity": "sha512-8NHi73otpWsZGBSZwwknTXS5pqMOrk9+Ssrna8xCaxkzEpU9OTf9R5ArQGVw03//Zmk9MOwLPng9WwndvpAJ5g==", + "dependencies": { + "dotenv": "^16.4.4" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dotenv-expand/node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/dottie": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.6.tgz",