feat(env): improve env loading logic for the api

.dockerignore

@@ -2,8 +2,30 @@
 **/data
 **/npm-debug.log
 **/.git
-**/.env
-**/.env.*
+
+**/.env.local
+**/.env.test
+**/.env.*.local
+**/.env.*.test
+
+apps/deploy-web/.env
+apps/deploy-web/.env.*
+
+apps/indexer/.env
+apps/indexer/.env.*
+
+apps/landing/.env
+apps/landing/.env.*
+
+apps/provider-console/.env
+apps/provider-console/.env.*
+
+apps/provider-proxy/.env
+apps/provider-proxy/.env.*
+
+apps/stats-web/.env
+apps/stats-web/.env.*
+
 **/.next
 *.md
 build.ps1

.env.sandbox.docker-compose-dev

@@ -2,7 +2,6 @@
 AkashSandboxDatabaseCS=postgres://postgres:password@db:5432/console-akash-sandbox
 UserDatabaseCS=postgres://postgres:password@db:5432/console-users
 Network=sandbox
-ActiveChain=akashSandbox
 POSTGRES_DB_URI: postgres://postgres:password@db:5432/console-users
 
 # Deploy Web
@@ -15,6 +14,9 @@ API_MAINNET_BASE_URL: http://api:3000
 API_TESTNET_BASE_URL: http://api:3000
 API_SANDBOX_BASE_URL: http://api:3000
 
+# Indexer
+ActiveChain=akashSandbox
+
 # DB
 POSTGRES_USER: postgres
 POSTGRES_PASSWORD: password

.gitignore

@@ -65,10 +65,12 @@ build.ps1
 .yarn-integrity
 
 # dotenv environment variable files
-.env
-.env.development.local
-.env.test.local
-.env.production.local
+apps/deploy-web/.env
+apps/indexer/.env
+apps/provider-console/.env
+apps/provider-proxy/.env
+apps/stats-web/.env
+.env.*.local
 .env.local
 
 # IDE files

apps/api/.gitignore

@@ -35,8 +35,7 @@ node_modules/
 .eslintcache
 
 #dotenv
-.env
-.env.local
+env/.env.local
 
 # akash
 /api/data

apps/api/README.md

@@ -16,25 +16,20 @@ You can make sure the api is working by accessing the status endpoint: `http://l
 
 ## Environment Variables
 
-When running the api locally the following environment variables can be set in a `.env` file.
-
-|Name|Value|Note|
-|-|-|-
-Network|`mainnet` or `testnet`|Specify if the api should be in mainnet or testnet mode. Default: `mainnet`.
-RestApiNodeUrl|ex: `"https://api.akashnet.net"`|Rest api to use. Will default to `"https://rest.cosmos.directory/akash"` for mainnet and `"https://api.testnet-02.aksh.pw:443"` for testnet.
-ServerOrigin|ex: `http://localhost:3080`|Origin of the api server. Will be used to populate the swagger server list.
-HealthchecksEnabled|`true` or `false`|Specify if the [Scheduler](./src/index.ts#L42) should send health check pings.
-SentryDSN|ex: `"https://[email protected]/1234"`|[Sentry DSN](https://docs.sentry.io/product/sentry-basics/dsn-explainer/) used when [initializing](./src/index.ts#L29) Sentry
-AkashDatabaseCS|ex: `postgres://user:password@localhost:5432/cloudmos-akash`|Akash Database Connection String
-AkashTestnetDatabaseCS|ex: `postgres://user:password@localhost:5432/cloudmos-akash-testnet`|Akash Testnet Database Connection String
-UserDatabaseCS|ex: `postgres://user:password@localhost:5432/cloudmos-users`|User Database Connection String
-Auth0JWKSUri|ex: `'https://1a2b3c.us.auth0.com/.well-known/jwks.json'`|
-Auth0Audience|ex: `'https://api.cloudmos.io'`
-Auth0Issuer|ex: `'https://dev-5aprb0lr.us.auth0.com/'`
-Auth0Issuer|ex: `'https://auth.cloudmos.io/'`
-StripeSecretKey|ex: `sk_test_12aw315wdawd3...293d12d32df8jf`
-WebsiteUrl|`http://localhost:3001`
+This app utilizes `.env*` files to manage environment variables. The list of environment variables can be found in the `env/.env.sample` file. These files are included in version control and should only contain non-sensitive values. Sensitive values are provided by the deployment system.
 
+### Important Notes:
+- **Sensitive Values**: The only env file that's ignored by Git is `env/.env.local`, which is intended for sensitive values used in development.
+- **Loading Order**: Environment files are loaded in a specific order, depending on two environment variables: `DEPLOYMENT_ENV` and `NETWORK`.
+
+### Loading Order:
+1. `env/.env.local` - Contains sensitive values for development.
+2. `env/.env` - Default values applicable to all environments.
+3. `env/.env.${DEPLOYMENT_ENV}` - Values specific to the deployment environment.
+4. `env/.env.${NETWORK}` - Values specific to the network.
+
+### Additional Details:
+- **Variable Precedence**: If a variable is already set in the environment, it will not be overridden by values in the `.env*` files. This behavior is critical when adjusting the loading order of these files.
 
 ## Testing
 Project is configured to use [Jest](https://jestjs.io/) for testing. It is intended to be covered with unit and functional tests where applicable.

apps/api/env/.env

@@ -0,0 +1,9 @@
+AKASHLYTICS_CORS_WEBSITE_URLS=https://cloudmos.io,https://www.cloudmos.io,http://debug.cloudmos.io,https://deploy.cloudmos.io,https://beta.cloudmos.io,https://stats.akash.network,https://console.akash.network,https://akash.network,https://akash.hooman.digital,http://localhost:3000,https://beta2.cloudmos.io,https://beta3.cloudmos.io,https://akashconsole.vercel.app
+WebsiteUrl=https://cloudmos.io
+TRIAL_DEPLOYMENT_ALLOWANCE_AMOUNT=6000000
+DEPLOYMENT_ALLOWANCE_REFILL_AMOUNT=6000000
+DEPLOYMENT_ALLOWANCE_REFILL_THRESHOLD=600000
+TRIAL_FEES_ALLOWANCE_AMOUNT=1000000
+FEE_ALLOWANCE_REFILL_AMOUNT=1000000
+FEE_ALLOWANCE_REFILL_THRESHOLD=100000
+LOG_LEVEL=debug

apps/api/env/.env.mainnet

@@ -0,0 +1,2 @@
+RPC_NODE_ENDPOINT=https://rpc.akashnet.net
+DEPLOYMENT_GRANT_DENOM=ibc/170C677610AC31DF0904FFE09CD3B5C657492170E7E52372E48756B71E56F2F1

apps/api/env/.env.production

@@ -0,0 +1,8 @@
+Auth0JWKSUri=https://cloudmos-prod.us.auth0.com/.well-known/jwks.json
+Auth0Audience=https://api.cloudmos.io
+Auth0Issuer=https://auth.cloudmos.io/
+PORT=80
+ServerOrigin=https://api.cloudmos.io
+DRIZZLE_MIGRATIONS_FOLDER=./dist/drizzle
+SENTRY_ENABLED=true
+BILLING_ENABLED=false

apps/api/env/.env.sample

@@ -0,0 +1,39 @@
+# Secrets
+AkashDatabaseCS=
+AkashlyticsGithubPAT=
+ANONYMOUS_USER_TOKEN_SECRET=
+HealthChecks_SyncAKTMarketData=
+MASTER_WALLET_MNEMONIC=
+POSTGRES_DB_URI=
+SecretToken=
+SentryDSN=
+SENTRY_DSN=
+StripeSecretKey=
+UserDatabaseCS=
+
+# Configuration
+AKASHLYTICS_CORS_WEBSITE_URLS=
+Auth0Audience=
+Auth0Issuer=
+Auth0JWKSUri=
+BILLING_ENABLED=
+DEPLOYMENT_ALLOWANCE_REFILL_AMOUNT=
+DEPLOYMENT_ALLOWANCE_REFILL_THRESHOLD=
+DEPLOYMENT_ENV=
+DEPLOYMENT_GRANT_DENOM=
+DRIZZLE_MIGRATIONS_FOLDER=
+FEE_ALLOWANCE_REFILL_AMOUNT=
+FEE_ALLOWANCE_REFILL_THRESHOLD=
+HealthchecksEnabled=
+LOG_LEVEL=
+NETWORK=
+PORT=
+RPC_NODE_ENDPOINT=
+SENTRY_ENABLED=
+SENTRY_SERVER_NAME=
+SentryServerName=
+SENTRY_TRACES_RATE=
+ServerOrigin=
+TRIAL_DEPLOYMENT_ALLOWANCE_AMOUNT=
+TRIAL_FEES_ALLOWANCE_AMOUNT=
+WebsiteUrl=

apps/api/env/.env.sandbox

@@ -0,0 +1,2 @@
+RPC_NODE_ENDPOINT=https://rpc.sandbox-01.aksh.pw:443
+DEPLOYMENT_GRANT_DENOM=ibc/12C6A0C374171B595A0A9E18B83FA09D295FB1F2D8C6DAA3AC28683471752D84

apps/api/env/.env.staging

@@ -0,0 +1,8 @@
+ServerOrigin=https://api-sandbox.cloudmos.io
+DRIZZLE_MIGRATIONS_FOLDER=./dist/drizzle
+Auth0JWKSUri=https://dev-5aprb0lr.us.auth0.com/.well-known/jwks.json
+Auth0Audience=https://api.cloudmos.io
+Auth0Issuer=https://dev-5aprb0lr.us.auth0.com/
+SENTRY_TRACES_RATE=1.0
+SENTRY_ENABLED=true
+BILLING_ENABLED=true

apps/api/package.json

@@ -60,6 +60,7 @@
     "date-fns": "^2.29.2",
     "date-fns-tz": "^1.3.6",
     "dotenv": "^12.0.4",
+    "dotenv-expand": "^11.0.6",
     "drizzle-orm": "^0.31.2",
     "hono": "3.12.0",
     "http-assert": "^1.5.0",

apps/api/src/db/dbConnection.ts

@@ -18,19 +18,19 @@ const csMap = {
   sandbox: env.AkashSandboxDatabaseCS
 };
 
-if (!isValidNetwork(env.Network)) {
-  throw new Error(`Invalid network: ${env.Network}`);
+if (!isValidNetwork(env.NETWORK)) {
+  throw new Error(`Invalid network: ${env.NETWORK}`);
 }
 
-if (!csMap[env.Network]) {
-  throw new Error(`Missing connection string for network: ${env.Network}`);
+if (!csMap[env.NETWORK]) {
+  throw new Error(`Missing connection string for network: ${env.NETWORK}`);
 }
 
 const logger = new PostgresLoggerService({ orm: "sequelize" });
 const logging = (msg: string) => logger.write(msg);
 
 pg.defaults.parseInt8 = true;
-export const chainDb = new Sequelize(csMap[env.Network], {
+export const chainDb = new Sequelize(csMap[env.NETWORK], {
   dialectModule: pg,
   logging,
   logQueryParameters: true,

apps/api/src/dotenv.ts

@@ -1,4 +1,24 @@
 import dotenv from "dotenv";
+import dotenvExpand from "dotenv-expand";
+import fs from "fs";
+import pino from "pino";
 
-dotenv.config({ path: ".env.local" });
-dotenv.config();
+const logger = pino().child({ context: "ENV" });
+
+const config = (path: string) => {
+  if (fs.existsSync(path)) {
+    dotenvExpand.expand(dotenv.config({ path }));
+    logger.info(`Loaded ${path}`);
+  }
+};
+config("env/.env.local");
+config("env/.env");
+
+const deploymentEnv = process.env.DEPLOYMENT_ENV;
+
+if (deploymentEnv && deploymentEnv !== "local") {
+  config(`env/.env.${deploymentEnv}`);
+}
+
+const network = process.env.NETWORK || "mainnet";
+config(`env/.env.${network}`);

apps/api/src/utils/constants.ts

@@ -20,6 +20,6 @@ export const defaultNodeUrlMapping: { [key: string]: string } = {
   testnet: "https://api.testnet-02.aksh.pw"
 };
 
-export const apiNodeUrl = env.RestApiNodeUrl ?? defaultNodeUrlMapping[env.Network] ?? defaultNodeUrlMapping.mainnet;
+export const apiNodeUrl = env.RestApiNodeUrl ?? defaultNodeUrlMapping[env.NETWORK] ?? defaultNodeUrlMapping.mainnet;
 export const betaTypeVersion = "v1beta3";
 export const betaTypeVersionMarket = "v1beta4";

apps/api/src/utils/env.ts

@@ -9,7 +9,7 @@ export const env = z
     AkashTestnetDatabaseCS: z.string().optional(),
     AkashSandboxDatabaseCS: z.string().optional(),
     UserDatabaseCS: z.string().optional(),
-    Network: z.string().default("mainnet"),
+    NETWORK: z.string().default("mainnet"),
     RestApiNodeUrl: z.string().optional(),
     ServerOrigin: z.string().optional().default("http://localhost:3080"),
     AkashlyticsGithubPAT: z.string().optional(),

apps/api/test/setup-functional-tests.ts

@@ -4,7 +4,7 @@ import dotenv from "dotenv";
 
 import { closeConnections, migratePG } from "@src/core";
 
-dotenv.config({ path: ".env.functional.test" });
+dotenv.config({ path: "env/.env.functional.test" });
 
 beforeAll(async () => {
   await migratePG();

apps/api/test/setup.ts

@@ -1,4 +1,4 @@
 import dotenv from "dotenv";
 
-dotenv.config({ path: ".env.local" });
-dotenv.config();
+dotenv.config({ path: "env/.env.local" });
+dotenv.config({ path: "env/.env" });

docker-compose.prod.yml

@@ -1,10 +1,9 @@
 services:
   api:
     restart: always
-    env_file:
-      - .env.sandbox.docker-compose-dev
     environment:
       PORT: 3000
+      DB_HOST: db
     ports:
       - '3080:3000'
 

docker/Dockerfile.node

@@ -31,6 +31,7 @@ RUN addgroup --system --gid $APP_GROUP_ID $APP_GROUP \
     && adduser --system --uid $APP_GROUP_ID --ingroup $APP_GROUP $APP_USER
 
 COPY --from=builder /app/$WORKSPACE/dist /app/$WORKSPACE/dist
+COPY --from=builder /app/$WORKSPACE/.env* /app/$WORKSPACE
 COPY --from=builder /app/packages /app/packages
 COPY --from=builder /app/package.json /app/package.json
 COPY --from=builder /app/package-lock.json /app/package-lock.json