refactor/provider-endpoint-attributes (#20)

* fix timeout to stop refreshing * added uptime tracker task + add new endpoint for provider list + mapping * filter duplicate providers with host uri * fix markdown code highlight missing language * improve provider list * improve provider list styling * added link to provider from deployment list * gpu list improvements * added gpu sorting * added sorting with query param * temp fix for typing * pr fix * improve network capacity title * small bugfix dom utils * remove unused vars * link to docs for provider * fix sql query for network capacity graph * fixes
akash-network · Sep 19, 2023 · 69068c3 · 69068c3
1 parent 786cbf7
commit 69068c3
Show file tree

Hide file tree

Showing 65 changed files with 1,004 additions and 527 deletions.
diff --git a/api/package-lock.json b/api/package-lock.json
diff --git a/api/package.json b/api/package.json
@@ -54,7 +54,7 @@
     "@types/node": "^16.6.0",
     "@types/node-fetch": "^2.6.2",
     "@types/pg": "^8.6.5",
-    "@types/semver": "^7.5.1",
+    "@types/semver": "^7.5.2",
     "@types/uuid": "^8.3.1",
     "alias-hq": "^5.1.6",
     "nodemon": "^2.0.7",

diff --git a/api/src/db/blocksProvider.ts b/api/src/db/blocksProvider.ts
@@ -1,6 +1,6 @@
 import { AkashBlock as Block, AkashMessage as Message } from "@shared/dbSchemas/akash";
 import { Transaction, Validator } from "@shared/dbSchemas/base";
-import { averageBlockTime } from "@src/shared/constants";
+import { averageBlockTime } from "@src/utils/constants";
 import { add } from "date-fns";
 
 export async function getBlocks(limit: number) {

diff --git a/api/src/db/dbConnection.ts b/api/src/db/dbConnection.ts
@@ -1,5 +1,5 @@
 import { defaults as pgDefaults } from "pg";
-import { env } from "@src/shared/utils/env";
+import { env } from "@src/utils/env";
 import { Transaction as DbTransaction } from "sequelize";
 import { Sequelize } from "sequelize-typescript";
 import { chainModels, getChainModels, userModels } from "@shared/dbSchemas";

diff --git a/api/src/db/deploymentProvider.ts b/api/src/db/deploymentProvider.ts
@@ -1,6 +1,6 @@
 import * as v1 from "@src/proto/akash/v1beta1";
 import * as v2 from "@src/proto/akash/v1beta2";
-import { decodeMsg } from "@src/shared/utils/protobuf";
+import { decodeMsg } from "@src/utils/protobuf";
 import { Transaction } from "@shared/dbSchemas/base";
 import { Deployment } from "@shared/dbSchemas/akash";
 import { Op } from "sequelize";

diff --git a/api/src/db/networkRevenueProvider.ts b/api/src/db/networkRevenueProvider.ts
@@ -2,8 +2,8 @@ import { Op } from "sequelize";
 import { Day } from "@shared/dbSchemas/base";
 import { AkashBlock as Block } from "@shared/dbSchemas/akash";
 import { add } from "date-fns";
-import { getTodayUTC } from "@src/shared/utils/date";
-import { round, uaktToAKT, udenomToDenom } from "@src/shared/utils/math";
+import { getTodayUTC } from "@src/utils/date";
+import { round, uaktToAKT, udenomToDenom } from "@src/utils/math";
 
 export const getWeb3IndexRevenue = async (debug?: boolean) => {
   const dailyNetworkRevenues = await getDailyRevenue();

diff --git a/api/src/db/statsProvider.ts b/api/src/db/statsProvider.ts
@@ -160,15 +160,16 @@ export const getProviderGraphData = async (dataName: ProviderStatsKey) => {
     async () => {
       return (await chainDb.query(
         `SELECT d."date", (SUM("activeCPU") + SUM("pendingCPU") + SUM("availableCPU")) AS "cpu", (SUM("activeGPU") + SUM("pendingGPU") + SUM("availableGPU")) AS "gpu", (SUM("activeMemory") + SUM("pendingMemory") + SUM("availableMemory")) AS memory, (SUM("activeStorage") + SUM("pendingStorage") + SUM("availableStorage")) as storage, COUNT(*) as count
-      FROM "day" d
-      INNER JOIN (
-          SELECT DISTINCT ON("owner",DATE("checkDate")) DATE("checkDate") AS date, "activeCPU", "pendingCPU", "availableCPU", "activeGPU", "pendingGPU", "availableGPU", "activeMemory", "pendingMemory", "availableMemory", "activeStorage", "pendingStorage", "availableStorage", "isOnline"
-                      FROM "providerSnapshot"
-                      ORDER BY "owner",DATE("checkDate"),"checkDate" DESC
-          ) "dailyProviderStats"
-      ON d."date"="dailyProviderStats"."date" AND "isOnline" IS TRUE
-      GROUP BY d."date"
-      ORDER BY d."date" ASC`,
+        FROM "day" d
+        INNER JOIN (
+            SELECT DISTINCT ON("hostUri",DATE("checkDate")) DATE("checkDate") AS date, ps."activeCPU", ps."pendingCPU", ps."availableCPU", ps."activeGPU", ps."pendingGPU", ps."availableGPU", ps."activeMemory", ps."pendingMemory", ps."availableMemory", ps."activeStorage", ps."pendingStorage", ps."availableStorage", ps."isOnline"
+                        FROM "providerSnapshot" ps
+                        INNER JOIN "provider" ON "provider"."owner"=ps."owner"
+                        ORDER BY "hostUri",DATE("checkDate"),"checkDate" DESC
+            ) "dailyProviderStats"
+        ON d."date"="dailyProviderStats"."date" AND "isOnline" IS TRUE
+        GROUP BY d."date"
+        ORDER BY d."date" ASC`,
         {
           type: QueryTypes.SELECT
         }

diff --git a/api/src/db/templateProvider.ts b/api/src/db/templateProvider.ts
@@ -1,7 +1,7 @@
 import { Template, TemplateFavorite, UserSetting } from "@shared/dbSchemas/user";
 import * as uuid from "uuid";
 import { Op } from "sequelize";
-import { toUTC } from "@src/shared/utils/date";
+import { toUTC } from "@src/utils/date";
 
 export async function getTemplateById(id: string, userId: string = "") {
   const template = await Template.findOne({

diff --git a/api/src/db/transactionsProvider.ts b/api/src/db/transactionsProvider.ts
@@ -1,6 +1,6 @@
 import { Transaction, AddressReference } from "@shared/dbSchemas/base";
 import { AkashBlock as Block, AkashMessage as Message } from "@shared/dbSchemas/akash";
-import { msgToJSON } from "@src/shared/utils/protobuf";
+import { msgToJSON } from "@src/utils/protobuf";
 import { QueryTypes } from "sequelize";
 import { chainDb } from "./dbConnection";
 

diff --git a/api/src/index.ts b/api/src/index.ts
@@ -1,15 +1,14 @@
 import express from "express";
 import cors from "cors";
 import packageJson from "../package.json";
-import { isProd } from "./shared/constants";
+import { isProd } from "./utils/constants";
 import * as Sentry from "@sentry/node";
-import * as marketDataProvider from "./providers/marketDataProvider";
 import { Scheduler } from "./scheduler";
 import { apiRouter } from "./routers/apiRouter";
 import { userRouter } from "./routers/userRouter";
 import { web3IndexRouter } from "./routers/web3indexRouter";
-import { bytesToHumanReadableSize } from "./shared/utils/files";
-import { env } from "./shared/utils/env";
+import { bytesToHumanReadableSize } from "./utils/files";
+import { env } from "./utils/env";
 import { chainDb, syncUserSchema, userDb } from "./db/dbConnection";
 import { dashboardRouter } from "./routers/dashboardRouter";
 

diff --git a/api/src/middlewares/privateMiddleware.ts b/api/src/middlewares/privateMiddleware.ts
@@ -1,4 +1,4 @@
-import { env } from "@src/shared/utils/env";
+import { env } from "@src/utils/env";
 
 export function privateMiddleware(req, res, next) {
   if (!env.SecretToken) {

diff --git a/api/src/middlewares/userMiddleware.ts b/api/src/middlewares/userMiddleware.ts
@@ -1,4 +1,4 @@
-import { env } from "@src/shared/utils/env";
+import { env } from "@src/utils/env";
 import { expressjwt as jwt, GetVerificationKey, Request as JWTRequest } from "express-jwt";
 import { expressJwtSecret } from "jwks-rsa";
 

diff --git a/api/src/providers/apiNodeProvider.ts b/api/src/providers/apiNodeProvider.ts
@@ -1,15 +1,14 @@
 import fetch from "node-fetch";
 import { getDeploymentRelatedMessages } from "../db/deploymentProvider";
-import { averageBlockCountInAMonth } from "@src/shared/constants";
-import { getMarketData } from "./marketDataProvider";
-import { coinToAsset } from "@src/shared/utils/coin";
+import { averageBlockCountInAMonth } from "@src/utils/constants";
+import { coinToAsset } from "@src/utils/coin";
 import { getTransactionByAddress } from "@src/db/transactionsProvider";
 import axios from "axios";
 import { Validator } from "@shared/dbSchemas/base";
 import { Op } from "sequelize";
 import { Provider, ProviderAttribute } from "@shared/dbSchemas/akash";
 import { cacheKeys, cacheResponse } from "@src/caching/helpers";
-import { env } from "@src/shared/utils/env";
+import { env } from "@src/utils/env";
 import { RestDeploymentInfoResponse, RestLeaseListResponse } from "@src/types/rest";
 
 const defaultNodeUrlMapping = {
@@ -349,8 +348,6 @@ export async function getDeployment(owner: string, dseq: string) {
     },
     include: [{ model: ProviderAttribute }]
   });
-
-  const marketData = await cacheResponse(60 * 5, cacheKeys.getMarketData, getMarketData);
   const deploymentDenom = deploymentData.escrow_account.balance.denom;
 
   const leases = leasesData.leases.map((x) => {

diff --git a/api/src/providers/githubProvider.ts b/api/src/providers/githubProvider.ts
@@ -1,5 +1,8 @@
 import { Octokit } from "@octokit/rest";
-import { env } from "@src/shared/utils/env";
+import { cacheKeys, cacheResponse } from "@src/caching/helpers";
+import { ProviderAttributesSchema } from "@src/types/provider";
+import { env } from "@src/utils/env";
+import axios from "axios";
 
 export function getOctokit() {
   const githubPAT = env.AkashlyticsGithubPAT;
@@ -14,3 +17,23 @@ export function getOctokit() {
     baseUrl: "https://api.github.com"
   });
 }
+
+export const getProviderAttributesSchema = async (): Promise<ProviderAttributesSchema> => {
+  // Fetching provider attributes schema
+  const response = await cacheResponse(
+    30,
+    cacheKeys.getProviderAttributesSchema,
+    async () => await axios.get("https://raw.githubusercontent.com/akash-network/cloudmos/main/config/provider-attributes.json")
+  );
+
+  return response.data;
+};
+
+export async function getAuditors() {
+  const response = await cacheResponse(60 * 5, cacheKeys.getAuditors, async () => {
+    const res = await axios.get("https://raw.githubusercontent.com/akash-network/cloudmos/main/config/auditors.json");
+    return res.data;
+  });
+
+  return response;
+}
diff --git a/api/src/providers/providerAttributesProvider.ts b/api/src/providers/providerAttributesProvider.ts
diff --git a/api/src/providers/providerStatusProvider.ts b/api/src/providers/providerStatusProvider.ts
@@ -1,9 +1,11 @@
 import { Provider, ProviderAttribute, ProviderAttributeSignature } from "@shared/dbSchemas/akash";
 import { ProviderSnapshot } from "@shared/dbSchemas/akash/providerSnapshot";
-import { toUTC } from "@src/shared/utils/date";
+import { toUTC } from "@src/utils/date";
 import { add } from "date-fns";
 import { Op } from "sequelize";
 import semver from "semver";
+import { mapProviderToList } from "@src/utils/map/provider";
+import { getAuditors, getProviderAttributesSchema } from "./githubProvider";
 
 export async function getNetworkCapacity() {
   const providers = await Provider.findAll({
@@ -12,21 +14,22 @@ export async function getNetworkCapacity() {
       deletedHeight: null
     }
   });
+  const filteredProviders = providers.filter((value, index, self) => self.map((x) => x.hostUri).indexOf(value.hostUri) === index);
 
   const stats = {
-    activeProviderCount: providers.length,
-    activeCPU: providers.map((x) => x.activeCPU).reduce((a, b) => a + b, 0),
-    activeGPU: providers.map((x) => x.activeGPU).reduce((a, b) => a + b, 0),
-    activeMemory: providers.map((x) => x.activeMemory).reduce((a, b) => a + b, 0),
-    activeStorage: providers.map((x) => x.activeStorage).reduce((a, b) => a + b, 0),
-    pendingCPU: providers.map((x) => x.pendingCPU).reduce((a, b) => a + b, 0),
-    pendingGPU: providers.map((x) => x.pendingGPU).reduce((a, b) => a + b, 0),
-    pendingMemory: providers.map((x) => x.pendingMemory).reduce((a, b) => a + b, 0),
-    pendingStorage: providers.map((x) => x.pendingStorage).reduce((a, b) => a + b, 0),
-    availableCPU: providers.map((x) => x.availableCPU).reduce((a, b) => a + b, 0),
-    availableGPU: providers.map((x) => x.availableGPU).reduce((a, b) => a + b, 0),
-    availableMemory: providers.map((x) => x.availableMemory).reduce((a, b) => a + b, 0),
-    availableStorage: providers.map((x) => x.availableStorage).reduce((a, b) => a + b, 0)
+    activeProviderCount: filteredProviders.length,
+    activeCPU: filteredProviders.map((x) => x.activeCPU).reduce((a, b) => a + b, 0),
+    activeGPU: filteredProviders.map((x) => x.activeGPU).reduce((a, b) => a + b, 0),
+    activeMemory: filteredProviders.map((x) => x.activeMemory).reduce((a, b) => a + b, 0),
+    activeStorage: filteredProviders.map((x) => x.activeStorage).reduce((a, b) => a + b, 0),
+    pendingCPU: filteredProviders.map((x) => x.pendingCPU).reduce((a, b) => a + b, 0),
+    pendingGPU: filteredProviders.map((x) => x.pendingGPU).reduce((a, b) => a + b, 0),
+    pendingMemory: filteredProviders.map((x) => x.pendingMemory).reduce((a, b) => a + b, 0),
+    pendingStorage: filteredProviders.map((x) => x.pendingStorage).reduce((a, b) => a + b, 0),
+    availableCPU: filteredProviders.map((x) => x.availableCPU).reduce((a, b) => a + b, 0),
+    availableGPU: filteredProviders.map((x) => x.availableGPU).reduce((a, b) => a + b, 0),
+    availableMemory: filteredProviders.map((x) => x.availableMemory).reduce((a, b) => a + b, 0),
+    availableStorage: filteredProviders.map((x) => x.availableStorage).reduce((a, b) => a + b, 0)
   };
 
   return {
@@ -60,17 +63,17 @@ export async function getProviders() {
         separate: true,
         where: {
           checkDate: {
-            [Op.gte]: add(nowUtc, { days: -7 })
+            [Op.gte]: add(nowUtc, { days: -1 })
           }
         }
       }
     ]
   });
+  const filteredProviders = providers.filter((value, index, self) => self.map((x) => x.hostUri).indexOf(value.hostUri) === index);
 
-  return providers.map((x) => {
+  return filteredProviders.map((x) => {
     const isValidVersion = x.cosmosSdkVersion ? semver.gte(x.cosmosSdkVersion, "v0.45.9") : false;
-    const name = new URL(x.hostUri).hostname;
-    const uptime7d = x.providerSnapshots.some((ps) => ps.isOnline) ? x.providerSnapshots.filter((ps) => ps.isOnline).length / x.providerSnapshots.length : 0;
+    const name = x.isOnline ? new URL(x.hostUri).hostname : null;
 
     return {
       owner: x.owner,
@@ -113,15 +116,37 @@ export async function getProviders() {
         memory: isValidVersion ? x.availableMemory : 0,
         storage: isValidVersion ? x.availableStorage : 0
       },
-      uptime7d: uptime7d,
-      uptime: x.providerSnapshots
-        .filter((ps) => ps.checkDate > add(nowUtc, { days: -1 }))
-        .map((ps) => ({
-          id: ps.id,
-          isOnline: ps.isOnline,
-          checkDate: ps.checkDate
-        })),
-      isValidVersion
+      uptime7d: x.uptime7d,
+      uptime: x.providerSnapshots.map((ps) => ({
+        id: ps.id,
+        isOnline: ps.isOnline,
+        checkDate: ps.checkDate
+      })),
+      isValidVersion,
+      isOnline: x.isOnline
     };
   });
 }
+
+export const getProviderList = async () => {
+  const providers = await Provider.findAll({
+    where: {
+      deletedHeight: null
+    },
+    include: [
+      {
+        model: ProviderAttribute
+      },
+      {
+        model: ProviderAttributeSignature
+      }
+    ]
+  });
+  const filteredProviders = providers.filter((value, index, self) => self.map((x) => x.hostUri).indexOf(value.hostUri) === index);
+  const providerAttributeSchemaQuery = getProviderAttributesSchema();
+  const auditorsQuery = getAuditors();
+
+  const [auditors, providerAttributeSchema] = await Promise.all([auditorsQuery, providerAttributeSchemaQuery]);
+
+  return filteredProviders.map((x) => mapProviderToList(x, providerAttributeSchema, auditors));
+};
diff --git a/api/src/providers/stripeProvider.ts b/api/src/providers/stripeProvider.ts
@@ -1,6 +1,6 @@
 import { UserSetting } from "@shared/dbSchemas/user";
 import { PlanCode } from "@shared/plans";
-import { env } from "@src/shared/utils/env";
+import { env } from "@src/utils/env";
 
 import Stripe from "stripe";
 

diff --git a/api/src/providers/templateReposProvider.ts b/api/src/providers/templateReposProvider.ts
@@ -2,11 +2,11 @@ import fetch from "node-fetch";
 import { markdownToTxt } from "markdown-to-txt";
 import path from "path";
 import { getOctokit } from "./githubProvider";
-import { isUrlAbsolute } from "@src/shared/utils/urls";
+import { isUrlAbsolute } from "@src/utils/urls";
 import * as fs from "fs";
 import { Octokit } from "@octokit/rest";
 import { getLogoFromPath } from "./templateReposLogos";
-import { dataFolderPath } from "@src/shared/constants";
+import { dataFolderPath } from "@src/utils/constants";
 import { GithubChainRegistryAssetListResponse } from "@src/types";
 import { GithubDirectoryItem } from "@src/types/github";