feature: Add metadataTypes endpoint - readonly; available for everyone

package.json

@@ -34,6 +34,7 @@
     "@elastic/elasticsearch": "^8.15.0",
     "@nestjs-modules/mailer": "^2.0.2",
     "@nestjs/axios": "^3.0.0",
+    "@nestjs/cache-manager": "^2.3.0",
     "@nestjs/common": "^10.3.8",
     "@nestjs/config": "^3.0.0",
     "@nestjs/core": "^10.3.8",

src/app.module.ts

@@ -32,6 +32,7 @@ import { EventEmitterModule } from "@nestjs/event-emitter";
 import { AdminModule } from "./admin/admin.module";
 import { HealthModule } from "./health/health.module";
 import { LoggerModule } from "./loggers/logger.module";
+import { CacheModule } from "@nestjs/cache-manager";
 
 @Module({
   imports: [
@@ -94,6 +95,14 @@ import { LoggerModule } from "./loggers/logger.module";
     UsersModule,
     AdminModule,
     HealthModule,
+    CacheModule.register({
+      ttl: 3600, // cache duration in seconds (1 hour)
+      max: 100, // maximum number of items in cache
+      // Uncomment and configure Redis as needed
+      // store: redisStore,
+      // host: 'localhost',
+      // port: 6379,
+    }),
   ],
   controllers: [],
   providers: [

src/auth/auth.service.ts

@@ -10,7 +10,6 @@ import { flattenObject, parseBoolean } from "src/common/utils";
 import { Issuer } from "openid-client";
 import { ReturnedAuthLoginDto } from "./dto/returnedLogin.dto";
 import { ReturnedUserDto } from "src/users/dto/returned-user.dto";
-import { CreateUserSettingsDto } from "src/users/dto/create-user-settings.dto";
 
 @Injectable()
 export class AuthService {
@@ -44,7 +43,6 @@ export class AuthService {
   async login(user: Omit<User, "password">): Promise<ReturnedAuthLoginDto> {
     const expiresIn = this.configService.get<number>("jwt.expiresIn");
     const accessToken = this.jwtService.sign(user, { expiresIn });
-    await this.postLoginTasks(user);
     return {
       access_token: accessToken,
       id: accessToken,
@@ -124,30 +122,4 @@ export class AuthService {
 
     return { logout: "successful" };
   }
-  /**
-   * postLoginTasks: Executes additional tasks after user login.
-   *
-   * - Checks if the user has userSettings record.
-   * - If user has no userSetting, it creates default userSetting for the user.
-   * @param user - The logged-in user (without password).
-   */
-  async postLoginTasks(user: Omit<User, "password">) {
-    if (!user) return;
-
-    const userId = user._id;
-
-    const userSettings = await this.usersService.findByIdUserSettings(userId);
-
-    if (!userSettings) {
-      Logger.log(
-        `Adding default settings to user ${user.username} with userId: ${user._id}`,
-        "postLoginTasks",
-      );
-      const createUserSettingsDto: CreateUserSettingsDto = {
-        userId,
-        externalSettings: {},
-      };
-      await this.usersService.createUserSettings(userId, createUserSettingsDto);
-    }
-  }
 }

src/config/default-filters.config.json

@@ -0,0 +1,11 @@
+[
+  { "type": "LocationFilterComponent", "visible": true },
+  { "type": "PidFilterComponent", "visible": true },
+  { "type": "PidFilterContainsComponent", "visible": false },
+  { "type": "PidFilterStartsWithComponent", "visible": false },
+  { "type": "GroupFilterComponent", "visible": true },
+  { "type": "TypeFilterComponent", "visible": true },
+  { "type": "KeywordFilterComponent", "visible": true },
+  { "type": "DateRangeFilterComponent", "visible": true },
+  { "type": "TextFilterComponent", "visible": true }
+]

src/config/frontend.config.json

@@ -26,7 +26,87 @@
   "jsonMetadataEnabled": true,
   "jupyterHubUrl": "",
   "landingPage": "doi.ess.eu/detail/",
-  "lbBaseURL": "http://localhost:3000",
+  "lbBaseURL": "http://127.0.0.1:3000",
+  "localColumns": [
+    {
+      "name": "select",
+      "order": 0,
+      "type": "standard",
+      "enabled": true
+    },
+    {
+      "name": "pid",
+      "order": 1,
+      "type": "standard",
+      "enabled": true
+    },
+    {
+      "name": "datasetName",
+      "order": 2,
+      "type": "standard",
+      "enabled": true
+    },
+    {
+      "name": "runNumber",
+      "order": 3,
+      "type": "standard",
+      "enabled": true
+    },
+    {
+      "name": "sourceFolder",
+      "order": 4,
+      "type": "standard",
+      "enabled": true
+    },
+    {
+      "name": "size",
+      "order": 5,
+      "type": "standard",
+      "enabled": true
+    },
+    {
+      "name": "creationTime",
+      "order": 6,
+      "type": "standard",
+      "enabled": true
+    },
+    {
+      "name": "type",
+      "order": 7,
+      "type": "standard",
+      "enabled": true
+    },
+    {
+      "name": "image",
+      "order": 8,
+      "type": "standard",
+      "enabled": true
+    },
+    {
+      "name": "metadata",
+      "order": 9,
+      "type": "standard",
+      "enabled": false
+    },
+    {
+      "name": "proposalId",
+      "order": 10,
+      "type": "standard",
+      "enabled": true
+    },
+    {
+      "name": "ownerGroup",
+      "order": 11,
+      "type": "standard",
+      "enabled": false
+    },
+    {
+      "name": "dataStatus",
+      "order": 12,
+      "type": "standard",
+      "enabled": false
+    }
+  ],
   "logbookEnabled": true,
   "loginFormEnabled": true,
   "maxDirectDownloadSize": 5000000000,
@@ -100,108 +180,5 @@
       "enabled": "#Selected",
       "authorization": ["#datasetAccess", "#datasetPublic"]
     }
-  ],
-  "labelMaps": {
-    "filters": {
-      "LocationFilter": "Location",
-      "PidFilter": "Pid",
-      "GroupFilter": "Group",
-      "TypeFilter": "Type",
-      "KeywordFilter": "Keyword",
-      "DateRangeFilter": "Start Date - End Date",
-      "TextFilter": "Text"
-    }
-  },
-  "defaultDatasetsListSettings": {
-    "columns": [
-      {
-        "name": "select",
-        "order": 0,
-        "type": "standard",
-        "enabled": true
-      },
-      {
-        "name": "pid",
-        "order": 1,
-        "type": "standard",
-        "enabled": true
-      },
-      {
-        "name": "datasetName",
-        "order": 2,
-        "type": "standard",
-        "enabled": true
-      },
-      {
-        "name": "runNumber",
-        "order": 3,
-        "type": "standard",
-        "enabled": true
-      },
-      {
-        "name": "sourceFolder",
-        "order": 4,
-        "type": "standard",
-        "enabled": true
-      },
-      {
-        "name": "size",
-        "order": 5,
-        "type": "standard",
-        "enabled": true
-      },
-      {
-        "name": "creationTime",
-        "order": 6,
-        "type": "standard",
-        "enabled": true
-      },
-      {
-        "name": "type",
-        "order": 7,
-        "type": "standard",
-        "enabled": true
-      },
-      {
-        "name": "image",
-        "order": 8,
-        "type": "standard",
-        "enabled": true
-      },
-      {
-        "name": "metadata",
-        "order": 9,
-        "type": "standard",
-        "enabled": false
-      },
-      {
-        "name": "proposalId",
-        "order": 10,
-        "type": "standard",
-        "enabled": true
-      },
-      {
-        "name": "ownerGroup",
-        "order": 11,
-        "type": "standard",
-        "enabled": false
-      },
-      {
-        "name": "dataStatus",
-        "order": 12,
-        "type": "standard",
-        "enabled": false
-      }
-    ],
-    "filters": [
-      { "LocationFilter": true },
-      { "PidFilter": true },
-      { "GroupFilter": true },
-      { "TypeFilter": true },
-      { "KeywordFilter": true },
-      { "DateRangeFilter": true },
-      { "TextFilter": true }
-    ],
-    "conditions": []
-  }
+  ]
 }

src/datasets/datasets.controller.ts

@@ -98,6 +98,7 @@ import { JWTUser } from "src/auth/interfaces/jwt-user.interface";
 import { LogbooksService } from "src/logbooks/logbooks.service";
 import configuration from "src/config/configuration";
 import { DatasetType } from "./dataset-type.enum";
+import { CACHE_MANAGER, CacheInterceptor } from "@nestjs/cache-manager";
 
 @ApiBearerAuth()
 @ApiExtraModels(
@@ -888,6 +889,30 @@ export class DatasetsController {
     return this.datasetsService.metadataKeys(parsedFilters);
   }
 
+  // GET /datasets/metadataTypes
+  // @UseGuards(PoliciesGuard)
+  // @CheckPolicies((ability: AppAbility) =>
+  //   ability.can(Action.DatasetRead, DatasetClass),
+  // )
+  @Get("/metadataTypes")
+  @ApiOperation({
+    summary: "Retrieve all available scientific metadata field types.",
+    description:
+      "This endpoint returns the field names and their corresponding data types from the `scientificMetadata` fields of all datasets in the system. It aggregates all the unique field names and their associated types from the dataset collection.",
+  })
+  @ApiResponse({
+    status: 200,
+    type: Array,
+    description:
+      "Returns an array of objects where each object contains a metadata field name and its associated type.",
+  })
+  @UseInterceptors(CacheInterceptor)
+  async metadataTypes(): Promise<any[]> {
+    const result = this.datasetsService.getScientificMetadataTypes();
+
+    return result;
+  }
+
   // GET /datasets/findOne
   @UseGuards(PoliciesGuard)
   @CheckPolicies("datasets", (ability: AppAbility) =>

src/datasets/datasets.module.ts

@@ -13,6 +13,7 @@ import { LogbooksModule } from "src/logbooks/logbooks.module";
 import { PoliciesService } from "src/policies/policies.service";
 import { PoliciesModule } from "src/policies/policies.module";
 import { ElasticSearchModule } from "src/elastic-search/elastic-search.module";
+import { CacheModule } from "@nestjs/cache-manager";
 
 @Module({
   imports: [
@@ -62,6 +63,14 @@ import { ElasticSearchModule } from "src/elastic-search/elastic-search.module";
         inject: [PoliciesService],
       },
     ]),
+    CacheModule.register({
+      ttl: 3600000, // cache duration in seconds (1 hour)
+      max: 100, // maximum number of items in cache
+      // Uncomment and configure Redis as needed
+      // store: redisStore,
+      // host: 'localhost',
+      // port: 6379,
+    }),
   ],
   exports: [DatasetsService],
   controllers: [DatasetsController],

src/datasets/datasets.service.ts

@@ -476,6 +476,43 @@ export class DatasetsService {
     }
   }
 
+  async getScientificMetadataTypes() {
+    // TODO performance research required, say 1K records, 100K and 1M
+    return this.datasetModel.aggregate([
+      {
+        // Step 1: Convert the scientificMetadata object into an array of key-value pairs
+        $project: {
+          scientificMetadataArray: { $objectToArray: "$scientificMetadata" },
+        },
+      },
+      {
+        // Step 2: Unwind the array to treat each field individually
+        $unwind: "$scientificMetadataArray",
+      },
+      {
+        // Step 3: Group by the field name and accumulate types
+        $group: {
+          _id: "$scientificMetadataArray.k", // Group by field name (key)
+          types: { $addToSet: { $type: "$scientificMetadataArray.v.value" } }, // Add the data type of the value
+        },
+      },
+      {
+        // Step 4: Convert the array of types into a string for each field name
+        $project: {
+          _id: 0, // Don't include the default _id
+          metadataKey: "$_id",
+          metadataType: {
+            $cond: {
+              if: { $eq: [{ $size: "$types" }, 1] }, // If only one type is present
+              then: { $arrayElemAt: ["$types", 0] }, // Use the type
+              else: "mixed", // If there are multiple types, mark it as 'mixed'
+            },
+          },
+        },
+      },
+    ]);
+  }
+
   async isElasticSearchDBEmpty() {
     if (!this.ESClient) return;
     const count = await this.ESClient.getCount();