diff --git a/.github/workflows/release-and-publish-sdk.yml b/.github/workflows/release-and-publish-sdk.yml index 9a79e592f..980cbe8d6 100644 --- a/.github/workflows/release-and-publish-sdk.yml +++ b/.github/workflows/release-and-publish-sdk.yml @@ -121,7 +121,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: swagger-schema + name: swagger-schema-${{ github.sha }} path: ./swagger-schema.json generate-upload-sdk: @@ -139,7 +139,7 @@ jobs: - uses: actions/download-artifact@v4 with: - name: swagger-schema + name: swagger-schema-${{ github.sha }} path: . - name: Generate Client @@ -215,7 +215,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/upload-sdk-artifact.yml b/.github/workflows/upload-sdk-artifact.yml index 369426a08..a33c50a09 100644 --- a/.github/workflows/upload-sdk-artifact.yml +++ b/.github/workflows/upload-sdk-artifact.yml @@ -39,7 +39,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: swagger-schema + name: swagger-schema-${{ github.sha }} path: ./swagger-schema.json generate-and-upload-sdk: @@ -56,7 +56,7 @@ jobs: - uses: actions/download-artifact@v4 with: - name: swagger-schema + name: swagger-schema-${{ github.sha }} path: . - name: Generate Client diff --git a/README.md b/README.md index 103b8c3fd..b1ffe56a3 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![Test](https://github.com/SciCatProject/scicat-backend-next/actions/workflows/test.yml/badge.svg)](https://github.com/SciCatProject/scicat-backend-next/actions/workflows/test.yml) [![Deploy](https://github.com/SciCatProject/scicat-backend-next/actions/workflows/deploy.yml/badge.svg)](https://github.com/SciCatProject/scicat-backend-next/actions/workflows/deploy.yml) +[![Generate and upload latest SDK artifacts](https://github.com/SciCatProject/scicat-backend-next/actions/workflows/upload-sdk-artifact.yml/badge.svg?branch=master)](https://github.com/SciCatProject/scicat-backend-next/actions/workflows/upload-sdk-artifact.yml) [![DeepScan grade](https://deepscan.io/api/teams/8394/projects/19251/branches/494247/badge/grade.svg)](https://deepscan.io/dashboard#view=project&tid=8394&pid=19251&bid=494247) [![Known Vulnerabilities](https://snyk.io/test/github/SciCatProject/scicat-backend-next/master/badge.svg?targetFile=package.json)](https://snyk.io/test/github/SciCatProject/scicat-backend-next/master?targetFile=package.json) diff --git a/migrations/20240920111733-dataset-unified-schema.js b/migrations/20240920111733-dataset-unified-schema.js new file mode 100644 index 000000000..598d6db6c --- /dev/null +++ b/migrations/20240920111733-dataset-unified-schema.js @@ -0,0 +1,47 @@ +module.exports = { + async up(db, client) { + // TODO write your migration here. + // See https://github.com/seppevs/migrate-mongo/#creating-a-new-migration-script + // Example: + // await db.collection('albums').updateOne({artist: 'The Beatles'}, {$set: {blacklisted: true}}); + await db.collection("Dataset").updateMany({}, [ + { + $set: { + proposalIds: ["$proposalId"], + instrumentIds: ["$instrumentId"], + sampleIds: ["$sampleId"], + }, + }, + ]); + await db.collection("Dataset").updateMany({ type: "derived" }, [ + { + $set: { + principalInvestigator: "$investigator", + }, + }, + ]); + }, + + async down(db, client) { + // TODO write the statements to rollback your migration (if possible) + // Example: + // await db.collection('albums').updateOne({artist: 'The Beatles'}, {$set: {blacklisted: false}}); + + await db.collection("Dataset").updateMany({}, [ + { + $set: { + proposalId: "$proposalIds[0]", + instrumentId: "$instrumentId[0]", + sampleId: "$sampleId[0]", + }, + }, + ]); + await db.collection("Dataset").updateMany({ type: "derived" }, [ + { + $set: { + investigator: "$principalInvestigator", + }, + }, + ]); + }, +}; diff --git a/package.json b/package.json index c2d233fcf..2be6c26a5 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", "test:api": "npm run test:api:jest --maxWorkers=50% && concurrently -k -s first \"wait-on http://localhost:3000/explorer/ && npm run test:api:mocha\" \"npm run start\"", "test:api:jest": "jest --config ./test/config/jest-e2e.json --maxWorkers=50%", - "test:api:mocha": "mocha --config ./test/config/.mocharc.json -r chai/register-should.js ", + "test:api:mocha": "mocha --config ./test/config/.mocharc.json -r chai/register-should.js", "prepare:local": "docker-compose -f CI/E2E/docker-compose-local.yaml --env-file CI/E2E/.env.elastic-search up -d && cp functionalAccounts.json.test functionalAccounts.json" }, "dependencies": { diff --git a/src/config/configuration.ts b/src/config/configuration.ts index 96e9e9bbd..94a190d89 100644 --- a/src/config/configuration.ts +++ b/src/config/configuration.ts @@ -59,7 +59,7 @@ const configuration = () => { const config = { versions: { - api: "v3", + api: "3", }, swaggerPath: process.env.SWAGGER_PATH || "explorer", loggerConfigs: jsonConfigMap.loggers || [defaultLogger], diff --git a/src/datasets/datasets.controller.ts b/src/datasets/datasets.controller.ts index 33d798bc4..b278cfc4c 100644 --- a/src/datasets/datasets.controller.ts +++ b/src/datasets/datasets.controller.ts @@ -36,10 +36,10 @@ import { } from "@nestjs/swagger"; import { Request } from "express"; import { DatasetsService } from "./datasets.service"; -import { PartialUpdateDatasetDto } from "./dto/update-dataset.dto"; +import { PartialUpdateDatasetObsoleteDto } from "./dto/update-dataset-obsolete.dto"; import { DatasetClass, DatasetDocument } from "./schemas/dataset.schema"; -import { CreateRawDatasetDto } from "./dto/create-raw-dataset.dto"; -import { CreateDerivedDatasetDto } from "./dto/create-derived-dataset.dto"; +import { CreateRawDatasetObsoleteDto } from "./dto/create-raw-dataset-obsolete.dto"; +import { CreateDerivedDatasetObsoleteDto } from "./dto/create-derived-dataset-obsolete.dto"; import { PoliciesGuard } from "src/casl/guards/policies.guard"; import { CheckPolicies } from "src/casl/decorators/check-policies.decorator"; import { AppAbility, CaslAbilityFactory } from "src/casl/casl-ability.factory"; @@ -74,13 +74,13 @@ import { validate, ValidationError, ValidatorOptions } from "class-validator"; import { HistoryInterceptor } from "src/common/interceptors/history.interceptor"; import { CreateDatasetOrigDatablockDto } from "src/origdatablocks/dto/create-dataset-origdatablock"; import { - PartialUpdateRawDatasetDto, - UpdateRawDatasetDto, -} from "./dto/update-raw-dataset.dto"; + PartialUpdateRawDatasetObsoleteDto, + UpdateRawDatasetObsoleteDto, +} from "./dto/update-raw-dataset-obsolete.dto"; import { - PartialUpdateDerivedDatasetDto, - UpdateDerivedDatasetDto, -} from "./dto/update-derived-dataset.dto"; + PartialUpdateDerivedDatasetObsoleteDto, + UpdateDerivedDatasetObsoleteDto, +} from "./dto/update-derived-dataset-obsolete.dto"; import { CreateDatasetDatablockDto } from "src/datablocks/dto/create-dataset-datablock"; import { filterDescription, @@ -98,12 +98,18 @@ import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; import { LogbooksService } from "src/logbooks/logbooks.service"; import configuration from "src/config/configuration"; import { DatasetType } from "./dataset-type.enum"; +import { OutputDatasetObsoleteDto } from "./dto/output-dataset-obsolete.dto"; +import { CreateDatasetDto } from "./dto/create-dataset.dto"; +import { + PartialUpdateDatasetDto, + UpdateDatasetDto, +} from "./dto/update-dataset.dto"; @ApiBearerAuth() @ApiExtraModels( CreateAttachmentDto, - CreateDerivedDatasetDto, - CreateRawDatasetDto, + CreateDerivedDatasetObsoleteDto, + CreateRawDatasetObsoleteDto, HistoryClass, TechniqueClass, RelationshipClass, @@ -282,7 +288,7 @@ export class DatasetsController { return dataset; } - async checkPermissionsForDataset(request: Request, id: string) { + async checkPermissionsForDatasetObsolete(request: Request, id: string) { const dataset = await this.datasetsService.findOne({ where: { pid: id } }); const user: JWTUser = request.user as JWTUser; @@ -331,7 +337,10 @@ export class DatasetsController { } async generateDatasetInstanceForPermissions( - dataset: CreateRawDatasetDto | CreateDerivedDatasetDto | DatasetClass, + dataset: + | CreateRawDatasetObsoleteDto + | CreateDerivedDatasetObsoleteDto + | DatasetClass, ): Promise { const datasetInstance = new DatasetClass(); datasetInstance._id = ""; @@ -344,9 +353,9 @@ export class DatasetsController { return datasetInstance; } - async checkPermissionsForDatasetCreate( + async checkPermissionsForObsoleteDatasetCreate( request: Request, - dataset: CreateRawDatasetDto | CreateDerivedDatasetDto, + dataset: CreateRawDatasetObsoleteDto | CreateDerivedDatasetObsoleteDto, ) { const user: JWTUser = request.user as JWTUser; @@ -387,7 +396,110 @@ export class DatasetsController { return dataset; } - // POST /datasets + convertObsoleteToCurrentSchema( + inputObsoleteDataset: + | CreateRawDatasetObsoleteDto + | CreateDerivedDatasetObsoleteDto + | UpdateRawDatasetObsoleteDto + | UpdateDerivedDatasetObsoleteDto + | PartialUpdateRawDatasetObsoleteDto + | PartialUpdateDerivedDatasetObsoleteDto, + ): CreateDatasetDto | UpdateDatasetDto | PartialUpdateDatasetDto { + const propertiesModifier: Record = { + version: "v3", + }; + + if ("proposalId" in inputObsoleteDataset) { + propertiesModifier.proposalIds = [ + (inputObsoleteDataset as CreateRawDatasetObsoleteDto).proposalId, + ]; + } + if ( + inputObsoleteDataset instanceof CreateRawDatasetObsoleteDto || + inputObsoleteDataset instanceof UpdateRawDatasetObsoleteDto || + inputObsoleteDataset instanceof PartialUpdateRawDatasetObsoleteDto + ) { + if ("sampleId" in inputObsoleteDataset) { + propertiesModifier.sampleIds = [ + (inputObsoleteDataset as CreateRawDatasetObsoleteDto).sampleId, + ]; + } + if ("instrumentId" in inputObsoleteDataset) { + propertiesModifier.instrumentIds = [ + (inputObsoleteDataset as CreateRawDatasetObsoleteDto).instrumentId, + ]; + } + } else { + if ("investigator" in inputObsoleteDataset) { + propertiesModifier.principalInvestigator = ( + inputObsoleteDataset as CreateDerivedDatasetObsoleteDto + ).investigator; + } + } + + let outputDataset: + | CreateDatasetDto + | UpdateDatasetDto + | PartialUpdateDatasetDto = {}; + if ( + inputObsoleteDataset instanceof CreateRawDatasetObsoleteDto || + inputObsoleteDataset instanceof CreateDerivedDatasetObsoleteDto + ) { + outputDataset = { + ...(inputObsoleteDataset as CreateDatasetDto), + ...propertiesModifier, + } as CreateDatasetDto; + } else if ( + inputObsoleteDataset instanceof UpdateRawDatasetObsoleteDto || + inputObsoleteDataset instanceof UpdateDerivedDatasetObsoleteDto + ) { + outputDataset = { + ...(inputObsoleteDataset as UpdateDatasetDto), + ...propertiesModifier, + } as UpdateDatasetDto; + } else if ( + inputObsoleteDataset instanceof PartialUpdateRawDatasetObsoleteDto || + inputObsoleteDataset instanceof PartialUpdateDerivedDatasetObsoleteDto + ) { + outputDataset = { + ...(inputObsoleteDataset as PartialUpdateDatasetDto), + ...propertiesModifier, + } as PartialUpdateDatasetDto; + } + + return outputDataset; + } + + convertCurrentToObsoleteSchema( + inputDataset: DatasetClass | null, + ): OutputDatasetObsoleteDto { + const propertiesModifier: Record = {}; + if (inputDataset) { + if ("proposalIds" in inputDataset) { + propertiesModifier.proposalId = inputDataset.proposalIds![0]; + } + if ("sampleIds" in inputDataset) { + propertiesModifier.sampleId = inputDataset.sampleIds![0]; + } + if ("instrumentIds" in inputDataset) { + propertiesModifier.instrumentId = inputDataset.instrumentIds![0]; + } + if (inputDataset.type == "derived") { + if ("investigator" in inputDataset) { + propertiesModifier.investigator = inputDataset.principalInvestigator; + } + } + } + + const outputDataset: OutputDatasetObsoleteDto = { + ...(inputDataset as DatasetDocument).toObject(), + ...propertiesModifier, + }; + + return outputDataset; + } + + // POST https://scicat.ess.eu/api/v3/datasets @UseGuards(PoliciesGuard) @CheckPolicies("datasets", (ability: AppAbility) => ability.can(Action.DatasetCreate, DatasetClass), @@ -403,14 +515,14 @@ export class DatasetsController { description: "It creates a new dataset and returns it completed with systems fields.", }) - @ApiExtraModels(CreateRawDatasetDto, CreateDerivedDatasetDto) + @ApiExtraModels(CreateRawDatasetObsoleteDto, CreateDerivedDatasetObsoleteDto) @ApiBody({ description: "Input fields for the dataset to be created", required: true, schema: { oneOf: [ - { $ref: getSchemaPath(CreateRawDatasetDto) }, - { $ref: getSchemaPath(CreateDerivedDatasetDto) }, + { $ref: getSchemaPath(CreateRawDatasetObsoleteDto) }, + { $ref: getSchemaPath(CreateDerivedDatasetObsoleteDto) }, ], }, }) @@ -421,25 +533,34 @@ export class DatasetsController { }) async create( @Req() request: Request, - @Body() createDatasetDto: CreateRawDatasetDto | CreateDerivedDatasetDto, - ): Promise { + @Body() + createDatasetObsoleteDto: + | CreateRawDatasetObsoleteDto + | CreateDerivedDatasetObsoleteDto, + ): Promise { // validate dataset - await this.validateDataset( - createDatasetDto, - createDatasetDto.type === "raw" - ? CreateRawDatasetDto - : CreateDerivedDatasetDto, - ); - - const datasetDTO = await this.checkPermissionsForDatasetCreate( - request, - createDatasetDto, - ); + const validatedDatasetObsoleteDto = (await this.validateDatasetObsolete( + createDatasetObsoleteDto, + createDatasetObsoleteDto.type === "raw" + ? CreateRawDatasetObsoleteDto + : CreateDerivedDatasetObsoleteDto, + )) as CreateRawDatasetObsoleteDto | CreateDerivedDatasetObsoleteDto; + + const obsoleteDatasetDto = + await this.checkPermissionsForObsoleteDatasetCreate( + request, + validatedDatasetObsoleteDto, + ); try { - const createdDataset = await this.datasetsService.create(datasetDTO); - - return createdDataset; + const datasetDto = this.convertObsoleteToCurrentSchema( + obsoleteDatasetDto, + ) as CreateDatasetDto; + const createdDataset = await this.datasetsService.create(datasetDto); + const outputObsoleteDatasetDto = + this.convertCurrentToObsoleteSchema(createdDataset); + + return outputObsoleteDatasetDto; } catch (error) { if ((error as MongoError).code === 11000) { throw new ConflictException( @@ -451,21 +572,21 @@ export class DatasetsController { } } - async validateDataset( + async validateDatasetObsolete( inputDatasetDto: - | CreateRawDatasetDto - | CreateDerivedDatasetDto - | PartialUpdateRawDatasetDto - | PartialUpdateDerivedDatasetDto - | UpdateRawDatasetDto - | UpdateDerivedDatasetDto, + | CreateRawDatasetObsoleteDto + | CreateDerivedDatasetObsoleteDto + | PartialUpdateRawDatasetObsoleteDto + | PartialUpdateDerivedDatasetObsoleteDto + | UpdateRawDatasetObsoleteDto + | UpdateDerivedDatasetObsoleteDto, dto: ClassConstructor< - | CreateRawDatasetDto - | CreateDerivedDatasetDto - | PartialUpdateRawDatasetDto - | PartialUpdateDerivedDatasetDto - | UpdateRawDatasetDto - | UpdateDerivedDatasetDto + | CreateRawDatasetObsoleteDto + | CreateDerivedDatasetObsoleteDto + | PartialUpdateRawDatasetObsoleteDto + | PartialUpdateDerivedDatasetObsoleteDto + | UpdateRawDatasetObsoleteDto + | UpdateDerivedDatasetObsoleteDto >, ) { const validateOptions: ValidatorOptions = { @@ -478,11 +599,18 @@ export class DatasetsController { }, }; + // first we convert input object to the correct class + const outputDatasetDto = plainToInstance(dto, inputDatasetDto); + if ( - inputDatasetDto instanceof - (CreateRawDatasetDto || CreateDerivedDatasetDto) + outputDatasetDto instanceof + (CreateRawDatasetObsoleteDto || CreateDerivedDatasetObsoleteDto) ) { - if (!(inputDatasetDto.type in DatasetType)) { + if ( + !(Object.values(DatasetType) as string[]).includes( + outputDatasetDto.type, + ) + ) { throw new HttpException( { status: HttpStatus.BAD_REQUEST, @@ -493,7 +621,6 @@ export class DatasetsController { } } - const outputDatasetDto = plainToInstance(dto, inputDatasetDto); const errors = await validate(outputDatasetDto, validateOptions); if (errors.length > 0) { @@ -525,14 +652,14 @@ export class DatasetsController { description: "It validates the dataset provided as input, and returns true if the information is a valid dataset", }) - @ApiExtraModels(CreateRawDatasetDto, CreateDerivedDatasetDto) + @ApiExtraModels(CreateRawDatasetObsoleteDto, CreateDerivedDatasetObsoleteDto) @ApiBody({ description: "Input fields for the dataset that needs to be validated", required: true, schema: { oneOf: [ - { $ref: getSchemaPath(CreateRawDatasetDto) }, - { $ref: getSchemaPath(CreateDerivedDatasetDto) }, + { $ref: getSchemaPath(CreateRawDatasetObsoleteDto) }, + { $ref: getSchemaPath(CreateDerivedDatasetObsoleteDto) }, ], }, }) @@ -544,19 +671,25 @@ export class DatasetsController { }) async isValid( @Req() request: Request, - @Body() createDatasetDto: CreateRawDatasetDto | CreateDerivedDatasetDto, + @Body() + createDatasetObsoleteDto: + | CreateRawDatasetObsoleteDto + | CreateDerivedDatasetObsoleteDto, ): Promise<{ valid: boolean }> { - await this.checkPermissionsForDatasetCreate(request, createDatasetDto); + await this.checkPermissionsForObsoleteDatasetCreate( + request, + createDatasetObsoleteDto, + ); const dtoTestRawCorrect = plainToInstance( - CreateRawDatasetDto, - createDatasetDto, + CreateRawDatasetObsoleteDto, + createDatasetObsoleteDto, ); const errorsTestRawCorrect = await validate(dtoTestRawCorrect); const dtoTestDerivedCorrect = plainToInstance( - CreateDerivedDatasetDto, - createDatasetDto, + CreateDerivedDatasetObsoleteDto, + createDatasetObsoleteDto, ); const errorsTestDerivedCorrect = await validate(dtoTestDerivedCorrect); @@ -597,7 +730,7 @@ export class DatasetsController { @Req() request: Request, @Headers() headers: Record, @Query(new FilterPipe()) queryFilter: { filter?: string }, - ): Promise { + ): Promise { const mergedFilters = replaceLikeOperator( this.updateMergedFiltersForList( request, @@ -607,10 +740,14 @@ export class DatasetsController { // this should be implemented at database level const datasets = await this.datasetsService.findAll(mergedFilters); + let outputDatasets: OutputDatasetObsoleteDto[] = []; if (datasets && datasets.length > 0) { const includeFilters = mergedFilters.include ?? []; + outputDatasets = datasets.map((dataset) => + this.convertCurrentToObsoleteSchema(dataset), + ); await Promise.all( - datasets.map(async (dataset) => { + outputDatasets.map(async (dataset) => { if (includeFilters) { await Promise.all( includeFilters.map(async ({ relation }) => { @@ -645,7 +782,7 @@ export class DatasetsController { }), ); } - return datasets; + return outputDatasets; } // GET /datasets/fullquery @@ -687,7 +824,7 @@ export class DatasetsController { async fullquery( @Req() request: Request, @Query() filters: { fields?: string; limits?: string }, - ): Promise { + ): Promise { const user: JWTUser = request.user as JWTUser; const fields: IDatasetFields = JSON.parse(filters.fields ?? "{}"); @@ -724,7 +861,9 @@ export class DatasetsController { limits: JSON.parse(filters.limits ?? "{}"), }; - return this.datasetsService.fullquery(parsedFilters); + const results = await this.datasetsService.fullquery(parsedFilters); + + return results as OutputDatasetObsoleteDto[]; } // GET /fullfacets @@ -940,7 +1079,7 @@ export class DatasetsController { @Req() request: Request, @Headers() headers: Record, @Query(new FilterPipe()) queryFilter: { filter?: string }, - ): Promise { + ): Promise { const mergedFilters = replaceLikeOperator( this.updateMergedFiltersForList( request, @@ -948,30 +1087,34 @@ export class DatasetsController { ) as Record, ) as IFilters; - const dataset = (await this.datasetsService.findOne( - mergedFilters, - )) as DatasetClass; + const databaseDataset = await this.datasetsService.findOne(mergedFilters); - if (dataset) { + const outputDataset = + await this.convertCurrentToObsoleteSchema(databaseDataset); + + if (outputDataset) { const includeFilters = mergedFilters.include ?? []; await Promise.all( includeFilters.map(async ({ relation }) => { switch (relation) { case "attachments": { - dataset.attachments = await this.attachmentsService.findAll({ - datasetId: dataset.pid, - }); + outputDataset.attachments = await this.attachmentsService.findAll( + { + datasetId: outputDataset.pid, + }, + ); break; } case "origdatablocks": { - dataset.origdatablocks = await this.origDatablocksService.findAll( - { where: { datasetId: dataset.pid } }, - ); + outputDataset.origdatablocks = + await this.origDatablocksService.findAll({ + where: { datasetId: outputDataset.pid }, + }); break; } case "datablocks": { - dataset.datablocks = await this.datablocksService.findAll({ - datasetId: dataset.pid, + outputDataset.datablocks = await this.datablocksService.findAll({ + datasetId: outputDataset.pid, }); break; } @@ -979,7 +1122,7 @@ export class DatasetsController { }), ); } - return dataset; + return outputDataset; } // GET /datasets/count @@ -1046,10 +1189,12 @@ export class DatasetsController { async findById( @Req() request: Request, @Param("pid") id: string, - ): Promise { - const dataset = await this.checkPermissionsForDataset(request, id); + ): Promise { + const dataset = this.convertCurrentToObsoleteSchema( + await this.checkPermissionsForDatasetObsolete(request, id), + ); - return dataset; + return dataset as OutputDatasetObsoleteDto; } // PATCH /datasets/:id @@ -1075,15 +1220,18 @@ export class DatasetsController { description: "Id of the dataset to modify", type: String, }) - @ApiExtraModels(PartialUpdateRawDatasetDto, PartialUpdateDerivedDatasetDto) + @ApiExtraModels( + PartialUpdateRawDatasetObsoleteDto, + PartialUpdateDerivedDatasetObsoleteDto, + ) @ApiBody({ description: "Fields that needs to be updated in the dataset. Only the fields that needs to be updated have to be passed in.", required: true, schema: { oneOf: [ - { $ref: getSchemaPath(PartialUpdateRawDatasetDto) }, - { $ref: getSchemaPath(PartialUpdateDerivedDatasetDto) }, + { $ref: getSchemaPath(PartialUpdateRawDatasetObsoleteDto) }, + { $ref: getSchemaPath(PartialUpdateDerivedDatasetObsoleteDto) }, ], }, }) @@ -1098,9 +1246,9 @@ export class DatasetsController { @Param("pid") pid: string, @Body() updateDatasetDto: - | PartialUpdateRawDatasetDto - | PartialUpdateDerivedDatasetDto, - ): Promise { + | PartialUpdateRawDatasetObsoleteDto + | PartialUpdateDerivedDatasetObsoleteDto, + ): Promise { const foundDataset = await this.datasetsService.findOne({ where: { pid } }); if (!foundDataset) { @@ -1108,11 +1256,11 @@ export class DatasetsController { } // NOTE: Default validation pipe does not validate union types. So we need custom validation. - await this.validateDataset( + await this.validateDatasetObsolete( updateDatasetDto, foundDataset.type === "raw" - ? PartialUpdateRawDatasetDto - : PartialUpdateDerivedDatasetDto, + ? PartialUpdateRawDatasetObsoleteDto + : PartialUpdateDerivedDatasetObsoleteDto, ); // NOTE: We need DatasetClass instance because casl module can not recognize the type from dataset mongo database model. If other fields are needed can be added later. @@ -1131,7 +1279,9 @@ export class DatasetsController { throw new ForbiddenException("Unauthorized to update this dataset"); } - return this.datasetsService.findByIdAndUpdate(pid, updateDatasetDto); + return this.convertCurrentToObsoleteSchema( + await this.datasetsService.findByIdAndUpdate(pid, updateDatasetDto), + ); } // PUT /datasets/:id @@ -1158,15 +1308,15 @@ export class DatasetsController { description: "Id of the dataset to modify", type: String, }) - @ApiExtraModels(UpdateRawDatasetDto, UpdateDerivedDatasetDto) + @ApiExtraModels(UpdateRawDatasetObsoleteDto, UpdateDerivedDatasetObsoleteDto) @ApiBody({ description: "Dataset object that needs to be updated. The whole dataset object with updated fields have to be passed in.", required: true, schema: { oneOf: [ - { $ref: getSchemaPath(UpdateRawDatasetDto) }, - { $ref: getSchemaPath(UpdateDerivedDatasetDto) }, + { $ref: getSchemaPath(UpdateRawDatasetObsoleteDto) }, + { $ref: getSchemaPath(UpdateDerivedDatasetObsoleteDto) }, ], }, }) @@ -1179,8 +1329,11 @@ export class DatasetsController { async findByIdAndReplace( @Req() request: Request, @Param("pid") pid: string, - @Body() updateDatasetDto: UpdateRawDatasetDto | UpdateDerivedDatasetDto, - ): Promise { + @Body() + updateDatasetObsoleteDto: + | UpdateRawDatasetObsoleteDto + | UpdateDerivedDatasetObsoleteDto, + ): Promise { const foundDataset = await this.datasetsService.findOne({ where: { pid } }); if (!foundDataset) { @@ -1188,11 +1341,11 @@ export class DatasetsController { } // NOTE: Default validation pipe does not validate union types. So we need custom validation. - const outputDto = await this.validateDataset( - updateDatasetDto, + const updateValidatedDto = await this.validateDatasetObsolete( + updateDatasetObsoleteDto, foundDataset.type === "raw" - ? UpdateRawDatasetDto - : UpdateDerivedDatasetDto, + ? UpdateRawDatasetObsoleteDto + : UpdateDerivedDatasetObsoleteDto, ); const datasetInstance = @@ -1210,10 +1363,15 @@ export class DatasetsController { throw new ForbiddenException("Unauthorized to update this dataset"); } - return this.datasetsService.findByIdAndReplace( + const updateDatasetDto = + await this.convertObsoleteToCurrentSchema(updateValidatedDto); + + const outputDatasetDto = await this.datasetsService.findByIdAndReplace( pid, - outputDto as UpdateRawDatasetDto | UpdateDerivedDatasetDto, + updateDatasetDto as UpdateDatasetDto, ); + + return await this.convertCurrentToObsoleteSchema(outputDatasetDto); } // DELETE /datasets/:id @@ -1608,7 +1766,7 @@ export class DatasetsController { const datablock = await this.origDatablocksService.create(createOrigDatablock); - const updateDatasetDto: PartialUpdateDatasetDto = { + const updateDatasetDto: PartialUpdateDatasetObsoleteDto = { size: dataset.size + datablock.size, numberOfFiles: dataset.numberOfFiles + datablock.dataFileList.length, }; @@ -1827,7 +1985,7 @@ export class DatasetsController { where: { datasetId: pid }, }); // update dataset size and files number - const updateDatasetDto: PartialUpdateDatasetDto = { + const updateDatasetDto: PartialUpdateDatasetObsoleteDto = { size: odb.reduce((a, b) => a + b.size, 0), numberOfFiles: odb.reduce((a, b) => a + b.dataFileList.length, 0), }; @@ -2057,7 +2215,7 @@ export class DatasetsController { datasetId: pid, }); // update dataset size and files number - const updateDatasetDto: PartialUpdateDatasetDto = { + const updateDatasetDto: PartialUpdateDatasetObsoleteDto = { packedSize: remainingDatablocks.reduce((a, b) => a + b.packedSize, 0), numberOfFilesArchived: remainingDatablocks.reduce( (a, b) => a + b.dataFileList.length, @@ -2110,7 +2268,7 @@ export class DatasetsController { Action.DatasetLogbookRead, ); - const proposalId = dataset?.proposalId; + const proposalId = (dataset?.proposalIds || [])[0]; if (!proposalId) return null; diff --git a/src/datasets/datasets.service.ts b/src/datasets/datasets.service.ts index b49de4269..ed977bbf0 100644 --- a/src/datasets/datasets.service.ts +++ b/src/datasets/datasets.service.ts @@ -23,22 +23,14 @@ import { import { ElasticSearchService } from "src/elastic-search/elastic-search.service"; import { InitialDatasetsService } from "src/initial-datasets/initial-datasets.service"; import { LogbooksService } from "src/logbooks/logbooks.service"; -import { DatasetType } from "./dataset-type.enum"; import { CreateDatasetDto } from "./dto/create-dataset.dto"; +import { IDatasetFields } from "./interfaces/dataset-filters.interface"; +import { DatasetClass, DatasetDocument } from "./schemas/dataset.schema"; import { PartialUpdateDatasetDto, + PartialUpdateDatasetWithHistoryDto, UpdateDatasetDto, } from "./dto/update-dataset.dto"; -import { - PartialUpdateDerivedDatasetDto, - UpdateDerivedDatasetDto, -} from "./dto/update-derived-dataset.dto"; -import { - PartialUpdateRawDatasetDto, - UpdateRawDatasetDto, -} from "./dto/update-raw-dataset.dto"; -import { IDatasetFields } from "./interfaces/dataset-filters.interface"; -import { DatasetClass, DatasetDocument } from "./schemas/dataset.schema"; @Injectable({ scope: Scope.REQUEST }) export class DatasetsService { @@ -205,10 +197,7 @@ export class DatasetsService { // we update the full dataset if exist or create a new one if it does not async findByIdAndReplace( id: string, - updateDatasetDto: - | UpdateDatasetDto - | UpdateRawDatasetDto - | UpdateDerivedDatasetDto, + updateDatasetDto: UpdateDatasetDto, ): Promise { const username = (this.request.user as JWTUser).username; const existingDataset = await this.datasetModel.findOne({ pid: id }).exec(); @@ -253,9 +242,7 @@ export class DatasetsService { id: string, updateDatasetDto: | PartialUpdateDatasetDto - | PartialUpdateRawDatasetDto - | PartialUpdateDerivedDatasetDto - | UpdateQuery, + | PartialUpdateDatasetWithHistoryDto, ): Promise { const existingDataset = await this.datasetModel.findOne({ pid: id }).exec(); // check if we were able to find the dataset @@ -434,20 +421,28 @@ export class DatasetsService { async updateHistory( req: Request, dataset: DatasetClass, - data: UpdateDatasetDto, + data: PartialUpdateDatasetDto, ) { if (req.body.history) { delete req.body.history; } if (!req.body.size && !req.body.packedSize) { - const updatedFields: Omit = - data; + const updatedFields: Omit< + PartialUpdateDatasetDto, + "updatedAt" | "updatedBy" + > = data; const historyItem: Record = {}; Object.keys(updatedFields).forEach((updatedField) => { historyItem[updatedField as keyof UpdateDatasetDto] = { currentValue: data[updatedField as keyof UpdateDatasetDto], - previousValue: dataset[updatedField as keyof UpdateDatasetDto], + previousValue: + dataset[ + updatedField as keyof Omit< + UpdateDatasetDto, + "attachments" | "origdatablocks" | "datablocks" + > + ], }; }); dataset.history = dataset.history ?? []; @@ -460,18 +455,15 @@ export class DatasetsService { if (logbookEnabled) { const user = (req.user as JWTUser).username.replace("ldap.", ""); const datasetPid = dataset.pid; - const proposalId = - dataset.type === DatasetType.Raw - ? (dataset as unknown as DatasetClass).proposalId - : undefined; - if (proposalId) { + const proposalIds = dataset.proposalIds || []; + (proposalIds as Array).forEach(async (proposalId) => { await Promise.all( Object.keys(updatedFields).map(async (updatedField) => { const message = `${user} updated "${updatedField}" of dataset with PID ${datasetPid}`; await this.logbooksService.sendMessage(proposalId, { message }); }), ); - } + }); } } } diff --git a/src/datasets/dto/create-dataset-obsolete.dto.ts b/src/datasets/dto/create-dataset-obsolete.dto.ts new file mode 100644 index 000000000..a75fe5dfe --- /dev/null +++ b/src/datasets/dto/create-dataset-obsolete.dto.ts @@ -0,0 +1,34 @@ +import { IsEnum, IsOptional, IsString } from "class-validator"; +import { ApiProperty } from "@nestjs/swagger"; +import { DatasetType } from "../dataset-type.enum"; +import { UpdateDatasetObsoleteDto } from "./update-dataset-obsolete.dto"; + +export class CreateDatasetObsoleteDto extends UpdateDatasetObsoleteDto { + @ApiProperty({ + type: String, + required: false, + description: "Persistent identifier of the dataset.", + }) + @IsOptional() + @IsString() + pid?: string; + + @ApiProperty({ + type: String, + required: true, + enum: [DatasetType.Raw, DatasetType.Derived], + description: + "Characterize type of dataset, either 'raw' or 'derived'. Autofilled when choosing the proper inherited models.", + }) + @IsEnum(DatasetType) + readonly type: string; + + @ApiProperty({ + type: String, + required: false, + description: "Version of the API used in creation of the dataset.", + }) + @IsOptional() + @IsString() + readonly version?: string; +} diff --git a/src/datasets/dto/create-dataset.dto.ts b/src/datasets/dto/create-dataset.dto.ts index c9b5b7ad5..8ee18d733 100644 --- a/src/datasets/dto/create-dataset.dto.ts +++ b/src/datasets/dto/create-dataset.dto.ts @@ -22,13 +22,4 @@ export class CreateDatasetDto extends UpdateDatasetDto { }) @IsEnum(DatasetType) readonly type: string; - - @ApiProperty({ - type: String, - required: false, - description: "Version of the API used in creation of the dataset.", - }) - @IsOptional() - @IsString() - readonly version?: string; } diff --git a/src/datasets/dto/create-derived-dataset.dto.ts b/src/datasets/dto/create-derived-dataset-obsolete.dto.ts similarity index 77% rename from src/datasets/dto/create-derived-dataset.dto.ts rename to src/datasets/dto/create-derived-dataset-obsolete.dto.ts index f95240c82..f6fcc180f 100644 --- a/src/datasets/dto/create-derived-dataset.dto.ts +++ b/src/datasets/dto/create-derived-dataset-obsolete.dto.ts @@ -1,9 +1,9 @@ -import { UpdateDerivedDatasetDto } from "./update-derived-dataset.dto"; +import { UpdateDerivedDatasetObsoleteDto } from "./update-derived-dataset-obsolete.dto"; import { ApiProperty } from "@nestjs/swagger"; import { IsEnum, IsOptional, IsString } from "class-validator"; import { DatasetType } from "../dataset-type.enum"; -export class CreateDerivedDatasetDto extends UpdateDerivedDatasetDto { +export class CreateDerivedDatasetObsoleteDto extends UpdateDerivedDatasetObsoleteDto { @ApiProperty({ type: String, required: false, diff --git a/src/datasets/dto/create-raw-dataset.dto.ts b/src/datasets/dto/create-raw-dataset-obsolete.dto.ts similarity index 78% rename from src/datasets/dto/create-raw-dataset.dto.ts rename to src/datasets/dto/create-raw-dataset-obsolete.dto.ts index fafa70b78..57f7d6ba9 100644 --- a/src/datasets/dto/create-raw-dataset.dto.ts +++ b/src/datasets/dto/create-raw-dataset-obsolete.dto.ts @@ -1,9 +1,9 @@ -import { UpdateRawDatasetDto } from "./update-raw-dataset.dto"; +import { UpdateRawDatasetObsoleteDto } from "./update-raw-dataset-obsolete.dto"; import { ApiProperty } from "@nestjs/swagger"; import { IsEnum, IsOptional, IsString } from "class-validator"; import { DatasetType } from "../dataset-type.enum"; -export class CreateRawDatasetDto extends UpdateRawDatasetDto { +export class CreateRawDatasetObsoleteDto extends UpdateRawDatasetObsoleteDto { @ApiProperty({ type: String, required: false, diff --git a/src/datasets/dto/output-dataset-obsolete.dto.ts b/src/datasets/dto/output-dataset-obsolete.dto.ts new file mode 100644 index 000000000..f99a7fe94 --- /dev/null +++ b/src/datasets/dto/output-dataset-obsolete.dto.ts @@ -0,0 +1,228 @@ +import { + IsArray, + IsDateString, + IsEnum, + IsObject, + IsOptional, + IsString, +} from "class-validator"; +import { ApiProperty, getSchemaPath } from "@nestjs/swagger"; +import { DatasetType } from "../dataset-type.enum"; +import { UpdateDatasetObsoleteDto } from "./update-dataset-obsolete.dto"; +import { Attachment } from "src/attachments/schemas/attachment.schema"; +import { Type } from "class-transformer"; +import { OrigDatablock } from "src/origdatablocks/schemas/origdatablock.schema"; +import { Datablock } from "src/datablocks/schemas/datablock.schema"; + +export class OutputDatasetObsoleteDto extends UpdateDatasetObsoleteDto { + @ApiProperty({ + type: String, + required: false, + description: "Persistent identifier of the dataset.", + }) + @IsString() + readonly pid: string; + + @ApiProperty({ + type: String, + required: true, + enum: [DatasetType.Raw, DatasetType.Derived], + description: + "Characterize type of dataset, either 'raw' or 'derived'. Autofilled when choosing the proper inherited models.", + }) + @IsEnum(DatasetType) + readonly type: string; + + @ApiProperty({ + type: String, + required: false, + description: "Version of the API used in creation of the dataset.", + }) + @IsOptional() + @IsString() + readonly version?: string; + + @ApiProperty({ + type: String, + required: true, + description: + "First name and last name of principal investigator(s). If multiple PIs are present, use a semicolon separated list. This field is required if the dataset is a Raw dataset.", + }) + @IsString() + @IsOptional() + readonly principalInvestigator?: string; + + @ApiProperty({ + type: Date, + required: false, + description: + "Start time of data acquisition for the current dataset.
It is expected to be in ISO8601 format according to specifications for internet date/time format in RFC 3339, chapter 5.6 (https://www.rfc-editor.org/rfc/rfc3339#section-5).
Local times without timezone/offset info are automatically transformed to UTC using the timezone of the API server.", + }) + @IsOptional() + @IsDateString() + readonly startTime?: Date; + + @ApiProperty({ + type: Date, + required: false, + description: + "End time of data acquisition for the current dataset.
It is expected to be in ISO8601 format according to specifications for internet date/time format in RFC 3339, chapter 5.6 (https://www.rfc-editor.org/rfc/rfc3339#section-5).
Local times without timezone/offset info are automatically transformed to UTC using the timezone of the API server.", + }) + @IsOptional() + @IsDateString() + readonly endTime?: Date; + + @ApiProperty({ + type: String, + required: true, + description: + "Unique location identifier where data was taken, usually in the form /Site-name/facility-name/instrumentOrBeamline-name. This field is required if the dataset is a Raw dataset.", + }) + @IsString() + @IsOptional() + readonly creationLocation?: string; + + @ApiProperty({ + type: String, + required: false, + description: + "Defines the format of the data files in this dataset, e.g Nexus Version x.y.", + }) + @IsOptional() + @IsString() + readonly dataFormat?: string; + + @ApiProperty({ + type: String, + required: false, + description: "ID of the sample used when collecting the data.", + }) + @IsOptional() + @IsString() + readonly sampleId?: string; + + @ApiProperty({ + type: String, + required: false, + description: "ID of the instrument where the data was created.", + }) + @IsOptional() + @IsString() + readonly instrumentId?: string; + + @ApiProperty({ + type: String, + required: true, + description: + "First name and last name of the person or people pursuing the data analysis. The string may contain a list of names, which should then be separated by semicolons.", + }) + @IsString() + @IsOptional() + readonly investigator?: string; + + @ApiProperty({ + type: [String], + required: true, + description: + "Array of input dataset identifiers used in producing the derived dataset. Ideally these are the global identifier to existing datasets inside this or federated data catalogs.", + }) + @IsString({ + each: true, + }) + @IsOptional() + readonly inputDatasets?: string[]; + + @ApiProperty({ + type: [String], + required: true, + description: + "A list of links to software repositories which uniquely identifies the pieces of software, including versions, used for yielding the derived data.", + }) + @IsString({ + each: true, + }) + @IsOptional() + readonly usedSoftware?: string[]; + + @ApiProperty({ + type: Object, + required: false, + description: + "The creation process of the derived data will usually depend on input job parameters. The full structure of these input parameters are stored here.", + }) + @IsOptional() + @IsObject() + readonly jobParameters?: Record; + + @ApiProperty({ + type: String, + required: false, + description: + "The output job logfile. Keep the size of this log data well below 15 MB.", + }) + @IsOptional() + @IsString() + readonly jobLogData?: string; + + @ApiProperty({ + type: "array", + items: { $ref: getSchemaPath(Attachment) }, + required: false, + description: + "Small, less than 16 MB attachments, envisaged for png/jpeg previews.", + }) + @IsOptional() + @IsArray() + @Type(() => Attachment) + attachments?: Attachment[]; + + @ApiProperty({ + type: "array", + items: { $ref: getSchemaPath(OrigDatablock) }, + required: false, + description: + "Containers that list all files and their attributes which make up a dataset. Usually filled at the time the dataset's metadata is created in the data catalog. Can be used by subsequent archiving processes to create the archived datasets.", + }) + origdatablocks?: OrigDatablock[]; + + @ApiProperty({ + type: "array", + items: { $ref: getSchemaPath(Datablock) }, + required: false, + description: + "When archiving a dataset, all files contained in the dataset are listed here together with their checksum information. Several datablocks can be created if the file listing is too long for a single datablock. This partitioning decision is done by the archiving system to allow for chunks of datablocks with manageable sizes. E.g a datasets consisting of 10 TB of data could be split into 10 datablocks of about 1 TB each. The upper limit set by the data catalog system itself is given by the fact that documents must be smaller than 16 MB, which typically allows for datasets of about 100000 files.", + }) + datablocks?: Datablock[]; + + @ApiProperty({ + type: String, + required: true, + description: + "Indicate the user who created this record. This property is added and maintained by the system.", + }) + createdBy: string; + + @ApiProperty({ + type: String, + required: true, + description: + "Indicate the user who updated this record last. This property is added and maintained by the system.", + }) + updatedBy: string; + + @ApiProperty({ + type: Date, + required: true, + description: + "Date and time when this record was created. This field is managed by mongoose with through the timestamp settings. The field should be a string containing a date in ISO 8601 format (2024-02-27T12:26:57.313Z)", + }) + createdAt: Date; + + @ApiProperty({ + type: Date, + required: true, + description: + "Date and time when this record was updated last. This field is managed by mongoose with through the timestamp settings. The field should be a string containing a date in ISO 8601 format (2024-02-27T12:26:57.313Z)", + }) + updatedAt: Date; +} diff --git a/src/datasets/dto/output-dataset.dto.ts b/src/datasets/dto/output-dataset.dto.ts new file mode 100644 index 000000000..6c070a73c --- /dev/null +++ b/src/datasets/dto/output-dataset.dto.ts @@ -0,0 +1,41 @@ +import { ApiProperty } from "@nestjs/swagger"; +import { CreateDatasetDto } from "./create-dataset.dto"; +import { IsDateString, IsString } from "class-validator"; + +export class OutputDatasetDto extends CreateDatasetDto { + @ApiProperty({ + type: String, + required: true, + description: + "Indicate the user who created this record. This property is added and maintained by the system.", + }) + @IsString() + createdBy: string; + + @ApiProperty({ + type: String, + required: true, + description: + "Indicate the user who updated this record last. This property is added and maintained by the system.", + }) + @IsString() + updatedBy: string; + + @ApiProperty({ + type: Date, + required: true, + description: + "Date and time when this record was created. This field is managed by mongoose with through the timestamp settings. The field should be a string containing a date in ISO 8601 format (2024-02-27T12:26:57.313Z)", + }) + @IsDateString() + createdAt: Date; + + @ApiProperty({ + type: Date, + required: true, + description: + "Date and time when this record was updated last. This field is managed by mongoose with through the timestamp settings. The field should be a string containing a date in ISO 8601 format (2024-02-27T12:26:57.313Z)", + }) + @IsDateString() + updatedAt: Date; +} diff --git a/src/datasets/dto/update-dataset-obsolete.dto.ts b/src/datasets/dto/update-dataset-obsolete.dto.ts new file mode 100644 index 000000000..73f4b34bd --- /dev/null +++ b/src/datasets/dto/update-dataset-obsolete.dto.ts @@ -0,0 +1,304 @@ +import { + ApiProperty, + ApiTags, + getSchemaPath, + PartialType, +} from "@nestjs/swagger"; +import { OwnableDto } from "../../common/dto/ownable.dto"; +import { + IsArray, + IsBoolean, + IsDateString, + IsEmail, + IsFQDN, + IsInt, + IsNumber, + IsObject, + IsOptional, + IsString, + ValidateNested, +} from "class-validator"; +import { TechniqueClass } from "../schemas/technique.schema"; +import { Type } from "class-transformer"; +import { CreateTechniqueDto } from "./create-technique.dto"; +import { RelationshipClass } from "../schemas/relationship.schema"; +import { CreateRelationshipDto } from "./create-relationship.dto"; +import { LifecycleClass } from "../schemas/lifecycle.schema"; + +@ApiTags("datasets") +export class UpdateDatasetObsoleteDto extends OwnableDto { + @ApiProperty({ + type: String, + required: true, + description: + "Owner or custodian of the dataset, usually first name + last name. The string may contain a list of persons, which should then be separated by semicolons.", + }) + @IsString() + readonly owner: string; + + @ApiProperty({ + type: String, + required: false, + description: + "Email of the owner or custodian of the dataset. The string may contain a list of emails, which should then be separated by semicolons.", + }) + @IsOptional() + @IsEmail() + readonly ownerEmail?: string; + + @ApiProperty({ + type: String, + required: false, + description: + "ORCID of the owner or custodian. The string may contain a list of ORCIDs, which should then be separated by semicolons.", + }) + @IsOptional() + @IsString() + readonly orcidOfOwner?: string; + + @ApiProperty({ + type: String, + required: true, + description: + "Email of the contact person for this dataset. The string may contain a list of emails, which should then be separated by semicolons.", + }) + @IsEmail() + readonly contactEmail: string; + + @ApiProperty({ + type: String, + required: true, + description: + "Absolute file path on file server containing the files of this dataset, e.g. /some/path/to/sourcefolder. In case of a single file dataset, e.g. HDF5 data, it contains the path up to, but excluding the filename. Trailing slashes are removed.", + }) + @IsString() + readonly sourceFolder: string; + + @ApiProperty({ + type: String, + required: false, + description: + "DNS host name of file server hosting sourceFolder, optionally including a protocol e.g. [protocol://]fileserver1.example.com", + }) + @IsOptional() + @IsFQDN() + readonly sourceFolderHost?: string; + + /* + * size and number of files fields should be managed by the system + */ + @ApiProperty({ + type: Number, + default: 0, + required: false, + description: + "Total size of all source files contained in source folder on disk when unpacked.", + }) + @IsOptional() + @IsInt() + readonly size?: number = 0; + + @ApiProperty({ + type: Number, + default: 0, + required: false, + description: + "Total size of all datablock package files created for this dataset.", + }) + @IsOptional() + @IsInt() + readonly packedSize?: number = 0; + + @ApiProperty({ + type: Number, + default: 0, + required: false, + description: + "Total number of files in all OrigDatablocks for this dataset.", + }) + @IsOptional() + @IsInt() + readonly numberOfFiles?: number = 0; + + @ApiProperty({ + type: Number, + default: 0, + required: true, + description: "Total number of files in all Datablocks for this dataset.", + }) + @IsOptional() + @IsInt() + readonly numberOfFilesArchived?: number; + + @ApiProperty({ + type: Date, + required: true, + description: + "Time when dataset became fully available on disk, i.e. all containing files have been written, or the dataset was created in SciCat.
It is expected to be in ISO8601 format according to specifications for internet date/time format in RFC 3339, chapter 5.6 (https://www.rfc-editor.org/rfc/rfc3339#section-5).
Local times without timezone/offset info are automatically transformed to UTC using the timezone of the API server.", + }) + @IsDateString() + readonly creationTime: Date; + + @ApiProperty({ + type: String, + required: false, + description: + "Defines a level of trust, e.g. a measure of how much data was verified or used by other persons.", + }) + @IsOptional() + @IsString() + readonly validationStatus?: string; + + @ApiProperty({ + type: [String], + required: false, + description: + "Array of tags associated with the meaning or contents of this dataset. Values should ideally come from defined vocabularies, taxonomies, ontologies or knowledge graphs.", + }) + @IsOptional() + @IsString({ + each: true, + }) + readonly keywords?: string[]; + + @ApiProperty({ + type: String, + required: false, + description: "Free text explanation of contents of dataset.", + }) + @IsOptional() + @IsString() + readonly description?: string; + + @ApiProperty({ + type: String, + required: false, + description: + "A name for the dataset, given by the creator to carry some semantic meaning. Useful for display purposes e.g. instead of displaying the pid. Will be autofilled if missing using info from sourceFolder.", + }) + @IsOptional() + @IsString() + readonly datasetName?: string; + + @ApiProperty({ + type: String, + required: false, + description: + "ACIA information about AUthenticity,COnfidentiality,INtegrity and AVailability requirements of dataset. E.g. AV(ailabilty)=medium could trigger the creation of a two tape copies. Format 'AV=medium,CO=low'", + }) + @IsOptional() + @IsString() + readonly classification?: string; + + @ApiProperty({ + type: String, + required: false, + description: "Name of the license under which the data can be used.", + }) + @IsOptional() + @IsString() + readonly license?: string; + + // it needs to be discussed if this fields is managed by the user or by the system + @ApiProperty({ + type: Boolean, + required: false, + default: false, + description: "Flag is true when data are made publicly available.", + }) + @IsOptional() + @IsBoolean() + readonly isPublished?: boolean; + + @ApiProperty({ + type: "array", + items: { $ref: getSchemaPath(TechniqueClass) }, + required: false, + default: [], + description: "Stores the metadata information for techniques.", + }) + @IsOptional() + @IsArray() + @ValidateNested({ each: true }) + @Type(() => CreateTechniqueDto) + readonly techniques?: TechniqueClass[]; + + // it needs to be discussed if this fields is managed by the user or by the system + @ApiProperty({ + type: [String], + required: false, + default: [], + description: "List of users that the dataset has been shared with.", + }) + @IsOptional() + @IsString({ + each: true, + }) + readonly sharedWith?: string[]; + + // it needs to be discussed if this fields is managed by the user or by the system + @ApiProperty({ + type: "array", + items: { $ref: getSchemaPath(RelationshipClass) }, + required: false, + default: [], + description: "Stores the relationships with other datasets.", + }) + @IsArray() + @IsOptional() + @ValidateNested({ each: true }) + @Type(() => CreateRelationshipDto) + readonly relationships?: RelationshipClass[]; + + @ApiProperty({ + type: LifecycleClass, + required: false, + default: {}, + description: + "Describes the current status of the dataset during its lifetime with respect to the storage handling systems.", + }) + @IsOptional() + readonly datasetlifecycle: LifecycleClass; + + @ApiProperty({ + type: Object, + required: false, + default: {}, + description: "JSON object containing the scientific metadata.", + }) + @IsOptional() + @IsObject() + readonly scientificMetadata?: Record; + + @ApiProperty({ + type: String, + required: false, + description: "Comment the user has about a given dataset.", + }) + @IsOptional() + @IsString() + readonly comment?: string; + + @ApiProperty({ + type: Number, + required: false, + description: + "Data Quality Metrics is a number given by the user to rate the dataset.", + }) + @IsOptional() + @IsNumber() + readonly dataQualityMetrics?: number; + + @ApiProperty({ + type: String, + required: false, + description: "The ID of the proposal to which the dataset belongs.", + }) + @IsOptional() + @IsString() + readonly proposalId?: string; +} + +export class PartialUpdateDatasetObsoleteDto extends PartialType( + UpdateDatasetObsoleteDto, +) {} diff --git a/src/datasets/dto/update-dataset.dto.ts b/src/datasets/dto/update-dataset.dto.ts index aab14a7e5..99c6c3eaf 100644 --- a/src/datasets/dto/update-dataset.dto.ts +++ b/src/datasets/dto/update-dataset.dto.ts @@ -24,9 +24,7 @@ import { CreateTechniqueDto } from "./create-technique.dto"; import { RelationshipClass } from "../schemas/relationship.schema"; import { CreateRelationshipDto } from "./create-relationship.dto"; import { LifecycleClass } from "../schemas/lifecycle.schema"; -import { Attachment } from "../../attachments/schemas/attachment.schema"; -import { OrigDatablock } from "../../origdatablocks/schemas/origdatablock.schema"; -import { Datablock } from "../../datablocks/schemas/datablock.schema"; +import { HistoryClass } from "../schemas/history.schema"; @ApiTags("datasets") export class UpdateDatasetDto extends OwnableDto { @@ -37,7 +35,8 @@ export class UpdateDatasetDto extends OwnableDto { "Owner or custodian of the dataset, usually first name + last name. The string may contain a list of persons, which should then be separated by semicolons.", }) @IsString() - readonly owner: string; + @IsOptional() + readonly owner?: string; @ApiProperty({ type: String, @@ -175,13 +174,12 @@ export class UpdateDatasetDto extends OwnableDto { @ApiProperty({ type: String, - required: false, + required: true, description: "A name for the dataset, given by the creator to carry some semantic meaning. Useful for display purposes e.g. instead of displaying the pid. Will be autofilled if missing using info from sourceFolder.", }) - @IsOptional() @IsString() - readonly datasetName?: string; + readonly datasetName: string; @ApiProperty({ type: String, @@ -261,7 +259,7 @@ export class UpdateDatasetDto extends OwnableDto { "Describes the current status of the dataset during its lifetime with respect to the storage handling systems.", }) @IsOptional() - readonly datasetlifecycle: LifecycleClass; + readonly datasetlifecycle?: LifecycleClass; @ApiProperty({ type: Object, @@ -292,14 +290,153 @@ export class UpdateDatasetDto extends OwnableDto { @IsNumber() readonly dataQualityMetrics?: number; + @ApiProperty({ + type: String, + required: true, + description: + "First name and last name of principal investigator(s). If multiple PIs are present, use a semicolon separated list. This field is required if the dataset is a Raw dataset.", + }) + @IsString() + readonly principalInvestigator: string; + + @ApiProperty({ + type: Date, + required: false, + description: + "Start time of data acquisition for the current dataset.
It is expected to be in ISO8601 format according to specifications for internet date/time format in RFC 3339, chapter 5.6 (https://www.rfc-editor.org/rfc/rfc3339#section-5).
Local times without timezone/offset info are automatically transformed to UTC using the timezone of the API server.", + }) + @IsOptional() + @IsDateString() + readonly startTime?: Date; + + @ApiProperty({ + type: Date, + required: false, + description: + "End time of data acquisition for the current dataset.
It is expected to be in ISO8601 format according to specifications for internet date/time format in RFC 3339, chapter 5.6 (https://www.rfc-editor.org/rfc/rfc3339#section-5).
Local times without timezone/offset info are automatically transformed to UTC using the timezone of the API server.", + }) + @IsOptional() + @IsDateString() + readonly endTime?: Date; + + @ApiProperty({ + type: String, + required: false, + description: + "Unique location identifier where data was taken, usually in the form /Site-name/facility-name/instrumentOrBeamline-name. This field is required if the dataset is a Raw dataset.", + }) + @IsOptional() + @IsString() + readonly creationLocation?: string; + + @ApiProperty({ + type: String, + required: false, + description: + "Defines the format of the data files in this dataset, e.g Nexus Version x.y.", + }) + @IsOptional() + @IsString() + readonly dataFormat?: string; + + @ApiProperty({ + type: [String], + required: false, + description: + "ID of the proposal or proposals which the dataset belongs to.
This dataset might have been acquired under the listed proposals or is derived from datasets acquired from datasets belonging to the listed datasets.", + }) + @IsOptional() + @IsString({ + each: true, + }) + readonly proposalIds?: string[]; + + @ApiProperty({ + type: [String], + required: false, + description: + "ID of the sample or samples used when collecting the data included or used in this dataset.", + }) + @IsOptional() + @IsString({ + each: true, + }) + readonly sampleIds?: string[]; + + @ApiProperty({ + type: String, + required: false, + description: + "ID of the instrument or instruments where the data included or used in this datasets was collected on.", + }) @IsOptional() - attachments?: Attachment[]; + @IsString({ + each: true, + }) + readonly instrumentIds?: string[]; + @ApiProperty({ + type: [String], + required: true, + description: + "Array of input dataset identifiers used in producing the derived dataset. Ideally these are the global identifier to existing datasets inside this or federated data catalogs.", + }) @IsOptional() - origdatablocks?: OrigDatablock[]; + @IsString({ + each: true, + }) + readonly inputDatasets?: string[]; + @ApiProperty({ + type: [String], + required: false, + description: + "A list of links to software repositories which uniquely identifies the pieces of software, including versions, used for yielding the derived data.", + }) @IsOptional() - datablocks?: Datablock[]; + @IsString({ + each: true, + }) + readonly usedSoftware?: string[]; + + @ApiProperty({ + type: Object, + required: false, + description: + "The creation process of the derived data will usually depend on input job parameters. The full structure of these input parameters are stored here.", + }) + @IsOptional() + @IsObject() + readonly jobParameters?: Record; + + @ApiProperty({ + type: String, + required: false, + description: + "The output job logfile. Keep the size of this log data well below 15 MB.", + }) + @IsOptional() + @IsString() + readonly jobLogData?: string; } export class PartialUpdateDatasetDto extends PartialType(UpdateDatasetDto) {} + +export class UpdateDatasetWithHistoryDto extends UpdateDatasetDto { + @ApiProperty({ + type: "array", + items: { $ref: getSchemaPath(HistoryClass) }, + required: false, + default: [], + description: "List of history objects containing old and new values.", + }) + @IsArray() + @IsOptional() + @ValidateNested({ each: true }) + @Type(() => HistoryClass) + readonly history?: HistoryClass[]; +} + +export class PartialUpdateDatasetWithHistoryDto extends PartialType( + UpdateDatasetWithHistoryDto, +) {} diff --git a/src/datasets/dto/update-derived-dataset.dto.ts b/src/datasets/dto/update-derived-dataset-obsolete.dto.ts similarity index 86% rename from src/datasets/dto/update-derived-dataset.dto.ts rename to src/datasets/dto/update-derived-dataset-obsolete.dto.ts index 5f4d83a1d..b6ec0bf4c 100644 --- a/src/datasets/dto/update-derived-dataset.dto.ts +++ b/src/datasets/dto/update-derived-dataset-obsolete.dto.ts @@ -1,8 +1,8 @@ -import { UpdateDatasetDto } from "./update-dataset.dto"; +import { UpdateDatasetObsoleteDto } from "./update-dataset-obsolete.dto"; import { ApiProperty, PartialType } from "@nestjs/swagger"; import { IsObject, IsOptional, IsString } from "class-validator"; -export class UpdateDerivedDatasetDto extends UpdateDatasetDto { +export class UpdateDerivedDatasetObsoleteDto extends UpdateDatasetObsoleteDto { @ApiProperty({ type: String, required: true, @@ -55,6 +55,6 @@ export class UpdateDerivedDatasetDto extends UpdateDatasetDto { readonly jobLogData?: string; } -export class PartialUpdateDerivedDatasetDto extends PartialType( - UpdateDerivedDatasetDto, +export class PartialUpdateDerivedDatasetObsoleteDto extends PartialType( + UpdateDerivedDatasetObsoleteDto, ) {} diff --git a/src/datasets/dto/update-raw-dataset.dto.ts b/src/datasets/dto/update-raw-dataset-obsolete.dto.ts similarity index 83% rename from src/datasets/dto/update-raw-dataset.dto.ts rename to src/datasets/dto/update-raw-dataset-obsolete.dto.ts index a03fa14ad..075406908 100644 --- a/src/datasets/dto/update-raw-dataset.dto.ts +++ b/src/datasets/dto/update-raw-dataset-obsolete.dto.ts @@ -1,8 +1,8 @@ import { IsDateString, IsOptional, IsString } from "class-validator"; -import { UpdateDatasetDto } from "./update-dataset.dto"; +import { UpdateDatasetObsoleteDto } from "./update-dataset-obsolete.dto"; import { ApiProperty, PartialType } from "@nestjs/swagger"; -export class UpdateRawDatasetDto extends UpdateDatasetDto { +export class UpdateRawDatasetObsoleteDto extends UpdateDatasetObsoleteDto { /* we need to discuss if the naming is adequate. */ @ApiProperty({ type: String, @@ -52,14 +52,14 @@ export class UpdateRawDatasetDto extends UpdateDatasetDto { @IsString() readonly dataFormat?: string; - @ApiProperty({ - type: String, - required: false, - description: "The ID of the proposal to which the dataset belongs.", - }) - @IsOptional() - @IsString() - readonly proposalId?: string; + // @ApiProperty({ + // type: String, + // required: false, + // description: "The ID of the proposal to which the dataset belongs.", + // }) + // @IsOptional() + // @IsString() + // readonly proposalId?: string; @ApiProperty({ type: String, @@ -77,7 +77,7 @@ export class UpdateRawDatasetDto extends UpdateDatasetDto { }) @IsOptional() @IsString() - readonly instrumentId: string; + readonly instrumentId?: string; @IsOptional() investigator?: string; @@ -95,6 +95,6 @@ export class UpdateRawDatasetDto extends UpdateDatasetDto { jobLogData?: string; } -export class PartialUpdateRawDatasetDto extends PartialType( - UpdateRawDatasetDto, +export class PartialUpdateRawDatasetObsoleteDto extends PartialType( + UpdateRawDatasetObsoleteDto, ) {} diff --git a/src/datasets/schemas/dataset.schema.ts b/src/datasets/schemas/dataset.schema.ts index 74c29aaa7..a3fb81086 100644 --- a/src/datasets/schemas/dataset.schema.ts +++ b/src/datasets/schemas/dataset.schema.ts @@ -1,19 +1,7 @@ import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; import { ApiProperty, getSchemaPath } from "@nestjs/swagger"; import { Document } from "mongoose"; -import { - Attachment, - AttachmentSchema, -} from "src/attachments/schemas/attachment.schema"; import { OwnableClass } from "src/common/schemas/ownable.schema"; -import { - Datablock, - DatablockSchema, -} from "src/datablocks/schemas/datablock.schema"; -import { - OrigDatablock, - OrigDatablockSchema, -} from "src/origdatablocks/schemas/origdatablock.schema"; import { v4 as uuidv4 } from "uuid"; import { DatasetType } from "../dataset-type.enum"; import { HistoryClass, HistorySchema } from "./history.schema"; @@ -244,28 +232,21 @@ export class DatasetClass extends OwnableClass { @ApiProperty({ type: String, - required: false, + required: true, description: - "A name for the dataset, given by the creator to carry some semantic meaning. Useful for display purposes e.g. instead of displaying the pid. Will be autofilled if missing using info from sourceFolder.", + "A name for the dataset, given by the creator to carry some semantic meaning. Useful for display purposes e.g. instead of displaying the pid.", }) @Prop({ type: String, - required: false, - default: function datasetName() { - const sourceFolder = (this as DatasetDocument).sourceFolder; - if (!sourceFolder) return ""; - const arr = sourceFolder.split("/"); - if (arr.length == 1) return arr[0]; - else return arr[arr.length - 2] + "/" + arr[arr.length - 1]; - }, + required: true, }) - datasetName?: string; + datasetName: string; @ApiProperty({ type: String, required: false, description: - "ACIA information about AUthenticity,COnfidentiality,INtegrity and AVailability requirements of dataset. E.g. AV(ailabilty)=medium could trigger the creation of a two tape copies. Format 'AV=medium,CO=low'", + "ACIA information about AUthenticity,COnfidentiality,INtegrity and AVailability requirements of dataset. E.g. AV(ailabilty)=medium could trigger the creation of a two tape copies. Format 'AV=medium,CO=low'. Please check the following post for more info: https://en.wikipedia.org/wiki/Parkerian_Hexad", }) @Prop({ type: String, required: false }) classification?: string; @@ -280,11 +261,12 @@ export class DatasetClass extends OwnableClass { @ApiProperty({ type: String, - required: false, - description: "Version of the API used in creation of the dataset.", + required: true, + description: + "Version of the API used when the dataset was created or last updated. API version is defined in code for each release. Managed by the system.", }) - @Prop({ type: String, required: false }) - version?: string; + @Prop({ type: String, required: true }) + version: string; @ApiProperty({ type: "array", @@ -298,12 +280,12 @@ export class DatasetClass extends OwnableClass { @ApiProperty({ type: LifecycleClass, - required: false, + required: true, default: {}, description: "Describes the current status of the dataset during its lifetime with respect to the storage handling systems.", }) - @Prop({ type: LifecycleSchema, default: {}, required: false }) + @Prop({ type: LifecycleSchema, default: {}, required: true }) datasetlifecycle?: LifecycleClass; @ApiProperty({ @@ -311,7 +293,8 @@ export class DatasetClass extends OwnableClass { items: { $ref: getSchemaPath(TechniqueClass) }, required: false, default: [], - description: "Stores the metadata information for techniques.", + description: + "Array of techniques information, with technique name and pid.", }) @Prop({ type: [TechniqueSchema], required: false, default: [] }) techniques?: TechniqueClass[]; @@ -321,7 +304,8 @@ export class DatasetClass extends OwnableClass { items: { $ref: getSchemaPath(RelationshipClass) }, required: false, default: [], - description: "Stores the relationships with other datasets.", + description: + "Array of relationships with other datasets. It contains relationship type and destination dataset", }) @Prop({ type: [RelationshipSchema], required: false, default: [] }) relationships?: RelationshipClass[]; @@ -330,7 +314,8 @@ export class DatasetClass extends OwnableClass { type: [String], required: false, default: [], - description: "List of users that the dataset has been shared with.", + description: + "List of additional users that the dataset has been shared with.", }) @Prop({ type: [String], @@ -339,37 +324,37 @@ export class DatasetClass extends OwnableClass { }) sharedWith?: string[]; - @ApiProperty({ - type: "array", - items: { $ref: getSchemaPath(Attachment) }, - required: false, - description: - "Small, less than 16 MB attachments, envisaged for png/jpeg previews.", - }) - @Prop({ type: [AttachmentSchema], default: [] }) - attachments?: Attachment[]; - - @ApiProperty({ - isArray: true, - type: OrigDatablock, - items: { $ref: getSchemaPath(OrigDatablock) }, - required: false, - description: - "Containers that list all files and their attributes which make up a dataset. Usually filled at the time the dataset's metadata is created in the data catalog. Can be used by subsequent archiving processes to create the archived datasets.", - }) - @Prop({ type: [OrigDatablockSchema], default: [] }) - origdatablocks: OrigDatablock[]; - - @ApiProperty({ - isArray: true, - type: Datablock, - items: { $ref: getSchemaPath(Datablock) }, - required: false, - description: - "When archiving a dataset, all files contained in the dataset are listed here together with their checksum information. Several datablocks can be created if the file listing is too long for a single datablock. This partitioning decision is done by the archiving system to allow for chunks of datablocks with manageable sizes. E.g a datasets consisting of 10 TB of data could be split into 10 datablocks of about 1 TB each. The upper limit set by the data catalog system itself is given by the fact that documents must be smaller than 16 MB, which typically allows for datasets of about 100000 files.", - }) - @Prop({ type: [DatablockSchema], default: [] }) - datablocks: Datablock[]; + // @ApiProperty({ + // type: "array", + // items: { $ref: getSchemaPath(Attachment) }, + // required: false, + // description: + // "Small, less than 16 MB attachments, envisaged for png/jpeg previews.", + // }) + // @Prop({ type: [AttachmentSchema], default: [] }) + // attachments?: Attachment[]; + + // @ApiProperty({ + // isArray: true, + // type: OrigDatablock, + // items: { $ref: getSchemaPath(OrigDatablock) }, + // required: false, + // description: + // "Containers that list all files and their attributes which make up a dataset. Usually filled at the time the dataset's metadata is created in the data catalog. Can be used by subsequent archiving processes to create the archived datasets.", + // }) + // @Prop({ type: [OrigDatablockSchema], default: [] }) + // origdatablocks: OrigDatablock[]; + + // @ApiProperty({ + // isArray: true, + // type: Datablock, + // items: { $ref: getSchemaPath(Datablock) }, + // required: false, + // description: + // "When archiving a dataset, all files contained in the dataset are listed here together with their checksum information. Several datablocks can be created if the file listing is too long for a single datablock. This partitioning decision is done by the archiving system to allow for chunks of datablocks with manageable sizes. E.g a datasets consisting of 10 TB of data could be split into 10 datablocks of about 1 TB each. The upper limit set by the data catalog system itself is given by the fact that documents must be smaller than 16 MB, which typically allows for datasets of about 100000 files.", + // }) + // @Prop({ type: [DatablockSchema], default: [] }) + // datablocks: Datablock[]; @ApiProperty({ type: Object, @@ -383,7 +368,8 @@ export class DatasetClass extends OwnableClass { @ApiProperty({ type: String, required: false, - description: "Comment the user has about a given dataset.", + description: + "Short comment provided by the user about a given dataset. This is additional to the description field.", }) @Prop({ type: String, @@ -400,7 +386,7 @@ export class DatasetClass extends OwnableClass { type: Number, required: false, }) - dataQualityMetrics: number; + dataQualityMetrics?: number; /* * fields related to Raw Datasets @@ -436,7 +422,7 @@ export class DatasetClass extends OwnableClass { type: String, required: true, description: - "Unique location identifier where data was taken, usually in the form /Site-name/facility-name/instrumentOrBeamline-name. This field is required if the dataset is a Raw dataset.", + "Unique location identifier where data was acquired. Usually in the form /Site-name/facility-name/instrumentOrBeamline-name.", }) @Prop({ type: String, required: false, index: true }) creationLocation?: string; @@ -451,46 +437,49 @@ export class DatasetClass extends OwnableClass { dataFormat?: string; @ApiProperty({ - type: String, + type: [String], required: false, - description: "The ID of the proposal to which the dataset belongs.", + description: + "The ID of the proposal to which the dataset belongs to and it has been acquired under.", }) - @Prop({ type: String, ref: "Proposal", required: false }) - proposalId?: string; + @Prop({ type: [String], ref: "Proposal", required: false }) + proposalIds?: string[]; @ApiProperty({ - type: String, + type: [String], required: false, - description: "ID of the sample used when collecting the data.", + description: + "Single ID or array of IDS of the samples used when collecting the data.", }) - @Prop({ type: String, ref: "Sample", required: false }) - sampleId?: string; + @Prop({ type: [String], ref: "Sample", required: false }) + sampleIds?: string[]; @ApiProperty({ - type: String, + type: [String], required: false, - description: "ID of the instrument where the data was created.", + description: + "Id of the instrument or array of IDS of the instruments where the data contained in this dataset was created/acquired.", }) - @Prop({ type: String, ref: "Instrument", required: false }) - instrumentId?: string; + @Prop({ type: [String], ref: "Instrument", required: false }) + instrumentIds?: string[]; /* * Derived Dataset */ - @ApiProperty({ - type: String, - required: false, - description: - "First name and last name of the person or people pursuing the data analysis. The string may contain a list of names, which should then be separated by semicolons.", - }) - @Prop({ type: String, required: false, index: true }) - investigator?: string; + // @ApiProperty({ + // type: String, + // required: false, + // description: + // "First name and last name of the person or people pursuing the data analysis. The string may contain a list of names, which should then be separated by semicolons.", + // }) + // @Prop({ type: String, required: false, index: true }) + // investigator?: string; @ApiProperty({ type: [String], required: false, description: - "Array of input dataset identifiers used in producing the derived dataset. Ideally these are the global identifier to existing datasets inside this or federated data catalogs. This field is required if the dataset is a Derived dataset.", + "Array of input dataset identifiers used in producing the derived dataset. Ideally these are the global identifier to existing datasets inside this or federated data catalogs.", }) @Prop({ type: [String], required: false }) inputDatasets?: string[]; @@ -499,7 +488,7 @@ export class DatasetClass extends OwnableClass { type: [String], required: false, description: - "A list of links to software repositories which uniquely identifies the pieces of software, including versions, used for yielding the derived data. This field is required if the dataset is a Derived dataset.", + "A list of links to software repositories which uniquely identifies the pieces of software, including versions, used for yielding the derived data.", }) @Prop({ type: [String], required: false }) usedSoftware?: string[]; diff --git a/src/elastic-search/configuration/datasetFieldMapping.ts b/src/elastic-search/configuration/datasetFieldMapping.ts index 429555ee8..629d5f80f 100644 --- a/src/elastic-search/configuration/datasetFieldMapping.ts +++ b/src/elastic-search/configuration/datasetFieldMapping.ts @@ -42,11 +42,15 @@ export const datasetMappings: MappingObject = { type: "nested", dynamic: false, }, - proposalId: { + proposalIds: { type: "keyword", ignore_above: 256, }, - sampleId: { + sampleIds: { + type: "keyword", + ignore_above: 256, + }, + instrumentIds: { type: "keyword", ignore_above: 256, }, diff --git a/src/jobs/jobs.controller.ts b/src/jobs/jobs.controller.ts index 588f202f9..838fc724c 100644 --- a/src/jobs/jobs.controller.ts +++ b/src/jobs/jobs.controller.ts @@ -179,25 +179,26 @@ export class JobsController { // Indexing originDataBlock with pid and create set of files for each dataset const datasets = await this.datasetsService.findAll(filter); // Include origdatablocks - await Promise.all( + const aggregatedData = await Promise.all( datasets.map(async (dataset) => { - dataset.origdatablocks = await this.origDatablocksService.findAll( - { + return { + dataset: dataset, + origdatablocks: await this.origDatablocksService.findAll({ datasetId: dataset.pid, - }, - ); + }), + }; }), ); - const result: Record> = datasets.reduce( - (acc: Record>, dataset) => { + const result: Record> = aggregatedData.reduce( + (acc: Record>, data) => { // Using Set make searching more efficient - const files = dataset.origdatablocks.reduce((acc, block) => { + const files = data.origdatablocks.reduce((acc, block) => { block.dataFileList.forEach((file) => { acc.add(file.path); }); return acc; }, new Set()); - acc[dataset.pid] = files; + acc[data.dataset.pid] = files; return acc; }, {}, diff --git a/src/main.ts b/src/main.ts index 980dc66a8..3af41b802 100644 --- a/src/main.ts +++ b/src/main.ts @@ -7,7 +7,7 @@ import { SwaggerModule, } from "@nestjs/swagger"; import { AppModule } from "./app.module"; -import { Logger, ValidationPipe } from "@nestjs/common"; +import { Logger, ValidationPipe, VersioningType } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; import { AllExceptionsFilter, ScicatLogger } from "./loggers/logger.service"; @@ -29,11 +29,20 @@ async function bootstrap() { app.enableCors(); - app.setGlobalPrefix(`api/${apiVersion}`); + app.setGlobalPrefix("api"); + + // NOTE: This is a workaround to enable versioning for individual routes + // Version decorator can be used to specify the version for a route + // Read more on https://docs.nestjs.com/techniques/versioning + app.enableVersioning({ + type: VersioningType.URI, + defaultVersion: apiVersion, + }); + const config = new DocumentBuilder() .setTitle("SciCat backend API") .setDescription("This is the API for the SciCat Backend") - .setVersion(`api/${apiVersion}`) + .setVersion(`api/v${apiVersion}`) .addBearerAuth() .build(); diff --git a/src/origdatablocks/origdatablocks.controller.ts b/src/origdatablocks/origdatablocks.controller.ts index f1a0f13be..97e21d50b 100644 --- a/src/origdatablocks/origdatablocks.controller.ts +++ b/src/origdatablocks/origdatablocks.controller.ts @@ -45,8 +45,8 @@ import { PartialUpdateDatasetDto } from "src/datasets/dto/update-dataset.dto"; import { filterDescription, filterExample } from "src/common/utils"; import { JWTUser } from "src/auth/interfaces/jwt-user.interface"; import { DatasetClass } from "src/datasets/schemas/dataset.schema"; -import { CreateRawDatasetDto } from "src/datasets/dto/create-raw-dataset.dto"; -import { CreateDerivedDatasetDto } from "src/datasets/dto/create-derived-dataset.dto"; +import { CreateRawDatasetObsoleteDto } from "src/datasets/dto/create-raw-dataset-obsolete.dto"; +import { CreateDerivedDatasetObsoleteDto } from "src/datasets/dto/create-derived-dataset-obsolete.dto"; import { logger } from "@user-office-software/duo-logger"; @ApiBearerAuth() @@ -103,7 +103,10 @@ export class OrigDatablocksController { // } async generateOrigDatablockInstanceInstanceForPermissions( - dataset: CreateRawDatasetDto | CreateDerivedDatasetDto | DatasetClass, + dataset: + | CreateRawDatasetObsoleteDto + | CreateDerivedDatasetObsoleteDto + | DatasetClass, ): Promise { const origDatablockInstance = new OrigDatablock(); origDatablockInstance.datasetId = dataset.pid || ""; diff --git a/src/published-data/published-data.controller.ts b/src/published-data/published-data.controller.ts index 3d8199c7d..0786e9a53 100644 --- a/src/published-data/published-data.controller.ts +++ b/src/published-data/published-data.controller.ts @@ -166,13 +166,17 @@ export class PublishedDataController { }) async formPopulate(@Query("pid") pid: string) { const formData: IFormPopulateData = {}; - const dataset = await this.datasetsService.findOne({ where: { pid } }); + const dataset = (await this.datasetsService.findOne({ + where: { pid }, + })) as unknown as DatasetClass; let proposalId; if (dataset) { formData.resourceType = dataset.type; formData.description = dataset.description; - proposalId = (dataset as unknown as DatasetClass).proposalId; + if ("proposalIds" in dataset) { + proposalId = dataset.proposalIds![0]; + } } let proposal; diff --git a/test/DatasetAuthorization.js b/test/DatasetAuthorization.js index ab707200d..3aeb7f749 100644 --- a/test/DatasetAuthorization.js +++ b/test/DatasetAuthorization.js @@ -45,7 +45,7 @@ describe("0300: DatasetAuthorization: Test access to dataset", () => { before(() => { db.collection("Dataset").deleteMany({}); }); - beforeEach(async() => { + beforeEach(async () => { accessTokenAdminIngestor = await utils.getToken(appUrl, { username: "adminIngestor", password: TestData.Accounts["adminIngestor"]["password"], @@ -76,7 +76,7 @@ describe("0300: DatasetAuthorization: Test access to dataset", () => { password: TestData.Accounts["archiveManager"]["password"], }); }); - + afterEach((done) => { sandbox.restore(); done(); @@ -662,7 +662,6 @@ describe("0300: DatasetAuthorization: Test access to dataset", () => { pid: TestData.PidPrefix + "/" + uuidv4(), ownerGroup: "admin", }; - console.log("0502: pid : " + datasetWithPid["pid"]); return request(appUrl) .post("/api/v3/Datasets") diff --git a/test/DatasetLifecycle.js b/test/DatasetLifecycle.js index 3b26585de..0611be077 100644 --- a/test/DatasetLifecycle.js +++ b/test/DatasetLifecycle.js @@ -15,8 +15,9 @@ const raw2 = { ...TestData.RawCorrect }; describe("0500: DatasetLifecycle: Test facet and filter queries", () => { before(() => { db.collection("Dataset").deleteMany({}); + db.collection("Policy").deleteMany({}); }); - beforeEach(async() => { + beforeEach(async () => { accessTokenAdminIngestor = await utils.getToken(appUrl, { username: "adminIngestor", password: TestData.Accounts["adminIngestor"]["password"], diff --git a/test/DerivedDataset.js b/test/DerivedDataset.js index 1ccadc9ce..7e4c2e6ce 100644 --- a/test/DerivedDataset.js +++ b/test/DerivedDataset.js @@ -99,6 +99,9 @@ describe("0700: DerivedDataset: Derived Datasets", () => { .and.be.equal(TestData.DerivedCorrect.owner); res.body.should.have.property("type").and.be.equal("derived"); res.body.should.have.property("pid").and.be.string; + res.body.should.have.property("proposalId").and.be.string; + //res.body.should.have.property("sampleId").and.be.string; + //res.body.should.have.property("instrumentId").and.be.string; pid = res.body["pid"]; }); }); diff --git a/test/DerivedDatasetDatablock.js b/test/DerivedDatasetDatablock.js index 8ef6ff8c7..6c223246c 100644 --- a/test/DerivedDatasetDatablock.js +++ b/test/DerivedDatasetDatablock.js @@ -13,7 +13,7 @@ describe("0750: DerivedDatasetDatablock: Test Datablocks and their relation to d before(() => { db.collection("Dataset").deleteMany({}); }); - beforeEach(async() => { + beforeEach(async () => { accessTokenAdminIngestor = await utils.getToken(appUrl, { username: "adminIngestor", password: TestData.Accounts["adminIngestor"]["password"], diff --git a/test/ElasticSearch.js b/test/ElasticSearch.js index cfec71f29..a027564cb 100644 --- a/test/ElasticSearch.js +++ b/test/ElasticSearch.js @@ -45,12 +45,12 @@ const scientificMetadata = (values) => { before(() => { db.collection("Dataset").deleteMany({}); }); - beforeEach(async() => { + beforeEach(async () => { accessTokenAdminIngestor = await utils.getToken(appUrl, { username: "adminIngestor", password: TestData.Accounts["adminIngestor"]["password"], }); - + accessTokenArchiveManager = await utils.getToken(appUrl, { username: "archiveManager", password: TestData.Accounts["archiveManager"]["password"], @@ -178,11 +178,11 @@ const scientificMetadata = (values) => { }); }); - it("0030: should fetching dataset with correct proposalId and size", async () => { + it("0030: should fetching dataset with correct proposalIds and size", async () => { return request(appUrl) .post("/api/v3/elastic-search/search") .send({ - proposalId: TestData.ScientificMetadataForElasticSearch.proposalId, + proposalIds: TestData.ScientificMetadataForElasticSearch.proposalId, size: TestData.ScientificMetadataForElasticSearch.size, }) .set("Accept", "application/json") @@ -194,11 +194,11 @@ const scientificMetadata = (values) => { }); }); - it("0031: should fail fetching dataset with correct proposalId but wrong size", async () => { + it("0031: should fail fetching dataset with correct proposalIds but wrong size", async () => { return request(appUrl) .post("/api/v3/elastic-search/search") .send({ - proposalId: TestData.ScientificMetadataForElasticSearch.proposalId, + proposalIds: [TestData.ScientificMetadataForElasticSearch.proposalId], size: faker.number.int({ min: 100000001, max: 100400000 }), }) .set("Accept", "application/json") @@ -209,11 +209,11 @@ const scientificMetadata = (values) => { res.body.data.should.be.length(0); }); }); - it("0032: should fail fetching dataset with wrong proposalId but correct size", async () => { + it("0032: should fail fetching dataset with wrong proposalIds but correct size", async () => { return request(appUrl) .post("/api/v3/elastic-search/search") .send({ - proposalId: "wrongProposalId", + proposalIds: ["wrongProposalId"], size: TestData.ScientificMetadataForElasticSearch.size, }) .set("Accept", "application/json") @@ -225,11 +225,11 @@ const scientificMetadata = (values) => { }); }); - it("0033: should fail fetching dataset with incorrect proposalId and size", async () => { + it("0033: should fail fetching dataset with incorrect proposalIds and size", async () => { return request(appUrl) .post("/api/v3/elastic-search/search") .send({ - proposalId: "wrongProposalId", + proposalIds: ["wrongProposalId"], size: faker.number.int({ min: 100000001, max: 100400000 }), }) .set("Accept", "application/json") diff --git a/test/OrigDatablockForRawDataset.js b/test/OrigDatablockForRawDataset.js index 057134d35..d329d5867 100644 --- a/test/OrigDatablockForRawDataset.js +++ b/test/OrigDatablockForRawDataset.js @@ -23,7 +23,7 @@ describe("1200: OrigDatablockForRawDataset: Test OrigDatablocks and their relati db.collection("Dataset").deleteMany({}); db.collection("OrigDatablock").deleteMany({}); }); - beforeEach(async() => { + beforeEach(async () => { accessTokenAdminIngestor = await utils.getToken(appUrl, { username: "adminIngestor", password: TestData.Accounts["adminIngestor"]["password"], diff --git a/test/RawDataset.js b/test/RawDataset.js index c150faa40..6e9fcd748 100644 --- a/test/RawDataset.js +++ b/test/RawDataset.js @@ -93,6 +93,9 @@ describe("1900: RawDataset: Raw Datasets", () => { res.body.should.have.property("owner").and.be.string; res.body.should.have.property("type").and.equal("raw"); res.body.should.have.property("pid").and.be.string; + res.body.should.have.property("instrumentId").and.be.string; + res.body.should.have.property("proposalId").and.be.string; + res.body.should.have.property("sampleId").and.be.string; pid = encodeURIComponent(res.body["pid"]); }); }); diff --git a/test/TestData.js b/test/TestData.js index 2f5f0a22e..801a45768 100644 --- a/test/TestData.js +++ b/test/TestData.js @@ -150,6 +150,7 @@ const TestData = { sourceFolder: faker.system.directoryPath(), owner: faker.internet.userName(), contactEmail: faker.internet.email(), + datasetName: faker.string.sample(), }, RawCorrect: { @@ -231,6 +232,8 @@ const TestData = { ownerGroup: "p13388", accessGroups: [], proposalId: "10.540.16635/20110123", + instrumentId: "1f016ec4-7a73-11ef-ae3e-439013069377", + sampleId: "20c32b4e-7a73-11ef-9aec-5b9688aa3791i", type: "raw", keywords: ["sls", "protein"], }, @@ -415,13 +418,14 @@ const TestData = { DerivedCorrectMin: { investigator: faker.internet.email(), - inputDatasets: [faker.system.filePath()], + inputDatasets: [faker.string.uuid()], usedSoftware: [faker.internet.url()], owner: faker.internet.userName(), contactEmail: faker.internet.email(), sourceFolder: faker.system.directoryPath(), creationTime: faker.date.past(), ownerGroup: faker.string.alphanumeric(6), + datasetName: faker.string.sample(), type: "derived", }, @@ -449,6 +453,9 @@ const TestData = { ownerGroup: "p34123", accessGroups: [], type: "derived", + proposalId: "10.540.16635/20110123", + //instrumentId: "1f016ec4-7a73-11ef-ae3e-439013069377", + //sampleId: "20c32b4e-7a73-11ef-9aec-5b9688aa3791i", }, DerivedWrong: { @@ -847,6 +854,7 @@ const TestData = { creationLocation: faker.location.city(), principalInvestigator: faker.internet.userName(), type: "raw", + datasetName: faker.string.sample(), creationTime: faker.date.past(), sourceFolder: faker.system.directoryPath(), owner: faker.internet.userName(),