From 354fd9560b81f0bc7b02e1249f4c6e0ff3d9962b Mon Sep 17 00:00:00 2001 From: Carter Rollins Date: Wed, 16 Oct 2024 15:51:38 -0400 Subject: [PATCH] Adds ability to upload tsv to db --- .../data/features/[featureId]/page.tsx | 4 +- frontend/app/(dashboard)/data/page.tsx | 3 +- .../data/taxonomies/[taxonomy]/page.tsx | 2 +- frontend/components/AsvUpload.tsx | 66 ++++++++ frontend/helpers/utils.ts | 6 +- frontend/prisma/dbml/OpalERD.dbml | 2 +- .../migration.sql | 147 ++++++++++++++++++ frontend/prisma/schema.prisma | 2 +- 8 files changed, 223 insertions(+), 9 deletions(-) create mode 100644 frontend/components/AsvUpload.tsx create mode 100644 frontend/prisma/migrations/20241016194957_feature_unique/migration.sql diff --git a/frontend/app/(dashboard)/data/features/[featureId]/page.tsx b/frontend/app/(dashboard)/data/features/[featureId]/page.tsx index f7dcf1c..bdc2d21 100644 --- a/frontend/app/(dashboard)/data/features/[featureId]/page.tsx +++ b/frontend/app/(dashboard)/data/features/[featureId]/page.tsx @@ -11,7 +11,7 @@ export default async function FeatureId({ params }: { params: { featureId: strin const headers = lines[0].split("\t"); for (let i = 1; i < lines.length; i++) { - const obj = {} as { featureid: String, species: String }; + const obj = {} as { featureid: string, species: string }; const currentline = lines[i].split("\t"); for (let j = 0; j < headers.length; j++) { @@ -21,7 +21,7 @@ export default async function FeatureId({ params }: { params: { featureId: strin result.push(obj); } - const feature = result.find((f) => f.featureid === params.featureId) as { featureid: String, species: String }; + const feature = result.find((f) => f.featureid === params.featureId) as { featureid: string, species: string }; //const records = parse(content, { bom: true, columns: true }); //console.log(records) diff --git a/frontend/app/(dashboard)/data/page.tsx b/frontend/app/(dashboard)/data/page.tsx index 91a7b2e..1074e3a 100644 --- a/frontend/app/(dashboard)/data/page.tsx +++ b/frontend/app/(dashboard)/data/page.tsx @@ -1,6 +1,6 @@ +import AsvUpload from "@/components/AsvUpload"; import ProjectCatalogue from "@/components/ProjectCatalogue"; import Search from "@/components/Search"; -import { getRemoteUrl } from "@/helpers/utils"; import dynamic from "next/dynamic"; const Map = dynamic(() => import("@/components/Map"), { ssr: false, @@ -18,6 +18,7 @@ export default async function Dashboard() { return (
+
diff --git a/frontend/app/(dashboard)/data/taxonomies/[taxonomy]/page.tsx b/frontend/app/(dashboard)/data/taxonomies/[taxonomy]/page.tsx index a1df6c6..dd5ae1e 100644 --- a/frontend/app/(dashboard)/data/taxonomies/[taxonomy]/page.tsx +++ b/frontend/app/(dashboard)/data/taxonomies/[taxonomy]/page.tsx @@ -11,7 +11,7 @@ export default async function FeatureId({ params }: { params: { taxonomy: string const headers = lines[0].split("\t"); for (let i = 1; i < lines.length; i++) { - const obj = {} as { featureid: String, species: String }; + const obj = {} as { featureid: string, species: string }; const currentline = lines[i].split("\t"); for (let j = 0; j < headers.length; j++) { diff --git a/frontend/components/AsvUpload.tsx b/frontend/components/AsvUpload.tsx new file mode 100644 index 0000000..b651318 --- /dev/null +++ b/frontend/components/AsvUpload.tsx @@ -0,0 +1,66 @@ +import { prisma } from "@/helpers/prisma"; + +export default function AsvUpload() { + async function asvUpload(formData: FormData) { + "use server" + + const content = await (formData.get("asvFile") as File).text(); + const lines = content.split("\n"); + const entriesByTaxa = {} as { [key: string]: [{}] }; //object where values are arrays of objects + const headers = lines[0].split("\t"); + const taxonomies = new Set(); + + for (let i = 1; i < lines.length; i++) { + const obj = {} as { featureid: string, species: string, taxonomy: string }; + const currentline = lines[i].split("\t"); + + for (let j = 0; j < headers.length; j++) { + //@ts-ignore + obj[headers[j]] = currentline[j]; + } + + const taxa = obj.taxonomy as keyof typeof entriesByTaxa; + if (entriesByTaxa[taxa]) { + entriesByTaxa[taxa].push(obj); + } else { + entriesByTaxa[taxa] = [obj]; + } + taxonomies.add(obj.taxonomy); + } + + //console.log(entriesByTaxa[[...taxonomies][0] as string]) + for (const taxa in entriesByTaxa) { + console.log(taxa) + await prisma.taxonomy.upsert({ + where: { + stringIdentifier: taxa + }, + update: { + Feature: { + createMany: { + data: entriesByTaxa[taxa].map((obj: any) => ({ featureId: obj.featureid, sequence: obj.sequence })), + skipDuplicates: true + } + } + }, + create: { + stringIdentifier: taxa, + Feature: { + createMany: { + data: entriesByTaxa[taxa].map((obj: any) => ({ featureId: obj.featureid, sequence: obj.sequence })), + skipDuplicates: true + } + } + } + }); + } + console.log("success") + } + + return ( +
+ + +
+ ); +} \ No newline at end of file diff --git a/frontend/helpers/utils.ts b/frontend/helpers/utils.ts index 5d7bf10..d4be6e2 100644 --- a/frontend/helpers/utils.ts +++ b/frontend/helpers/utils.ts @@ -5,10 +5,10 @@ export async function fetcher(url: string) { } export function getBaseUrl() { - // if (process.env.NODE_ENV === "development") { + if (process.env.NODE_ENV === "development") { return "http://localhost:3000"; - // } - // return "https://opal-ochre.vercel.app"; + } + return "https://opaldb.vercel.app/"; } export function getRemoteUrl() { diff --git a/frontend/prisma/dbml/OpalERD.dbml b/frontend/prisma/dbml/OpalERD.dbml index 407eb48..321257d 100644 --- a/frontend/prisma/dbml/OpalERD.dbml +++ b/frontend/prisma/dbml/OpalERD.dbml @@ -167,7 +167,7 @@ Table Feature { Assignments Assignment [not null] consensusTaxonomyId String consensusTaxonomy Taxonomy - featureId String [not null] + featureId String [unique, not null] sequence String [not null] } diff --git a/frontend/prisma/migrations/20241016194957_feature_unique/migration.sql b/frontend/prisma/migrations/20241016194957_feature_unique/migration.sql new file mode 100644 index 0000000..23c2a6a --- /dev/null +++ b/frontend/prisma/migrations/20241016194957_feature_unique/migration.sql @@ -0,0 +1,147 @@ +/* + Warnings: + + - You are about to drop the column `step1Id` on the `Step2_16S` table. All the data in the column will be lost. + - You are about to drop the column `step1Id` on the `Step2_18S` table. All the data in the column will be lost. + - You are about to drop the `Step1` table. If the table is not empty, all the data it contains will be lost. + - You are about to drop the `Step1_Metadata` table. If the table is not empty, all the data it contains will be lost. + - A unique constraint covering the columns `[featureId]` on the table `Feature` will be added. If there are existing duplicate values, this will fail. + - Added the required column `sampleId` to the `Occurrence` table without a default value. This is not possible if the table is not empty. + - Added the required column `dateModified` to the `Run` table without a default value. This is not possible if the table is not empty. + - Added the required column `uploadedBy` to the `Run` table without a default value. This is not possible if the table is not empty. + - Added the required column `SampleId` to the `Step2_16S` table without a default value. This is not possible if the table is not empty. + - Added the required column `SampleId` to the `Step2_18S` table without a default value. This is not possible if the table is not empty. + +*/ +-- DropForeignKey +ALTER TABLE "Step1" DROP CONSTRAINT "Step1_projectId_fkey"; + +-- DropForeignKey +ALTER TABLE "Step1" DROP CONSTRAINT "Step1_step1_MetadataId_fkey"; + +-- DropForeignKey +ALTER TABLE "Step2_16S" DROP CONSTRAINT "Step2_16S_step1Id_fkey"; + +-- DropForeignKey +ALTER TABLE "Step2_18S" DROP CONSTRAINT "Step2_18S_step1Id_fkey"; + +-- AlterTable +ALTER TABLE "Occurrence" ADD COLUMN "sampleId" TEXT NOT NULL; + +-- AlterTable +ALTER TABLE "Run" ADD COLUMN "dateModified" TEXT NOT NULL, +ADD COLUMN "uploadedBy" TEXT NOT NULL; + +-- AlterTable +ALTER TABLE "Step2_16S" DROP COLUMN "step1Id", +ADD COLUMN "SampleId" TEXT NOT NULL; + +-- AlterTable +ALTER TABLE "Step2_18S" DROP COLUMN "step1Id", +ADD COLUMN "SampleId" TEXT NOT NULL; + +-- DropTable +DROP TABLE "Step1"; + +-- DropTable +DROP TABLE "Step1_Metadata"; + +-- CreateTable +CREATE TABLE "Sample_Metadata" ( + "id" TEXT NOT NULL, + "nucl_acid_ext" TEXT, + "nucl_acid_ext_kit" TEXT, + "nucl_acid_ext_modify" TEXT, + "precip_method_ext" TEXT, + "dna_cleanup_0_1" TEXT, + "dna_cleanup_method" TEXT, + "concentration_method" TEXT, + "seq_kit" TEXT, + "lib_layout" TEXT NOT NULL, + "adapter_forward" TEXT, + "adapter_reverse" TEXT, + "seq_facility" TEXT, + "lib_screen" TEXT, + "platform" TEXT NOT NULL, + "instrument_model" TEXT NOT NULL, + "seq_meth" TEXT, + "seq_method_additional" TEXT, + + CONSTRAINT "Sample_Metadata_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "Sample" ( + "id" TEXT NOT NULL, + "projectId" TEXT NOT NULL, + "sample_name" TEXT NOT NULL, + "sample_name_original" TEXT NOT NULL, + "sample_type" "sample_type", + "serial_number" TEXT, + "cruise_id" TEXT, + "line_id" TEXT, + "station" TEXT, + "locationID" TEXT, + "habitat_natural_artificial_0_1" TEXT, + "ctd_bottle_no" TEXT, + "sample_replicate" TEXT, + "source_material_id" TEXT, + "biological_replicates" TEXT, + "notes_sampling" TEXT, + "verbatimEventDate" TEXT, + "eventDate" TEXT NOT NULL, + "minimumDepthInMeters" DOUBLE PRECISION, + "maximumDepthInMeters" DOUBLE PRECISION, + "env_broad_scale" TEXT NOT NULL, + "env_local_scale" TEXT NOT NULL, + "env_medium" TEXT NOT NULL, + "geo_loc_name" TEXT NOT NULL, + "waterBody" TEXT, + "country" TEXT, + "decimalLatitude" DOUBLE PRECISION, + "decimalLongitude" DOUBLE PRECISION, + "collection_method" TEXT, + "samp_collect_device" TEXT, + "samp_size" DOUBLE PRECISION, + "samp_size_unit" TEXT, + "extract_number" TEXT, + "samp_mat_process" TEXT, + "filter_passive_active_0_1" TEXT, + "filter_onsite_dur" TEXT, + "size_frac" TEXT, + "samp_vol_we_dna_ext" TEXT, + "samp_vol_ext_unit" DOUBLE PRECISION, + "sample_title" TEXT, + "bioproject_accession" TEXT, + "biosample_accession" TEXT, + "organism" TEXT NOT NULL, + "description" TEXT, + "amplicon_sequenced" TEXT, + "metagenome_sequenced" TEXT, + "sterilise_method" TEXT, + "samp_store_dur" TEXT, + "samp_store_loc" TEXT, + "samp_store_temp" DOUBLE PRECISION, + "samp_store_sol" TEXT, + "sample_MetadataId" TEXT, + + CONSTRAINT "Sample_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "Feature_featureId_key" ON "Feature"("featureId"); + +-- AddForeignKey +ALTER TABLE "Sample" ADD CONSTRAINT "Sample_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Sample" ADD CONSTRAINT "Sample_sample_MetadataId_fkey" FOREIGN KEY ("sample_MetadataId") REFERENCES "Sample_Metadata"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Step2_16S" ADD CONSTRAINT "Step2_16S_SampleId_fkey" FOREIGN KEY ("SampleId") REFERENCES "Sample"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Step2_18S" ADD CONSTRAINT "Step2_18S_SampleId_fkey" FOREIGN KEY ("SampleId") REFERENCES "Sample"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Occurrence" ADD CONSTRAINT "Occurrence_sampleId_fkey" FOREIGN KEY ("sampleId") REFERENCES "Sample"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/frontend/prisma/schema.prisma b/frontend/prisma/schema.prisma index 19ec080..b22ef7d 100644 --- a/frontend/prisma/schema.prisma +++ b/frontend/prisma/schema.prisma @@ -454,7 +454,7 @@ model Feature { consensusTaxonomyId String? consensusTaxonomy Taxonomy? @relation(fields: [consensusTaxonomyId], references: [id]) - featureId String + featureId String @unique sequence String }