From 9d4c8c78976a136b0b73e777d3594e6b214163a0 Mon Sep 17 00:00:00 2001
From: nichenqin <nichenqin@hotmail.com>
Date: Thu, 31 Aug 2023 14:37:32 +0800
Subject: [PATCH 1/8] feat(merge): merge data modal

---
 apps/frontend/src/lib/import/MergeData.svelte    |  6 ++++++
 apps/frontend/src/lib/store/modal.ts             |  3 +++
 apps/frontend/src/lib/table/TableMenu.svelte     | 13 ++++++++++++-
 apps/frontend/src/routes/(authed)/+layout.svelte |  3 +++
 packages/authz/src/rbac/permissions.ts           |  5 +++++
 5 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 apps/frontend/src/lib/import/MergeData.svelte

diff --git a/apps/frontend/src/lib/import/MergeData.svelte b/apps/frontend/src/lib/import/MergeData.svelte
new file mode 100644
index 000000000..c1c312f28
--- /dev/null
+++ b/apps/frontend/src/lib/import/MergeData.svelte
@@ -0,0 +1,6 @@
+<script lang="ts">
+	import { mergeDataModal } from '$lib/store/modal'
+	import { Modal } from 'flowbite-svelte'
+</script>
+
+<Modal class="w-full" bind:open={$mergeDataModal.open}>merge</Modal>
diff --git a/apps/frontend/src/lib/store/modal.ts b/apps/frontend/src/lib/store/modal.ts
index 2125ccef5..7857787f5 100644
--- a/apps/frontend/src/lib/store/modal.ts
+++ b/apps/frontend/src/lib/store/modal.ts
@@ -72,6 +72,9 @@ export const createOptionModal = createModal(CREATE_OPTION)
 const UPDATE_OPTION = Symbol('UPDATE_OPTION')
 export const updateOptionModal = createModal(UPDATE_OPTION)
 
+const MERGE_DATA_MODAL = Symbol('MERGE_DATA_MODAL')
+export const mergeDataModal = createModal(MERGE_DATA_MODAL)
+
 const IMPORT_DATA_MODAL = Symbol('IMPORT_DATA_MODAL')
 export const importDataModal = createModal(IMPORT_DATA_MODAL)
 
diff --git a/apps/frontend/src/lib/table/TableMenu.svelte b/apps/frontend/src/lib/table/TableMenu.svelte
index 7b0fd134e..c2a523036 100644
--- a/apps/frontend/src/lib/table/TableMenu.svelte
+++ b/apps/frontend/src/lib/table/TableMenu.svelte
@@ -1,7 +1,7 @@
 <script lang="ts">
 	import { t } from '$lib/i18n'
 	import { webhookListDrawer } from '$lib/store/drawer'
-	import { erdModal, rlsModal } from '$lib/store/modal'
+	import { erdModal, mergeDataModal, rlsModal } from '$lib/store/modal'
 	import { currentRLSS } from '$lib/store/table'
 	import { Badge, Dropdown, DropdownItem } from 'flowbite-svelte'
 	import { hasPermission } from '$lib/store/authz'
@@ -47,4 +47,15 @@
 			{/if}
 		</DropdownItem>
 	{/if}
+	{#if $hasPermission('table:merge_data')}
+		<DropdownItem
+			on:click={() => {
+				mergeDataModal.open()
+			}}
+			class="text-xs font-normal inline-flex items-center gap-2"
+		>
+			<i class="ti ti-database-import text-gray-600 dark:text-gray-50" />
+			<span>{$t('merge data')}</span>
+		</DropdownItem>
+	{/if}
 </Dropdown>
diff --git a/apps/frontend/src/routes/(authed)/+layout.svelte b/apps/frontend/src/routes/(authed)/+layout.svelte
index 2b21174a8..0e264514b 100644
--- a/apps/frontend/src/routes/(authed)/+layout.svelte
+++ b/apps/frontend/src/routes/(authed)/+layout.svelte
@@ -22,6 +22,7 @@
 	import { DARK_THEME, LIGHT_THEME } from '$lib/store/ui.type'
 	import TablesNav from '$lib/table/TablesNav.svelte'
 	import { hasPermission } from '$lib/store/authz'
+	import MergeData from '$lib/import/MergeData.svelte'
 
 	$: navigation = [
 		{ name: $t('Tables', { ns: 'common' }), href: '/', icon: 'table', current: $page.url.pathname === '/' },
@@ -406,6 +407,8 @@
 	{#if $importDataModal.open}
 		<ImportData formData={$page.data.createTable} />
 	{/if}
+
+	<MergeData />
 </div>
 
 <Toast
diff --git a/packages/authz/src/rbac/permissions.ts b/packages/authz/src/rbac/permissions.ts
index 48caa17a6..f5700401f 100644
--- a/packages/authz/src/rbac/permissions.ts
+++ b/packages/authz/src/rbac/permissions.ts
@@ -29,6 +29,7 @@ export const tableActions = z.enum([
   'table:list_form',
   'table:create_form',
   'table:update_form',
+  'table:merge_data',
 ])
 export const recordActions = z.enum(['record:create', 'record:update', 'record:delete', 'record:list_trash'])
 export const webhookActions = z.enum(['webhook:create', 'webhook:update', 'webhook:delete'])
@@ -81,6 +82,7 @@ export const permissions: Record<IRoles, Record<PermissionAction, boolean>> = {
     'table:set_view_field': true,
     'table:delete_view': true,
     'table:duplicate_view': true,
+    'table:merge_data': true,
     'record:create': true,
     'record:delete': true,
     'record:update': true,
@@ -131,6 +133,7 @@ export const permissions: Record<IRoles, Record<PermissionAction, boolean>> = {
     'table:set_view_field': true,
     'table:delete_view': true,
     'table:duplicate_view': true,
+    'table:merge_data': true,
     'table:create_form': true,
     'table:update_form': true,
     'record:create': true,
@@ -185,6 +188,7 @@ export const permissions: Record<IRoles, Record<PermissionAction, boolean>> = {
     'table:list_form': true,
     'table:create_form': true,
     'table:update_form': true,
+    'table:merge_data': true,
     'record:create': true,
     'record:delete': true,
     'record:update': true,
@@ -237,6 +241,7 @@ export const permissions: Record<IRoles, Record<PermissionAction, boolean>> = {
     'table:list_form': false,
     'table:create_form': false,
     'table:update_form': false,
+    'table:merge_data': false,
     'record:create': false,
     'record:delete': false,
     'record:update': false,

From 1c017c199649b8e5ff7815f03d896d1619158371 Mon Sep 17 00:00:00 2001
From: nichenqin <nichenqin@hotmail.com>
Date: Thu, 31 Aug 2023 16:00:08 +0800
Subject: [PATCH 2/8] feat(merge): merge text data

---
 apps/frontend/src/lib/import/MergeData.svelte | 122 +++++++++++++++++-
 packages/trpc/src/router/record.router.ts     |   9 ++
 2 files changed, 129 insertions(+), 2 deletions(-)

diff --git a/apps/frontend/src/lib/import/MergeData.svelte b/apps/frontend/src/lib/import/MergeData.svelte
index c1c312f28..2a9886dd0 100644
--- a/apps/frontend/src/lib/import/MergeData.svelte
+++ b/apps/frontend/src/lib/import/MergeData.svelte
@@ -1,6 +1,124 @@
 <script lang="ts">
+	import { t } from '$lib/i18n'
 	import { mergeDataModal } from '$lib/store/modal'
-	import { Modal } from 'flowbite-svelte'
+	import { Button, Dropzone, Modal } from 'flowbite-svelte'
+	import { parse, type SheetData } from './import.helper'
+	import { getTable } from '$lib/store/table'
+	import { castFieldValue, type IMutateRecordValueSchema } from '@undb/core'
+	import { trpc } from '$lib/trpc/client'
+
+	const table = getTable()
+
+	let data: SheetData | undefined
+
+	const createRecords = trpc().record.buldCreate.mutation({
+		onSuccess(data, variables, context) {
+			mergeDataModal.close()
+		},
+	})
+
+	$: header = data?.[0]
+	$: body = data?.slice(1)
+
+	$: schema = $table.schema.toNameMap()
+
+	$: records = body
+		?.map((values) => {
+			return values.reduce((prev, value, index) => {
+				const title = header?.[index]
+				if (!title) return prev
+
+				const field = schema.get(String(title))
+				if (!field || field.controlled) return prev
+
+				const type = field.type
+
+				prev[field.id.value] = type ? castFieldValue(type, value) : value
+				return prev
+			}, {} as IMutateRecordValueSchema)
+		})
+		.filter(Boolean)
+
+	let ext: string | undefined
+	let file: File | undefined
+
+	const dropHandle = async (event: DragEvent) => {
+		event.preventDefault()
+		const files = event.dataTransfer?.files
+		if (!!files?.length) {
+			file = files[0]
+		}
+	}
+
+	const handleChange = async (event: Event) => {
+		const target = event.target as HTMLInputElement
+		const files = target.files
+		if (!!files?.length) {
+			file = files[0]
+		}
+	}
+
+	const handleFile = async (file: File | undefined) => {
+		if (!file) return
+
+		const parsed = await parse(file)
+		data = parsed.data
+		ext = parsed.extension
+	}
+
+	$: if (file) {
+		handleFile(file)
+	}
 </script>
 
-<Modal class="w-full" bind:open={$mergeDataModal.open}>merge</Modal>
+<Modal class="w-full" bind:open={$mergeDataModal.open}>
+	<Dropzone
+		accept=".csv, .json, application/vnd.openxmlformats-officedocument.spreadsheetml.sheet, application/vnd.ms-excel"
+		id="dropzone"
+		on:drop={dropHandle}
+		on:dragover={(event) => {
+			event.preventDefault()
+		}}
+		on:change={handleChange}
+	>
+		<svg
+			aria-hidden="true"
+			class="mb-3 w-10 h-10 text-gray-400 dark:text-gray-200"
+			fill="none"
+			stroke="currentColor"
+			viewBox="0 0 24 24"
+			xmlns="http://www.w3.org/2000/svg"
+		>
+			<path
+				stroke-linecap="round"
+				stroke-linejoin="round"
+				stroke-width="2"
+				d="M7 16a4 4 0 01-.88-7.903A5 5 0 1115.9 6L16 6a5 5 0 011 9.9M15 13l-3-3m0 0l-3 3m3-3v12"
+			/>
+		</svg>
+		<p class="mb-2 text-sm text-gray-500 dark:text-gray-400">
+			{@html $t('click to upload or dnd', { ns: 'common' })}
+		</p>
+	</Dropzone>
+
+	<div class="flex justify-end items-center gap-2">
+		<Button size="xs" type="button" outline color="alternative" on:click={() => mergeDataModal.close()}>
+			{$t('Cancel', { ns: 'common' })}
+		</Button>
+		<Button
+			size="xs"
+			disabled={!data || $createRecords.isLoading}
+			on:click={() => {
+				if (!records?.length) return
+				$createRecords.mutate({
+					tableId: $table.id.value,
+					records: records.map((record) => ({ values: record })),
+				})
+			}}
+		>
+			<div class="flex items-center">
+				{$t('Confirm', { ns: 'common' })}
+			</div>
+		</Button>
+	</div>
+</Modal>
diff --git a/packages/trpc/src/router/record.router.ts b/packages/trpc/src/router/record.router.ts
index dbeee7ac7..02b2579a3 100644
--- a/packages/trpc/src/router/record.router.ts
+++ b/packages/trpc/src/router/record.router.ts
@@ -4,6 +4,7 @@ import {
   BulkDeleteRecordsCommand,
   BulkDuplicateRecordsCommand,
   CreateRecordCommand,
+  CreateRecordsCommand,
   DeleteRecordCommand,
   DuplicateRecordCommand,
   GetForeignRecordsQuery,
@@ -42,6 +43,14 @@ export const createRecordRouter =
           const cmd = new CreateRecordCommand(input)
           return commandBus.execute(cmd)
         }),
+      buldCreate: procedure
+        .use(authz('record:create'))
+        .input(z.any())
+        .output(z.any())
+        .mutation(({ input }) => {
+          const cmd = new CreateRecordsCommand(input)
+          return commandBus.execute(cmd)
+        }),
       duplicate: procedure
         .use(authz('record:create'))
         .input(duplicateRecordCommandInput)

From 035a9023eae335a84d76d25c8d439f42268a00f1 Mon Sep 17 00:00:00 2001
From: nichenqin <nichenqin@hotmail.com>
Date: Thu, 31 Aug 2023 16:43:25 +0800
Subject: [PATCH 3/8] fix: fix merge data modal

---
 apps/frontend/src/routes/(authed)/+layout.svelte               | 3 ---
 .../src/routes/(authed)/t/[tableId]/[[viewId]]/+page.svelte    | 2 ++
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/apps/frontend/src/routes/(authed)/+layout.svelte b/apps/frontend/src/routes/(authed)/+layout.svelte
index 0e264514b..2b21174a8 100644
--- a/apps/frontend/src/routes/(authed)/+layout.svelte
+++ b/apps/frontend/src/routes/(authed)/+layout.svelte
@@ -22,7 +22,6 @@
 	import { DARK_THEME, LIGHT_THEME } from '$lib/store/ui.type'
 	import TablesNav from '$lib/table/TablesNav.svelte'
 	import { hasPermission } from '$lib/store/authz'
-	import MergeData from '$lib/import/MergeData.svelte'
 
 	$: navigation = [
 		{ name: $t('Tables', { ns: 'common' }), href: '/', icon: 'table', current: $page.url.pathname === '/' },
@@ -407,8 +406,6 @@
 	{#if $importDataModal.open}
 		<ImportData formData={$page.data.createTable} />
 	{/if}
-
-	<MergeData />
 </div>
 
 <Toast
diff --git a/apps/frontend/src/routes/(authed)/t/[tableId]/[[viewId]]/+page.svelte b/apps/frontend/src/routes/(authed)/t/[tableId]/[[viewId]]/+page.svelte
index b73c01207..418725722 100644
--- a/apps/frontend/src/routes/(authed)/t/[tableId]/[[viewId]]/+page.svelte
+++ b/apps/frontend/src/routes/(authed)/t/[tableId]/[[viewId]]/+page.svelte
@@ -23,6 +23,7 @@
 	import { match } from 'ts-pattern'
 	import RLSModal from '$lib/authz/rls/RLSModal.svelte'
 	import FlsModal from '$lib/authz/fls/FLSModal.svelte'
+	import MergeData from '$lib/import/MergeData.svelte'
 
 	const table = getTable()
 	export let data: PageData
@@ -85,6 +86,7 @@
 
 <TableIndex />
 
+<MergeData />
 {#key $table}
 	<UpdateTable data={data.updateTable} />
 {/key}

From 7aa623d5422d35faf724a5a1b30018b3ed123261 Mon Sep 17 00:00:00 2001
From: nichenqin <nichenqin@hotmail.com>
Date: Thu, 31 Aug 2023 16:49:09 +0800
Subject: [PATCH 4/8] feat: merge data cast field value o array

---
 packages/core/src/table/field/field.util.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/core/src/table/field/field.util.ts b/packages/core/src/table/field/field.util.ts
index d9decebac..3236177f4 100644
--- a/packages/core/src/table/field/field.util.ts
+++ b/packages/core/src/table/field/field.util.ts
@@ -1,5 +1,5 @@
 import type { TFunction } from 'i18next'
-import { isBoolean, isNumber, isPlainObject, isString, uniq } from 'lodash-es'
+import { castArray, isBoolean, isNumber, isPlainObject, isString, uniq } from 'lodash-es'
 import { match } from 'ts-pattern'
 import { z } from 'zod'
 import { Options } from '../option'
@@ -708,5 +708,6 @@ export const castFieldValue = (type: IFieldType, value: string | number | null |
         .otherwise(Boolean),
     )
     .with('select', () => value || null)
+    .with('multi-select', 'reference', 'tree', 'collaborator', () => (value ? castArray(value) : null))
     .otherwise(() => value)
 }

From f27005381d7a7238b65220b9931488699dbb1091 Mon Sep 17 00:00:00 2001
From: nichenqin <nichenqin@hotmail.com>
Date: Fri, 1 Sep 2023 11:05:21 +0800
Subject: [PATCH 5/8] fix: fix filter record values

---
 apps/frontend/src/lib/import/MergeData.svelte | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/apps/frontend/src/lib/import/MergeData.svelte b/apps/frontend/src/lib/import/MergeData.svelte
index 2a9886dd0..fb41cf1de 100644
--- a/apps/frontend/src/lib/import/MergeData.svelte
+++ b/apps/frontend/src/lib/import/MergeData.svelte
@@ -6,6 +6,7 @@
 	import { getTable } from '$lib/store/table'
 	import { castFieldValue, type IMutateRecordValueSchema } from '@undb/core'
 	import { trpc } from '$lib/trpc/client'
+	import { isEmpty } from 'lodash-es'
 
 	const table = getTable()
 
@@ -37,7 +38,7 @@
 				return prev
 			}, {} as IMutateRecordValueSchema)
 		})
-		.filter(Boolean)
+		.filter((value) => !isEmpty(value))
 
 	let ext: string | undefined
 	let file: File | undefined

From 9bc0b4835f8b436cc5e6bf10831735adad9972bf Mon Sep 17 00:00:00 2001
From: nichenqin <nichenqin@hotmail.com>
Date: Fri, 1 Sep 2023 11:41:35 +0800
Subject: [PATCH 6/8] feat: merge data add some warning message

---
 apps/frontend/src/lib/import/MergeData.svelte | 33 ++++++++++++++++---
 packages/i18n/src/index.ts                    |  2 ++
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/apps/frontend/src/lib/import/MergeData.svelte b/apps/frontend/src/lib/import/MergeData.svelte
index fb41cf1de..6523eb8bc 100644
--- a/apps/frontend/src/lib/import/MergeData.svelte
+++ b/apps/frontend/src/lib/import/MergeData.svelte
@@ -1,24 +1,29 @@
 <script lang="ts">
 	import { t } from '$lib/i18n'
 	import { mergeDataModal } from '$lib/store/modal'
-	import { Button, Dropzone, Modal } from 'flowbite-svelte'
+	import { Alert, Badge, Button, Dropzone, Modal, Tooltip } from 'flowbite-svelte'
 	import { parse, type SheetData } from './import.helper'
 	import { getTable } from '$lib/store/table'
-	import { castFieldValue, type IMutateRecordValueSchema } from '@undb/core'
+	import { castFieldValue, type IFieldType, type IMutateRecordValueSchema } from '@undb/core'
 	import { trpc } from '$lib/trpc/client'
-	import { isEmpty } from 'lodash-es'
+	import { includes, isEmpty } from 'lodash-es'
+	import FieldIcon from '$lib/field/FieldIcon.svelte'
 
 	const table = getTable()
 
 	let data: SheetData | undefined
 
+	const unsupportedMergeType: IFieldType[] = ['reference', 'attachment', 'parent', 'tree', 'collaborator']
+
 	const createRecords = trpc().record.buldCreate.mutation({
 		onSuccess(data, variables, context) {
 			mergeDataModal.close()
 		},
 	})
 
-	$: header = data?.[0]
+	$: header = data?.[0] ?? []
+	$: importHeaders = header.map((title) => schema.get(String(title))!).filter(Boolean)
+	$: unsupportedFields = importHeaders.filter((field) => unsupportedMergeType.includes(field.type))
 	$: body = data?.slice(1)
 
 	$: schema = $table.schema.toNameMap()
@@ -34,6 +39,9 @@
 
 				const type = field.type
 
+				// TODO: support these field types
+				if (unsupportedMergeType.includes(type)) return prev
+
 				prev[field.id.value] = type ? castFieldValue(type, value) : value
 				return prev
 			}, {} as IMutateRecordValueSchema)
@@ -102,6 +110,23 @@
 		</p>
 	</Dropzone>
 
+	{#if !!unsupportedFields.length}
+		<Alert color="yellow">
+			{$t('unsupport merge')}
+			<div class="flex items-center gap-2 mt-2">
+				{#each unsupportedFields as field}
+					<Badge color="dark" class="inline-flex items-center gap-2">
+						<FieldIcon type={field.type} />
+						{field.name.value}
+					</Badge>
+					<Tooltip>
+						{$t(field.type)}
+					</Tooltip>
+				{/each}
+			</div>
+		</Alert>
+	{/if}
+
 	<div class="flex justify-end items-center gap-2">
 		<Button size="xs" type="button" outline color="alternative" on:click={() => mergeDataModal.close()}>
 			{$t('Cancel', { ns: 'common' })}
diff --git a/packages/i18n/src/index.ts b/packages/i18n/src/index.ts
index cd7ec0b6f..d1a4a960d 100644
--- a/packages/i18n/src/index.ts
+++ b/packages/i18n/src/index.ts
@@ -369,6 +369,7 @@ export const config: InitOptions = {
         'set date': 'set date',
         'recycle bin': 'recycle bin',
         'show form conditions': 'show conditions',
+        'unsupport merge': 'unsupport field types to merge into table',
       },
       webhook: {
         Webhook: 'Webhook',
@@ -767,6 +768,7 @@ export const config: InitOptions = {
         'set date': '设置日期',
         'recycle bin': '回收站',
         'show form conditions': '显示过滤',
+        'unsupport merge': '不支持部分列合并到表格',
       },
       webhook: {
         Webhook: 'Webhook',

From 6d7dc831e07e5b0c3f421ff0662efe8544a22227 Mon Sep 17 00:00:00 2001
From: nichenqin <nichenqin@hotmail.com>
Date: Fri, 1 Sep 2023 12:33:40 +0800
Subject: [PATCH 7/8] style: merge records count

---
 apps/frontend/src/lib/import/MergeData.svelte | 5 +++++
 packages/i18n/src/index.ts                    | 2 ++
 2 files changed, 7 insertions(+)

diff --git a/apps/frontend/src/lib/import/MergeData.svelte b/apps/frontend/src/lib/import/MergeData.svelte
index 6523eb8bc..51f685443 100644
--- a/apps/frontend/src/lib/import/MergeData.svelte
+++ b/apps/frontend/src/lib/import/MergeData.svelte
@@ -146,5 +146,10 @@
 				{$t('Confirm', { ns: 'common' })}
 			</div>
 		</Button>
+		{#if records?.length}
+			<Tooltip>
+				{$t('merge record count', { count: records.length })}
+			</Tooltip>
+		{/if}
 	</div>
 </Modal>
diff --git a/packages/i18n/src/index.ts b/packages/i18n/src/index.ts
index d1a4a960d..8a957822f 100644
--- a/packages/i18n/src/index.ts
+++ b/packages/i18n/src/index.ts
@@ -370,6 +370,7 @@ export const config: InitOptions = {
         'recycle bin': 'recycle bin',
         'show form conditions': 'show conditions',
         'unsupport merge': 'unsupport field types to merge into table',
+        'merge record count': 'will merge {{count}} record(s)',
       },
       webhook: {
         Webhook: 'Webhook',
@@ -769,6 +770,7 @@ export const config: InitOptions = {
         'recycle bin': '回收站',
         'show form conditions': '显示过滤',
         'unsupport merge': '不支持部分列合并到表格',
+        'merge record count': '合并 {{count}} 条记录',
       },
       webhook: {
         Webhook: 'Webhook',

From a37ea4ba7bfb31fb4e6b217e83eced7299532826 Mon Sep 17 00:00:00 2001
From: nichenqin <nichenqin@hotmail.com>
Date: Fri, 1 Sep 2023 12:37:48 +0800
Subject: [PATCH 8/8] chore: i18n

---
 packages/i18n/src/index.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/i18n/src/index.ts b/packages/i18n/src/index.ts
index 8a957822f..d4e721f35 100644
--- a/packages/i18n/src/index.ts
+++ b/packages/i18n/src/index.ts
@@ -369,6 +369,7 @@ export const config: InitOptions = {
         'set date': 'set date',
         'recycle bin': 'recycle bin',
         'show form conditions': 'show conditions',
+        'merge data': 'merge data',
         'unsupport merge': 'unsupport field types to merge into table',
         'merge record count': 'will merge {{count}} record(s)',
       },
@@ -769,6 +770,7 @@ export const config: InitOptions = {
         'set date': '设置日期',
         'recycle bin': '回收站',
         'show form conditions': '显示过滤',
+        'merge data': '合并数据',
         'unsupport merge': '不支持部分列合并到表格',
         'merge record count': '合并 {{count}} 条记录',
       },