Create ClickHouse mirror: New sorting key UI and logic (#2142)

![Screenshot 2024-10-22 at 4 04 20 AM](https://github.com/user-attachments/assets/b63865f1-99cc-40b2-88e8-b78fb5c61260) User can now specify order key list by selecting from a dropdown. If they do this, then a warning banner also pops up. This links to a document explaining the various caveats of custom order keys Instead of suffixing the primary keys at the end of the ordering key, the behaviour is now: If user doesn't define order key - PKEY and ORDER BY in CH is PKEY in Postgres. If user defines order key - PKEY and ORDER BY in CH is defined Order Key in Postgres.
PeerDB-io · Oct 24, 2024 · 22addcb · 22addcb
1 parent 8dc6c98
commit 22addcb
Show file tree

Hide file tree

Showing 5 changed files with 294 additions and 66 deletions.
diff --git a/flow/connectors/clickhouse/normalize.go b/flow/connectors/clickhouse/normalize.go
@@ -200,12 +200,16 @@ func getOrderedOrderByColumns(
 	orderby := make([]*protos.ColumnSetting, 0)
 	if tableMapping != nil {
 		for _, col := range tableMapping.Columns {
-			if col.Ordering > 0 && !slices.Contains(pkeys, col.SourceName) {
+			if col.Ordering > 0 {
 				orderby = append(orderby, col)
 			}
 		}
 	}
 
+	if len(orderby) == 0 {
+		return pkeys
+	}
+
 	slices.SortStableFunc(orderby, func(a *protos.ColumnSetting, b *protos.ColumnSetting) int {
 		return cmp.Compare(a.Ordering, b.Ordering)
 	})
@@ -215,10 +219,6 @@ func getOrderedOrderByColumns(
 		orderbyColumns[idx] = getColName(colNameMap, col.SourceName)
 	}
 
-	// Typically primary keys are not what aggregates are performed on and hence
-	// having them at the start of the order by clause is not beneficial.
-	orderbyColumns = append(orderbyColumns, pkeys...)
-
 	return orderbyColumns
 }
 

diff --git a/ui/app/mirrors/create/cdc/columnbox.tsx b/ui/app/mirrors/create/cdc/columnbox.tsx
@@ -3,7 +3,6 @@ import { TableMapRow } from '@/app/dto/MirrorsDTO';
 import { Checkbox } from '@/lib/Checkbox';
 import { Label } from '@/lib/Label';
 import { RowWithCheckbox } from '@/lib/Layout';
-import { TextField } from '@/lib/TextField';
 import { Dispatch, SetStateAction } from 'react';
 
 interface ColumnProps {
@@ -20,7 +19,6 @@ export default function ColumnBox({
   rows,
   setRows,
   disabled,
-  showOrdering,
 }: ColumnProps) {
   const handleColumnExclusion = (column: string, include: boolean) => {
     const source = tableRow.source;
@@ -41,36 +39,15 @@ export default function ColumnBox({
       setRows(currRows);
     }
   };
-  const handleColumnOrdering = (column: string, ordering: number) => {
-    const source = tableRow.source;
-    const currRows = [...rows];
-    const rowIndex = currRows.findIndex((row) => row.source === source);
-    if (rowIndex !== -1) {
-      const sourceRow = currRows[rowIndex];
-      const columns = [...sourceRow.columns];
-      const colIndex = columns.findIndex((col) => col.sourceName === column);
-      if (colIndex !== -1) {
-        columns[colIndex] = { ...columns[colIndex], ordering };
-      } else {
-        columns.push({
-          sourceName: column,
-          destinationName: '',
-          destinationType: '',
-          nullableEnabled: false,
-          ordering,
-        });
-      }
-      currRows[rowIndex] = {
-        ...sourceRow,
-        columns,
-      };
-      setRows(currRows);
-    }
-  };
 
   return columns.map((column) => {
     const [columnName, columnType, isPkeyStr] = column.split(':');
     const isPkey = isPkeyStr === 'true';
+    const partOfOrderingKey = rows
+      .find((row) => row.source == tableRow.source)
+      ?.columns.some(
+        (col) => col.sourceName === columnName && col.ordering <= 0
+      );
     return (
       <RowWithCheckbox
         key={columnName}
@@ -92,26 +69,12 @@ export default function ColumnBox({
             >
               {columnType}
             </p>
-            {showOrdering && !disabled && !isPkey && (
-              <TextField
-                variant='simple'
-                type='number'
-                style={{ width: '3rem', marginLeft: '1rem', fontSize: 13 }}
-                value={
-                  tableRow.columns.find((col) => col.sourceName === columnName)
-                    ?.ordering ?? 0
-                }
-                onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
-                  handleColumnOrdering(columnName, +e.target.value)
-                }
-              />
-            )}
           </Label>
         }
         action={
           <Checkbox
             style={{ cursor: 'pointer' }}
-            disabled={isPkey || disabled}
+            disabled={isPkey || disabled || partOfOrderingKey}
             checked={!tableRow.exclude.has(columnName)}
             onCheckedChange={(state: boolean) =>
               handleColumnExclusion(columnName, state)

diff --git a/ui/app/mirrors/create/cdc/schemabox.tsx b/ui/app/mirrors/create/cdc/schemabox.tsx
@@ -37,6 +37,7 @@ import {
 
 import { Divider } from '@tremor/react';
 import ReactSelect from 'react-select';
+import SelectSortingKeys from './sortingkey';
 
 interface SchemaBoxProps {
   sourcePeer: string;
@@ -69,7 +70,6 @@ export default function SchemaBox({
   const [tableQuery, setTableQuery] = useState<string>('');
   const [defaultTargetSchema, setDefaultTargetSchema] =
     useState<string>(schema);
-
   const searchedTables = useMemo(() => {
     const tableQueryLower = tableQuery.toLowerCase();
     return rows
@@ -389,25 +389,70 @@ export default function SchemaBox({
                     {row.selected && (
                       <div className='ml-5 mt-3' style={{ width: '100%' }}>
                         <Divider style={columnBoxDividerStyle} />
-                        <Label
-                          as='label'
-                          colorName='lowContrast'
-                          style={{ fontSize: 13 }}
+
+                        <div
+                          style={{
+                            display: 'flex',
+                            flexDirection: 'column',
+                            rowGap: '0.5rem',
+                            width: '100%',
+                          }}
                         >
-                          Columns
-                        </Label>
+                          <Label
+                            as='label'
+                            colorName='lowContrast'
+                            style={{ fontSize: 13 }}
+                          >
+                            Columns
+                          </Label>
+                        </div>
                         {columns ? (
-                          <ColumnBox
-                            columns={columns}
-                            tableRow={row}
-                            rows={rows}
-                            setRows={setRows}
-                            disabled={row.editingDisabled}
-                            showOrdering={
-                              peerType?.toString() ===
-                              DBType[DBType.CLICKHOUSE].toString()
-                            }
-                          />
+                          <>
+                            <ColumnBox
+                              columns={columns}
+                              tableRow={row}
+                              rows={rows}
+                              setRows={setRows}
+                              disabled={row.editingDisabled}
+                              showOrdering={
+                                peerType?.toString() ===
+                                DBType[DBType.CLICKHOUSE].toString()
+                              }
+                            />
+                            {peerType?.toString() ===
+                              DBType[DBType.CLICKHOUSE].toString() && (
+                              <div
+                                style={{
+                                  width: '50%',
+                                  display: 'flex',
+                                  flexDirection: 'column',
+                                  rowGap: '0.5rem',
+                                }}
+                              >
+                                <Divider
+                                  style={{
+                                    ...columnBoxDividerStyle,
+                                    marginTop: '0.5rem',
+                                  }}
+                                />
+                                <SelectSortingKeys
+                                  columns={
+                                    columns?.map((column) => {
+                                      const [
+                                        columnName,
+                                        columnType,
+                                        isPkeyStr,
+                                      ] = column.split(':');
+                                      return columnName;
+                                    }) ?? []
+                                  }
+                                  loading={columnsLoading}
+                                  tableRow={row}
+                                  setRows={setRows}
+                                />
+                              </div>
+                            )}
+                          </>
                         ) : columnsLoading ? (
                           <BarLoader />
                         ) : (