Skip to content

Commit

Permalink
fix: index length is to long
Browse files Browse the repository at this point in the history
  • Loading branch information
@caoxing9
caoxing9 committed Jan 17, 2025
1 parent f06c42c commit 7a12aff
Show file tree
Hide file tree
Showing 9 changed files with 73 additions and 54 deletions.
8 changes: 4 additions & 4 deletions apps/nestjs-backend/src/db-provider/index-query/index-abstract-builder.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { IGetAbnormalVo, TableIndex } from '@teable/openapi';
import type { IGetAbnormalVo } from '@teable/openapi';
import type { IFieldInstance } from '../../features/field/model/factory';

export abstract class IndexBuilderAbstract {
Expand All @@ -8,12 +8,12 @@ export abstract class IndexBuilderAbstract {

abstract getExistTableIndexSql(dbTableName: string): string;

abstract getDeleteSingleIndexSql(dbTableName: string, dbFieldName: string): string;
abstract getDeleteSingleIndexSql(dbTableName: string, field: IFieldInstance): string;

abstract getUpdateSingleIndexNameSql(
dbTableName: string,
oldDbFieldName: string,
newDbFieldName: string
oldField: Pick<IFieldInstance, 'id' | 'dbFieldName'>,
newField: Pick<IFieldInstance, 'id' | 'dbFieldName'>
): string;

abstract createSingleIndexSql(dbTableName: string, field: IFieldInstance): string | null;
Expand Down
34 changes: 22 additions & 12 deletions apps/nestjs-backend/src/db-provider/search-query/search-index-builder.postgres.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,18 +66,28 @@ export class FieldFormatter {
}

export class IndexBuilderPostgres extends IndexBuilderAbstract {
static PG_MAX_INDEX_LEN = 63;
private getIndexPrefix() {
return `idx_trgm`;
}

private getIndexName(table: string, dbFieldName: string): string {
private getIndexName(table: string, field: Pick<IFieldInstance, 'id' | 'dbFieldName'>): string {
const { dbFieldName, id } = field;
const prefix = this.getIndexPrefix();
return `${prefix}_${table}_${dbFieldName}`;
// 3 is space character
const len =
IndexBuilderPostgres.PG_MAX_INDEX_LEN -
id.length -
this.getIndexPrefix().length -
table.length -
3;
const abbDbFieldName = dbFieldName.slice(0, len);
return `${prefix}_${table}_${abbDbFieldName}_${id}`;
}

createSingleIndexSql(dbTableName: string, field: IFieldInstance): string | null {
const [schema, table] = dbTableName.split('.');
const indexName = this.getIndexName(table, field.dbFieldName);
const indexName = this.getIndexName(table, field);
const expression = FieldFormatter.getIndexExpression(field);
if (expression === null) {
return null;
Expand Down Expand Up @@ -131,21 +141,21 @@ export class IndexBuilderPostgres extends IndexBuilderAbstract {
)`;
}

getDeleteSingleIndexSql(dbTableName: string, dbFieldName: string): string {
getDeleteSingleIndexSql(dbTableName: string, field: IFieldInstance): string {
const [schema, table] = dbTableName.split('.');
const indexName = this.getIndexName(table, dbFieldName);
const indexName = this.getIndexName(table, field);

return `DROP INDEX IF EXISTS "${schema}"."${indexName}"`;
}

getUpdateSingleIndexNameSql(
dbTableName: string,
oldDbFieldName: string,
newDbFieldName: string
oldField: Pick<IFieldInstance, 'id' | 'dbFieldName'>,
newField: Pick<IFieldInstance, 'id' | 'dbFieldName'>
): string {
const [schema, table] = dbTableName.split('.');
const oldIndexName = this.getIndexName(table, oldDbFieldName);
const newIndexName = this.getIndexName(table, newDbFieldName);
const oldIndexName = this.getIndexName(table, oldField);
const newIndexName = this.getIndexName(table, newField);

return `
ALTER INDEX IF EXISTS "${schema}"."${oldIndexName}"
Expand All @@ -166,8 +176,8 @@ AND indexname like '%${prefix}_${table}_%'`;
const [, table] = dbTableName.split('.');
const expectExistIndex = fields
.filter(({ cellValueType }) => !unSupportCellValueType.includes(cellValueType))
.map(({ dbFieldName }) => {
return this.getIndexName(table, dbFieldName);
.map((field) => {
return this.getIndexName(table, field);
});

// 1: find the lack or redundant index
Expand All @@ -189,7 +199,7 @@ AND indexname like '%${prefix}_${table}_%'`;
.filter(({ cellValueType }) => !unSupportCellValueType.includes(cellValueType))
.map((f) => {
return {
indexName: this.getIndexName(dbTableName, f.dbFieldName),
indexName: this.getIndexName(dbTableName, f),
indexDef: this.createSingleIndexSql(dbTableName, f) as string,
};
});
Expand Down
7 changes: 4 additions & 3 deletions apps/nestjs-backend/src/db-provider/search-query/search-index-builder.sqlite.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint-disable @typescript-eslint/no-unused-vars */
import { CellValueType } from '@teable/core';
import type { IGetAbnormalVo } from '@teable/openapi';
import type { IFieldInstance } from '../../features/field/model/factory';
Expand Down Expand Up @@ -92,14 +93,14 @@ export class IndexBuilderSqlite extends IndexBuilderAbstract {
)`;
}

getDeleteSingleIndexSql(dbTableName: string, dbFieldName: string): string {
getDeleteSingleIndexSql(dbTableName: string, field: IFieldInstance): string {
return NO_OPERATION_SQL;
}

getUpdateSingleIndexNameSql(
dbTableName: string,
oldDbFieldName: string,
newDbFieldName: string
oldField: IFieldInstance,
newField: IFieldInstance
): string {
return NO_OPERATION_SQL;
}
Expand Down
2 changes: 1 addition & 1 deletion apps/nestjs-backend/src/features/field/field-calculate/field-deleting.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ export class FieldDeletingService {
}

// delete index first
await this.tableIndexService.deleteSearchFieldIndex(tableId, field.dbFieldName);
await this.tableIndexService.deleteSearchFieldIndex(tableId, field);

if (type === FieldType.Link && !isLookup) {
const linkFieldOptions = field.options;
Expand Down
12 changes: 6 additions & 6 deletions apps/nestjs-backend/src/features/field/open-api/field-open-api.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,14 +373,14 @@ export class FieldOpenApiService {
},
select: {
dbFieldName: true,
id: true,
},
});
// do not need in transaction, causing just index name
await this.tableIndexService.updateSearchFieldIndexName(
tableId,
oldField.dbFieldName,
updateFieldRo.dbFieldName
);
await this.tableIndexService.updateSearchFieldIndexName(tableId, oldField, {
id: oldField.id,
dbFieldName: updateFieldRo?.dbFieldName ?? oldField.dbFieldName,
});
ops.push(op);
}

Expand Down Expand Up @@ -443,7 +443,7 @@ export class FieldOpenApiService {
// 3. stage apply record changes and calculate field
await this.prismaService.$tx(
async () => {
await this.tableIndexService.deleteSearchFieldIndex(tableId, oldField.dbFieldName);
await this.tableIndexService.deleteSearchFieldIndex(tableId, oldField);
await this.fieldConvertingService.stageCalculate(tableId, newField, oldField, modifiedOps);

if (supplementChange) {
Expand Down
2 changes: 0 additions & 2 deletions apps/nestjs-backend/src/features/record/record.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -430,8 +430,6 @@ export class RecordService {
}
});

console.log('ooooooooo', [searchValue, fieldIdOrName, hideNotMatchRow]);

return [searchValue, fieldIdOrName, hideNotMatchRow];
}

Expand Down
1 change: 0 additions & 1 deletion apps/nestjs-backend/src/features/table/open-api/table-open-api.controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import {
toggleIndexRoSchema,
TableIndex,
} from '@teable/openapi';
import { Timing } from '../../../utils/timing';
import { ZodValidationPipe } from '../../../zod.validation.pipe';
import { Permissions } from '../../auth/decorators/permissions.decorator';
import { TableIndexService } from '../table-index.service';
Expand Down
32 changes: 17 additions & 15 deletions apps/nestjs-backend/src/features/table/table-index.service.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { BadRequestException, Injectable } from '@nestjs/common';
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
import { CellValueType } from '@teable/core';
import { PrismaService } from '@teable/db-main-prisma';
import { TableIndex } from '@teable/openapi';
Expand All @@ -17,6 +17,8 @@ const unSupportTableIndex = 'Unsupport table index type';

@Injectable()
export class TableIndexService {
private logger = new Logger(TableIndexService.name);

constructor(
private readonly cls: ClsService<IClsStore>,
private readonly prismaService: PrismaService,
Expand Down Expand Up @@ -118,15 +120,15 @@ export class TableIndexService {
}
}

async deleteSearchFieldIndex(tableId: string, dbFieldName: string) {
async deleteSearchFieldIndex(tableId: string, field: IFieldInstance) {
const tableRaw = await this.prismaService.txClient().tableMeta.findFirstOrThrow({
where: { id: tableId, deletedTime: null },
select: { dbTableName: true },
});
const { dbTableName } = tableRaw;
const index = await this.getActivatedTableIndexes(tableId);
if (index.includes(TableIndex.search)) {
const sql = this.dbProvider.searchIndex().getDeleteSingleIndexSql(dbTableName, dbFieldName);
const sql = this.dbProvider.searchIndex().getDeleteSingleIndexSql(dbTableName, field);
await this.prismaService.$executeRawUnsafe(sql);
}
}
Expand All @@ -146,8 +148,8 @@ export class TableIndexService {

async updateSearchFieldIndexName(
tableId: string,
oldDbFieldName: string,
newDbFieldName: string
oldField: Pick<IFieldInstance, 'id' | 'dbFieldName'>,
newField: Pick<IFieldInstance, 'id' | 'dbFieldName'>
) {
const tableRaw = await this.prismaService.txClient().tableMeta.findFirstOrThrow({
where: { id: tableId, deletedTime: null },
Expand All @@ -158,7 +160,7 @@ export class TableIndexService {
if (index.includes(TableIndex.search)) {
const sql = this.dbProvider
.searchIndex()
.getUpdateSingleIndexNameSql(dbTableName, oldDbFieldName, newDbFieldName);
.getUpdateSingleIndexNameSql(dbTableName, oldField, newField);
await this.prismaService.$executeRawUnsafe(sql);
}
}
Expand Down Expand Up @@ -238,14 +240,14 @@ export class TableIndexService {
isStructuredCellValue: field.isStructuredCellValue,
})) as IFieldInstance[];
const createSqls = this.dbProvider.searchIndex().getCreateIndexSql(dbTableName, fieldInstances);
await this.prismaService.$tx(async (prisma) => {
await prisma.$executeRawUnsafe(dropSql);
for (let i = 0; i < createSqls.length; i++) {
await prisma.$executeRawUnsafe(createSqls[i]);
}
{
this.thresholdConfig.bigTransactionTimeout;
}
});
await this.prismaService.$tx(
async (prisma) => {
await prisma.$executeRawUnsafe(dropSql);
for (let i = 0; i < createSqls.length; i++) {
await prisma.$executeRawUnsafe(createSqls[i]);
}
},
{ timeout: this.thresholdConfig.bigTransactionTimeout }
);
}
}
29 changes: 19 additions & 10 deletions apps/nestjs-backend/test/record-search-query.e2e-spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
convertField,
} from '@teable/openapi';
import { difference, differenceWith, isEqual } from 'lodash';
import type { IFieldInstance } from '../src/features/field/model/factory';
import { x_20 } from './data-helpers/20x';
import { x_20_link, x_20_link_from_lookups } from './data-helpers/20x-link';
import {
Expand Down Expand Up @@ -235,12 +236,16 @@ describe('OpenAPI Record-Search-Query (e2e)', async () => {
'search index relative',
() => {
let table: ITableFullVo;
let tableName: string;
let dbFieldNameIndexLen: number;
beforeEach(async () => {
table = await createTable(baseId, {
name: 'record_query_x_20',
fields: x_20.fields,
records: x_20.records,
});
tableName = table?.dbTableName?.split('.').pop() as string;
dbFieldNameIndexLen = 63 - 'idx_trgm'.length - tableName.length - 22;
});

afterEach(async () => {
Expand All @@ -257,36 +262,40 @@ describe('OpenAPI Record-Search-Query (e2e)', async () => {
});

it('should get abnormal index list', async () => {
const textfield = table.fields.find((f) => f.cellValueType === CellValueType.String)!;
const textfield = table.fields.find(
(f) => f.cellValueType === CellValueType.String
)! as IFieldInstance;
// enable search index
await toggleTableIndex(baseId, table.id, { type: TableIndex.search });

// delete or update abnormal index
const tableIndexService = await getTableIndexService(app);
await tableIndexService.deleteSearchFieldIndex(table.id, textfield.dbFieldName);
await tableIndexService.deleteSearchFieldIndex(table.id, textfield);

// expect get the abnormal list
const result = await getTableAbnormalIndex(baseId, table.id, TableIndex.search);
expect(result.data.length).toBe(1);
expect(result.data[0]).toEqual({
indexName: `idx_trgm_${table.dbTableName.split('.').pop()}_${textfield.dbFieldName}`,
indexName: `idx_trgm_${tableName}_${textfield.dbFieldName.slice(0, dbFieldNameIndexLen)}_${textfield.id}`,
});
});

it('should repair abnormal index', async () => {
const textfield = table.fields.find((f) => f.cellValueType === CellValueType.String)!;
const textfield = table.fields.find(
(f) => f.cellValueType === CellValueType.String
)! as IFieldInstance;
// enable search index
await toggleTableIndex(baseId, table.id, { type: TableIndex.search });

// delete or update abnormal index
const tableIndexService = await getTableIndexService(app);
await tableIndexService.deleteSearchFieldIndex(table.id, textfield.dbFieldName);
await tableIndexService.deleteSearchFieldIndex(table.id, textfield);

// expect get the abnormal list
const result = await getTableAbnormalIndex(baseId, table.id, TableIndex.search);
expect(result.data.length).toBe(1);
expect(result.data[0]).toEqual({
indexName: `idx_trgm_${table.dbTableName.split('.').pop()}_${textfield.dbFieldName}`,
indexName: `idx_trgm_${tableName}_${textfield.dbFieldName.slice(0, dbFieldNameIndexLen)}_${textfield.id}`,
});

await repairTableIndex(baseId, table.id, TableIndex.search);
Expand All @@ -308,7 +317,7 @@ describe('OpenAPI Record-Search-Query (e2e)', async () => {
const index2 = (await tableIndexService.getIndexInfo(table.id)) as { indexname: string }[];
const diffIndex = differenceWith(index, index2, (a, b) => a?.indexname === b?.indexname);
expect(diffIndex[0]?.indexname).toEqual(
`idx_trgm_${table.dbTableName.split('.').pop()}_${textfield.dbFieldName}`
`idx_trgm_${tableName}_${textfield.dbFieldName.slice(0, dbFieldNameIndexLen)}_${textfield.id}`
);
const result2 = await getTableAbnormalIndex(baseId, table.id, TableIndex.search);
expect(result2.data.length).toBe(0);
Expand All @@ -325,7 +334,7 @@ describe('OpenAPI Record-Search-Query (e2e)', async () => {
const index2 = (await tableIndexService.getIndexInfo(table.id)) as { indexname: string }[];
const diffIndex = differenceWith(index2, index, (a, b) => a?.indexname === b?.indexname);
expect(diffIndex[0]?.indexname).toEqual(
`idx_trgm_${table.dbTableName.split('.').pop()}_${newField?.data?.dbFieldName}`
`idx_trgm_${tableName}_${newField?.data?.dbFieldName.slice(0, dbFieldNameIndexLen)}_${newField?.data?.id}`
);
const result2 = await getTableAbnormalIndex(baseId, table.id, TableIndex.search);
expect(result2.data.length).toBe(0);
Expand All @@ -344,7 +353,7 @@ describe('OpenAPI Record-Search-Query (e2e)', async () => {
const index2 = (await tableIndexService.getIndexInfo(table.id)) as { indexname: string }[];
const diffIndex = differenceWith(index, index2, (a, b) => a?.indexname === b?.indexname);
expect(diffIndex[0]?.indexname).toEqual(
`idx_trgm_${table.dbTableName.split('.').pop()}_${textfield?.dbFieldName}`
`idx_trgm_${tableName}_${textfield.dbFieldName.slice(0, dbFieldNameIndexLen)}_${textfield.id}`
);

const result2 = await getTableAbnormalIndex(baseId, table.id, TableIndex.search);
Expand All @@ -364,7 +373,7 @@ describe('OpenAPI Record-Search-Query (e2e)', async () => {
const index2 = (await tableIndexService.getIndexInfo(table.id)) as { indexname: string }[];
const diffIndex = differenceWith(index2, index, (a, b) => a?.indexname === b?.indexname);
expect(diffIndex[0]?.indexname).toEqual(
`idx_trgm_${table.dbTableName.split('.').pop()}_Test_Field`
`idx_trgm_${tableName}_${textfield?.dbFieldName.slice(0, dbFieldNameIndexLen)}_${textfield.id}`
);
const result2 = await getTableAbnormalIndex(baseId, table.id, TableIndex.search);
expect(result2.data.length).toBe(0);
Expand Down

0 comments on commit 7a12aff

Please sign in to comment.