From 0daea3a1db59a3fd8732276d91717556520ea19b Mon Sep 17 00:00:00 2001 From: Ruihao Chen Date: Fri, 17 Jan 2025 14:18:49 +0800 Subject: [PATCH 1/2] Set pebble block size in local sort --- lightning/pkg/importer/table_import.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lightning/pkg/importer/table_import.go b/lightning/pkg/importer/table_import.go index 17f342fc535c9..672f66a794dfb 100644 --- a/lightning/pkg/importer/table_import.go +++ b/lightning/pkg/importer/table_import.go @@ -664,6 +664,9 @@ func (tr *TableImporter) preprocessEngine( logTask := tr.logger.With(zap.Int32("engineNumber", engineID)).Begin(zap.InfoLevel, "encode kv data and write") dataEngineCfg := &backend.EngineConfig{ TableInfo: tr.tableInfo, + Local: backend.LocalEngineConfig{ + BlockSize: int(rc.cfg.TikvImporter.BlockSize), + }, } if !tr.tableMeta.IsRowOrdered { dataEngineCfg.Local.Compact = true From a891740376c1787c27a02826aa8b3f5c71139ee7 Mon Sep 17 00:00:00 2001 From: Ruihao Chen Date: Fri, 17 Jan 2025 15:47:37 +0800 Subject: [PATCH 2/2] Add block size for disk sorter --- pkg/util/extsort/disk_sorter.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/util/extsort/disk_sorter.go b/pkg/util/extsort/disk_sorter.go index 3401e3e4299a6..f0818981b2ef0 100644 --- a/pkg/util/extsort/disk_sorter.go +++ b/pkg/util/extsort/disk_sorter.go @@ -168,6 +168,7 @@ func newSSTWriter( } writable := objstorageprovider.NewFileWritable(f) w := sstable.NewWriter(writable, sstable.WriterOptions{ + BlockSize: 16 * 1024, TablePropertyCollectors: []func() sstable.TablePropertyCollector{ func() sstable.TablePropertyCollector { return newKVStatsCollector(kvStatsBucketSize)