Skip to content

Commit

Permalink
Apply patch: Add partial index iterator
Browse files Browse the repository at this point in the history 
Summary:
This adds the partial index iterator. It is a special iterator that sorts groups from the primary key on the fly as needed, and if it exceeds a certain threshold, it will materialize the rows on the secondary key as well.

For point queries, the secondary key read is not found, we simply extract the primary key parts and read from the primary key. This means that point queries don't trigger materialization though.

Test Plan: mtr

Reviewers: luqun, herman, yzha, #mysql_eng

Subscribers: phabricatorlinter, pgl

Differential Revision: https://phabricator.intern.facebook.com/D25933178
  • Loading branch information
@spetrunia
spetrunia committed May 17, 2021
1 parent aaebd62 commit d1d0c15
Show file tree
Hide file tree
Showing 18 changed files with 1,672 additions and 8 deletions.
14 changes: 14 additions & 0 deletions mysql-test/include/diff_queries.inc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
--disable_query_log

--output $MYSQL_TMP_DIR/A
--eval $query1

--output $MYSQL_TMP_DIR/B
--eval $query2

--enable_query_log

--diff_files $MYSQL_TMP_DIR/A $MYSQL_TMP_DIR/B

--remove_file $MYSQL_TMP_DIR/A
--remove_file $MYSQL_TMP_DIR/B
146 changes: 146 additions & 0 deletions mysql-test/suite/rocksdb/r/partial_index.result
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
set optimizer_force_index_for_range = on;
CREATE TABLE t (i varchar(64), j varchar(64), k varchar(64), l varchar(64),
PRIMARY KEY (i, j),
KEY ik1 (i, k) COMMENT 'cfname=cf;partial_group_keyparts=1;partial_group_threshold=5',
KEY ik2 (i, k) COMMENT 'cfname=cf'
) ENGINE=ROCKSDB;
Warnings:
Warning 1831 Duplicate index 'ik2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
INSERT INTO t values ("1", "1", "2", "1");
INSERT INTO t values ("1", "2", "1", "1");
INSERT INTO t values ("11111111", "1", "9", "1");
INSERT INTO t values ("11111111", "2", "8", "1");
INSERT INTO t values ("11111111", "3", "7", "1");
INSERT INTO t values ("11111111", "4", "5", "1");
INSERT INTO t values ("11111111", "5", "4", "1");
INSERT INTO t values ("11111111", "6", "2", "1");
INSERT INTO t values ("111111111", "1", "9", "1");
INSERT INTO t values ("111111111", "2", "2", "1");
INSERT INTO t values ("11111112", "1", "1", "1");
DROP TABLE t;
CREATE TABLE t (i int, j int, k int, l int,
PRIMARY KEY (i, j),
KEY ik1 (i, k) COMMENT 'cfname=cf;partial_group_keyparts=1;partial_group_threshold=5',
KEY ik2 (i, k) COMMENT 'cfname=cf'
) ENGINE=ROCKSDB;
Warnings:
Warning 1831 Duplicate index 'ik2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
INSERT INTO t values (1, 1, 2, 1);
INSERT INTO t values (1, 2, 1, 1);
INSERT INTO t values (2, 1, 9, 1);
INSERT INTO t values (2, 2, 8, 1);
INSERT INTO t values (2, 3, 7, 1);
INSERT INTO t values (2, 4, 5, 1);
INSERT INTO t values (2, 5, 4, 1);
INSERT INTO t values (2, 6, 2, 1);
INSERT INTO t values (4, 1, 1, 1);
DROP TABLE t;
CREATE TABLE t (i varchar(64), j varchar(64), k varchar(64), l varchar(64),
PRIMARY KEY (i, j),
KEY ik1 (i, k) COMMENT 'cfname=rev:cf;partial_group_keyparts=1;partial_group_threshold=5',
KEY ik2 (i, k) COMMENT 'cfname=rev:cf'
) ENGINE=ROCKSDB;
Warnings:
Warning 1831 Duplicate index 'ik2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
INSERT INTO t values ("1", "1", "2", "1");
INSERT INTO t values ("1", "2", "1", "1");
INSERT INTO t values ("11111111", "1", "9", "1");
INSERT INTO t values ("11111111", "2", "8", "1");
INSERT INTO t values ("11111111", "3", "7", "1");
INSERT INTO t values ("11111111", "4", "5", "1");
INSERT INTO t values ("11111111", "5", "4", "1");
INSERT INTO t values ("11111111", "6", "2", "1");
INSERT INTO t values ("111111111", "1", "9", "1");
INSERT INTO t values ("111111111", "2", "2", "1");
INSERT INTO t values ("11111112", "1", "1", "1");
DROP TABLE t;
CREATE TABLE t (i int, j int, k int, l int,
PRIMARY KEY (i, j),
KEY ik1 (i, k) COMMENT 'cfname=rev:cf;partial_group_keyparts=1;partial_group_threshold=5',
KEY ik2 (i, k) COMMENT 'cfname=rev:cf'
) ENGINE=ROCKSDB;
Warnings:
Warning 1831 Duplicate index 'ik2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
INSERT INTO t values (1, 1, 2, 1);
INSERT INTO t values (1, 2, 1, 1);
INSERT INTO t values (2, 1, 9, 1);
INSERT INTO t values (2, 2, 8, 1);
INSERT INTO t values (2, 3, 7, 1);
INSERT INTO t values (2, 4, 5, 1);
INSERT INTO t values (2, 5, 4, 1);
INSERT INTO t values (2, 6, 2, 1);
INSERT INTO t values (4, 1, 1, 1);
DROP TABLE t;
CREATE TABLE t (i varchar(64), j varchar(64), k varchar(64), l varchar(64),
PRIMARY KEY (i, j),
KEY ik1 (i, k) COMMENT 'cfname=cf;partial_group_keyparts=1;partial_group_threshold=5',
KEY ik2 (i, k) COMMENT 'cfname=cf'
) ENGINE=ROCKSDB;
Warnings:
Warning 1831 Duplicate index 'ik2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
INSERT INTO t values ("1", "1", "2", "1");
INSERT INTO t values ("1", "2", "1", "1");
INSERT INTO t values ("11111111", "1", "9", "1");
INSERT INTO t values ("11111111", "2", "8", "1");
INSERT INTO t values ("11111111", "3", "7", "1");
INSERT INTO t values ("11111111", "4", "5", "1");
INSERT INTO t values ("11111111", "5", "4", "1");
INSERT INTO t values ("11111111", "6", "2", "1");
INSERT INTO t values ("111111111", "1", "9", "1");
INSERT INTO t values ("111111111", "2", "2", "1");
INSERT INTO t values ("11111112", "1", "1", "1");
DROP TABLE t;
CREATE TABLE t (i int, j int, k int, l int,
PRIMARY KEY (i, j),
KEY ik1 (i, k) COMMENT 'cfname=cf;partial_group_keyparts=1;partial_group_threshold=5',
KEY ik2 (i, k) COMMENT 'cfname=cf'
) ENGINE=ROCKSDB;
Warnings:
Warning 1831 Duplicate index 'ik2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
INSERT INTO t values (1, 1, 2, 1);
INSERT INTO t values (1, 2, 1, 1);
INSERT INTO t values (2, 1, 9, 1);
INSERT INTO t values (2, 2, 8, 1);
INSERT INTO t values (2, 3, 7, 1);
INSERT INTO t values (2, 4, 5, 1);
INSERT INTO t values (2, 5, 4, 1);
INSERT INTO t values (2, 6, 2, 1);
INSERT INTO t values (4, 1, 1, 1);
DROP TABLE t;
CREATE TABLE t (i varchar(64), j varchar(64), k varchar(64), l varchar(64),
PRIMARY KEY (i, j),
KEY ik1 (i, k) COMMENT 'cfname=rev:cf;partial_group_keyparts=1;partial_group_threshold=5',
KEY ik2 (i, k) COMMENT 'cfname=rev:cf'
) ENGINE=ROCKSDB;
Warnings:
Warning 1831 Duplicate index 'ik2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
INSERT INTO t values ("1", "1", "2", "1");
INSERT INTO t values ("1", "2", "1", "1");
INSERT INTO t values ("11111111", "1", "9", "1");
INSERT INTO t values ("11111111", "2", "8", "1");
INSERT INTO t values ("11111111", "3", "7", "1");
INSERT INTO t values ("11111111", "4", "5", "1");
INSERT INTO t values ("11111111", "5", "4", "1");
INSERT INTO t values ("11111111", "6", "2", "1");
INSERT INTO t values ("111111111", "1", "9", "1");
INSERT INTO t values ("111111111", "2", "2", "1");
INSERT INTO t values ("11111112", "1", "1", "1");
DROP TABLE t;
CREATE TABLE t (i int, j int, k int, l int,
PRIMARY KEY (i, j),
KEY ik1 (i, k) COMMENT 'cfname=rev:cf;partial_group_keyparts=1;partial_group_threshold=5',
KEY ik2 (i, k) COMMENT 'cfname=rev:cf'
) ENGINE=ROCKSDB;
Warnings:
Warning 1831 Duplicate index 'ik2' defined on the table 'test.t'. This is deprecated and will be disallowed in a future release.
INSERT INTO t values (1, 1, 2, 1);
INSERT INTO t values (1, 2, 1, 1);
INSERT INTO t values (2, 1, 9, 1);
INSERT INTO t values (2, 2, 8, 1);
INSERT INTO t values (2, 3, 7, 1);
INSERT INTO t values (2, 4, 5, 1);
INSERT INTO t values (2, 5, 4, 1);
INSERT INTO t values (2, 6, 2, 1);
INSERT INTO t values (4, 1, 1, 1);
DROP TABLE t;
set optimizer_force_index_for_range = off;
171 changes: 171 additions & 0 deletions mysql-test/suite/rocksdb/r/partial_index_assoc.result
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
set optimizer_force_index_for_range = on;
CREATE TABLE `assoc_table` (
`id1` bigint(20) unsigned NOT NULL DEFAULT '0',
`id1_type` int(10) unsigned NOT NULL DEFAULT '0',
`id2` bigint(20) unsigned NOT NULL DEFAULT '0',
`id2_type` int(10) unsigned NOT NULL DEFAULT '0',
`assoc_type` bigint(20) unsigned NOT NULL DEFAULT '0',
`visibility` tinyint(3) NOT NULL DEFAULT '0',
`data` varchar(255) COLLATE latin1_bin NOT NULL DEFAULT '',
`time` int(10) unsigned NOT NULL DEFAULT '0',
`version` bigint(20) unsigned NOT NULL DEFAULT '0',
PRIMARY KEY (`assoc_type`,`id1`,`id2`) COMMENT 'cf_assoc',
KEY `id1_type` (`assoc_type`, `id1`, `visibility`,`time`,`id2`,`version`,`data`) COMMENT 'rev:cf_assoc_id1_type',
KEY `id1_type2` (`assoc_type`, `id1`, `visibility`,`time`,`id2`,`version`,`data`) COMMENT 'cfname=rev:cf_assoc_id1_type;partial_group_keyparts=2;partial_group_threshold=10'
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8;
Warnings:
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1831 Duplicate index 'id1_type2' defined on the table 'test.assoc_table'. This is deprecated and will be disallowed in a future release.
ALTER TABLE assoc_table ENGINE=ROCKSDB;
CREATE TEMPORARY TABLE t1 AS
SELECT * FROM performance_schema.global_status
WHERE variable_name LIKE 'rocksdb_partial_index%';
CREATE TEMPORARY TABLE t2 AS
SELECT * FROM performance_schema.global_status
WHERE variable_name LIKE 'rocksdb_partial_index%';
SELECT variable_name, t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name);
variable_name diff
rocksdb_partial_index_groups_materialized 40
rocksdb_partial_index_groups_sorted 47
rocksdb_partial_index_rows_materialized 955
rocksdb_partial_index_rows_sorted 1000
include/assert.inc [Check that materialized groups are non-zero.]
include/assert.inc [Check that materialized rows are non-zero.]
DROP TABLE t1, t2;
CREATE TEMPORARY TABLE t1 AS
SELECT * FROM performance_schema.global_status
WHERE variable_name LIKE 'rocksdb_partial_index%';
SELECT id1, id2, assoc_type, visibility, time, version FROM assoc_table FORCE INDEX (id1_type2);
CREATE TEMPORARY TABLE t2 AS
SELECT * FROM performance_schema.global_status
WHERE variable_name LIKE 'rocksdb_partial_index%';
SELECT variable_name, t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name);
variable_name diff
rocksdb_partial_index_groups_materialized 0
rocksdb_partial_index_groups_sorted 7
rocksdb_partial_index_rows_materialized 0
rocksdb_partial_index_rows_sorted 45
include/assert.inc [Check that materialized groups are zero.]
include/assert.inc [Check that materialized rows are zero.]
DROP TABLE t1, t2;
DROP TABLE assoc_table;
CREATE TABLE `assoc_table` (
`id1` binary(16) NOT NULL DEFAULT '\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0',
`raw_key` text COLLATE latin1_bin,
`id2` bigint(20) unsigned NOT NULL DEFAULT '0',
`id2_type` int(10) unsigned NOT NULL DEFAULT '0',
`assoc_type` bigint(20) unsigned NOT NULL DEFAULT '0',
`visibility` tinyint(3) NOT NULL DEFAULT '0',
`data` varchar(255) COLLATE latin1_bin NOT NULL DEFAULT '',
`time` int(10) unsigned NOT NULL DEFAULT '0',
`version` bigint(20) unsigned NOT NULL DEFAULT '0',
PRIMARY KEY (`assoc_type`,`id1`,`id2`) COMMENT 'cf_assoc',
KEY `id1_type` (`assoc_type`,`id1`,`visibility`,`time`,`id2`,`version`,`data`) COMMENT 'rev:cf_assoc_id1_type',
KEY `id1_type2` (`assoc_type`, `id1`, `visibility`,`time`,`id2`,`version`,`data`) COMMENT 'cfname=rev:cf_assoc_id1_type;partial_group_keyparts=2;partial_group_threshold=10'
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=8;
Warnings:
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1831 Duplicate index 'id1_type2' defined on the table 'test.assoc_table'. This is deprecated and will be disallowed in a future release.
ALTER TABLE assoc_table ENGINE=ROCKSDB;
CREATE TEMPORARY TABLE t1 AS
SELECT * FROM performance_schema.global_status
WHERE variable_name LIKE 'rocksdb_partial_index%';
CREATE TEMPORARY TABLE t2 AS
SELECT * FROM performance_schema.global_status
WHERE variable_name LIKE 'rocksdb_partial_index%';
SELECT variable_name, t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name);
variable_name diff
rocksdb_partial_index_groups_materialized 40
rocksdb_partial_index_groups_sorted 47
rocksdb_partial_index_rows_materialized 955
rocksdb_partial_index_rows_sorted 1000
include/assert.inc [Check that materialized groups are non-zero.]
include/assert.inc [Check that materialized rows are non-zero.]
DROP TABLE t1, t2;
CREATE TEMPORARY TABLE t1 AS
SELECT * FROM performance_schema.global_status
WHERE variable_name LIKE 'rocksdb_partial_index%';
SELECT id1, id2, assoc_type, visibility, time, version FROM assoc_table FORCE INDEX (id1_type2);
CREATE TEMPORARY TABLE t2 AS
SELECT * FROM performance_schema.global_status
WHERE variable_name LIKE 'rocksdb_partial_index%';
SELECT variable_name, t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name);
variable_name diff
rocksdb_partial_index_groups_materialized 0
rocksdb_partial_index_groups_sorted 7
rocksdb_partial_index_rows_materialized 0
rocksdb_partial_index_rows_sorted 45
include/assert.inc [Check that materialized groups are zero.]
include/assert.inc [Check that materialized rows are zero.]
DROP TABLE t1, t2;
DROP TABLE assoc_table;
CREATE TABLE `assoc_table` (
`id1` bigint(20) unsigned NOT NULL DEFAULT '0',
`id1_type` int(10) unsigned NOT NULL DEFAULT '0',
`id2` bigint(20) unsigned NOT NULL DEFAULT '0',
`id2_type` int(10) unsigned NOT NULL DEFAULT '0',
`assoc_type` bigint(20) unsigned NOT NULL DEFAULT '0',
`visibility` tinyint(4) NOT NULL DEFAULT '0',
`data` text COLLATE latin1_bin NOT NULL,
`time` int(10) unsigned NOT NULL DEFAULT '0',
`version` bigint(20) unsigned NOT NULL DEFAULT '0',
PRIMARY KEY (`assoc_type`,`id1`,`id2`) COMMENT 'cf_assoc',
KEY `id1_type` (`assoc_type`, `id1`, `visibility`,`time`,`id2`,`version`,`data`(255)) COMMENT 'rev:cf_assoc_id1_type',
KEY `id1_type2` (`assoc_type`, `id1`, `visibility`,`time`,`id2`,`version`,`data`(255)) COMMENT 'cfname=rev:cf_assoc_id1_type;partial_group_keyparts=2;partial_group_threshold=10'
) ENGINE=ROCKSDB DEFAULT CHARSET=latin1 COLLATE=latin1_bin ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=4;
Warnings:
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1681 Integer display width is deprecated and will be removed in a future release.
Warning 1831 Duplicate index 'id1_type2' defined on the table 'test.assoc_table'. This is deprecated and will be disallowed in a future release.
ALTER TABLE assoc_table ENGINE=ROCKSDB;
CREATE TEMPORARY TABLE t1 AS
SELECT * FROM performance_schema.global_status
WHERE variable_name LIKE 'rocksdb_partial_index%';
CREATE TEMPORARY TABLE t2 AS
SELECT * FROM performance_schema.global_status
WHERE variable_name LIKE 'rocksdb_partial_index%';
SELECT variable_name, t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name);
variable_name diff
rocksdb_partial_index_groups_materialized 40
rocksdb_partial_index_groups_sorted 47
rocksdb_partial_index_rows_materialized 955
rocksdb_partial_index_rows_sorted 1000
include/assert.inc [Check that materialized groups are non-zero.]
include/assert.inc [Check that materialized rows are non-zero.]
DROP TABLE t1, t2;
CREATE TEMPORARY TABLE t1 AS
SELECT * FROM performance_schema.global_status
WHERE variable_name LIKE 'rocksdb_partial_index%';
SELECT id1, id2, assoc_type, visibility, time, version FROM assoc_table FORCE INDEX (id1_type2);
CREATE TEMPORARY TABLE t2 AS
SELECT * FROM performance_schema.global_status
WHERE variable_name LIKE 'rocksdb_partial_index%';
SELECT variable_name, t2.variable_value - t1.variable_value AS diff FROM t1 JOIN t2 USING (variable_name);
variable_name diff
rocksdb_partial_index_groups_materialized 0
rocksdb_partial_index_groups_sorted 7
rocksdb_partial_index_rows_materialized 0
rocksdb_partial_index_rows_sorted 45
include/assert.inc [Check that materialized groups are zero.]
include/assert.inc [Check that materialized rows are zero.]
DROP TABLE t1, t2;
DROP TABLE assoc_table;
set optimizer_force_index_for_range = off;
Loading

0 comments on commit d1d0c15

Please sign in to comment.