Skip to content

Commit

Permalink
Fix mysql longblob wrong column type returned by proxy protocol
Browse files Browse the repository at this point in the history
  • Loading branch information
strongduanmu committed Dec 22, 2024
1 parent bfde8d8 commit 8d80b59
Show file tree
Hide file tree
Showing 10 changed files with 224 additions and 11 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,9 @@ public enum MySQLBinaryColumnType implements BinaryColumnType {
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.DATE, DATE);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.TIME, TIME);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.TIMESTAMP, TIMESTAMP);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.BINARY, STRING);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.VARBINARY, VAR_STRING);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.LONGVARBINARY, VAR_STRING);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.BINARY, LONG_BLOB);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.VARBINARY, TINY_BLOB);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.LONGVARBINARY, BLOB);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.NULL, NULL);
JDBC_TYPE_AND_COLUMN_TYPE_MAP.put(Types.BLOB, BLOB);
for (MySQLBinaryColumnType each : values()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ public enum MySQLCharacterSet {
UTF32_GENERAL_CI(60, () -> Charset.forName("utf32")),
UTF32_BIN(61, () -> Charset.forName("utf32")),
UTF16LE_BIN(62, () -> StandardCharsets.UTF_16LE),
BINARY(63, () -> Charset.forName("binary")),
ARMSCII8_BIN(64, () -> Charset.forName("armscii8")),
ASCII_BIN(65, () -> StandardCharsets.US_ASCII),
CP1250_BIN(66, () -> Charset.forName("cp1250")),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,9 @@ void assertValueOfJDBC() {
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.DATE), is(MySQLBinaryColumnType.DATE));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.TIME), is(MySQLBinaryColumnType.TIME));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.TIMESTAMP), is(MySQLBinaryColumnType.TIMESTAMP));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.BINARY), is(MySQLBinaryColumnType.STRING));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.VARBINARY), is(MySQLBinaryColumnType.VAR_STRING));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.LONGVARBINARY), is(MySQLBinaryColumnType.VAR_STRING));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.BINARY), is(MySQLBinaryColumnType.LONG_BLOB));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.VARBINARY), is(MySQLBinaryColumnType.TINY_BLOB));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.LONGVARBINARY), is(MySQLBinaryColumnType.BLOB));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.NULL), is(MySQLBinaryColumnType.NULL));
assertThat(MySQLBinaryColumnType.valueOfJDBCType(Types.BLOB), is(MySQLBinaryColumnType.BLOB));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import org.apache.shardingsphere.db.protocol.mysql.constant.MySQLBinaryColumnType;
import org.apache.shardingsphere.db.protocol.mysql.constant.MySQLCharacterSet;
import org.apache.shardingsphere.db.protocol.mysql.packet.command.query.MySQLColumnDefinition41Packet;
import org.apache.shardingsphere.db.protocol.mysql.packet.command.query.MySQLColumnDefinitionFlag;
import org.apache.shardingsphere.db.protocol.mysql.packet.command.query.MySQLFieldCountPacket;
Expand All @@ -30,8 +31,11 @@
import org.apache.shardingsphere.proxy.backend.response.header.query.QueryResponseHeader;
import org.apache.shardingsphere.proxy.backend.response.header.update.UpdateResponseHeader;

import java.sql.Types;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;

Expand All @@ -45,19 +49,22 @@ public final class ResponsePacketBuilder {

private static final String BLOB_COLUMN_TYPE_KEYWORD = "BLOB";

private static final Collection<Integer> BINARY_TYPES = new HashSet<>(Arrays.asList(Types.BINARY, Types.VARBINARY, Types.LONGVARBINARY));

/**
* Build query response packets.
*
* @param queryResponseHeader query response header
* @param characterSet MySQL character set id
* @param sessionCharacterSet MySQL character set id
* @param statusFlags server status flags
* @return query response packets
*/
public static Collection<DatabasePacket> buildQueryResponsePackets(final QueryResponseHeader queryResponseHeader, final int characterSet, final int statusFlags) {
public static Collection<DatabasePacket> buildQueryResponsePackets(final QueryResponseHeader queryResponseHeader, final int sessionCharacterSet, final int statusFlags) {
Collection<DatabasePacket> result = new LinkedList<>();
List<QueryHeader> queryHeaders = queryResponseHeader.getQueryHeaders();
result.add(new MySQLFieldCountPacket(queryHeaders.size()));
for (QueryHeader each : queryHeaders) {
int characterSet = BINARY_TYPES.contains(each.getColumnType()) ? MySQLCharacterSet.BINARY.getId() : sessionCharacterSet;
result.add(new MySQLColumnDefinition41Packet(characterSet, getColumnDefinitionFlag(each), each.getSchema(), each.getTable(), each.getTable(),
each.getColumnLabel(), each.getColumnName(), each.getColumnLength(), MySQLBinaryColumnType.valueOfJDBCType(each.getColumnType()), each.getDecimals(), false));
}
Expand All @@ -82,6 +89,9 @@ private static int getColumnDefinitionFlag(final QueryHeader header) {
if (header.getColumnTypeName().contains(BINARY_COLUMN_TYPE_KEYWORD) || header.getColumnTypeName().contains(BLOB_COLUMN_TYPE_KEYWORD)) {
result += MySQLColumnDefinitionFlag.BINARY_COLLATION.getValue();
}
if (BINARY_TYPES.contains(header.getColumnType())) {
result += MySQLColumnDefinitionFlag.BLOB.getValue();
}
return result;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@
import org.apache.shardingsphere.test.e2e.cases.dataset.metadata.DataSetColumn;
import org.apache.shardingsphere.test.e2e.cases.dataset.metadata.DataSetMetaData;
import org.apache.shardingsphere.test.e2e.cases.dataset.row.DataSetRow;
import org.apache.shardingsphere.test.e2e.env.E2EEnvironmentAware;
import org.apache.shardingsphere.test.e2e.env.E2EEnvironmentEngine;
import org.apache.shardingsphere.test.e2e.engine.context.E2ETestContext;
import org.apache.shardingsphere.test.e2e.env.DataSetEnvironmentManager;
import org.apache.shardingsphere.test.e2e.env.E2EEnvironmentAware;
import org.apache.shardingsphere.test.e2e.env.E2EEnvironmentEngine;
import org.apache.shardingsphere.test.e2e.env.runtime.scenario.path.ScenarioDataPath;
import org.apache.shardingsphere.test.e2e.env.runtime.scenario.path.ScenarioDataPath.Type;
import org.apache.shardingsphere.test.e2e.framework.param.model.AssertionTestParameter;
Expand Down Expand Up @@ -136,7 +136,8 @@ private void assertMetaData(final ResultSetMetaData actualResultSetMetaData, fin
if ("db_tbl_sql_federation".equals(testParam.getScenario())) {
continue;
}
if ("jdbc".equals(testParam.getAdapter()) && "Cluster".equals(testParam.getMode()) && "encrypt".equals(testParam.getScenario())) {
if ("jdbc".equals(testParam.getAdapter()) && "Cluster".equals(testParam.getMode()) && "encrypt".equals(testParam.getScenario())
|| "passthrough".equals(testParam.getScenario())) {
// FIXME correct columnType with proxy adapter and other jdbc scenario
assertThat(actualResultSetMetaData.getColumnType(i + 1), is(expectedResultSetMetaData.getColumnType(i + 1)));
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<!-- SPEX ADDED: BEGIN -->
<e2e-test-cases>
<!--<test-case sql="SELECT * FROM t_data_type_binary WHERE id = ?" db-types="MySQL" scenario-types="passthrough">
<assertion parameters="1:int" expected-data-source-name="expected_dataset" />
</test-case>-->

<test-case sql="SELECT * FROM t_data_type_varbinary WHERE id = ?" db-types="MySQL" scenario-types="passthrough">
<assertion parameters="1:int" expected-data-source-name="expected_dataset" />
</test-case>

<test-case sql="SELECT * FROM t_data_type_longblob WHERE id = ?" db-types="MySQL" scenario-types="passthrough">
<assertion parameters="1:int" expected-data-source-name="expected_dataset" />
</test-case>
</e2e-test-cases>
<!-- SPEX ADDED: END -->
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<dataset>
<metadata data-nodes="passthrough.t_data_type_integer">
<column name="id" type="numeric" />
<column name="col_bigint" type="numeric" />
<column name="col_int" type="numeric" />
<column name="col_mediumint" type="numeric" />
<column name="col_smallint" type="numeric" />
<column name="col_tinyint" type="numeric" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_integer_unsigned">
<column name="id" type="numeric" />
<column name="col_bigint_unsigned" type="decimal" />
<column name="col_int_unsigned" type="numeric" />
<column name="col_mediumint_unsigned" type="numeric" />
<column name="col_smallint_unsigned" type="numeric" />
<column name="col_tinyint_unsigned" type="numeric" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_floating_point">
<column name="id" type="numeric" />
<column name="col_float" type="numeric" />
<column name="col_double" type="numeric" />
</metadata>
<metadata data-nodes="passthrough.t_with_generated_id">
<column name="id" type="numeric" />
<column name="val" type="varchar" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_money">
<column name="id" type="numeric" />
<column name="val" type="cast#money" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_bytea">
<column name="id" type="numeric" />
<column name="val" type="varchar" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_uuid">
<column name="id" type="numeric" />
<column name="val" type="varchar" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_date">
<column name="id" type="numeric" />
<column name="creation_date" type="Date" />
<column name="update_date" type="datetime" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_binary">
<column name="id" type="numeric" />
<column name="val" type="binary" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_varbinary">
<column name="id" type="numeric" />
<column name="val" type="varbinary" />
</metadata>
<metadata data-nodes="passthrough.t_data_type_longblob">
<column name="id" type="numeric" />
<column name="val" type="longblob" />
</metadata>
<row data-node="passthrough.t_data_type_integer_unsigned" values="1001, 18446744073709551615, 4294967295, 16777215, 65535, 255" />
<row data-node="passthrough.t_data_type_integer_unsigned" values="1002, 0, 0, 0, 0, 0" />
<row data-node="passthrough.t_data_type_money" values="1001, 123" />
<row data-node="passthrough.t_data_type_money" values="1002, 456" />
<row data-node="passthrough.t_data_type_date" values="1, 2017-08-08, 2017-08-08 00:00:00" />
<row data-node="passthrough.t_data_type_binary" values="1, '123'" />
<row data-node="passthrough.t_data_type_varbinary" values="1, '123'" />
<row data-node="passthrough.t_data_type_longblob" values="1, '123'" />
</dataset>
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,6 @@ CREATE TABLE passthrough.t_data_type_money (id INT PRIMARY KEY, val NUMERIC(16,
CREATE TABLE passthrough.t_data_type_bytea (id INT PRIMARY KEY, val BLOB NOT NULL);
CREATE TABLE passthrough.t_data_type_date (id INT PRIMARY KEY, creation_date DATE NOT NULL, update_date DATETIME NOT NULL);
CREATE TABLE passthrough.t_data_type_uuid (id INT PRIMARY KEY, val VARCHAR(36) NOT NULL);
CREATE TABLE passthrough.t_data_type_binary (id INT PRIMARY KEY, val BINARY(10) NOT NULL);
CREATE TABLE passthrough.t_data_type_varbinary (id INT PRIMARY KEY, val VARBINARY(10) NOT NULL);
CREATE TABLE passthrough.t_data_type_longblob (id INT PRIMARY KEY, val LONGBLOB NOT NULL);
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<dataset>
<metadata data-nodes="expected_dataset.t_data_type_integer">
<column name="id" type="numeric" />
<column name="col_bigint" type="numeric" />
<column name="col_int" type="numeric" />
<column name="col_mediumint" type="numeric" />
<column name="col_smallint" type="numeric" />
<column name="col_tinyint" type="numeric" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_integer_unsigned">
<column name="id" type="numeric" />
<column name="col_bigint_unsigned" type="decimal" />
<column name="col_int_unsigned" type="numeric" />
<column name="col_mediumint_unsigned" type="numeric" />
<column name="col_smallint_unsigned" type="numeric" />
<column name="col_tinyint_unsigned" type="numeric" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_floating_point">
<column name="id" type="numeric" />
<column name="col_float" type="numeric" />
<column name="col_double" type="numeric" />
</metadata>
<metadata data-nodes="expected_dataset.t_with_generated_id">
<column name="id" type="numeric" />
<column name="val" type="varchar" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_money">
<column name="id" type="numeric" />
<column name="val" type="cast#money" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_bytea">
<column name="id" type="numeric" />
<column name="val" type="varchar" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_uuid">
<column name="id" type="numeric" />
<column name="val" type="varchar" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_date">
<column name="id" type="numeric" />
<column name="creation_date" type="Date" />
<column name="update_date" type="datetime" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_binary">
<column name="id" type="numeric" />
<column name="val" type="binary" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_varbinary">
<column name="id" type="numeric" />
<column name="val" type="varbinary" />
</metadata>
<metadata data-nodes="expected_dataset.t_data_type_longblob">
<column name="id" type="numeric" />
<column name="val" type="longblob" />
</metadata>
<row data-node="expected_dataset.t_data_type_integer_unsigned" values="1001, 18446744073709551615, 4294967295, 16777215, 65535, 255" />
<row data-node="expected_dataset.t_data_type_integer_unsigned" values="1002, 0, 0, 0, 0, 0" />
<row data-node="expected_dataset.t_data_type_money" values="1001, 123" />
<row data-node="expected_dataset.t_data_type_money" values="1002, 456" />
<row data-node="expected_dataset.t_data_type_date" values="1, 2017-08-08, 2017-08-08 00:00:00" />
<row data-node="expected_dataset.t_data_type_binary" values="1, '123'" />
<row data-node="expected_dataset.t_data_type_varbinary" values="1, '123'" />
<row data-node="expected_dataset.t_data_type_longblob" values="1, '123'" />
</dataset>
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,6 @@ CREATE TABLE expected_dataset.t_data_type_money (id INT PRIMARY KEY, val NUMERI
CREATE TABLE expected_dataset.t_data_type_bytea (id INT PRIMARY KEY, val BLOB NOT NULL);
CREATE TABLE expected_dataset.t_data_type_date (id INT PRIMARY KEY, creation_date DATE NOT NULL, update_date DATETIME NOT NULL);
CREATE TABLE expected_dataset.t_data_type_uuid (id INT PRIMARY KEY, val VARCHAR(36) NOT NULL);
CREATE TABLE expected_dataset.t_data_type_binary (id INT PRIMARY KEY, val BINARY(10) NOT NULL);
CREATE TABLE expected_dataset.t_data_type_varbinary (id INT PRIMARY KEY, val VARBINARY(10) NOT NULL);
CREATE TABLE expected_dataset.t_data_type_longblob (id INT PRIMARY KEY, val LONGBLOB NOT NULL);

0 comments on commit 8d80b59

Please sign in to comment.