From 8a137f25e432f13efcb0ea002a726582824bce7a Mon Sep 17 00:00:00 2001 From: Vitaly Stoyan Date: Thu, 18 Jul 2024 14:56:04 +0000 Subject: [PATCH] init --- .../yql/parser/pg_wrapper/comp_factory.cpp | 15 ++++++++-- .../yql/parser/pg_wrapper/interface/codec.h | 2 +- .../yql/providers/common/codec/yql_codec.cpp | 2 +- .../yt/codec/codegen/yt_codec_cg.cpp | 4 +-- ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp | 3 +- .../sql/dq_file/part17/canondata/result.json | 25 +++++++++++++++++ .../extracted | 3 ++ .../sql/dq_file/part18/canondata/result.json | 25 +++++++++++++++++ .../extracted | 3 ++ .../hybrid_file/part2/canondata/result.json | 14 ++++++++++ .../hybrid_file/part6/canondata/result.json | 14 ++++++++++ .../tests/sql/sql2yql/canondata/result.json | 28 +++++++++++++++++++ ydb/library/yql/tests/sql/suites/pg/nulls.sql | 13 +++++++++ .../yql/tests/sql/suites/pg/nulls_native.sql | 14 ++++++++++ .../part17/canondata/result.json | 21 ++++++++++++++ .../part18/canondata/result.json | 21 ++++++++++++++ 16 files changed, 200 insertions(+), 7 deletions(-) create mode 100644 ydb/library/yql/tests/sql/dq_file/part17/canondata/test.test_pg-nulls-default.txt-Analyze_/extracted create mode 100644 ydb/library/yql/tests/sql/dq_file/part18/canondata/test.test_pg-nulls_native-default.txt-Analyze_/extracted create mode 100644 ydb/library/yql/tests/sql/suites/pg/nulls.sql create mode 100644 ydb/library/yql/tests/sql/suites/pg/nulls_native.sql diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp index a7c7c159adac..2c6eea0347aa 100644 --- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp +++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp @@ -3591,10 +3591,15 @@ TString PgValueToString(const NUdf::TUnboxedValuePod& value, ui32 pgTypeId) { } } -void WriteYsonValueInTableFormatPg(TOutputBuf& buf, TPgType* type, const NUdf::TUnboxedValuePod& value) { +void WriteYsonValueInTableFormatPg(TOutputBuf& buf, TPgType* type, const NUdf::TUnboxedValuePod& value, bool topLevel) { using namespace NYson::NDetail; if (!value) { - buf.Write(EntitySymbol); + if (topLevel) { + buf.Write(BeginListSymbol); + buf.Write(EndListSymbol); + } else { + buf.Write(EntitySymbol); + } return; } @@ -3670,6 +3675,12 @@ NUdf::TUnboxedValue ReadYsonValueInTableFormatPg(TPgType* type, char cmd, TInput return NUdf::TUnboxedValuePod(); } + if (cmd == BeginListSymbol) { + cmd = buf.Read(); + YQL_ENSURE(cmd == EndListSymbol); + return NUdf::TUnboxedValuePod(); + } + switch (type->GetTypeId()) { case BOOLOID: { YQL_ENSURE(cmd == FalseMarker || cmd == TrueMarker, "Expected either true or false, but got: " << TString(cmd).Quote()); diff --git a/ydb/library/yql/parser/pg_wrapper/interface/codec.h b/ydb/library/yql/parser/pg_wrapper/interface/codec.h index f672a8cad1d7..f4541ba70b9a 100644 --- a/ydb/library/yql/parser/pg_wrapper/interface/codec.h +++ b/ydb/library/yql/parser/pg_wrapper/interface/codec.h @@ -34,7 +34,7 @@ TString PgValueCoerce(const NUdf::TUnboxedValuePod& value, ui32 pgTypeId, i32 ty void WriteYsonValuePg(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod& value, NKikimr::NMiniKQL::TPgType* type, const TVector* structPositions); -void WriteYsonValueInTableFormatPg(TOutputBuf& buf, NKikimr::NMiniKQL::TPgType* type, const NKikimr::NUdf::TUnboxedValuePod& value); +void WriteYsonValueInTableFormatPg(TOutputBuf& buf, NKikimr::NMiniKQL::TPgType* type, const NKikimr::NUdf::TUnboxedValuePod& value, bool topLevel); NUdf::TUnboxedValue ReadYsonValueInTableFormatPg(NKikimr::NMiniKQL::TPgType* type, char cmd, TInputBuf& buf); NUdf::TUnboxedValue ReadYsonValuePg(NKikimr::NMiniKQL::TPgType* type, char cmd, TInputBuf& buf); diff --git a/ydb/library/yql/providers/common/codec/yql_codec.cpp b/ydb/library/yql/providers/common/codec/yql_codec.cpp index 6e03af7dcce0..383e3beb9df3 100644 --- a/ydb/library/yql/providers/common/codec/yql_codec.cpp +++ b/ydb/library/yql/providers/common/codec/yql_codec.cpp @@ -2497,7 +2497,7 @@ void WriteYsonValueInTableFormat(TOutputBuf& buf, TType* type, ui64 nativeYtType case TType::EKind::Pg: { auto pgType = static_cast(type); - WriteYsonValueInTableFormatPg(buf, pgType, value); + WriteYsonValueInTableFormatPg(buf, pgType, value, topLevel); break; } diff --git a/ydb/library/yql/providers/yt/codec/codegen/yt_codec_cg.cpp b/ydb/library/yql/providers/yt/codec/codegen/yt_codec_cg.cpp index 628e3f311255..cb560e0bfeee 100644 --- a/ydb/library/yql/providers/yt/codec/codegen/yt_codec_cg.cpp +++ b/ydb/library/yql/providers/yt/codec/codegen/yt_codec_cg.cpp @@ -112,7 +112,7 @@ class TYtCodecCgWriter : public IYtCodecCgWriter { bool isOptional; auto unwrappedType = UnpackOptional(type, isOptional); - if (!isOptional) { + if (!isOptional && !type->IsPg()) { GenerateRequired(elemPtr, buf, type, nativeYtTypeFlags, false); } else { const auto just = BasicBlock::Create(context, "just", Func_); @@ -127,7 +127,7 @@ class TYtCodecCgWriter : public IYtCodecCgWriter { { Block_ = just; CallInst::Create(module.getFunction("WriteJust"), { buf }, "", Block_); - if (unwrappedType->IsOptional()) { + if (unwrappedType->IsOptional() || unwrappedType->IsPg()) { const auto unwrappedElem = GetOptionalValue(context, elem, Block_); const auto unwrappedElemPtr = new AllocaInst(valueType, 0U, "unwrapped", Block_); new StoreInst(unwrappedElem, unwrappedElemPtr, Block_); diff --git a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp index 8bfdd0b9cbbd..d7580de5ea1b 100644 --- a/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp +++ b/ydb/library/yql/sql/pg_dummy/pg_sql_dummy.cpp @@ -79,10 +79,11 @@ void WriteYsonValuePg(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod& v throw yexception() << "WriteYsonValuePg: PG types are not supported"; } -void WriteYsonValueInTableFormatPg(TOutputBuf& buf, NKikimr::NMiniKQL::TPgType* type, const NKikimr::NUdf::TUnboxedValuePod& value) { +void WriteYsonValueInTableFormatPg(TOutputBuf& buf, NKikimr::NMiniKQL::TPgType* type, const NKikimr::NUdf::TUnboxedValuePod& value, bool topLevel) { Y_UNUSED(buf); Y_UNUSED(type); Y_UNUSED(value); + Y_UNUSED(topLevel); throw yexception() << "WriteYsonValueInTableFormatPg: PG types are not supported"; } diff --git a/ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json index 753c3b460808..03678a159b05 100644 --- a/ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json @@ -1950,6 +1950,31 @@ } ], "test.test[pg-join_using_multiple3-default.txt-Results]": [], + "test.test[pg-nulls-default.txt-Analyze]": [ + { + "checksum": "6f55f046c1fa846e9a504c79cffe6486", + "size": 14397, + "uri": "https://{canondata_backend}/1925842/b93fc4ae84f4cb762b966478e854456851aa6f1f/resource.tar.gz#test.test_pg-nulls-default.txt-Analyze_/plan.txt" + }, + { + "uri": "file://test.test_pg-nulls-default.txt-Analyze_/extracted" + } + ], + "test.test[pg-nulls-default.txt-Debug]": [ + { + "checksum": "78892c99b809f77ec5da42283b011dc1", + "size": 4318, + "uri": "https://{canondata_backend}/1925842/b93fc4ae84f4cb762b966478e854456851aa6f1f/resource.tar.gz#test.test_pg-nulls-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[pg-nulls-default.txt-Plan]": [ + { + "checksum": "9696d88960b250adcfdbab9d1e9f6799", + "size": 14950, + "uri": "https://{canondata_backend}/1925842/b93fc4ae84f4cb762b966478e854456851aa6f1f/resource.tar.gz#test.test_pg-nulls-default.txt-Plan_/plan.txt" + } + ], + "test.test[pg-nulls-default.txt-Results]": [], "test.test[pg-order_by_agg_extra_for_keys-default.txt-Analyze]": [ { "checksum": "b4dd508a329723c74293d80f0278c705", diff --git a/ydb/library/yql/tests/sql/dq_file/part17/canondata/test.test_pg-nulls-default.txt-Analyze_/extracted b/ydb/library/yql/tests/sql/dq_file/part17/canondata/test.test_pg-nulls-default.txt-Analyze_/extracted new file mode 100644 index 000000000000..3169272a6e72 --- /dev/null +++ b/ydb/library/yql/tests/sql/dq_file/part17/canondata/test.test_pg-nulls-default.txt-Analyze_/extracted @@ -0,0 +1,3 @@ +/program.sql:
: Info: Optimization + + /program.sql:
: Info: DQ cannot execute the query. Cause: table without statistics \ No newline at end of file diff --git a/ydb/library/yql/tests/sql/dq_file/part18/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part18/canondata/result.json index e91ad14e3524..e5decd2c1bb8 100644 --- a/ydb/library/yql/tests/sql/dq_file/part18/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part18/canondata/result.json @@ -2038,6 +2038,31 @@ } ], "test.test[pg-nothing-default.txt-Results]": [], + "test.test[pg-nulls_native-default.txt-Analyze]": [ + { + "checksum": "6dc2925be134caed98091b33fe46837d", + "size": 14407, + "uri": "https://{canondata_backend}/1903885/cb1ae8567826c8726613cbec6958ece74666470d/resource.tar.gz#test.test_pg-nulls_native-default.txt-Analyze_/plan.txt" + }, + { + "uri": "file://test.test_pg-nulls_native-default.txt-Analyze_/extracted" + } + ], + "test.test[pg-nulls_native-default.txt-Debug]": [ + { + "checksum": "fb613eb480003088dcb8ed4582bd0d80", + "size": 4457, + "uri": "https://{canondata_backend}/1903885/cb1ae8567826c8726613cbec6958ece74666470d/resource.tar.gz#test.test_pg-nulls_native-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[pg-nulls_native-default.txt-Plan]": [ + { + "checksum": "8aecbc0b84719f81f7025df6e11eb25e", + "size": 14960, + "uri": "https://{canondata_backend}/1903885/cb1ae8567826c8726613cbec6958ece74666470d/resource.tar.gz#test.test_pg-nulls_native-default.txt-Plan_/plan.txt" + } + ], + "test.test[pg-nulls_native-default.txt-Results]": [], "test.test[pg-pg_array_cast-default.txt-Analyze]": [ { "checksum": "b4dd508a329723c74293d80f0278c705", diff --git a/ydb/library/yql/tests/sql/dq_file/part18/canondata/test.test_pg-nulls_native-default.txt-Analyze_/extracted b/ydb/library/yql/tests/sql/dq_file/part18/canondata/test.test_pg-nulls_native-default.txt-Analyze_/extracted new file mode 100644 index 000000000000..3169272a6e72 --- /dev/null +++ b/ydb/library/yql/tests/sql/dq_file/part18/canondata/test.test_pg-nulls_native-default.txt-Analyze_/extracted @@ -0,0 +1,3 @@ +/program.sql:
: Info: Optimization + + /program.sql:
: Info: DQ cannot execute the query. Cause: table without statistics \ No newline at end of file diff --git a/ydb/library/yql/tests/sql/hybrid_file/part2/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part2/canondata/result.json index 21c2edd8a97a..013d9fd5deaf 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part2/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part2/canondata/result.json @@ -1931,6 +1931,20 @@ "uri": "https://{canondata_backend}/1937429/f07b84667ac042b441b980139436c1568397aded/resource.tar.gz#test.test_pg-join_using_multiple3-default.txt-Plan_/plan.txt" } ], + "test.test[pg-nulls-default.txt-Debug]": [ + { + "checksum": "452ca8efc692a3347c470be4f370a540", + "size": 4227, + "uri": "https://{canondata_backend}/1920236/0eac73c31a05c1a04134788b280b88cd1b7de73c/resource.tar.gz#test.test_pg-nulls-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[pg-nulls-default.txt-Plan]": [ + { + "checksum": "def9c8aa55dc1c86d4db2d81e6ec91c9", + "size": 14338, + "uri": "https://{canondata_backend}/1920236/0eac73c31a05c1a04134788b280b88cd1b7de73c/resource.tar.gz#test.test_pg-nulls-default.txt-Plan_/plan.txt" + } + ], "test.test[pg-numbers_to_pg-default.txt-Debug]": [ { "checksum": "a729cc20e300134e2c134e1c0b1da2c7", diff --git a/ydb/library/yql/tests/sql/hybrid_file/part6/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part6/canondata/result.json index 241701262e61..89d2cfab9126 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part6/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part6/canondata/result.json @@ -1917,6 +1917,20 @@ "uri": "https://{canondata_backend}/1937001/a770b7e950bbaeaf08ef4bbb336b7e3683a914ce/resource.tar.gz#test.test_pg-nullif-default.txt-Plan_/plan.txt" } ], + "test.test[pg-nulls_native-default.txt-Debug]": [ + { + "checksum": "42fd175664a0cb4114f67242cd8cf19c", + "size": 4380, + "uri": "https://{canondata_backend}/1937150/aa6ebab753c815bad08c1b6b3896a13eb323c6b4/resource.tar.gz#test.test_pg-nulls_native-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[pg-nulls_native-default.txt-Plan]": [ + { + "checksum": "4b3eb456315f50af908ff2de4e6d47bb", + "size": 14344, + "uri": "https://{canondata_backend}/1937150/aa6ebab753c815bad08c1b6b3896a13eb323c6b4/resource.tar.gz#test.test_pg-nulls_native-default.txt-Plan_/plan.txt" + } + ], "test.test[pg-order_by_distinct_same_expr-default.txt-Debug]": [ { "checksum": "e01341f460d0a6d02f36a145b54a8e87", diff --git a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json index dfe66b121d62..2d2e546e686c 100644 --- a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json +++ b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json @@ -12270,6 +12270,20 @@ "uri": "https://{canondata_backend}/1925821/85af800a282697ba458d7b0567031a1eadd214a4/resource.tar.gz#test_sql2yql.test_pg-nullif_/sql.yql" } ], + "test_sql2yql.test[pg-nulls]": [ + { + "checksum": "bbdf86c4f0f32147a110082c47c3fa13", + "size": 4079, + "uri": "https://{canondata_backend}/1942100/3287718f5dbfc64c909294682e6e046711e86046/resource.tar.gz#test_sql2yql.test_pg-nulls_/sql.yql" + } + ], + "test_sql2yql.test[pg-nulls_native]": [ + { + "checksum": "e59cdfb17a68dec47c4e046a41f6d201", + "size": 4169, + "uri": "https://{canondata_backend}/1899731/1a4633fa5c299daa94c0cd1621f3e998810c0b83/resource.tar.gz#test_sql2yql.test_pg-nulls_native_/sql.yql" + } + ], "test_sql2yql.test[pg-numbers_to_pg]": [ { "checksum": "cab84d866e037bed91a0a711ef8b4a4c", @@ -31121,6 +31135,20 @@ "uri": "https://{canondata_backend}/1925842/993f03733a95d23632abd85d6850807a0e17b52b/resource.tar.gz#test_sql_format.test_pg-interval_to_pg_/formatted.sql" } ], + "test_sql_format.test[pg-nulls]": [ + { + "checksum": "052a686d648056bfc7801f4a0f661822", + "size": 341, + "uri": "https://{canondata_backend}/1942100/3287718f5dbfc64c909294682e6e046711e86046/resource.tar.gz#test_sql_format.test_pg-nulls_/formatted.sql" + } + ], + "test_sql_format.test[pg-nulls_native]": [ + { + "checksum": "601f4035e9f67d2e091a5d01c864c6a5", + "size": 369, + "uri": "https://{canondata_backend}/1899731/1a4633fa5c299daa94c0cd1621f3e998810c0b83/resource.tar.gz#test_sql_format.test_pg-nulls_native_/formatted.sql" + } + ], "test_sql_format.test[pg-numbers_to_pg]": [ { "checksum": "1cf61d59e7e627b3451a3e67cff6b9f1", diff --git a/ydb/library/yql/tests/sql/suites/pg/nulls.sql b/ydb/library/yql/tests/sql/suites/pg/nulls.sql new file mode 100644 index 000000000000..32634c7e2a91 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/pg/nulls.sql @@ -0,0 +1,13 @@ +use plato; + +insert into @foo +select 1 as x,Nothing(pgcstring) as i1,Just(Nothing(pgcstring)) as i2,Just(Just(Nothing(pgcstring))) as i3; + +commit; + +insert into @bar +select x+1 as x,i1,i2,i3,i1 is null as i1n,i2 is null as i2n,i3 is null as i3n from @foo; +commit; + +select * from @bar; + diff --git a/ydb/library/yql/tests/sql/suites/pg/nulls_native.sql b/ydb/library/yql/tests/sql/suites/pg/nulls_native.sql new file mode 100644 index 000000000000..cfef625aef93 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/pg/nulls_native.sql @@ -0,0 +1,14 @@ +use plato; +pragma yt.UseNativeYtTypes; + +insert into @foo +select 1 as x,Nothing(pgcstring) as i1,Just(Nothing(pgcstring)) as i2,Just(Just(Nothing(pgcstring))) as i3; + +commit; + +insert into @bar +select x+1 as x,i1,i2,i3,i1 is null as i1n,i2 is null as i2n,i3 is null as i3n from @foo; +commit; + +select * from @bar; + diff --git a/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json index fb88b6436fd8..f65eeea0753b 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json @@ -1706,6 +1706,27 @@ "uri": "https://{canondata_backend}/937458/419a96ea577b829313b24bc6972184dd67aa552e/resource.tar.gz#test.test_pg-join_using_multiple3-default.txt-Results_/results.txt" } ], + "test.test[pg-nulls-default.txt-Debug]": [ + { + "checksum": "ada02188942381fe8d8dd33ae2555164", + "size": 3221, + "uri": "https://{canondata_backend}/1031349/dae70efc9692ed0b363116ee7ad8e201a153fa69/resource.tar.gz#test.test_pg-nulls-default.txt-Debug_/opt.yql" + } + ], + "test.test[pg-nulls-default.txt-Plan]": [ + { + "checksum": "1645d4823fc10616a6aff49c22f11c8d", + "size": 14391, + "uri": "https://{canondata_backend}/1031349/dae70efc9692ed0b363116ee7ad8e201a153fa69/resource.tar.gz#test.test_pg-nulls-default.txt-Plan_/plan.txt" + } + ], + "test.test[pg-nulls-default.txt-Results]": [ + { + "checksum": "b1ee09819814dc45ba746da440feb998", + "size": 3153, + "uri": "https://{canondata_backend}/1031349/dae70efc9692ed0b363116ee7ad8e201a153fa69/resource.tar.gz#test.test_pg-nulls-default.txt-Results_/results.txt" + } + ], "test.test[pg-order_by_agg_extra_for_keys-default.txt-Debug]": [ { "checksum": "93b7c305f6e56c902d34f42050cd72d2", diff --git a/ydb/library/yql/tests/sql/yt_native_file/part18/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part18/canondata/result.json index dea0eade6619..3a12e5b8a714 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part18/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part18/canondata/result.json @@ -1877,6 +1877,27 @@ "uri": "https://{canondata_backend}/1899731/c92a452de9406ab0662eb9deef9f0799dbd3673d/resource.tar.gz#test.test_pg-nothing-default.txt-Results_/results.txt" } ], + "test.test[pg-nulls_native-default.txt-Debug]": [ + { + "checksum": "7e949a666a936950f633bd3f6c32115e", + "size": 3360, + "uri": "https://{canondata_backend}/1784117/8d207d76ab1fd2062cf29be93d0ab0d2f86741a3/resource.tar.gz#test.test_pg-nulls_native-default.txt-Debug_/opt.yql" + } + ], + "test.test[pg-nulls_native-default.txt-Plan]": [ + { + "checksum": "cfca4330a57e3c71ee877d1f73068260", + "size": 14407, + "uri": "https://{canondata_backend}/1784117/8d207d76ab1fd2062cf29be93d0ab0d2f86741a3/resource.tar.gz#test.test_pg-nulls_native-default.txt-Plan_/plan.txt" + } + ], + "test.test[pg-nulls_native-default.txt-Results]": [ + { + "checksum": "55c31d70ebb794e5f9b78342dc43d1b1", + "size": 3089, + "uri": "https://{canondata_backend}/1784117/8d207d76ab1fd2062cf29be93d0ab0d2f86741a3/resource.tar.gz#test.test_pg-nulls_native-default.txt-Results_/results.txt" + } + ], "test.test[pg-pg_array_cast-default.txt-Debug]": [ { "checksum": "b1808b6d98c49f704ce24af91c7b9b88",