Merge remote-tracking branch 'origin/main' into wrj/catalog-macro

Signed-off-by: Runji Wang <[email protected]>

ci/Dockerfile

@@ -29,6 +29,7 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-mo
 ENV PATH /root/.cargo/bin/:$PATH
 
 RUN rustup show
+RUN rustup default `rustup show active-toolchain | awk '{print $1}'`
 
 RUN curl -sSL "https://github.com/bufbuild/buf/releases/download/v1.4.0/buf-$(uname -s)-$(uname -m).tar.gz" | \
     tar -xvzf - -C /usr/local --strip-components 1
@@ -48,7 +49,7 @@ ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 # install build tools
 RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
 RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \
-    cargo-make@0.36.10 \
+    cargo-make@0.37.9 \
     [email protected] \
     [email protected] \
     && cargo cache -a \

ci/build-ci-image.sh

@@ -10,7 +10,7 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20240204
+export BUILD_ENV_VERSION=v20240213
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -71,7 +71,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213
     depends_on:
       - mysql
       - db
@@ -81,7 +81,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213
     depends_on:
       - mysql
       - db
@@ -93,12 +93,12 @@ services:
       - ..:/risingwave
 
   rw-build-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -109,7 +109,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240213
     depends_on:
       db:
         condition: service_healthy

ci/scripts/sql/nexmark/q14.drop.sql

@@ -1,3 +1,4 @@
 -- noinspection SqlNoDataSourceInspectionForFile
 -- noinspection SqlResolveForFile
 DROP SINK nexmark_q14;
+DROP FUNCTION count_char;

ci/scripts/sql/nexmark/q14.sql

@@ -1,5 +1,16 @@
 -- noinspection SqlNoDataSourceInspectionForFile
 -- noinspection SqlResolveForFile
+
+CREATE FUNCTION count_char(s varchar, c varchar) RETURNS int LANGUAGE javascript AS $$
+    var count = 0;
+    for (var cc of s) {
+        if (cc === c) {
+            count++;
+        }
+    }
+    return count;
+$$;
+
 CREATE SINK nexmark_q14 AS
 SELECT auction,
        bidder,
@@ -15,11 +26,8 @@ SELECT auction,
                THEN 'nightTime'
            ELSE 'otherTime'
            END       AS bidTimeType,
-       date_time
-       -- extra
-       -- TODO: count_char is an UDF, add it back when we support similar functionality.
-       -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java
-       -- count_char(extra, 'c') AS c_counts
+       date_time,
+       count_char(extra, 'c') AS c_counts
 FROM bid
 WHERE 0.908 * price > 1000000
   AND 0.908 * price < 50000000

ci/workflows/pull-request.yml

@@ -96,7 +96,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 16
+    timeout_in_minutes: 17
     retry: *auto-retry
 
   - label: "end-to-end test (parallel)"

e2e_test/batch/catalog/pg_database.slt.part

@@ -1,4 +1,4 @@
 query ITIITT
-SELECT oid, datname, encoding, datcollate, datctype, datistemplate, datallowconn, datconnlimit, dattablespace FROM pg_catalog.pg_database where oid = 0;
+SELECT datname, encoding, datcollate, datctype, datistemplate, datallowconn, datconnlimit, dattablespace FROM pg_catalog.pg_database where datname = 'dev';
 ----
-0 dev 6 C C f t -1 1663
+dev 6 C C f t -1 1663

e2e_test/source/nexmark_endless_sinks/nexmark_endless_part3.slt

@@ -16,6 +16,9 @@ drop sink nexmark_q10;
 statement ok
 drop sink nexmark_q14;
 
+statement ok
+drop function count_char;
+
 statement ok
 drop sink nexmark_q15;
 

e2e_test/streaming/nexmark/drop_views.slt.part

@@ -34,6 +34,9 @@ drop materialized view nexmark_q10;
 statement ok
 drop materialized view nexmark_q14;
 
+statement ok
+drop function count_char;
+
 statement ok
 drop materialized view nexmark_q15;
 

e2e_test/streaming/nexmark/q14.slt.part

@@ -1,15 +1,15 @@
 query IIRTT rowsort
 SELECT * FROM nexmark_q14;
 ----
-1000	1001	12696452.316	nightTime	2015-07-15 00:00:11.003
-1000	1001	1753269.004	nightTime	2015-07-15 00:00:14.004
-1000	1001	2196534.628	nightTime	2015-07-15 00:00:04.001
-1000	1001	26038438.152	nightTime	2015-07-15 00:00:13.003
-1000	1001	39654430.240	nightTime	2015-07-15 00:00:04.001
-1000	1001	9503126.184	nightTime	2015-07-15 00:00:22.005
-1000	1006	1301754.200	nightTime	2015-07-15 00:00:12.003
-1006	1001	19666916.800	nightTime	2015-07-15 00:00:16.004
-1006	1001	3099951.044	nightTime	2015-07-15 00:00:16.004
-1008	1002	26289625.456	nightTime	2015-07-15 00:00:04.001
-1012	1001	3067656.208	nightTime	2015-07-15 00:00:20.005
-1012	1001	3299152.624	nightTime	2015-07-15 00:00:20.005
+1000	1001	12696452.316	nightTime	2015-07-15 00:00:11.003 1
+1000	1001	1753269.004		nightTime	2015-07-15 00:00:14.004 0
+1000	1001	2196534.628		nightTime	2015-07-15 00:00:04.001 3
+1000	1001	26038438.152	nightTime	2015-07-15 00:00:13.003 3
+1000	1001	39654430.240	nightTime	2015-07-15 00:00:04.001 2
+1000	1001	9503126.184		nightTime	2015-07-15 00:00:22.005 1
+1000	1006	1301754.200		nightTime	2015-07-15 00:00:12.003 2
+1006	1001	19666916.800	nightTime	2015-07-15 00:00:16.004 4
+1006	1001	3099951.044		nightTime	2015-07-15 00:00:16.004 3
+1008	1002	26289625.456	nightTime	2015-07-15 00:00:04.001 4
+1012	1001	3067656.208		nightTime	2015-07-15 00:00:20.005 3
+1012	1001	3299152.624		nightTime	2015-07-15 00:00:20.005 3

e2e_test/streaming/nexmark/sinks/q14.slt.part

@@ -1,3 +1,14 @@
+statement ok
+CREATE FUNCTION count_char(s varchar, c varchar) RETURNS int LANGUAGE javascript AS $$
+    var count = 0;
+    for (var cc of s) {
+        if (cc === c) {
+            count++;
+        }
+    }
+    return count;
+$$;
+
 statement ok
 CREATE SINK nexmark_q14 AS
 SELECT
@@ -15,11 +26,8 @@ SELECT
     THEN 'nightTime'
     ELSE 'otherTime'
   END AS bidTimeType,
-  date_time
-  -- extra
-  -- TODO: count_char is an UDF, add it back when we support similar functionality.
-  -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java
-  -- count_char(extra, 'c') AS c_counts
+  date_time,
+  count_char(extra, 'c') AS c_counts
 FROM bid
 WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000
 WITH ( connector = 'blackhole', type = 'append-only', force_append_only = 'true');

e2e_test/streaming/nexmark/views/q14.slt.part

@@ -1,3 +1,14 @@
+statement ok
+CREATE FUNCTION count_char(s varchar, c varchar) RETURNS int LANGUAGE javascript AS $$
+    var count = 0;
+    for (var cc of s) {
+        if (cc === c) {
+            count++;
+        }
+    }
+    return count;
+$$;
+
 statement ok
 CREATE MATERIALIZED VIEW nexmark_q14 AS
 SELECT
@@ -15,10 +26,7 @@ SELECT
     THEN 'nightTime'
     ELSE 'otherTime'
   END AS bidTimeType,
-  date_time
-  -- extra
-  -- TODO: count_char is an UDF, add it back when we support similar functionality.
-  -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java
-  -- count_char(extra, 'c') AS c_counts
+  date_time,
+  count_char(extra, 'c') AS c_counts
 FROM bid
 WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000;

e2e_test/udf/sql_udf.slt

@@ -274,9 +274,9 @@ select type_match(114514);
 ----
 $1 + 114514 + $1
 
-#################################################################################
-# Invalid definition (and maybe not yet supported features 🤪) / use case tests #
-#################################################################################
+##################################################
+# Invalid definition tests when creating sql udf #
+##################################################
 
 # Named sql udf with invalid parameter in body definition
 # Will be rejected at creation time
@@ -306,14 +306,6 @@ create function fib(INT) returns int
       else fib($1 - 1) + fib($1 - 2)
       end;';
 
-# The execution will eventually exceed the pre-defined max stack depth
-# statement error function fib calling stack depth limit exceeded
-# select fib(100);
-
-# Currently create a materialized view with a recursive sql udf will be rejected
-# statement error function fib calling stack depth limit exceeded
-# create materialized view foo_mv as select fib(100);
-
 # Calling a non-existent function
 statement error failed to conduct semantic check, please see if you are calling non-existent functions
 create function non_exist(INT) returns int language sql as 'select yo(114514)';
@@ -326,6 +318,20 @@ create function type_mismatch(INT) returns varchar language sql as 'select $1 +
 statement error Expected an expression:, found: EOF at the end
 create function add_error(INT, INT) returns int language sql as $$select $1 + $2 +$$;
 
+######################################################################
+# Not yet supported features 🤪 (and potential basic use case tests) #
+######################################################################
+
+# 1. Recursion Support for SQL UDF
+
+# The execution will eventually exceed the pre-defined max stack depth
+# statement error function fib calling stack depth limit exceeded
+# select fib(100);
+
+# Currently create a materialized view with a recursive sql udf will be rejected
+# statement error function fib calling stack depth limit exceeded
+# create materialized view foo_mv as select fib(100);
+
 # Rejected deep calling stack
 # statement error function recursive calling stack depth limit exceeded
 # select recursive(1, 1);
@@ -342,6 +348,116 @@ create function add_error(INT, INT) returns int language sql as $$select $1 + $2
 # statement error function fib calling stack depth limit exceeded
 # create materialized view bar_mv as select fib(c1) from t1;
 
+# 2. Select from table inside SQL UDF (potential concerns: naming conflict)
+
+# statment ok
+# create funciton from_table_single_row() returns int language sql as 'select c1 from t1';
+
+# Do NOT need explicit `select ... from ...`
+# query I
+# select from_table_single_row();
+# ----
+# 1
+
+# statement ok
+# create function from_table_single_row_1() returns int language sql as 'select c1 from t1 order by c1 desc';
+
+# Need explict `select ... from ...`
+# query I
+# select * from from_table_single_row_1();
+# ----
+# 5
+
+# Should add parser support
+# statement ok
+# create function from_table_multiple_rows() returns setof int language sql as 'select c1 from t1';
+
+# P.S. postgres shadows the `c1` parameter by the actual column `c1` to resolve the naming conflict
+# statement ok
+# create function from_table_conflict(c1 INT) returns int language sql as 'select c1 from t1';
+
+# 3. Output multiple columns / expressions from a single SQL UDF
+
+# Parser support is needed
+# statement ok
+# create function out_parameters_without_name(out INT, out INT) language sql as 'select 1919810, 114514';
+
+# query II
+# select out_parameters_without_name();
+# ----
+# 1919810 114514
+
+# query II
+# select * out_parameters_without_name();
+# ----
+# 1919810 114514
+
+# statement error non-existent column
+# select a, b from out_parameters_without_name();
+# ----
+# 1919810 114514
+
+# statement ok
+# create function out_parameters_with_name(out a INT, out b INT) language sql as 'select 1919810, 114514';
+
+# query II
+# select out_parameters_with_name();
+# ----
+# 1919810 114514
+
+# query II
+# select a, b from out_parameters_with_name();
+# ----
+# 1919810 114514
+
+# statement ok
+# create function multiple_cols() returns setof record as 'select c1, c2, c3 from t2' language sql;
+
+# query III
+# select * from multiple_cols() as t(c1 INT, c2 FLOAT, c3 INT);
+# ----
+# corresponding results
+# may need to order the sequence
+
+# 4. Polymorphic arguments
+
+# statement ok
+# create function is_greater(anyelement, anyelement) returns boolean language sql as 'select $1 > $2';
+
+# query I
+# select is_greater(1, 2);
+# ----
+# f
+
+# query I
+# select is_greater(3.14, 2.95);
+# ----
+# t
+
+# 5. Overloading functions
+
+# statement ok
+# create function overload(c1 INT) returns int language sql as 'select c1';
+
+# statement ok
+# create function overload(c1 VARCHAR) returns varchar language sql as 'select c1';
+
+# query I
+# select overload(114514), overload('114514');
+# ----
+# 114514 114514
+
+# statement error naming conflict with first overload
+# create function overload(c1 INT, out VARCHAR)
+
+# This definition will cause ambiguity with the second overload during runtime if `c2` is not specified
+# statement ok
+# create function overload(c1 VARCHAR, c2 VARCHAR default '114514', out VARCHAR, out VARCHAR) language sql as 'select c1, c2';
+
+# statement error can not choose a best candidate function, need explicit type cast
+# query TT
+# select overload('114514');
+
 ##################################################
 # Clean up the funtions / mock tables at the end #
 ##################################################