s3ext: refactor regress codes (#909)

1. add expected output files 2. add schedule file 3. use pg_regress to check results automatically 4. refactor gpcheckcloud regress shell script Signed-off-by: Haozhou Wang <[email protected]>
jet2007 · Jul 6, 2016 · 2f10abe · 2f10abe
1 parent 0a13b23
commit 2f10abe
Show file tree

Hide file tree

Showing 37 changed files with 742 additions and 207 deletions.
diff --git a/gpAux/extensions/gps3ext/.gitignore b/gpAux/extensions/gps3ext/.gitignore
@@ -53,3 +53,10 @@ s3test
 gpcheckcloud
 
 s3.conf
+
+sql
+expected
+results
+
+*.out
+*.diffs
diff --git a/gpAux/extensions/gps3ext/regress/Makefile b/gpAux/extensions/gps3ext/regress/Makefile
@@ -0,0 +1,10 @@
+top_builddir = $(shell cd ../../../.. && pwd)
+include $(top_builddir)/src/Makefile.global
+
+installcheck:
+	$(top_builddir)/src/test/regress/pg_regress --psqldir=$$GPHOME/bin/ --schedule=s3_regress_schedule
+
+clean:
+	rm -rf sql expected results regression.* optimizer_status.out
+
+.PHONY: installcheck clean
diff --git a/gpAux/extensions/gps3ext/regress/gpcheckcloud_regress.sh b/gpAux/extensions/gps3ext/regress/gpcheckcloud_regress.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+EXIT_CODE=0
+GPCHECKCLOUD=./bin/gpcheckcloud/gpcheckcloud
+
+PREFIX[0]="http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/data0014"
+PREFIX[1]="http://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/data0016"
+PREFIX[2]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/data0014"
+PREFIX[3]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/data0016"
+PREFIX[4]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/gzipped/data0001.gz"
+PREFIX[5]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/threebytes/"
+PREFIX[6]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/small17/"
+PREFIX[7]="s3://s3-us-east-1.amazonaws.com/useast1.s3test.pivotal.io/small17/"
+PREFIX[8]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/gzipped/"
+PREFIX[9]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/gzipped_normal1/"
+
+HASHSUM[0]="68c4a63b721e7af0ae945ce109ca87ad"
+HASHSUM[1]="0fd502a303eb8f138f5916ec357721b1"
+HASHSUM[2]="68c4a63b721e7af0ae945ce109ca87ad"
+HASHSUM[3]="0fd502a303eb8f138f5916ec357721b1"
+HASHSUM[4]="b958fb80b98605a6095e6ebc4b9b4786"
+HASHSUM[5]="fe7d81814e02eb1296757e75bb3c6be9"
+HASHSUM[6]="138fc555074671912125ba692c678246"
+HASHSUM[7]="138fc555074671912125ba692c678246"
+HASHSUM[8]="7b2260e9a3a3f26e84aa28dc2124f68f"
+HASHSUM[9]="eacb7b210d3f7703ee06d16f520b103e"
+
+#HUGE FILES
+PREFIX[10]="https://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/airlinedata1.csv"
+PREFIX[11]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/"
+PREFIX[12]="s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/gzipped_normal2/"
+
+HASHSUM[10]="f5811ad92c994f1d6913d5338575fe38"
+HASHSUM[11]="75baaa39f2b1544ed8af437c2cad86b7"
+HASHSUM[12]="a930794bc885bccf6eed45bd40367a7d"
+
+for ((i=0; i<${#PREFIX[@]}; i++))
+do
+	[ `$GPCHECKCLOUD -d "${PREFIX[$i]} config=/home/gpadmin/s3.conf" 2>/dev/null |md5sum |cut -d ' ' -f 1` = "${HASHSUM[$i]}" ] \
+		&& echo test ${PREFIX[$i]} ... ok \
+		|| { EXIT_CODE=1; echo test ${PREFIX[$i]} ... failed; }
+done
+
+exit $EXIT_CODE
diff --git a/...tensions/gps3ext/regress/regress_1.01.sql → .../gps3ext/regress/input/1_01_normal.source b/...tensions/gps3ext/regress/regress_1.01.sql → .../gps3ext/regress/input/1_01_normal.source
@@ -1,7 +1,3 @@
--- ========
--- PROTOCOL
--- ========
-
 -- create the database functions
 CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
         '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
@@ -11,9 +7,6 @@ CREATE PROTOCOL s3 (
         readfunc  = read_from_s3
 );
 
--- Check out the catalog table
-select * from pg_extprotocol;
-
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
         low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'csv';
@@ -22,9 +15,6 @@ SELECT count(*) FROM s3example;
 SELECT sum(open) FROM s3example;
 SELECT avg(open) FROM s3example;
 
--- =======
--- CLEANUP
--- =======
 DROP EXTERNAL TABLE s3example;
 
 DROP PROTOCOL s3;
diff --git a/...tensions/gps3ext/regress/regress_1.02.sql → ...s3ext/regress/input/1_02_log_error.source b/...tensions/gps3ext/regress/regress_1.02.sql → ...s3ext/regress/input/1_02_log_error.source
@@ -1,7 +1,3 @@
--- ========
--- PROTOCOL
--- ========
-
 -- create the database functions
 CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
         '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
@@ -11,19 +7,13 @@ CREATE PROTOCOL s3 (
         readfunc  = read_from_s3
 );
 
--- Check out the catalog table
-select * from pg_extprotocol;
-
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
         low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/normal/ config=/home/gpadmin/s3.conf') FORMAT 'text' ( DELIMITER ',' NULL '') LOG ERRORS SEGMENT REJECT LIMIT 5;
 
 SELECT count(*) FROM s3example;
 SELECT gp_read_error_log('s3example');
 
--- =======
--- CLEANUP
--- =======
 DROP EXTERNAL TABLE s3example;
 
 DROP PROTOCOL s3;
diff --git a/...tensions/gps3ext/regress/regress_1.03.sql → ...ps3ext/regress/input/1_03_bad_data.source b/...tensions/gps3ext/regress/regress_1.03.sql → ...ps3ext/regress/input/1_03_bad_data.source
@@ -1,7 +1,3 @@
--- ========
--- PROTOCOL
--- ========
-
 -- create the database functions
 CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
         '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
@@ -11,20 +7,16 @@ CREATE PROTOCOL s3 (
         readfunc  = read_from_s3
 );
 
--- Check out the catalog table
-select * from pg_extprotocol;
-
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
 	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset1/baddata/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
 
 SELECT count(*) FROM s3example;
 
+-- start_ignore
 SELECT gp_read_error_log('s3example');
+-- end_ignore
 
--- =======
--- CLEANUP
--- =======
 DROP EXTERNAL TABLE s3example;
 
 DROP PROTOCOL s3;
diff --git a/...tensions/gps3ext/regress/regress_1.04.sql → ...xt/regress/input/1_04_empty_prefix.source b/...tensions/gps3ext/regress/regress_1.04.sql → ...xt/regress/input/1_04_empty_prefix.source
@@ -1,7 +1,3 @@
--- ========
--- PROTOCOL
--- ========
-
 -- create the database functions
 CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
         '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
@@ -11,19 +7,13 @@ CREATE PROTOCOL s3 (
         readfunc  = read_from_s3
 );
 
--- Check out the catalog table
-select * from pg_extprotocol;
-
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
 	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/emptyfile/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
 
 SELECT count(*) FROM s3example;
 SELECT gp_read_error_log('s3example');
 
--- =======
--- CLEANUP
--- =======
 DROP EXTERNAL TABLE s3example;
 
 DROP PROTOCOL s3;
diff --git a/...tensions/gps3ext/regress/regress_1.05.sql → ...ps3ext/regress/input/1_05_one_line.source b/...tensions/gps3ext/regress/regress_1.05.sql → ...ps3ext/regress/input/1_05_one_line.source
@@ -1,7 +1,3 @@
--- ========
--- PROTOCOL
--- ========
-
 -- create the database functions
 CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
         '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
@@ -11,18 +7,12 @@ CREATE PROTOCOL s3 (
         readfunc  = read_from_s3
 );
 
--- Check out the catalog table
-select * from pg_extprotocol;
-
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
 	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/oneline/ config=/home/gpadmin/s3.conf') format 'csv';
 
 SELECT count(*) FROM s3example;
 
--- =======
--- CLEANUP
--- =======
 DROP EXTERNAL TABLE s3example;
 
 DROP PROTOCOL s3;
diff --git a/...tensions/gps3ext/regress/regress_1.06.sql → ...regress/input/1_06_1correct_1wrong.source b/...tensions/gps3ext/regress/regress_1.06.sql → ...regress/input/1_06_1correct_1wrong.source
@@ -1,7 +1,3 @@
--- ========
--- PROTOCOL
--- ========
-
 -- create the database functions
 CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
         '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
@@ -11,19 +7,15 @@ CREATE PROTOCOL s3 (
         readfunc  = read_from_s3
 );
 
--- Check out the catalog table
-select * from pg_extprotocol;
-
 drop external table s3example;
 create READABLE external table s3example (date text, time text, open float, high float,
 	low float, volume int) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/1correct1wrong/ config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 5;
 
 SELECT count(*) FROM s3example;
+-- start_ignore
 SELECT gp_read_error_log('s3example');
+-- end_ignore
 
--- =======
--- CLEANUP
--- =======
 DROP EXTERNAL TABLE s3example;
 
 DROP PROTOCOL s3;
diff --git a/...tensions/gps3ext/regress/regress_1.07.sql → ...t/regress/input/1_07_huge_bad_data.source b/...tensions/gps3ext/regress/regress_1.07.sql → ...t/regress/input/1_07_huge_bad_data.source
@@ -1,7 +1,3 @@
--- ========
--- PROTOCOL
--- ========
-
 -- create the database functions
 CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
         '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
@@ -11,18 +7,11 @@ CREATE PROTOCOL s3 (
         readfunc  = read_from_s3
 );
 
--- Check out the catalog table
-select * from pg_extprotocol;
-
 drop external table s3example;
--- create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile config=/home/gpadmin/s3.conf') format 'csv' LOG ERRORS SEGMENT REJECT LIMIT 100 PERCENT;
 create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
 
 SELECT count(*) FROM s3example;
 
--- =======
--- CLEANUP
--- =======
 DROP EXTERNAL TABLE s3example;
 
 DROP PROTOCOL s3;
diff --git a/...tensions/gps3ext/regress/regress_1.08.sql → ...gress/input/1_08_huge_correct_data.source b/...tensions/gps3ext/regress/regress_1.08.sql → ...gress/input/1_08_huge_correct_data.source
@@ -1,7 +1,3 @@
--- ========
--- PROTOCOL
--- ========
-
 -- create the database functions
 CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
         '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
@@ -11,17 +7,11 @@ CREATE PROTOCOL s3 (
         readfunc  = read_from_s3
 );
 
--- Check out the catalog table
-select * from pg_extprotocol;
-
 drop external table s3example;
 create READABLE external table s3example (Year text, Month text, DayofMonth text, DayOfWeek text, DepTime text, CRSDepTime text, ArrTime text,CRSArrTime text, UniqueCarrier text, FlightNum text,TailNum text, ActualElapsedTime text, CRSElapsedTime text, AirTime text, ArrDelay text, DepDelay text, Origin text, Dest text, Distance text, TaxiIn text, TaxiOut text, Cancelled text, CancellationCode text, Diverted text, CarrierDelay text, WeatherDelay text, NASDelay text, SecurityDelay text, LateAircraftDelay text) location('s3://s3-us-west-2.amazonaws.com/s3test.pivotal.io/dataset2/hugefile_correct_data/ config=/home/gpadmin/s3.conf') format 'csv' SEGMENT REJECT LIMIT 100 PERCENT;
 
 SELECT count(*) FROM s3example;
 
--- =======
--- CLEANUP
--- =======
 DROP EXTERNAL TABLE s3example;
 
 DROP PROTOCOL s3;
diff --git a/...tensions/gps3ext/regress/regress_1.09.sql → ...s3ext/regress/input/1_09_partition.source b/...tensions/gps3ext/regress/regress_1.09.sql → ...s3ext/regress/input/1_09_partition.source
@@ -1,7 +1,3 @@
--- ========
--- PROTOCOL
--- ========
-
 -- create the database functions
 CREATE OR REPLACE FUNCTION read_from_s3() RETURNS integer AS
 '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;
@@ -11,9 +7,6 @@ CREATE PROTOCOL s3 (
 	readfunc  = read_from_s3
 );
 
--- Check out the catalog table
-select * from pg_extprotocol;
-
 drop external table s3example;
 
 CREATE TABLE stock (date text, time text, open float, high float, low float, volume int) DISTRIBUTED BY (date) PARTITION BY RANGE (volume)
@@ -35,9 +28,6 @@ ALTER TABLE stock EXCHANGE PARTITION FOR (0) WITH TABLE ext_stock000 WITHOUT VAL
 SELECT count(*) FROM stock_1_prt_stock000;
 SELECT count(*) FROM stock;
 
--- =======
--- CLEANUP
--- =======
 DROP TABLE ext_stock000;
 DROP TABLE stock;
-Original file line number
+Diff line change
@@ Expand Up / @@ -53,3 +53,10 @@ s3test @@
     gpcheckcloud
     s3.conf
+    sql
+    expected
+    results
+    *.out
+    *.diffs