test: generic sql

test/parser/sql/contextCollect/fixtures/common.sql

@@ -0,0 +1,31 @@
+CREATE TABLE IF NOT EXISTS new_tb1 like like_old_tb;
+
+CREATE TABLE new_tb2 (new_col1 INT COMMENT 'this is new col1', new_col2 STRING)
+    PARTITIONED BY (YEAR STRING)
+    CLUSTERED BY (new_col1, NAME)
+    SORTED BY (new_col1 ASC)
+    INTO 3 BUCKETS
+    STORED AS PARQUET
+    COMMENT 'this is new_tb2 comment';
+
+CREATE TABLE student_copy USING CSV AS SELECT * FROM student;
+
+CREATE VIEW new_view1 (ID COMMENT 'Unique identification number', Name) 
+COMMENT 'View for experienced employees' 
+AS SELECT id, name FROM old_tb_1 WHERE working_years > 5;
+
+SELECT id, name, em.deptno, deptname FROM employee AS em CROSS JOIN department AS dept;
+
+INSERT INTO insert_tb (address, name, student_id) VALUES ('Hangzhou, China', 'Kent Yao', 11215016);
+
+INSERT OVERWRITE target_tb TABLE source_tb;
+
+INSERT OVERWRITE DIRECTORY '/path/to/output/directory' SELECT * FROM from_tb WHERE condition;
+
+CREATE DATABASE IF NOT EXISTS customer_db COMMENT 'this is database comment';
+
+USE NAMESPACE ns1;
+
+CREATE OR REPLACE FUNCTION simple_udf AS 'SimpleUdfR';
+
+CREATE FUNCTION simple_udf AS 'SimpleUdfR';

test/parser/sql/errorListener.test.ts

@@ -0,0 +1,67 @@
+import { Sql } from 'src/parser/sql';
+
+const randomText = `dhsdansdnkla ndjnsla ndnalks`;
+const sql1 = `ALTER VIEW`;
+const sql2 = `SELECT * FROM `;
+const sql3 = `DROP SCHEMA aaa aaa`;
+
+describe('Sql validate invalid sql and test msg', () => {
+    const sql = new Sql();
+
+    test('validate random text', () => {
+        const errors = sql.validate(randomText);
+        expect(errors.length).toBe(1);
+        expect(errors[0].message).toBe(
+            `'dhsdansdnkla' is not valid at this position, expecting a keyword`
+        );
+    });
+
+    test('validate unComplete sql1', () => {
+        const errors = sql.validate(sql1);
+        expect(errors.length).toBe(1);
+        expect(errors[0].message).toBe('Statement is incomplete');
+    });
+
+    test('validate unComplete sql2', () => {
+        const errors = sql.validate(sql2);
+        expect(errors.length).toBe(1);
+        expect(errors[0].message).toBe(
+            'Statement is incomplete, expecting an existing table or an existing view or an existing function or a keyword'
+        );
+    });
+
+    test('validate unComplete sql3', () => {
+        const errors = sql.validate(sql3);
+        expect(errors.length).toBe(1);
+        expect(errors[0].message).toBe(
+            `'aaa' is not valid at this position, expecting an existing namespace or a keyword`
+        );
+    });
+
+    test('validate random text cn', () => {
+        sql.locale = 'zh_CN';
+        const errors = sql.validate(randomText);
+        expect(errors.length).toBe(1);
+        expect(errors[0].message).toBe(`'dhsdansdnkla' 在此位置无效，期望一个关键字`);
+    });
+
+    test('validate unComplete sql1 cn', () => {
+        const errors = sql.validate(sql1);
+        expect(errors.length).toBe(1);
+        expect(errors[0].message).toBe('语句不完整');
+    });
+
+    test('validate unComplete sql2 cn', () => {
+        const errors = sql.validate(sql2);
+        expect(errors.length).toBe(1);
+        expect(errors[0].message).toBe(
+            '语句不完整，期望一个存在的table或者一个存在的view或者一个存在的function或者一个关键字'
+        );
+    });
+
+    test('validate unComplete sql3 cn', () => {
+        const errors = sql.validate(sql3);
+        expect(errors.length).toBe(1);
+        expect(errors[0].message).toBe(`'aaa' 在此位置无效，期望一个存在的namespace或者一个关键字`);
+    });
+});

test/parser/sql/errorStrategy.test.ts

@@ -0,0 +1,64 @@
+import { Sql, SqlSplitListener } from 'src/parser/sql';
+import { SqlParserListener } from 'src/lib/sql/SqlParserListener';
+
+const validSQL1 = `INSERT INTO country_page_view
+VALUES ('Chinese', 'mumiao', 18),
+    ('Amercian', 'georage', 22);`;
+const validSQL2 = 'SELECT * FROM tb;';
+const inValidSQL = 'CREATE TABLE;';
+
+describe('Sql ErrorStrategy test', () => {
+    const sql = new Sql();
+
+    // TODO: handle unexpected case
+    // test('begin inValid', () => {
+    //     const sqlText = [inValidSQL, validSQL1, validSQL2].join('\n');
+    //     // parse with empty errorListener
+    //     const parseTree = sql.parse(sqlText, () => {});
+    //     const splitListener = new SqlSplitListener();
+    //     sql.listen(splitListener as SqlParserListener, parseTree);
+
+    //     const statementCount = splitListener.statementsContext.length;
+    //     splitListener.statementsContext.map((item, index) => {
+    //         if (index !== statementCount - 1 && index !== statementCount - 2) {
+    //             expect(item.exception).not.toBe(null);
+    //         } else {
+    //             expect(item.exception).toBe(null);
+    //         }
+    //     });
+    // });
+
+    // TODO: handle unexpected case
+    // test('middle inValid', () => {
+    //     const sqlText = [validSQL1, inValidSQL, validSQL2].join('\n');
+    //     // parse with empty errorListener
+    //     const parseTree = sql.parse(sqlText, () => {});
+    //     const splitListener = new SqlSplitListener();
+    //     sql.listen(splitListener as SqlParserListener, parseTree);
+
+    //     const statementCount = splitListener.statementsContext.length;
+    //     splitListener.statementsContext.map((item, index) => {
+    //         if (index !== statementCount - 1 && index !== 0) {
+    //             expect(item.exception).not.toBe(null);
+    //         } else {
+    //             expect(item.exception).toBe(null);
+    //         }
+    //     });
+    // });
+
+    test('end inValid', () => {
+        const sqlText = [validSQL1, validSQL2, inValidSQL].join('\n');
+        // parse with empty errorListener
+        const parseTree = sql.parse(sqlText, () => {});
+        const splitListener = new SqlSplitListener();
+        sql.listen(splitListener as SqlParserListener, parseTree);
+
+        splitListener.statementsContext.map((item, index) => {
+            if (index !== 0 && index !== 1) {
+                expect(item.exception).not.toBe(null);
+            } else {
+                expect(item.exception).toBe(null);
+            }
+        });
+    });
+});

test/parser/sql/lexer.test.ts

@@ -0,0 +1,17 @@
+import { Sql } from 'src/parser/sql';
+
+describe('Sql Lexer tests', () => {
+    const sql = new Sql();
+
+    test('select id,name from user1;', () => {
+        const sqlText = `select id,name from user1;`;
+        const tokens = sql.getAllTokens(sqlText);
+        expect(tokens.length).toBe(10);
+    });
+
+    test('SELECT * FROM t WHERE x = 1 AND y = 2;', () => {
+        const sqlText = `SELECT * FROM t WHERE x = 1 AND y = 2;`;
+        const tokens = sql.getAllTokens(sqlText);
+        expect(tokens.length).toBe(24);
+    });
+});

test/parser/sql/listener.test.ts

@@ -0,0 +1,63 @@
+import { Sql } from 'src/parser/sql';
+import { SqlParserListener } from 'src/lib/sql/SqlParserListener';
+
+describe('Sql Listener Tests', () => {
+    const expectTableName = 'user1';
+    const sqlText = `select id,name,sex from ${expectTableName};`;
+    const sql = new Sql();
+
+    const parseTree = sql.parse(sqlText);
+
+    test('Listener exitTableName', () => {
+        class MyListener extends SqlParserListener {
+            result = '';
+            exitTableName = (ctx): void => {
+                this.result = ctx.getText().toLowerCase();
+            };
+        }
+        const listener = new MyListener();
+
+        sql.listen(listener, parseTree);
+        expect(listener.result).toBe(expectTableName);
+    });
+
+    test('Split sql listener', async () => {
+        const singleStatementArr = [
+            `SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person SORT BY name ASC, age DESC;`,
+
+            `INSERT INTO students FROM applicants SELECT name, address, student_id WHERE qualified = true;`,
+
+            `CREATE TABLE student_bucket
+                USING parquet
+                CLUSTERED BY (id) INTO 4 buckets (
+                WITH tmpTable AS (
+                    SELECT * FROM student WHERE id > 100
+                )
+                SELECT * FROM tmpTable
+            );`,
+        ];
+
+        const sqlText = singleStatementArr.join('\n');
+        const sqlSlices = sql.splitSQLByStatement(sqlText);
+
+        expect(sqlSlices).not.toBeNull();
+
+        // check text in result
+        expect(sqlSlices.map((item) => item.text)).toEqual(singleStatementArr);
+
+        // check startIndex and endIndex in result
+        sqlSlices.forEach((slice, index) => {
+            expect(sqlText.slice(slice.startIndex, slice.endIndex + 1)).toBe(
+                singleStatementArr[index]
+            );
+        });
+
+        // check lineNumber in result
+        expect(sqlSlices[0].startLine).toBe(1);
+        expect(sqlSlices[0].endLine).toBe(1);
+        expect(sqlSlices[1].startLine).toBe(2);
+        expect(sqlSlices[1].endLine).toBe(2);
+        expect(sqlSlices[2].startLine).toBe(3);
+        expect(sqlSlices[2].endLine).toBe(10);
+    });
+});

test/parser/sql/suggestion/fixtures/multipleStatement.sql

@@ -0,0 +1,21 @@
+CREATE TABLE  VALUES -- unfinished
+
+CREATE TABLE student (id INT, name STRING, age INT) STORED AS ORC;
+
+CREATE SCHEMA customer_db WITH DBPROPERTIES (ID=001, Name='John');
+
+ALTER TABLE StudentInfo ADD COLUMNS (LastName string, DOB timestamp);
+
+SELECT * FROM db. ; -- unfinished
+
+INSERT INTO weather (date, city, temp_hi, temp_lo) VALUES ('1994-11-29', 'Hayward', 54, 37);
+
+DESC EXTENDED students name;
+
+INSERT INTO  weather (date, city, temp_hi, temp_lo) VALUES ('1994-11-29', 'Hayward', 54, 37); -- unfinished
+
+DROP TABLE IF EXISTS employable;
+
+DROP TEMPORARY FUNCTION test_avg;
+
+INSERT INTO products (product_no, name, price) SELECT * FROM db. ; -- unfinished

test/parser/sql/suggestion/fixtures/suggestionWithEntity.sql

@@ -0,0 +1,11 @@
+SELECT  FROM my_db.tb;
+
+SELECT name, calculate_age(birthdate) AS age,  FROM students;
+
+INSERT INTO insert_tb SELECT  FROM from_tb;
+
+INSERT INTO insert_tb SELECT id, age,  FROM from_tb;
+
+CREATE TABLE sorted_census_data AS SELECT  FROM unsorted_census_data;
+
+CREATE TABLE sorted_census_data AS SELECT id, age,  FROM unsorted_census_data;

test/parser/sql/suggestion/fixtures/syntaxSuggestion.sql

@@ -0,0 +1,55 @@
+INSERT INTO db.tb ;
+
+SELECT * FROM db.;
+
+CREATE TABLE db. VALUES;
+
+DROP TABLE IF EXISTS db.a;
+
+CREATE OR REPLACE VIEW db.v;
+
+DROP VIEW db.v ;
+
+CREATE FUNCTION fn1;
+
+SELECT name, calculate_age(birthday) AS age FROM students;
+
+CREATE DATABASE db;
+
+DROP SCHEMA IF EXISTS sch;
+
+ANALYZE TABLE students COMPUTE STATISTICS FOR COLUMNS name, co ;
+
+ALTER TABLE StudentInfo ADD COLUMNS (LastName string,  );
+
+ALTER TABLE StudentInfo RENAME COLUMN ;
+
+ALTER TABLE StudentInfo RENAME COLUMN name TO t;
+
+ALTER TABLE StudentInfo DROP COLUMNS (LastName, );
+
+ALTER TABLE StudentInfo CHANGE FirstName;
+
+INSERT INTO students (  );
+
+INSERT INTO students ( id, n );
+
+SELECT ;
+
+SELECT id, n;
+
+SELECT  FROM tbl;
+
+SELECT id, n FROM tbl;
+
+SELECT id, n FROM tbl GROUP BY  ;
+
+SELECT id, n FROM tbl ORDER BY name, i ;
+
+SELECT id FROM tb1 GROUP BY ROLLUP(  );
+
+OPTIMIZE db.tb;
+
+OPTIMIZE db.tb ZORDER BY  ;
+
+OPTIMIZE db.tb ZORDER BY name, i;

test/parser/sql/suggestion/fixtures/tokenSuggestion.sql

@@ -0,0 +1,18 @@
+ALTER 
+;
+CREATE 
+;
+DELETE 
+;
+DESCRIBE 
+;
+DROP 
+;
+INSERT 
+;
+LOAD 
+;
+SHOW 
+;
+EXPORT 
+;