From 435166d88a0eacea5bb9f96e76394de012966d12 Mon Sep 17 00:00:00 2001 From: worksofliam Date: Thu, 3 Oct 2024 10:38:27 -0400 Subject: [PATCH] Improved support for parser conditional and additional statement types Signed-off-by: worksofliam --- src/language/sql/document.ts | 25 +- src/language/sql/statement.ts | 84 ++++-- src/language/sql/tests/blocks.test.ts | 78 +++-- src/language/sql/tests/statements.test.ts | 332 +++++++++++++--------- src/language/sql/tokens.ts | 15 +- src/language/sql/types.ts | 42 ++- 6 files changed, 391 insertions(+), 185 deletions(-) diff --git a/src/language/sql/document.ts b/src/language/sql/document.ts index 2bbf41df..6664035e 100644 --- a/src/language/sql/document.ts +++ b/src/language/sql/document.ts @@ -35,6 +35,7 @@ export default class Document { let statementStart = 0; for (let i = 0; i < tokens.length; i++) { + const upperValue = tokens[i].value?.toUpperCase(); switch (tokens[i].type) { case `semicolon`: const statementTokens = tokens.slice(statementStart, i); @@ -45,19 +46,25 @@ export default class Document { break; case `statementType`: - currentStatementType = StatementTypeWord[tokens[i].value?.toUpperCase()]; + currentStatementType = StatementTypeWord[upperValue]; break; case `keyword`: - switch (tokens[i].value?.toUpperCase()) { + switch (upperValue) { case `LOOP`: - // This handles the case that 'END LOOP' is supported. - if (currentStatementType === StatementType.End) { - break; - } + case `THEN`: case `BEGIN`: case `DO`: - case `THEN`: + // This handles the case that 'END LOOP' is supported. + if (upperValue === `LOOP` && currentStatementType === StatementType.End) { + break; + } + + // Support for THEN in conditionals + if (upperValue === `THEN` && !Statement.typeIsConditional(currentStatementType)) { + break; + } + // We include BEGIN in the current statement // then the next statement beings const statementTokens = tokens.slice(statementStart, i+1); @@ -102,7 +109,7 @@ export default class Document { let depth = 0; for (const statement of this.statements) { - if (statement.isBlockEnder()) { + if (statement.isCompoundEnd()) { if (depth > 0) { currentGroup.push(statement); @@ -118,7 +125,7 @@ export default class Document { currentGroup = []; } } else - if (statement.isBlockOpener()) { + if (statement.isCompoundStart()) { if (depth > 0) { currentGroup.push(statement); } else { diff --git a/src/language/sql/statement.ts b/src/language/sql/statement.ts index e40a84a1..a9a14182 100644 --- a/src/language/sql/statement.ts +++ b/src/language/sql/statement.ts @@ -7,6 +7,7 @@ const tokenIs = (token: Token|undefined, type: string, value?: string) => { export default class Statement { public type: StatementType = StatementType.Unknown; + private label: string|undefined; constructor(public tokens: Token[], public range: IRange) { this.tokens = this.tokens.filter(newToken => newToken.type !== `newline`); @@ -19,11 +20,15 @@ export default class Statement { first = this.tokens[2]; } - if (tokenIs(first, `statementType`) || tokenIs(first, `keyword`, `END`) || tokenIs(first, `keyword`, `BEGIN`)) { - const wordValue = first.value?.toUpperCase(); - - this.type = StatementTypeWord[wordValue]; + if (tokenIs(first, `word`) && tokenIs(this.tokens[1], `colon`)) { + // Possible label? + this.label = first.value; + first = this.tokens[2]; } + + const wordValue = first.value?.toUpperCase(); + + this.type = StatementTypeWord[wordValue] || StatementType.Unknown; switch (this.type) { case StatementType.Create: @@ -35,7 +40,7 @@ export default class Statement { } } - isBlockOpener() { + isCompoundStart() { if (this.tokens.length === 1 && tokenIs(this.tokens[0], `keyword`, `BEGIN`)) { return true; } @@ -51,7 +56,19 @@ export default class Statement { return false; } - isBlockEnder() { + static typeIsConditional(type: StatementType) { + return [StatementType.If, StatementType.While, StatementType.Loop, StatementType.For].includes(type); + } + + isConditionStart() { + return Statement.typeIsConditional(this.type); + } + + isConditionEnd() { + return this.type === StatementType.End && this.tokens.length > 1; + } + + isCompoundEnd() { return this.type === StatementType.End && this.tokens.length === 1; } @@ -314,17 +331,21 @@ export default class Statement { } const basicQueryFinder = (startIndex: number): void => { + let currentClause: undefined|"select"|"from"; for (let i = startIndex; i < this.tokens.length; i++) { if (tokenIs(this.tokens[i], `clause`, `FROM`)) { - inFromClause = true; - } else if (inFromClause && tokenIs(this.tokens[i], `clause`) || tokenIs(this.tokens[i], `join`) || tokenIs(this.tokens[i], `closebracket`)) { - inFromClause = false; + currentClause = `from`; + } + else if (tokenIs(this.tokens[i], `statementType`, `SELECT`)) { + currentClause = `select`; + } else if (currentClause === `from` && tokenIs(this.tokens[i], `clause`) || tokenIs(this.tokens[i], `join`) || tokenIs(this.tokens[i], `closebracket`)) { + currentClause = undefined; } if (tokenIs(this.tokens[i], `clause`, `FROM`) || (this.type !== StatementType.Select && tokenIs(this.tokens[i], `clause`, `INTO`)) || tokenIs(this.tokens[i], `join`) || - (inFromClause && tokenIs(this.tokens[i], `comma`) + (currentClause === `from` && tokenIs(this.tokens[i], `comma`) )) { const sqlObj = this.getRefAtToken(i+1); if (sqlObj) { @@ -334,6 +355,15 @@ export default class Statement { i += 3; //For the brackets } } + } else if (currentClause === `select` && tokenIs(this.tokens[i], `function`)) { + const sqlObj = this.getRefAtToken(i); + if (sqlObj) { + doAdd(sqlObj); + i += sqlObj.tokens.length; + if (sqlObj.isUDTF || sqlObj.fromLateral) { + i += 3; //For the brackets + } + } } } } @@ -592,7 +622,7 @@ export default class Statement { } if (options.withSystemName) { - if (tokenIs(this.tokens[endIndex+1], `keyword`, `FOR`) && tokenIs(this.tokens[endIndex+2], `word`, `SYSTEM`) && tokenIs(this.tokens[endIndex+3], `word`, `NAME`)) { + if (tokenIs(this.tokens[endIndex+1], `statementType`, `FOR`) && tokenIs(this.tokens[endIndex+2], `word`, `SYSTEM`) && tokenIs(this.tokens[endIndex+3], `word`, `NAME`)) { if (this.tokens[endIndex+4] && NameTypes.includes(this.tokens[endIndex+4].type)) { sqlObj.object.system = this.tokens[endIndex+4].value; } @@ -624,10 +654,25 @@ export default class Statement { switch (currentToken.type) { case `statementType`: - if (declareStmt) continue; + const currentValue = currentToken.value.toLowerCase(); + if (declareStmt) { + if (currentValue === `for`) { + ranges.push({ + type: `remove`, + range: { + start: declareStmt.range.start, + end: currentToken.range.end + } + }); + + declareStmt = undefined; + } + + continue; + }; // If we're in a DECLARE, it's likely a cursor definition - if (currentToken.value.toLowerCase() === `declare`) { + if (currentValue === `declare`) { declareStmt = currentToken; } break; @@ -716,19 +761,6 @@ export default class Statement { } }); } - } else - if (declareStmt && tokenIs(currentToken, `keyword`, `FOR`)) { - // If we're a DECLARE, and we found the FOR keyword, the next - // set of tokens should be the select. - ranges.push({ - type: `remove`, - range: { - start: declareStmt.range.start, - end: currentToken.range.end - } - }); - - declareStmt = undefined; } break; } diff --git a/src/language/sql/tests/blocks.test.ts b/src/language/sql/tests/blocks.test.ts index 1da9a81d..1b74cb55 100644 --- a/src/language/sql/tests/blocks.test.ts +++ b/src/language/sql/tests/blocks.test.ts @@ -1,9 +1,12 @@ -import { assert, describe, expect, test } from 'vitest' +import { describe, expect, test } from 'vitest' import Document from '../document'; -import { StatementType } from '../types'; -describe(`Block statement tests`, () => { +const parserScenarios = describe.each([ + {newDoc: (content: string) => new Document(content), isFormatted: false}, +]); + +parserScenarios(`Block statement tests`, ({newDoc, isFormatted}) => { test('Block start tests', () => { const lines = [ `CREATE ALIAS "TestDelimiters"."Delimited Alias" FOR "TestDelimiters"."Delimited Table";`, @@ -15,19 +18,19 @@ describe(`Block statement tests`, () => { `LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`, ].join(`\n`); - const doc = new Document(lines); + const doc = newDoc(lines); // CREATE, CREATE, RETURN, END, CREATE, SET, END expect(doc.statements.length).toBe(7); const aliasDef = doc.statements[0]; - expect(aliasDef.isBlockOpener()).toBeFalsy(); + expect(aliasDef.isCompoundStart()).toBeFalsy(); const functionDef = doc.statements[1]; - expect(functionDef.isBlockOpener()).toBeTruthy(); + expect(functionDef.isCompoundStart()).toBeTruthy(); const procedureDef = doc.statements[4]; - expect(procedureDef.isBlockOpener()).toBeTruthy(); + expect(procedureDef.isCompoundStart()).toBeTruthy(); }); test('Compound statement test', () => { @@ -53,21 +56,21 @@ describe(`Block statement tests`, () => { `LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`, ].join(`\n`); - const doc = new Document(lines); + const doc = newDoc(lines); const t = doc.statements.length; const aliasDef = doc.statements[0]; - expect(aliasDef.isBlockOpener()).toBeFalsy(); + expect(aliasDef.isCompoundStart()).toBeFalsy(); const functionDef = doc.statements[1]; - expect(functionDef.isBlockOpener()).toBeTruthy(); + expect(functionDef.isCompoundStart()).toBeTruthy(); const functionEnd = doc.statements[3]; - expect(functionEnd.isBlockEnder()).toBeTruthy(); + expect(functionEnd.isCompoundEnd()).toBeTruthy(); const beginBlock = doc.statements[4]; - expect(beginBlock.isBlockOpener()).toBeTruthy(); + expect(beginBlock.isCompoundStart()).toBeTruthy(); }); test('Statement groups', () => { @@ -96,23 +99,58 @@ describe(`Block statement tests`, () => { `LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`, ].join(`\r\n`); - const doc = new Document(lines); + const doc = newDoc(lines); const groups = doc.getStatementGroups(); expect(groups.length).toBe(4); const aliasStatement = groups[0]; - const aliasSubstring = lines.substring(aliasStatement.range.start, aliasStatement.range.end); + const aliasSubstring = doc.content.substring(aliasStatement.range.start, aliasStatement.range.end); expect(aliasSubstring).toBe(`CREATE ALIAS "TestDelimiters"."Delimited Alias" FOR "TestDelimiters"."Delimited Table"`); + const functionStatement = groups[1]; + const functionSubstring = doc.content.substring(functionStatement.range.start, functionStatement.range.end); + + if (isFormatted) { + expect(functionSubstring).toBe([ + `CREATE FUNCTION "TestDelimiters"."Delimited Function"(`, + ` "Delimited Parameter" INTEGER`, + `) RETURNS INTEGER LANGUAGE SQL BEGIN`, + ` RETURN "Delimited Parameter";`, + `END`, + ].join(`\r\n`)); + } else { + expect(functionSubstring).toBe([ + `CREATE FUNCTION "TestDelimiters"."Delimited Function" ("Delimited Parameter" INTEGER) `, + `RETURNS INTEGER LANGUAGE SQL BEGIN RETURN "Delimited Parameter"; END` + ].join(`\r\n`)) + } const beginStatement = groups[2]; - const compoundSubstring = lines.substring(beginStatement.range.start, beginStatement.range.end); - expect(compoundSubstring).toBe(compoundStatement); + expect(beginStatement.statements.length).toBe(9); + const compoundSubstring = doc.content.substring(beginStatement.range.start, beginStatement.range.end); + + if (isFormatted) { + expect(compoundSubstring).toBe([ + `BEGIN`, + ` DECLARE already_exists SMALLINT DEFAULT 0;`, + ` DECLARE dup_object_hdlr CONDITION FOR SQLSTATE '42710';`, + ` DECLARE CONTINUE HANDLER FOR dup_object_hdlr SET already_exists = 1;`, + ` CREATE TABLE table1(`, + ` col1 INT`, + ` );`, + ` IF already_exists > 0 THEN;`, + ` DELETE FROM table1;`, + ` END IF;`, + `END`, + ].join(`\r\n`)); + } else { + expect(compoundSubstring).toBe(compoundStatement); + } }); }); -describe(`Definition tests`, () => { +parserScenarios(`Definition tests`, ({newDoc}) => { test(`Alias, function, procedure`, () => { const lines = [ `CREATE ALIAS "TestDelimiters"."Delimited Alias" FOR "TestDelimiters"."Delimited Table";`, @@ -124,7 +162,7 @@ describe(`Definition tests`, () => { `LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`, ].join(`\n`); - const doc = new Document(lines); + const doc = newDoc(lines); const defs = doc.getDefinitions(); @@ -161,7 +199,7 @@ describe(`Definition tests`, () => { `END;`, ].join(`\r\n`); - const doc = new Document(lines); + const doc = newDoc(lines); const defs = doc.getDefinitions(); @@ -245,7 +283,7 @@ describe(`Definition tests`, () => { `END ; `, ].join(`\n`); - const doc = new Document(lines); + const doc = newDoc(lines); const groups = doc.getStatementGroups(); expect(groups.length).toBe(1); diff --git a/src/language/sql/tests/statements.test.ts b/src/language/sql/tests/statements.test.ts index f06008aa..cd046ceb 100644 --- a/src/language/sql/tests/statements.test.ts +++ b/src/language/sql/tests/statements.test.ts @@ -3,16 +3,20 @@ import SQLTokeniser from '../tokens' import Document from '../document'; import { ClauseType, StatementType } from '../types'; -describe(`Basic statements`, () => { +const parserScenarios = describe.each([ + {newDoc: (content: string) => new Document(content)}, +]); + +parserScenarios(`Basic statements`, ({newDoc}) => { test('One statement, no end', () => { - const document = new Document(`select * from sample`); + const document = newDoc(`select * from sample`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(4); }); test('One statement, with end', () => { - const document = new Document(`select * from sample;`); + const document = newDoc(`select * from sample;`); expect(document.statements.length).toBe(1); expect(document.statements[0].type).toBe(StatementType.Select); @@ -20,7 +24,7 @@ describe(`Basic statements`, () => { }); test('Two statements, one end', () => { - const document = new Document([ + const document = newDoc([ `select * from sample;`, `select a from b.b` ].join(`\n`)); @@ -34,7 +38,7 @@ describe(`Basic statements`, () => { }); test('Two statements, both end', () => { - const document = new Document([ + const document = newDoc([ `select * from sample;`, `select a from b.b;` ].join(`\n`)); @@ -45,7 +49,7 @@ describe(`Basic statements`, () => { }); test('Two statements, both end, with comments', () => { - const document = new Document([ + const document = newDoc([ `select * from sample; --Yep`, `select a from b.b; -- Nope` ].join(`\n`)); @@ -56,7 +60,7 @@ describe(`Basic statements`, () => { }); test('Two statements, both end, with comments, trimmed', () => { - const document = new Document([ + const document = newDoc([ ``, `select * from sample; --Yep`, ``, @@ -72,9 +76,9 @@ describe(`Basic statements`, () => { }); }); -describe(`Object references`, () => { +parserScenarios(`Object references`, ({newDoc}) => { test('SELECT: Simple unqualified object', () => { - const document = new Document(`select * from sample;`); + const document = newDoc(`select * from sample;`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(4); @@ -92,7 +96,7 @@ describe(`Object references`, () => { }); test('SELECT: Simple qualified object', () => { - const document = new Document(`select * from myschema.sample;`); + const document = newDoc(`select * from myschema.sample;`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(6); @@ -110,7 +114,7 @@ describe(`Object references`, () => { }); test('SELECT: Simple qualified object with alias', () => { - const document = new Document(`select * from myschema.sample as a;`); + const document = newDoc(`select * from myschema.sample as a;`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(8); @@ -128,7 +132,7 @@ describe(`Object references`, () => { }); test('SELECT: Simple unqualified object with alias (no AS)', () => { - const document = new Document(`select * from sample a;`); + const document = newDoc(`select * from sample a;`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(5); @@ -146,7 +150,7 @@ describe(`Object references`, () => { }); test('SELECT: Simple qualified object with alias (no AS)', () => { - const document = new Document(`select * from myschema.sample a;`); + const document = newDoc(`select * from myschema.sample a;`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(7); @@ -164,7 +168,7 @@ describe(`Object references`, () => { }); test('SELECT: Simple qualified object with alias (system naming)', () => { - const document = new Document(`select * from myschema/sample as a;`); + const document = newDoc(`select * from myschema/sample as a;`); expect(document.statements.length).toBe(1); expect(document.statements[0].tokens.length).toBe(8); @@ -192,14 +196,14 @@ describe(`Object references`, () => { `WHERE ORCUID = CUID`, ].join(`\r\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); const statement = document.statements[0]; const refs = statement.getObjectReferences(); - expect(refs.length).toBe(3); + expect(refs.length).toBe(5); expect(statement.getClauseForOffset(10)).toBe(ClauseType.Unknown); expect(statement.getClauseForOffset(125)).toBe(ClauseType.From); @@ -216,7 +220,7 @@ describe(`Object references`, () => { `WHERE LASTNAME > 'S';`, ].join(`\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); @@ -246,7 +250,7 @@ describe(`Object references`, () => { `WHERE LASTNAME > 'S'`, ].join(`\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); @@ -279,7 +283,7 @@ describe(`Object references`, () => { `WHERE LASTNAME > 'S';`, ].join(`\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); @@ -309,7 +313,7 @@ describe(`Object references`, () => { `WHERE LASTNAME > 'S'`, ].join(`\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); @@ -336,7 +340,7 @@ describe(`Object references`, () => { `SELECT * FROM A CROSS JOIN B`, ].join(`\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); @@ -366,7 +370,7 @@ describe(`Object references`, () => { `WHERE LASTNAME > 'S'`, ].join(`\n`); - const document = new Document(query); + const document = newDoc(query); expect(document.statements.length).toBe(1); @@ -398,7 +402,7 @@ describe(`Object references`, () => { `insert into "myschema".hashtags (tag, base_talk) values('#hi', 2);`, ].join(`\r\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(2); @@ -423,7 +427,7 @@ describe(`Object references`, () => { `delete from talks where id > 2;` ].join(`\r\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -441,7 +445,7 @@ describe(`Object references`, () => { `call create_Sql_sample('MYNEWSCHEMA');` ].join(`\r\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -459,7 +463,7 @@ describe(`Object references`, () => { `call "QSYS".create_Sql_sample('MYNEWSCHEMA');` ].join(`\r\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -480,7 +484,7 @@ describe(`Object references`, () => { ` ON DELETE SET NULL;`, ].join(`\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -509,7 +513,7 @@ describe(`Object references`, () => { ` ON DELETE SET NULL;`, ].join(`\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -539,7 +543,7 @@ describe(`Object references`, () => { ` ON DEPARTMENT (MGRNO);`, ].join(`\r\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(2); @@ -588,7 +592,7 @@ describe(`Object references`, () => { ` ON other.DEPARTMENT (MGRNO);`, ].join(`\r\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(2); @@ -635,7 +639,7 @@ describe(`Object references`, () => { ` PRIMARY KEY( COL_B ) );`, ].join(`\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -658,7 +662,7 @@ describe(`Object references`, () => { `);`, ].join(`\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -701,7 +705,7 @@ describe(`Object references`, () => { ` ;`, ].join(`\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -747,7 +751,7 @@ describe(`Object references`, () => { ` ;`, ].join(`\n`); - const document = new Document(content); + const document = newDoc(content); expect(document.statements.length).toBe(1); @@ -797,7 +801,7 @@ describe(`Object references`, () => { `end;`, ].join(`\n`); - const document = new Document(lines); + const document = newDoc(lines); const groups = document.getStatementGroups(); expect(groups.length).toBe(1); @@ -829,9 +833,11 @@ describe(`Object references`, () => { const selectStatement = group.statements[2]; expect(selectStatement.type).toBe(StatementType.Select); const refsC = selectStatement.getObjectReferences(); - expect(refsC.length).toBe(1); + expect(refsC.length).toBe(2); expect(refsC[0].createType).toBeUndefined(); - expect(refsC[0].object.name).toBe(`employee`); + expect(refsC[0].object.name).toBe(`sum`); + expect(refsC[1].createType).toBeUndefined(); + expect(refsC[1].object.name).toBe(`employee`); }); test(`CREATE FUNCTION: with multiple parameters`, () => { @@ -858,7 +864,7 @@ describe(`Object references`, () => { `end;`, ].join(`\n`); - const document = new Document(lines); + const document = newDoc(lines); const groups = document.getStatementGroups(); expect(groups.length).toBe(1); @@ -886,7 +892,7 @@ describe(`Object references`, () => { `EXTERNAL NAME LIB.PROGRAM GENERAL;`, ].join(`\n`); - const document = new Document(lines); + const document = newDoc(lines); const groups = document.getStatementGroups(); expect(groups.length).toBe(1); @@ -913,7 +919,7 @@ describe(`Object references`, () => { }); test(`DECLARE VARIABLE`, () => { - const document = new Document(`declare watsonx_response Varchar(10000) CCSID 1208;`); + const document = newDoc(`declare watsonx_response Varchar(10000) CCSID 1208;`); const groups = document.getStatementGroups(); expect(groups.length).toBe(1); @@ -928,7 +934,7 @@ describe(`Object references`, () => { }); test(`CREATE OR REPLACE VARIABLE`, () => { - const document = new Document(`create or replace variable watsonx.apiVersion varchar(10) ccsid 1208 default '2023-07-07';`); + const document = newDoc(`create or replace variable watsonx.apiVersion varchar(10) ccsid 1208 default '2023-07-07';`); const groups = document.getStatementGroups(); @@ -1059,10 +1065,10 @@ describe(`Object references`, () => { expect(group.statements.map(s => s.type)).toEqual([ 'Create', 'Declare', 'Declare', 'Call', + 'Loop', 'Set', 'Unknown', 'Unknown', - 'Unknown', 'Unknown', - 'Unknown', 'Call', - 'Unknown', 'End', + 'If', 'Call', + 'Leave', 'End', 'Call', 'End', 'Unknown', 'End' ]); @@ -1086,9 +1092,9 @@ describe(`Object references`, () => { }) }); -describe(`Offset reference tests`, () => { +parserScenarios(`Offset reference tests`, ({newDoc}) => { test(`Writing select`, () => { - const document = new Document(`select * from sample.;`); + const document = newDoc(`select * from sample.;`); expect(document.statements.length).toBe(1); @@ -1101,7 +1107,7 @@ describe(`Offset reference tests`, () => { }); test(`Writing select, invalid middle`, () => { - const document = new Document(`select b. from department b;`); + const document = newDoc(`select b. from department b;`); expect(document.statements.length).toBe(1); @@ -1120,7 +1126,7 @@ describe(`Offset reference tests`, () => { }); }); -describe(`PL body tests`, () => { +parserScenarios(`PL body tests`, ({newDoc}) => { test(`CREATE PROCEDURE: with body`, () => { const lines = [ `CREATE PROCEDURE MEDIAN_RESULT_SET (OUT medianSalary DECIMAL(7,2))`, @@ -1152,12 +1158,12 @@ describe(`PL body tests`, () => { `END`, ].join(`\r\n`); - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; const medianResultSetProc = statements[0]; expect(medianResultSetProc.type).toBe(StatementType.Create); - expect(medianResultSetProc.isBlockOpener()).toBe(true); + expect(medianResultSetProc.isCompoundStart()).toBe(true); const parms = medianResultSetProc.getRoutineParameters(); expect(parms.length).toBe(1); @@ -1178,6 +1184,12 @@ describe(`PL body tests`, () => { // END expect(endStatements[1].tokens.length).toBe(1); + + const whileStatement = statements.find(stmt => stmt.type === StatementType.While); + expect(whileStatement).toBeDefined(); + + const fetchStatement = statements.find(stmt => stmt.type === StatementType.Fetch); + expect(fetchStatement).toBeDefined(); }); test(`CREATE PROCEDURE followed by CALL statement`, () => { @@ -1213,12 +1225,12 @@ describe(`PL body tests`, () => { `CALL MEDIAN_RESULT_SET(12345.55);`, ].join(`\r\n`); - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; const medianResultSetProc = statements[0]; expect(medianResultSetProc.type).toBe(StatementType.Create); - expect(medianResultSetProc.isBlockOpener()).toBe(true); + expect(medianResultSetProc.isCompoundStart()).toBe(true); const parms = medianResultSetProc.getRoutineParameters(); expect(parms.length).toBe(1); @@ -1237,7 +1249,7 @@ describe(`PL body tests`, () => { const callStatement = statements[statements.length - 1]; expect(callStatement.type).toBe(StatementType.Call); - expect(callStatement.isBlockOpener()).toBe(false); + expect(callStatement.isCompoundStart()).toBe(false); const blockParent = callStatement.getCallableDetail(callStatement.tokens[3].range.start); expect(blockParent).toBeDefined(); @@ -1268,7 +1280,7 @@ describe(`PL body tests`, () => { `select * from Temp02`, ].join(`\n`); - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1279,27 +1291,36 @@ describe(`PL body tests`, () => { const refs = statement.getObjectReferences(); const ctes = statement.getCTEReferences(); - expect(refs.length).toBe(7); + expect(refs.length).toBe(10); expect(refs[0].object.name).toBe(`shipments`); expect(refs[0].alias).toBe(`s`); expect(refs[1].object.name).toBe(`BillingDate`); expect(refs[1].alias).toBeUndefined(); - expect(refs[2].object.name).toBe(`Temp01`); - expect(refs[2].alias).toBe(`t1`); + expect(refs[2].object.name).toBe(`sum`); + expect(refs[2].alias).toBeUndefined(); expect(refs[3].object.name).toBe(`Temp01`); expect(refs[3].alias).toBe(`t1`); - expect(refs[4].object.name).toBe(`Temp02`); - expect(refs[4].alias).toBe(`t2`); + expect(refs[4].object.name).toBe(`dec`); + expect(refs[4].alias).toBeUndefined(); + + expect(refs[5].object.name).toBe(`round`); + expect(refs[5].alias).toBeUndefined(); + + expect(refs[6].object.name).toBe(`Temp01`); + expect(refs[6].alias).toBe(`t1`); + + expect(refs[7].object.name).toBe(`Temp02`); + expect(refs[7].alias).toBe(`t2`); - expect(refs[5].object.name).toBe(`customers`); - expect(refs[5].alias).toBe(`c`); + expect(refs[8].object.name).toBe(`customers`); + expect(refs[8].alias).toBe(`c`); - expect(refs[6].object.name).toBe(`Temp02`); - expect(refs[6].alias).toBeUndefined(); + expect(refs[9].object.name).toBe(`Temp02`); + expect(refs[9].alias).toBeUndefined(); expect(ctes.length).toBe(3); expect(ctes[0].name).toBe(`Temp01`); @@ -1310,11 +1331,13 @@ describe(`PL body tests`, () => { expect(ctes[2].name).toBe(`Temp03`); expect(ctes[2].columns.length).toBe(0); + const temp03Stmt = ctes[2].statement.getObjectReferences(); - expect(temp03Stmt.length).toBe(3); - expect(temp03Stmt[0].object.name).toBe(`Temp01`); - expect(temp03Stmt[1].object.name).toBe(`Temp02`); - expect(temp03Stmt[2].object.name).toBe(`customers`); + + expect(temp03Stmt[0].object.name).toBe(`dec`); + expect(temp03Stmt[1].object.name).toBe(`Temp01`); + expect(temp03Stmt[2].object.name).toBe(`Temp02`); + expect(temp03Stmt[3].object.name).toBe(`customers`); }) test(`WITH: explicit columns`, () => { @@ -1326,7 +1349,7 @@ describe(`PL body tests`, () => { `select * from cteme` ].join(`\r\n`); - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1355,7 +1378,7 @@ describe(`PL body tests`, () => { test(`SELECT: table function`, () => { const lines = `select * from table(qsys2.mti_info());`; - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1373,7 +1396,7 @@ describe(`PL body tests`, () => { test(`SELECT: table function with name (no AS)`, () => { const lines = `select * from table(qsys2.mti_info()) x;`; - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1391,7 +1414,7 @@ describe(`PL body tests`, () => { test(`SELECT: table function with name (with AS)`, () => { const lines = `select * from table(qsys2.mti_info()) as x;`; - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(1); @@ -1419,7 +1442,7 @@ describe(`PL body tests`, () => { `stop;`, ].join(`\n`); - const document = new Document(lines); + const document = newDoc(lines); const statements = document.statements; expect(statements.length).toBe(2); @@ -1427,10 +1450,65 @@ describe(`PL body tests`, () => { expect(statement.type).toBe(StatementType.Select); const objs = statement.getObjectReferences(); - expect(objs.length).toBe(1); - expect(objs[0].object.schema).toBe(`qsys2`); - expect(objs[0].object.name).toBe(`ACTIVE_JOB_INFO`); - }) + + expect(objs.length).toBe(2); + expect(objs[1].object.schema).toBe(`qsys2`); + expect(objs[1].object.name).toBe(`ACTIVE_JOB_INFO`); + }); + + test('CASE, WHEN, END', () => { + const lines = [ + `--`, + ``, + `--`, + `-- Hold any jobs that started running an SQL statement more than 2 hours ago.`, + `--`, + `select JOB_NAME,`, + ` case`, + ` when QSYS2.QCMDEXC('HLDJOB ' concat JOB_NAME) = 1 then 'Job Held'`, + ` else 'Job not held'`, + ` end as HLDJOB_RESULT`, + ` from table (`, + ` QSYS2.ACTIVE_JOB_INFO(DETAILED_INFO => 'ALL')`, + ` )`, + ` where SQL_STATEMENT_START_TIMESTAMP < current timestamp - 2 hours;`, + ].join(`\n`); + + const document = newDoc(lines); + + + const statements = document.statements; + expect(statements.length).toBe(1); + + const statement = statements[0]; + expect(statement.type).toBe(StatementType.Select); + + const objs = statement.getObjectReferences(); + + expect(objs.length).toBe(2); + }); + + + + test('SELECT statement with CASE', () => { + const content = [ + `SELECT`, + ` CLE,`, + ` CASE`, + ` WHEN CLE = 1 THEN 'FIRST' Else VALEUR End As VALEUR`, + `FROM`, + ` QTEMP.Test`, + ].join(` `); + + const document = new Document(content); + const statements = document.statements; + expect(statements.length).toBe(1); + + const statement = statements[0]; + expect(statement.type).toBe(StatementType.Select); + + + }); }); describe(`Parameter statement tests`, () => { @@ -1670,10 +1748,10 @@ describe(`Parameter statement tests`, () => { test(`Exec with basic DECLARE`, () => { const lines = [ `EXEC SQL`, - `DECLARE cursor-name SCROLL CURSOR FOR`, - `SELECT column-1, column-2`, - ` FROM table-name`, - ` WHERE column-1 = expression`, + `DECLARE cursor_name SCROLL CURSOR FOR`, + `SELECT column_1, column_2`, + `FROM table_name`, + `WHERE column_1 = expression`, ].join(`\n`); const document = new Document(lines); @@ -1685,9 +1763,9 @@ describe(`Parameter statement tests`, () => { const result = document.removeEmbeddedAreas(statement); expect(result.parameterCount).toBe(0); expect(result.content).toBe([ - `SELECT column-1, column-2`, - ` FROM table-name`, - ` WHERE column-1 = expression`, + `SELECT column_1, column_2`, + `FROM table_name`, + `WHERE column_1 = expression`, ].join(`\n`)); }); @@ -1704,6 +1782,50 @@ describe(`Parameter statement tests`, () => { expect(result.parameterCount).toBe(0); expect(result.content).toBe(content); }); + + test(`Callable blocks`, () => { + const lines = [ + `call qsys2.create_abcd();`, + `call qsys2.create_abcd(a, cool(a + b));`, + ].join(` `); + + const document = new Document(lines); + const statements = document.statements; + + expect(statements.length).toBe(2); + + const a = statements[0]; + expect(a.type).toBe(StatementType.Call); + + const b = statements[1]; + expect(b.type).toBe(StatementType.Call); + + const blockA = a.getBlockRangeAt(23); + expect(blockA).toMatchObject({ start: 5, end: 5 }); + + const callableA = a.getCallableDetail(23); + expect(callableA).toBeDefined(); + expect(callableA.parentRef.object.schema).toBe(`qsys2`); + expect(callableA.parentRef.object.name).toBe(`create_abcd`); + + const blockB = a.getBlockRangeAt(24); + expect(blockB).toMatchObject({ start: 5, end: 5 }); + + const callableB = a.getCallableDetail(24); + expect(callableB).toBeDefined(); + expect(callableB.parentRef.object.schema).toBe(`qsys2`); + expect(callableB.parentRef.object.name).toBe(`create_abcd`); + + const blockC = b.getBlockRangeAt(49); + expect(blockC).toMatchObject({ start: 5, end: 13 }); + + const callableC = b.getCallableDetail(49, true); + expect(callableC).toBeDefined(); + expect(callableC.tokens.length).toBe(4); + expect(callableC.tokens.some(t => t.type === `block` && t.block.length === 3)).toBeTruthy(); + expect(callableC.parentRef.object.schema).toBe(`qsys2`); + expect(callableC.parentRef.object.name).toBe(`create_abcd`); + }); }); describe(`Prefix tests`, () => { @@ -1725,48 +1847,4 @@ describe(`Prefix tests`, () => { expect(statement.type).toBe(StatementType.Select); }); -}); - -test(`Callable blocks`, () => { - const lines = [ - `call qsys2.create_abcd();`, - `call qsys2.create_abcd(a, cool(a + b));`, - ].join(` `); - - const document = new Document(lines); - const statements = document.statements; - - expect(statements.length).toBe(2); - - const a = statements[0]; - expect(a.type).toBe(StatementType.Call); - - const b = statements[1]; - expect(b.type).toBe(StatementType.Call); - - const blockA = a.getBlockRangeAt(23); - expect(blockA).toMatchObject({ start: 5, end: 5 }); - - const callableA = a.getCallableDetail(23); - expect(callableA).toBeDefined(); - expect(callableA.parentRef.object.schema).toBe(`qsys2`); - expect(callableA.parentRef.object.name).toBe(`create_abcd`); - - const blockB = a.getBlockRangeAt(24); - expect(blockB).toMatchObject({ start: 5, end: 5 }); - - const callableB = a.getCallableDetail(24); - expect(callableB).toBeDefined(); - expect(callableB.parentRef.object.schema).toBe(`qsys2`); - expect(callableB.parentRef.object.name).toBe(`create_abcd`); - - const blockC = b.getBlockRangeAt(49); - expect(blockC).toMatchObject({ start: 5, end: 13 }); - - const callableC = b.getCallableDetail(49, true); - expect(callableC).toBeDefined(); - expect(callableC.tokens.length).toBe(4); - expect(callableC.tokens.some(t => t.type === `block` && t.block.length === 3)).toBeTruthy(); - expect(callableC.parentRef.object.schema).toBe(`qsys2`); - expect(callableC.parentRef.object.name).toBe(`create_abcd`); }); \ No newline at end of file diff --git a/src/language/sql/tokens.ts b/src/language/sql/tokens.ts index e462c064..dc153c90 100644 --- a/src/language/sql/tokens.ts +++ b/src/language/sql/tokens.ts @@ -34,10 +34,18 @@ export default class SQLTokeniser { { name: `STATEMENTTYPE`, match: [{ type: `word`, match: (value: string) => { - return [`CREATE`, `ALTER`, `SELECT`, `WITH`, `INSERT`, `UPDATE`, `DELETE`, `DROP`, `CALL`, `DECLARE`].includes(value.toUpperCase()); + return [`CREATE`, `ALTER`, `SELECT`, `WITH`, `INSERT`, `UPDATE`, `DELETE`, `DROP`, `CALL`, `DECLARE`, `IF`, `FOR`, `WHILE`].includes(value.toUpperCase()); } }], becomes: `statementType`, }, + { + name: `CLAUSE-ORDER`, + match: [ + {type: `word`, match: (value: string) => {return value.toUpperCase() === `ORDER`}}, + {type: `word`, match: (value: string) => {return value.toUpperCase() === `BY`}} + ], + becomes: `clause`, + }, { name: `CLAUSE`, match: [{ type: `word`, match: (value: string) => { @@ -98,6 +106,11 @@ export default class SQLTokeniser { match: [{ type: `equals` }, { type: `morethan` }], becomes: `rightpipe`, }, + { + name: `NOT`, + match: [{type: `lessthan`}, {type: `morethan`}], + becomes: `not` + } ]; readonly spaces = [`\t`, ` `]; readonly splitParts: string[] = [`(`, `)`, `/`, `.`, `*`, `-`, `+`, `;`, `"`, `&`, `%`, `,`, `|`, `?`, `:`, `=`, `<`, `>`, `\n`, `\r`, ...this.spaces]; diff --git a/src/language/sql/types.ts b/src/language/sql/types.ts index b79f1479..0084b6e9 100644 --- a/src/language/sql/types.ts +++ b/src/language/sql/types.ts @@ -3,6 +3,7 @@ import Statement from "./statement"; export enum StatementType { Unknown = "Unknown", Create = "Create", + Close = "Close", Insert = "Insert", Select = "Select", With = "With", @@ -12,8 +13,27 @@ export enum StatementType { Begin = "Begin", Drop = "Drop", End = "End", + Else = "Else", + Elseif = "Elseif", Call = "Call", - Alter = "Alter" + Alter = "Alter", + Fetch = "Fetch", + For = "For", + Get = "Get", + Goto = "Goto", + If = "If", + Include = "Include", + Iterate = "Iterate", + Leave = "Leave", + Loop = "Loop", + Open = "Open", + Pipe = "Pipe", + Repeat = "Repeat", + Resignal = "Resignal", + Return = "Return", + Signal = "Signal", + Set = "Set", + While = "While" } export const StatementTypeWord = { @@ -26,9 +46,27 @@ export const StatementTypeWord = { 'DECLARE': StatementType.Declare, 'DROP': StatementType.Drop, 'END': StatementType.End, + 'ELSE': StatementType.Else, + 'ELSEIF': StatementType.Elseif, 'CALL': StatementType.Call, 'BEGIN': StatementType.Begin, - 'ALTER': StatementType.Alter + 'ALTER': StatementType.Alter, + 'FOR': StatementType.For, + 'FETCH': StatementType.Fetch, + 'GET': StatementType.Get, + 'GOTO': StatementType.Goto, + 'IF': StatementType.If, + 'INCLUDE': StatementType.Include, + 'ITERATE': StatementType.Iterate, + 'LEAVE': StatementType.Leave, + 'LOOP': StatementType.Loop, + 'PIPE': StatementType.Pipe, + 'REPEAT': StatementType.Repeat, + 'RESIGNAL': StatementType.Resignal, + 'RETURN': StatementType.Return, + 'SIGNAL': StatementType.Signal, + 'SET': StatementType.Set, + 'WHILE': StatementType.While, }; export enum ClauseType {