Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improved support for parser conditional and additional statement types #283

Merged
merged 1 commit into from
Oct 3, 2024
+391 −185
Merged

Improved support for parser conditional and additional statement types #283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 16 additions & 9 deletions src/language/sql/document.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ export default class Document {
let statementStart = 0;

for (let i = 0; i < tokens.length; i++) {
const upperValue = tokens[i].value?.toUpperCase();
switch (tokens[i].type) {
case `semicolon`:
const statementTokens = tokens.slice(statementStart, i);
Expand All @@ -45,19 +46,25 @@ export default class Document {
break;

case `statementType`:
currentStatementType = StatementTypeWord[tokens[i].value?.toUpperCase()];
currentStatementType = StatementTypeWord[upperValue];
break;

case `keyword`:
switch (tokens[i].value?.toUpperCase()) {
switch (upperValue) {
case `LOOP`:
// This handles the case that 'END LOOP' is supported.
if (currentStatementType === StatementType.End) {
break;
}
case `THEN`:
case `BEGIN`:
case `DO`:
case `THEN`:
// This handles the case that 'END LOOP' is supported.
if (upperValue === `LOOP` && currentStatementType === StatementType.End) {
break;
}

// Support for THEN in conditionals
if (upperValue === `THEN` && !Statement.typeIsConditional(currentStatementType)) {
break;
}

// We include BEGIN in the current statement
// then the next statement beings
const statementTokens = tokens.slice(statementStart, i+1);
Expand Down Expand Up @@ -102,7 +109,7 @@ export default class Document {
let depth = 0;

for (const statement of this.statements) {
if (statement.isBlockEnder()) {
if (statement.isCompoundEnd()) {
if (depth > 0) {
currentGroup.push(statement);

Expand All @@ -118,7 +125,7 @@ export default class Document {
currentGroup = [];
}
} else
if (statement.isBlockOpener()) {
if (statement.isCompoundStart()) {
if (depth > 0) {
currentGroup.push(statement);
} else {
Expand Down
84 changes: 58 additions & 26 deletions src/language/sql/statement.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ const tokenIs = (token: Token|undefined, type: string, value?: string) => {

export default class Statement {
public type: StatementType = StatementType.Unknown;
private label: string|undefined;

constructor(public tokens: Token[], public range: IRange) {
this.tokens = this.tokens.filter(newToken => newToken.type !== `newline`);
Expand All @@ -19,11 +20,15 @@ export default class Statement {
first = this.tokens[2];
}

if (tokenIs(first, `statementType`) || tokenIs(first, `keyword`, `END`) || tokenIs(first, `keyword`, `BEGIN`)) {
const wordValue = first.value?.toUpperCase();

this.type = StatementTypeWord[wordValue];
if (tokenIs(first, `word`) && tokenIs(this.tokens[1], `colon`)) {
// Possible label?
this.label = first.value;
first = this.tokens[2];
}

const wordValue = first.value?.toUpperCase();

this.type = StatementTypeWord[wordValue] || StatementType.Unknown;

switch (this.type) {
case StatementType.Create:
Expand All @@ -35,7 +40,7 @@ export default class Statement {
}
}

isBlockOpener() {
isCompoundStart() {
if (this.tokens.length === 1 && tokenIs(this.tokens[0], `keyword`, `BEGIN`)) {
return true;
}
Expand All @@ -51,7 +56,19 @@ export default class Statement {
return false;
}

isBlockEnder() {
static typeIsConditional(type: StatementType) {
return [StatementType.If, StatementType.While, StatementType.Loop, StatementType.For].includes(type);
}

isConditionStart() {
return Statement.typeIsConditional(this.type);
}

isConditionEnd() {
return this.type === StatementType.End && this.tokens.length > 1;
}

isCompoundEnd() {
return this.type === StatementType.End && this.tokens.length === 1;
}

Expand Down Expand Up @@ -314,17 +331,21 @@ export default class Statement {
}

const basicQueryFinder = (startIndex: number): void => {
let currentClause: undefined|"select"|"from";
for (let i = startIndex; i < this.tokens.length; i++) {
if (tokenIs(this.tokens[i], `clause`, `FROM`)) {
inFromClause = true;
} else if (inFromClause && tokenIs(this.tokens[i], `clause`) || tokenIs(this.tokens[i], `join`) || tokenIs(this.tokens[i], `closebracket`)) {
inFromClause = false;
currentClause = `from`;
}
else if (tokenIs(this.tokens[i], `statementType`, `SELECT`)) {
currentClause = `select`;
} else if (currentClause === `from` && tokenIs(this.tokens[i], `clause`) || tokenIs(this.tokens[i], `join`) || tokenIs(this.tokens[i], `closebracket`)) {
currentClause = undefined;
}

if (tokenIs(this.tokens[i], `clause`, `FROM`) ||
(this.type !== StatementType.Select && tokenIs(this.tokens[i], `clause`, `INTO`)) ||
tokenIs(this.tokens[i], `join`) ||
(inFromClause && tokenIs(this.tokens[i], `comma`)
(currentClause === `from` && tokenIs(this.tokens[i], `comma`)
)) {
const sqlObj = this.getRefAtToken(i+1);
if (sqlObj) {
Expand All @@ -334,6 +355,15 @@ export default class Statement {
i += 3; //For the brackets
}
}
} else if (currentClause === `select` && tokenIs(this.tokens[i], `function`)) {
const sqlObj = this.getRefAtToken(i);
if (sqlObj) {
doAdd(sqlObj);
i += sqlObj.tokens.length;
if (sqlObj.isUDTF || sqlObj.fromLateral) {
i += 3; //For the brackets
}
}
}
}
}
Expand Down Expand Up @@ -592,7 +622,7 @@ export default class Statement {
}

if (options.withSystemName) {
if (tokenIs(this.tokens[endIndex+1], `keyword`, `FOR`) && tokenIs(this.tokens[endIndex+2], `word`, `SYSTEM`) && tokenIs(this.tokens[endIndex+3], `word`, `NAME`)) {
if (tokenIs(this.tokens[endIndex+1], `statementType`, `FOR`) && tokenIs(this.tokens[endIndex+2], `word`, `SYSTEM`) && tokenIs(this.tokens[endIndex+3], `word`, `NAME`)) {
if (this.tokens[endIndex+4] && NameTypes.includes(this.tokens[endIndex+4].type)) {
sqlObj.object.system = this.tokens[endIndex+4].value;
}
Expand Down Expand Up @@ -624,10 +654,25 @@ export default class Statement {

switch (currentToken.type) {
case `statementType`:
if (declareStmt) continue;
const currentValue = currentToken.value.toLowerCase();
if (declareStmt) {
if (currentValue === `for`) {
ranges.push({
type: `remove`,
range: {
start: declareStmt.range.start,
end: currentToken.range.end
}
});

declareStmt = undefined;
}

continue;
};

// If we're in a DECLARE, it's likely a cursor definition
if (currentToken.value.toLowerCase() === `declare`) {
if (currentValue === `declare`) {
declareStmt = currentToken;
}
break;
Expand Down Expand Up @@ -716,19 +761,6 @@ export default class Statement {
}
});
}
} else
if (declareStmt && tokenIs(currentToken, `keyword`, `FOR`)) {
// If we're a DECLARE, and we found the FOR keyword, the next
// set of tokens should be the select.
ranges.push({
type: `remove`,
range: {
start: declareStmt.range.start,
end: currentToken.range.end
}
});

declareStmt = undefined;
}
break;
}
Expand Down
78 changes: 58 additions & 20 deletions src/language/sql/tests/blocks.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@

import { assert, describe, expect, test } from 'vitest'
import { describe, expect, test } from 'vitest'
import Document from '../document';
import { StatementType } from '../types';

describe(`Block statement tests`, () => {
const parserScenarios = describe.each([
{newDoc: (content: string) => new Document(content), isFormatted: false},
]);

parserScenarios(`Block statement tests`, ({newDoc, isFormatted}) => {
test('Block start tests', () => {
const lines = [
`CREATE ALIAS "TestDelimiters"."Delimited Alias" FOR "TestDelimiters"."Delimited Table";`,
Expand All @@ -15,19 +18,19 @@ describe(`Block statement tests`, () => {
`LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`,
].join(`\n`);

const doc = new Document(lines);
const doc = newDoc(lines);

// CREATE, CREATE, RETURN, END, CREATE, SET, END
expect(doc.statements.length).toBe(7);

const aliasDef = doc.statements[0];
expect(aliasDef.isBlockOpener()).toBeFalsy();
expect(aliasDef.isCompoundStart()).toBeFalsy();

const functionDef = doc.statements[1];
expect(functionDef.isBlockOpener()).toBeTruthy();
expect(functionDef.isCompoundStart()).toBeTruthy();

const procedureDef = doc.statements[4];
expect(procedureDef.isBlockOpener()).toBeTruthy();
expect(procedureDef.isCompoundStart()).toBeTruthy();
});

test('Compound statement test', () => {
Expand All @@ -53,21 +56,21 @@ describe(`Block statement tests`, () => {
`LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`,
].join(`\n`);

const doc = new Document(lines);
const doc = newDoc(lines);

const t = doc.statements.length;

const aliasDef = doc.statements[0];
expect(aliasDef.isBlockOpener()).toBeFalsy();
expect(aliasDef.isCompoundStart()).toBeFalsy();

const functionDef = doc.statements[1];
expect(functionDef.isBlockOpener()).toBeTruthy();
expect(functionDef.isCompoundStart()).toBeTruthy();

const functionEnd = doc.statements[3];
expect(functionEnd.isBlockEnder()).toBeTruthy();
expect(functionEnd.isCompoundEnd()).toBeTruthy();

const beginBlock = doc.statements[4];
expect(beginBlock.isBlockOpener()).toBeTruthy();
expect(beginBlock.isCompoundStart()).toBeTruthy();
});

test('Statement groups', () => {
Expand Down Expand Up @@ -96,23 +99,58 @@ describe(`Block statement tests`, () => {
`LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`,
].join(`\r\n`);

const doc = new Document(lines);
const doc = newDoc(lines);

const groups = doc.getStatementGroups();

expect(groups.length).toBe(4);

const aliasStatement = groups[0];
const aliasSubstring = lines.substring(aliasStatement.range.start, aliasStatement.range.end);
const aliasSubstring = doc.content.substring(aliasStatement.range.start, aliasStatement.range.end);
expect(aliasSubstring).toBe(`CREATE ALIAS "TestDelimiters"."Delimited Alias" FOR "TestDelimiters"."Delimited Table"`);

const functionStatement = groups[1];
const functionSubstring = doc.content.substring(functionStatement.range.start, functionStatement.range.end);

if (isFormatted) {
expect(functionSubstring).toBe([
`CREATE FUNCTION "TestDelimiters"."Delimited Function"(`,
` "Delimited Parameter" INTEGER`,
`) RETURNS INTEGER LANGUAGE SQL BEGIN`,
` RETURN "Delimited Parameter";`,
`END`,
].join(`\r\n`));
} else {
expect(functionSubstring).toBe([
`CREATE FUNCTION "TestDelimiters"."Delimited Function" ("Delimited Parameter" INTEGER) `,
`RETURNS INTEGER LANGUAGE SQL BEGIN RETURN "Delimited Parameter"; END`
].join(`\r\n`))
}
const beginStatement = groups[2];
const compoundSubstring = lines.substring(beginStatement.range.start, beginStatement.range.end);
expect(compoundSubstring).toBe(compoundStatement);
expect(beginStatement.statements.length).toBe(9);
const compoundSubstring = doc.content.substring(beginStatement.range.start, beginStatement.range.end);

if (isFormatted) {
expect(compoundSubstring).toBe([
`BEGIN`,
` DECLARE already_exists SMALLINT DEFAULT 0;`,
` DECLARE dup_object_hdlr CONDITION FOR SQLSTATE '42710';`,
` DECLARE CONTINUE HANDLER FOR dup_object_hdlr SET already_exists = 1;`,
` CREATE TABLE table1(`,
` col1 INT`,
` );`,
` IF already_exists > 0 THEN;`,
` DELETE FROM table1;`,
` END IF;`,
`END`,
].join(`\r\n`));
} else {
expect(compoundSubstring).toBe(compoundStatement);
}
});
});

describe(`Definition tests`, () => {
parserScenarios(`Definition tests`, ({newDoc}) => {
test(`Alias, function, procedure`, () => {
const lines = [
`CREATE ALIAS "TestDelimiters"."Delimited Alias" FOR "TestDelimiters"."Delimited Table";`,
Expand All @@ -124,7 +162,7 @@ describe(`Definition tests`, () => {
`LANGUAGE SQL BEGIN SET "Delimited Parameter" = 13; END;`,
].join(`\n`);

const doc = new Document(lines);
const doc = newDoc(lines);

const defs = doc.getDefinitions();

Expand Down Expand Up @@ -161,7 +199,7 @@ describe(`Definition tests`, () => {
`END;`,
].join(`\r\n`);

const doc = new Document(lines);
const doc = newDoc(lines);

const defs = doc.getDefinitions();

Expand Down Expand Up @@ -245,7 +283,7 @@ describe(`Definition tests`, () => {
`END ; `,
].join(`\n`);

const doc = new Document(lines);
const doc = newDoc(lines);

const groups = doc.getStatementGroups();
expect(groups.length).toBe(1);
Expand Down
Loading
Loading