From 7dc6c680d3835c7309b70f2ad3ae8b67ba3cefd8 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Wed, 5 May 2021 15:36:40 +0300 Subject: [PATCH 1/2] refactor(managers): Use moo fallback for unknown fragments --- lib/manager/bazel/extract.ts | 2 +- lib/manager/cake/index.ts | 2 +- lib/manager/gradle-lite/common.ts | 1 - lib/manager/gradle-lite/tokenizer.spec.ts | 8 +++----- lib/manager/gradle-lite/tokenizer.ts | 22 ++-------------------- 5 files changed, 7 insertions(+), 28 deletions(-) diff --git a/lib/manager/bazel/extract.ts b/lib/manager/bazel/extract.ts index 74c3f3d1c407e0..56ca1bebd75e1e 100644 --- a/lib/manager/bazel/extract.ts +++ b/lib/manager/bazel/extract.ts @@ -85,7 +85,7 @@ const lexer = moo.states({ ].join('|') ), }, - unknown: { match: /[^]/, lineBreaks: true }, + unknown: moo.fallback, }, longDoubleQuoted: { stringFinish: { match: '"""', pop: 1 }, diff --git a/lib/manager/cake/index.ts b/lib/manager/cake/index.ts index 50d04007f97a92..f7352cb1f54eae 100644 --- a/lib/manager/cake/index.ts +++ b/lib/manager/cake/index.ts @@ -21,7 +21,7 @@ const lexer = moo.states({ match: /^#(?:addin|tool|module)\s+"(?:nuget|dotnet):[^"]+"\s*$/, value: (s: string) => s.trim().slice(1, -1), }, - unknown: { match: /[^]/, lineBreaks: true }, + unknown: moo.fallback, }, }); diff --git a/lib/manager/gradle-lite/common.ts b/lib/manager/gradle-lite/common.ts index 59ade8fc7f0920..b8c15b7b19cf80 100644 --- a/lib/manager/gradle-lite/common.ts +++ b/lib/manager/gradle-lite/common.ts @@ -60,7 +60,6 @@ export enum TokenType { EscapedChar = 'escapedChar', String = 'string', - UnknownLexeme = 'unknownChar', UnknownFragment = 'unknownFragment', } diff --git a/lib/manager/gradle-lite/tokenizer.spec.ts b/lib/manager/gradle-lite/tokenizer.spec.ts index d7378abdd1aa08..980af130f3a75c 100644 --- a/lib/manager/gradle-lite/tokenizer.spec.ts +++ b/lib/manager/gradle-lite/tokenizer.spec.ts @@ -35,7 +35,7 @@ describe(getName(), () => { TokenType.RightBrace, TokenType.RightBrace, ], - '@': [TokenType.UnknownLexeme], + '@': [TokenType.UnknownFragment], "'\\''": [ TokenType.SingleQuotedStart, TokenType.EscapedChar, @@ -130,9 +130,7 @@ describe(getName(), () => { '"${x()}"': [ TokenType.DoubleQuotedStart, TokenType.IgnoredInterpolationStart, - TokenType.UnknownLexeme, - TokenType.UnknownLexeme, - TokenType.UnknownLexeme, + TokenType.UnknownFragment, TokenType.RightBrace, TokenType.DoubleQuotedFinish, ], @@ -140,7 +138,7 @@ describe(getName(), () => { '"${x{}}"': [ TokenType.DoubleQuotedStart, TokenType.IgnoredInterpolationStart, - TokenType.UnknownLexeme, + TokenType.UnknownFragment, TokenType.LeftBrace, TokenType.RightBrace, TokenType.RightBrace, diff --git a/lib/manager/gradle-lite/tokenizer.ts b/lib/manager/gradle-lite/tokenizer.ts index b37722fae2b873..6fba5144b12821 100644 --- a/lib/manager/gradle-lite/tokenizer.ts +++ b/lib/manager/gradle-lite/tokenizer.ts @@ -55,7 +55,7 @@ const lexer = moo.states({ match: '"', push: TokenType.DoubleQuotedStart, }, - [TokenType.UnknownLexeme]: { match: /./ }, + [TokenType.UnknownFragment]: moo.fallback, }, // Tokenize triple-quoted string literal characters @@ -102,27 +102,10 @@ const lexer = moo.states({ push: TokenType.IgnoredInterpolationStart, }, [TokenType.RightBrace]: { match: '}', pop: 1 }, - [TokenType.UnknownLexeme]: { match: /[^]/, lineBreaks: true }, + [TokenType.UnknownFragment]: moo.fallback, }, }); -/* - Turn UnknownLexeme chars to UnknownFragment strings - */ -function processUnknownLexeme(acc: Token[], token: Token): Token[] { - if (token.type === TokenType.UnknownLexeme) { - const prevToken: Token = acc[acc.length - 1]; - if (prevToken?.type === TokenType.UnknownFragment) { - prevToken.value += token.value; - } else { - acc.push({ ...token, type: TokenType.UnknownFragment }); - } - } else { - acc.push(token); - } - return acc; -} - // // Turn separated chars of string literal to single String token // @@ -221,7 +204,6 @@ export function extractRawTokens(input: string): Token[] { export function processTokens(tokens: Token[]): Token[] { return tokens - .reduce(processUnknownLexeme, []) .reduce(processChar, []) .reduce(processInterpolation, []) .filter(filterTokens); From c6bf8d19cd0fd125c42a1986c73e5edb90f271ec Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Wed, 5 May 2021 18:13:42 +0300 Subject: [PATCH 2/2] Use fallback for strings too --- lib/manager/bazel/extract.ts | 8 ++++---- lib/manager/gradle-lite/common.ts | 2 +- lib/manager/gradle-lite/tokenizer.spec.ts | 15 +++++++-------- lib/manager/gradle-lite/tokenizer.ts | 16 ++++++++-------- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/lib/manager/bazel/extract.ts b/lib/manager/bazel/extract.ts index 56ca1bebd75e1e..197e801b666bd8 100644 --- a/lib/manager/bazel/extract.ts +++ b/lib/manager/bazel/extract.ts @@ -89,19 +89,19 @@ const lexer = moo.states({ }, longDoubleQuoted: { stringFinish: { match: '"""', pop: 1 }, - char: { match: /[^]/, lineBreaks: true }, + char: moo.fallback, }, doubleQuoted: { stringFinish: { match: '"', pop: 1 }, - char: { match: /[^]/, lineBreaks: true }, + char: moo.fallback, }, longSingleQuoted: { stringFinish: { match: "'''", pop: 1 }, - char: { match: /[^]/, lineBreaks: true }, + char: moo.fallback, }, singleQuoted: { stringFinish: { match: "'", pop: 1 }, - char: { match: /[^]/, lineBreaks: true }, + char: moo.fallback, }, }); diff --git a/lib/manager/gradle-lite/common.ts b/lib/manager/gradle-lite/common.ts index b8c15b7b19cf80..3ef9d3c020dccc 100644 --- a/lib/manager/gradle-lite/common.ts +++ b/lib/manager/gradle-lite/common.ts @@ -56,7 +56,7 @@ export enum TokenType { TripleDoubleQuotedStart = 'tripleDoubleQuotedStart', TripleQuotedFinish = 'tripleQuotedFinish', - Char = 'char', + Chars = 'chars', EscapedChar = 'escapedChar', String = 'string', diff --git a/lib/manager/gradle-lite/tokenizer.spec.ts b/lib/manager/gradle-lite/tokenizer.spec.ts index 980af130f3a75c..6c73c1f5ed3046 100644 --- a/lib/manager/gradle-lite/tokenizer.spec.ts +++ b/lib/manager/gradle-lite/tokenizer.spec.ts @@ -54,23 +54,22 @@ describe(getName(), () => { ], "'x'": [ TokenType.SingleQuotedStart, - TokenType.Char, + TokenType.Chars, TokenType.SingleQuotedFinish, ], "'\n'": [ TokenType.SingleQuotedStart, - TokenType.Char, + TokenType.Chars, TokenType.SingleQuotedFinish, ], "'$x'": [ TokenType.SingleQuotedStart, - TokenType.Char, - TokenType.Char, + TokenType.Chars, TokenType.SingleQuotedFinish, ], "''''''": ['tripleQuotedStart', 'tripleQuotedFinish'], - "'''x'''": ['tripleQuotedStart', TokenType.Char, 'tripleQuotedFinish'], - "'''\n'''": ['tripleQuotedStart', TokenType.Char, 'tripleQuotedFinish'], + "'''x'''": ['tripleQuotedStart', TokenType.Chars, 'tripleQuotedFinish'], + "'''\n'''": ['tripleQuotedStart', TokenType.Chars, 'tripleQuotedFinish'], "'''\\''''": [ 'tripleQuotedStart', TokenType.EscapedChar, @@ -106,12 +105,12 @@ describe(getName(), () => { ], '"x"': [ TokenType.DoubleQuotedStart, - TokenType.Char, + TokenType.Chars, TokenType.DoubleQuotedFinish, ], '"\n"': [ TokenType.DoubleQuotedStart, - TokenType.Char, + TokenType.Chars, TokenType.DoubleQuotedFinish, ], // eslint-disable-next-line no-template-curly-in-string diff --git a/lib/manager/gradle-lite/tokenizer.ts b/lib/manager/gradle-lite/tokenizer.ts index 6fba5144b12821..271a80958336d8 100644 --- a/lib/manager/gradle-lite/tokenizer.ts +++ b/lib/manager/gradle-lite/tokenizer.ts @@ -62,19 +62,19 @@ const lexer = moo.states({ [TokenType.TripleSingleQuotedStart]: { ...escapedChars, [TokenType.TripleQuotedFinish]: { match: "'''", pop: 1 }, - [TokenType.Char]: { match: /[^]/, lineBreaks: true }, + [TokenType.Chars]: moo.fallback, }, [TokenType.TripleDoubleQuotedStart]: { ...escapedChars, [TokenType.TripleQuotedFinish]: { match: '"""', pop: 1 }, - [TokenType.Char]: { match: /[^]/, lineBreaks: true }, + [TokenType.Chars]: moo.fallback, }, // Tokenize single-quoted string literal characters [TokenType.SingleQuotedStart]: { ...escapedChars, [TokenType.SingleQuotedFinish]: { match: "'", pop: 1 }, - [TokenType.Char]: { match: /[^]/, lineBreaks: true }, + [TokenType.Chars]: moo.fallback, }, // Tokenize double-quoted string literal chars and interpolations @@ -91,7 +91,7 @@ const lexer = moo.states({ match: /\${/, push: TokenType.IgnoredInterpolationStart, }, - [TokenType.Char]: { match: /[^]/, lineBreaks: true }, + [TokenType.Chars]: moo.fallback, }, // Ignore interpolation of complex expressions˙, @@ -107,12 +107,12 @@ const lexer = moo.states({ }); // -// Turn separated chars of string literal to single String token +// Turn substrings of chars and escaped chars into single String token // -function processChar(acc: Token[], token: Token): Token[] { +function processChars(acc: Token[], token: Token): Token[] { const tokenType = token.type; const prevToken: Token = acc[acc.length - 1]; - if ([TokenType.Char, TokenType.EscapedChar].includes(tokenType)) { + if ([TokenType.Chars, TokenType.EscapedChar].includes(tokenType)) { if (prevToken?.type === TokenType.String) { prevToken.value += token.value; } else { @@ -204,7 +204,7 @@ export function extractRawTokens(input: string): Token[] { export function processTokens(tokens: Token[]): Token[] { return tokens - .reduce(processChar, []) + .reduce(processChars, []) .reduce(processInterpolation, []) .filter(filterTokens); }