diff --git a/AUTHORS b/AUTHORS index 3013a362d8..5c174b179a 100644 --- a/AUTHORS +++ b/AUTHORS @@ -13,6 +13,7 @@ bingou Boris Verkhovskiy Christian Jorgensen Christopher Manouvrier +cnbei Damon Davison Daniël van Eeden Davut Can Abacigil diff --git a/src/lexer/Tokenizer.ts b/src/lexer/Tokenizer.ts index ba761de0ae..7f85bc893e 100644 --- a/src/lexer/Tokenizer.ts +++ b/src/lexer/Tokenizer.ts @@ -170,7 +170,8 @@ export default class Tokenizer { type: TokenType.VARIABLE, regex: cfg.variableTypes ? regex.variable(cfg.variableTypes) : undefined, }, - { type: TokenType.STRING, regex: regex.string(cfg.stringTypes) }, + // Typographic ‘…’ quotes (U+2018/U+2019), commonly inserted by copy-paste. See #942. + { type: TokenType.STRING, regex: regex.string([...cfg.stringTypes, '‘’']) }, { type: TokenType.IDENTIFIER, regex: regex.identifier(cfg.identChars), diff --git a/src/lexer/regexFactory.ts b/src/lexer/regexFactory.ts index a466e0ce57..4bdacd8fb2 100644 --- a/src/lexer/regexFactory.ts +++ b/src/lexer/regexFactory.ts @@ -112,6 +112,8 @@ export const quotePatterns = { "''-bs": String.raw`(?:'[^'\\]*(?:\\.[^'\\]*)*')`, // with backslash escapes "''-qq-bs": String.raw`(?:'[^'\\]*(?:\\.[^'\\]*)*')+`, // with repeated quote or backslash escapes "''-raw": String.raw`(?:'[^']*')`, // no escaping + // typographic single quotes (U+2018 / U+2019), common in copy-pasted SQL + '‘’': String.raw`\u2018[^\u2019]*\u2019`, // PostgreSQL dollar-quoted '$$': String.raw`(?\$\w*\$)[\s\S]*?\k`, // BigQuery '''triple-quoted''' (using \' to escape) diff --git a/src/lexer/regexUtil.ts b/src/lexer/regexUtil.ts index ddece97a2e..99f2185dd5 100644 --- a/src/lexer/regexUtil.ts +++ b/src/lexer/regexUtil.ts @@ -3,7 +3,9 @@ import { PrefixedQuoteType } from './TokenizerOptions.js'; // Escapes regex special chars export const escapeRegExp = (string: string) => string.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&'); -export const WHITESPACE_REGEX = /\s+/uy; +// \s covers ordinary Unicode whitespace (including NBSP and BOM), but not +// U+200B ZERO WIDTH SPACE, which word processors often insert around quotes. +export const WHITESPACE_REGEX = /[\s\u200B]+/uy; export const patternToRegex = (pattern: string): RegExp => new RegExp(`(?:${pattern})`, 'uy'); diff --git a/test/behavesLikeSqlFormatter.ts b/test/behavesLikeSqlFormatter.ts index 1b9e636ba6..5b351063ae 100644 --- a/test/behavesLikeSqlFormatter.ts +++ b/test/behavesLikeSqlFormatter.ts @@ -277,4 +277,22 @@ export default function behavesLikeSqlFormatter(format: FormatFn) { tbl; `); }); + + it('treats zero-width space as whitespace', () => { + const zwsp = '\u200B'; + expect(format(`SELECT${zwsp}foo FROM bar;`)).toBe(dedent` + SELECT + foo + FROM + bar; + `); + }); + + it('supports typographic single-quoted strings', () => { + expect(format('SELECT \u2018foo JOIN bar\u2019')).toBe('SELECT\n \u2018foo JOIN bar\u2019'); + }); + + it('does not treat a curly apostrophe inside an ASCII string as a delimiter', () => { + expect(format("SELECT 'don\u2019t'")).toBe("SELECT\n 'don\u2019t'"); + }); } diff --git a/test/transactsql.test.ts b/test/transactsql.test.ts index f3041c23fe..81e6b8b71f 100644 --- a/test/transactsql.test.ts +++ b/test/transactsql.test.ts @@ -258,6 +258,33 @@ describe('TransactSqlFormatter', () => { `); }); + // Issue #942 + it('supports typographic quotes and zero-width spaces around a parameter', () => { + const zwsp = '\u200B'; + // U+200B ZERO WIDTH SPACE + U+2018 LEFT SINGLE QUOTATION MARK + @p0 + // + U+200B + U+2019 RIGHT SINGLE QUOTATION MARK + const quotedParam = `${zwsp}\u2018@p0${zwsp}\u2019`; + expect(format(`SELECT * FROM t WHERE x <= ${quotedParam} AND y >= ${quotedParam}`)).toBe(dedent` + SELECT + * + FROM + t + WHERE + x <= \u2018@p0${zwsp}\u2019 + AND y >= \u2018@p0${zwsp}\u2019 + `); + + // Original playground input from the issue, with the same surrounding characters + expect(() => + format(`select +top @p1 ltrim(rtrim(long_matt_name)) +from sl_hbm_matter where matter_uno = @p2 and + CONVERT(DATETIME,CONVERT(VARCHAR(@p3), last_modified)) +<= ${quotedParam} and CONVERT(DATETIME,CONVERT(VARCHAR(@p3), sl_entry_date)) + >= ${quotedParam} order by sl_entry_date desc`) + ).not.toThrow(); + }); + // Issue #877 it('allows the use of the ODBC date format', () => { const result = format( diff --git a/test/unit/Tokenizer.test.ts b/test/unit/Tokenizer.test.ts index 4f85695e2e..0cc9ba6592 100644 --- a/test/unit/Tokenizer.test.ts +++ b/test/unit/Tokenizer.test.ts @@ -1,4 +1,5 @@ import Tokenizer from '../../src/lexer/Tokenizer.js'; +import { TokenType } from '../../src/lexer/token.js'; describe('Tokenizer', () => { const tokenize = (sql: string) => @@ -21,6 +22,18 @@ describe('Tokenizer', () => { expect(tokenize(' \t\n \n\r ')).toEqual([]); }); + it('treats zero-width space as whitespace', () => { + expect(tokenize('\u200B \u200B')).toEqual([]); + }); + + it('tokenizes typographic single-quoted strings', () => { + const tokens = tokenize('SELECT \u2018foo\u2019'); + expect(tokens.map(t => ({ type: t.type, text: t.text }))).toEqual([ + { type: TokenType.RESERVED_SELECT, text: 'SELECT' }, + { type: TokenType.STRING, text: '\u2018foo\u2019' }, + ]); + }); + it('tokenizes single line SQL tokens', () => { expect(tokenize('SELECT * FROM foo;')).toMatchSnapshot(); });