fix: support typographic quotes and zero-width spaces

AUTHORS

@@ -13,6 +13,7 @@ bingou
 Boris Verkhovskiy <boris.verk@gmail.com>
 Christian Jorgensen <chr.jorgensen1@gmail.com>
 Christopher Manouvrier <chris@dovetailapp.com>
+cnbei
 Damon Davison <ddavison@avalere.com>
 Daniël van Eeden <daniel.van.eeden@pingcap.com>
 Davut Can Abacigil <can@teamsql.io>

src/lexer/Tokenizer.ts

@@ -170,7 +170,8 @@ export default class Tokenizer {
         type: TokenType.VARIABLE,
         regex: cfg.variableTypes ? regex.variable(cfg.variableTypes) : undefined,
       },
-      { type: TokenType.STRING, regex: regex.string(cfg.stringTypes) },
+      // Typographic ‘…’ quotes (U+2018/U+2019), commonly inserted by copy-paste. See #942.
+      { type: TokenType.STRING, regex: regex.string([...cfg.stringTypes, '‘’']) },
       {
         type: TokenType.IDENTIFIER,
         regex: regex.identifier(cfg.identChars),

src/lexer/regexFactory.ts

@@ -112,6 +112,8 @@ export const quotePatterns = {
   "''-bs": String.raw`(?:'[^'\\]*(?:\\.[^'\\]*)*')`, // with backslash escapes
   "''-qq-bs": String.raw`(?:'[^'\\]*(?:\\.[^'\\]*)*')+`, // with repeated quote or backslash escapes
   "''-raw": String.raw`(?:'[^']*')`, // no escaping
+  // typographic single quotes (U+2018 / U+2019), common in copy-pasted SQL
+  '‘’': String.raw`\u2018[^\u2019]*\u2019`,
   // PostgreSQL dollar-quoted
   '$$': String.raw`(?<tag>\$\w*\$)[\s\S]*?\k<tag>`,
   // BigQuery '''triple-quoted''' (using \' to escape)

src/lexer/regexUtil.ts

@@ -3,7 +3,9 @@ import { PrefixedQuoteType } from './TokenizerOptions.js';
 // Escapes regex special chars
 export const escapeRegExp = (string: string) => string.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
 
-export const WHITESPACE_REGEX = /\s+/uy;
+// \s covers ordinary Unicode whitespace (including NBSP and BOM), but not
+// U+200B ZERO WIDTH SPACE, which word processors often insert around quotes.
+export const WHITESPACE_REGEX = /[\s\u200B]+/uy;
 
 export const patternToRegex = (pattern: string): RegExp => new RegExp(`(?:${pattern})`, 'uy');
 

test/behavesLikeSqlFormatter.ts

@@ -277,4 +277,22 @@ export default function behavesLikeSqlFormatter(format: FormatFn) {
         tbl;
     `);
   });
+
+  it('treats zero-width space as whitespace', () => {
+    const zwsp = '\u200B';
+    expect(format(`SELECT${zwsp}foo FROM bar;`)).toBe(dedent`
+      SELECT
+        foo
+      FROM
+        bar;
+    `);
+  });
+
+  it('supports typographic single-quoted strings', () => {
+    expect(format('SELECT \u2018foo JOIN bar\u2019')).toBe('SELECT\n  \u2018foo JOIN bar\u2019');
+  });
+
+  it('does not treat a curly apostrophe inside an ASCII string as a delimiter', () => {
+    expect(format("SELECT 'don\u2019t'")).toBe("SELECT\n  'don\u2019t'");
+  });
 }

test/transactsql.test.ts

@@ -258,6 +258,33 @@ describe('TransactSqlFormatter', () => {
     `);
   });
 
+  // Issue #942
+  it('supports typographic quotes and zero-width spaces around a parameter', () => {
+    const zwsp = '\u200B';
+    // U+200B ZERO WIDTH SPACE + U+2018 LEFT SINGLE QUOTATION MARK + @p0
+    // + U+200B + U+2019 RIGHT SINGLE QUOTATION MARK
+    const quotedParam = `${zwsp}\u2018@p0${zwsp}\u2019`;
+    expect(format(`SELECT * FROM t WHERE x <= ${quotedParam} AND y >= ${quotedParam}`)).toBe(dedent`
+      SELECT
+        *
+      FROM
+        t
+      WHERE
+        x <= \u2018@p0${zwsp}\u2019
+        AND y >= \u2018@p0${zwsp}\u2019
+    `);
+
+    // Original playground input from the issue, with the same surrounding characters
+    expect(() =>
+      format(`select
+top @p1 ltrim(rtrim(long_matt_name))
+from sl_hbm_matter where matter_uno = @p2 and
+ CONVERT(DATETIME,CONVERT(VARCHAR(@p3), last_modified))
+<= ${quotedParam} and CONVERT(DATETIME,CONVERT(VARCHAR(@p3), sl_entry_date))
+ >= ${quotedParam} order by sl_entry_date desc`)
+    ).not.toThrow();
+  });
+
   // Issue #877
   it('allows the use of the ODBC date format', () => {
     const result = format(

test/unit/Tokenizer.test.ts

@@ -1,4 +1,5 @@
 import Tokenizer from '../../src/lexer/Tokenizer.js';
+import { TokenType } from '../../src/lexer/token.js';
 
 describe('Tokenizer', () => {
   const tokenize = (sql: string) =>
@@ -21,6 +22,18 @@ describe('Tokenizer', () => {
     expect(tokenize(' \t\n \n\r ')).toEqual([]);
   });
 
+  it('treats zero-width space as whitespace', () => {
+    expect(tokenize('\u200B \u200B')).toEqual([]);
+  });
+
+  it('tokenizes typographic single-quoted strings', () => {
+    const tokens = tokenize('SELECT \u2018foo\u2019');
+    expect(tokens.map(t => ({ type: t.type, text: t.text }))).toEqual([
+      { type: TokenType.RESERVED_SELECT, text: 'SELECT' },
+      { type: TokenType.STRING, text: '\u2018foo\u2019' },
+    ]);
+  });
+
   it('tokenizes single line SQL tokens', () => {
     expect(tokenize('SELECT * FROM foo;')).toMatchSnapshot();
   });