From 4238599896a0b6c4f40e5f2454ce370ab426e548 Mon Sep 17 00:00:00 2001 From: wojpadlo Date: Wed, 29 Jul 2026 15:44:20 +0200 Subject: [PATCH] Snowflake: decode octal/hex/unicode string-literal escapes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decode \ooo, \xhh and \uhhhh as Unicode code points, keep \0 as NUL, collapse unrecognized letters (\a, \Z) to the bare letter, and raise a tokenizer error on malformed \x / \u — gated behind a new supports_snowflake_string_literal_escapes dialect capability so other backslash-escaping dialects keep MySQL/BigQuery semantics. Co-Authored-By: Claude Opus 4.8 --- src/dialect/mod.rs | 16 ++++ src/dialect/snowflake.rs | 5 + src/tokenizer.rs | 197 +++++++++++++++++++++++++++++++++++++- tests/sqlparser_common.rs | 6 +- 4 files changed, 218 insertions(+), 6 deletions(-) diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index a958d33caa..2be89f4215 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -283,6 +283,18 @@ pub trait Dialect: Debug + Any { false } + /// Determine whether the dialect decodes Snowflake's extended backslash + /// escape set inside single-quoted string literals: octal `\ooo`, hex + /// `\xhh` and unicode `\uhhhh` (each yielding a Unicode code point), with + /// unrecognized letters such as `\a` / `\Z` collapsing to the bare letter + /// and malformed `\x` / `\u` raising a tokenizer error. This differs from + /// the MySQL/BigQuery-style mapping used by + /// [`Self::supports_string_literal_backslash_escape`], so a dialect that + /// enables this must also enable that one. + fn supports_snowflake_string_literal_escapes(&self) -> bool { + false + } + /// Determine whether the dialect strips the backslash when escaping LIKE wildcards (%, _). /// /// [MySQL] has a special case when escaping single quoted strings which leaves these unescaped @@ -2066,6 +2078,10 @@ mod tests { self.0.supports_string_literal_backslash_escape() } + fn supports_snowflake_string_literal_escapes(&self) -> bool { + self.0.supports_snowflake_string_literal_escapes() + } + fn supports_filter_during_aggregation(&self) -> bool { self.0.supports_filter_during_aggregation() } diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index 9304750890..fc4a541b69 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -179,6 +179,11 @@ impl Dialect for SnowflakeDialect { true } + // See https://docs.snowflake.com/en/sql-reference/data-types-text#escape-sequences-in-single-quoted-string-constants + fn supports_snowflake_string_literal_escapes(&self) -> bool { + true + } + fn supports_within_after_array_aggregation(&self) -> bool { true } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index d9f131f8fc..4b6fda3c29 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -2287,17 +2287,21 @@ impl<'a> Tokenizer<'a> { num_consecutive_quotes = 0; - if let Some(next) = chars.peek() { + if let Some(&next) = chars.peek() { if !self.unescape || (self.dialect.ignores_wildcard_escapes() - && (*next == '%' || *next == '_')) + && (next == '%' || next == '_')) { // In no-escape mode, the given query has to be saved completely // including backslashes. Similarly, with ignore_like_wildcard_escapes, // the backslash is not stripped. s.push(ch); - s.push(*next); + s.push(next); chars.next(); // consume next + } else if self.dialect.supports_snowflake_string_literal_escapes() { + let escape_loc = chars.location(); + chars.next(); // consume the escape indicator + s.push(self.unescape_snowflake_escape(next, chars, escape_loc)?); } else { let n = match next { '0' => '\0', @@ -2308,7 +2312,7 @@ impl<'a> Tokenizer<'a> { 'r' => '\r', 't' => '\t', 'Z' => '\u{1a}', - _ => *next, + _ => next, }; s.push(n); chars.next(); // consume next @@ -2331,6 +2335,172 @@ impl<'a> Tokenizer<'a> { self.tokenizer_error(error_loc, "Unterminated string literal") } + /// Decode a single Snowflake backslash escape (the backslash and the + /// `indicator` char have already been consumed). Octal `\ooo`, hex `\xhh` + /// and unicode `\uhhhh` each yield a Unicode code point; recognized control + /// letters map to their control character; every other letter collapses to + /// itself. Malformed `\x` / `\u` raise a tokenizer error, matching real + /// Snowflake. + fn unescape_snowflake_escape( + &self, + indicator: char, + chars: &mut State, + loc: Location, + ) -> Result { + match indicator { + 'x' => { + let mut digits = String::new(); + while digits.len() < 2 { + match chars.peek() { + Some(c) if c.is_ascii_hexdigit() => { + if let Some(c) = chars.next() { + digits.push(c); + } + } + _ => break, + } + } + if digits.len() < 2 { + return self.tokenizer_error( + loc, + format!( + "Invalid hex escape sequence '\\x{digits}'; should be exactly 2 digits." + ), + ); + } + self.code_point(&digits, 16, loc) + } + 'u' => self.unescape_unicode(chars, loc), + // Octal `\ooo`: up to three octal digits, but a third digit is only + // consumed when the running value stays within a single byte + // (<= 0xFF); otherwise it is left as a literal character. + '0'..='7' => { + let mut value = indicator.to_digit(8).unwrap_or(0); + // Second octal digit always fits in a byte (max 0o77 = 63). + if let Some(d) = chars.peek().and_then(|c| c.to_digit(8)) { + value = value * 8 + d; + chars.next(); + // Third digit only when the byte does not overflow. + if let Some(d) = chars.peek().and_then(|c| c.to_digit(8)) { + if value * 8 + d <= 0xFF { + value = value * 8 + d; + chars.next(); + } + } + } + char::from_u32(value).map_or_else( + || self.tokenizer_error(loc, "Invalid octal escape sequence."), + Ok, + ) + } + 'b' => Ok('\u{8}'), + 'f' => Ok('\u{c}'), + 'n' => Ok('\n'), + 'r' => Ok('\r'), + 't' => Ok('\t'), + other => Ok(other), + } + } + + fn code_point(&self, digits: &str, radix: u32, loc: Location) -> Result { + u32::from_str_radix(digits, radix) + .ok() + .and_then(char::from_u32) + .map_or_else(|| self.tokenizer_error(loc, "Invalid escape sequence."), Ok) + } + + /// Decode a `\uhhhh` escape (the `\u` has already been consumed). A high + /// surrogate must be followed by a `\uhhhh` low surrogate and combines with + /// it into an astral code point; an unpaired high or low surrogate is an + /// error, matching real Snowflake. + fn unescape_unicode(&self, chars: &mut State, loc: Location) -> Result { + let value = self.read_hex4(chars, loc)?; + if (0xD800..=0xDBFF).contains(&value) { + return match self.read_low_surrogate(chars) { + Some(low) => { + let cp = 0x1_0000 + ((value - 0xD800) << 10) + (low - 0xDC00); + char::from_u32(cp).map_or_else( + || self.tokenizer_error(loc, "Invalid escape sequence."), + Ok, + ) + } + None => self.tokenizer_error( + loc, + format!( + "Invalid Unicode string literal; high surrogate '\\u{value:04X}' \ + must be followed by a low surrogate ('\\uDC00'-'\\uDFFF')." + ), + ), + }; + } + if (0xDC00..=0xDFFF).contains(&value) { + return self.tokenizer_error( + loc, + format!( + "Invalid Unicode string literal; low surrogate '\\u{value:04X}' \ + must be preceded by a high surrogate ('\\uD800'-'\\uDBFF')." + ), + ); + } + char::from_u32(value) + .map_or_else(|| self.tokenizer_error(loc, "Invalid escape sequence."), Ok) + } + + /// Read exactly four hex digits as a `u32`, erroring like Snowflake when + /// fewer than four are present. + fn read_hex4(&self, chars: &mut State, loc: Location) -> Result { + let mut digits = String::new(); + while digits.len() < 4 { + match chars.peek() { + Some(c) if c.is_ascii_hexdigit() => { + if let Some(c) = chars.next() { + digits.push(c); + } + } + _ => break, + } + } + if digits.len() < 4 { + return self.tokenizer_error( + loc, + format!("Invalid unicode escape sequence '\\u{digits}'; should be exactly 4 digits."), + ); + } + u32::from_str_radix(&digits, 16) + .map_or_else(|_| self.tokenizer_error(loc, "Invalid unicode escape sequence."), Ok) + } + + /// Consume a trailing `\uhhhh` low surrogate, returning its value when + /// present and in range. Consumes greedily on the error path — the caller + /// aborts the whole token, so partial consumption is irrelevant. + fn read_low_surrogate(&self, chars: &mut State) -> Option { + if chars.peek() != Some(&'\\') { + return None; + } + chars.next(); + if chars.peek() != Some(&'u') { + return None; + } + chars.next(); + let mut digits = String::new(); + while digits.len() < 4 { + match chars.peek() { + Some(c) if c.is_ascii_hexdigit() => { + if let Some(c) = chars.next() { + digits.push(c); + } + } + _ => break, + } + } + if digits.len() < 4 { + return None; + } + u32::from_str_radix(&digits, 16) + .ok() + .filter(|low| (0xDC00..=0xDFFF).contains(low)) + } + fn tokenize_multiline_comment( &self, chars: &mut State, @@ -3878,10 +4048,13 @@ mod tests { (r#"'%a\'%b'"#, r#"%a\'%b"#, r#"%a'%b"#), (r#"'a\'\'b\'c\'d'"#, r#"a\'\'b\'c\'d"#, r#"a''b'c'd"#), (r#"'\\'"#, r#"\\"#, r#"\"#), + // Snowflake keeps `\0` as NUL, maps the control-letter escapes, and + // collapses unrecognized letters (`\a` -> a, `\Z` -> Z) to the bare + // letter rather than a control character. ( r#"'\0\a\b\f\n\r\t\Z'"#, r#"\0\a\b\f\n\r\t\Z"#, - "\0\u{7}\u{8}\u{c}\n\r\t\u{1a}", + "\0a\u{8}\u{c}\n\r\tZ", ), (r#"'\"'"#, r#"\""#, "\""), (r#"'\\a\\b\'c'"#, r#"\\a\\b\'c"#, r#"\a\b'c"#), @@ -3890,6 +4063,20 @@ mod tests { (r#"'\q'"#, r#"\q"#, r#"q"#), (r#"'\%\_'"#, r#"\%\_"#, r#"%_"#), (r#"'\\%\\_'"#, r#"\\%\\_"#, r#"\%\_"#), + // Hex `\xhh` and unicode `\uhhhh` yield a code point; octal `\ooo` + // takes a third digit only when the byte does not overflow. + (r#"'\x41'"#, r#"\x41"#, "A"), + (r#"'\u0041'"#, r#"\u0041"#, "A"), + (r#"'\u00e9'"#, r#"\u00e9"#, "\u{e9}"), + (r#"'\ud83d\ude00'"#, r#"\ud83d\ude00"#, "\u{1f600}"), + (r#"'\xff'"#, r#"\xff"#, "\u{ff}"), + (r#"'A'"#, r#"A"#, "A"), + (r#"'é'"#, r#"é"#, "\u{e9}"), + (r#"'ꯍ'"#, r#"ꯍ"#, "\u{abcd}"), + (r#"'\101'"#, r#"\101"#, "A"), + (r#"'\1012'"#, r#"\1012"#, "A2"), + (r#"'\777'"#, r#"\777"#, "\u{3f}7"), + (r#"'\400'"#, r#"\400"#, "\u{20}0"), ] { let tokens = Tokenizer::new(&dialect, sql) .with_unescape(false) diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 0b8fbe84a7..7761701859 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11283,7 +11283,11 @@ fn parse_escaped_string_with_unescape() { let escaping_dialects = &all_dialects_where(|dialect| dialect.supports_string_literal_backslash_escape()); let no_wildcard_exception = &all_dialects_where(|dialect| { - dialect.supports_string_literal_backslash_escape() && !dialect.ignores_wildcard_escapes() + dialect.supports_string_literal_backslash_escape() + && !dialect.ignores_wildcard_escapes() + // Snowflake decodes `\Z` / `\a` as the bare letter, not a control + // character, so it is exercised separately. + && !dialect.supports_snowflake_string_literal_escapes() }); let with_wildcard_exception = &all_dialects_where(|dialect| { dialect.supports_string_literal_backslash_escape() && dialect.ignores_wildcard_escapes()