From 468639d6b14e5453c03cb1abf180dde065820365 Mon Sep 17 00:00:00 2001 From: wojpadlo Date: Wed, 29 Jul 2026 15:44:20 +0200 Subject: [PATCH 1/2] Snowflake: decode octal/hex/unicode string-literal escapes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decode \ooo, \xhh and \uhhhh as Unicode code points, keep \0 as NUL, collapse unrecognized letters (\a, \Z) to the bare letter, and raise a tokenizer error on malformed \x / \u — gated behind a new supports_snowflake_string_literal_escapes dialect capability so other backslash-escaping dialects keep MySQL/BigQuery semantics. Co-Authored-By: Claude Opus 4.8 --- src/dialect/mod.rs | 16 +++ src/dialect/snowflake.rs | 5 + src/tokenizer.rs | 199 +++++++++++++++++++++++++++++++++++++- tests/sqlparser_common.rs | 6 +- 4 files changed, 220 insertions(+), 6 deletions(-) diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index 8afe594dc0..209827d564 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -283,6 +283,18 @@ pub trait Dialect: Debug + Any { false } + /// Determine whether the dialect decodes Snowflake's extended backslash + /// escape set inside single-quoted string literals: octal `\ooo`, hex + /// `\xhh` and unicode `\uhhhh` (each yielding a Unicode code point), with + /// unrecognized letters such as `\a` / `\Z` collapsing to the bare letter + /// and malformed `\x` / `\u` raising a tokenizer error. This differs from + /// the MySQL/BigQuery-style mapping used by + /// [`Self::supports_string_literal_backslash_escape`], so a dialect that + /// enables this must also enable that one. + fn supports_snowflake_string_literal_escapes(&self) -> bool { + false + } + /// Determine whether the dialect strips the backslash when escaping LIKE wildcards (%, _). /// /// [MySQL] has a special case when escaping single quoted strings which leaves these unescaped @@ -2016,6 +2028,10 @@ mod tests { self.0.supports_string_literal_backslash_escape() } + fn supports_snowflake_string_literal_escapes(&self) -> bool { + self.0.supports_snowflake_string_literal_escapes() + } + fn supports_filter_during_aggregation(&self) -> bool { self.0.supports_filter_during_aggregation() } diff --git a/src/dialect/snowflake.rs b/src/dialect/snowflake.rs index 7cb6f2190e..3ff1bab6ed 100644 --- a/src/dialect/snowflake.rs +++ b/src/dialect/snowflake.rs @@ -172,6 +172,11 @@ impl Dialect for SnowflakeDialect { true } + // See https://docs.snowflake.com/en/sql-reference/data-types-text#escape-sequences-in-single-quoted-string-constants + fn supports_snowflake_string_literal_escapes(&self) -> bool { + true + } + fn supports_within_after_array_aggregation(&self) -> bool { true } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index d9f131f8fc..196defa5bc 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -2287,17 +2287,21 @@ impl<'a> Tokenizer<'a> { num_consecutive_quotes = 0; - if let Some(next) = chars.peek() { + if let Some(&next) = chars.peek() { if !self.unescape || (self.dialect.ignores_wildcard_escapes() - && (*next == '%' || *next == '_')) + && (next == '%' || next == '_')) { // In no-escape mode, the given query has to be saved completely // including backslashes. Similarly, with ignore_like_wildcard_escapes, // the backslash is not stripped. s.push(ch); - s.push(*next); + s.push(next); chars.next(); // consume next + } else if self.dialect.supports_snowflake_string_literal_escapes() { + let escape_loc = chars.location(); + chars.next(); // consume the escape indicator + s.push(self.unescape_snowflake_escape(next, chars, escape_loc)?); } else { let n = match next { '0' => '\0', @@ -2308,7 +2312,7 @@ impl<'a> Tokenizer<'a> { 'r' => '\r', 't' => '\t', 'Z' => '\u{1a}', - _ => *next, + _ => next, }; s.push(n); chars.next(); // consume next @@ -2331,6 +2335,174 @@ impl<'a> Tokenizer<'a> { self.tokenizer_error(error_loc, "Unterminated string literal") } + /// Decode a single Snowflake backslash escape (the backslash and the + /// `indicator` char have already been consumed). Octal `\ooo`, hex `\xhh` + /// and unicode `\uhhhh` each yield a Unicode code point; recognized control + /// letters map to their control character; every other letter collapses to + /// itself. Malformed `\x` / `\u` raise a tokenizer error, matching real + /// Snowflake. + fn unescape_snowflake_escape( + &self, + indicator: char, + chars: &mut State, + loc: Location, + ) -> Result { + match indicator { + 'x' => { + let mut digits = String::new(); + while digits.len() < 2 { + match chars.peek() { + Some(c) if c.is_ascii_hexdigit() => { + if let Some(c) = chars.next() { + digits.push(c); + } + } + _ => break, + } + } + if digits.len() < 2 { + return self.tokenizer_error( + loc, + format!( + "Invalid hex escape sequence '\\x{digits}'; should be exactly 2 digits." + ), + ); + } + self.code_point(&digits, 16, loc) + } + 'u' => self.unescape_unicode(chars, loc), + // Octal `\ooo`: up to three octal digits, but a third digit is only + // consumed when the running value stays within a single byte + // (<= 0xFF); otherwise it is left as a literal character. + '0'..='7' => { + let mut value = indicator.to_digit(8).unwrap_or(0); + // Second octal digit always fits in a byte (max 0o77 = 63). + if let Some(d) = chars.peek().and_then(|c| c.to_digit(8)) { + value = value * 8 + d; + chars.next(); + // Third digit only when the byte does not overflow. + if let Some(d) = chars.peek().and_then(|c| c.to_digit(8)) { + if value * 8 + d <= 0xFF { + value = value * 8 + d; + chars.next(); + } + } + } + char::from_u32(value).map_or_else( + || self.tokenizer_error(loc, "Invalid octal escape sequence."), + Ok, + ) + } + 'b' => Ok('\u{8}'), + 'f' => Ok('\u{c}'), + 'n' => Ok('\n'), + 'r' => Ok('\r'), + 't' => Ok('\t'), + other => Ok(other), + } + } + + fn code_point(&self, digits: &str, radix: u32, loc: Location) -> Result { + u32::from_str_radix(digits, radix) + .ok() + .and_then(char::from_u32) + .map_or_else(|| self.tokenizer_error(loc, "Invalid escape sequence."), Ok) + } + + /// Decode a `\uhhhh` escape (the `\u` has already been consumed). A high + /// surrogate must be followed by a `\uhhhh` low surrogate and combines with + /// it into an astral code point; an unpaired high or low surrogate is an + /// error, matching real Snowflake. + fn unescape_unicode(&self, chars: &mut State, loc: Location) -> Result { + let value = self.read_hex4(chars, loc)?; + if (0xD800..=0xDBFF).contains(&value) { + return match self.read_low_surrogate(chars) { + Some(low) => { + let cp = 0x1_0000 + ((value - 0xD800) << 10) + (low - 0xDC00); + char::from_u32(cp) + .map_or_else(|| self.tokenizer_error(loc, "Invalid escape sequence."), Ok) + } + None => self.tokenizer_error( + loc, + format!( + "Invalid Unicode string literal; high surrogate '\\u{value:04X}' \ + must be followed by a low surrogate ('\\uDC00'-'\\uDFFF')." + ), + ), + }; + } + if (0xDC00..=0xDFFF).contains(&value) { + return self.tokenizer_error( + loc, + format!( + "Invalid Unicode string literal; low surrogate '\\u{value:04X}' \ + must be preceded by a high surrogate ('\\uD800'-'\\uDBFF')." + ), + ); + } + char::from_u32(value) + .map_or_else(|| self.tokenizer_error(loc, "Invalid escape sequence."), Ok) + } + + /// Read exactly four hex digits as a `u32`, erroring like Snowflake when + /// fewer than four are present. + fn read_hex4(&self, chars: &mut State, loc: Location) -> Result { + let mut digits = String::new(); + while digits.len() < 4 { + match chars.peek() { + Some(c) if c.is_ascii_hexdigit() => { + if let Some(c) = chars.next() { + digits.push(c); + } + } + _ => break, + } + } + if digits.len() < 4 { + return self.tokenizer_error( + loc, + format!( + "Invalid unicode escape sequence '\\u{digits}'; should be exactly 4 digits." + ), + ); + } + u32::from_str_radix(&digits, 16).map_or_else( + |_| self.tokenizer_error(loc, "Invalid unicode escape sequence."), + Ok, + ) + } + + /// Consume a trailing `\uhhhh` low surrogate, returning its value when + /// present and in range. Consumes greedily on the error path — the caller + /// aborts the whole token, so partial consumption is irrelevant. + fn read_low_surrogate(&self, chars: &mut State) -> Option { + if chars.peek() != Some(&'\\') { + return None; + } + chars.next(); + if chars.peek() != Some(&'u') { + return None; + } + chars.next(); + let mut digits = String::new(); + while digits.len() < 4 { + match chars.peek() { + Some(c) if c.is_ascii_hexdigit() => { + if let Some(c) = chars.next() { + digits.push(c); + } + } + _ => break, + } + } + if digits.len() < 4 { + return None; + } + u32::from_str_radix(&digits, 16) + .ok() + .filter(|low| (0xDC00..=0xDFFF).contains(low)) + } + fn tokenize_multiline_comment( &self, chars: &mut State, @@ -3878,10 +4050,13 @@ mod tests { (r#"'%a\'%b'"#, r#"%a\'%b"#, r#"%a'%b"#), (r#"'a\'\'b\'c\'d'"#, r#"a\'\'b\'c\'d"#, r#"a''b'c'd"#), (r#"'\\'"#, r#"\\"#, r#"\"#), + // Snowflake keeps `\0` as NUL, maps the control-letter escapes, and + // collapses unrecognized letters (`\a` -> a, `\Z` -> Z) to the bare + // letter rather than a control character. ( r#"'\0\a\b\f\n\r\t\Z'"#, r#"\0\a\b\f\n\r\t\Z"#, - "\0\u{7}\u{8}\u{c}\n\r\t\u{1a}", + "\0a\u{8}\u{c}\n\r\tZ", ), (r#"'\"'"#, r#"\""#, "\""), (r#"'\\a\\b\'c'"#, r#"\\a\\b\'c"#, r#"\a\b'c"#), @@ -3890,6 +4065,20 @@ mod tests { (r#"'\q'"#, r#"\q"#, r#"q"#), (r#"'\%\_'"#, r#"\%\_"#, r#"%_"#), (r#"'\\%\\_'"#, r#"\\%\\_"#, r#"\%\_"#), + // Hex `\xhh` and unicode `\uhhhh` yield a code point; octal `\ooo` + // takes a third digit only when the byte does not overflow. + (r#"'\x41'"#, r#"\x41"#, "A"), + (r#"'\u0041'"#, r#"\u0041"#, "A"), + (r#"'\u00e9'"#, r#"\u00e9"#, "\u{e9}"), + (r#"'\ud83d\ude00'"#, r#"\ud83d\ude00"#, "\u{1f600}"), + (r#"'\xff'"#, r#"\xff"#, "\u{ff}"), + (r#"'A'"#, r#"A"#, "A"), + (r#"'é'"#, r#"é"#, "\u{e9}"), + (r#"'ꯍ'"#, r#"ꯍ"#, "\u{abcd}"), + (r#"'\101'"#, r#"\101"#, "A"), + (r#"'\1012'"#, r#"\1012"#, "A2"), + (r#"'\777'"#, r#"\777"#, "\u{3f}7"), + (r#"'\400'"#, r#"\400"#, "\u{20}0"), ] { let tokens = Tokenizer::new(&dialect, sql) .with_unescape(false) diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 9f2311be92..fd9da4bdc4 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -11254,7 +11254,11 @@ fn parse_escaped_string_with_unescape() { let escaping_dialects = &all_dialects_where(|dialect| dialect.supports_string_literal_backslash_escape()); let no_wildcard_exception = &all_dialects_where(|dialect| { - dialect.supports_string_literal_backslash_escape() && !dialect.ignores_wildcard_escapes() + dialect.supports_string_literal_backslash_escape() + && !dialect.ignores_wildcard_escapes() + // Snowflake decodes `\Z` / `\a` as the bare letter, not a control + // character, so it is exercised separately. + && !dialect.supports_snowflake_string_literal_escapes() }); let with_wildcard_exception = &all_dialects_where(|dialect| { dialect.supports_string_literal_backslash_escape() && dialect.ignores_wildcard_escapes() From dba3a43a13465f68569fd593d87473abd80f7625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wojciech=20Pad=C5=82o?= Date: Thu, 30 Jul 2026 13:07:22 +0200 Subject: [PATCH 2/2] Satisfy rust-1.97 clippy on main (useless_borrows_in_formatting, needless_return_with_question_mark) Pre-existing lints in ast/parser/tests/examples surfaced by the stable toolchain advancing to rust-1.97 (main last passed CI on the older stable). Mechanical fixes only; no behaviour change. Unrelated to the escape-parity commit but required to get this PR's lint job green. Co-Authored-By: Claude Opus 4.8 --- examples/cli.rs | 4 ++-- src/ast/ddl.rs | 2 +- src/ast/mod.rs | 2 +- src/ast/query.rs | 2 +- src/parser/mod.rs | 8 ++++---- tests/sqlparser_common.rs | 6 +++--- tests/sqlparser_postgres.rs | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/cli.rs b/examples/cli.rs index 3c4299b209..51a63a7ea1 100644 --- a/examples/cli.rs +++ b/examples/cli.rs @@ -71,9 +71,9 @@ $ cargo run --example cli - [--dialectname] .expect("failed to read from stdin"); String::from_utf8(buf).expect("stdin content wasn't valid utf8") } else { - println!("Parsing from file '{}' using {:?}", &filename, dialect); + println!("Parsing from file '{}' using {:?}", filename, dialect); fs::read_to_string(&filename) - .unwrap_or_else(|_| panic!("Unable to read the file {}", &filename)) + .unwrap_or_else(|_| panic!("Unable to read the file {}", filename)) }; let without_bom = if contents.chars().next().unwrap() as u64 != 0xfeff { contents.as_str() diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index a8821fc5b2..8493323453 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -4735,7 +4735,7 @@ impl fmt::Display for AlterTable { if self.only { write!(f, "ONLY ")?; } - write!(f, "{} ", &self.name)?; + write!(f, "{} ", self.name)?; if let Some(cluster) = &self.on_cluster { write!(f, "ON CLUSTER {cluster} ")?; } diff --git a/src/ast/mod.rs b/src/ast/mod.rs index dbbc9103fc..0d25be2806 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -13462,7 +13462,7 @@ impl fmt::Display for AlterUser { let has_props = !self.set_props.options.is_empty(); if has_props { write!(f, " SET")?; - write!(f, " {}", &self.set_props)?; + write!(f, " {}", self.set_props)?; } if !self.unset_props.is_empty() { write!(f, " UNSET {}", display_comma_separated(&self.unset_props))?; diff --git a/src/ast/query.rs b/src/ast/query.rs index 6b8b36eb4c..af17340fd6 100644 --- a/src/ast/query.rs +++ b/src/ast/query.rs @@ -3530,7 +3530,7 @@ pub struct LockClause { impl fmt::Display for LockClause { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "FOR {}", &self.lock_type)?; + write!(f, "FOR {}", self.lock_type)?; if let Some(ref of) = self.of { write!(f, " OF {of}")?; } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 309e1ffb8a..53128543b8 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -5010,7 +5010,7 @@ impl<'a> Parser<'a> { if self.parse_keyword(expected) { Ok(self.get_current_token().clone()) } else { - self.expected_ref(format!("{:?}", &expected).as_str(), self.peek_token_ref()) + self.expected_ref(format!("{:?}", expected).as_str(), self.peek_token_ref()) } } @@ -5023,7 +5023,7 @@ impl<'a> Parser<'a> { if self.parse_keyword(expected) { Ok(()) } else { - self.expected_ref(format!("{:?}", &expected).as_str(), self.peek_token_ref()) + self.expected_ref(format!("{:?}", expected).as_str(), self.peek_token_ref()) } } @@ -13974,7 +13974,7 @@ impl<'a> Parser<'a> { } Token::EOF => break, token => { - return Err(ParserError::ParserError(format!( + Err(ParserError::ParserError(format!( "Unexpected token in identifier: {token}" )))?; } @@ -17298,7 +17298,7 @@ impl<'a> Parser<'a> { where_clause = Some(self.parse_expr()?); } else { let tok = self.peek_token_ref(); - return parser_err!( + parser_err!( format!( "Expected one of DIMENSIONS, METRICS, FACTS or WHERE, got {}", tok.token diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index fd9da4bdc4..e01acbe412 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -1719,7 +1719,7 @@ fn parse_json_ops_without_colon() { ]; for (str_op, op, dialects) in binary_ops { - let select = dialects.verified_only_select(&format!("SELECT a {} b", &str_op)); + let select = dialects.verified_only_select(&format!("SELECT a {} b", str_op)); assert_eq!( SelectItem::UnnamedExpr(Expr::BinaryOp { left: Box::new(Expr::Identifier(Ident::new("a"))), @@ -2415,7 +2415,7 @@ fn parse_bitwise_ops() { ]; for (str_op, op, dialects) in bitwise_ops { - let select = dialects.verified_only_select(&format!("SELECT a {} b", &str_op)); + let select = dialects.verified_only_select(&format!("SELECT a {} b", str_op)); assert_eq!( SelectItem::UnnamedExpr(Expr::BinaryOp { left: Box::new(Expr::Identifier(Ident::new("a"))), @@ -18599,7 +18599,7 @@ fn parse_generic_unary_ops() { ("+", UnaryOperator::Plus), ]; for (str_op, op) in unary_ops { - let select = verified_only_select(&format!("SELECT {}expr", &str_op)); + let select = verified_only_select(&format!("SELECT {}expr", str_op)); assert_eq!( UnnamedExpr(UnaryOp { op: *op, diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 141ca0bec6..dbee5970d9 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -2461,7 +2461,7 @@ fn parse_pg_unary_ops() { ("@", UnaryOperator::PGAbs), ]; for (str_op, op) in pg_unary_ops { - let select = pg().verified_only_select(&format!("SELECT {}a", &str_op)); + let select = pg().verified_only_select(&format!("SELECT {}a", str_op)); assert_eq!( SelectItem::UnnamedExpr(Expr::UnaryOp { op: *op, @@ -2477,7 +2477,7 @@ fn parse_pg_postfix_factorial() { let postfix_factorial = &[("!", UnaryOperator::PGPostfixFactorial)]; for (str_op, op) in postfix_factorial { - let select = pg().verified_only_select(&format!("SELECT a{}", &str_op)); + let select = pg().verified_only_select(&format!("SELECT a{}", str_op)); assert_eq!( SelectItem::UnnamedExpr(Expr::UnaryOp { op: *op,