Skip to content

Commit 0df8011

Browse files
committed
use dialect_of to limit tokenizer to PostgreSql dialect
1 parent c1dd27d commit 0df8011

File tree

1 file changed

+22
-7
lines changed

1 file changed

+22
-7
lines changed

src/tokenizer.rs

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ use std::str::Chars;
2121

2222
use super::dialect::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX};
2323
use super::dialect::Dialect;
24+
use super::dialect::PostgreSqlDialect;
2425
use super::dialect::SnowflakeDialect;
2526
#[cfg(feature = "serde")]
2627
use serde::{Deserialize, Serialize};
@@ -433,11 +434,15 @@ impl<'a> Tokenizer<'a> {
433434
'|' => {
434435
chars.next(); // consume the '|'
435436
match chars.peek() {
436-
Some('/') => self.consume_and_return(chars, Token::SquareRoot),
437+
Some('/') if dialect_of!(self is PostgreSqlDialect) => {
438+
self.consume_and_return(chars, Token::SquareRoot)
439+
}
437440
Some('|') => {
438441
chars.next(); // consume the second '|'
439442
match chars.peek() {
440-
Some('/') => self.consume_and_return(chars, Token::CubeRoot),
443+
Some('/') if dialect_of!(self is PostgreSqlDialect) => {
444+
self.consume_and_return(chars, Token::CubeRoot)
445+
}
441446
_ => Ok(Some(Token::StringConcat)),
442447
}
443448
}
@@ -457,7 +462,9 @@ impl<'a> Tokenizer<'a> {
457462
chars.next(); // consume
458463
match chars.peek() {
459464
Some('=') => self.consume_and_return(chars, Token::Neq),
460-
Some('!') => self.consume_and_return(chars, Token::DoubleExclamationMark),
465+
Some('!') if dialect_of!(self is PostgreSqlDialect) => {
466+
self.consume_and_return(chars, Token::DoubleExclamationMark)
467+
}
461468
_ => Ok(Some(Token::ExclamationMark)),
462469
}
463470
}
@@ -466,7 +473,9 @@ impl<'a> Tokenizer<'a> {
466473
match chars.peek() {
467474
Some('=') => self.consume_and_return(chars, Token::LtEq),
468475
Some('>') => self.consume_and_return(chars, Token::Neq),
469-
Some('<') => self.consume_and_return(chars, Token::ShiftLeft),
476+
Some('<') if dialect_of!(self is PostgreSqlDialect) => {
477+
self.consume_and_return(chars, Token::ShiftLeft)
478+
}
470479
_ => Ok(Some(Token::Lt)),
471480
}
472481
}
@@ -501,9 +510,15 @@ impl<'a> Tokenizer<'a> {
501510
comment,
502511
})))
503512
}
504-
'~' => self.consume_and_return(chars, Token::Tilde),
505-
'#' => self.consume_and_return(chars, Token::Sharp),
506-
'@' => self.consume_and_return(chars, Token::Ampersat),
513+
'~' if dialect_of!(self is PostgreSqlDialect) => {
514+
self.consume_and_return(chars, Token::Tilde)
515+
}
516+
'#' if dialect_of!(self is PostgreSqlDialect) => {
517+
self.consume_and_return(chars, Token::Sharp)
518+
}
519+
'@' if dialect_of!(self is PostgreSqlDialect) => {
520+
self.consume_and_return(chars, Token::Ampersat)
521+
}
507522
other => self.consume_and_return(chars, Token::Char(other)),
508523
},
509524
None => Ok(None),

0 commit comments

Comments
 (0)