Struct Tokenizer

pub struct Tokenizer<'a> {
    sql: &'a str,
    chars: Peekable<Chars<'a>>,
    line: u64,
    col: u64,
}

SQL Tokenizer

Fields

sql: &'a str

chars: Peekable<Chars<'a>>

line: u64

col: u64

Implementations

impl<'a> Tokenizer<'a>

pub fn new(query: &'a str) -> Self

Create a new SQL tokenizer for the specified SQL statement

fn next(&mut self) -> Option<char>

Consume the next character.

fn peek(&mut self) -> Option<char>

Return the next character without consuming it.

pub fn tokenize_with_location( &mut self, ) -> Result<Vec<TokenWithLocation>, TokenizerError>

Tokenize the statement and produce a vector of tokens with locations.

Whitespaces are skipped.

fn tokenize_with_whitespace(&mut self) -> Result<Vec<Token>, TokenizerError>

Tokenize the statement and produce a vector of tokens.

Whitespaces are included.

fn tokenize(&mut self) -> Result<Vec<TokenWithLocation>, TokenizerError>

Tokenize the statement and produce a vector of tokens.

Whitespaces are included.

fn next_token_with_location( &mut self, ) -> Result<Option<TokenWithLocation>, TokenizerError>

Get the next token or return None

fn next_token(&mut self) -> Result<Option<Token>, TokenizerError>

Get the next token or return None

fn tokenize_dollar_preceded_value(&mut self) -> Result<Token, TokenizerError>

Tokenize dollar preceded value (i.e: a string/placeholder)

fn error<R>(&self, message: impl Into<String>) -> Result<R, TokenizerError>

fn reject_number_junk(&mut self) -> Result<(), TokenizerError>

fn tokenize_single_line_comment(&mut self) -> String

fn tokenize_word(&mut self, first_char: char) -> String

Tokenize an identifier or keyword, after the first char is already consumed.

fn tokenize_single_quoted_string(&mut self) -> Result<String, TokenizerError>

Read a single quoted string, starting with the opening quote.

fn tokenize_single_quoted_string_with_escape( &mut self, ) -> Result<CstyleEscapedString, TokenizerError>

Read a single qutoed string with escape

fn unescape_c_style(s: &str) -> Result<String, String>

Helper function used to convert string with c-style escapes into a normal string e.g. ‘hello\x3fworld’ -> ‘hello?world’

Detail of c-style escapes refer from: https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-STRINGS-UESCAPE:~:text=4.1.2.2.%C2%A0String%20Constants%20With%20C%2DStyle%20Escapes

fn tokenize_multiline_comment( &mut self, ) -> Result<Option<Token>, TokenizerError>

fn consume_and_return( &mut self, t: Token, ) -> Result<Option<Token>, TokenizerError>

fn peeking_take_while(&mut self, predicate: impl FnMut(char) -> bool) -> String

Read from self until predicate returns false or EOF is hit. Return the characters read as String, and keep the first non-matching char available as self.next().