Skip to content

Commit 38f1864

Browse files
committed
Clean up JSON operator tokenizing code
1 parent 518149c commit 38f1864

File tree

1 file changed

+15
-22
lines changed

1 file changed

+15
-22
lines changed

src/tokenizer.rs

Lines changed: 15 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -497,12 +497,14 @@ impl<'a> Tokenizer<'a> {
497497
Ok(tokens)
498498
}
499499

500+
// Tokenize the identifer or keywords in `ch`
500501
fn tokenize_identifier_or_keyword(
501502
&self,
502-
ch: String,
503+
ch: impl IntoIterator<Item = char>,
503504
chars: &mut State,
504505
) -> Result<Option<Token>, TokenizerError> {
505506
chars.next(); // consume the first char
507+
let ch: String = ch.into_iter().collect();
506508
let word = self.tokenize_word(ch, chars);
507509

508510
// TODO: implement parsing of exponent here
@@ -550,7 +552,7 @@ impl<'a> Tokenizer<'a> {
550552
}
551553
_ => {
552554
// regular identifier starting with an "b" or "B"
553-
let s = self.tokenize_word(b.to_string(), chars);
555+
let s = self.tokenize_word(b, chars);
554556
Ok(Some(Token::make_word(&s, None)))
555557
}
556558
}
@@ -569,7 +571,7 @@ impl<'a> Tokenizer<'a> {
569571
}
570572
_ => {
571573
// regular identifier starting with an "r" or "R"
572-
let s = self.tokenize_word(b.to_string(), chars);
574+
let s = self.tokenize_word(b, chars);
573575
Ok(Some(Token::make_word(&s, None)))
574576
}
575577
}
@@ -585,7 +587,7 @@ impl<'a> Tokenizer<'a> {
585587
}
586588
_ => {
587589
// regular identifier starting with an "N"
588-
let s = self.tokenize_word(n.to_string(), chars);
590+
let s = self.tokenize_word(n, chars);
589591
Ok(Some(Token::make_word(&s, None)))
590592
}
591593
}
@@ -602,7 +604,7 @@ impl<'a> Tokenizer<'a> {
602604
}
603605
_ => {
604606
// regular identifier starting with an "E" or "e"
605-
let s = self.tokenize_word(x.to_string(), chars);
607+
let s = self.tokenize_word(x, chars);
606608
Ok(Some(Token::make_word(&s, None)))
607609
}
608610
}
@@ -619,7 +621,7 @@ impl<'a> Tokenizer<'a> {
619621
}
620622
_ => {
621623
// regular identifier starting with an "X"
622-
let s = self.tokenize_word(x.to_string(), chars);
624+
let s = self.tokenize_word(x, chars);
623625
Ok(Some(Token::make_word(&s, None)))
624626
}
625627
}
@@ -794,9 +796,7 @@ impl<'a> Tokenizer<'a> {
794796
match chars.peek() {
795797
Some(' ') => self.consume_and_return(chars, Token::Mod),
796798
Some(sch) if self.dialect.is_identifier_start('%') => {
797-
let mut s = ch.to_string();
798-
s.push_str(&sch.to_string());
799-
self.tokenize_identifier_or_keyword(s, chars)
799+
self.tokenize_identifier_or_keyword([ch, *sch], chars)
800800
}
801801
_ => self.consume_and_return(chars, Token::Mod),
802802
}
@@ -917,9 +917,7 @@ impl<'a> Tokenizer<'a> {
917917
}
918918
Some(' ') => Ok(Some(Token::Sharp)),
919919
Some(sch) if self.dialect.is_identifier_start('#') => {
920-
let mut s = ch.to_string();
921-
s.push_str(&sch.to_string());
922-
self.tokenize_identifier_or_keyword(s, chars)
920+
self.tokenize_identifier_or_keyword([ch, *sch], chars)
923921
}
924922
_ => Ok(Some(Token::Sharp)),
925923
}
@@ -934,19 +932,14 @@ impl<'a> Tokenizer<'a> {
934932
match chars.peek() {
935933
Some(' ') => Ok(Some(Token::AtAt)),
936934
Some(tch) if self.dialect.is_identifier_start('@') => {
937-
let mut s = ch.to_string();
938-
s.push('@');
939-
s.push_str(&tch.to_string());
940-
self.tokenize_identifier_or_keyword(s, chars)
935+
self.tokenize_identifier_or_keyword([ch, '@', *tch], chars)
941936
}
942937
_ => Ok(Some(Token::AtAt)),
943938
}
944939
}
945940
Some(' ') => Ok(Some(Token::AtSign)),
946941
Some(sch) if self.dialect.is_identifier_start('@') => {
947-
let mut s = ch.to_string();
948-
s.push_str(&sch.to_string());
949-
self.tokenize_identifier_or_keyword(s, chars)
942+
self.tokenize_identifier_or_keyword([ch, *sch], chars)
950943
}
951944
_ => Ok(Some(Token::AtSign)),
952945
}
@@ -959,7 +952,7 @@ impl<'a> Tokenizer<'a> {
959952

960953
// identifier or keyword
961954
ch if self.dialect.is_identifier_start(ch) => {
962-
self.tokenize_identifier_or_keyword(ch.to_string(), chars)
955+
self.tokenize_identifier_or_keyword([ch], chars)
963956
}
964957
'$' => Ok(Some(self.tokenize_dollar_preceded_value(chars)?)),
965958

@@ -1086,8 +1079,8 @@ impl<'a> Tokenizer<'a> {
10861079
}
10871080

10881081
/// Tokenize an identifier or keyword, after the first char is already consumed.
1089-
fn tokenize_word(&self, first_chars: String, chars: &mut State) -> String {
1090-
let mut s = first_chars;
1082+
fn tokenize_word(&self, first_chars: impl Into<String>, chars: &mut State) -> String {
1083+
let mut s = first_chars.into();
10911084
s.push_str(&peeking_take_while(chars, |ch| {
10921085
self.dialect.is_identifier_part(ch)
10931086
}));

0 commit comments

Comments
 (0)