syntax: convert the lexer to use Option<char> over transmute(-1).

The transmute was unsound.

There are many instances of .unwrap_or('\x00') for "ignoring" EOF which
either do not make the situation worse than it was (well, actually make
it better, since it's easy to grep for places that don't handle EOF) or
can never ever be read.

Fixes #8971.

Author: huonw

src/libsyntax/parse/comments.rs

@@ -12,7 +12,7 @@ use ast;
 use codemap::{BytePos, CharPos, CodeMap, Pos};
 use diagnostic;
 use parse::lexer::{is_whitespace, with_str_from, Reader};
-use parse::lexer::{StringReader, bump, is_eof, nextch, TokenAndSpan};
+use parse::lexer::{StringReader, bump, is_eof, nextch_is, TokenAndSpan};
 use parse::lexer::{is_line_non_doc_comment, is_block_non_doc_comment};
 use parse::lexer;
 use parse::token;
@@ -136,11 +136,11 @@ pub fn strip_doc_comment_decoration(comment: &str) -> ~str {
 
 fn read_to_eol(rdr: &StringReader) -> ~str {
     let mut val = ~"";
-    while rdr.curr.get() != '\n' && !is_eof(rdr) {
-        val.push_char(rdr.curr.get());
+    while !rdr.curr_is('\n') && !is_eof(rdr) {
+        val.push_char(rdr.curr.get().unwrap());
         bump(rdr);
     }
-    if rdr.curr.get() == '\n' { bump(rdr); }
+    if rdr.curr_is('\n') { bump(rdr); }
     return val;
 }
 
@@ -152,7 +152,7 @@ fn read_one_line_comment(rdr: &StringReader) -> ~str {
 }
 
 fn consume_non_eol_whitespace(rdr: &StringReader) {
-    while is_whitespace(rdr.curr.get()) && rdr.curr.get() != '\n' &&
+    while is_whitespace(rdr.curr.get()) && !rdr.curr_is('\n') &&
             !is_eof(rdr) {
         bump(rdr);
     }
@@ -171,7 +171,7 @@ fn push_blank_line_comment(rdr: &StringReader, comments: &mut ~[Comment]) {
 fn consume_whitespace_counting_blank_lines(rdr: &StringReader,
                                            comments: &mut ~[Comment]) {
     while is_whitespace(rdr.curr.get()) && !is_eof(rdr) {
-        if rdr.col.get() == CharPos(0u) && rdr.curr.get() == '\n' {
+        if rdr.col.get() == CharPos(0u) && rdr.curr_is('\n') {
             push_blank_line_comment(rdr, &mut *comments);
         }
         bump(rdr);
@@ -196,7 +196,7 @@ fn read_line_comments(rdr: &StringReader, code_to_the_left: bool,
     debug!(">>> line comments");
     let p = rdr.last_pos.get();
     let mut lines: ~[~str] = ~[];
-    while rdr.curr.get() == '/' && nextch(rdr) == '/' {
+    while rdr.curr_is('/') && nextch_is(rdr, '/') {
         let line = read_one_line_comment(rdr);
         debug!("{}", line);
         if is_doc_comment(line) { // doc-comments are not put in comments
@@ -261,9 +261,9 @@ fn read_block_comment(rdr: &StringReader,
     let mut curr_line = ~"/*";
 
     // doc-comments are not really comments, they are attributes
-    if rdr.curr.get() == '*' || rdr.curr.get() == '!' {
-        while !(rdr.curr.get() == '*' && nextch(rdr) == '/') && !is_eof(rdr) {
-            curr_line.push_char(rdr.curr.get());
+    if rdr.curr_is('*') || rdr.curr_is('!') {
+        while !(rdr.curr_is('*') && nextch_is(rdr, '/')) && !is_eof(rdr) {
+            curr_line.push_char(rdr.curr.get().unwrap());
             bump(rdr);
         }
         if !is_eof(rdr) {
@@ -281,20 +281,20 @@ fn read_block_comment(rdr: &StringReader,
             if is_eof(rdr) {
                 rdr.fatal(~"unterminated block comment");
             }
-            if rdr.curr.get() == '\n' {
+            if rdr.curr_is('\n') {
                 trim_whitespace_prefix_and_push_line(&mut lines, curr_line,
                                                      col);
                 curr_line = ~"";
                 bump(rdr);
             } else {
-                curr_line.push_char(rdr.curr.get());
-                if rdr.curr.get() == '/' && nextch(rdr) == '*' {
+                curr_line.push_char(rdr.curr.get().unwrap());
+                if rdr.curr_is('/') && nextch_is(rdr, '*') {
                     bump(rdr);
                     bump(rdr);
                     curr_line.push_char('*');
                     level += 1;
                 } else {
-                    if rdr.curr.get() == '*' && nextch(rdr) == '/' {
+                    if rdr.curr_is('*') && nextch_is(rdr, '/') {
                         bump(rdr);
                         bump(rdr);
                         curr_line.push_char('/');
@@ -310,28 +310,28 @@ fn read_block_comment(rdr: &StringReader,
 
     let mut style = if code_to_the_left { Trailing } else { Isolated };
     consume_non_eol_whitespace(rdr);
-    if !is_eof(rdr) && rdr.curr.get() != '\n' && lines.len() == 1u {
+    if !is_eof(rdr) && !rdr.curr_is('\n') && lines.len() == 1u {
         style = Mixed;
     }
     debug!("<<< block comment");
     comments.push(Comment {style: style, lines: lines, pos: p});
 }
 
 fn peeking_at_comment(rdr: &StringReader) -> bool {
-    return ((rdr.curr.get() == '/' && nextch(rdr) == '/') ||
-         (rdr.curr.get() == '/' && nextch(rdr) == '*')) ||
-         (rdr.curr.get() == '#' && nextch(rdr) == '!');
+    return (rdr.curr_is('/') && nextch_is(rdr, '/')) ||
+         (rdr.curr_is('/') && nextch_is(rdr, '*')) ||
+         (rdr.curr_is('#') && nextch_is(rdr, '!'));
 }
 
 fn consume_comment(rdr: &StringReader,
                    code_to_the_left: bool,
                    comments: &mut ~[Comment]) {
     debug!(">>> consume comment");
-    if rdr.curr.get() == '/' && nextch(rdr) == '/' {
+    if rdr.curr_is('/') && nextch_is(rdr, '/') {
         read_line_comments(rdr, code_to_the_left, comments);
-    } else if rdr.curr.get() == '/' && nextch(rdr) == '*' {
+    } else if rdr.curr_is('/') && nextch_is(rdr, '*') {
         read_block_comment(rdr, code_to_the_left, comments);
-    } else if rdr.curr.get() == '#' && nextch(rdr) == '!' {
+    } else if rdr.curr_is('#') && nextch_is(rdr, '!') {
         read_shebang_comment(rdr, code_to_the_left, comments);
     } else { fail!(); }
     debug!("<<< consume comment");
@@ -363,7 +363,7 @@ pub fn gather_comments_and_literals(span_diagnostic:
         loop {
             let mut code_to_the_left = !first_read;
             consume_non_eol_whitespace(&rdr);
-            if rdr.curr.get() == '\n' {
+            if rdr.curr_is('\n') {
                 code_to_the_left = false;
                 consume_whitespace_counting_blank_lines(&rdr, &mut comments);
             }