Start letting the parser catch interpolated ASTs.

Author: paulstansifer

src/libsyntax/ext/tt/earley_parser.rs

@@ -259,7 +259,11 @@ fn parse_nt(p: parser, name: str) -> whole_nt {
       "expr" { token::w_expr(p.parse_expr()) }
       "ty" { token::w_ty(p.parse_ty(false /* no need to disambiguate*/)) }
       // this could be handled like a token, since it is one
-      "ident" { token::w_ident(p.parse_ident()) }
+      "ident" { alt copy p.token {
+          token::IDENT(sn,b) { p.bump(); token::w_ident(sn,b) }
+          _ { p.fatal("expected ident, found "
+                      + token::to_str(*p.reader.interner(), copy p.token)) }
+      } }
       "path" { token::w_path(p.parse_path_with_tps(false)) }
       _ { p.fatal("Unsupported builtin nonterminal parser: " + name)}
     }

src/libsyntax/ext/tt/transcribe.rs

@@ -3,7 +3,7 @@ import diagnostic::span_handler;
 import ast::{token_tree,tt_delim,tt_flat,tt_dotdotdot,tt_interpolate,ident};
 import earley_parser::{arb_depth,seq,leaf};
 import codemap::span;
-import parse::token::{EOF,ACTUALLY,token};
+import parse::token::{EOF,ACTUALLY,IDENT,token,w_ident};
 import std::map::{hashmap,box_str_hash};
 
 export tt_reader,  new_tt_reader, dup_tt_reader, tt_next_token;
@@ -193,8 +193,17 @@ fn tt_next_token(&&r: tt_reader) -> {tok: token, sp: span} {
           // TODO: think about span stuff here
           tt_interpolate(sp, ident) {
             alt *lookup_cur_ad(r, ident) {
+              /* sidestep the interpolation tricks for ident because
+              (a) idents can be in lots of places, so it'd be a pain
+              (b) we actually can, since it's a token. */
+              leaf(w_ident(sn,b)) {
+                r.cur_span = sp; r.cur_tok = IDENT(sn,b);
+                r.cur.idx += 1u;
+                ret ret_val;
+              }
               leaf(w_nt) {
                 r.cur_span = sp; r.cur_tok = ACTUALLY(w_nt);
+                r.cur.idx += 1u;
                 ret ret_val;
               }
               seq(*) {

src/libsyntax/parse/common.rs

@@ -50,6 +50,8 @@ impl parser_common for parser {
     fn parse_ident() -> ast::ident {
         alt copy self.token {
           token::IDENT(i, _) { self.bump(); ret self.get_str(i); }
+          token::ACTUALLY(token::w_ident(*)) { self.bug(
+              "ident interpolation not converted to real token"); }
           _ { self.fatal("expecting ident, found "
                       + token_to_str(self.reader, self.token)); }
         }

src/libsyntax/parse/lexer.rs

@@ -459,8 +459,7 @@ fn next_token_inner(rdr: string_reader) -> token::token {
         let is_mod_name = c == ':' && nextch(rdr) == ':';
 
         // FIXME: perform NFKC normalization here. (Issue #2253)
-        ret token::IDENT(intern(*rdr.interner,
-                                          @accum_str), is_mod_name);
+        ret token::IDENT(intern(*rdr.interner, @accum_str), is_mod_name);
     }
     if is_dec_digit(c) {
         ret scan_number(c, rdr);

src/libsyntax/parse/parser.rs

@@ -1,7 +1,7 @@
 import result::result;
 import either::{either, left, right};
 import std::map::{hashmap, str_hash};
-import token::{can_begin_expr, is_ident, is_plain_ident};
+import token::{can_begin_expr, is_ident, is_plain_ident, ACTUALLY};
 import codemap::{span,fss_none};
 import util::interner;
 import ast_util::{spanned, respan, mk_sp, ident_to_path, operator_prec};
@@ -59,6 +59,78 @@ enum class_contents { ctor_decl(fn_decl, blk, codemap::span),
 type arg_or_capture_item = either<arg, capture_item>;
 type item_info = (ident, item_, option<~[attribute]>);
 
+fn dummy() {
+
+/* We need to position the macros to capture the ACTUALLY tokens before they
+get bumped away. So two bumps in a row is bad. (the first lookahead also
+counts as a bump).
+
+Events happen L to R; 'B' indicates a bump before hand:
+           ._____________________________.________________________.
+           ↓                            B|                        |
+   parse_expr -> parse_expr_res -> parse_assign_expr              |
+                                    ↓                             |
+                            parse_binops -> parse_more_binops     |
+                             ↓             B↓                    B|
+                            parse_prefix_expr B-> parse_dot_or_call_expr
+                             B|_↑                  ↓
+                                                  parse_bottom_expr
+                                                  B↓
+                                            ⋯->parse_ident
+...so we've hit parse_prefix_expr, parse_more_binops, and parse_bottom_expr.
+*/
+
+    #macro[[#maybe_whole_item[p],
+            alt copy p.token {
+                ACTUALLY(token::w_item(i)) { p.bump(); ret i; }
+                _ {} }]];
+    #macro[[#maybe_whole_block[p],
+            alt copy p.token {
+                ACTUALLY(token::w_block(b)) { p.bump(); ret b; }
+                _ {} }]];
+    #macro[[#maybe_whole_stmt[p],
+            alt copy p.token {
+                ACTUALLY(token::w_stmt(s)) { p.bump(); ret s; }
+                _ {} }]];
+    #macro[[#maybe_whole_pat[p],
+            alt copy p.token {
+                ACTUALLY(token::w_pat(pt)) { p.bump(); ret pt; }
+                _ {} }]];
+    #macro[[#maybe_whole_expr[p],
+            alt copy p.token {
+                ACTUALLY(token::w_expr(e)) {
+                    p.bump();
+                    ret e;
+                }
+                ACTUALLY(token::w_path(pt)) {
+                    p.bump();
+                    ret p.mk_expr(p.span.lo, p.span.lo,
+                                  expr_path(pt));
+                }
+                _ {} }]];
+    #macro[[#maybe_whole_expr_pexpr[p], /* ack! */
+            alt copy p.token {
+                ACTUALLY(token::w_expr(e)) {
+                    p.bump();
+                    ret pexpr(e);
+                }
+                ACTUALLY(token::w_path(pt)) {
+                    p.bump();
+                    ret p.mk_pexpr(p.span.lo, p.span.lo,
+                                   expr_path(pt));
+                }
+                _ {} }]];
+    #macro[[#maybe_whole_ty[p],
+            alt copy p.token {
+                ACTUALLY(token::w_ty(t)) { p.bump(); ret t; }
+                _ {} }]];
+    /* ident is handled by common.rs */
+    #macro[[#maybe_whole_path[p],
+            alt p.token {
+                ACTUALLY(token::w_path(pt)) { p.bump(); ret pt; }
+                _ {} }]];
+}
+
 class parser {
     let sess: parse_sess;
     let cfg: crate_cfg;
@@ -718,6 +790,7 @@ class parser {
     }
 
     fn parse_bottom_expr() -> pexpr {
+        #maybe_whole_expr_pexpr[self];
         let lo = self.span.lo;
         let mut hi = self.span.hi;
 
@@ -1181,6 +1254,7 @@ class parser {
 
 
     fn parse_prefix_expr() -> pexpr {
+        #maybe_whole_expr_pexpr[self];
         let lo = self.span.lo;
         let mut hi;
 
@@ -1258,6 +1332,7 @@ class parser {
 
     fn parse_more_binops(plhs: pexpr, min_prec: uint) ->
         @expr {
+        #maybe_whole_expr[self];
         let lhs = self.to_expr(plhs);
         if self.expr_is_complete(plhs) { ret lhs; }
         let peeked = self.token;

src/libsyntax/parse/token.rs

@@ -94,7 +94,7 @@ enum whole_nt {
     w_pat( @ast::pat),
     w_expr(@ast::expr),
     w_ty(  @ast::ty),
-    w_ident(ast::ident),
+    w_ident(str_num, bool),
     w_path(@ast::path),
 }