support universal characters

Author: Daniel Kroening

regression/ansi-c/Universal_characters1/test.desc

@@ -1,4 +1,4 @@
-KNOWNBUG
+CORE
 main.c
 
 ^EXIT=0$

src/ansi-c/literals/convert_string_literal.cpp

@@ -34,7 +34,7 @@ void convert_one_string_literal(
   std::basic_string<unsigned int> &value)
 {
   assert(src.size()>=2);
-
+  
   if(src[0]=='u' && src[1]=='8')
   {
     assert(src[src.size()-1]=='"');
@@ -63,6 +63,14 @@ void convert_one_string_literal(
     assert(src[src.size()-1]=='"');
 
     unescape_wide_string(std::string(src, 1, src.size()-2), value);
+
+    // turn into utf-8
+    std::string utf8_value=utf32_to_utf8(value);
+
+    // pad into wide string
+    value.resize(utf8_value.size());
+    for(unsigned i=0; i<utf8_value.size(); i++)
+      value[i]=utf8_value[i];
   }
 }
 

src/ansi-c/scanner.l

@@ -150,7 +150,8 @@ digit           [0-9]
 bindigit        [01]
 octdigit        [0-7]
 hexdigit        [0-9a-fA-F]
-identifier      (({letter}|"_"|"$")({letter}|{digit}|"_"|"$")*)
+utf8letter      [\x80-\xff]
+identifier      (({letter}|"_"|"$"|{utf8letter})({letter}|{digit}|"_"|"$"|{utf8letter})*)
 integer         {digit}+
 binary          {bindigit}+
 msiw_suffix     ([iI]("8"|"16"|"32"|"64"|"128"))