add serialization library; convert ebml lib to use u64 internally

Author: nikomatsakis

src/comp/metadata/decoder.rs

@@ -42,14 +42,14 @@ fn lookup_hash(d: ebml::doc, eq_fn: fn@([u8]) -> bool, hash: uint) ->
     let index = ebml::get_doc(d, tag_index);
     let table = ebml::get_doc(index, tag_index_table);
     let hash_pos = table.start + hash % 256u * 4u;
-    let pos = ebml::be_uint_from_bytes(d.data, hash_pos, 4u);
+    let pos = ebml::be_u64_from_bytes(d.data, hash_pos, 4u) as uint;
     let {tag:_, doc:bucket} = ebml::doc_at(d.data, pos);
     // Awkward logic because we can't ret from foreach yet
 
     let result: [ebml::doc] = [];
     let belt = tag_index_buckets_bucket_elt;
     ebml::tagged_docs(bucket, belt) {|elt|
-        let pos = ebml::be_uint_from_bytes(elt.data, elt.start, 4u);
+        let pos = ebml::be_u64_from_bytes(elt.data, elt.start, 4u) as uint;
         if eq_fn(vec::slice::<u8>(*elt.data, elt.start + 4u, elt.end)) {
             result += [ebml::doc_at(d.data, pos).doc];
         }
@@ -59,7 +59,7 @@ fn lookup_hash(d: ebml::doc, eq_fn: fn@([u8]) -> bool, hash: uint) ->
 
 fn maybe_find_item(item_id: int, items: ebml::doc) -> option<ebml::doc> {
     fn eq_item(bytes: [u8], item_id: int) -> bool {
-        ret ebml::be_uint_from_bytes(@bytes, 0u, 4u) as int == item_id;
+        ret ebml::be_u64_from_bytes(@bytes, 0u, 4u) as int == item_id;
     }
     let eqer = bind eq_item(_, item_id);
     let found = lookup_hash(items, eqer, hash_node_id(item_id));
@@ -81,7 +81,7 @@ fn lookup_item(item_id: int, data: @[u8]) -> ebml::doc {
 
 fn item_family(item: ebml::doc) -> u8 {
     let fam = ebml::get_doc(item, tag_items_data_item_family);
-    ret ebml::doc_as_uint(fam) as u8;
+    ret ebml::doc_as_u8(fam);
 }
 
 fn item_symbol(item: ebml::doc) -> str {
@@ -183,7 +183,7 @@ fn item_path(item_doc: ebml::doc) -> ast_map::path {
     let path_doc = ebml::get_doc(item_doc, tag_path);
 
     let len_doc = ebml::get_doc(path_doc, tag_path_len);
-    let len = ebml::doc_as_uint(len_doc);
+    let len = ebml::doc_as_vuint(len_doc);
 
     let result = [];
     vec::reserve(result, len);
@@ -355,7 +355,7 @@ fn family_names_type(fam_ch: u8) -> bool {
 
 fn read_path(d: ebml::doc) -> {path: str, pos: uint} {
     let desc = ebml::doc_data(d);
-    let pos = ebml::be_uint_from_bytes(@desc, 0u, 4u);
+    let pos = ebml::be_u64_from_bytes(@desc, 0u, 4u) as uint;
     let pathbytes = vec::slice::<u8>(desc, 4u, vec::len::<u8>(desc));
     let path = str::from_bytes(pathbytes);
     ret {path: path, pos: pos};

src/comp/metadata/encoder.rs

@@ -281,7 +281,7 @@ fn encode_path(ebml_w: ebml::writer,
 
     ebml_w.wr_tag(tag_path) {||
         ebml_w.wr_tag(tag_path_len) {||
-            ebml_w.wr_uint(vec::len(path) + 1u);
+            ebml_w.wr_vuint(vec::len(path) + 1u);
         }
         vec::iter(path) {|pe| encode_path_elt(ebml_w, pe); }
         encode_path_elt(ebml_w, name);

src/libstd/ebml.rs

@@ -19,35 +19,42 @@ type doc = {data: @[u8], start: uint, end: uint};
 
 type tagged_doc = {tag: uint, doc: doc};
 
-fn vint_at(data: [u8], start: uint) -> {val: uint, next: uint} {
+fn vu64_at(data: [u8], start: uint) -> {val: u64, next: uint} {
     let a = data[start];
     if a & 0x80u8 != 0u8 {
-        ret {val: (a & 0x7fu8) as uint, next: start + 1u};
+        ret {val: (a & 0x7fu8) as u64, next: start + 1u};
     }
     if a & 0x40u8 != 0u8 {
-        ret {val: ((a & 0x3fu8) as uint) << 8u | (data[start + 1u] as uint),
+        ret {val: ((a & 0x3fu8) as u64) << 8u64 |
+                 (data[start + 1u] as u64),
              next: start + 2u};
     } else if a & 0x20u8 != 0u8 {
-        ret {val: ((a & 0x1fu8) as uint) << 16u |
-                 (data[start + 1u] as uint) << 8u |
-                 (data[start + 2u] as uint),
+        ret {val: ((a & 0x1fu8) as u64) << 16u64 |
+                 (data[start + 1u] as u64) << 8u64 |
+                 (data[start + 2u] as u64),
              next: start + 3u};
     } else if a & 0x10u8 != 0u8 {
-        ret {val: ((a & 0x0fu8) as uint) << 24u |
-                 (data[start + 1u] as uint) << 16u |
-                 (data[start + 2u] as uint) << 8u |
-                 (data[start + 3u] as uint),
+        ret {val: ((a & 0x0fu8) as u64) << 24u64 |
+                 (data[start + 1u] as u64) << 16u64 |
+                 (data[start + 2u] as u64) << 8u64 |
+                 (data[start + 3u] as u64),
              next: start + 4u};
     } else { #error("vint too big"); fail; }
 }
 
+fn vuint_at(data: [u8], start: uint) -> {val: uint, next: uint} {
+    let {val, next} = vu64_at(data, start);
+    ret {val: val as uint, next: next};
+}
+
+
 fn new_doc(data: @[u8]) -> doc {
     ret {data: data, start: 0u, end: vec::len::<u8>(*data)};
 }
 
 fn doc_at(data: @[u8], start: uint) -> tagged_doc {
-    let elt_tag = vint_at(*data, start);
-    let elt_size = vint_at(*data, elt_tag.next);
+    let elt_tag = vuint_at(*data, start);
+    let elt_size = vuint_at(*data, elt_tag.next);
     let end = elt_size.next + elt_size.val;
     ret {tag: elt_tag.val,
          doc: {data: data, start: elt_size.next, end: end}};
@@ -56,8 +63,8 @@ fn doc_at(data: @[u8], start: uint) -> tagged_doc {
 fn maybe_get_doc(d: doc, tg: uint) -> option<doc> {
     let pos = d.start;
     while pos < d.end {
-        let elt_tag = vint_at(*d.data, pos);
-        let elt_size = vint_at(*d.data, elt_tag.next);
+        let elt_tag = vuint_at(*d.data, pos);
+        let elt_size = vuint_at(*d.data, elt_tag.next);
         pos = elt_size.next + elt_size.val;
         if elt_tag.val == tg {
             ret some::<doc>({data: d.data, start: elt_size.next, end: pos});
@@ -79,8 +86,8 @@ fn get_doc(d: doc, tg: uint) -> doc {
 fn docs(d: doc, it: fn(uint, doc)) {
     let pos = d.start;
     while pos < d.end {
-        let elt_tag = vint_at(*d.data, pos);
-        let elt_size = vint_at(*d.data, elt_tag.next);
+        let elt_tag = vuint_at(*d.data, pos);
+        let elt_size = vuint_at(*d.data, elt_tag.next);
         pos = elt_size.next + elt_size.val;
         it(elt_tag.val, {data: d.data, start: elt_size.next, end: pos});
     }
@@ -89,8 +96,8 @@ fn docs(d: doc, it: fn(uint, doc)) {
 fn tagged_docs(d: doc, tg: uint, it: fn(doc)) {
     let pos = d.start;
     while pos < d.end {
-        let elt_tag = vint_at(*d.data, pos);
-        let elt_size = vint_at(*d.data, elt_tag.next);
+        let elt_tag = vuint_at(*d.data, pos);
+        let elt_size = vuint_at(*d.data, elt_tag.next);
         pos = elt_size.next + elt_size.val;
         if elt_tag.val == tg {
             it({data: d.data, start: elt_size.next, end: pos});
@@ -102,28 +109,36 @@ fn doc_data(d: doc) -> [u8] { ret vec::slice::<u8>(*d.data, d.start, d.end); }
 
 fn doc_str(d: doc) -> str { ret str::from_bytes(doc_data(d)); }
 
-fn be_uint_from_bytes(data: @[u8], start: uint, size: uint) -> uint {
+fn be_u64_from_bytes(data: @[u8], start: uint, size: uint) -> u64 {
     let sz = size;
     assert (sz <= 4u);
-    let val = 0u;
+    let val = 0_u64;
     let pos = start;
     while sz > 0u {
         sz -= 1u;
-        val += (data[pos] as uint) << sz * 8u;
+        val += (data[pos] as u64) << ((sz * 8u) as u64);
         pos += 1u;
     }
     ret val;
 }
 
-fn doc_as_uint(d: doc) -> uint {
-    ret be_uint_from_bytes(d.data, d.start, d.end - d.start);
+fn doc_as_u8(d: doc) -> u8 {
+    assert d.end == d.start + 1u;
+    ret (*d.data)[d.start];
+}
+
+fn doc_as_vu64(d: doc) -> u64 {
+    ret vu64_at(*d.data, d.start).val;
 }
 
+fn doc_as_vuint(d: doc) -> uint {
+    ret vuint_at(*d.data, d.start).val;
+}
 
 // ebml writing
 type writer = {writer: io::writer, mutable size_positions: [uint]};
 
-fn write_sized_vint(w: io::writer, n: u64, size: uint) {
+fn write_sized_vu64(w: io::writer, n: u64, size: uint) {
     let buf: [u8];
     alt size {
       1u { buf = [0x80u8 | (n as u8)]; }
@@ -141,11 +156,11 @@ fn write_sized_vint(w: io::writer, n: u64, size: uint) {
     w.write(buf);
 }
 
-fn write_vint(w: io::writer, n: u64) {
-    if n < 0x7f_u64 { write_sized_vint(w, n, 1u); ret; }
-    if n < 0x4000_u64 { write_sized_vint(w, n, 2u); ret; }
-    if n < 0x200000_u64 { write_sized_vint(w, n, 3u); ret; }
-    if n < 0x10000000_u64 { write_sized_vint(w, n, 4u); ret; }
+fn write_vu64(w: io::writer, n: u64) {
+    if n < 0x7f_u64 { write_sized_vu64(w, n, 1u); ret; }
+    if n < 0x4000_u64 { write_sized_vu64(w, n, 2u); ret; }
+    if n < 0x200000_u64 { write_sized_vu64(w, n, 3u); ret; }
+    if n < 0x10000000_u64 { write_sized_vu64(w, n, 4u); ret; }
     #error("vint to write too big");
     fail;
 }
@@ -158,8 +173,10 @@ fn create_writer(w: io::writer) -> writer {
 
 // TODO: Provide a function to write the standard ebml header.
 fn start_tag(w: writer, tag_id: uint) {
+    #debug["Start tag %u", tag_id];
+
     // Write the enum ID:
-    write_vint(w.writer, tag_id as u64);
+    write_vu64(w.writer, tag_id as u64);
 
     // Write a placeholder four-byte size.
     w.size_positions += [w.writer.tell()];
@@ -171,8 +188,11 @@ fn end_tag(w: writer) {
     let last_size_pos = vec::pop::<uint>(w.size_positions);
     let cur_pos = w.writer.tell();
     w.writer.seek(last_size_pos as int, io::seek_set);
-    write_sized_vint(w.writer, (cur_pos - last_size_pos - 4u) as u64, 4u);
+    let size = (cur_pos - last_size_pos - 4u);
+    write_sized_vu64(w.writer, size as u64, 4u);
     w.writer.seek(cur_pos as int, io::seek_set);
+
+    #debug["End tag (size = %u)", size];
 }
 
 impl writer_util for writer {
@@ -182,20 +202,26 @@ impl writer_util for writer {
         end_tag(self);
     }
 
-    fn wr_u64(id: u64) {
-        write_vint(self.writer, id);
+    fn wr_vu64(id: u64) {
+        #debug["Write u64 0x%02x%02x",
+               (id >> 32u64) as uint,
+               (id & 0xFFFFFFFFu64) as uint];
+        write_vu64(self.writer, id);
     }
 
-    fn wr_uint(id: uint) {
-        self.wr_u64(id as u64);
+    fn wr_vuint(id: uint) {
+        #debug["Write uint: %u", id];
+        write_vu64(self.writer, id as u64);
     }
 
     fn wr_bytes(b: [u8]) {
+        #debug["Write %u bytes", vec::len(b)];
         self.writer.write(b);
     }
 
     fn wr_str(s: str) {
-        self.wr_bytes(str::bytes(s));
+        #debug["Write str: %?", s];
+        self.writer.write(str::bytes(s));
     }
 }