Minimize IRep Serialization memory consumption (rust-lang#621)

* Use serde to serialize irep to reduce mem consumption

This change allow us to serialize the irep without having to buffer the
entire json in memory.

I'm not removing the old code yet because this new logic is slower and
still need some prunning.

* Add a BufWriter to speed up serialization.

* Address PR comments

 - Add more test coverage.
 - Added utility function.

Author: celinval

Cargo.lock

@@ -501,6 +501,8 @@ dependencies = [
  "rustc_session",
  "rustc_span",
  "rustc_target",
+ "serde",
+ "serde_test",
  "smallvec",
  "snap",
  "tracing",
@@ -3873,6 +3875,8 @@ dependencies = [
  "rustc_session",
  "rustc_span",
  "rustc_target",
+ "serde",
+ "serde_json",
  "smallvec",
  "snap",
  "tracing",
@@ -5006,6 +5010,15 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "serde_test"
+version = "1.0.130"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d82178225dbdeae2d5d190e8649287db6a3a32c6d24da22ae3146325aa353e4c"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "sha-1"
 version = "0.8.2"

compiler/cbmc/Cargo.toml

@@ -16,6 +16,7 @@ cstr = "0.2"
 libc = "0.2"
 measureme = "9.1.0"
 num = "0.4.0"
+serde = {version = "1", features = ["derive"]}
 snap = "1"
 tracing = "0.1"
 rustc_middle = { path = "../rustc_middle" }
@@ -36,3 +37,6 @@ rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
 rustc_ast = { path = "../rustc_ast" }
 rustc_span = { path = "../rustc_span" }
+
+[dev-dependencies]
+serde_test = "1"

compiler/cbmc/src/irep/mod.rs

@@ -16,6 +16,7 @@
 
 mod irep;
 mod irep_id;
+pub mod serialize;
 mod symbol;
 mod symbol_table;
 mod to_irep;

compiler/cbmc/src/irep/serialize.rs

@@ -0,0 +1,244 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+//! This crate implements irep serialization using serde Serializer.
+use crate::irep::{Irep, IrepId, Symbol, SymbolTable};
+use serde::ser::{SerializeMap, Serializer};
+use serde::Serialize;
+
+impl Serialize for Irep {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut obj = serializer.serialize_map(None)?;
+        obj.serialize_entry("id", &self.id)?;
+        if !self.sub.is_empty() {
+            obj.serialize_entry("sub", &self.sub)?;
+        }
+        if !self.named_sub.is_empty() {
+            obj.serialize_entry("namedSub", &self.named_sub)?;
+        }
+        obj.end()
+    }
+}
+
+impl Serialize for IrepId {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        self.to_string().serialize(serializer)
+    }
+}
+
+impl Serialize for SymbolTable {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut obj = serializer.serialize_map(None)?;
+        obj.serialize_entry("symbolTable", &self.symbol_table)?;
+        obj.end()
+    }
+}
+
+impl Serialize for Symbol {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let mut obj = serializer.serialize_map(None)?;
+        obj.serialize_entry(&IrepId::Type.to_string(), &self.typ)?;
+        obj.serialize_entry(&IrepId::Value.to_string(), &self.value)?;
+        obj.serialize_entry("location", &self.location)?;
+        obj.serialize_entry(&IrepId::Name.to_string(), &self.name)?;
+        obj.serialize_entry(&IrepId::Module.to_string(), &self.module)?;
+        obj.serialize_entry("baseName", &self.base_name)?;
+        obj.serialize_entry("prettyName", &self.pretty_name)?;
+        obj.serialize_entry(&IrepId::Mode.to_string(), &self.mode)?;
+        obj.serialize_entry("isType", &self.is_type)?;
+        obj.serialize_entry("isMacro", &self.is_macro)?;
+        obj.serialize_entry("isExported", &self.is_exported)?;
+        obj.serialize_entry("isInput", &self.is_input)?;
+        obj.serialize_entry("isOutput", &self.is_output)?;
+        obj.serialize_entry("isStateVar", &self.is_state_var)?;
+        obj.serialize_entry("isProperty", &self.is_property)?;
+        obj.serialize_entry("isStaticLifetime", &self.is_static_lifetime)?;
+        obj.serialize_entry("isThreadLocal", &self.is_thread_local)?;
+        obj.serialize_entry("isLvalue", &self.is_lvalue)?;
+        obj.serialize_entry("isFileLocal", &self.is_file_local)?;
+        obj.serialize_entry("isExtern", &self.is_extern)?;
+        obj.serialize_entry("isVolatile", &self.is_volatile)?;
+        obj.serialize_entry("isParameter", &self.is_parameter)?;
+        obj.serialize_entry("isAuxiliary", &self.is_auxiliary)?;
+        obj.serialize_entry("isWeak", &self.is_weak)?;
+
+        obj.end()
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use serde_test::{assert_ser_tokens, Token};
+    #[test]
+    fn serialize_irep() {
+        let irep = Irep::empty();
+        assert_ser_tokens(
+            &irep,
+            &[Token::Map { len: None }, Token::String("id"), Token::String("empty"), Token::MapEnd],
+        );
+    }
+
+    #[test]
+    fn serialize_sym_table() {
+        let mut sym_table = SymbolTable::new();
+        let symbol = Symbol {
+            typ: Irep::empty(),
+            value: Irep::empty(),
+            location: Irep::empty(),
+            name: String::from("my_name"),
+            module: String::new(),
+            base_name: String::new(),
+            pretty_name: String::new(),
+            mode: String::new(),
+            is_type: false,
+            is_macro: false,
+            is_exported: false,
+            is_input: false,
+            is_output: false,
+            is_state_var: false,
+            is_property: false,
+
+            // ansi-C properties
+            is_static_lifetime: false,
+            is_thread_local: false,
+            is_lvalue: false,
+            is_file_local: false,
+            is_extern: false,
+            is_volatile: false,
+            is_parameter: false,
+            is_auxiliary: false,
+            is_weak: false,
+        };
+        sym_table.insert(symbol.clone());
+        assert_ser_tokens(
+            &sym_table,
+            &[
+                Token::Map { len: None },
+                Token::String("symbolTable"),
+                Token::Map { len: Some(1) },
+                Token::String("my_name"),
+                // symbol start
+                Token::Map { len: None },
+                // type irep
+                Token::String("type"),
+                Token::Map { len: None },
+                Token::String("id"),
+                Token::String("empty"),
+                Token::MapEnd,
+                // value irep
+                Token::String("value"),
+                Token::Map { len: None },
+                Token::String("id"),
+                Token::String("empty"),
+                Token::MapEnd,
+                // value locaton
+                Token::String("location"),
+                Token::Map { len: None },
+                Token::String("id"),
+                Token::String("empty"),
+                Token::MapEnd,
+                Token::String("name"),
+                Token::String("my_name"),
+                Token::String("module"),
+                Token::String(""),
+                Token::String("baseName"),
+                Token::String(""),
+                Token::String("prettyName"),
+                Token::String(""),
+                Token::String("mode"),
+                Token::String(""),
+                Token::String("isType"),
+                Token::Bool(false),
+                Token::String("isMacro"),
+                Token::Bool(false),
+                Token::String("isExported"),
+                Token::Bool(false),
+                Token::String("isInput"),
+                Token::Bool(false),
+                Token::String("isOutput"),
+                Token::Bool(false),
+                Token::String("isStateVar"),
+                Token::Bool(false),
+                Token::String("isProperty"),
+                Token::Bool(false),
+                Token::String("isStaticLifetime"),
+                Token::Bool(false),
+                Token::String("isThreadLocal"),
+                Token::Bool(false),
+                Token::String("isLvalue"),
+                Token::Bool(false),
+                Token::String("isFileLocal"),
+                Token::Bool(false),
+                Token::String("isExtern"),
+                Token::Bool(false),
+                Token::String("isVolatile"),
+                Token::Bool(false),
+                Token::String("isParameter"),
+                Token::Bool(false),
+                Token::String("isAuxiliary"),
+                Token::Bool(false),
+                Token::String("isWeak"),
+                Token::Bool(false),
+                Token::MapEnd,
+                Token::MapEnd,
+                Token::MapEnd,
+            ],
+        );
+    }
+
+    #[test]
+    fn serialize_irep_sub() {
+        let empty_irep = Irep::empty();
+        let one_irep = Irep::one();
+        let sub_irep = Irep::just_sub(vec![empty_irep.clone(), one_irep]);
+        let top_irep = Irep::just_sub(vec![sub_irep, empty_irep]);
+        assert_ser_tokens(
+            &top_irep,
+            &[
+                // top_irep
+                Token::Map { len: None },
+                Token::String("id"),
+                Token::String(""),
+                Token::String("sub"),
+                Token::Seq { len: Some(2) },
+                // sub_irep
+                Token::Map { len: None },
+                Token::String("id"),
+                Token::String(""),
+                Token::String("sub"),
+                Token::Seq { len: Some(2) },
+                // empty_irep
+                Token::Map { len: None },
+                Token::String("id"),
+                Token::String("empty"),
+                Token::MapEnd,
+                // one_irep
+                Token::Map { len: None },
+                Token::String("id"),
+                Token::String("1"),
+                Token::MapEnd,
+                Token::SeqEnd,
+                Token::MapEnd,
+                // empty_irep
+                Token::Map { len: None },
+                Token::String("id"),
+                Token::String("empty"),
+                Token::MapEnd,
+                Token::SeqEnd,
+                Token::MapEnd,
+            ],
+        );
+    }
+}

compiler/cbmc/src/lib.rs

@@ -30,4 +30,5 @@ pub mod goto_program;
 pub mod irep;
 mod machine_model;
 pub mod utils;
+pub use irep::serialize;
 pub use machine_model::{MachineModel, RoundingMode};

compiler/rustc_codegen_rmc/Cargo.toml

@@ -17,6 +17,8 @@ cstr = "0.2"
 libc = "0.2"
 measureme = "9.1.0"
 num = "0.4.0"
+serde = "1"
+serde_json = "1"
 snap = "1"
 tracing = "0.1"
 rustc_middle = { path = "../rustc_middle" }

compiler/rustc_codegen_rmc/src/compiler_interface.rs

@@ -16,12 +16,13 @@ use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, TyCtxt};
-use rustc_serialize::json::ToJson;
 use rustc_session::config::{OutputFilenames, OutputType};
 use rustc_session::cstore::MetadataLoaderDyn;
 use rustc_session::Session;
 use std::collections::BTreeMap;
+use std::io::BufWriter;
 use std::iter::FromIterator;
+use std::path::PathBuf;
 use tracing::{debug, warn};
 
 // #[derive(RustcEncodable, RustcDecodable)]
@@ -150,8 +151,6 @@ impl CodegenBackend for GotocCodegenBackend {
         codegen_results: Box<dyn Any>,
         outputs: &OutputFilenames,
     ) -> Result<(), ErrorReported> {
-        use std::io::Write;
-
         let result = codegen_results
             .downcast::<GotocCodegenResult>()
             .expect("in link: codegen_results is not a GotocCodegenResult");
@@ -160,18 +159,21 @@ impl CodegenBackend for GotocCodegenBackend {
         if !sess.opts.debugging_opts.no_codegen && sess.opts.output_types.should_codegen() {
             // "path.o"
             let base_filename = outputs.path(OutputType::Object);
-
-            let symtab_filename = base_filename.with_extension("symtab.json");
-            debug!("output to {:?}", symtab_filename);
-            let mut out_file = ::std::fs::File::create(&symtab_filename).unwrap();
-            write!(out_file, "{}", result.symtab.to_irep().to_json().pretty().to_string()).unwrap();
-
-            let type_map_filename = base_filename.with_extension("type_map.json");
-            debug!("type_map to {:?}", type_map_filename);
-            let mut out_file = ::std::fs::File::create(&type_map_filename).unwrap();
-            write!(out_file, "{}", result.type_map.to_json().pretty().to_string()).unwrap();
+            write_file(&base_filename, "symtab.json", &result.symtab.to_irep());
+            write_file(&base_filename, "type_map.json", &result.type_map);
         }
 
         Ok(())
     }
 }
+
+fn write_file<T>(base_filename: &PathBuf, extension: &str, source: &T)
+where
+    T: serde::Serialize,
+{
+    let filename = base_filename.with_extension(extension);
+    debug!("output to {:?}", filename);
+    let out_file = ::std::fs::File::create(&filename).unwrap();
+    let writer = BufWriter::new(out_file);
+    serde_json::to_writer(writer, &source).unwrap();
+}