Quickchecking crate CLI

ci/script.sh

@@ -41,7 +41,7 @@ case "$BINDGEN_JOB" in
     "quickchecking")
         cd ./tests/quickchecking
         # TODO: Actually run quickchecks once `bindgen` is reliable enough.
-        cargo check
+        cargo test
         ;;
     *)
         echo "Error! Unknown \$BINDGEN_JOB: '$BINDGEN_JOB'"

tests/quickchecking/Cargo.toml

@@ -2,9 +2,19 @@
 name = "quickchecking"
 description = "Bindgen property tests with quickcheck. Generate random valid C code and pass it to the csmith/predicate.py script"
 version = "0.1.0"
-authors = ["Shea Newton <[email protected]>"]
+authors = ["Shea Newton <[email protected]>"]
+
+[lib]
+name = "quickchecking"
+path = "src/lib.rs"
+
+[[bin]]
+name = "quickchecking"
+path = "src/bin.rs"
 
 [dependencies]
+clap = "2.28"
+lazy_static = "1.0"
 quickcheck = "0.4"
-tempdir = "0.3"
 rand = "0.3"
+tempdir = "0.3"

tests/quickchecking/src/bin.rs

@@ -0,0 +1,110 @@
+//! An application to run property tests for `bindgen` with _fuzzed_ C headers 
+//! using `quickcheck` 
+//!
+//! ## Usage
+//!
+//! Print help
+//! ```bash
+//! $ cargo run --bin=quickchecking -- -h
+//! ```
+//!
+//! Run with default values
+//! ```bash
+//! $ cargo run --bin=quickchecking
+//! ```
+//!
+#![deny(missing_docs)]
+extern crate clap;
+extern crate quickchecking;
+
+use clap::{App, Arg};
+use std::path::Path;
+
+// Validate CLI argument input for generation range.
+fn validate_generate_range(v: String) -> Result<(), String> {
+    match v.parse::<usize>() {
+        Ok(_) => Ok(()),
+        Err(_) => Err(String::from(
+            "Generate range could not be converted to a usize.",
+        )),
+    }
+}
+
+// Validate CLI argument input for tests count.
+fn validate_tests_count(v: String) -> Result<(), String> {
+    match v.parse::<usize>() {
+        Ok(_) => Ok(()),
+        Err(_) => Err(String::from(
+            "Tests count could not be converted to a usize.",
+        )),
+    }
+}
+
+// Validate CLI argument input for fuzzed headers output path.
+fn validate_path(v: String) -> Result<(), String> {
+    match Path::new(&v).is_dir() {
+        true => Ok(()),
+        false => Err(String::from("Provided directory path does not exist.")),
+    }
+}
+
+fn main() {
+    let matches = App::new("quickchecking")
+        .version("0.2.0")
+        .about(
+            "Bindgen property tests with quickcheck. \
+             Generate random valid C code and pass it to the \
+             csmith/predicate.py script",
+        )
+        .arg(
+            Arg::with_name("path")
+                .short("p")
+                .long("path")
+                .value_name("PATH")
+                .help(
+                    "Optional. Preserve generated headers for inspection, \
+                     provide directory path for header output. [default: None] ",
+                )
+                .takes_value(true)
+                .validator(validate_path),
+        )
+        .arg(
+            Arg::with_name("range")
+                .short("r")
+                .long("range")
+                .value_name("RANGE")
+                .help(
+                    "Sets the range quickcheck uses during generation. \
+                     Corresponds to things like arbitrary usize and \
+                     arbitrary vector length. This number doesn't have \
+                     to grow much for that execution time to increase \
+                     significantly.",
+                )
+                .takes_value(true)
+                .default_value("32")
+                .validator(validate_generate_range),
+        )
+        .arg(
+            Arg::with_name("count")
+                .short("c")
+                .long("count")
+                .value_name("COUNT")
+                .help(
+                    "Count / number of tests to run. Running a fuzzed \
+                     header through the predicate.py script can take a \
+                     long time, especially if the generation range is \
+                     large. Increase this number if you're willing to \
+                     wait a while.",
+                )
+                .takes_value(true)
+                .default_value("2")
+                .validator(validate_tests_count),
+        )
+        .get_matches();
+
+    let output_path: Option<&str> = matches.value_of("path");
+    let generate_range: usize = matches.value_of("range").unwrap().parse::<usize>().unwrap();
+    let tests: usize = matches.value_of("count").unwrap().parse::<usize>().unwrap();
+
+    quickchecking::test_bindgen(generate_range, tests, output_path)
+}

tests/quickchecking/src/lib.rs

@@ -20,9 +20,107 @@
 //! ```
 //!
 #![deny(missing_docs)]
+#[macro_use]
+extern crate lazy_static;
 extern crate quickcheck;
 extern crate rand;
 extern crate tempdir;
 
+use std::sync::Mutex;
+use quickcheck::{QuickCheck, StdGen, TestResult};
+use std::fs::File;
+use std::io::Write;
+use tempdir::TempDir;
+use std::process::{Command, Output};
+use std::path::PathBuf;
+use std::error::Error;
+use rand::thread_rng;
+
 /// Contains definitions of and impls for types used to fuzz C declarations.
 pub mod fuzzers;
+
+// Global singleton, manages context across tests. For now that context is
+// only the output_path for inspecting fuzzed headers (if specified).
+struct Context {
+    output_path: Option<String>,
+}
+
+// Initialize global context.
+lazy_static! {
+    static ref CONTEXT: Mutex<Context> = Mutex::new(Context { output_path: None });
+}
+
+// Passes fuzzed header to the `csmith-fuzzing/predicate.py` script, returns
+// output of the associated command.
+fn run_predicate_script(header: fuzzers::HeaderC) -> Result<Output, Box<Error>> {
+    let dir = TempDir::new("bindgen_prop")?;
+    let header_path = dir.path().join("prop_test.h");
+
+    let mut header_file = File::create(&header_path)?;
+    header_file.write_all(header.to_string().as_bytes())?;
+    header_file.sync_all()?;
+
+    let header_path_string;
+    match header_path.into_os_string().into_string() {
+        Ok(s) => header_path_string = s,
+        Err(_) => return Err(From::from("error converting path into String")),
+    }
+
+    let mut predicate_script_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    predicate_script_path.push("../../csmith-fuzzing/predicate.py");
+
+    let predicate_script_path_string;
+    match predicate_script_path.into_os_string().into_string() {
+        Ok(s) => predicate_script_path_string = s,
+        Err(_) => return Err(From::from("error converting path into String")),
+    }
+
+    // Copy generated temp files to output_path directory for inspection.
+    // If `None`, output path not specified, don't copy.
+    match CONTEXT.lock().unwrap().output_path {
+        Some(ref path) => {
+            Command::new("cp")
+                .arg("-a")
+                .arg(&dir.path().to_str().unwrap())
+                .arg(&path)
+                .output()?;
+        }
+        None => {}
+    }
+
+    Ok(Command::new(&predicate_script_path_string)
+        .arg(&header_path_string)
+        .output()?)
+}
+
+// Generatable property. Pass generated headers off to run through the
+// `csmith-fuzzing/predicate.py` script. Success is measured by the success
+// status of that command.
+fn bindgen_prop(header: fuzzers::HeaderC) -> TestResult {
+    match run_predicate_script(header) {
+        Ok(o) => return TestResult::from_bool(o.status.success()),
+        Err(e) => {
+            println!("{:?}", e);
+            return TestResult::from_bool(false);
+        }
+    }
+}
+
+/// Instantiate a Quickcheck object and use it to run property tests using
+/// fuzzed C headers generated with types defined in the `fuzzers` module.
+/// Success/Failure is dictated by the result of passing the fuzzed headers
+/// to the `csmith-fuzzing/predicate.py` script.
+pub fn test_bindgen(generate_range: usize, tests: usize, output_path: Option<&str>) {
+    match output_path {
+        Some(path) => {
+            CONTEXT.lock().unwrap().output_path =
+                Some(String::from(PathBuf::from(path).to_str().unwrap()));
+        }
+        None => {} // Path not specified, don't provide output.
+    }
+
+    QuickCheck::new()
+        .tests(tests)
+        .gen(StdGen::new(thread_rng(), generate_range))
+        .quickcheck(bindgen_prop as fn(fuzzers::HeaderC) -> TestResult)
+}