Auto merge of #1177 - snewt:feat/quickcheck-as-bin, r=fitzgen

Quickchecking crate CLI

Prior to this commit the quickchecking crate used for generating proprty tests for bindgen was a [lib] target and had configurations that required commenting/uncommenting code to enable/disable. This meant it was inconvienent/prohibitive to configure the property tests on a per-run basis. This commit reorganizes the `quickchecking` crate to provide both [lib] and [[bin]] targets in order to expose those configurations through a CLI.

The configurations that are exposed through the [[bin]] target's CLI are:
* Count/number of tests to run.
* Directory to provide fuzzed headers
* Generation range corresponding to the range quickcheck uses to
* generate arbitrary.

__Usage from the__ `tests/quickchecking` __directory__
```bash
quickchecking 0.2.0
Bindgen property tests with quickcheck. Generate random valid C code and pass it to the csmith/predicate.py script

USAGE:
    quickchecking [OPTIONS]

FLAGS:
    -h, --help       Prints help information
    -V, --version    Prints version information

OPTIONS:
    -c, --count <COUNT>    Count / number of tests to run. Running a fuzzed header through the predicate.py script can
                           take a long time, especially if the generation range is large. Increase this number if you're
                           willing to wait a while. [default: 2]
    -p, --path <PATH>      Optional. Preserve generated headers for inspection, provide directory path for header
                           output. [default: None]
    -r, --range <RANGE>    Sets the range quickcheck uses during generation. Corresponds to things like arbitrary usize
                           and arbitrary vector length. This number doesn't have to grow much for that execution time to
                           increase significantly. [default: 32]
```

Because the actual work of running the property tests moved to the [[bin]]
target, rather than duplicate that code in the `quickchecking` crate's tests
directory, some actual (very basic) tests for the `quickchecking` crate were
added.

*Note: I'm not attached to any of the option flags, if there are better characters/words for any of the options I've exposed I'll be happy to revise!

Also, I'm not sure how palatable the "global singleton" is for managing context (output path) across tests in the `lib.rs` file. Very open to suggestions on how to manage that if it's not an acceptable approach.

Thanks for taking a look, looking forward to feedback!

Closes #1168

r? @fitzgen

Author: bors-servo

ci/script.sh

@@ -41,7 +41,7 @@ case "$BINDGEN_JOB" in
     "quickchecking")
         cd ./tests/quickchecking
         # TODO: Actually run quickchecks once `bindgen` is reliable enough.
-        cargo check
+        cargo test
         ;;
     *)
         echo "Error! Unknown \$BINDGEN_JOB: '$BINDGEN_JOB'"

tests/quickchecking/Cargo.toml

@@ -2,9 +2,19 @@
 name = "quickchecking"
 description = "Bindgen property tests with quickcheck. Generate random valid C code and pass it to the csmith/predicate.py script"
 version = "0.1.0"
-authors = ["Shea Newton <[email protected]>"]
+authors = ["Shea Newton <[email protected]>"]
+
+[lib]
+name = "quickchecking"
+path = "src/lib.rs"
+
+[[bin]]
+name = "quickchecking"
+path = "src/bin.rs"
 
 [dependencies]
+clap = "2.28"
+lazy_static = "1.0"
 quickcheck = "0.4"
-tempdir = "0.3"
 rand = "0.3"
+tempdir = "0.3"

tests/quickchecking/src/bin.rs

@@ -0,0 +1,110 @@
+//! An application to run property tests for `bindgen` with _fuzzed_ C headers 
+//! using `quickcheck` 
+//!
+//! ## Usage
+//!
+//! Print help
+//! ```bash
+//! $ cargo run --bin=quickchecking -- -h
+//! ```
+//!
+//! Run with default values
+//! ```bash
+//! $ cargo run --bin=quickchecking
+//! ```
+//!
+#![deny(missing_docs)]
+extern crate clap;
+extern crate quickchecking;
+
+use clap::{App, Arg};
+use std::path::Path;
+
+// Validate CLI argument input for generation range.
+fn validate_generate_range(v: String) -> Result<(), String> {
+    match v.parse::<usize>() {
+        Ok(_) => Ok(()),
+        Err(_) => Err(String::from(
+            "Generate range could not be converted to a usize.",
+        )),
+    }
+}
+
+// Validate CLI argument input for tests count.
+fn validate_tests_count(v: String) -> Result<(), String> {
+    match v.parse::<usize>() {
+        Ok(_) => Ok(()),
+        Err(_) => Err(String::from(
+            "Tests count could not be converted to a usize.",
+        )),
+    }
+}
+
+// Validate CLI argument input for fuzzed headers output path.
+fn validate_path(v: String) -> Result<(), String> {
+    match Path::new(&v).is_dir() {
+        true => Ok(()),
+        false => Err(String::from("Provided directory path does not exist.")),
+    }
+}
+
+fn main() {
+    let matches = App::new("quickchecking")
+        .version("0.2.0")
+        .about(
+            "Bindgen property tests with quickcheck. \
+             Generate random valid C code and pass it to the \
+             csmith/predicate.py script",
+        )
+        .arg(
+            Arg::with_name("path")
+                .short("p")
+                .long("path")
+                .value_name("PATH")
+                .help(
+                    "Optional. Preserve generated headers for inspection, \
+                     provide directory path for header output. [default: None] ",
+                )
+                .takes_value(true)
+                .validator(validate_path),
+        )
+        .arg(
+            Arg::with_name("range")
+                .short("r")
+                .long("range")
+                .value_name("RANGE")
+                .help(
+                    "Sets the range quickcheck uses during generation. \
+                     Corresponds to things like arbitrary usize and \
+                     arbitrary vector length. This number doesn't have \
+                     to grow much for that execution time to increase \
+                     significantly.",
+                )
+                .takes_value(true)
+                .default_value("32")
+                .validator(validate_generate_range),
+        )
+        .arg(
+            Arg::with_name("count")
+                .short("c")
+                .long("count")
+                .value_name("COUNT")
+                .help(
+                    "Count / number of tests to run. Running a fuzzed \
+                     header through the predicate.py script can take a \
+                     long time, especially if the generation range is \
+                     large. Increase this number if you're willing to \
+                     wait a while.",
+                )
+                .takes_value(true)
+                .default_value("2")
+                .validator(validate_tests_count),
+        )
+        .get_matches();
+
+    let output_path: Option<&str> = matches.value_of("path");
+    let generate_range: usize = matches.value_of("range").unwrap().parse::<usize>().unwrap();
+    let tests: usize = matches.value_of("count").unwrap().parse::<usize>().unwrap();
+
+    quickchecking::test_bindgen(generate_range, tests, output_path)
+}

tests/quickchecking/src/lib.rs

@@ -20,9 +20,107 @@
 //! ```
 //!
 #![deny(missing_docs)]
+#[macro_use]
+extern crate lazy_static;
 extern crate quickcheck;
 extern crate rand;
 extern crate tempdir;
 
+use std::sync::Mutex;
+use quickcheck::{QuickCheck, StdGen, TestResult};
+use std::fs::File;
+use std::io::Write;
+use tempdir::TempDir;
+use std::process::{Command, Output};
+use std::path::PathBuf;
+use std::error::Error;
+use rand::thread_rng;
+
 /// Contains definitions of and impls for types used to fuzz C declarations.
 pub mod fuzzers;
+
+// Global singleton, manages context across tests. For now that context is
+// only the output_path for inspecting fuzzed headers (if specified).
+struct Context {
+    output_path: Option<String>,
+}
+
+// Initialize global context.
+lazy_static! {
+    static ref CONTEXT: Mutex<Context> = Mutex::new(Context { output_path: None });
+}
+
+// Passes fuzzed header to the `csmith-fuzzing/predicate.py` script, returns
+// output of the associated command.
+fn run_predicate_script(header: fuzzers::HeaderC) -> Result<Output, Box<Error>> {
+    let dir = TempDir::new("bindgen_prop")?;
+    let header_path = dir.path().join("prop_test.h");
+
+    let mut header_file = File::create(&header_path)?;
+    header_file.write_all(header.to_string().as_bytes())?;
+    header_file.sync_all()?;
+
+    let header_path_string;
+    match header_path.into_os_string().into_string() {
+        Ok(s) => header_path_string = s,
+        Err(_) => return Err(From::from("error converting path into String")),
+    }
+
+    let mut predicate_script_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    predicate_script_path.push("../../csmith-fuzzing/predicate.py");
+
+    let predicate_script_path_string;
+    match predicate_script_path.into_os_string().into_string() {
+        Ok(s) => predicate_script_path_string = s,
+        Err(_) => return Err(From::from("error converting path into String")),
+    }
+
+    // Copy generated temp files to output_path directory for inspection.
+    // If `None`, output path not specified, don't copy.
+    match CONTEXT.lock().unwrap().output_path {
+        Some(ref path) => {
+            Command::new("cp")
+                .arg("-a")
+                .arg(&dir.path().to_str().unwrap())
+                .arg(&path)
+                .output()?;
+        }
+        None => {}
+    }
+
+    Ok(Command::new(&predicate_script_path_string)
+        .arg(&header_path_string)
+        .output()?)
+}
+
+// Generatable property. Pass generated headers off to run through the
+// `csmith-fuzzing/predicate.py` script. Success is measured by the success
+// status of that command.
+fn bindgen_prop(header: fuzzers::HeaderC) -> TestResult {
+    match run_predicate_script(header) {
+        Ok(o) => return TestResult::from_bool(o.status.success()),
+        Err(e) => {
+            println!("{:?}", e);
+            return TestResult::from_bool(false);
+        }
+    }
+}
+
+/// Instantiate a Quickcheck object and use it to run property tests using
+/// fuzzed C headers generated with types defined in the `fuzzers` module.
+/// Success/Failure is dictated by the result of passing the fuzzed headers
+/// to the `csmith-fuzzing/predicate.py` script.
+pub fn test_bindgen(generate_range: usize, tests: usize, output_path: Option<&str>) {
+    match output_path {
+        Some(path) => {
+            CONTEXT.lock().unwrap().output_path =
+                Some(String::from(PathBuf::from(path).to_str().unwrap()));
+        }
+        None => {} // Path not specified, don't provide output.
+    }
+
+    QuickCheck::new()
+        .tests(tests)
+        .gen(StdGen::new(thread_rng(), generate_range))
+        .quickcheck(bindgen_prop as fn(fuzzers::HeaderC) -> TestResult)
+}