Allow asserting expectations across different libclang versions

Rather than having a tests that we only run if libclang >= 3.9, this makes the
test suite dynamically detect when we have different expectations for different
libclang versions. It does this by adding `tests/expectations/tests/libclang-$VERSION`
directories, and `testing_only_libclang_$VERSION` features that are consulted
when the usual expectation file does not exist.

Fixes rust-lang#697

Author: fitzgen

.travis.yml

@@ -17,16 +17,15 @@ env:
   global:
     - CARGO_TARGET_DIR=/tmp/bindgen
   matrix:
-    - LLVM_VERSION=3.7.1 BINDGEN_FEATURES=testing_only_llvm_stable
-    - LLVM_VERSION=3.8.1 BINDGEN_FEATURES=testing_only_llvm_stable
-    - LLVM_VERSION=3.9.0 BINDGEN_FEATURES=
-    - LLVM_VERSION=4.0.0 BINDGEN_FEATURES=
+    - LLVM_VERSION=3.7.1 BINDGEN_FEATURES=testing_only_libclang_3_8
+    - LLVM_VERSION=3.8.1 BINDGEN_FEATURES=testing_only_libclang_3_8
+    - LLVM_VERSION=3.9.0 BINDGEN_FEATURES=testing_only_libclang_3_9
+    - LLVM_VERSION=4.0.0 BINDGEN_FEATURES=testing_only_libclang_4
 
 matrix:
   fast_finish: true
   allow_failures:
-    - env: LLVM_VERSION=4.0.0 BINDGEN_FEATURES=
-    - env: LLVM_VERSION=3.7.1 BINDGEN_FEATURES=testing_only_llvm_stable
+    - env: LLVM_VERSION=3.7.1 BINDGEN_FEATURES=testing_only_libclang_3_8
 
 cache:
   directories:

CONTRIBUTING.md

@@ -14,7 +14,9 @@ out to us in a GitHub issue, or stop by
 - [Testing](#testing)
   - [Overview](#overview)
   - [Running All Tests](#running-all-tests)
+  - [Running a Single Test](#running-a-single-test)
   - [Authoring New Tests](#authoring-new-tests)
+  - [Test Expectations and `libclang` Versions](#test-expectations-and-libclang-versions)
 - [Automatic code formatting](#automatic-code-formatting)
 - [Generating Graphviz Dot Files](#generating-graphviz-dot-files)
 - [Debug Logging](#debug-logging)
@@ -91,9 +93,17 @@ Run `cargo test` to compare generated Rust bindings to the expectations.
 ### Running All Tests
 
 ```
-$ cargo test [--all-features]
+$ cargo test --features testing_only_libclang_$VERSION
 ```
 
+Where `$VERSION` is one of:
+
+* `4`
+* `3_9`
+* `3_8`
+
+depending on which version of `libclang` you have installed.
+
 ### Running a Single Test
 
 To generate bindings for a single test header, compile the bindings, and run the
@@ -124,6 +134,25 @@ Then verify the new Rust bindings compile and pass some basic tests:
 $ cargo test -p tests_expectations
 ```
 
+### Test Expectations and `libclang` Versions
+
+If a test generates different bindings across different `libclang` versions (for
+example, because we take advantage of better/newer APIs when possible), then you
+can add multiple test expectations, one for each supported `libclang`
+version. Instead of having a single `tests/expectations/tests/my_test.rs` file,
+add each of:
+
+* `tests/expectations/tests/libclang-4/my_test.rs`
+* `tests/expectations/tests/libclang-3.9/my_test.rs`
+* `tests/expectations/tests/libclang-3.8/my_test.rs`
+
+If you need to update the test expectations for a test file that generates
+different bindings for different `libclang` versions, you *don't* need to have
+many version of `libclang` installed locally. Just make a work-in-progress pull
+request, and then when Travis CI fails, it will log a diff of the
+expectations. Use the diff to patch the appropriate expectation file locally and
+then update your pull request.
+
 ## Automatic code formatting
 
 We use [`rustfmt`](https://github.com/rust-lang-nursery/rustfmt) to enforce a

Cargo.toml

@@ -77,4 +77,6 @@ static = []
 # on bindgen!
 testing_only_docs = []
 testing_only_extra_assertions = []
-testing_only_llvm_stable = []
+testing_only_libclang_4 = []
+testing_only_libclang_3_9 = []
+testing_only_libclang_3_8 = []

bindgen-integration/Cargo.toml

@@ -11,4 +11,6 @@ bindgen = { path = ".." }
 gcc = "0.3"
 
 [features]
-testing_only_llvm_stable = ["bindgen/testing_only_llvm_stable"]
+testing_only_libclang_4 = ["bindgen/testing_only_libclang_4"]
+testing_only_libclang_3_9 = ["bindgen/testing_only_libclang_3_9"]
+testing_only_libclang_3_8 = ["bindgen/testing_only_libclang_3_8"]

src/main.rs

@@ -31,9 +31,12 @@ pub fn main() {
     let bind_args: Vec<_> = env::args().collect();
 
     let version = clang_version();
-    let expected_version = if cfg!(feature = "testing_only_llvm_stable") {
+    let expected_version = if cfg!(feature = "testing_only_libclang_4") {
+        (4, 0)
+    } else if cfg!(feature = "testing_only_libclang_3_8") {
         (3, 8)
     } else {
+        // Default to 3.9.
         (3, 9)
     };
 

tests/expectations/tests/libclang-3.8/auto.rs

@@ -14,7 +14,9 @@ extern crate bindgen;
 #[test]
 #[cfg(not(any(debug_assertions,
               feature = "testing_only_extra_assertions",
-              feature = "testing_only_llvm_stable")))]
+              feature = "testing_only_libclang_3_8")))]
+#[cfg(any(feature = "testing_only_libclang_3_9",
+          feature = "testing_only_libclang_4"))]
 fn sanity_check_can_generate_stylo_bindings() {
     use std::time::Instant;
 

tests/expectations/tests/libclang-3.8/call-conv-field.rs

@@ -26,6 +26,30 @@ fn compare_generated_header(header: &PathBuf,
     expected.push(file_name);
     expected.set_extension("rs");
 
+    // If the expectation file doesn't exist, see if we have different test
+    // expectations for different libclang versions.
+    if !expected.is_file() {
+        let file_name = expected.file_name().unwrap().to_owned();
+        expected.pop();
+
+        if cfg!(feature = "testing_only_libclang_4") {
+            expected.push("libclang-4");
+        } else if cfg!(feature = "testing_only_libclang_3_9") {
+            expected.push("libclang-3.9");
+        } else if cfg!(feature = "testing_only_libclang_3_8") {
+            expected.push("libclang-3.8");
+        }
+
+        expected.push(file_name);
+
+        if !expected.is_file() {
+            panic!("missing test expectation file and/or 'testing_only_libclang_$VERSION' \
+                    feature for header '{}'; looking for expectation file at '{}'",
+                   header.display(),
+                   expected.display());
+        }
+    }
+
     // We skip the generate() error here so we get a full diff below
     let output = match builder.generate() {
         Ok(bindings) => bindings.to_string(),
@@ -87,9 +111,6 @@ fn create_bindgen_builder(header: &PathBuf) -> Result<Option<Builder>, Error> {
                 .and_then(shlex::split)
                 .unwrap();
             flags.extend(extra_flags.into_iter());
-        } else if line.contains("bindgen-unstable") &&
-                  cfg!(feature = "testing_only_llvm_stable") {
-            return Ok(None);
         } else if line.contains("bindgen-osx-only") {
             let prepend_flags = ["--raw-line", "#![cfg(target_os=\"macos\")]"];
             flags = prepend_flags.into_iter()