Merge pull request #29 from qryxip/ja-all-enabled-add-a-tool-to-test-rustc-dep-option-generator

Add a tool to test `rustc-dep-option-generator`

Author: statiolake

.cargo/config

@@ -1,2 +1,3 @@
 [alias]
-dep-tests = ["run", "--manifest-path", "./dep-tests/Cargo.toml", "--"]
+dep-tests = ["run", "--manifest-path", "./tools/dep-tests/Cargo.toml", "--"]
+test-with-generated-opts = ["run", "--manifest-path", "./tools/test-with-generated-opts/Cargo.toml", "--"]

.github/workflows/ci.yml

@@ -37,11 +37,55 @@ jobs:
           command: fmt
           args: -- --check
 
-      - name: '`cargo fmt --manifest-path ./dep-tests/Cargo.toml -- --check`'
+      - name: '`cargo fmt --manifest-path ./tools/dep-tests/Cargo.toml -- --check`'
         uses: actions-rs/cargo@v1
         with:
           command: fmt
-          args: --manifest-path ./dep-tests/Cargo.toml -- --check
+          args: --manifest-path ./tools/dep-tests/Cargo.toml -- --check
+
+      - name: '`cargo fmt --manifest-path ./tools/test-with-generated-opts/Cargo.toml -- --check`'
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --manifest-path ./tools/test-with-generated-opts/Cargo.toml -- --check
+
+  test-with-generated-opts:
+    name: test-with-generated-opts
+    runs-on: ubuntu-18.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+
+      - name: rust-toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.38.0-x86_64-unknown-linux-gnu
+          default: true
+          profile: default
+
+      - name: '`cargo install --git https://github.com/rust-lang-ja/atcoder-rustc-dep-option-generator`'
+        uses: actions-rs/cargo@v1
+        with:
+          command: install
+          args: --git https://github.com/rust-lang-ja/atcoder-rustc-dep-option-generator
+
+      - name: '`cargo clippy --all-features --manifest-path ./tools/test-with-generated-opts/Cargo.toml -- -D warnings`'
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --all-features --manifest-path ./tools/test-with-generated-opts/Cargo.toml -- -D warnings
+
+      - name: '`cargo build --all-features --release`'
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --all-features --release
+
+      - name: '`cargo test-with-generated-opts`'
+        uses: actions-rs/cargo@v1
+        with:
+          command: test-with-generated-opts
 
   build:
     strategy:
@@ -147,18 +191,18 @@ jobs:
           command: run
           args: ${{ matrix.features }} --release
 
-      - name: '`cargo clippy ${{ matrix.features }} --manifest-path ./dep-tests/Cargo.toml -- -D warnings`'
+      - name: '`cargo clippy ${{ matrix.features }} --manifest-path ./tools/dep-tests/Cargo.toml -- -D warnings`'
         uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: ${{ matrix.features }} --manifest-path ./dep-tests/Cargo.toml -- -D warnings
+          args: ${{ matrix.features }} --manifest-path ./tools/dep-tests/Cargo.toml -- -D warnings
         if: matrix.dep_tests
 
-      - name: '`cargo test ${{ matrix.features }} --manifest-path ./dep-tests/Cargo.toml --no-fail-fast`'
+      - name: '`cargo test ${{ matrix.features }} --manifest-path ./tools/dep-tests/Cargo.toml --no-fail-fast`'
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: ${{ matrix.features }} --manifest-path ./dep-tests/Cargo.toml --no-fail-fast
+          args: ${{ matrix.features }} --manifest-path ./tools/dep-tests/Cargo.toml --no-fail-fast
         if: matrix.dep_tests
 
       - name: '`cargo dep-tests --all-features -d 1`'

.gitignore

@@ -1,5 +1,5 @@
-/target/
-/dep-tests/Cargo.lock
-/dep-tests/target/
+/tools/dep-tests/Cargo.lock
+/tools/test-with-generated-opts/Cargo.lock
+**/target/
 **/*.rs.bk
 **/*~

Cargo.toml

@@ -6,7 +6,7 @@ version = "0.1.0"
 edition = "2018"
 
 [workspace]
-exclude = ["./dep-tests"]
+exclude = ["./tools"]
 
 [[bin]]
 name = "main"

examples/arc065-c.rs

@@ -0,0 +1,17 @@
+// https://atcoder.jp/contests/arc065/tasks/arc065_a
+
+use lazy_static::lazy_static;
+use proconio::input;
+use proconio::marker::Bytes;
+use regex::bytes::Regex;
+
+fn main() {
+    input! {
+        s: Bytes,
+    }
+
+    lazy_static! {
+        static ref R: Regex = Regex::new(r"\A(dream(er)?|eraser?)*\z").unwrap();
+    };
+    println!("{}", if R.is_match(&s) { "YES" } else { "NO" });
+}

examples/practice-a.rs

@@ -0,0 +1,14 @@
+// https://atcoder.jp/contests/practice/tasks/practice_1
+
+use proconio::input;
+
+fn main() {
+    input! {
+        a: u32,
+        b: u32,
+        c: u32,
+        s: String,
+    }
+
+    println!("{} {}", a + b + c, s);
+}

examples/tests.ron

@@ -0,0 +1,12 @@
+(
+    tests: {
+        "practice-a": (
+            name: "practice contest: A - Welcome to AtCoder",
+            word_match: Exact,
+        ),
+        "arc065-c": (
+            name: "ABC049 / ARC065: C - 白昼夢 / Daydream",
+            word_match: Exact,
+        ),
+    }
+)

examples/testsets/arc065-c/in/subtask0_0.txt

@@ -0,0 +1 @@
+erasedream

examples/testsets/arc065-c/in/subtask0_1.txt

@@ -0,0 +1 @@
+dreameraser

examples/testsets/arc065-c/in/subtask0_2.txt

@@ -0,0 +1 @@
+dreamerer

examples/testsets/arc065-c/out/subtask0_0.txt

@@ -0,0 +1 @@
+YES

examples/testsets/arc065-c/out/subtask0_1.txt

@@ -0,0 +1 @@
+YES

examples/testsets/arc065-c/out/subtask0_2.txt

@@ -0,0 +1 @@
+NO

examples/testsets/practice-a/in/00_sample_1.txt

@@ -0,0 +1,3 @@
+1
+2 3
+test

examples/testsets/practice-a/in/00_sample_2.txt

@@ -0,0 +1,3 @@
+72
+128 256
+myonmyon

examples/testsets/practice-a/out/00_sample_1.txt

@@ -0,0 +1 @@
+6 test

examples/testsets/practice-a/out/00_sample_2.txt

@@ -0,0 +1 @@
+456 myonmyon

dep-tests/Cargo.toml renamed to tools/dep-tests/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "test-with-generated-opts"
+version = "0.0.0"
+edition = "2018"
+publish = false
+description = "Test with `rustc-dep-option-generator`."
+
+[dependencies]
+anyhow = "1.0.25"
+env_logger = "0.7.1"
+indexmap = { version = "1.3.0", features = ["serde-1"] }
+itertools = "0.8.2"
+log = "0.4.8"
+ron = "0.5.1"
+serde = { version = "1.0.103", features = ["derive"] }
+serde_json = "1.0.42"
+shell-escape = "0.1.4"
+structopt = "0.3.5"
+tempdir = "0.3.7"
+which = { version = "3.1.0", default-features = false }

dep-tests/src/main.rs renamed to tools/dep-tests/src/main.rs

@@ -0,0 +1,240 @@
+use anyhow::{anyhow, Context as _};
+use env_logger::fmt::Color;
+use indexmap::IndexMap;
+use itertools::Itertools as _;
+use log::{info, Level, LevelFilter};
+use serde::Deserialize;
+use structopt::StructOpt;
+use tempdir::TempDir;
+
+use std::collections::{BTreeMap, HashMap};
+use std::env;
+use std::ffi::{OsStr, OsString};
+use std::fs::{self, File};
+use std::io::{self, Read as _, Write as _};
+use std::path::{Path, PathBuf};
+use std::process::{Command, Output, Stdio};
+use std::str::SplitWhitespace;
+use std::time::Instant;
+
+#[derive(StructOpt, Debug)]
+struct Opt {}
+
+fn main() -> anyhow::Result<()> {
+    Opt::from_args();
+
+    env_logger::builder()
+        .format(|buf, record| {
+            let mut style = buf.style();
+            let mut write_with_style = |color, bold, intense, value| -> _ {
+                let value = style
+                    .set_color(color)
+                    .set_bold(bold)
+                    .set_intense(intense)
+                    .value(value);
+                write!(buf, "{}", value)
+            };
+            write_with_style(Color::Black, false, true, "[")?;
+            match record.level() {
+                Level::Info => write_with_style(Color::Cyan, true, false, "INFO"),
+                Level::Warn => write_with_style(Color::Yellow, true, false, "WARN"),
+                Level::Error => write_with_style(Color::Red, true, false, "ERROR"),
+                _ => unreachable!(),
+            }?;
+            write_with_style(Color::Black, false, true, "]")?;
+            writeln!(buf, " {}", record.args())
+        })
+        .filter_module("test_with_generated_opts", LevelFilter::Info)
+        .init();
+
+    let Tests { tests } = File::open("./examples/tests.ron")
+        .map_err(anyhow::Error::from)
+        .and_then(|h| ron::de::from_reader(h).map_err(Into::into))
+        .with_context(|| "Failed to read ./examples/tests.ron")?;
+
+    let tempdir = TempDir::new("atcoder-rust-base-test-with-generated-opts")?;
+
+    let tests = tests
+        .into_iter()
+        .map(|(slug, Test { name, word_match })| {
+            let src = Path::new("./examples").join(&slug).with_extension("rs");
+            let testsets = Path::new("./examples/testsets").join(&slug);
+            let binary = compile(&src, tempdir.path(), &slug)?;
+            Ok((name, word_match, testsets, binary))
+        })
+        .collect::<anyhow::Result<Vec<_>>>()?;
+
+    for (name, word_match, testsets, binary) in tests {
+        test(&name, word_match, &testsets, &binary)?;
+    }
+    Ok(())
+}
+
+fn compile(src: &Path, tempdir: &Path, dir_name: &str) -> anyhow::Result<PathBuf> {
+    fn run_command<S1: AsRef<OsStr>, S2: AsRef<OsStr>, I: IntoIterator<Item = S2>>(
+        program: S1,
+        args: I,
+    ) -> anyhow::Result<Vec<u8>> {
+        let program = program.as_ref();
+        let args = args.into_iter().collect::<Vec<_>>();
+
+        info!(
+            "Running `{}{}`",
+            shell_escape::escape(program.to_string_lossy()),
+            args.iter()
+                .map(AsRef::as_ref)
+                .map(OsStr::to_string_lossy)
+                .map(shell_escape::escape)
+                .format_with("", |s, f| f(&format_args!(" {}", s))),
+        );
+
+        let Output { status, stdout, .. } = Command::new(program)
+            .args(&args)
+            .stdin(Stdio::null())
+            .stderr(Stdio::inherit())
+            .output()?;
+
+        if !status.success() {
+            return Err(anyhow!("{}: {}", program.to_string_lossy(), status));
+        }
+        Ok(stdout)
+    }
+
+    let generated_opts = {
+        let program = which::which("rustc-dep-option-generator")?;
+        let stdout = run_command(&program, &["--format", "json"])?;
+        serde_json::from_slice::<Vec<String>>(&stdout)
+            .with_context(|| format!("{}: invalid output", program.to_string_lossy()))?
+    };
+
+    let out = tempdir
+        .join(dir_name)
+        .with_extension(if cfg!(windows) { "exe" } else { "" });
+
+    let program = env::var_os("RUSTC")
+        .map(Ok)
+        .unwrap_or_else(|| which::which("rustc").map(Into::into))?;
+
+    let args = {
+        let mut args = vec![
+            OsString::from("--edition"),
+            OsString::from("2018"),
+            OsString::from("-C"),
+            OsString::from("opt-level=3"),
+            OsString::from("-o"),
+            OsString::from(&out),
+        ];
+        for opt in generated_opts {
+            args.push(opt.into());
+        }
+        args.push(src.to_owned().into());
+        args
+    };
+
+    run_command(program, args)?;
+    Ok(out)
+}
+
+fn test(
+    task_name: &str,
+    word_match: WordMatch,
+    testsets: &Path,
+    binary: &Path,
+) -> anyhow::Result<()> {
+    let testsets = {
+        let find_files = |dir: &str| -> _ {
+            fs::read_dir(testsets.join(dir))?
+                .map(|entry| {
+                    let path = entry?.path();
+                    let name = path
+                        .file_stem()
+                        .unwrap_or_default()
+                        .to_string_lossy()
+                        .into_owned();
+                    Ok((name, path))
+                })
+                .collect::<io::Result<HashMap<_, _>>>()
+        };
+
+        let (ins, outs) = (find_files("in")?, find_files("out")?);
+
+        ins.into_iter()
+            .flat_map(|(stem, path_in)| {
+                outs.get(&stem)
+                    .map(|path_out| (stem, (path_in, path_out.clone())))
+            })
+            .collect::<BTreeMap<_, _>>()
+    };
+
+    info!("Testing {} for {:?}", binary.display(), task_name);
+
+    for (test_name, (path_in, path_out)) in testsets {
+        fn read_to_string(path: &Path) -> anyhow::Result<String> {
+            fs::read_to_string(path).with_context(|| format!("Failed to read {}", path.display()))
+        }
+
+        let input = read_to_string(&path_in)?;
+        let expected = read_to_string(&path_out)?;
+        let start = Instant::now();
+
+        let mut child = Command::new(binary)
+            .stdin(Stdio::piped())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::inherit())
+            .spawn()
+            .with_context(|| format!("Failed to execute {}", binary.display()))?;
+
+        child.stdin.as_mut().unwrap().write_all(input.as_ref())?;
+        let status = child.wait()?;
+        let stop = Instant::now();
+        let actual = {
+            let mut actual = "".to_owned();
+            child
+                .stdout
+                .as_mut()
+                .unwrap()
+                .read_to_string(&mut actual)
+                .with_context(|| format!("{} outputted invalid UTF-8", binary.display()))?;
+            actual
+        };
+
+        let time = (stop - start).as_millis();
+        let (expected, actual) = (expected.split_whitespace(), actual.split_whitespace());
+        let verdict = if status.success() && word_match.accepts(expected, actual) {
+            "AC"
+        } else if status.success() {
+            "WA"
+        } else {
+            "RE"
+        };
+        info!("{}: {} in {}ms", test_name, verdict, time);
+        if verdict != "AC" {
+            return Err(anyhow!("Test failed"));
+        }
+    }
+    Ok(())
+}
+
+#[derive(Debug, Deserialize)]
+struct Tests {
+    tests: IndexMap<String, Test>,
+}
+
+#[derive(Debug, Deserialize)]
+struct Test {
+    name: String,
+    word_match: WordMatch,
+}
+
+#[derive(Debug, Clone, Copy, Deserialize)]
+enum WordMatch {
+    Exact,
+}
+
+impl WordMatch {
+    fn accepts(self, expected: SplitWhitespace, actual: SplitWhitespace) -> bool {
+        match self {
+            WordMatch::Exact => itertools::equal(expected, actual),
+        }
+    }
+}