runner all-test

Needed for #34

Author: ligurio

src/tarantool/runner.clj

@@ -47,8 +47,17 @@
    ;:monotonic       monotonic/workload
    ;:multimonotonic  multimonotonic/workload
    ;:pages           pages/workload
+   :none            (fn [_] tests/noop-test)
    :register        register/workload})
 
+(def standard-workloads
+  "The workload names we run for test-all by default."
+  (remove #{:none} (keys workloads)))
+
+(def workloads-expected-to-pass
+  "A collection of workload names which we expect should actually pass."
+  (remove #{:register :counter-dec} standard-workloads))
+
 (def workload-options
   "For each workload, a map of workload options to all the values that option
   supports."
@@ -73,14 +82,27 @@
     [nil "--mvcc"
      "Enable MVCC engine"
      :default false]
-    ["-w" "--workload NAME" "Test workload to run"
-     :parse-fn keyword
-     :missing (str "--workload " (cli/one-of workloads))
-     :validate [workloads (cli/one-of workloads)]]
     ["-e" "--engine NAME"
      "What Tarantool data engine should we use?"
      :default "memtx"]])
 
+(def test-all-opts
+  "Command line options for testing everything."
+  [[nil "--only-workloads-expected-to-pass" "Don't run tests which we know fail."
+     :default false]
+   ["-w" "--workload NAME"
+    "Test workload to run. If omitted, runs all workloads."
+    :parse-fn keyword
+    :default nil
+    :validate [workloads (cli/one-of workloads)]]])
+
+(def single-test-opts
+  "Command line options for single tests."
+  [["-w" "--workload NAME" "Test workload to run"
+    :parse-fn keyword
+    :missing (str "--workload " (cli/one-of workloads))
+    :validate [workloads (cli/one-of workloads)]]])
+
 (def crash-pattern
   "An egrep pattern we use to find crashes in the Tarantool logs."
   (str/join "|"
@@ -148,10 +170,31 @@
                                          :exceptions  (checker/unhandled-exceptions)
                                          :workload    (:checker workload)})})))
 
+(defn all-test-options
+  "Takes base cli options, a collection of workloads, and a test count,
+  and constructs a sequence of test options."
+  [cli workloads]
+  (for [w workloads, i (range (:test-count cli))]
+    (assoc cli
+           :workload  w)))
+
+(defn all-tests
+  "Takes parsed CLI options and constructs a sequence of test options, by
+  combining all workloads."
+  [test-fn opts]
+  (let [workloads (if-let [w (:workload opts)] [w]
+                    (if (:only-workloads-expected-to-pass opts)
+                      workloads-expected-to-pass
+                      standard-workloads))]
+    (->> (all-test-options opts workloads)
+         (map test-fn))))
+
 (defn -main
   "Handles command line arguments."
   [& args]
   (cli/run! (merge (cli/single-test-cmd {:test-fn tarantool-test
-                                         :opt-spec cli-opts})
+                                         :opt-spec (concat cli-opts single-test-opts)})
+                   (cli/test-all-cmd {:tests-fn  (partial all-tests tarantool-test)
+                                      :opt-spec (concat cli-opts test-all-opts)})
                    (cli/serve-cmd))
             args))