Add nemeses support to runner

Closes #16
Closes #17
Closes #21
Closes #25
Closes #34

Author: ligurio

src/tarantool/db.clj

@@ -247,6 +247,31 @@
       (stop! test node)
       (wipe! test node))
 
+    ;db/Primary
+    ;(setup-primary! [_ test node])
+
+    ;(primaries      [_ test]
+    ;  (->> (node-state test)
+    ;       (filter (comp #{:leader} :role))
+    ;       (map :node)))
+
+    db/Process
+    (start! [_ test node]
+      (start! node {}))
+              ;{:initial-cluster-state (if @(:initialized? test)
+              ;                          :existing
+              ;                          :new)
+              ; :nodes                 @(:members test)}))
+
+    (kill! [_ test node]
+      (c/su
+        ;(cu/stop-daemon! binary pidfile)))
+        (cu/grepkill! :kill "tarantool")))
+
+    db/Pause
+      (pause!  [_ test node] (c/su (cu/grepkill! :stop "tarantool")))
+      (resume! [_ test node] (c/su (cu/grepkill! :cont "tarantool")))
+
     db/LogFiles
     (log-files [_ test node]
       [logfile])))

src/tarantool/nemesis.clj

@@ -0,0 +1,17 @@
+(ns tarantool.nemesis
+  "Nemeses for Tarantool"
+  (:require [clojure.pprint :refer [pprint]]
+            [clojure.tools.logging :refer [info warn]]
+            [dom-top.core :refer [real-pmap]]
+            [jepsen [nemesis :as n]
+                    [net :as net]
+                    [util :as util]]
+            [jepsen.generator :as gen]
+            [jepsen.nemesis [combined :as nc]
+                            [time :as nt]]))
+
+(defn nemesis-package
+  "Constructs a nemesis and generators for Tarantool"
+  [opts]
+  (let [opts (update opts :faults set)]
+    (nc/nemesis-package opts)))

src/tarantool/runner.clj

@@ -12,13 +12,13 @@
                     [control :as c]
                     [independent :as independent]
                     [generator :as gen]
-                    [nemesis :as nemesis]
                     [tests :as tests]
                     [util :refer [timeout meh]]]
             [jepsen.checker.timeline :as timeline]
             ;[knossos.model :as model]
             [jepsen.os.ubuntu :as ubuntu]
             [tarantool [db :as db]
+                       [nemesis :as nemesis]
                        [register :as register]
                        [sets :as sets]
                        [counter :as counter]]))
@@ -68,6 +68,32 @@
    ;:pages       {:serialized-indices  [true false]}
    :register    {}})
 
+(def nemeses
+  "Types of faults a nemesis can create."
+   #{:pause :kill :partition :clock})
+
+(def standard-nemeses
+  "Combinations of nemeses for tests"
+  [[]
+   [:pause]
+   [:kill]
+   [:partition]
+   [:pause :kill :partition :clock]])
+
+(def special-nemeses
+  "A map of special nemesis names to collections of faults"
+  {:none      []
+   :standard  [:pause :kill :partition :clock :member]
+   :all       [:pause :kill :partition :clock :member]})
+
+(defn parse-nemesis-spec
+  "Takes a comma-separated nemesis string and returns a collection of keyword
+  faults."
+  [spec]
+  (->> (str/split spec #",")
+       (map keyword)
+       (mapcat #(get special-nemeses % [%]))))
+
 (def cli-opts
   "Options for test runners."
    [["-v" "--version VERSION"
@@ -80,13 +106,22 @@
      :parse-fn keyword
      :missing (str "--workload " (cli/one-of workloads))
      :validate [workloads (cli/one-of workloads)]]
+    [nil "--nemesis FAULTS" "A comma-separated list of nemesis faults to enable"
+     :parse-fn parse-nemesis-spec
+     :validate [(partial every? (into nemeses (keys special-nemeses)))
+                (str "Faults must be one of " nemeses " or "
+                     (cli/one-of special-nemeses))]]
+    [nil "--nemesis-interval SECONDS" "How long to wait between nemesis faults"
+     :default  3
+     :parse-fn read-string
+     :validate [#(and (number? %) (pos? %)) "must be a positive number"]]
     ["-e" "--engine NAME"
      "What Tarantool data engine should we use?"
      :default "memtx"]])
 
 (def crash-pattern
   "An egrep pattern we use to find crashes in the Tarantool logs."
-  "Segmentation fault|too long WAL write|F>")
+  "Segmentation fault|too long WAL write|F>|ER_TUPLE_FOUND")
 
 (defn logged-crashes
   "Takes a test, and returns a map of nodes to strings from their Tarantool logs
@@ -119,7 +154,15 @@
 (defn tarantool-test
   [opts]
   (let [workload ((get workloads (:workload opts)) opts)
-        nemesis  nemesis/noop
+        nemesis  (nemesis/nemesis-package
+                   {:db        db/db
+                    :nodes     (:nodes opts)
+                    :faults    (:nemesis opts)
+                    :partition {:targets [:primaries]}
+                    :pause     {:targets [nil :one :primaries :majority :all]}
+                    :kill      {:targets [nil :one :primaries :majority :all]}
+                    :interval  (:nemesis-interval opts)})
+        _ (info (pr-str nemesis))
         gen      (->> (:generator workload)
                       (gen/nemesis (:generator nemesis))
                       (gen/time-limit (:time-limit opts)))
@@ -133,15 +176,15 @@
     (merge tests/noop-test
            opts
            {:client    (:client workload)
-            :nemesis   nemesis
+            :nemesis   (:nemesis nemesis)
             :name      (str "tarantool-" (:version opts))
             :os        ubuntu/os
             :db        (db/db (:version opts))
             :engine    (:engine opts)
             :mvcc      (:mvcc opts)
             :pure-generators true
             :generator gen
-            :checker   (checker/compose {:perf        (checker/perf)
+            :checker   (checker/compose {:perf        (checker/perf {:nemeses (:perf nemesis)})
                                          :clock-skew  (checker/clock-plot)
                                          :crash       (crash-checker)
                                          :timeline    (timeline/html)