Add nemeses support

Write via leader only

Closes #16
Closes #17
Closes #21
Closes #25
Closes #34

Author: ligurio

src/tarantool/counter.clj

@@ -38,8 +38,9 @@
     (cl/with-error-handling op
       (cl/with-txn-aborts op
         (case (:f op)
-          :add (do (j/execute! conn
-                      [(str "UPDATE " table-name " SET count = count + ? WHERE id = 0") (:value op)])
+          :add (let [con (cl/open (jepsen/primary test) test)]
+                 (do (j/execute! con
+                      [(str "UPDATE " table-name " SET count = count + ? WHERE id = 0") (:value op)]))
                (assoc op :type :ok))
 
           :read (let [value (:COUNT

src/tarantool/db.clj

@@ -247,6 +247,31 @@
       (stop! test node)
       (wipe! test node))
 
+    ;db/Primary
+    ;(setup-primary! [_ test node])
+
+    ;(primaries      [_ test]
+    ;  (->> (node-state test)
+    ;       (filter (comp #{:leader} :role))
+    ;       (map :node)))
+
+    db/Process
+    (start! [_ test node]
+      (start! node {}))
+              ;{:initial-cluster-state (if @(:initialized? test)
+              ;                          :existing
+              ;                          :new)
+              ; :nodes                 @(:members test)}))
+
+    (kill! [_ test node]
+      (c/su
+        ;(cu/stop-daemon! binary pidfile)))
+        (cu/grepkill! :kill "tarantool")))
+
+    db/Pause
+      (pause!  [_ test node] (c/su (cu/grepkill! :stop "tarantool")))
+      (resume! [_ test node] (c/su (cu/grepkill! :cont "tarantool")))
+
     db/LogFiles
     (log-files [_ test node]
       [logfile])))

src/tarantool/nemesis.clj

@@ -0,0 +1,17 @@
+(ns tarantool.nemesis
+  "Nemeses for Tarantool"
+  (:require [clojure.pprint :refer [pprint]]
+            [clojure.tools.logging :refer [info warn]]
+            [dom-top.core :refer [real-pmap]]
+            [jepsen [nemesis :as n]
+                    [net :as net]
+                    [util :as util]]
+            [jepsen.generator :as gen]
+            [jepsen.nemesis [combined :as nc]
+                            [time :as nt]]))
+
+(defn nemesis-package
+  "Constructs a nemesis and generators for Tarantool"
+  [opts]
+  (let [opts (update opts :faults set)]
+    (nc/nemesis-package opts)))

src/tarantool/runner.clj

@@ -12,13 +12,13 @@
                     [control :as c]
                     [independent :as independent]
                     [generator :as gen]
-                    [nemesis :as nemesis]
                     [tests :as tests]
                     [util :refer [timeout meh]]]
             [jepsen.checker.timeline :as timeline]
             ;[knossos.model :as model]
             [jepsen.os.ubuntu :as ubuntu]
             [tarantool [db :as db]
+                       [nemesis :as nemesis]
                        [register :as register]
                        [sets :as sets]
                        [counter :as counter]]))
@@ -48,6 +48,10 @@
    ;:pages           pages/workload
    :register        register/workload})
 
+(def standard-workloads
+  "The workload names we run for test-all by default."
+  (keys workloads))
+
 (def workload-options
   "For each workload, a map of workload options to all the values that option
   supports."
@@ -64,6 +68,32 @@
    ;:pages       {:serialized-indices  [true false]}
    :register    {}})
 
+(def nemeses
+  "Types of faults a nemesis can create."
+   #{:pause :kill :partition :clock})
+
+(def standard-nemeses
+  "Combinations of nemeses for tests"
+  [[]
+   [:pause]
+   [:kill]
+   [:partition]
+   [:pause :kill :partition :clock]])
+
+(def special-nemeses
+  "A map of special nemesis names to collections of faults"
+  {:none      []
+   :standard  [:pause :kill :partition :clock :member]
+   :all       [:pause :kill :partition :clock :member]})
+
+(defn parse-nemesis-spec
+  "Takes a comma-separated nemesis string and returns a collection of keyword
+  faults."
+  [spec]
+  (->> (str/split spec #",")
+       (map keyword)
+       (mapcat #(get special-nemeses % [%]))))
+
 (def cli-opts
   "Options for test runners."
    [["-v" "--version VERSION"
@@ -76,13 +106,22 @@
      :parse-fn keyword
      :missing (str "--workload " (cli/one-of workloads))
      :validate [workloads (cli/one-of workloads)]]
+    [nil "--nemesis FAULTS" "A comma-separated list of nemesis faults to enable"
+     :parse-fn parse-nemesis-spec
+     :validate [(partial every? (into nemeses (keys special-nemeses)))
+                (str "Faults must be one of " nemeses " or "
+                     (cli/one-of special-nemeses))]]
+    [nil "--nemesis-interval SECONDS" "How long to wait between nemesis faults"
+     :default  3
+     :parse-fn read-string
+     :validate [#(and (number? %) (pos? %)) "must be a positive number"]]
     ["-e" "--engine NAME"
      "What Tarantool data engine should we use?"
      :default "memtx"]])
 
 (def crash-pattern
   "An egrep pattern we use to find crashes in the Tarantool logs."
-  "Segmentation fault|too long WAL write|F>")
+  "Segmentation fault|too long WAL write|F>|ER_TUPLE_FOUND")
 
 (defn logged-crashes
   "Takes a test, and returns a map of nodes to strings from their Tarantool logs
@@ -115,7 +154,15 @@
 (defn tarantool-test
   [opts]
   (let [workload ((get workloads (:workload opts)) opts)
-        nemesis  nemesis/noop
+        nemesis  (nemesis/nemesis-package
+                   {:db        db/db
+                    :nodes     (:nodes opts)
+                    :faults    (:nemesis opts)
+                    :partition {:targets [:primaries]}
+                    :pause     {:targets [nil :one :primaries :majority :all]}
+                    :kill      {:targets [nil :one :primaries :majority :all]}
+                    :interval  (:nemesis-interval opts)})
+        _ (info (pr-str nemesis))
         gen      (->> (:generator workload)
                       (gen/nemesis (:generator nemesis))
                       (gen/time-limit (:time-limit opts)))
@@ -129,26 +176,39 @@
     (merge tests/noop-test
            opts
            {:client    (:client workload)
-            :nemesis   nemesis
+            :nemesis   (:nemesis nemesis)
             :name      (str "tarantool-" (:version opts))
             :os        ubuntu/os
             :db        (db/db (:version opts))
             :engine    (:engine opts)
             :mvcc      (:mvcc opts)
             :pure-generators true
             :generator gen
-            :checker   (checker/compose {:perf        (checker/perf)
+            :checker   (checker/compose {:perf        (checker/perf {:nemeses (:perf nemesis)})
                                          :clock-skew  (checker/clock-plot)
                                          :crash       (crash-checker)
                                          :timeline    (timeline/html)
                                          :stats       (checker/stats)
                                          :exceptions  (checker/unhandled-exceptions)
                                          :workload    (:checker workload)})})))
 
+(defn all-tests
+  "Takes parsed CLI options and constructs a sequence of test options, by
+  combining all workloads and nemeses."
+  [opts]
+  (let [nemeses     (if-let [n (:nemesis opts)]  [n] standard-nemeses)
+        workloads   (if-let [w (:workload opts)] [w] standard-workloads)
+        counts      (range (:test-count opts))]
+    (->> (for [i counts, n nemeses, w workloads]
+           (assoc opts :nemesis n :workload w))
+         (map tarantool-test))))
+
 (defn -main
   "Handles command line arguments."
   [& args]
   (cli/run! (merge (cli/single-test-cmd {:test-fn tarantool-test
                                          :opt-spec cli-opts})
+                   (cli/test-all-cmd {:test-fn  all-tests
+                                      :opt-spec cli-opts})
                    (cli/serve-cmd))
             args))

src/tarantool/sets.clj

@@ -6,6 +6,7 @@
                     [checker :as checker]
                     [generator :as gen]
                     [independent :as independent]]
+            [clojure.tools.logging :refer [info warn]]
             [next.jdbc :as j]
             [next.jdbc.sql :as sql]
             [tarantool.client :as cl]
@@ -37,8 +38,11 @@
     (cl/with-error-handling op
       (cl/with-txn-aborts op
         (case (:f op)
-          :add  (do (sql/insert! conn table-name {:value v})
-                    (assoc op :type :ok))
+          :add (let [con (cl/open (jepsen/primary test) test)]
+                 (info (jepsen/primary test))
+                 ;(do (sql/insert! con table-name {:value v})
+                 (do (j/execute! con [(str "INSERT INTO " table-name "(value) VALUES (" v ")")])
+                    (assoc op :type :ok)))
 
           :read (->> (sql/query conn [(str "SELECT * FROM " table-name)])
                      (mapv :VALUE)