Enable Raft consensus and primary discovery in workloads

- Raft will work only when quorum is >= N / 2, therefore automatic quorum
calculation was added, it depends on initial cluster size;
- Raft manage read-only status automatically, so setting `read_only` has been
removed in an instance file;
- Enable primary instance discovery in workloads;

Closes #42

Author: ligurio

resources/tarantool/jepsen.lua

@@ -14,9 +14,11 @@ else
 box.cfg {
     listen = '%TARANTOOL_IP_ADDRESS%:3301';
     replication = { %TARANTOOL_REPLICATION% };
-    read_only = %TARANTOOL_IS_READ_ONLY%;
-    replication_synchro_quorum = 2;
+    replication_synchro_quorum = %TARANTOOL_QUORUM%;
     replication_synchro_timeout = 0.2;
+    replication_timeout = 1;
+    election_mode = 'candidate';
+    election_timeout = 0.5;
     log_level = 6;
     log_nonblock = false;
     too_long_threshold = 0.5;

src/tarantool/counter.clj

@@ -8,7 +8,8 @@
             [clojure.tools.logging :refer [debug info warn]]
             [next.jdbc :as j]
             [next.jdbc.sql :as sql]
-            [tarantool.client :as cl]
+            [tarantool [client :as cl]
+                       [db :as db]]
             [jepsen.core :as jepsen]
             [knossos.model :as model]
             [knossos.op :as op]))
@@ -26,7 +27,8 @@
   (setup! [this test node]
     (let [conn (cl/open node test)]
       (assert conn)
-      (when (= node (jepsen/primary test))
+      (Thread/sleep 10000) ; wait for leader election and joining to a cluster
+      (when (= node (first (db/primaries test)))
         (cl/with-conn-failure-retry conn
           (j/execute! conn [(str "CREATE TABLE IF NOT EXISTS " table-name
                             " (id INT NOT NULL PRIMARY KEY,
@@ -40,7 +42,7 @@
     (cl/with-error-handling op
       (cl/with-txn-aborts op
         (case (:f op)
-          :add (let [con (cl/open (jepsen/primary test) test)]
+          :add (let [con (cl/open (first (db/primaries test)) test)]
                  (do (j/execute! con
                       [(str "UPDATE " table-name " SET count = count + ? WHERE id = 0") (:value op)]))
                (assoc op :type :ok))

src/tarantool/db.clj

@@ -192,11 +192,6 @@
   [b]
   (if (true? b) "true" "false"))
 
-(defn is-primary?
-  [test node]
-  (let [p (jepsen/primary test)]
-    (if (= node p) true false)))
-
 (defn is-single-mode?
   [test]
   (let [n (count (:nodes test))]
@@ -214,32 +209,29 @@
          (into [])
          (remove nil?))))
 
+(defn calculate-quorum
+  "Calculate quorum for a given number of nodes."
+  [test]
+  (->> (/ (count (:nodes test)) 2)
+       (double)
+       (Math/round)))
+
 (defn configure!
   "Configure instance"
   [test node]
-  (let [read-only (not (is-primary? test node))]
-    (info "Joining" node "as" (if (true? read-only) "replica" "leader"))
+    (info "Configure instance on" node)
     (c/exec :mkdir :-p "/etc/tarantool/instances.available")
     (c/exec :mkdir :-p "/etc/tarantool/instances.enabled")
     (c/exec :usermod :-a :-G :tarantool :ubuntu)
     (c/exec :echo (-> "tarantool/jepsen.lua" io/resource slurp
+                      (str/replace #"%TARANTOOL_QUORUM%" (str (calculate-quorum test)))
                       (str/replace #"%TARANTOOL_IP_ADDRESS%" node)
                       (str/replace #"%TARANTOOL_REPLICATION%" (replica-set test))
-                      (str/replace #"%TARANTOOL_IS_READ_ONLY%" (boolean-to-str read-only))
                       (str/replace #"%TARANTOOL_MVCC%" (boolean-to-str (:mvcc test)))
                       (str/replace #"%TARANTOOL_SINGLE_MODE%" (boolean-to-str (is-single-mode? test)))
                       (str/replace #"%TARANTOOL_DATA_ENGINE%" (:engine test)))
             :> "/etc/tarantool/instances.enabled/jepsen.lua")
-    (c/exec :cp "/etc/tarantool/instances.enabled/jepsen.lua" "/etc/tarantool/instances.available")))
-
-(defn is-read-only
-  [conn]
-  (j/execute! conn ["SELECT lua('return box.info().ro') IS NOT NULL"]))
-
-(defn set-read-only-mode
-  "Disable and enable read only mode"
-  [conn mode]
-  (j/execute! conn ["SELECT LUA('box.cfg{read_only=true}; return true')"]))
+    (c/exec :cp "/etc/tarantool/instances.enabled/jepsen.lua" "/etc/tarantool/instances.available"))
 
 (defn db
   "Tarantool DB for a particular version."

src/tarantool/register.clj

@@ -17,8 +17,8 @@
             [knossos.model :as model]
             [jepsen.checker.timeline :as timeline]
             [jepsen.os.ubuntu :as ubuntu]
-            [tarantool.client :as cl]
-            [tarantool.db :as db]))
+            [tarantool [client :as cl]
+                       [db :as db]]))
 
 (def table-name "register")
 
@@ -37,7 +37,8 @@
   (setup! [this test node]
     (let [conn (cl/open node test)]
       (assert conn)
-      (if (= node (jepsen/primary test))
+      (Thread/sleep 10000) ; wait for leader election and joining to a cluster
+      (if (= node (first (db/primaries test)))
         (cl/with-conn-failure-retry conn
           (j/execute! conn [(str "CREATE TABLE IF NOT EXISTS " table-name
                             " (id INT NOT NULL PRIMARY KEY,
@@ -54,7 +55,7 @@
                    v (:value r)]
                (assoc op :type :ok, :value (independent/tuple k v)))
 
-       :write (do (let [con (cl/open (jepsen/primary test) test)
+       :write (do (let [con (cl/open (first (db/primaries test)) test)
                         table (clojure.string/upper-case table-name)]
                     (j/execute! con
                       [(str "SELECT _UPSERT(" k ", " value ", '" table "')")])
@@ -63,7 +64,7 @@
                            :value (independent/tuple k value))))
 
        :cas (do (let [[old new] value
-                  con (cl/open (jepsen/primary test) test)
+                  con (cl/open (first (db/primaries test)) test)
                   table (clojure.string/upper-case table-name)
                   r (->> (j/execute! con
                            [(str "SELECT _CAS(" k ", " old ", " new ", '" table "')")])

src/tarantool/sets.clj

@@ -9,7 +9,8 @@
             [clojure.tools.logging :refer [info warn]]
             [next.jdbc :as j]
             [next.jdbc.sql :as sql]
-            [tarantool.client :as cl]
+            [tarantool [client :as cl]
+                       [db :as db]]
             [jepsen.core :as jepsen]
             [knossos.model :as model]
             [knossos.op :as op]))
@@ -27,7 +28,8 @@
   (setup! [this test node]
     (let [conn (cl/open node test)]
       (assert conn)
-      (if (= node (jepsen/primary test))
+      (Thread/sleep 10000) ; wait for leader election and joining to a cluster
+      (if (= node (first (db/primaries test)))
         (cl/with-conn-failure-retry conn
           (j/execute! conn [(str "CREATE TABLE IF NOT EXISTS " table-name
                             " (id INT NOT NULL PRIMARY KEY AUTOINCREMENT,
@@ -41,7 +43,7 @@
     (cl/with-error-handling op
       (cl/with-txn-aborts op
         (case (:f op)
-          :add (let [con (cl/open (jepsen/primary test) test)]
+          :add (let [con (cl/open (first (db/primaries test)) test)]
                  (do (sql/insert! con table-name {:value v})
                     (assoc op :type :ok)))