Skip to content

Commit e1f47a5

Browse files
ligurioligurio
committed
Add bank workload
In the bank test, we create a pool of simulated bank accounts, and transfer money between them using transactions which read two randomly selected accounts, subtract and increment their balances accordingly, and write the new account values back. Under snapshot isolation, the total of all accounts should be constant over time. We read the state of all accounts concurrently, and check for changes in the total, which suggests read skew or other snapshot isolation anomalies. BIGINT support Interactive transactions in IPROTO #2016 Closes #67
1 parent f411d3b commit e1f47a5

File tree

3 files changed

+249
-1
lines changed

3 files changed

+249
-1
lines changed

‎src/tarantool/bank.clj

+172
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
(ns tarantool.bank
2+
"Simulates transfers between bank accounts."
3+
(:require [clojure.tools.logging :refer [info warn]]
4+
[clojure.string :as str]
5+
[clojure.core.reducers :as r]
6+
[jepsen [cli :as cli]
7+
[client :as client]
8+
[checker :as checker]
9+
[control :as c]
10+
[generator :as gen]]
11+
[jepsen.tests.bank :as bank]
12+
[next.jdbc :as j]
13+
[next.jdbc.sql :as sql]
14+
[knossos.op :as op]
15+
[jepsen.checker.timeline :as timeline]
16+
[tarantool [client :as cl]
17+
[db :as db]]))
18+
19+
(def table-name "accounts")
20+
21+
(defrecord BankClient [conn]
22+
client/Client
23+
24+
(open! [this test node]
25+
(let [conn (cl/open node test)]
26+
(assoc this :conn conn :node node)))
27+
28+
(setup! [this test node]
29+
(locking BankClient
30+
(let [conn (cl/open node test)]
31+
(Thread/sleep 10000) ; wait for leader election and joining to a cluster
32+
(when (= node (first (db/primaries test)))
33+
(cl/with-conn-failure-retry conn
34+
(info (str "Creating table " table-name))
35+
(j/execute! conn [(str "CREATE TABLE IF NOT EXISTS " table-name
36+
"(id INT NOT NULL PRIMARY KEY,
37+
balance INT NOT NULL)")])
38+
(doseq [a (:accounts test)]
39+
(info "Populating account")
40+
(sql/insert! conn table-name {:id a
41+
:balance (if (= a (first (:accounts test)))
42+
(:total-amount test)
43+
0)}))))
44+
(assoc this :conn conn :node node))))
45+
46+
(invoke! [this test op]
47+
;(with-txn op [c conn]
48+
(try
49+
(case (:f op)
50+
:read (->> (sql/query conn [(str "SELECT * FROM " table-name)])
51+
(map (juxt :ID :BALANCE))
52+
(into (sorted-map))
53+
(assoc op :type :ok, :value))
54+
55+
:transfer
56+
(let [{:keys [from to amount]} (:value op)
57+
con (cl/open (first (db/primaries test)) test)
58+
b1 (-> con
59+
(sql/query [(str "SELECT * FROM " table-name " WHERE id = ? ") from])
60+
first
61+
:BALANCE
62+
(- amount))
63+
b2 (-> con
64+
(sql/query [(str "SELECT * FROM " table-name " WHERE id = ? ") to])
65+
first
66+
:BALANCE
67+
(+ amount))]
68+
(cond (or (neg? b1) (neg? b2))
69+
(assoc op :type :fail, :value {:from from :to to :amount amount})
70+
true
71+
; TODO: with-transaction is not supported due to
72+
; https://github.com/tarantool/tarantool/issues/2016
73+
;(cl/with-txn op [c con]
74+
(do (j/execute! con [(str "UPDATE " table-name " SET balance = balance - ? WHERE id = ?") amount from])
75+
(j/execute! con [(str "UPDATE " table-name " SET balance = balance + ? WHERE id = ?") amount to])
76+
(assoc op :type :ok)))))))
77+
78+
(teardown! [_ test]
79+
(when-not (:leave-db-running? test)
80+
(info (str "Drop table" table-name))
81+
(cl/with-conn-failure-retry conn
82+
(j/execute! conn [(str "DROP TABLE IF EXISTS " table-name)]))))
83+
84+
(close! [_ test]))
85+
86+
(defn workload
87+
[opts]
88+
(assoc (bank/test opts)
89+
:client (BankClient. nil)))
90+
91+
; One bank account per table
92+
(defrecord MultiBankClient [conn tbl-created?]
93+
client/Client
94+
(open! [this test node]
95+
(assoc this :conn (cl/open node test)))
96+
97+
(setup! [this test node]
98+
(locking tbl-created?
99+
(let [conn (cl/open node test)]
100+
(Thread/sleep 10000) ; wait for leader election and joining to a cluster
101+
(when (= node (first (db/primaries test)))
102+
(when (compare-and-set! tbl-created? false true)
103+
(cl/with-conn-failure-retry conn
104+
(doseq [a (:accounts test)]
105+
(info "Creating table" table-name a)
106+
(j/execute! conn [(str "CREATE TABLE IF NOT EXISTS " table-name a
107+
"(id INT NOT NULL PRIMARY KEY,"
108+
"balance INT NOT NULL)")])
109+
(info "Populating account" a)
110+
(sql/insert! conn (str table-name a)
111+
{:id 0
112+
:balance (if (= a (first (:accounts test)))
113+
(:total-amount test)
114+
0)})))))
115+
(assoc this :conn conn :node node))))
116+
117+
(invoke! [this test op]
118+
;(with-txn op [c conn]
119+
(try
120+
(case (:f op)
121+
:read
122+
(->> (:accounts test)
123+
(map (fn [x]
124+
[x (->> (sql/query conn [(str "SELECT balance FROM " table-name
125+
x)]
126+
{:row-fn :BALANCE})
127+
first)]))
128+
(into (sorted-map))
129+
(map (fn [[k {b :BALANCE}]] [k b]))
130+
(into {})
131+
(assoc op :type :ok, :value))
132+
133+
:transfer
134+
(let [{:keys [from to amount]} (:value op)
135+
from (str table-name from)
136+
to (str table-name to)
137+
con (cl/open (first (db/primaries test)) test)
138+
b1 (-> con
139+
(sql/query [(str "SELECT balance FROM " from)])
140+
first
141+
:BALANCE
142+
(- amount))
143+
b2 (-> con
144+
(sql/query [(str "SELECT balance FROM " to)])
145+
first
146+
:BALANCE
147+
(+ amount))]
148+
(cond (neg? b1)
149+
(assoc op :type :fail, :error [:negative from b1])
150+
(neg? b2)
151+
(assoc op :type :fail, :error [:negative to b2])
152+
true
153+
; TODO: with-transaction is not supported due to
154+
; https://github.com/tarantool/tarantool/issues/2016
155+
;(cl/with-txn op [c con]
156+
(do (j/execute! con [(str "UPDATE " from " SET balance = balance - ? WHERE id = 0") amount])
157+
(j/execute! con [(str "UPDATE " to " SET balance = balance + ? WHERE id = 0") amount])
158+
(assoc op :type :ok)))))))
159+
160+
(teardown! [_ test]
161+
(when-not (:leave-db-running? test)
162+
(cl/with-conn-failure-retry conn
163+
(doseq [a (:accounts test)]
164+
(info "Drop table" table-name a)
165+
(j/execute! conn [(str "DROP TABLE IF EXISTS " table-name a)])))))
166+
167+
(close! [_ test]))
168+
169+
(defn multitable-workload
170+
[opts]
171+
(assoc (workload opts)
172+
:client (MultiBankClient. nil (atom false))))

‎src/tarantool/client.clj

+70
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,18 @@
55
[next.jdbc :as j]
66
[next.jdbc.sql :as sql]
77
[dom-top.core :as dt]
8+
[jepsen [util :as util :refer [default timeout]]]
89
[next.jdbc.connection :as connection]))
910

1011
(def max-timeout "Longest timeout, in ms" 10000000)
1112

13+
(def txn-timeout 5000)
14+
(def connect-timeout 10000)
15+
(def socket-timeout 10000)
16+
(def open-timeout
17+
"How long will we wait for an open call by default"
18+
5000)
19+
1220
(defn conn-spec
1321
"JDBC connection spec for a node."
1422
[node]
@@ -72,6 +80,68 @@
7280
(assoc ~op :type :fail, :error :conflict)
7381
res#)))
7482

83+
(defmacro with-txn-retries
84+
"Retries body on rollbacks."
85+
[& body]
86+
`(loop []
87+
(let [res# (capture-txn-abort ~@body)]
88+
(if (= ::abort res#)
89+
(recur)
90+
res#))))
91+
92+
(defmacro capture-txn-abort
93+
"Converts aborted transactions to an ::abort keyword"
94+
[& body]
95+
`(try ~@body
96+
(catch java.sql.SQLTransactionRollbackException e#
97+
(if (= (.getMessage e#) rollback-msg)
98+
::abort
99+
(throw e#)))
100+
(catch java.sql.BatchUpdateException e#
101+
(if (= (.getMessage e#) rollback-msg)
102+
::abort
103+
(throw e#)))
104+
(catch java.sql.SQLException e#
105+
(condp re-find (.getMessage e#)
106+
#"can not retry select for update statement" ::abort
107+
#"\[try again later\]" ::abort
108+
(throw e#)))))
109+
110+
(defmacro with-txn
111+
"Executes body in a transaction, with a timeout, automatically retrying
112+
conflicts and handling common errors."
113+
[op [c conn] & body]
114+
`(timeout (+ 1000 socket-timeout) (assoc ~op :type :info, :error :timed-out)
115+
(with-error-handling ~op
116+
(with-txn-retries
117+
; PingCAP says that the default isolation level for
118+
; transactions is snapshot isolation
119+
; (https://github.com/pingcap/docs/blob/master/sql/transaction.md),
120+
; and also that TiDB uses repeatable read to mean SI
121+
; (https://github.com/pingcap/docs/blob/master/sql/transaction-isolation.md).
122+
; I've tried testing both with an explicitly provided
123+
; repeatable read isolation level, and without an explicit
124+
; level; both report the current transaction isolation level as
125+
; 4 (repeatable read), and have identical effects.
126+
;(j/with-db-transaction [~c ~conn :isolation :repeatable-read]
127+
(j/with-transaction [~c ~conn]
128+
; PingCAP added this start-transaction statement below. I
129+
; have concerns about this--it's not clear to me whether
130+
; starting, and not committing, this nested transaction does
131+
; the right thing. In particular, PingCAP has some docs
132+
; (https://github.com/pingcap/docs/blob/master/sql/transaction.md)
133+
; which say "If at this time, the current Session is in the
134+
; process of a transaction, a new transaction is started
135+
; after the current transaction is committed." which does NOT
136+
; seem like it's what we want, because at this point, we're
137+
; already inside a transaction!
138+
; (j/execute! ~c ["start transaction with consistent snapshot"])
139+
;(info :isolation (-> ~c
140+
; j/db-find-connection
141+
; .getTransactionIsolation))
142+
~@body)))))
143+
144+
75145
(defmacro with-conn-failure-retry
76146
"DBMS tends to be flaky for a few seconds after starting up, which can wind
77147
up breaking our setup code. This macro adds a little bit of backoff and retry

‎src/tarantool/runner.clj

+7-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
[jepsen.checker.timeline :as timeline]
1818
[jepsen.os.ubuntu :as ubuntu]
1919
[tarantool [db :as db]
20+
[bank :as bank]
2021
[errcode :as err]
2122
[nemesis :as nemesis]
2223
[register :as register]
@@ -39,7 +40,9 @@
3940
4041
Or, for some special cases where nemeses and workloads are coupled, we return
4142
a keyword here instead."
42-
{:set sets/workload
43+
{:bank bank/workload
44+
:bank-multitable bank/multitable-workload
45+
:set sets/workload
4346
:counter-inc counter/workload-inc
4447
:register register/workload})
4548

@@ -185,6 +188,9 @@
185188
(= (:workload opts) :register))
186189
10
187190
(:concurrency opts))
191+
:accounts (vec (range 10)) ; bank-specific option
192+
:max-transfer 5 ; bank-specific option
193+
:total-amount 100 ; bank-specific option
188194
:generator gen
189195
:checker (checker/compose {:perf (checker/perf {:nemeses (:perf nemesis)})
190196
:clock-skew (checker/clock-plot)

0 commit comments

Comments
 (0)