Merge pull request #125 from tarantool/gh-111-support-shard-2.0

Support shard-2.1

Author: Totktonada

.luacheckrc

@@ -3,7 +3,7 @@ std = {
         'tonumber', 'type', 'assert', 'ipairs', 'math', 'error', 'string',
         'table', 'pairs', 'os', 'select', 'unpack', 'dofile', 'next',
         'getmetatable', 'setmetatable', 'rawget', 'print', 'shard_status',
-        'loadstring',
+        'loadstring', 'arg',
     },
     globals = {'package'}
 }

.travis.yml

@@ -4,6 +4,10 @@ sudo: true
 
 dist: trusty
 
+env:
+    - SHARD_VERSION=1.2
+    - SHARD_VERSION=2.1
+
 branches:
   only:
     - master
@@ -32,7 +36,7 @@ install:
   - cmake .
   - sudo make install
   - cd ..
-  - tarantoolctl rocks install shard 1.2
+  - tarantoolctl rocks install shard "${SHARD_VERSION}"
   - tarantoolctl rocks install avro-schema
   - cd graphql
   - git submodule update --recursive --init

Makefile

@@ -10,10 +10,10 @@ default:
 lint:
 	luacheck graphql/*.lua \
 		test/bench/*.lua \
-		test/local/*.lua \
+		test/space/*.lua \
 		test/testdata/*.lua \
-		test/common/*.test.lua test/common/lua/*.lua \
-		test/extra/*.test.lua \
+		test/common/*.lua \
+		test/extra/*.lua \
 		test/*.lua \
 		--no-redefined --no-unused-args
 

README.md

@@ -101,13 +101,15 @@ make test
   * tarantool,
   * lulpeg,
   * >=tarantool/avro-schema-2.0-71-gfea0ead,
-  * >=tarantool/shard-1.1-91-gfa88bf8 (but < 2.0) (optional),
+  * >=tarantool/shard-1.1-91-gfa88bf8 (but < 2.0) or
+    >=tarantool/shard-2.1-0-g0a7d98f (optional),
   * lrexlib-pcre2 or lrexlib-pcre (optional).
 * For test (additionally to 'for use'):
   * python 2.7,
   * virtualenv,
   * luacheck,
-  * >=tarantool/shard-1.1-92-gec1a27e (but < 2.0),
+  * >=tarantool/shard-1.1-92-gec1a27e (but < 2.0) or
+    >=tarantool/shard-2.1-0-g0a7d98f.
   * >=tarantool/avro-schema-2.2.2-4-g1145e3e.
 * For building apidoc (additionally to 'for use'):
   * ldoc.

graphql/accessor_shard.lua

@@ -2,22 +2,30 @@
 --- (@{accessor_general}) behaves as shard accessor and provides the
 --- `accessor_shard.new` function to create a new shard data accessor instance.
 
+local json = require('json')
 local shard = require('shard')
 local accessor_general = require('graphql.accessor_general')
 
 local accessor_shard = {}
 
 local LIMIT = 100000 -- XXX: we need to raise an error when a limit reached
 
+local function shard_check_error(func_name, result, err)
+    if result ~= nil then return end
+    error(('%s: %s'):format(func_name, json.encode(err)))
+end
+
 -- Check whether a collection (it is sharded space for that accessor) exists.
 local function is_collection_exists(collection_name)
+    local func_name = 'accessor_shard.is_collection_exists'
     local exists
     for _, zone in ipairs(shard.shards) do
         for _, node in ipairs(zone) do
-            local cur = shard:space_call(collection_name, node,
+            local cur, err = shard:space_call(collection_name, node,
                 function(space_obj)
                     return space_obj ~= nil
                 end)
+            shard_check_error(func_name, cur, err)
             assert(exists == nil or cur == exists,
                 ('space "%s" exists on some shards, ' ..
                 'but does not on others'):format(collection_name))
@@ -31,13 +39,15 @@ end
 --- determining whether the index exists within a shard cluster is
 --- not-so-trivial as for local spaces.
 local function is_index_exists(collection_name, index_name)
+    local func_name = 'accessor_shard.is_index_exists'
     local exists
     for _, zone in ipairs(shard.shards) do
         for _, node in ipairs(zone) do
-            local cur = shard:space_call(collection_name, node,
+            local cur, err = shard:space_call(collection_name, node,
                 function(space_obj)
                     return space_obj.index[index_name] ~= nil
                 end)
+            shard_check_error(func_name, cur, err)
             assert(exists == nil or cur == exists,
                 ('index "%s" of space "%s" exists on some shards, ' ..
                 'but does not on others'):format(index_name, collection_name))
@@ -57,10 +67,12 @@ local function get_index(collection_name, index_name)
     local index = setmetatable({}, {
         __index = {
             pairs = function(self, value, opts)
+                local func_name = 'accessor_shard.get_index.<index>.pairs'
                 local opts = opts or {}
                 opts.limit = opts.limit or LIMIT
-                local tuples = shard:secondary_select(collection_name,
+                local tuples, err = shard:secondary_select(collection_name,
                     index_name, opts, value, 0)
+                shard_check_error(func_name, tuples, err)
                 local cur = 1
                 local function gen()
                     if cur > #tuples then return nil end

graphql/tarantool_graphql.lua

@@ -1329,7 +1329,10 @@ local function create_default_accessor(cfg)
             collections = cfg.collections,
             service_fields = cfg.service_fields,
             indexes = cfg.indexes,
-            collection_use_tomap = cfg.collection_use_tomap
+            collection_use_tomap = cfg.collection_use_tomap,
+            resulting_object_cnt_max = cfg.resulting_object_cnt_max,
+            fetched_object_cnt_max = cfg.fetched_object_cnt_max,
+            timeout_ms = cfg.timeout_ms,
         }, cfg.accessor_funcs)
     end
 
@@ -1339,7 +1342,10 @@ local function create_default_accessor(cfg)
             collections = cfg.collections,
             service_fields = cfg.service_fields,
             indexes = cfg.indexes,
-            collection_use_tomap = cfg.collection_use_tomap
+            collection_use_tomap = cfg.collection_use_tomap,
+            resulting_object_cnt_max = cfg.resulting_object_cnt_max,
+            fetched_object_cnt_max = cfg.fetched_object_cnt_max,
+            timeout_ms = cfg.timeout_ms,
         }, cfg.accessor_funcs);
     end
 end

test/bench/bench.lua

@@ -12,9 +12,10 @@ local yaml = require('yaml')
 local clock = require('clock')
 local fiber = require('fiber')
 local digest = require('digest')
-local multirunner = require('test.common.lua.multirunner')
+local multirunner = require('test.common.multirunner')
 local graphql = require('graphql')
 local utils = require('graphql.utils')
+local test_utils = require('test.utils')
 local test_run = utils.optional_require('test_run')
 test_run = test_run and test_run.new()
 
@@ -29,29 +30,6 @@ local SCRIPT_DIR = fio.abspath(debug.getinfo(1).source:match("@?(.*/)")
 
 local bench = {}
 
-function bench.graphql_from_testdata(testdata, shard)
-    local accessor_class = shard and graphql.accessor_shard or
-        graphql.accessor_space
-
-    local meta = testdata.get_test_metadata()
-
-    local accessor = accessor_class.new({
-        schemas = meta.schemas,
-        collections = meta.collections,
-        service_fields = meta.service_fields,
-        indexes = meta.indexes,
-        timeout_ms = graphql.TIMEOUT_INFINITY,
-    })
-
-    local gql_wrapper = graphql.new({
-        schemas = meta.schemas,
-        collections = meta.collections,
-        accessor = accessor,
-    })
-
-    return gql_wrapper
-end
-
 local function workload(shard, bench_prepare, bench_iter, opts)
     local iterations = opts.iterations
     local exp_checksum = opts.checksum
@@ -178,4 +156,14 @@ function bench.run(test_name, opts)
     end
 end
 
+-- helper for preparing benchmarking environment
+function bench.bench_prepare_helper(testdata, shard)
+    testdata.fill_test_data(shard or box.space)
+    return test_utils.graphql_from_testdata(testdata, shard, {
+        graphql_opts = {
+            timeout_ms = graphql.TIMEOUT_INFINITY,
+        }
+    })
+end
+
 return bench

test/bench/nesting-1-1.test.lua

@@ -16,11 +16,7 @@ local testdata = require('test.testdata.bench_testdata')
 -- ---------
 
 local function bench_prepare(state)
-    local virtbox = state.shard or box.space
-
-    state.gql_wrapper = bench.graphql_from_testdata(testdata, state.shard)
-    testdata.fill_test_data(virtbox)
-
+    state.gql_wrapper = bench.bench_prepare_helper(testdata, state.shard)
     local query = [[
         query match_by_user_id($user_id: String) {
             user(user_id: $user_id) {

test/bench/nesting-1-100.test.lua

@@ -16,11 +16,7 @@ local testdata = require('test.testdata.bench_testdata')
 -- ---------
 
 local function bench_prepare(state)
-    local virtbox = state.shard or box.space
-
-    state.gql_wrapper = bench.graphql_from_testdata(testdata, state.shard)
-    testdata.fill_test_data(virtbox)
-
+    state.gql_wrapper = bench.bench_prepare_helper(testdata, state.shard)
     local query = [[
         query match_users {
             user {

test/bench/nesting-2-1-1.test.lua

@@ -16,11 +16,7 @@ local testdata = require('test.testdata.bench_testdata')
 -- ---------
 
 local function bench_prepare(state)
-    local virtbox = state.shard or box.space
-
-    state.gql_wrapper = bench.graphql_from_testdata(testdata, state.shard)
-    testdata.fill_test_data(virtbox)
-
+    state.gql_wrapper = bench.bench_prepare_helper(testdata, state.shard)
     local query = [[
         query match_by_user_and_passport_id($user_id: String,
                 $passport_id: String) {

test/bench/nesting-2-100-1.test.lua

@@ -16,11 +16,7 @@ local testdata = require('test.testdata.bench_testdata')
 -- ---------
 
 local function bench_prepare(state)
-    local virtbox = state.shard or box.space
-
-    state.gql_wrapper = bench.graphql_from_testdata(testdata, state.shard)
-    testdata.fill_test_data(virtbox)
-
+    state.gql_wrapper = bench.bench_prepare_helper(testdata, state.shard)
     local query = [[
         query match_by_passport_id($passport_id: String) {
             user(user_to_passport_c: {passport_id: $passport_id}) {

test/bench/nesting-3-1-1-1.test.lua

@@ -16,11 +16,7 @@ local testdata = require('test.testdata.bench_testdata')
 -- ---------
 
 local function bench_prepare(state)
-    local virtbox = state.shard or box.space
-
-    state.gql_wrapper = bench.graphql_from_testdata(testdata, state.shard)
-    testdata.fill_test_data(virtbox)
-
+    state.gql_wrapper = bench.bench_prepare_helper(testdata, state.shard)
     local query = [[
         query match_by_user_and_passport($user_id: String, $number: String) {
             user(user_id: $user_id, user_to_passport_c: {

test/bench/nesting-3-100-100-1.test.lua

@@ -16,11 +16,7 @@ local testdata = require('test.testdata.bench_testdata')
 -- ---------
 
 local function bench_prepare(state)
-    local virtbox = state.shard or box.space
-
-    state.gql_wrapper = bench.graphql_from_testdata(testdata, state.shard)
-    testdata.fill_test_data(virtbox)
-
+    state.gql_wrapper = bench.bench_prepare_helper(testdata, state.shard)
     local query = [[
         query match_by_passport($number: String) {
             user(user_to_passport_c: {passport_c: {number: $number}}) {

test/common/avro_refs.test.lua

@@ -7,7 +7,7 @@ package.path = fio.abspath(debug.getinfo(1).source:match("@?(.*/)")
     :gsub('/./', '/'):gsub('/+$', '')) .. '/../../?.lua' .. ';' .. package.path
 
 local utils = require('test.utils')
-local testdata = require('test.common.lua.test_data_avro_refs')
+local testdata = require('test.testdata.avro_refs_testdata')
 
 box.cfg({})
 

test/common/common.test.lua

@@ -0,0 +1,16 @@
+#!/usr/bin/env tarantool
+
+local fio = require('fio')
+
+-- require in-repo version of graphql/ sources despite current working directory
+package.path = fio.abspath(debug.getinfo(1).source:match("@?(.*/)")
+    :gsub('/./', '/'):gsub('/+$', '')) .. '/../../?.lua' .. ';' .. package.path
+
+local utils = require('test.utils')
+local testdata = require('test.testdata.common_testdata')
+
+box.cfg({})
+
+utils.run_testdata(testdata)
+
+os.exit()

test/common/compound_index.test.lua

@@ -0,0 +1,16 @@
+#!/usr/bin/env tarantool
+
+local fio = require('fio')
+
+-- require in-repo version of graphql/ sources despite current working directory
+package.path = fio.abspath(debug.getinfo(1).source:match("@?(.*/)")
+    :gsub('/./', '/'):gsub('/+$', '')) .. '/../../?.lua' .. ';' .. package.path
+
+local utils = require('test.utils')
+local testdata = require('test.testdata.compound_index_testdata')
+
+box.cfg({})
+
+utils.run_testdata(testdata)
+
+os.exit()