Feature: convert graphql query to avro schema

Extend query object: add `avro_schema` method, which produces Avro
schema which can be used to verify or flatten any `query_exequte`
result.

Closes #7

Author: Khatskevich

Makefile

@@ -3,8 +3,11 @@ default:
 
 .PHONY: lint
 lint:
-	luacheck graphql/*.lua test/local/*.lua test/testdata/*.lua \
+	luacheck graphql/*.lua \
+		test/local/*.lua \
+		test/testdata/*.lua \
 		test/common/*.test.lua test/common/lua/*.lua \
+		test/extra/*.test.lua \
 		--no-redefined --no-unused-args
 
 .PHONY: test

graphql/query_to_avro.lua

@@ -0,0 +1,171 @@
+--- Module for convertion GraphQL query to Avro schema.
+---
+--- Random notes:
+---
+--- * The best way to use this module is to just call `avro_schema` methon on
+---   compiled query object.
+local path = "graphql.core"
+local introspection = require(path .. '.introspection')
+local query_util = require(path .. '.query_util')
+
+-- module functions
+local query_to_avro = {}
+
+local gql_scalar_to_avro_index = {
+    String = "string",
+    Int = "int",
+    Long = "long",
+    -- GraphQL Float is double precision according to graphql.org.
+    -- More info http://graphql.org/learn/schema/#scalar-types
+    Float = "double",
+    Boolean = "boolean"
+}
+local function gql_scalar_to_avro(fieldType)
+    assert(fieldType.__type == "Scalar", "GraphQL scalar field expected")
+    assert(fieldType.name ~= "Map", "Map type is not supported")
+    local result = gql_scalar_to_avro_index[fieldType.name]
+    assert(result ~= nil, "Unexpected scalar type: " .. fieldType.name)
+    return result
+end
+
+-- The function converts avro type to nullable.
+-- In current tarantool/avro-schema implementation we simply add '*'
+-- to the end of type name.
+-- The function do not copy the resulting type but changes it in place.
+--
+-- @tparam table avro schema node to be converted to nullable
+--
+-- @tresult table schema node; basically it is the passed schema node,
+-- however in nullable type implementation through unions it can be different
+-- node
+local function make_avro_type_nullable(avro)
+    assert(avro.type ~= nil, "Avro `type` field is necessary")
+    local type_type = type(avro.type)
+    if type_type == "string" then
+        assert(avro.type:endswith("*") == false,
+            "Avro type should not be nullable already")
+        avro.type = avro.type .. '*'
+        return avro
+    end
+    if type_type == "table" then
+        avro.type = make_avro_type_nullable(avro.type)
+        return avro
+    end
+    error("Avro type should be a string or table, got :" .. type_type)
+end
+
+local object_to_avro
+
+local function complete_field_to_avro(fieldType, result, subSelections, context,
+        NonNull)
+    local fieldTypeName = fieldType.__type
+    if fieldTypeName == 'NonNull' then
+        -- In case the field is NonNull, the real type is in ofType attribute.
+        fieldType = fieldType.ofType
+        fieldTypeName = fieldType.__type
+    elseif NonNull ~= true then
+        -- Call complete_field second time and make result nullable.
+        result = complete_field_to_avro(fieldType, result, subSelections,
+            context, true)
+        result = make_avro_type_nullable(result)
+        return result
+    end
+
+    if fieldTypeName == 'List' then
+        local innerType = fieldType.ofType
+        -- Steal type from virtual object.
+        -- This is necessary because in case of arrays type should be
+        -- "completed" into results `items` field, but in other cases (Object,
+        -- Scalar) it should be completed into `type` field.
+        local items = complete_field_to_avro(innerType, {}, subSelections,
+            context).type
+        result.type = {
+            type = "array",
+            items = items
+        }
+        return result
+    end
+
+    if fieldTypeName == 'Scalar' then
+        result.type = gql_scalar_to_avro(fieldType)
+        return result
+    end
+
+    if fieldTypeName == 'Object' then
+        result.type = object_to_avro(fieldType, subSelections, context)
+        return result
+    elseif fieldTypeName == 'Interface' or fieldTypeName == 'Union' then
+        error('Interfaces and Unions are not supported yet')
+    end
+    error(string.format('Unknown type "%s"', fieldTypeName))
+end
+
+--- The function converts a single Object field to avro format
+local function field_to_avro(object_type, fields, context)
+    local firstField = fields[1]
+    assert(#fields == 1, "The aliases are not considered yet")
+    local fieldName = firstField.name.value
+    local fieldType = introspection.fieldMap[fieldName] or
+        object_type.fields[fieldName]
+    assert(fieldType ~= nil)
+    local subSelections = query_util.mergeSelectionSets(fields)
+    local result = {}
+    result.name = fieldName
+    result = complete_field_to_avro(fieldType.kind, result, subSelections,
+        context)
+    return result
+end
+
+--- Convert GraphQL object to avro record.
+---
+--- @tparam table object_type GraphQL type object to be converted to Avro schema
+---
+--- @tparam table selections GraphQL representations of fields which should be
+--- in the output of the query
+---
+--- @tparam table context additional information for Avro schema generation; one
+--- of the fields is `namespace_parts` -- table of names of records from the
+--- root to the current object
+---
+--- @treturn table corresponding Avro schema
+object_to_avro = function(object_type, selections, context)
+    local groupedFieldSet = query_util.collectFields(object_type, selections,
+        {}, {}, context)
+    local result = {
+        type = 'record',
+        name = object_type.name,
+        fields = {}
+    }
+    if #context.namespace_parts ~= 0 then
+        result.namespace = table.concat(context.namespace_parts, ".")
+    end
+    table.insert(context.namespace_parts, result.name)
+    for _, fields in pairs(groupedFieldSet) do
+        local avro_field = field_to_avro(object_type, fields, context)
+        table.insert(result.fields, avro_field)
+    end
+    context.namespace_parts[#context.namespace_parts] = nil
+    return result
+end
+
+--- Create an Avro schema for a given query.
+---
+--- @tparam table query object which avro schema should be created for
+---
+--- @treturn table `avro_schema` avro schema for any `query:execute()` result.
+function query_to_avro.convert(query)
+    assert(type(query) == "table",
+        'query should be a table, got: ' .. type(table)
+        .. '; hint: use ":" instead of "."')
+    local state = query.state
+    local context = query_util.buildContext(state.schema, query.ast, {}, {},
+        query.operation_name)
+    -- The variable is necessary to avoid fullname interferention.
+    -- Each nested Avro record creates it's namespace.
+    context.namespace_parts = {}
+    local rootType = state.schema[context.operation.operation]
+    local selections = context.operation.selectionSet.selections
+    return object_to_avro(rootType, selections, context)
+end
+
+return query_to_avro

graphql/tarantool_graphql.lua

@@ -14,6 +14,7 @@ local schema = require('graphql.core.schema')
 local types = require('graphql.core.types')
 local validate = require('graphql.core.validate')
 local execute = require('graphql.core.execute')
+local query_to_avro = require('graphql.query_to_avro')
 
 local utils = require('graphql.utils')
 
@@ -661,6 +662,7 @@ local function gql_compile(state, query)
     local gql_query = setmetatable(qstate, {
         __index = {
             execute = gql_execute,
+            avro_schema = query_to_avro.convert
         }
     })
     return gql_query

test/extra/suite.ini

@@ -0,0 +1,5 @@
+[default]
+core = app
+description = tests on features which are not related to specific executor
+lua_libs = ../common/lua/test_data_user_order.lua ../testdata/array_and_map_testdata.lua \
+	../testdata/nullable_index_testdata.lua

test/extra/to_avro_arrays.test.lua

@@ -0,0 +1,112 @@
+#!/usr/bin/env tarantool
+local fio = require('fio')
+local yaml = require('yaml')
+local avro = require('avro_schema')
+local testdata = require('array_and_map_testdata')
+local test = require('tap').test('to avro schema')
+-- require in-repo version of graphql/ sources despite current working directory
+package.path = fio.abspath(debug.getinfo(1).source:match("@?(.*/)")
+    :gsub('/./', '/'):gsub('/+$', '')) .. '/../../?.lua' .. ';' ..
+    package.path
+
+local graphql = require('graphql')
+
+box.cfg{wal_mode="none"}
+test:plan(4)
+
+testdata.init_spaces()
+testdata.fill_test_data()
+local meta = testdata.get_test_metadata()
+
+local accessor = graphql.accessor_space.new({
+    schemas = meta.schemas,
+    collections = meta.collections,
+    service_fields = meta.service_fields,
+    indexes = meta.indexes,
+})
+
+local gql_wrapper = graphql.new({
+    schemas = meta.schemas,
+    collections = meta.collections,
+    accessor = accessor,
+})
+
+-- We do not select `customer_balances` and `favorite_holidays` because thay are
+-- is of `Map` type, which is not supported.
+local query = [[
+    query user_holidays($user_id: String) {
+        user_collection(user_id: $user_id) {
+            user_id
+            favorite_food
+            user_balances {
+                value
+            }
+        }
+    }
+]]
+local expected_avro_schema = [[
+type: record
+name: Query
+fields:
+- name: user_collection
+  type:
+    type: array
+    items:
+      type: record
+      fields:
+      - name: user_id
+        type: string
+      - name: user_balances
+        type:
+          type: array
+          items:
+            type: record
+            fields:
+            - name: value
+              type: int
+            name: balance
+            namespace: Query.user_collection
+      - name: favorite_food
+        type:
+          type: array
+          items: string
+      name: user_collection
+      namespace: Query
+]]
+expected_avro_schema = yaml.decode(expected_avro_schema)
+local gql_query = gql_wrapper:compile(query)
+local variables = {
+    user_id = 'user_id_1',
+}
+
+local avros = gql_query:avro_schema()
+
+test:is_deeply(avros, expected_avro_schema, "generated avro schema")
+local result_expected = [[
+user_collection:
+- user_id: user_id_1
+  user_balances:
+  - value: 33
+  - value: 44
+  favorite_food:
+  - meat
+  - potato
+]]
+result_expected = yaml.decode(result_expected)
+local result = gql_query:execute(variables)
+test:is_deeply(result, result_expected, 'graphql qury exec result')
+local ok, ash, r, fs, _
+ok, ash = avro.create(avros)
+assert(ok)
+ok, _ = avro.validate(ash, result)
+assert(ok)
+test:is(ok, true, 'gql result validation by avro')
+ok, fs = avro.compile(ash)
+assert(ok)
+ok, r = fs.flatten(result)
+assert(ok)
+ok, r = fs.unflatten(r)
+assert(ok)
+test:is_deeply(r, result_expected, 'res = unflatten(flatten(res))')
+
+os.exit(test:check() == true and 0 or 1)

test/extra/to_avro_huge.test.lua

@@ -0,0 +1,267 @@
+#!/usr/bin/env tarantool
+local fio = require('fio')
+local yaml = require('yaml')
+local avro = require('avro_schema')
+local data = require('test_data_user_order')
+local test = require('tap').test('to avro schema')
+-- require in-repo version of graphql/ sources despite current working directory
+package.path = fio.abspath(debug.getinfo(1).source:match("@?(.*/)")
+    :gsub('/./', '/'):gsub('/+$', '')) .. '/../../?.lua' .. ';' ..
+    package.path
+
+local graphql = require('graphql')
+
+box.cfg{wal_mode="none"}
+test:plan(4)
+
+data.init_spaces()
+data.fill_test_data(box.space)
+
+local accessor = graphql.accessor_space.new({
+    schemas = data.meta.schemas,
+    collections = data.meta.collections,
+    service_fields = data.meta.service_fields,
+    indexes = data.meta.indexes,
+})
+
+local gql_wrapper = graphql.new({
+    schemas = data.meta.schemas,
+    collections = data.meta.collections,
+    accessor = accessor,
+})
+
+local query = [[
+    query object_result_max($user_id: Int, $order_id: Int) {
+        user_collection(id: $user_id) {
+            id
+            last_name
+            first_name
+            order_connection(limit: 1) {
+                id
+                user_id
+                description
+                order__order_item {
+                    order_id
+                    item_id
+                    order_item__item{
+                        id
+                        name
+                        description
+                        price
+                    }
+                }
+            }
+        },
+        order_collection(id: $order_id) {
+            id
+            description
+            user_connection {
+                id
+                first_name
+                last_name
+                order_connection(limit: 1) {
+                    id
+                    order__order_item {
+                        order_item__item {
+                            name
+                            price
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+]]
+local expected_avro_schema = [[
+type: record
+name: Query
+fields:
+- name: user_collection
+  type:
+    type: array
+    items:
+      type: record
+      fields:
+      - name: order_connection
+        type:
+          type: array
+          items:
+            type: record
+            fields:
+            - name: user_id
+              type: int
+            - name: order__order_item
+              type:
+                type: array
+                items:
+                  type: record
+                  fields:
+                  - name: order_id
+                    type: int
+                  - name: item_id
+                    type: int
+                  - name: order_item__item
+                    type:
+                      type: record
+                      fields:
+                      - name: description
+                        type: string
+                      - name: price
+                        type: string
+                      - name: name
+                        type: string
+                      - name: id
+                        type: int
+                      name: item_collection
+                      namespace: Query.user_collection.order_collection.order_item_collection
+                  name: order_item_collection
+                  namespace: Query.user_collection.order_collection
+            - name: description
+              type: string
+            - name: id
+              type: int
+            name: order_collection
+            namespace: Query.user_collection
+      - name: last_name
+        type: string
+      - name: first_name
+        type: string
+      - name: id
+        type: int
+      name: user_collection
+      namespace: Query
+- name: order_collection
+  type:
+    type: array
+    items:
+      type: record
+      fields:
+      - name: user_connection
+        type:
+          type: record
+          fields:
+          - name: order_connection
+            type:
+              type: array
+              items:
+                type: record
+                fields:
+                - name: order__order_item
+                  type:
+                    type: array
+                    items:
+                      type: record
+                      fields:
+                      - name: order_item__item
+                        type:
+                          type: record
+                          fields:
+                          - name: name
+                            type: string
+                          - name: price
+                            type: string
+                          name: item_collection
+                          namespace: Query.order_collection.user_collection.order_collection.order_item_collection
+                      name: order_item_collection
+                      namespace: Query.order_collection.user_collection.order_collection
+                - name: id
+                  type: int
+                name: order_collection
+                namespace: Query.order_collection.user_collection
+          - name: last_name
+            type: string
+          - name: first_name
+            type: string
+          - name: id
+            type: int
+          name: user_collection
+          namespace: Query.order_collection
+      - name: id
+        type: int
+      - name: description
+        type: string
+      name: order_collection
+      namespace: Query
+
+]]
+expected_avro_schema = yaml.decode(expected_avro_schema)
+local gql_query = gql_wrapper:compile(query)
+local variables = {
+    user_id = 5,
+    order_id = 20
+}
+
+local avros = gql_query:avro_schema()
+test:is_deeply(avros, expected_avro_schema, "generated avro schema")
+local result_expected = [[
+user_collection:
+- order_connection:
+  - user_id: 5
+    id: 11
+    description: order of user 5
+    order__order_item:
+    - order_id: 1
+      item_id: 11
+      order_item__item:
+        id: 11
+        price: '9.74'
+        name: Money
+        description: lobortis ultrices. Vivamus rhoncus.
+    - order_id: 29
+      item_id: 11
+      order_item__item:
+        id: 11
+        price: '9.74'
+        name: Money
+        description: lobortis ultrices. Vivamus rhoncus.
+    - order_id: 30
+      item_id: 11
+      order_item__item:
+        id: 11
+        price: '9.74'
+        name: Money
+        description: lobortis ultrices. Vivamus rhoncus.
+  last_name: user ln 5
+  first_name: user fn 5
+  id: 5
+order_collection:
+- description: order of user 6
+  user_connection:
+    order_connection:
+    - id: 16
+      order__order_item:
+      - order_item__item:
+          name: Cup
+          price: '8.05'
+      - order_item__item:
+          name: Cup
+          price: '8.05'
+      - order_item__item:
+          name: Cup
+          price: '8.05'
+    last_name: user ln 6
+    first_name: user fn 6
+    id: 6
+  id: 20
+]]
+result_expected = yaml.decode(result_expected)
+local result = gql_query:execute(variables)
+test:is_deeply(result, result_expected, 'graphql qury exec result')
+local ok, ash, r, fs, _
+ok, ash = avro.create(avros)
+assert(ok)
+ok, _ = avro.validate(ash, result)
+assert(ok)
+test:is(ok, true, 'gql result validation by avro')
+ok, fs = avro.compile(ash)
+assert(ok)
+ok, r = fs.flatten(result)
+assert(ok)
+ok, r = fs.unflatten(r)
+-- The test can fail if wrong avro-schema version is installed.
+-- Please install avro-schema >= fea0ead9d1.
+assert(ok)
+test:is_deeply(r, result_expected, 'res = unflatten(flatten(res))')
+
+os.exit(test:check() == true and 0 or 1)

test/extra/to_avro_nested.test.lua

@@ -0,0 +1,110 @@
+#!/usr/bin/env tarantool
+local fio = require('fio')
+local yaml = require('yaml')
+local avro = require('avro_schema')
+local data = require('test_data_nested_record')
+local test = require('tap').test('to avro schema')
+-- require in-repo version of graphql/ sources despite current working directory
+package.path = fio.abspath(debug.getinfo(1).source:match("@?(.*/)")
+    :gsub('/./', '/'):gsub('/+$', '')) .. '/../../?.lua' .. ';' ..
+    package.path
+
+local graphql = require('graphql')
+
+box.cfg{wal_mode="none"}
+test:plan(4)
+
+data.init_spaces()
+data.fill_test_data(box.space)
+
+local accessor = graphql.accessor_space.new({
+    schemas = data.meta.schemas,
+    collections = data.meta.collections,
+    service_fields = data.meta.service_fields,
+    indexes = data.meta.indexes,
+})
+
+local gql_wrapper = graphql.new({
+    schemas = data.meta.schemas,
+    collections = data.meta.collections,
+    accessor = accessor,
+})
+
+local query = [[
+    query getUserByUid($uid: Long) {
+        user(uid: $uid) {
+            uid
+            p1
+            p2
+            nested {
+                x
+                y
+            }
+        }
+    }
+]]
+local expected_avro_schema = [[
+type: record
+name: Query
+fields:
+- name: user
+  type:
+    type: array
+    items:
+      type: record
+      fields:
+      - name: p2
+        type: string
+      - name: p1
+        type: string
+      - name: uid
+        type: long
+      - name: nested
+        type:
+          type: record
+          fields:
+          - name: y
+            type: long
+          - name: x
+            type: long
+          name: nested
+          namespace: Query.user
+      name: user
+      namespace: Query
+]]
+expected_avro_schema = yaml.decode(expected_avro_schema)
+local gql_query = gql_wrapper:compile(query)
+local variables = {
+    uid = 1,
+}
+
+local avros = gql_query:avro_schema()
+
+test:is_deeply(avros, expected_avro_schema, "generated avro schema")
+local result_expected = [[
+user:
+- p2: p2 1
+  p1: p1 1
+  uid: 1
+  nested:
+    y: 2001
+    x: 1001
+]]
+result_expected = yaml.decode(result_expected)
+local result = gql_query:execute(variables)
+test:is_deeply(result, result_expected, 'graphql qury exec result')
+local ok, ash, r, fs, _
+ok, ash = avro.create(avros)
+assert(ok)
+ok, _ = avro.validate(ash, result)
+assert(ok)
+test:is(ok, true, 'gql result validation by avro')
+ok, fs = avro.compile(ash)
+assert(ok)
+ok, r = fs.flatten(result)
+assert(ok)
+ok, r = fs.unflatten(r)
+assert(ok)
+test:is_deeply(r, result_expected, 'res = unflatten(flatten(res))')
+
+os.exit(test:check() == true and 0 or 1)

test/extra/to_avro_nullable.test.lua

@@ -0,0 +1,99 @@
+#!/usr/bin/env tarantool
+local fio = require('fio')
+local yaml = require('yaml')
+local avro = require('avro_schema')
+local testdata = require('nullable_index_testdata')
+local test = require('tap').test('to avro schema')
+-- require in-repo version of graphql/ sources despite current working directory
+package.path = fio.abspath(debug.getinfo(1).source:match("@?(.*/)")
+    :gsub('/./', '/'):gsub('/+$', '')) .. '/../../?.lua' .. ';' ..
+    package.path
+
+local graphql = require('graphql')
+
+box.cfg{wal_mode="none"}
+test:plan(4)
+
+testdata.init_spaces()
+testdata.fill_test_data()
+local meta = testdata.get_test_metadata()
+
+local accessor = graphql.accessor_space.new({
+    schemas = meta.schemas,
+    collections = meta.collections,
+    service_fields = meta.service_fields,
+    indexes = meta.indexes,
+})
+
+local gql_wrapper = graphql.new({
+    schemas = meta.schemas,
+    collections = meta.collections,
+    accessor = accessor,
+})
+
+-- We do not select `customer_balances` and `favorite_holidays` because thay are
+-- is of `Map` type, which is not supported.
+local query = [[
+   query get_foo($id: String) {
+        bar(id: $id) {
+            id
+            id_or_null_1
+            id_or_null_2
+            id_or_null_3
+        }
+    }
+]]
+local expected_avro_schema = [[
+type: record
+name: Query
+fields:
+- name: bar
+  type:
+    type: array
+    items:
+      type: record
+      fields:
+      - name: id_or_null_1
+        type: string*
+      - name: id_or_null_3
+        type: string*
+      - name: id_or_null_2
+        type: string*
+      - name: id
+        type: string
+      name: bar
+      namespace: Query
+]]
+expected_avro_schema = yaml.decode(expected_avro_schema)
+local gql_query = gql_wrapper:compile(query)
+local variables = {
+    id = '101',
+}
+
+local avros = gql_query:avro_schema()
+
+test:is_deeply(avros, expected_avro_schema, "generated avro schema")
+local result_expected = [[
+bar:
+- id_or_null_3: '101'
+  id_or_null_2: '101'
+  id: '101'
+]]
+result_expected = yaml.decode(result_expected)
+local result = gql_query:execute(variables)
+test:is_deeply(result, result_expected, 'graphql qury exec result')
+local ok, ash, r, fs, _
+ok, ash = avro.create(avros)
+assert(ok)
+ok, _ = avro.validate(ash, result)
+assert(ok)
+test:is(ok, true, 'gql result validation by avro')
+ok, fs = avro.compile(ash)
+assert(ok)
+ok, r = fs.flatten(result)
+assert(ok)
+ok, r = fs.unflatten(r)
+assert(ok)
+test:is_deeply(r, result_expected, 'res = unflatten(flatten(res))')
+
+os.exit(test:check() == true and 0 or 1)