api: native crud module support

Adds native api support for crud module [1]
to use it from a connection object.

1. github.com/tarantool/crud

Closes #205

Author: GRISHNOV

.github/workflows/packing.yml

@@ -99,6 +99,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Run tests
         run: make test-pure-install
 
@@ -154,6 +157,12 @@ jobs:
           curl -L https://tarantool.io/release/2/installer.sh | bash -s
           sudo apt install -y tarantool tarantool-dev
 
+      - name: Setup crud module
+        shell: wsl-bash_Ubuntu-20.04 {0}
+        run: |
+          sudo apt install -y unzip
+          tarantoolctl rocks install crud
+
       - name: Setup test tarantool instance
         shell: wsl-bash_Ubuntu-20.04 {0}
         run: |
@@ -162,6 +171,10 @@ jobs:
           touch tarantool.pid
           echo $TNT_PID > ./tarantool.pid
 
+      - name: Check crud exist
+        shell: wsl-bash_Ubuntu-20.04 {0}
+        run: tarantoolctl rocks list
+
       - name: Run tests
         env:
           REMOTE_TARANTOOL_HOST: localhost
@@ -331,6 +344,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Run tests
         run: make test-pure-install
 
@@ -492,6 +508,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Run tests
         run: make test-pure-install
 

.github/workflows/reusable_testing.yml

@@ -39,4 +39,7 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - run: make test

.github/workflows/testing.yml

@@ -85,6 +85,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Run tests
         run: make test
 
@@ -143,6 +146,11 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: |
+          source tarantool-enterprise/env.sh
+          tarantoolctl rocks install crud
+
       - name: Run tests
         run: |
           source tarantool-enterprise/env.sh
@@ -196,6 +204,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Run tests
         run: make test-pure-install
 
@@ -257,6 +268,12 @@ jobs:
           curl -L https://tarantool.io/release/2/installer.sh | bash -s
           sudo apt install -y tarantool=${{ matrix.tarantool }} tarantool-dev=${{ matrix.tarantool }}
 
+      - name: Setup crud module
+        shell: wsl-bash_Ubuntu-20.04 {0}
+        run: |
+          sudo apt install -y unzip
+          tarantoolctl rocks install crud
+
       - name: Setup test tarantool instance
         shell: wsl-bash_Ubuntu-20.04 {0}
         run: |
@@ -265,6 +282,10 @@ jobs:
           touch tarantool.pid
           echo $TNT_PID > ./tarantool.pid
 
+      - name: Check crud exist
+        shell: wsl-bash_Ubuntu-20.04 {0}
+        run: tarantoolctl rocks list
+
       - name: Run tests
         env:
           REMOTE_TARANTOOL_HOST: localhost
@@ -329,6 +350,12 @@ jobs:
           curl -L https://tarantool.io/release/2/installer.sh | bash -s
           sudo apt install -y tarantool=${{ matrix.tarantool }} tarantool-dev=${{ matrix.tarantool }}
 
+      - name: Setup crud module
+        shell: wsl-bash_Ubuntu-20.04 {0}
+        run: |
+          sudo apt install -y unzip
+          tarantoolctl rocks install crud
+
       - name: Setup test tarantool instance
         shell: wsl-bash_Ubuntu-20.04 {0}
         run: |
@@ -337,6 +364,10 @@ jobs:
           touch tarantool.pid
           echo $TNT_PID > ./tarantool.pid
 
+      - name: Check crud exist
+        shell: wsl-bash_Ubuntu-20.04 {0}
+        run: tarantoolctl rocks list
+
       - name: Run tests
         env:
           REMOTE_TARANTOOL_HOST: localhost

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Support custom packer and unpacker factories (#191).
 
+- Support [crud module](https://github.com/tarantool/crud) native API (#205).
+
 ### Changed
 
 ### Fixed

docs/source/api/submodule-crud.rst

@@ -0,0 +1,4 @@
+module :py:mod:`tarantool.crud`
+================================
+
+.. automodule:: tarantool.crud

docs/source/index.rst

@@ -63,6 +63,7 @@ API Reference
    api/module-tarantool.rst
    api/submodule-connection.rst
    api/submodule-connection-pool.rst
+   api/submodule-crud.rst
    api/submodule-dbapi.rst
    api/submodule-error.rst
    api/submodule-mesh-connection.rst

docs/source/quick-start.rst

@@ -347,3 +347,140 @@ with out-of-band message processing:
 
     print(callback_res)
     >>> [[[100, 1]], [[200, 1]], [[300, 1]]]
+
+
+Interaction with the crud module
+----------------------------------
+
+Through the :class:`~tarantool.Connection` object, you can access 
+`crud module <https://github.com/tarantool/crud>`_ methods:
+
+.. code-block:: python
+
+    >>> import tarantool
+    >>> from tarantool.error import CrudModuleError, DatabaseError
+    >>> conn = tarantool.Connection(host='localhost',port=3301)
+
+    >>> conn.crud_
+    conn.crud_count(                conn.crud_insert(               conn.crud_insert_object_many(   
+    conn.crud_min(                  conn.crud_replace_object(       conn.crud_stats(                
+    conn.crud_unflatten_rows(       conn.crud_upsert_many(          conn.crud_delete(               
+    conn.crud_insert_many(          conn.crud_len(                  conn.crud_replace(              
+    conn.crud_replace_object_many(  conn.crud_storage_info(         conn.crud_update(               
+    conn.crud_upsert_object(        conn.crud_get(                  conn.crud_insert_object(        
+    conn.crud_max(                  conn.crud_replace_many(         conn.crud_select(               
+    conn.crud_truncate(             conn.crud_upsert(               conn.crud_upsert_object_many(
+
+As an example, consider :meth:`~tarantool.Connection.crud_insert` and :meth:`~tarantool.Connection.crud_insert_object_many`. 
+It is recommended to enclose calls in the try-except construction as follows:
+
+.. code-block:: python
+
+    # Insert without exception:
+    >>> res = conn.crud_insert('tester', (3500,300,'Rob'))
+    >>> res
+    <tarantool.crud.CrudResult object at 0x11a56e320>
+    >>> res.
+    res.metadata  res.rows # fields like in Lua implementation.
+    >>> res.rows
+    [[3500, 300, 'Rob']]
+    >>> res.metadata
+    [{'name': 'id', 'type': 'unsigned'}, {'name': 'bucket_id', 'type': 'unsigned'}, {'name': 'name', 'type': 'string'}]
+
+    # Insert with exception (duplicate key exists):
+    >>> try:
+    ...     res = conn.crud_insert('tester', (3500,300,'Rob'))
+    ... except CrudModuleError as e:
+    ...     exc_crud = e
+    ... 
+    >>> exc_crud
+    CrudModuleError(None, <tarantool.crud.CrudError object at 0x10a276950>)
+    >>> exc_crud.
+    exc_crud.args             exc_crud.err              exc_crud.res              exc_crud.with_traceback(  
+    >>> exc_crud.args
+    (None, <tarantool.crud.CrudError object at 0x10a276950>)
+    >>> exc_crud.err
+    <tarantool.crud.CrudError object at 0x10a276950>
+    >>> exc_crud.err. # fields like in Lua implementation.
+    exc_crud.err.class_name  exc_crud.err.err         exc_crud.err.file        exc_crud.err.line        exc_crud.err.str         
+    >>> exc_crud.err.class_name
+    'InsertError'
+    >>> exc_crud.err.str
+    'InsertError: Failed to insert: Duplicate key exists in unique index "primary_index" in space "tester" with old tuple - [3500, 300, "Rob"] and new tuple - [3500, 300, "Rob"]'
+    >>> exc_crud.res is None # using only in case of *_many.
+    True
+
+    # In case of batch operation (*_many), CrudModuleError contains both result and errors:
+    >>> try:
+    ...     res = conn.crud_insert_object_many('tester', ({'id':1800,'bucket_id':100,'name':'Mike'}, {'id':2800,'bucket_id':100,'second_name':'Bill'}, {'id':3800,'bucket_id':100,'name':'Bob'}, {'id':1,'bucket_id':100,'name':'Rob'}), {'timeout':100, 'rollback_on_error':False})
+    ... except CrudModuleError as e:
+    ...     exc_crud = e
+    ... 
+    >>> exc_crud
+    CrudModuleError(<tarantool.crud.CrudResult object at 0x11a56f310>, [<tarantool.crud.CrudError object at 0x11a56e9e0>, <tarantool.crud.CrudError object at 0x11a56f490>])
+    >>> exc_crud.
+    exc_crud.args             exc_crud.errs             exc_crud.res              exc_crud.with_traceback(  
+    >>> exc_crud.res # some of the rows were inserted.
+    <tarantool.crud.CrudResult object at 0x11a56f310>
+    >>> exc_crud.res.rows
+    [[1800, 100, 'Mike'], [3800, 100, 'Bob']]
+    >>> exc_crud.errs # some of the rows were not inserted.
+    [<tarantool.crud.CrudError object at 0x11a56e9e0>, <tarantool.crud.CrudError object at 0x11a56f490>]
+    >>> exc_crud.errs[0].str
+    'CallError: Failed for 56d9c861-a8ac-459c-9715-09094ebb94a7: Function returned an error: Duplicate key exists in unique index "primary_index" in space "tester" with old tuple - [1, 100, "Mike"] and new tuple - [1, 100, "Rob"]'
+    >>> exc_crud.errs[1].str
+    'InsertManyError: Failed to flatten object: FlattenError: Object is specified in bad format: FlattenError: Unknown field "second_name" is specified'
+    >>> exc_crud.args
+    (<tarantool.crud.CrudResult object at 0x11a56f310>, [<tarantool.crud.CrudError object at 0x11a56e9e0>, <tarantool.crud.CrudError object at 0x11a56f490>])
+
+If module crud not found on the router:
+
+.. code-block:: python
+
+    >>> try:
+    ...     res = conn.crud_insert('tester', (22221,300,'Rob'))
+    ... except CrudModuleError as e:
+    ...     exc_crud = e
+    ... except DatabaseError as e:
+    ...     exc_db = e
+    ... 
+    >>> exc_db
+    DatabaseError(33, "Procedure 'crud.insert' is not defined. Ensure that you're calling crud.router and user have enough grants")
+
+Using :meth:`~tarantool.Connection.crud_select` and :meth:`~tarantool.Connection.crud_unflatten_rows`:
+
+.. code-block:: python
+
+    >>> try:
+    ...     res = conn.crud_select('tester', {}, {'first':2})
+    ... except CrudModuleError as e:
+    ...     exc_crud = e
+    ... 
+    >>> res
+    <tarantool.crud.CrudResult object at 0x10a276d10>
+    >>> res.
+    res.metadata  res.rows      
+    >>> res.rows
+    [[1, 100, 'Mike'], [2, 100, 'Mike']]
+    >>> res.metadata
+    [{'name': 'id', 'type': 'unsigned'}, {'name': 'bucket_id', 'type': 'unsigned'}, {'name': 'name', 'type': 'string'}]
+    >>> r = conn.crud_unflatten_rows(res.rows, res.metadata)
+    >>> r
+    [{'id': 1, 'bucket_id': 100, 'name': 'Mike'}, {'id': 2, 'bucket_id': 100, 'name': 'Mike'}]
+
+Using :meth:`~tarantool.Connection.crud_truncate` and :meth:`~tarantool.Connection.crud_len`:
+
+.. code-block:: python
+
+    >>> try:
+    ...     res = conn.crud_len('tester')
+    ... except CrudModuleError as e:
+    ...     exc_crud = e
+    >>> res
+    26
+    >>> try:
+    ...     res = conn.crud_truncate('tester')
+    ... except CrudModuleError as e:
+    ...     exc_crud = e
+    >>> res
+    True

tarantool/connection.py

@@ -0,0 +1,87 @@
+"""
+This module provides API for interaction with the `crud`_ module.
+
+.. _crud: https://github.com/tarantool/crud/
+"""
+
+from tarantool.error import DatabaseError
+
+
+class CrudResponse(object):
+    """
+    Contains response fields from the `crud`_ module that correspond 
+    to the Lua implementation.
+
+    .. _crud: https://github.com/tarantool/crud/
+    """
+
+    def __init__(self, response):
+        """
+        Sets response fields as in Lua implementation.
+
+        :param response: The response object of the crud module call.
+        :type response: :class:`~tarantool.response.Response`
+        """
+
+        if isinstance(response, dict):
+            for response_field_name in response.keys():
+                if isinstance(response_field_name, bytes):
+                    setattr(self, response_field_name.decode(), response[response_field_name])
+                else:
+                    setattr(self, response_field_name, response[response_field_name])
+        else:
+            raise RuntimeError('Unable to decode response to object due to unknown type')
+
+
+class CrudResult(CrudResponse):
+    """
+    Contains result's fields from result variable 
+    of crud module operation.
+    """
+
+
+class CrudError(CrudResponse):
+    """
+    Contains error's fields from error variable 
+    of crud module operation.
+    """
+
+
+def wrapCrudUndefinedProcedureError(e: DatabaseError):
+    """
+    Supplements information about an unsuccessful attempt 
+    to find the crud module procedure.
+
+    :param e: The error object of the crud module call.
+    :type e: :class:`~tarantool.error.DatabaseError`
+    """
+
+    info = ". Ensure that you're calling crud.router and user has sufficient grants"
+    e.message = e.message + info
+    e.extra_info.message = e.extra_info.message + info
+    e.args = (e.args[0], e.args[1] + info)
+
+    return e
+
+def call_crud(conn, *args):
+    """
+    Calls the crud via connection.call with try/except block.
+
+    :param conn: The connection object for the crud module call.
+    :type conn: :class:`~tarantool.connection.Connection`
+
+    :param args: The arguments to pass to the crud module.
+    :type args: :obj:`tuple`
+    """
+
+    # Response error code for case "Requested procedure is not defined".
+    ER_NO_SUCH_PROC = 33
+
+    try:
+        crud_resp = conn.call(*args)
+    except DatabaseError as e:
+        if e.code == ER_NO_SUCH_PROC:
+            e = wrapCrudUndefinedProcedureError(e)
+        raise e
+
+    return crud_resp

tarantool/connection_pool.py

@@ -330,6 +330,33 @@ class PoolTolopogyError(DatabaseError):
     pass
 
 
+class CrudModuleError(DatabaseError):
+    """
+    Exception raised for errors that are related to the operation result of `crud`_ module.
+
+    .. _crud: https://github.com/tarantool/crud
+    """
+
+    def __init__(self, *args):
+        """
+        Sets fields with result and errors.
+        
+        :param args: The tuple from the crud module with result and errors.
+        :type args: :obj:`tuple`
+        """
+
+        # Sets tarantool.crud.CrudResult object.
+        self.res = args[0]
+
+        if isinstance(args[1], (tuple, list)):
+            # Case for crud_*_many().
+            # Sets list of tarantool.crud.CrudError objects.
+            self.errs = args[1]
+        else:
+            # Sets tarantool.crud.CrudError object.
+            self.err = args[1]
+
+
 # always print this warnings
 warnings.filterwarnings("always", category=NetworkWarning)
 

tarantool/crud.py

@@ -23,6 +23,7 @@
 from .test_error_ext import TestSuite_ErrorExt
 from .test_push import TestSuite_Push
 from .test_connection import TestSuite_Connection
+from .test_crud import TestSuite_Crud
 
 test_cases = (TestSuite_Schema_UnicodeConnection,
               TestSuite_Schema_BinaryConnection,
@@ -31,7 +32,7 @@
               TestSuite_Encoding, TestSuite_Pool, TestSuite_Ssl,
               TestSuite_Decimal, TestSuite_UUID, TestSuite_Datetime,
               TestSuite_Interval, TestSuite_ErrorExt, TestSuite_Push,
-              TestSuite_Connection,)
+              TestSuite_Connection, TestSuite_Crud,)
 
 def load_tests(loader, tests, pattern):
     suite = unittest.TestSuite()

tarantool/error.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env tarantool
+
+local crud = require('crud')
+local vshard = require('vshard')
+
+local admin_listen = os.getenv("ADMIN")
+local primary_listen = os.getenv("LISTEN")
+
+require('console').listen(admin_listen)
+box.cfg{
+    listen       = primary_listen,
+    memtx_memory = 0.1 * 1024^3, -- 0.1 GiB
+    pid_file     = "box.pid",
+}
+
+box.schema.user.grant(
+    'guest',
+    'read,write,execute',
+    'universe'
+)
+box.schema.create_space(
+ 'tester', {
+  format = {
+    {name = 'id', type = 'unsigned'},
+    {name = 'bucket_id', type = 'unsigned'},
+    {name = 'name', type = 'string'},
+  }
+})
+box.space.tester:create_index('primary_index', {
+    parts = {
+        {field = 1, type = 'unsigned'},
+    },
+})
+box.space.tester:create_index('bucket_id', {
+    parts = {
+        {field = 2, type = 'unsigned'},
+    },
+    unique = false,
+})
+
+-- Setup vshard.
+_G.vshard = vshard
+box.once('guest', function()
+    box.schema.user.grant('guest', 'super')
+end)
+local uri = 'guest@0.0.0.0:' .. primary_listen
+local cfg = {
+    bucket_count = 300,
+    sharding = {
+        [box.info().cluster.uuid] = {
+            replicas = {
+                [box.info().uuid] = {
+                    uri = uri,
+                    name = 'storage',
+                    master = true,
+                },
+            },
+        },
+    },
+}
+vshard.storage.cfg(cfg, box.info().uuid)
+vshard.router.cfg(cfg)
+vshard.router.bootstrap()
+
+-- Initialize crud.
+crud.init_storage()
+crud.init_router()
+crud.cfg{stats = true}

test/suites/__init__.py

@@ -182,6 +182,32 @@ clean = function()
     local _FUNC_LANGUAGE = 5
     local allowed_funcs = {
         ['box.schema.user.info'] = true,
+        ['crud.insert'] = true,
+        ['crud.insert_object'] = true,
+        ['crud.insert_many'] = true,
+        ['crud.insert_object_many'] = true,
+        ['crud.get'] = true,
+        ['crud.update'] = true,
+        ['crud.delete'] = true,
+        ['crud.replace'] = true,
+        ['crud.replace_object'] = true,
+        ['crud.replace_many'] = true,
+        ['crud.replace_object_many'] = true,
+        ['crud.upsert'] = true,
+        ['crud.upsert_object'] = true,
+        ['crud.upsert_many'] = true,
+        ['crud.upsert_object_many'] = true,
+        ['crud.select'] = true,
+        ['crud.min'] = true,
+        ['crud.max'] = true,
+        ['crud.truncate'] = true,
+        ['crud.len'] = true,
+        ['crud.storage_info'] = true,
+        ['crud.count'] = true,
+        ['crud.stats'] = true,
+        ['crud.init_storage'] = true,
+        ['crud.init_router'] = true,
+        ['crud.cfg'] = true,
     }
     local allowed_langs = {
         ['SQL_BUILTIN'] = true,
@@ -220,6 +246,7 @@ clean = function()
         -- modules
         bit = true,
         coroutine = true,
+        crud = true,
         debug = true,
         io = true,
         jit = true,
@@ -289,6 +316,33 @@ clean = function()
         console = true,
         coroutine = true,
         crypto = true,
+        crud = true,
+        ['crud.insert'] = true,
+        ['crud.insert_object'] = true,
+        ['crud.insert_many'] = true,
+        ['crud.insert_object_many'] = true,
+        ['crud.get'] = true,
+        ['crud.update'] = true,
+        ['crud.delete'] = true,
+        ['crud.replace'] = true,
+        ['crud.replace_object'] = true,
+        ['crud.replace_many'] = true,
+        ['crud.replace_object_many'] = true,
+        ['crud.upsert'] = true,
+        ['crud.upsert_object'] = true,
+        ['crud.upsert_many'] = true,
+        ['crud.upsert_object_many'] = true,
+        ['crud.select'] = true,
+        ['crud.min'] = true,
+        ['crud.max'] = true,
+        ['crud.truncate'] = true,
+        ['crud.len'] = true,
+        ['crud.storage_info'] = true,
+        ['crud.count'] = true,
+        ['crud.stats'] = true,
+        ['crud.init_storage'] = true,
+        ['crud.init_router'] = true,
+        ['crud.cfg'] = true,
         csv = true,
         debug = true,
         digest = true,