api: native crud module support

Adds native api support for crud module [1]
to use it from a connection object.

1. github.com/tarantool/crud

Closes #205

Author: GRISHNOV

.github/workflows/packing.yml

@@ -99,6 +99,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Run tests
         run: make test-pure-install
 
@@ -143,6 +146,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Setup WSL for tarantool
         uses: Vampire/setup-wsl@v1
         with:
@@ -331,6 +337,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Run tests
         run: make test-pure-install
 
@@ -492,6 +501,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Run tests
         run: make test-pure-install
 

.github/workflows/reusable_testing.yml

@@ -39,4 +39,7 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - run: make test

.github/workflows/testing.yml

@@ -85,6 +85,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Run tests
         run: make test
 
@@ -141,6 +144,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Run tests
         run: |
           source tarantool-enterprise/env.sh
@@ -194,6 +200,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Run tests
         run: make test-pure-install
 
@@ -236,6 +245,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Setup WSL for tarantool
         uses: Vampire/setup-wsl@v1
         with:
@@ -316,6 +328,9 @@ jobs:
       - name: Install test requirements
         run: pip3 install -r requirements-test.txt
 
+      - name: Install the crud module for testing purposes
+        run: tarantoolctl rocks install crud
+
       - name: Setup WSL for tarantool
         uses: Vampire/setup-wsl@v1
         with:

CHANGELOG.md

@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Support [crud module](https://github.com/tarantool/crud) native API (#205).
+
 ### Changed
 
 ### Fixed

docs/source/api/submodule-crud.rst

@@ -0,0 +1,4 @@
+module :py:mod:`tarantool.crud`
+================================
+
+.. automodule:: tarantool.crud

docs/source/index.rst

@@ -63,6 +63,7 @@ API Reference
    api/module-tarantool.rst
    api/submodule-connection.rst
    api/submodule-connection-pool.rst
+   api/submodule-crud.rst
    api/submodule-dbapi.rst
    api/submodule-error.rst
    api/submodule-mesh-connection.rst

docs/source/quick-start.rst

@@ -347,3 +347,154 @@ with out-of-band message processing:
 
     print(callback_res)
     >>> [[[100, 1]], [[200, 1]], [[300, 1]]]
+
+
+Interaction with the crud module
+----------------------------------
+
+Through the :class:`~tarantool.Connection` object, you can access 
+`crud module <https://github.com/tarantool/crud>`_ methods:
+
+.. code-block:: python
+
+    >>> import tarantool
+    >>> from tarantool.error import CrudModuleError, DatabaseError
+    >>> conn = tarantool.Connection(host='localhost',port=3301)
+
+    >>> conn.crud_
+    conn.crud_count(                conn.crud_insert(               conn.crud_insert_object_many(   
+    conn.crud_min(                  conn.crud_replace_object(       conn.crud_stats(                
+    conn.crud_unflatten_rows(       conn.crud_upsert_many(          conn.crud_delete(               
+    conn.crud_insert_many(          conn.crud_len(                  conn.crud_replace(              
+    conn.crud_replace_object_many(  conn.crud_storage_info(         conn.crud_update(               
+    conn.crud_upsert_object(        conn.crud_get(                  conn.crud_insert_object(        
+    conn.crud_max(                  conn.crud_replace_many(         conn.crud_select(               
+    conn.crud_truncate(             conn.crud_upsert(               conn.crud_upsert_object_many(
+
+As an example, consider :class:`~tarantool.Connection.crud_insert` and :class:`~tarantool.Connection.crud_insert_object_many`. 
+It is recommended to enclose calls in the try-except construction as follows:
+
+.. code-block:: python
+
+    # Try without exception:
+    >>> try:
+    ...     res = conn.crud_insert('tester', (3500,300,'Rob'))
+    ... except CrudModuleError as e:
+    ...     exc_crud = e
+    ... 
+    >>> res
+    <tarantool.crud.CrudResult object at 0x11a56e320>
+    >>> res.
+    res.metadata  res.rows # fields like in lua implementation.
+    >>> res.rows
+    [[3500, 300, 'Rob']]
+    >>> res.metadata
+    [{'name': 'id', 'type': 'unsigned'}, {'name': 'bucket_id', 'type': 'unsigned'}, {'name': 'name', 'type': 'string'}]
+
+    # Try with exception:
+    >>> try:
+    ...     res = conn.crud_insert('tester', (3500,300,'Rob'))
+    ... except CrudModuleError as e:
+    ...     exc_crud = e
+    ... 
+    >>> exc_crud
+    CrudModuleError(None, <tarantool.crud.CrudError object at 0x10a276950>)
+    >>> exc_crud.
+    exc_crud.args             exc_crud.err              exc_crud.res              exc_crud.with_traceback(  
+    >>> exc_crud.args
+    (None, <tarantool.crud.CrudError object at 0x10a276950>)
+    >>> exc_crud.err
+    <tarantool.crud.CrudError object at 0x10a276950>
+    >>> exc_crud.err. # fields like in lua implementation.
+    exc_crud.err.class_name  exc_crud.err.err         exc_crud.err.file        exc_crud.err.line        exc_crud.err.str         
+    >>> exc_crud.err.class_name
+    'InsertError'
+    >>> exc_crud.err.str
+    'InsertError: Failed to insert: Duplicate key exists in unique index "primary_index" in space "tester" with old tuple - [3500, 300, "Rob"] and new tuple - [3500, 300, "Rob"]'
+    >>> exc_crud.res is None # using only in case of *_many.
+    True
+
+    # In case of batch operation (*_many), CrudModuleError contains both result and errors:
+    >>> try:
+    ...     res = conn.crud_insert_object_many('tester', ({'id':1800,'bucket_id':100,'name':'Mike'}, {'id':2800,'bucket_id':100,'second_name':'Bill'}, {'id':3800,'bucket_id':100,'name':'Bob'}, {'id':1,'bucket_id':100,'name':'Rob'}), {'timeout':100, 'rollback_on_error':False})
+    ... except CrudModuleError as e:
+    ...     exc_crud = e
+    ... 
+    >>> exc_crud
+    CrudModuleError(<tarantool.crud.CrudResult object at 0x11a56f310>, [<tarantool.crud.CrudError object at 0x11a56e9e0>, <tarantool.crud.CrudError object at 0x11a56f490>])
+    >>> exc_crud.
+    exc_crud.args             exc_crud.errs             exc_crud.res              exc_crud.with_traceback(  
+    >>> exc_crud.res # some of the lines were inserted.
+    <tarantool.crud.CrudResult object at 0x11a56f310>
+    >>> exc_crud.res.rows
+    [[1800, 100, 'Mike'], [3800, 100, 'Bob']]
+    >>> exc_crud.errs # some of the lines were not inserted.
+    [<tarantool.crud.CrudError object at 0x11a56e9e0>, <tarantool.crud.CrudError object at 0x11a56f490>]
+    >>> exc_crud.errs[0].str
+    'CallError: Failed for 56d9c861-a8ac-459c-9715-09094ebb94a7: Function returned an error: Duplicate key exists in unique index "primary_index" in space "tester" with old tuple - [1, 100, "Mike"] and new tuple - [1, 100, "Rob"]'
+    >>> exc_crud.errs[1].str
+    'InsertManyError: Failed to flatten object: FlattenError: Object is specified in bad format: FlattenError: Unknown field "second_name" is specified'
+    >>> exc_crud.args
+    (<tarantool.crud.CrudResult object at 0x11a56f310>, [<tarantool.crud.CrudError object at 0x11a56e9e0>, <tarantool.crud.CrudError object at 0x11a56f490>])
+
+If module crud not found on the router:
+
+.. code-block:: python
+
+    >>> try:
+    ...     res = conn.crud_insert('tester', (22221,300,'Rob'))
+    ... except CrudModuleError as e:
+    ...     exc_crud = e
+    ... except DatabaseError as e:
+    ...     exc_db = e
+    ... 
+    >>> exc_db
+    DatabaseError(33, "Procedure 'crud.insert' is not defined. Ensure that you're calling crud.router and user have enough grants")
+    >>> exc_db.
+    exc_db.args             exc_db.code             exc_db.extra_info       exc_db.message          exc_db.with_traceback(  
+    >>> exc_db.args
+    (33, "Procedure 'crud.insert' is not defined. Ensure that you're calling crud.router and user have enough grants")
+    >>> exc_db.code
+    33
+    >>> exc_db.message
+    "Procedure 'crud.insert' is not defined. Ensure that you're calling crud.router and user have enough grants"
+    >>> exc_db.extra_info
+    BoxError(type='ClientError', file='/tmp/tarantool-20221003-6335-edruh3/tarantool-2.10.3/src/box/lua/call.c', line=112, message="Procedure 'crud.insert' is not defined. Ensure that you're calling crud.router and user have enough grants", errno=0, errcode=33, fields=None, prev=None)
+
+Using :class:`~tarantool.Connection.crud_select` and :class:`~tarantool.Connection.crud_unflatten_rows`:
+
+.. code-block:: python
+
+    >>> try:
+    ...     res = conn.crud_select('tester', {}, {'first':2})
+    ... except CrudModuleError as e:
+    ...     exc_crud = e
+    ... 
+    >>> res
+    <tarantool.crud.CrudResult object at 0x10a276d10>
+    >>> res.
+    res.metadata  res.rows      
+    >>> res.rows
+    [[1, 100, 'Mike'], [2, 100, 'Mike']]
+    >>> res.metadata
+    [{'name': 'id', 'type': 'unsigned'}, {'name': 'bucket_id', 'type': 'unsigned'}, {'name': 'name', 'type': 'string'}]
+    >>> r = conn.crud_unflatten_rows(res.rows, res.metadata)
+    >>> r
+    [{'id': 1, 'bucket_id': 100, 'name': 'Mike'}, {'id': 2, 'bucket_id': 100, 'name': 'Mike'}]
+
+Using :class:`~tarantool.Connection.crud_truncate` and :class:`~tarantool.Connection.crud_len`:
+
+.. code-block:: python
+
+    >>> try:
+    ...     res = conn.crud_len('tester')
+    ... except CrudModuleError as e:
+    ...     exc_crud = e
+    >>> res
+    26
+    >>> try:
+    ...     res = conn.crud_truncate('tester')
+    ... except CrudModuleError as e:
+    ...     exc_crud = e
+    >>> res
+    True

tarantool/connection.py

@@ -0,0 +1,81 @@
+"""
+This module provides API for interaction with a crud module.
+"""
+
+from tarantool.error import DatabaseError
+
+
+class CrudResponse(object):
+    """
+    Contains response fields from the `crud`_ module that correspond to the lua implementation.
+
+    .. _crud: https://github.com/tarantool/crud/
+    """
+
+    def __init__(self, response):
+        """
+        Sets response fields as in lua implementation.
+
+        :param response: The response object of the crud module call.
+        :type response: :class:`~tarantool.response.Response`
+        """
+
+        if isinstance(response, dict):
+            for response_field_name in response.keys():
+                if isinstance(response_field_name, bytes):
+                    setattr(self, response_field_name.decode(), response[response_field_name])
+                else:
+                    setattr(self, response_field_name, response[response_field_name])
+        elif isinstance(response, str) or isinstance(response, bytes):
+            self.str = response
+            self.err = response
+        else:
+            raise RuntimeError('Unable to decode response to object due to unknown type')
+
+
+class CrudResult(CrudResponse):
+    """
+    Contains result's fields from result variable of crud module operation.
+    """
+
+
+class CrudError(CrudResponse):
+    """
+    Contains error's fields from error variable of crud module operation.
+    """
+
+
+def wrapCrudUndefinedProcedureError(e: DatabaseError):
+    """
+    Supplements information about an unsuccessful attempt to find the crud module procedure.
+
+    :param e: The error object of the crud module call.
+    :type e: :class:`~tarantool.error.DatabaseError`
+    """
+
+    info = ". Ensure that you're calling crud.router and user have enough grants"
+    e.message = e.message + info
+    e.extra_info.message = e.extra_info.message + info
+    e.args = (e.args[0], e.args[1] + info)
+
+    return e
+
+def call_crud(conn, *args):
+    """
+    Calls the crud via connection.call with try/except block.
+
+    :param conn: The connection object for the crud module call.
+    :type conn: :class:`~tarantool.connection.Connection`
+
+    :param args: The arguments to pass to the crud module.
+    :type args: :obj:`tuple`
+    """
+
+    try:
+        crud_resp = conn.call(*args)
+    except DatabaseError as e:
+        if e.code == 33:
+            e = wrapCrudUndefinedProcedureError(e)
+        raise e
+
+    return crud_resp

tarantool/crud.py

@@ -330,6 +330,33 @@ class PoolTolopogyError(DatabaseError):
     pass
 
 
+class CrudModuleError(DatabaseError):
+    """
+    Exception raised for errors that are related to the operation result of `crud`_ module.
+
+    .. _crud: https://github.com/tarantool/crud
+    """
+
+    def __init__(self, *args):
+        """
+        Sets fields with result and errors.
+        
+        :param args: The tuple from the crud module with result and errors.
+        :type args: :obj:`tuple`
+        """
+
+        # Sets tarantool.crud.CrudResult object.
+        self.res = args[0]
+
+        if isinstance(args[1], (tuple, list)):
+            # Case for crud_*_many().
+            # Sets list of tarantool.crud.CrudError objects.
+            self.errs = args[1]
+        else:
+            # Sets tarantool.crud.CrudError object.
+            self.err = args[1]
+
+
 # always print this warnings
 warnings.filterwarnings("always", category=NetworkWarning)
 

tarantool/error.py

@@ -22,14 +22,16 @@
 from .test_package import TestSuite_Package
 from .test_error_ext import TestSuite_ErrorExt
 from .test_push import TestSuite_Push
+from .test_crud import TestSuite_Crud
 
 test_cases = (TestSuite_Schema_UnicodeConnection,
               TestSuite_Schema_BinaryConnection,
               TestSuite_Request, TestSuite_Protocol, TestSuite_Reconnect,
               TestSuite_Mesh, TestSuite_Execute, TestSuite_DBAPI,
               TestSuite_Encoding, TestSuite_Pool, TestSuite_Ssl,
               TestSuite_Decimal, TestSuite_UUID, TestSuite_Datetime,
-              TestSuite_Interval, TestSuite_ErrorExt, TestSuite_Push,)
+              TestSuite_Interval, TestSuite_ErrorExt, TestSuite_Push,
+              TestSuite_Crud,)
 
 def load_tests(loader, tests, pattern):
     suite = unittest.TestSuite()

test/suites/__init__.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env tarantool
+
+os = require('os')
+crud = require('crud')
+vshard = require('vshard')
+
+admin_listen = os.getenv("ADMIN")
+primary_listen = os.getenv("LISTEN")
+
+require('console').listen(admin_listen)
+box.cfg{
+    listen       = primary_listen,
+    memtx_memory = 0.1 * 1024^3, -- 0.1 GiB
+    pid_file     = "box.pid",
+}
+
+box.schema.user.grant(
+    'guest',
+    'read,write,execute',
+    'universe'
+)
+box.schema.create_space(
+ 'tester', {
+  format = {
+    {name = 'id', type = 'unsigned'},
+    {name = 'bucket_id', type = 'unsigned'},
+    {name = 'name', type = 'string'},
+  }
+})
+box.space.tester:create_index('primary_index', {
+    parts = {
+        {field = 1, type = 'unsigned'},
+    },
+})
+box.space.tester:create_index('bucket_id', {
+    parts = {
+        {field = 2, type = 'unsigned'},
+    },
+    unique = false,
+})
+
+-- Setup vshard.
+_G.vshard = vshard
+box.once('guest', function()
+    box.schema.user.grant('guest', 'super')
+end)
+local uri = 'guest@localhost:' .. primary_listen
+local cfg = {
+    bucket_count = 300,
+    sharding = {
+        [box.info().cluster.uuid] = {
+            replicas = {
+                [box.info().uuid] = {
+                    uri = uri,
+                    name = 'storage',
+                    master = true,
+                },
+            },
+        },
+    },
+}
+vshard.storage.cfg(cfg, box.info().uuid)
+vshard.router.cfg(cfg)
+vshard.router.bootstrap()
+
+-- Initialize crud.
+crud.init_storage()
+crud.init_router()

test/suites/crud_server.lua

@@ -0,0 +1,64 @@
+#!/usr/bin/env tarantool
+
+os = require('os')
+crud = require('crud')
+vshard = require('vshard')
+
+box.cfg{
+    listen = 3301,
+}
+
+box.schema.user.grant(
+    'guest',
+    'read,write,execute',
+    'universe'
+)
+box.schema.create_space(
+ 'tester', {
+  format = {
+    {name = 'id', type = 'unsigned'},
+    {name = 'bucket_id', type = 'unsigned'},
+    {name = 'name', type = 'string'},
+  }
+})
+box.space.tester:create_index('primary_index', {
+    parts = {
+        {field = 1, type = 'unsigned'},
+    },
+})
+box.space.tester:create_index('bucket_id', {
+    parts = {
+        {field = 2, type = 'unsigned'},
+    },
+    unique = false,
+})
+
+-- Setup vshard.
+_G.vshard = vshard
+box.once('guest', function()
+    box.schema.user.grant('guest', 'super')
+end)
+local uri = 'guest@localhost:' .. '3301'
+local cfg = {
+    bucket_count = 3000,
+    sharding = {
+        [box.info().cluster.uuid] = {
+            replicas = {
+                [box.info().uuid] = {
+                    uri = uri,
+                    name = 'storage',
+                    master = true,
+                },
+            },
+        },
+    },
+}
+vshard.storage.cfg(cfg, box.info().uuid)
+vshard.router.cfg(cfg)
+vshard.router.bootstrap()
+
+-- Initialize crud.
+crud.init_storage()
+crud.init_router()
+
+require('console').start()