diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..20276f9f --- /dev/null +++ b/.gitignore @@ -0,0 +1,204 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/python,macos +# Edit at https://www.toptal.com/developers/gitignore?templates=python,macos + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### macOS Patch ### +# iCloud generated files +*.icloud + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# End of https://www.toptal.com/developers/gitignore/api/python,macos + +databricks_sql_connector.egg-info/ +dist/ +build/ diff --git a/cmdexec/clients/python/.gitignore b/cmdexec/clients/python/.gitignore deleted file mode 100644 index 277abef8..00000000 --- a/cmdexec/clients/python/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -databricks_sql_connector.egg-info/ -dist/ -build/ diff --git a/cmdexec/clients/python/poetry.lock b/cmdexec/clients/python/poetry.lock new file mode 100644 index 00000000..9bc3ae8a --- /dev/null +++ b/cmdexec/clients/python/poetry.lock @@ -0,0 +1,575 @@ +[[package]] +name = "atomicwrites" +version = "1.4.0" +description = "Atomic file writes." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "attrs" +version = "21.4.0" +description = "Classes Without Boilerplate" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[package.extras] +dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] +docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] +tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] +tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] + +[[package]] +name = "black" +version = "22.3.0" +description = "The uncompromising code formatter." +category = "dev" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""} +typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "importlib-metadata" +version = "4.11.3" +description = "Read metadata from Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +perf = ["ipython"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] + +[[package]] +name = "iniconfig" +version = "1.1.1" +description = "iniconfig: brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "mypy" +version = "0.950" +description = "Optional static typing for Python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +mypy-extensions = ">=0.4.3" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} +typing-extensions = ">=3.10" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "0.4.3" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "numpy" +version = "1.21.1" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "packaging" +version = "21.3" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" + +[[package]] +name = "pandas" +version = "1.3.5" +description = "Powerful data structures for data analysis, time series, and statistics" +category = "main" +optional = false +python-versions = ">=3.7.1" + +[package.dependencies] +numpy = [ + {version = ">=1.17.3", markers = "platform_machine != \"aarch64\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, + {version = ">=1.19.2", markers = "platform_machine == \"aarch64\" and python_version < \"3.10\""}, + {version = ">=1.20.0", markers = "platform_machine == \"arm64\" and python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, +] +python-dateutil = ">=2.7.3" +pytz = ">=2017.3" + +[package.extras] +test = ["hypothesis (>=3.58)", "pytest (>=6.0)", "pytest-xdist"] + +[[package]] +name = "pathspec" +version = "0.9.0" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[[package]] +name = "platformdirs" +version = "2.5.2" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4)"] +test = ["appdirs (==1.4.4)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)", "pytest (>=6)"] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "pyarrow" +version = "5.0.0" +description = "Python library for Apache Arrow" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +numpy = ">=1.16.6" + +[[package]] +name = "pyparsing" +version = "3.0.9" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "dev" +optional = false +python-versions = ">=3.6.8" + +[package.extras] +diagrams = ["railroad-diagrams", "jinja2"] + +[[package]] +name = "pytest" +version = "7.1.2" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +tomli = ">=1.0.0" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2022.1" +description = "World timezone definitions, modern and historical" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "thrift" +version = "0.13.0" +description = "Python bindings for the Apache Thrift RPC system" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +six = ">=1.7.2" + +[package.extras] +all = ["tornado (>=4.0)", "twisted"] +tornado = ["tornado (>=4.0)"] +twisted = ["twisted"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "typed-ast" +version = "1.5.3" +description = "a fork of Python 2 and 3 ast modules with type comment support" +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "typing-extensions" +version = "4.2.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "zipp" +version = "3.8.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.7.1" +content-hash = "9a8934a880c7e31bf7dc9673ee9a9eafe4111ec26ef98298cbe20aa2b7533b52" + +[metadata.files] +atomicwrites = [ + {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, + {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, +] +attrs = [ + {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, + {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, +] +black = [ + {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, + {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"}, + {file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"}, + {file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"}, + {file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"}, + {file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"}, + {file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"}, + {file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"}, + {file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"}, + {file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"}, + {file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"}, + {file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"}, + {file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"}, + {file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"}, + {file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"}, + {file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"}, + {file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"}, + {file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"}, + {file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"}, + {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"}, + {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"}, +] +click = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +importlib-metadata = [ + {file = "importlib_metadata-4.11.3-py3-none-any.whl", hash = "sha256:1208431ca90a8cca1a6b8af391bb53c1a2db74e5d1cef6ddced95d4b2062edc6"}, + {file = "importlib_metadata-4.11.3.tar.gz", hash = "sha256:ea4c597ebf37142f827b8f39299579e31685c31d3a438b59f469406afd0f2539"}, +] +iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, + {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, +] +mypy = [ + {file = "mypy-0.950-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cf9c261958a769a3bd38c3e133801ebcd284ffb734ea12d01457cb09eacf7d7b"}, + {file = "mypy-0.950-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5b5bd0ffb11b4aba2bb6d31b8643902c48f990cc92fda4e21afac658044f0c0"}, + {file = "mypy-0.950-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5e7647df0f8fc947388e6251d728189cfadb3b1e558407f93254e35abc026e22"}, + {file = "mypy-0.950-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:eaff8156016487c1af5ffa5304c3e3fd183edcb412f3e9c72db349faf3f6e0eb"}, + {file = "mypy-0.950-cp310-cp310-win_amd64.whl", hash = "sha256:563514c7dc504698fb66bb1cf897657a173a496406f1866afae73ab5b3cdb334"}, + {file = "mypy-0.950-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dd4d670eee9610bf61c25c940e9ade2d0ed05eb44227275cce88701fee014b1f"}, + {file = "mypy-0.950-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ca75ecf2783395ca3016a5e455cb322ba26b6d33b4b413fcdedfc632e67941dc"}, + {file = "mypy-0.950-cp36-cp36m-win_amd64.whl", hash = "sha256:6003de687c13196e8a1243a5e4bcce617d79b88f83ee6625437e335d89dfebe2"}, + {file = "mypy-0.950-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4c653e4846f287051599ed8f4b3c044b80e540e88feec76b11044ddc5612ffed"}, + {file = "mypy-0.950-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e19736af56947addedce4674c0971e5dceef1b5ec7d667fe86bcd2b07f8f9075"}, + {file = "mypy-0.950-cp37-cp37m-win_amd64.whl", hash = "sha256:ef7beb2a3582eb7a9f37beaf38a28acfd801988cde688760aea9e6cc4832b10b"}, + {file = "mypy-0.950-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0112752a6ff07230f9ec2f71b0d3d4e088a910fdce454fdb6553e83ed0eced7d"}, + {file = "mypy-0.950-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ee0a36edd332ed2c5208565ae6e3a7afc0eabb53f5327e281f2ef03a6bc7687a"}, + {file = "mypy-0.950-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:77423570c04aca807508a492037abbd72b12a1fb25a385847d191cd50b2c9605"}, + {file = "mypy-0.950-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5ce6a09042b6da16d773d2110e44f169683d8cc8687e79ec6d1181a72cb028d2"}, + {file = "mypy-0.950-cp38-cp38-win_amd64.whl", hash = "sha256:5b231afd6a6e951381b9ef09a1223b1feabe13625388db48a8690f8daa9b71ff"}, + {file = "mypy-0.950-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0384d9f3af49837baa92f559d3fa673e6d2652a16550a9ee07fc08c736f5e6f8"}, + {file = "mypy-0.950-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1fdeb0a0f64f2a874a4c1f5271f06e40e1e9779bf55f9567f149466fc7a55038"}, + {file = "mypy-0.950-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:61504b9a5ae166ba5ecfed9e93357fd51aa693d3d434b582a925338a2ff57fd2"}, + {file = "mypy-0.950-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a952b8bc0ae278fc6316e6384f67bb9a396eb30aced6ad034d3a76120ebcc519"}, + {file = "mypy-0.950-cp39-cp39-win_amd64.whl", hash = "sha256:eaea21d150fb26d7b4856766e7addcf929119dd19fc832b22e71d942835201ef"}, + {file = "mypy-0.950-py3-none-any.whl", hash = "sha256:a4d9898f46446bfb6405383b57b96737dcfd0a7f25b748e78ef3e8c576bba3cb"}, + {file = "mypy-0.950.tar.gz", hash = "sha256:1b333cfbca1762ff15808a0ef4f71b5d3eed8528b23ea1c3fb50543c867d68de"}, +] +mypy-extensions = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] +numpy = [ + {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e"}, + {file = "numpy-1.21.1-cp37-cp37m-win32.whl", hash = "sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172"}, + {file = "numpy-1.21.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8"}, + {file = "numpy-1.21.1-cp38-cp38-win32.whl", hash = "sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd"}, + {file = "numpy-1.21.1-cp38-cp38-win_amd64.whl", hash = "sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a"}, + {file = "numpy-1.21.1-cp39-cp39-win32.whl", hash = "sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2"}, + {file = "numpy-1.21.1-cp39-cp39-win_amd64.whl", hash = "sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33"}, + {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"}, + {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, +] +packaging = [ + {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, + {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, +] +pandas = [ + {file = "pandas-1.3.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:62d5b5ce965bae78f12c1c0df0d387899dd4211ec0bdc52822373f13a3a022b9"}, + {file = "pandas-1.3.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:adfeb11be2d54f275142c8ba9bf67acee771b7186a5745249c7d5a06c670136b"}, + {file = "pandas-1.3.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:60a8c055d58873ad81cae290d974d13dd479b82cbb975c3e1fa2cf1920715296"}, + {file = "pandas-1.3.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd541ab09e1f80a2a1760032d665f6e032d8e44055d602d65eeea6e6e85498cb"}, + {file = "pandas-1.3.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2651d75b9a167cc8cc572cf787ab512d16e316ae00ba81874b560586fa1325e0"}, + {file = "pandas-1.3.5-cp310-cp310-win_amd64.whl", hash = "sha256:aaf183a615ad790801fa3cf2fa450e5b6d23a54684fe386f7e3208f8b9bfbef6"}, + {file = "pandas-1.3.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:344295811e67f8200de2390093aeb3c8309f5648951b684d8db7eee7d1c81fb7"}, + {file = "pandas-1.3.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:552020bf83b7f9033b57cbae65589c01e7ef1544416122da0c79140c93288f56"}, + {file = "pandas-1.3.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cce0c6bbeb266b0e39e35176ee615ce3585233092f685b6a82362523e59e5b4"}, + {file = "pandas-1.3.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d28a3c65463fd0d0ba8bbb7696b23073efee0510783340a44b08f5e96ffce0c"}, + {file = "pandas-1.3.5-cp37-cp37m-win32.whl", hash = "sha256:a62949c626dd0ef7de11de34b44c6475db76995c2064e2d99c6498c3dba7fe58"}, + {file = "pandas-1.3.5-cp37-cp37m-win_amd64.whl", hash = "sha256:8025750767e138320b15ca16d70d5cdc1886e8f9cc56652d89735c016cd8aea6"}, + {file = "pandas-1.3.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fe95bae4e2d579812865db2212bb733144e34d0c6785c0685329e5b60fcb85dd"}, + {file = "pandas-1.3.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f261553a1e9c65b7a310302b9dbac31cf0049a51695c14ebe04e4bfd4a96f02"}, + {file = "pandas-1.3.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b6dbec5f3e6d5dc80dcfee250e0a2a652b3f28663492f7dab9a24416a48ac39"}, + {file = "pandas-1.3.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3bc49af96cd6285030a64779de5b3688633a07eb75c124b0747134a63f4c05f"}, + {file = "pandas-1.3.5-cp38-cp38-win32.whl", hash = "sha256:b6b87b2fb39e6383ca28e2829cddef1d9fc9e27e55ad91ca9c435572cdba51bf"}, + {file = "pandas-1.3.5-cp38-cp38-win_amd64.whl", hash = "sha256:a395692046fd8ce1edb4c6295c35184ae0c2bbe787ecbe384251da609e27edcb"}, + {file = "pandas-1.3.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bd971a3f08b745a75a86c00b97f3007c2ea175951286cdda6abe543e687e5f2f"}, + {file = "pandas-1.3.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37f06b59e5bc05711a518aa10beaec10942188dccb48918bb5ae602ccbc9f1a0"}, + {file = "pandas-1.3.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c21778a688d3712d35710501f8001cdbf96eb70a7c587a3d5613573299fdca6"}, + {file = "pandas-1.3.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3345343206546545bc26a05b4602b6a24385b5ec7c75cb6059599e3d56831da2"}, + {file = "pandas-1.3.5-cp39-cp39-win32.whl", hash = "sha256:c69406a2808ba6cf580c2255bcf260b3f214d2664a3a4197d0e640f573b46fd3"}, + {file = "pandas-1.3.5-cp39-cp39-win_amd64.whl", hash = "sha256:32e1a26d5ade11b547721a72f9bfc4bd113396947606e00d5b4a5b79b3dcb006"}, + {file = "pandas-1.3.5.tar.gz", hash = "sha256:1e4285f5de1012de20ca46b188ccf33521bff61ba5c5ebd78b4fb28e5416a9f1"}, +] +pathspec = [ + {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, + {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, +] +platformdirs = [ + {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, + {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, +] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] +py = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] +pyarrow = [ + {file = "pyarrow-5.0.0-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:e9ec80f4a77057498cf4c5965389e42e7f6a618b6859e6dd615e57505c9167a6"}, + {file = "pyarrow-5.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b1453c2411b5062ba6bf6832dbc4df211ad625f678c623a2ee177aee158f199b"}, + {file = "pyarrow-5.0.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:9e04d3621b9f2f23898eed0d044203f66c156d880f02c5534a7f9947ebb1a4af"}, + {file = "pyarrow-5.0.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:64f30aa6b28b666a925d11c239344741850eb97c29d3aa0f7187918cf82494f7"}, + {file = "pyarrow-5.0.0-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:99c8b0f7e2ce2541dd4c0c0101d9944bb8e592ae3295fe7a2f290ab99222666d"}, + {file = "pyarrow-5.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:456a4488ae810a0569d1adf87dbc522bcc9a0e4a8d1809b934ca28c163d8edce"}, + {file = "pyarrow-5.0.0-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:c5493d2414d0d690a738aac8dd6d38518d1f9b870e52e24f89d8d7eb3afd4161"}, + {file = "pyarrow-5.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:1832709281efefa4f199c639e9f429678286329860188e53beeda71750775923"}, + {file = "pyarrow-5.0.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:b6387d2058d95fa48ccfedea810a768187affb62f4a3ef6595fa30bf9d1a65cf"}, + {file = "pyarrow-5.0.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:bbe2e439bec2618c74a3bb259700c8a7353dc2ea0c5a62686b6cf04a50ab1e0d"}, + {file = "pyarrow-5.0.0-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:5c0d1b68e67bb334a5af0cecdf9b6a702aaa4cc259c5cbb71b25bbed40fcedaf"}, + {file = "pyarrow-5.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:6e937ce4a40ea0cc7896faff96adecadd4485beb53fbf510b46858e29b2e75ae"}, + {file = "pyarrow-5.0.0-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:7560332e5846f0e7830b377c14c93624e24a17f91c98f0b25dafb0ca1ea6ba02"}, + {file = "pyarrow-5.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:53e550dec60d1ab86cba3afa1719dc179a8bc9632a0e50d9fe91499cf0a7f2bc"}, + {file = "pyarrow-5.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2d26186ca9748a1fb89ae6c1fa04fb343a4279b53f118734ea8096f15d66c820"}, + {file = "pyarrow-5.0.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:7c4edd2bacee3eea6c8c28bddb02347f9d41a55ec9692c71c6de6e47c62a7f0d"}, + {file = "pyarrow-5.0.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:601b0aabd6fb066429e706282934d4d8d38f53bdb8d82da9576be49f07eedf5c"}, + {file = "pyarrow-5.0.0-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:ff21711f6ff3b0bc90abc8ca8169e676faeb2401ddc1a0bc1c7dc181708a3406"}, + {file = "pyarrow-5.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:ed135a99975380c27077f9d0e210aea8618ed9fadcec0e71f8a3190939557afe"}, + {file = "pyarrow-5.0.0-cp39-cp39-macosx_10_13_universal2.whl", hash = "sha256:6e1f0e4374061116f40e541408a8a170c170d0a070b788717e18165ebfdd2a54"}, + {file = "pyarrow-5.0.0-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:4341ac0f552dc04c450751e049976940c7f4f8f2dae03685cc465ebe0a61e231"}, + {file = "pyarrow-5.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c3fc856f107ca2fb3c9391d7ea33bbb33f3a1c2b4a0e2b41f7525c626214cc03"}, + {file = "pyarrow-5.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:357605665fbefb573d40939b13a684c2490b6ed1ab4a5de8dd246db4ab02e5a4"}, + {file = "pyarrow-5.0.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f4db312e9ba80e730cefcae0a05b63ea5befc7634c28df56682b628ad8e1c25c"}, + {file = "pyarrow-5.0.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:1d9485741e497ccc516cb0a0c8f56e22be55aea815be185c3f9a681323b0e614"}, + {file = "pyarrow-5.0.0-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:b3115df938b8d7a7372911a3cb3904196194bcea8bb48911b4b3eafee3ab8d90"}, + {file = "pyarrow-5.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:4d8adda1892ef4553c4804af7f67cce484f4d6371564e2d8374b8e2bc85293e2"}, + {file = "pyarrow-5.0.0.tar.gz", hash = "sha256:24e64ea33eed07441cc0e80c949e3a1b48211a1add8953268391d250f4d39922"}, +] +pyparsing = [ + {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, + {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, +] +pytest = [ + {file = "pytest-7.1.2-py3-none-any.whl", hash = "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c"}, + {file = "pytest-7.1.2.tar.gz", hash = "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"}, +] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] +pytz = [ + {file = "pytz-2022.1-py2.py3-none-any.whl", hash = "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"}, + {file = "pytz-2022.1.tar.gz", hash = "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +thrift = [ + {file = "thrift-0.13.0.tar.gz", hash = "sha256:9af1c86bf73433afc6010ed376a6c6aca2b54099cc0d61895f640870a9ae7d89"}, +] +tomli = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] +typed-ast = [ + {file = "typed_ast-1.5.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ad3b48cf2b487be140072fb86feff36801487d4abb7382bb1929aaac80638ea"}, + {file = "typed_ast-1.5.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:542cd732351ba8235f20faa0fc7398946fe1a57f2cdb289e5497e1e7f48cfedb"}, + {file = "typed_ast-1.5.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5dc2c11ae59003d4a26dda637222d9ae924387f96acae9492df663843aefad55"}, + {file = "typed_ast-1.5.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fd5df1313915dbd70eaaa88c19030b441742e8b05e6103c631c83b75e0435ccc"}, + {file = "typed_ast-1.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:e34f9b9e61333ecb0f7d79c21c28aa5cd63bec15cb7e1310d7d3da6ce886bc9b"}, + {file = "typed_ast-1.5.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f818c5b81966d4728fec14caa338e30a70dfc3da577984d38f97816c4b3071ec"}, + {file = "typed_ast-1.5.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3042bfc9ca118712c9809201f55355479cfcdc17449f9f8db5e744e9625c6805"}, + {file = "typed_ast-1.5.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4fff9fdcce59dc61ec1b317bdb319f8f4e6b69ebbe61193ae0a60c5f9333dc49"}, + {file = "typed_ast-1.5.3-cp36-cp36m-win_amd64.whl", hash = "sha256:8e0b8528838ffd426fea8d18bde4c73bcb4167218998cc8b9ee0a0f2bfe678a6"}, + {file = "typed_ast-1.5.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8ef1d96ad05a291f5c36895d86d1375c0ee70595b90f6bb5f5fdbee749b146db"}, + {file = "typed_ast-1.5.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed44e81517364cb5ba367e4f68fca01fba42a7a4690d40c07886586ac267d9b9"}, + {file = "typed_ast-1.5.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f60d9de0d087454c91b3999a296d0c4558c1666771e3460621875021bf899af9"}, + {file = "typed_ast-1.5.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9e237e74fd321a55c90eee9bc5d44be976979ad38a29bbd734148295c1ce7617"}, + {file = "typed_ast-1.5.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ee852185964744987609b40aee1d2eb81502ae63ee8eef614558f96a56c1902d"}, + {file = "typed_ast-1.5.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:27e46cdd01d6c3a0dd8f728b6a938a6751f7bd324817501c15fb056307f918c6"}, + {file = "typed_ast-1.5.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d64dabc6336ddc10373922a146fa2256043b3b43e61f28961caec2a5207c56d5"}, + {file = "typed_ast-1.5.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8cdf91b0c466a6c43f36c1964772918a2c04cfa83df8001ff32a89e357f8eb06"}, + {file = "typed_ast-1.5.3-cp38-cp38-win_amd64.whl", hash = "sha256:9cc9e1457e1feb06b075c8ef8aeb046a28ec351b1958b42c7c31c989c841403a"}, + {file = "typed_ast-1.5.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e20d196815eeffb3d76b75223e8ffed124e65ee62097e4e73afb5fec6b993e7a"}, + {file = "typed_ast-1.5.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:37e5349d1d5de2f4763d534ccb26809d1c24b180a477659a12c4bde9dd677d74"}, + {file = "typed_ast-1.5.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9f1a27592fac87daa4e3f16538713d705599b0a27dfe25518b80b6b017f0a6d"}, + {file = "typed_ast-1.5.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8831479695eadc8b5ffed06fdfb3e424adc37962a75925668deeb503f446c0a3"}, + {file = "typed_ast-1.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:20d5118e494478ef2d3a2702d964dae830aedd7b4d3b626d003eea526be18718"}, + {file = "typed_ast-1.5.3.tar.gz", hash = "sha256:27f25232e2dd0edfe1f019d6bfaaf11e86e657d9bdb7b0956db95f560cceb2b3"}, +] +typing-extensions = [ + {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, + {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, +] +zipp = [ + {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, + {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, +] diff --git a/cmdexec/clients/python/pyproject.toml b/cmdexec/clients/python/pyproject.toml index e77f3626..a1926e3c 100644 --- a/cmdexec/clients/python/pyproject.toml +++ b/cmdexec/clients/python/pyproject.toml @@ -1,7 +1,30 @@ +[tool.poetry] +name = "databricks-sql-connector" +version = "2.0.2" +description = "Databricks SQL Connector for Python" +authors = ["Databricks "] +license = "Apache-2.0" +readme = "README.md" +packages = [{include = "databricks", from = "src"}] + +[tool.poetry.dependencies] +python = "^3.7.1" +thrift = "^0.13.0" +pyarrow = "^5.0.0" +pandas = "^1.3.0" + +[tool.poetry.dev-dependencies] +pytest = "^7.1.2" +mypy = "^0.950" +black = "^22.3.0" + [build-system] -requires = [ - "setuptools>=58", - "wheel", -] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.mypy] +ignore_missing_imports = "true" +exclude = ['ttypes\.py$', 'TCLIService\.py$'] -build-backend = "setuptools.build_meta" +[tool.black] +exclude = '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist|thrift_api)/' \ No newline at end of file diff --git a/cmdexec/clients/python/src/databricks/sql/__init__.py b/cmdexec/clients/python/src/databricks/sql/__init__.py index be141bc2..8f67d465 100644 --- a/cmdexec/clients/python/src/databricks/sql/__init__.py +++ b/cmdexec/clients/python/src/databricks/sql/__init__.py @@ -3,9 +3,9 @@ from databricks.sql.exc import * # PEP 249 module globals -apilevel = '2.0' +apilevel = "2.0" threadsafety = 1 # Threads may share the module, but not connections. -paramstyle = 'pyformat' # Python extended format codes, e.g. ...WHERE name=%(name)s +paramstyle = "pyformat" # Python extended format codes, e.g. ...WHERE name=%(name)s class DBAPITypeObject(object): @@ -19,12 +19,13 @@ def __repr__(self): return "DBAPITypeObject({})".format(self.values) -STRING = DBAPITypeObject('string') -BINARY = DBAPITypeObject('binary') -NUMBER = DBAPITypeObject('boolean', 'tinyint', 'smallint', 'int', 'bigint', 'float', 'double', - 'decimal') -DATETIME = DBAPITypeObject('timestamp') -DATE = DBAPITypeObject('date') +STRING = DBAPITypeObject("string") +BINARY = DBAPITypeObject("binary") +NUMBER = DBAPITypeObject( + "boolean", "tinyint", "smallint", "int", "bigint", "float", "double", "decimal" +) +DATETIME = DBAPITypeObject("timestamp") +DATE = DBAPITypeObject("date") ROWID = DBAPITypeObject() __version__ = "2.0.2" @@ -45,4 +46,5 @@ def TimestampFromTicks(ticks): def connect(server_hostname, http_path, access_token, **kwargs): from .client import Connection + return Connection(server_hostname, http_path, access_token, **kwargs) diff --git a/cmdexec/clients/python/src/databricks/sql/client.py b/cmdexec/clients/python/src/databricks/sql/client.py index 82557333..76fcc704 100644 --- a/cmdexec/clients/python/src/databricks/sql/client.py +++ b/cmdexec/clients/python/src/databricks/sql/client.py @@ -22,15 +22,17 @@ class Connection: - def __init__(self, - server_hostname: str, - http_path: str, - access_token: str, - http_headers: Optional[List[Tuple[str, str]]] = None, - session_configuration: Dict[str, Any] = None, - catalog: Optional[str] = None, - schema: Optional[str] = None, - **kwargs) -> None: + def __init__( + self, + server_hostname: str, + http_path: str, + access_token: str, + http_headers: Optional[List[Tuple[str, str]]] = None, + session_configuration: Dict[str, Any] = None, + catalog: Optional[str] = None, + schema: Optional[str] = None, + **kwargs + ) -> None: """ Connect to a Databricks SQL endpoint or a Databricks cluster. @@ -39,7 +41,7 @@ def __init__(self, or to a DBR interactive cluster (e.g. /sql/protocolv1/o/1234567890123456/1234-123456-slid123) :param access_token: Http Bearer access token, e.g. Databricks Personal Access Token. :param http_headers: An optional list of (k, v) pairs that will be set as Http headers on every request - :param session_configuration: An optional dictionary of Spark session parameters. Defaults to None. + :param session_configuration: An optional dictionary of Spark session parameters. Defaults to None. Execute the SQL command `SET -v` to get a full list of available commands. :param catalog: An optional initial catalog to use. Requires DBR version 9.0+ :param schema: An optional initial schema to use. Requires DBR version 9.0+ @@ -91,26 +93,42 @@ def __init__(self, authorization_header = [] if kwargs.get("_username") and kwargs.get("_password"): auth_credentials = "{username}:{password}".format( - username=kwargs.get("_username"), password=kwargs.get("_password")).encode("UTF-8") - auth_credentials_base64 = base64.standard_b64encode(auth_credentials).decode("UTF-8") - authorization_header = [("Authorization", "Basic {}".format(auth_credentials_base64))] + username=kwargs.get("_username"), password=kwargs.get("_password") + ).encode("UTF-8") + auth_credentials_base64 = base64.standard_b64encode( + auth_credentials + ).decode("UTF-8") + authorization_header = [ + ("Authorization", "Basic {}".format(auth_credentials_base64)) + ] elif access_token: authorization_header = [("Authorization", "Bearer {}".format(access_token))] - elif not (kwargs.get("_use_cert_as_auth") and kwargs.get("_tls_client_cert_file")): - raise ValueError("No valid authentication settings. Please provide an access token.") + elif not ( + kwargs.get("_use_cert_as_auth") and kwargs.get("_tls_client_cert_file") + ): + raise ValueError( + "No valid authentication settings. Please provide an access token." + ) if not kwargs.get("_user_agent_entry"): useragent_header = "{}/{}".format(USER_AGENT_NAME, __version__) else: - useragent_header = "{}/{} ({})".format(USER_AGENT_NAME, __version__, - kwargs.get("_user_agent_entry")) + useragent_header = "{}/{} ({})".format( + USER_AGENT_NAME, __version__, kwargs.get("_user_agent_entry") + ) base_headers = [("User-Agent", useragent_header)] + authorization_header - self.thrift_backend = ThriftBackend(self.host, self.port, http_path, - (http_headers or []) + base_headers, **kwargs) - - self._session_handle = self.thrift_backend.open_session(session_configuration, catalog, - schema) + self.thrift_backend = ThriftBackend( + self.host, + self.port, + http_path, + (http_headers or []) + base_headers, + **kwargs + ) + + self._session_handle = self.thrift_backend.open_session( + session_configuration, catalog, schema + ) self.open = True logger.info("Successfully opened session " + str(self.get_session_id())) self._cursors = [] # type: List[Cursor] @@ -123,8 +141,10 @@ def __exit__(self, exc_type, exc_value, traceback): def __del__(self): if self.open: - logger.debug("Closing unclosed connection for session " - "{}".format(self.get_session_id())) + logger.debug( + "Closing unclosed connection for session " + "{}".format(self.get_session_id()) + ) try: self._close(close_cursors=False) except OperationalError as e: @@ -134,9 +154,11 @@ def __del__(self): def get_session_id(self): return self.thrift_backend.handle_to_id(self._session_handle) - def cursor(self, - arraysize: int = DEFAULT_ARRAY_SIZE, - buffer_size_bytes: int = DEFAULT_RESULT_BUFFER_SIZE_BYTES) -> "Cursor": + def cursor( + self, + arraysize: int = DEFAULT_ARRAY_SIZE, + buffer_size_bytes: int = DEFAULT_RESULT_BUFFER_SIZE_BYTES, + ) -> "Cursor": """ Return a new Cursor object using the connection. @@ -149,7 +171,8 @@ def cursor(self, self, self.thrift_backend, arraysize=arraysize, - result_buffer_size_bytes=buffer_size_bytes) + result_buffer_size_bytes=buffer_size_bytes, + ) self._cursors.append(cursor) return cursor @@ -174,11 +197,13 @@ def rollback(self): class Cursor: - def __init__(self, - connection: Connection, - thrift_backend: ThriftBackend, - result_buffer_size_bytes: int = DEFAULT_RESULT_BUFFER_SIZE_BYTES, - arraysize: int = DEFAULT_ARRAY_SIZE) -> None: + def __init__( + self, + connection: Connection, + thrift_backend: ThriftBackend, + result_buffer_size_bytes: int = DEFAULT_RESULT_BUFFER_SIZE_BYTES, + arraysize: int = DEFAULT_ARRAY_SIZE, + ) -> None: """ These objects represent a database cursor, which is used to manage the context of a fetch operation. @@ -189,7 +214,7 @@ def __init__(self, self.connection = connection self.rowcount = -1 # Return -1 as this is not supported self.buffer_size_bytes = result_buffer_size_bytes - self.active_result_set = None + self.active_result_set[ResultSet, None] = None self.arraysize = arraysize # Note that Cursor closed => active result set closed, but not vice versa self.open = True @@ -223,7 +248,9 @@ def _check_not_closed(self): if not self.open: raise Error("Attempting operation on closed cursor") - def execute(self, operation: str, parameters: Optional[Dict[str, str]] = None) -> "Cursor": + def execute( + self, operation: str, parameters: Optional[Dict[str, str]] = None + ) -> "Cursor": """ Execute a query and wait for execution to complete. Parameters should be given in extended param format style: %(...). @@ -243,9 +270,15 @@ def execute(self, operation: str, parameters: Optional[Dict[str, str]] = None) - session_handle=self.connection._session_handle, max_rows=self.arraysize, max_bytes=self.buffer_size_bytes, - cursor=self) - self.active_result_set = ResultSet(self.connection, execute_response, self.thrift_backend, - self.buffer_size_bytes, self.arraysize) + cursor=self, + ) + self.active_result_set = ResultSet( + self.connection, + execute_response, + self.thrift_backend, + self.buffer_size_bytes, + self.arraysize, + ) return self def executemany(self, operation, seq_of_parameters): @@ -273,13 +306,20 @@ def catalogs(self) -> "Cursor": session_handle=self.connection._session_handle, max_rows=self.arraysize, max_bytes=self.buffer_size_bytes, - cursor=self) - self.active_result_set = ResultSet(self.connection, execute_response, self.thrift_backend, - self.buffer_size_bytes, self.arraysize) + cursor=self, + ) + self.active_result_set = ResultSet( + self.connection, + execute_response, + self.thrift_backend, + self.buffer_size_bytes, + self.arraysize, + ) return self - def schemas(self, catalog_name: Optional[str] = None, - schema_name: Optional[str] = None) -> "Cursor": + def schemas( + self, catalog_name: Optional[str] = None, schema_name: Optional[str] = None + ) -> "Cursor": """ Get schemas corresponding to the catalog_name and schema_name. @@ -294,16 +334,24 @@ def schemas(self, catalog_name: Optional[str] = None, max_bytes=self.buffer_size_bytes, cursor=self, catalog_name=catalog_name, - schema_name=schema_name) - self.active_result_set = ResultSet(self.connection, execute_response, self.thrift_backend, - self.buffer_size_bytes, self.arraysize) + schema_name=schema_name, + ) + self.active_result_set = ResultSet( + self.connection, + execute_response, + self.thrift_backend, + self.buffer_size_bytes, + self.arraysize, + ) return self - def tables(self, - catalog_name: Optional[str] = None, - schema_name: Optional[str] = None, - table_name: Optional[str] = None, - table_types: List[str] = None) -> "Cursor": + def tables( + self, + catalog_name: Optional[str] = None, + schema_name: Optional[str] = None, + table_name: Optional[str] = None, + table_types: List[str] = None, + ) -> "Cursor": """ Get tables corresponding to the catalog_name, schema_name and table_name. @@ -321,16 +369,24 @@ def tables(self, catalog_name=catalog_name, schema_name=schema_name, table_name=table_name, - table_types=table_types) - self.active_result_set = ResultSet(self.connection, execute_response, self.thrift_backend, - self.buffer_size_bytes, self.arraysize) + table_types=table_types, + ) + self.active_result_set = ResultSet( + self.connection, + execute_response, + self.thrift_backend, + self.buffer_size_bytes, + self.arraysize, + ) return self - def columns(self, - catalog_name: Optional[str] = None, - schema_name: Optional[str] = None, - table_name: Optional[str] = None, - column_name: Optional[str] = None) -> "Cursor": + def columns( + self, + catalog_name: Optional[str] = None, + schema_name: Optional[str] = None, + table_name: Optional[str] = None, + column_name: Optional[str] = None, + ) -> "Cursor": """ Get columns corresponding to the catalog_name, schema_name, table_name and column_name. @@ -348,9 +404,15 @@ def columns(self, catalog_name=catalog_name, schema_name=schema_name, table_name=table_name, - column_name=column_name) - self.active_result_set = ResultSet(self.connection, execute_response, self.thrift_backend, - self.buffer_size_bytes, self.arraysize) + column_name=column_name, + ) + self.active_result_set = ResultSet( + self.connection, + execute_response, + self.thrift_backend, + self.buffer_size_bytes, + self.arraysize, + ) return self def fetchall(self) -> List[Row]: @@ -426,8 +488,10 @@ def cancel(self) -> None: if self.active_op_handle is not None: self.thrift_backend.cancel_command(self.active_op_handle) else: - logger.warning("Attempting to cancel a command, but there is no " - "currently executing command") + logger.warning( + "Attempting to cancel a command, but there is no " + "currently executing command" + ) def close(self) -> None: """Close cursor""" @@ -480,12 +544,14 @@ def setoutputsize(self, size, column=None): class ResultSet: - def __init__(self, - connection: Connection, - execute_response: ExecuteResponse, - thrift_backend: ThriftBackend, - result_buffer_size_bytes: int = DEFAULT_RESULT_BUFFER_SIZE_BYTES, - arraysize: int = 10000): + def __init__( + self, + connection: Connection, + execute_response: ExecuteResponse, + thrift_backend: ThriftBackend, + result_buffer_size_bytes: int = DEFAULT_RESULT_BUFFER_SIZE_BYTES, + arraysize: int = 10000, + ): """ A ResultSet manages the results of a single command. @@ -529,7 +595,8 @@ def _fill_results_buffer(self): max_bytes=self.buffer_size_bytes, expected_row_start_offset=self._next_row_index, arrow_schema_bytes=self._arrow_schema_bytes, - description=self.description) + description=self.description, + ) self.results = results self.has_more_rows = has_more_rows @@ -538,7 +605,9 @@ def _convert_arrow_table(self, table): ResultRow = Row(*column_names) if self.connection.disable_pandas is True: - return [ResultRow(*[v.as_py() for v in r]) for r in zip(*table.itercolumns())] + return [ + ResultRow(*[v.as_py() for v in r]) for r in zip(*table.itercolumns()) + ] # Need to use nullable types, as otherwise type can change when there are missing values. # See https://arrow.apache.org/docs/python/pandas.html#nullable-types @@ -561,7 +630,10 @@ def _convert_arrow_table(self, table): # Need to rename columns, as the to_pandas function cannot handle duplicate column names table_renamed = table.rename_columns([str(c) for c in range(table.num_columns)]) df = table_renamed.to_pandas( - types_mapper=dtype_mapping.get, date_as_object=True, timestamp_as_object=True) + types_mapper=dtype_mapping.get, + date_as_object=True, + timestamp_as_object=True, + ) res = df.to_numpy(na_value=None) return [ResultRow(*v) for v in res] @@ -582,7 +654,11 @@ def fetchmany_arrow(self, size: int) -> pyarrow.Table: n_remaining_rows = size - results.num_rows self._next_row_index += results.num_rows - while n_remaining_rows > 0 and not self.has_been_closed_server_side and self.has_more_rows: + while ( + n_remaining_rows > 0 + and not self.has_been_closed_server_side + and self.has_more_rows + ): self._fill_results_buffer() partial_results = self.results.next_n_rows(n_remaining_rows) results = pyarrow.concat_tables([results, partial_results]) @@ -637,8 +713,11 @@ def close(self) -> None: been closed on the server for some other reason, issue a request to the server to close it. """ try: - if self.op_state != self.thrift_backend.CLOSED_OP_STATE and not self.has_been_closed_server_side \ - and self.connection.open: + if ( + self.op_state != self.thrift_backend.CLOSED_OP_STATE + and not self.has_been_closed_server_side + and self.connection.open + ): self.thrift_backend.close_command(self.command_id) finally: self.has_been_closed_server_side = True @@ -651,10 +730,12 @@ def _get_schema_description(table_schema_message): """ def map_col_type(type_): - if type_.startswith('decimal'): - return 'decimal' + if type_.startswith("decimal"): + return "decimal" else: return type_ - return [(column.name, map_col_type(column.datatype), None, None, None, None, None) - for column in table_schema_message.columns] + return [ + (column.name, map_col_type(column.datatype), None, None, None, None, None) + for column in table_schema_message.columns + ] diff --git a/cmdexec/clients/python/src/databricks/sql/exc.py b/cmdexec/clients/python/src/databricks/sql/exc.py index 66f5b747..bb1e203e 100644 --- a/cmdexec/clients/python/src/databricks/sql/exc.py +++ b/cmdexec/clients/python/src/databricks/sql/exc.py @@ -62,6 +62,7 @@ class NotSupportedError(DatabaseError): ### Custom error classes ### class InvalidServerResponseError(OperationalError): """Thrown if the server does not set the initial namespace correctly""" + pass @@ -72,6 +73,7 @@ class ServerOperationError(DatabaseError): "diagnostic-info": The full Spark stack trace (if available) "operation-id": The Thrift ID of the operation """ + pass @@ -89,4 +91,5 @@ class RequestError(OperationalError): "attempt": current retry number / maximum number of retries "elapsed-seconds": time that has elapsed since first attempting the RPC request """ + pass diff --git a/cmdexec/clients/python/src/databricks/sql/thrift_backend.py b/cmdexec/clients/python/src/databricks/sql/thrift_backend.py index 9f62dc1d..d812f93b 100644 --- a/cmdexec/clients/python/src/databricks/sql/thrift_backend.py +++ b/cmdexec/clients/python/src/databricks/sql/thrift_backend.py @@ -15,7 +15,13 @@ from databricks.sql.thrift_api.TCLIService import TCLIService, ttypes from databricks.sql import * -from databricks.sql.utils import ArrowQueue, ExecuteResponse, _bound, RequestErrorInfo, NoRetryReason +from databricks.sql.utils import ( + ArrowQueue, + ExecuteResponse, + _bound, + RequestErrorInfo, + NoRetryReason, +) logger = logging.getLogger(__name__) @@ -28,11 +34,11 @@ # see Connection.__init__ for parameter descriptions. # - Min/Max avoids unsustainable configs (sane values are far more constrained) # - 900s attempts-duration lines up w ODBC/JDBC drivers (for cluster startup > 10 mins) -_retry_policy = { # (type, default, min, max) - "_retry_delay_min": (float, 1, 0.1, 60), - "_retry_delay_max": (float, 60, 5, 3600), - "_retry_stop_after_attempts_count": (int, 30, 1, 60), - "_retry_stop_after_attempts_duration": (float, 900, 1, 86400), +_retry_policy = { # (type, default, min, max) + "_retry_delay_min": (float, 1, 0.1, 60), + "_retry_delay_max": (float, 60, 5, 3600), + "_retry_stop_after_attempts_count": (int, 30, 1, 60), + "_retry_stop_after_attempts_duration": (float, 900, 1, 86400), } @@ -41,7 +47,9 @@ class ThriftBackend: ERROR_OP_STATE = ttypes.TOperationState.ERROR_STATE BIT_MASKS = [1, 2, 4, 8, 16, 32, 64, 128] - def __init__(self, server_hostname: str, port, http_path: str, http_headers, **kwargs): + def __init__( + self, server_hostname: str, port, http_path: str, http_headers, **kwargs + ): # Internal arguments in **kwargs: # _user_agent_entry # Tag to add to User-Agent header. For use by partners. @@ -82,14 +90,19 @@ def __init__(self, server_hostname: str, port, http_path: str, http_headers, **k uri = kwargs.get("_connection_uri") elif server_hostname and http_path: uri = "https://{host}:{port}/{path}".format( - host=server_hostname, port=port, path=http_path.lstrip("/")) + host=server_hostname, port=port, path=http_path.lstrip("/") + ) else: raise ValueError("No valid connection settings.") self._initialize_retry_args(kwargs) - self._use_arrow_native_complex_types = kwargs.get("_use_arrow_native_complex_types", True) + self._use_arrow_native_complex_types = kwargs.get( + "_use_arrow_native_complex_types", True + ) self._use_arrow_native_decimals = kwargs.get("_use_arrow_native_decimals", True) - self._use_arrow_native_timestamps = kwargs.get("_use_arrow_native_timestamps", True) + self._use_arrow_native_timestamps = kwargs.get( + "_use_arrow_native_timestamps", True + ) # Configure tls context ssl_context = create_default_context(cafile=kwargs.get("_tls_trusted_ca_file")) @@ -110,7 +123,8 @@ def __init__(self, server_hostname: str, port, http_path: str, http_headers, **k ssl_context.load_cert_chain( certfile=tls_client_cert_file, keyfile=tls_client_cert_key_file, - password=tls_client_cert_key_password) + password=tls_client_cert_key_password, + ) self._transport = thrift.transport.THttpClient.THttpClient( uri_or_host=uri, @@ -140,22 +154,34 @@ def _initialize_retry_args(self, kwargs): given_or_default = type_(kwargs.get(key, default)) bound = _bound(min, max, given_or_default) setattr(self, key, bound) - logger.debug('retry parameter: {} given_or_default {}'.format(key, given_or_default)) + logger.debug( + "retry parameter: {} given_or_default {}".format(key, given_or_default) + ) if bound != given_or_default: - logger.warn('Override out of policy retry parameter: ' + - '{} given {}, restricted to {}'.format(key, given_or_default, bound)) + logger.warn( + "Override out of policy retry parameter: " + + "{} given {}, restricted to {}".format( + key, given_or_default, bound + ) + ) # Fail on retry delay min > max; consider later adding fail on min > duration? - if self._retry_stop_after_attempts_count > 1 \ - and self._retry_delay_min > self._retry_delay_max: + if ( + self._retry_stop_after_attempts_count > 1 + and self._retry_delay_min > self._retry_delay_max + ): raise ValueError( "Invalid configuration enables retries with retry delay min(={}) > max(={})".format( - self._retry_delay_min, self._retry_delay_max)) + self._retry_delay_min, self._retry_delay_max + ) + ) @staticmethod def _check_response_for_error(response): - if response.status and response.status.statusCode in \ - [ttypes.TStatusCode.ERROR_STATUS, ttypes.TStatusCode.INVALID_HANDLE_STATUS]: + if response.status and response.status.statusCode in [ + ttypes.TStatusCode.ERROR_STATUS, + ttypes.TStatusCode.INVALID_HANDLE_STATUS, + ]: raise DatabaseError(response.status.errorMessage) @staticmethod @@ -164,9 +190,12 @@ def _extract_error_message_from_headers(headers): if THRIFT_ERROR_MESSAGE_HEADER in headers: err_msg = headers[THRIFT_ERROR_MESSAGE_HEADER] if DATABRICKS_ERROR_OR_REDIRECT_HEADER in headers: - if err_msg: # We don't expect both to be set, but log both here just in case + if ( + err_msg + ): # We don't expect both to be set, but log both here just in case err_msg = "Thriftserver error: {}, Databricks error: {}".format( - err_msg, headers[DATABRICKS_ERROR_OR_REDIRECT_HEADER]) + err_msg, headers[DATABRICKS_ERROR_OR_REDIRECT_HEADER] + ) else: err_msg = headers[DATABRICKS_ERROR_OR_REDIRECT_HEADER] if DATABRICKS_REASON_HEADER in headers: @@ -177,7 +206,10 @@ def _handle_request_error(self, error_info, attempt, elapsed): max_attempts = self._retry_stop_after_attempts_count max_duration_s = self._retry_stop_after_attempts_duration - if error_info.retry_delay is not None and elapsed + error_info.retry_delay > max_duration_s: + if ( + error_info.retry_delay is not None + and elapsed + error_info.retry_delay > max_duration_s + ): no_retry_reason = NoRetryReason.OUT_OF_TIME elif error_info.retry_delay is not None and attempt >= max_attempts: no_retry_reason = NoRetryReason.OUT_OF_ATTEMPTS @@ -187,19 +219,25 @@ def _handle_request_error(self, error_info, attempt, elapsed): no_retry_reason = None full_error_info_context = error_info.full_info_logging_context( - no_retry_reason, attempt, max_attempts, elapsed, max_duration_s) + no_retry_reason, attempt, max_attempts, elapsed, max_duration_s + ) if no_retry_reason is not None: user_friendly_error_message = error_info.user_friendly_error_message( - no_retry_reason, attempt, elapsed) - network_request_error = RequestError(user_friendly_error_message, - full_error_info_context, error_info.error) + no_retry_reason, attempt, elapsed + ) + network_request_error = RequestError( + user_friendly_error_message, full_error_info_context, error_info.error + ) logger.info(network_request_error.message_with_context()) raise network_request_error - logger.info("Retrying request after error in {} seconds: {}".format( - error_info.retry_delay, full_error_info_context)) + logger.info( + "Retrying request after error in {} seconds: {}".format( + error_info.retry_delay, full_error_info_context + ) + ) time.sleep(error_info.retry_delay) # FUTURE: Consider moving to https://github.com/litl/backoff or @@ -249,14 +287,16 @@ def attempt_request(attempt): except Exception as error: retry_delay = extract_retry_delay(attempt) error_message = ThriftBackend._extract_error_message_from_headers( - getattr(self._transport, "headers", {})) + getattr(self._transport, "headers", {}) + ) return RequestErrorInfo( error=error, error_message=error_message, retry_delay=retry_delay, http_code=getattr(self._transport, "code", None), method=method.__name__, - request=request) + request=request, + ) # The real work: # - for each available attempt: @@ -290,45 +330,61 @@ def _check_protocol_version(self, t_open_session_resp): protocol_version = t_open_session_resp.serverProtocolVersion if protocol_version < ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V2: - raise OperationalError("Error: expected server to use a protocol version >= " - "SPARK_CLI_SERVICE_PROTOCOL_V2, " - "instead got: {}".format(protocol_version)) + raise OperationalError( + "Error: expected server to use a protocol version >= " + "SPARK_CLI_SERVICE_PROTOCOL_V2, " + "instead got: {}".format(protocol_version) + ) def _check_initial_namespace(self, catalog, schema, response): if not (catalog or schema): return - if response.serverProtocolVersion < \ - ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V4: + if ( + response.serverProtocolVersion + < ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V4 + ): raise InvalidServerResponseError( "Setting initial namespace not supported by the DBR version, " - "Please use a Databricks SQL endpoint or a cluster with DBR >= 9.0.") + "Please use a Databricks SQL endpoint or a cluster with DBR >= 9.0." + ) if catalog: if not response.canUseMultipleCatalogs: raise InvalidServerResponseError( - "Unexpected response from server: Trying to set initial catalog to {}, " + - "but server does not support multiple catalogs.".format(catalog)) + "Unexpected response from server: Trying to set initial catalog to {}, " + + "but server does not support multiple catalogs.".format(catalog) # type: ignore + ) def _check_session_configuration(self, session_configuration): # This client expects timetampsAsString to be false, so we do not allow users to modify that - if session_configuration.get(TIMESTAMP_AS_STRING_CONFIG, "false").lower() != "false": - raise Error("Invalid session configuration: {} cannot be changed " - "while using the Databricks SQL connector, it must be false not {}".format( - TIMESTAMP_AS_STRING_CONFIG, - session_configuration[TIMESTAMP_AS_STRING_CONFIG])) + if ( + session_configuration.get(TIMESTAMP_AS_STRING_CONFIG, "false").lower() + != "false" + ): + raise Error( + "Invalid session configuration: {} cannot be changed " + "while using the Databricks SQL connector, it must be false not {}".format( + TIMESTAMP_AS_STRING_CONFIG, + session_configuration[TIMESTAMP_AS_STRING_CONFIG], + ) + ) def open_session(self, session_configuration, catalog, schema): try: self._transport.open() - session_configuration = {k: str(v) for (k, v) in (session_configuration or {}).items()} + session_configuration = { + k: str(v) for (k, v) in (session_configuration or {}).items() + } self._check_session_configuration(session_configuration) # We want to receive proper Timestamp arrow types. # We set it also in confOverlay in TExecuteStatementReq on a per query basic, # but it doesn't hurt to also set for the whole session. session_configuration[TIMESTAMP_AS_STRING_CONFIG] = "false" if catalog or schema: - initial_namespace = ttypes.TNamespace(catalogName=catalog, schemaName=schema) + initial_namespace = ttypes.TNamespace( + catalogName=catalog, schemaName=schema + ) else: initial_namespace = None @@ -337,7 +393,8 @@ def open_session(self, session_configuration, catalog, schema): client_protocol=None, initialNamespace=initial_namespace, canUseMultipleCatalogs=True, - configuration=session_configuration) + configuration=session_configuration, + ) response = self.make_request(self._client.OpenSession, open_session_req) self._check_initial_namespace(catalog, schema, response) self._check_protocol_version(response) @@ -353,24 +410,33 @@ def close_session(self, session_handle) -> None: finally: self._transport.close() - def _check_command_not_in_error_or_closed_state(self, op_handle, get_operations_resp): + def _check_command_not_in_error_or_closed_state( + self, op_handle, get_operations_resp + ): if get_operations_resp.operationState == ttypes.TOperationState.ERROR_STATE: if get_operations_resp.displayMessage: raise ServerOperationError( - get_operations_resp.displayMessage, { + get_operations_resp.displayMessage, + { "operation-id": op_handle and op_handle.operationId.guid, - "diagnostic-info": get_operations_resp.diagnosticInfo - }) + "diagnostic-info": get_operations_resp.diagnosticInfo, + }, + ) else: - raise ServerOperationError(get_operations_resp.errorMessage, { - "operation-id": op_handle and op_handle.operationId.guid, - "diagnostic-info": None - }) + raise ServerOperationError( + get_operations_resp.errorMessage, + { + "operation-id": op_handle and op_handle.operationId.guid, + "diagnostic-info": None, + }, + ) elif get_operations_resp.operationState == ttypes.TOperationState.CLOSED_STATE: raise DatabaseError( - "Command {} unexpectedly closed server side".format(op_handle and - op_handle.operationId.guid), - {"operation-id": op_handle and op_handle.operationId.guid}) + "Command {} unexpectedly closed server side".format( + op_handle and op_handle.operationId.guid + ), + {"operation-id": op_handle and op_handle.operationId.guid}, + ) def _poll_for_status(self, op_handle): req = ttypes.TGetOperationStatusReq( @@ -381,11 +447,19 @@ def _poll_for_status(self, op_handle): def _create_arrow_table(self, t_row_set, schema_bytes, description): if t_row_set.columns is not None: - arrow_table, num_rows = ThriftBackend._convert_column_based_set_to_arrow_table( - t_row_set.columns, description) + ( + arrow_table, + num_rows, + ) = ThriftBackend._convert_column_based_set_to_arrow_table( + t_row_set.columns, description + ) elif t_row_set.arrowBatches is not None: - arrow_table, num_rows = ThriftBackend._convert_arrow_based_set_to_arrow_table( - t_row_set.arrowBatches, schema_bytes) + ( + arrow_table, + num_rows, + ) = ThriftBackend._convert_arrow_based_set_to_arrow_table( + t_row_set.arrowBatches, schema_bytes + ) else: raise OperationalError("Unsupported TRowSet instance {}".format(t_row_set)) return self._convert_decimals_in_arrow_table(arrow_table, description), num_rows @@ -393,8 +467,10 @@ def _create_arrow_table(self, t_row_set, schema_bytes, description): @staticmethod def _convert_decimals_in_arrow_table(table, description): for (i, col) in enumerate(table.itercolumns()): - if description[i][1] == 'decimal': - decimal_col = col.to_pandas().apply(lambda v: v if v is None else Decimal(v)) + if description[i][1] == "decimal": + decimal_col = col.to_pandas().apply( + lambda v: v if v is None else Decimal(v) + ) precision, scale = description[i][4], description[i][5] assert scale is not None assert precision is not None @@ -424,7 +500,8 @@ def _convert_column_based_set_to_arrow_table(columns, description): # Only use the column names from the schema, the types are determined by the # physical types used in column based set, as they can differ from the # mapping used in _hive_schema_to_arrow_schema. - names=[c[0] for c in description]) + names=[c[0] for c in description], + ) return arrow_table, arrow_table.num_rows @staticmethod @@ -442,12 +519,14 @@ def _convert_column_to_arrow_array(t_col): "i64Val": pyarrow.int64(), "doubleVal": pyarrow.float64(), "stringVal": pyarrow.string(), - "binaryVal": pyarrow.binary() + "binaryVal": pyarrow.binary(), } for field in field_name_to_arrow_type.keys(): wrapper = getattr(t_col, field) if wrapper: - return ThriftBackend._create_arrow_array(wrapper, field_name_to_arrow_type[field]) + return ThriftBackend._create_arrow_array( + wrapper, field_name_to_arrow_type[field] + ) raise OperationalError("Empty TColumn instance {}".format(t_col)) @@ -484,7 +563,7 @@ def map_type(t_type_entry): ttypes.TTypeId.FLOAT_TYPE: pyarrow.float32(), ttypes.TTypeId.DOUBLE_TYPE: pyarrow.float64(), ttypes.TTypeId.STRING_TYPE: pyarrow.string(), - ttypes.TTypeId.TIMESTAMP_TYPE: pyarrow.timestamp('us', None), + ttypes.TTypeId.TIMESTAMP_TYPE: pyarrow.timestamp("us", None), ttypes.TTypeId.BINARY_TYPE: pyarrow.binary(), ttypes.TTypeId.ARRAY_TYPE: pyarrow.string(), ttypes.TTypeId.MAP_TYPE: pyarrow.string(), @@ -497,16 +576,19 @@ def map_type(t_type_entry): ttypes.TTypeId.VARCHAR_TYPE: pyarrow.string(), ttypes.TTypeId.CHAR_TYPE: pyarrow.string(), ttypes.TTypeId.INTERVAL_YEAR_MONTH_TYPE: pyarrow.string(), - ttypes.TTypeId.INTERVAL_DAY_TIME_TYPE: pyarrow.string() + ttypes.TTypeId.INTERVAL_DAY_TIME_TYPE: pyarrow.string(), }[t_type_entry.primitiveEntry.type] else: # Current thriftserver implementation should always return a primitiveEntry, # even for complex types - raise OperationalError("Thrift protocol error: t_type_entry not a primitiveEntry") + raise OperationalError( + "Thrift protocol error: t_type_entry not a primitiveEntry" + ) def convert_col(t_column_desc): - return pyarrow.field(t_column_desc.columnName, - map_type(t_column_desc.typeDesc.types[0])) + return pyarrow.field( + t_column_desc.columnName, map_type(t_column_desc.typeDesc.types[0]) + ) return pyarrow.schema([convert_col(col) for col in t_table_schema.columns]) @@ -519,16 +601,22 @@ def _col_to_description(col): # Drop _TYPE suffix cleaned_type = (name[:-5] if name.endswith("_TYPE") else name).lower() else: - raise OperationalError("Thrift protocol error: t_type_entry not a primitiveEntry") + raise OperationalError( + "Thrift protocol error: t_type_entry not a primitiveEntry" + ) if type_entry.primitiveEntry.type == ttypes.TTypeId.DECIMAL_TYPE: qualifiers = type_entry.primitiveEntry.typeQualifiers.qualifiers if qualifiers and "precision" in qualifiers and "scale" in qualifiers: - precision, scale = qualifiers["precision"].i32Value, qualifiers["scale"].i32Value + precision, scale = ( + qualifiers["precision"].i32Value, + qualifiers["scale"].i32Value, + ) else: raise OperationalError( "Decimal type did not provide typeQualifier precision, scale in " - "primitiveEntry {}".format(type_entry.primitiveEntry)) + "primitiveEntry {}".format(type_entry.primitiveEntry) + ) else: precision, scale = None, None @@ -536,7 +624,9 @@ def _col_to_description(col): @staticmethod def _hive_schema_to_description(t_table_schema): - return [ThriftBackend._col_to_description(col) for col in t_table_schema.columns] + return [ + ThriftBackend._col_to_description(col) for col in t_table_schema.columns + ] def _results_message_to_execute_response(self, resp, operation_state): if resp.directResults and resp.directResults.resultSetMetadata: @@ -545,26 +635,41 @@ def _results_message_to_execute_response(self, resp, operation_state): t_result_set_metadata_resp = self._get_metadata_resp(resp.operationHandle) if t_result_set_metadata_resp.resultFormat not in [ - ttypes.TSparkRowSetType.ARROW_BASED_SET, ttypes.TSparkRowSetType.COLUMN_BASED_SET + ttypes.TSparkRowSetType.ARROW_BASED_SET, + ttypes.TSparkRowSetType.COLUMN_BASED_SET, ]: - raise OperationalError("Expected results to be in Arrow or column based format, " - "instead they are: {}".format( - ttypes.TSparkRowSetType._VALUES_TO_NAMES[ - t_result_set_metadata_resp.resultFormat])) + raise OperationalError( + "Expected results to be in Arrow or column based format, " + "instead they are: {}".format( + ttypes.TSparkRowSetType._VALUES_TO_NAMES[ + t_result_set_metadata_resp.resultFormat + ] + ) + ) direct_results = resp.directResults has_been_closed_server_side = direct_results and direct_results.closeOperation - has_more_rows = (not direct_results) or (not direct_results.resultSet) \ - or direct_results.resultSet.hasMoreRows - description = self._hive_schema_to_description(t_result_set_metadata_resp.schema) - schema_bytes = (t_result_set_metadata_resp.arrowSchema or self._hive_schema_to_arrow_schema( - t_result_set_metadata_resp.schema).serialize().to_pybytes()) + has_more_rows = ( + (not direct_results) + or (not direct_results.resultSet) + or direct_results.resultSet.hasMoreRows + ) + description = self._hive_schema_to_description( + t_result_set_metadata_resp.schema + ) + schema_bytes = ( + t_result_set_metadata_resp.arrowSchema + or self._hive_schema_to_arrow_schema(t_result_set_metadata_resp.schema) + .serialize() + .to_pybytes() + ) if direct_results and direct_results.resultSet: - assert (direct_results.resultSet.results.startRowOffset == 0) - assert (direct_results.resultSetMetadata) - arrow_results, n_rows = self._create_arrow_table(direct_results.resultSet.results, - schema_bytes, description) + assert direct_results.resultSet.results.startRowOffset == 0 + assert direct_results.resultSetMetadata + arrow_results, n_rows = self._create_arrow_table( + direct_results.resultSet.results, schema_bytes, description + ) arrow_queue_opt = ArrowQueue(arrow_results, n_rows, 0) else: arrow_queue_opt = None @@ -575,15 +680,21 @@ def _results_message_to_execute_response(self, resp, operation_state): has_more_rows=has_more_rows, command_handle=resp.operationHandle, description=description, - arrow_schema_bytes=schema_bytes) + arrow_schema_bytes=schema_bytes, + ) def _wait_until_command_done(self, op_handle, initial_operation_status_resp): if initial_operation_status_resp: - self._check_command_not_in_error_or_closed_state(op_handle, - initial_operation_status_resp) - operation_state = initial_operation_status_resp and initial_operation_status_resp.operationState + self._check_command_not_in_error_or_closed_state( + op_handle, initial_operation_status_resp + ) + operation_state = ( + initial_operation_status_resp + and initial_operation_status_resp.operationState + ) while not operation_state or operation_state in [ - ttypes.TOperationState.RUNNING_STATE, ttypes.TOperationState.PENDING_STATE + ttypes.TOperationState.RUNNING_STATE, + ttypes.TOperationState.PENDING_STATE, ]: poll_resp = self._poll_for_status(op_handle) operation_state = poll_resp.operationState @@ -594,16 +705,24 @@ def _wait_until_command_done(self, op_handle, initial_operation_status_resp): def _check_direct_results_for_error(t_spark_direct_results): if t_spark_direct_results: if t_spark_direct_results.operationStatus: - ThriftBackend._check_response_for_error(t_spark_direct_results.operationStatus) + ThriftBackend._check_response_for_error( + t_spark_direct_results.operationStatus + ) if t_spark_direct_results.resultSetMetadata: - ThriftBackend._check_response_for_error(t_spark_direct_results.resultSetMetadata) + ThriftBackend._check_response_for_error( + t_spark_direct_results.resultSetMetadata + ) if t_spark_direct_results.resultSet: - ThriftBackend._check_response_for_error(t_spark_direct_results.resultSet) + ThriftBackend._check_response_for_error( + t_spark_direct_results.resultSet + ) if t_spark_direct_results.closeOperation: - ThriftBackend._check_response_for_error(t_spark_direct_results.closeOperation) + ThriftBackend._check_response_for_error( + t_spark_direct_results.closeOperation + ) def execute_command(self, operation, session_handle, max_rows, max_bytes, cursor): - assert (session_handle is not None) + assert session_handle is not None spark_arrow_types = ttypes.TSparkArrowTypes( timestampAsArrow=self._use_arrow_native_timestamps, @@ -611,12 +730,15 @@ def execute_command(self, operation, session_handle, max_rows, max_bytes, cursor complexTypesAsArrow=self._use_arrow_native_complex_types, # TODO: The current Arrow type used for intervals can not be deserialised in PyArrow # DBR should be changed to use month_day_nano_interval - intervalTypesAsArrow=False) + intervalTypesAsArrow=False, + ) req = ttypes.TExecuteStatementReq( sessionHandle=session_handle, statement=operation, runAsync=True, - getDirectResults=ttypes.TSparkGetDirectResults(maxRows=max_rows, maxBytes=max_bytes), + getDirectResults=ttypes.TSparkGetDirectResults( + maxRows=max_rows, maxBytes=max_bytes + ), canReadArrowResult=True, canDecompressLZ4Result=False, canDownloadResult=False, @@ -624,76 +746,94 @@ def execute_command(self, operation, session_handle, max_rows, max_bytes, cursor # We want to receive proper Timestamp arrow types. "spark.thriftserver.arrowBasedRowSet.timestampAsString": "false" }, - useArrowNativeTypes=spark_arrow_types) + useArrowNativeTypes=spark_arrow_types, + ) resp = self.make_request(self._client.ExecuteStatement, req) return self._handle_execute_response(resp, cursor) def get_catalogs(self, session_handle, max_rows, max_bytes, cursor): - assert (session_handle is not None) + assert session_handle is not None req = ttypes.TGetCatalogsReq( sessionHandle=session_handle, - getDirectResults=ttypes.TSparkGetDirectResults(maxRows=max_rows, maxBytes=max_bytes)) + getDirectResults=ttypes.TSparkGetDirectResults( + maxRows=max_rows, maxBytes=max_bytes + ), + ) resp = self.make_request(self._client.GetCatalogs, req) return self._handle_execute_response(resp, cursor) - def get_schemas(self, - session_handle, - max_rows, - max_bytes, - cursor, - catalog_name=None, - schema_name=None): - assert (session_handle is not None) + def get_schemas( + self, + session_handle, + max_rows, + max_bytes, + cursor, + catalog_name=None, + schema_name=None, + ): + assert session_handle is not None req = ttypes.TGetSchemasReq( sessionHandle=session_handle, - getDirectResults=ttypes.TSparkGetDirectResults(maxRows=max_rows, maxBytes=max_bytes), + getDirectResults=ttypes.TSparkGetDirectResults( + maxRows=max_rows, maxBytes=max_bytes + ), catalogName=catalog_name, schemaName=schema_name, ) resp = self.make_request(self._client.GetSchemas, req) return self._handle_execute_response(resp, cursor) - def get_tables(self, - session_handle, - max_rows, - max_bytes, - cursor, - catalog_name=None, - schema_name=None, - table_name=None, - table_types=None): - assert (session_handle is not None) + def get_tables( + self, + session_handle, + max_rows, + max_bytes, + cursor, + catalog_name=None, + schema_name=None, + table_name=None, + table_types=None, + ): + assert session_handle is not None req = ttypes.TGetTablesReq( sessionHandle=session_handle, - getDirectResults=ttypes.TSparkGetDirectResults(maxRows=max_rows, maxBytes=max_bytes), + getDirectResults=ttypes.TSparkGetDirectResults( + maxRows=max_rows, maxBytes=max_bytes + ), catalogName=catalog_name, schemaName=schema_name, tableName=table_name, - tableTypes=table_types) + tableTypes=table_types, + ) resp = self.make_request(self._client.GetTables, req) return self._handle_execute_response(resp, cursor) - def get_columns(self, - session_handle, - max_rows, - max_bytes, - cursor, - catalog_name=None, - schema_name=None, - table_name=None, - column_name=None): - assert (session_handle is not None) + def get_columns( + self, + session_handle, + max_rows, + max_bytes, + cursor, + catalog_name=None, + schema_name=None, + table_name=None, + column_name=None, + ): + assert session_handle is not None req = ttypes.TGetColumnsReq( sessionHandle=session_handle, - getDirectResults=ttypes.TSparkGetDirectResults(maxRows=max_rows, maxBytes=max_bytes), + getDirectResults=ttypes.TSparkGetDirectResults( + maxRows=max_rows, maxBytes=max_bytes + ), catalogName=catalog_name, schemaName=schema_name, tableName=table_name, - columnName=column_name) + columnName=column_name, + ) resp = self.make_request(self._client.GetColumns, req) return self._handle_execute_response(resp, cursor) @@ -702,13 +842,22 @@ def _handle_execute_response(self, resp, cursor): self._check_direct_results_for_error(resp.directResults) final_operation_state = self._wait_until_command_done( - resp.operationHandle, resp.directResults and resp.directResults.operationStatus) + resp.operationHandle, + resp.directResults and resp.directResults.operationStatus, + ) return self._results_message_to_execute_response(resp, final_operation_state) - def fetch_results(self, op_handle, max_rows, max_bytes, expected_row_start_offset, - arrow_schema_bytes, description): - assert (op_handle is not None) + def fetch_results( + self, + op_handle, + max_rows, + max_bytes, + expected_row_start_offset, + arrow_schema_bytes, + description, + ): + assert op_handle is not None req = ttypes.TFetchResultsReq( operationHandle=ttypes.TOperationHandle( @@ -719,14 +868,19 @@ def fetch_results(self, op_handle, max_rows, max_bytes, expected_row_start_offse ), maxRows=max_rows, maxBytes=max_bytes, - orientation=ttypes.TFetchOrientation.FETCH_NEXT) + orientation=ttypes.TFetchOrientation.FETCH_NEXT, + ) resp = self.make_request(self._client.FetchResults, req) if resp.results.startRowOffset > expected_row_start_offset: - logger.warning("Expected results to start from {} but they instead start at {}".format( - expected_row_start_offset, resp.results.startRowOffset)) - arrow_results, n_rows = self._create_arrow_table(resp.results, arrow_schema_bytes, - description) + logger.warning( + "Expected results to start from {} but they instead start at {}".format( + expected_row_start_offset, resp.results.startRowOffset + ) + ) + arrow_results, n_rows = self._create_arrow_table( + resp.results, arrow_schema_bytes, description + ) arrow_queue = ArrowQueue(arrow_results, n_rows) return arrow_queue, resp.hasMoreRows diff --git a/cmdexec/clients/python/src/databricks/sql/types.py b/cmdexec/clients/python/src/databricks/sql/types.py index 1a1b93a8..b44704cd 100644 --- a/cmdexec/clients/python/src/databricks/sql/types.py +++ b/cmdexec/clients/python/src/databricks/sql/types.py @@ -16,7 +16,7 @@ # # Row class was taken from Apache Spark pyspark. -from typing import (Any, Dict, List, Optional, Tuple, Union) +from typing import Any, Dict, List, Optional, Tuple, Union class Row(tuple): @@ -137,8 +137,10 @@ def __contains__(self, item: Any) -> bool: def __call__(self, *args: Any) -> "Row": """create new Row object""" if len(args) > len(self): - raise ValueError("Can not create Row with fields %s, expected %d values " - "but got %s" % (self, len(self), args)) + raise ValueError( + "Can not create Row with fields %s, expected %d values " + "but got %s" % (self, len(self), args) + ) return _create_row(self, args) def __getitem__(self, item: Any) -> Any: @@ -172,7 +174,9 @@ def __setattr__(self, key: Any, value: Any) -> None: raise RuntimeError("Row is read-only") self.__dict__[key] = value - def __reduce__(self, ) -> Union[str, Tuple[Any, ...]]: + def __reduce__( + self, + ) -> Union[str, Tuple[Any, ...]]: """Returns a tuple so Python knows how to pickle Row.""" if hasattr(self, "__fields__"): return (_create_row, (self.__fields__, tuple(self))) @@ -182,14 +186,16 @@ def __reduce__(self, ) -> Union[str, Tuple[Any, ...]]: def __repr__(self) -> str: """Printable representation of Row used in Python REPL.""" if hasattr(self, "__fields__"): - return "Row(%s)" % ", ".join("%s=%r" % (k, v) - for k, v in zip(self.__fields__, tuple(self))) + return "Row(%s)" % ", ".join( + "%s=%r" % (k, v) for k, v in zip(self.__fields__, tuple(self)) + ) else: return "" % ", ".join("%r" % field for field in self) -def _create_row(fields: Union["Row", List[str]], - values: Union[Tuple[Any, ...], List[Any]]) -> "Row": +def _create_row( + fields: Union["Row", List[str]], values: Union[Tuple[Any, ...], List[Any]] +) -> "Row": row = Row(*values) row.__fields__ = fields return row diff --git a/cmdexec/clients/python/src/databricks/sql/utils.py b/cmdexec/clients/python/src/databricks/sql/utils.py index 9d04af9b..2961a1f5 100644 --- a/cmdexec/clients/python/src/databricks/sql/utils.py +++ b/cmdexec/clients/python/src/databricks/sql/utils.py @@ -7,7 +7,9 @@ class ArrowQueue: - def __init__(self, arrow_table: pyarrow.Table, n_valid_rows: int, start_row_index: int = 0): + def __init__( + self, arrow_table: pyarrow.Table, n_valid_rows: int, start_row_index: int = 0 + ): """ A queue-like wrapper over an Arrow table @@ -29,14 +31,18 @@ def next_n_rows(self, num_rows: int) -> pyarrow.Table: return slice def remaining_rows(self) -> pyarrow.Table: - slice = self.arrow_table.slice(self.cur_row_index, self.n_valid_rows - self.cur_row_index) + slice = self.arrow_table.slice( + self.cur_row_index, self.n_valid_rows - self.cur_row_index + ) self.cur_row_index += slice.num_rows return slice ExecuteResponse = namedtuple( - 'ExecuteResponse', 'status has_been_closed_server_side has_more_rows description ' - 'command_handle arrow_queue arrow_schema_bytes') + "ExecuteResponse", + "status has_been_closed_server_side has_more_rows description " + "command_handle arrow_queue arrow_schema_bytes", +) def _bound(min_x, max_x, x): @@ -60,8 +66,10 @@ class NoRetryReason(Enum): class RequestErrorInfo( - namedtuple("RequestErrorInfo_", - "error error_message retry_delay http_code method request")): + namedtuple( + "RequestErrorInfo_", "error error_message retry_delay http_code method request" + ) +): @property def request_session_id(self): if hasattr(self.request, "sessionHandle"): @@ -76,18 +84,23 @@ def request_query_id(self): else: return None - def full_info_logging_context(self, no_retry_reason, attempt, max_attempts, elapsed, - max_duration): - log_base_data_dict = OrderedDict([ - ("method", self.method), - ("session-id", self.request_session_id), - ("query-id", self.request_query_id), - ("http-code", self.http_code), - ("error-message", self.error_message), - ("original-exception", str(self.error)), - ]) - - log_base_data_dict["no-retry-reason"] = no_retry_reason and no_retry_reason.value + def full_info_logging_context( + self, no_retry_reason, attempt, max_attempts, elapsed, max_duration + ): + log_base_data_dict = OrderedDict( + [ + ("method", self.method), + ("session-id", self.request_session_id), + ("query-id", self.request_query_id), + ("http-code", self.http_code), + ("error-message", self.error_message), + ("original-exception", str(self.error)), + ] + ) + + log_base_data_dict["no-retry-reason"] = ( + no_retry_reason and no_retry_reason.value + ) log_base_data_dict["bounded-retry-delay"] = self.retry_delay log_base_data_dict["attempt"] = "{}/{}".format(attempt, max_attempts) log_base_data_dict["elapsed-seconds"] = "{}/{}".format(elapsed, max_duration) @@ -98,8 +111,9 @@ def user_friendly_error_message(self, no_retry_reason, attempt, elapsed): # This should be kept at the level that is appropriate to return to a Redash user user_friendly_error_message = "Error during request to server" if self.error_message: - user_friendly_error_message = "{}: {}".format(user_friendly_error_message, - self.error_message) + user_friendly_error_message = "{}: {}".format( + user_friendly_error_message, self.error_message + ) return user_friendly_error_message @@ -115,7 +129,9 @@ def escape_args(self, parameters): elif isinstance(parameters, (list, tuple)): return tuple(self.escape_item(x) for x in parameters) else: - raise exc.ProgrammingError("Unsupported param format: {}".format(parameters)) + raise exc.ProgrammingError( + "Unsupported param format: {}".format(parameters) + ) def escape_number(self, item): return item @@ -126,7 +142,7 @@ def escape_string(self, item): # as byte strings. The old version always encodes Unicode as byte strings, which breaks # string formatting here. if isinstance(item, bytes): - item = item.decode('utf-8') + item = item.decode("utf-8") # This is good enough when backslashes are literal, newlines are just followed, and the way # to escape a single quote is to put two single quotes. # (i.e. only special character is single quote) @@ -134,7 +150,7 @@ def escape_string(self, item): def escape_sequence(self, item): l = map(str, map(self.escape_item, item)) - return '(' + ','.join(l) + ')' + return "(" + ",".join(l) + ")" def escape_datetime(self, item, format, cutoff=0): dt_str = item.strftime(format) @@ -143,7 +159,7 @@ def escape_datetime(self, item, format, cutoff=0): def escape_item(self, item): if item is None: - return 'NULL' + return "NULL" elif isinstance(item, (int, float)): return self.escape_number(item) elif isinstance(item, str):