[PECO-1803] Splitting the PySql connector into the core and the non core part

.gitignore

@@ -48,7 +48,7 @@ __pycache__/
 .Python
 build/
 develop-eggs/
-dist/
+#dist/
 downloads/
 eggs/
 .eggs/
@@ -200,7 +200,7 @@ cython_debug/
 # End of https://www.toptal.com/developers/gitignore/api/python,macos
 
 databricks_sql_connector.egg-info/
-dist/
+#dist/
 build/
 
 # vs code stuff

databricks_sql_connector/pyproject.toml

@@ -0,0 +1,47 @@
+[tool.poetry]
+name = "databricks-sql-connector"
+version = "1.0.0"
+description = "Databricks SQL Connector for Python"
+authors = ["Databricks <[email protected]>"]
+license = "Apache-2.0"
+readme = "README.md"
+packages = [{ include = "databricks", from = "src" }]
+include = ["CHANGELOG.md"]
+
+[tool.poetry.dependencies]
+python = "^3.8.0"
+pyarrow = ">=14.0.1,<17"
+databricks_sql_connector_core = { version = ">=1.0.0"}
+databricks_sqlalchemy = { version = ">=1.0.0", optional = true }
+
+[tool.poetry.extras]
+databricks_sqlalchemy = ["databricks_sqlalchemy"]
+
+[tool.poetry.dev-dependencies]
+pytest = "^7.1.2"
+mypy = "^1.10.1"
+pylint = ">=2.12.0"
+black = "^22.3.0"
+pytest-dotenv = "^0.5.2"
+
+[tool.poetry.urls]
+"Homepage" = "https://github.com/databricks/databricks-sql-python"
+"Bug Tracker" = "https://github.com/databricks/databricks-sql-python/issues"
+
+[tool.poetry.plugins."sqlalchemy.dialects"]
+"databricks" = "databricks_sqlalchemy:DatabricksDialect"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.black]
+exclude = '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist|thrift_api)/'
+
+#[tool.pytest.ini_options]
+#markers = {"reviewed" = "Test case has been reviewed by Databricks"}
+#minversion = "6.0"
+#log_cli = "false"
+#log_cli_level = "INFO"
+#testpaths = ["tests", "databricks_sql_connector/src"]
+#env_files = ["test.env"]

databricks_sql_connector/src/databricks/sql/__init__.py

@@ -0,0 +1 @@
+from databricks_sql_connector_core.sql import *

databricks_sql_connector/src/databricks/sql/utils.py

@@ -0,0 +1 @@
+from databricks_sql_connector_core.sql.utils import *

databricks_sql_connector/src/databricks/sqlalchemy/__init__.py

@@ -0,0 +1,6 @@
+try:
+    from databricks_sqlalchemy import *
+except:
+    import warnings
+
+    warnings.warn("Install databricks_sqlalchemy plugin before using this")

README.md → databricks_sql_connector_core/README.md

@@ -65,8 +65,8 @@ or to a Databricks Runtime interactive cluster (e.g. /sql/protocolv1/o/123456789
 
 ## Contributing
 
-See [CONTRIBUTING.md](CONTRIBUTING.md)
+See [CONTRIBUTING.md](../CONTRIBUTING.md)
 
 ## License
 
-[Apache License 2.0](LICENSE)
+[Apache License 2.0](../LICENSE)

pyproject.toml → databricks_sql_connector_core/pyproject.toml

@@ -1,11 +1,11 @@
 [tool.poetry]
-name = "databricks-sql-connector"
-version = "3.3.0"
-description = "Databricks SQL Connector for Python"
+name = "databricks-sql-connector-core"
+version = "1.0.0"
+description = "Databricks SQL Connector core for Python"
 authors = ["Databricks <[email protected]>"]
 license = "Apache-2.0"
 readme = "README.md"
-packages = [{ include = "databricks", from = "src" }]
+packages = [{ include = "databricks_sql_connector_core", from = "src" }]
 include = ["CHANGELOG.md"]
 
 [tool.poetry.dependencies]
@@ -14,23 +14,17 @@ thrift = ">=0.16.0,<0.21.0"
 pandas = [
     { version = ">=1.2.5,<2.2.0", python = ">=3.8" }
 ]
-pyarrow = ">=14.0.1,<17"
-
 lz4 = "^4.0.2"
 requests = "^2.18.1"
 oauthlib = "^3.1.0"
-numpy = [
-    { version = "^1.16.6", python = ">=3.8,<3.11" },
-    { version = "^1.23.4", python = ">=3.11" },
-]
-sqlalchemy = { version = ">=2.0.21", optional = true }
 openpyxl = "^3.0.10"
+databricks_sqlalchemy = { version = ">=1.0.0", optional = true }
 alembic = { version = "^1.0.11", optional = true }
 urllib3 = ">=1.26"
 
 [tool.poetry.extras]
-sqlalchemy = ["sqlalchemy"]
-alembic = ["sqlalchemy", "alembic"]
+databricks_sqlalchemy = ["databricks_sqlalchemy"]
+alembic = ["databricks_sqlalchemy", "alembic"]
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.1.2"
@@ -44,7 +38,7 @@ pytest-dotenv = "^0.5.2"
 "Bug Tracker" = "https://github.com/databricks/databricks-sql-python/issues"
 
 [tool.poetry.plugins."sqlalchemy.dialects"]
-"databricks" = "databricks.sqlalchemy:DatabricksDialect"
+"databricks" = "databricks_sqlalchemy:DatabricksDialect"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
@@ -62,5 +56,5 @@ markers = {"reviewed" = "Test case has been reviewed by Databricks"}
 minversion = "6.0"
 log_cli = "false"
 log_cli_level = "INFO"
-testpaths = ["tests", "src/databricks/sqlalchemy/test_local"]
+testpaths = ["tests", "databricks_sql_connector_core/tests"]
 env_files = ["test.env"]

src/databricks/sql/__init__.py → databricks_sql_connector_core/src/databricks_sql_connector_core/sql/__init__.py

@@ -1,6 +1,6 @@
 import datetime
 
-from databricks.sql.exc import *
+from databricks_sql_connector_core.sql.exc import *
 
 # PEP 249 module globals
 apilevel = "2.0"

src/databricks/sql/auth/auth.py → databricks_sql_connector_core/src/databricks_sql_connector_core/sql/auth/auth.py

@@ -1,7 +1,7 @@
 from enum import Enum
 from typing import Optional, List
 
-from databricks.sql.auth.authenticators import (
+from databricks_sql_connector_core.sql.auth.authenticators import (
     AuthProvider,
     AccessTokenAuthProvider,
     ExternalAuthProvider,
@@ -107,9 +107,11 @@ def get_python_sql_connector_auth_provider(hostname: str, **kwargs):
         tls_client_cert_file=kwargs.get("_tls_client_cert_file"),
         oauth_scopes=PYSQL_OAUTH_SCOPES,
         oauth_client_id=kwargs.get("oauth_client_id") or client_id,
-        oauth_redirect_port_range=[kwargs["oauth_redirect_port"]]
-        if kwargs.get("oauth_client_id") and kwargs.get("oauth_redirect_port")
-        else redirect_port_range,
+        oauth_redirect_port_range=(
+            [kwargs["oauth_redirect_port"]]
+            if kwargs.get("oauth_client_id") and kwargs.get("oauth_redirect_port")
+            else redirect_port_range
+        ),
         oauth_persistence=kwargs.get("experimental_oauth_persistence"),
         credentials_provider=kwargs.get("credentials_provider"),
     )

src/databricks/sql/auth/authenticators.py → databricks_sql_connector_core/src/databricks_sql_connector_core/sql/auth/authenticators.py

@@ -3,12 +3,18 @@
 import logging
 from typing import Callable, Dict, List
 
-from databricks.sql.auth.oauth import OAuthManager
-from databricks.sql.auth.endpoint import get_oauth_endpoints, infer_cloud_from_host
+from databricks_sql_connector_core.sql.auth.oauth import OAuthManager
+from databricks_sql_connector_core.sql.auth.endpoint import (
+    get_oauth_endpoints,
+    infer_cloud_from_host,
+)
 
 # Private API: this is an evolving interface and it will change in the future.
 # Please must not depend on it in your applications.
-from databricks.sql.experimental.oauth_persistence import OAuthToken, OAuthPersistence
+from databricks_sql_connector_core.sql.experimental.oauth_persistence import (
+    OAuthToken,
+    OAuthPersistence,
+)
 
 
 class AuthProvider:
@@ -25,12 +31,10 @@ class CredentialsProvider(abc.ABC):
     for authenticating requests to Databricks REST APIs"""
 
     @abc.abstractmethod
-    def auth_type(self) -> str:
-        ...
+    def auth_type(self) -> str: ...
 
     @abc.abstractmethod
-    def __call__(self, *args, **kwargs) -> HeaderFactory:
-        ...
+    def __call__(self, *args, **kwargs) -> HeaderFactory: ...
 
 
 # Private API: this is an evolving interface and it will change in the future.

src/databricks/sql/auth/oauth.py → databricks_sql_connector_core/src/databricks_sql_connector_core/sql/auth/oauth.py

@@ -13,8 +13,10 @@
 from oauthlib.oauth2.rfc6749.errors import OAuth2Error
 from requests.exceptions import RequestException
 
-from databricks.sql.auth.oauth_http_handler import OAuthHttpSingleRequestHandler
-from databricks.sql.auth.endpoint import OAuthEndpointCollection
+from databricks_sql_connector_core.sql.auth.oauth_http_handler import (
+    OAuthHttpSingleRequestHandler,
+)
+from databricks_sql_connector_core.sql.auth.endpoint import OAuthEndpointCollection
 
 logger = logging.getLogger(__name__)
 

src/databricks/sql/auth/retry.py → databricks_sql_connector_core/src/databricks_sql_connector_core/sql/auth/retry.py

@@ -14,7 +14,7 @@
 from urllib3 import Retry
 from urllib3.util.retry import RequestHistory
 
-from databricks.sql.exc import (
+from databricks_sql_connector_core.sql.exc import (
     CursorAlreadyClosedError,
     MaxRetryDurationError,
     NonRecoverableNetworkError,

src/databricks/sql/auth/thrift_http_client.py → databricks_sql_connector_core/src/databricks_sql_connector_core/sql/auth/thrift_http_client.py

@@ -15,7 +15,10 @@
 
 from urllib3 import HTTPConnectionPool, HTTPSConnectionPool, ProxyManager
 from urllib3.util import make_headers
-from databricks.sql.auth.retry import CommandType, DatabricksRetryPolicy
+from databricks_sql_connector_core.sql.auth.retry import (
+    CommandType,
+    DatabricksRetryPolicy,
+)
 
 
 class THttpClient(thrift.transport.THttpClient.THttpClient):