Skip to content

Commit 629a510

Browse files
overcoilovercoil
committed
various minor items
-clean up Makefile for pytest -explore passing in --requirements to help with suite run -exploration of DDLCompiler
1 parent 6655473 commit 629a510

File tree

6 files changed

+557
-30
lines changed

6 files changed

+557
-30
lines changed

‎Makefile

Lines changed: 26 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ SUITE_PATH=tests/sqlalchemy
1515

1616
SUITE=test_suite.py
1717

18+
REQ=--requirements src.databricks.sqlalchemy.requirements:Requirements
19+
DBURI=--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
20+
1821
.PHONY=all clean showtables full reflection simple str num drop_simpletest drop_reflectiontest
1922

2023

@@ -30,68 +33,69 @@ showtables:
3033

3134
full:
3235
$(PYTEST) $(SUITE_PATH) \
33-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)" \
36+
$(DBURI) \
3437
--log-file=~/.pytestlogs/full.log
3538

36-
sa-bool: drop_booleantest
39+
sa-bool: drop_booleantest drop_t
3740
$(PYTEST) $(SUITE_PATH)/test_full_sa.py::BooleanTest \
38-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
41+
$(DBURI)
3942

4043
sa-date: drop_datetest
4144
$(PYTEST) $(SUITE_PATH)/test_full_sa.py::DateTest \
42-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
45+
$(DBURI)
4346

4447
sa-dt: drop_datetimetest
4548
$(PYTEST) $(SUITE_PATH)/test_full_sa.py::DateTimeTest \
46-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
49+
$(DBURI)
4750

4851
sa-int: drop_integertest
4952
$(PYTEST) $(SUITE_PATH)/test_full_sa.py::IntegerTest \
50-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
53+
$(DBURI)
5154

5255
sa-num: drop_numerictest
5356
$(PYTEST) $(SUITE_PATH)/test_full_sa.py::NumericTest \
54-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
57+
$(DBURI)
5558

5659
sa-str: drop_stringtest
5760
$(PYTEST) $(SUITE_PATH)/test_full_sa.py::StringTest \
58-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
61+
$(DBURI)
5962

6063
sa-ddl: drop_tableddl
6164
$(PYTEST) $(SUITE_PATH)/test_full_sa.py::TableDDLTest \
62-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
65+
$(REQ) \
66+
$(DBURI)
6367

6468
sa-ddl1: drop_tableddl
6569
$(PYTEST) $(SUITE_PATH)/test_full_sa.py::TableDDLTest:test_create_table \
66-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
70+
$(DBURI)
6771

6872
sa-ddl2: drop_tableddl
6973
$(PYTEST) $(SUITE_PATH)/test_full_sa.py::TableDDLTest:test_create_table_schema \
70-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
74+
$(DBURI)
7175

7276
sa-ddl3: drop_tableddl
7377
$(PYTEST) $(SUITE_PATH)/test_full_sa.py::TableDDLTest:test_drop_table \
74-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
78+
$(DBURI)
7579

7680
sa-join: drop_jointest
7781
$(PYTEST) $(SUITE_PATH)/test_full_sa.py::JoinTest \
78-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
82+
$(DBURI)
7983

8084
reflection:
8185
$(PYTEST) $(SUITE_PATH)/$(SUITE)::ReflectionTest \
82-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
86+
$(DBURI)
8387

8488
num:
8589
$(PYTEST) $(SUITE_PATH)/$(SUITE)::ReflectionTest::test_numtypes \
86-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
90+
$(DBURI)
8791

8892
str:
8993
$(PYTEST) $(SUITE_PATH)/$(SUITE)::ReflectionTest::test_strtypes \
90-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
94+
$(DBURI)
9195

9296
simple:
9397
$(PYTEST) $(SUITE_PATH)/$(SUITE)::SimpleTest \
94-
--dburi "databricks+thrift://token:$(DATABRICKS_TOKEN)@$(DATABRICKS_SERVER_HOSTNAME)/$(DATABRICKS_SCHEMA)?http_path=$(DATABRICKS_HTTP_PATH)"
98+
$(DBURI)
9599

96100
# clean up after SimpleTest run
97101
drop_simpletest:
@@ -111,7 +115,7 @@ drop_datetest: drop_date_table
111115

112116
drop_datetimetest: drop_date_table
113117

114-
drop_integertest: drop_t drop_tabletest
118+
drop_integertest: drop_t drop_tabletest drop_integer_table
115119

116120
drop_numerictest: drop_t drop_tabletest
117121

@@ -121,7 +125,7 @@ drop_tableddl: drop__test_table drop_test_table
121125

122126
drop_jointest: drop_a drop_b
123127

124-
128+
125129
drop_t:
126130
echo y | $(DBSCLI) -e "USE $(DATABRICKS_SCHEMA); DROP TABLE IF EXISTS t;"
127131

@@ -146,6 +150,9 @@ drop_b:
146150
drop_date_table:
147151
echo y | $(DBSCLI) -e "USE $(DATABRICKS_SCHEMA); DROP TABLE IF EXISTS date_table;"
148152

153+
drop_integer_table:
154+
echo y | $(DBSCLI) -e "USE $(DATABRICKS_SCHEMA); DROP TABLE IF EXISTS integer_table;"
155+
149156

150157
# these two schemas are baked into SQLAlchemy's test suite
151158
satestdb:

‎src/databricks/sqlalchemy/dialect.py

Lines changed: 119 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,11 @@ class DatabricksDate(DatabricksStringTypeBase):
3333
"""Translates date strings to date objects"""
3434
impl = types.DATE
3535

36+
# ref: https://docs.sqlalchemy.org/en/14/core/custom_types.html
37+
def process_bind_param(self, value, dialect):
38+
# handle string
39+
return "PREFIX:" + value
40+
3641
def process_result_value(self, value, dialect):
3742
return processors.str_to_date(value)
3843

@@ -201,14 +206,121 @@ def visit_primary_key_constraint(self, constraint, **kw):
201206
def visit_foreign_key_constraint(self, constraint, **kw):
202207
return ""
203208

204-
# stripped down from DDLCompiler::get_column_specification
205-
# def get_column_specification(self, column, **kwargs):
206-
# colspec = (
207-
# self.preparer.format_column(column)
208-
# + " "
209-
# )
210-
# return colspec
209+
def visit_unique_constraint(self, constraint, **kw):
210+
return ""
211211

212+
# def visit_create_table(self, create, **kw):
213+
# # if debugbreakpoint:
214+
# # breakpoint()
215+
216+
# table = create.element
217+
# preparer = self.preparer
218+
219+
# text = "\nCREATE "
220+
# if table._prefixes:
221+
# text += " ".join(table._prefixes) + " "
222+
223+
# text += "TABLE "
224+
# if create.if_not_exists:
225+
# text += "IF NOT EXISTS "
226+
227+
# text += preparer.format_table(table) + " "
228+
229+
# create_table_suffix = self.create_table_suffix(table)
230+
# if create_table_suffix:
231+
# text += create_table_suffix + " "
232+
233+
# text += "("
234+
235+
# separator = "\n"
236+
237+
# # if only one primary key, specify it along with the column
238+
# first_pk = False
239+
# for create_column in create.columns:
240+
# column = create_column.element
241+
# try:
242+
# processed = self.process(
243+
# create_column, first_pk=column.primary_key and not first_pk
244+
# )
245+
# if processed is not None:
246+
# text += separator
247+
# separator = ", \n"
248+
# text += "\t" + processed
249+
# if column.primary_key:
250+
# first_pk = True
251+
# except exc.CompileError as ce:
252+
# raise exc.CompileError(
253+
# "(in table '%s', column '%s'): %s"
254+
# % (table.description, column.name, ce.args[0])
255+
# ) from ce
256+
257+
# const = self.create_table_constraints(
258+
# table,
259+
# _include_foreign_key_constraints=create.include_foreign_key_constraints, # noqa
260+
# )
261+
# if const:
262+
# text += separator + "\t" + const
263+
264+
# text += "\n)%s\n\n" % self.post_create_table(table)
265+
# return text
266+
267+
# def visit_create_column(self, create, first_pk=False, **kw):
268+
# # if debugbreakpoint:
269+
# # breakpoint()
270+
271+
# column = create.element
272+
273+
# if column.system:
274+
# return None
275+
276+
# text = self.get_column_specification(column, first_pk=first_pk)
277+
# const = " ".join(
278+
# self.process(constraint) for constraint in column.constraints
279+
# )
280+
# if const:
281+
# text += " " + const
282+
283+
# return text
284+
285+
# def create_table_constraints(
286+
# self, table, _include_foreign_key_constraints=None, **kw
287+
# ):
288+
# if debugbreakpoint:
289+
# breakpoint()
290+
291+
# # On some DB order is significant: visit PK first, then the
292+
# # other constraints (engine.ReflectionTest.testbasic failed on FB2)
293+
# constraints = []
294+
# if table.primary_key:
295+
# constraints.append(table.primary_key)
296+
297+
# all_fkcs = table.foreign_key_constraints
298+
# if _include_foreign_key_constraints is not None:
299+
# omit_fkcs = all_fkcs.difference(_include_foreign_key_constraints)
300+
# else:
301+
# omit_fkcs = set()
302+
303+
# constraints.extend(
304+
# [
305+
# c
306+
# for c in table._sorted_constraints
307+
# if c is not table.primary_key and c not in omit_fkcs
308+
# ]
309+
# )
310+
311+
# return ", \n\t".join(
312+
# p
313+
# for p in (
314+
# self.process(constraint)
315+
# for constraint in constraints
316+
# if (constraint._should_create_for_compiler(self))
317+
# and (
318+
# not self.dialect.supports_alter
319+
# or not getattr(constraint, "use_alter", False)
320+
# )
321+
# )
322+
# if p is not None
323+
# )
212324

213325
# The following lookup table is by DATA_TYPE and is rather nice since Decimal can be detected directly.
214326
# However, as DATA_TYPE is rather obtuse... going forward, we switched to use COLUMN_TYPE_NAME instead (the table below)

‎src/databricks/sqlalchemy/requirements.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,24 @@ class Requirements(SuiteRequirements):
1616
def two_phase_transactions(self):
1717
# Databricks SQL doesn't support transactions
1818
return exclusions.closed()
19+
20+
@property
21+
def table_ddl_if_exists(self):
22+
"""target platform supports IF NOT EXISTS / IF EXISTS for tables."""
23+
24+
return exclusions.open()
25+
26+
@property
27+
def foreign_keys(self):
28+
# Databricks SQL doesn't support foreign keys
29+
return exclusions.closed()
30+
31+
@property
32+
def self_referential_foreign_keys(self):
33+
34+
return exclusions.closed()
35+
36+
@property
37+
def foreign_key_ddl(self):
38+
39+
return exclusions.closed()

‎src/databricks/sqlalchemy/sample-app-insert.py

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# sample-app-insert.py
2+
#
3+
# Program to demonstrate the simplest INSERT statement
4+
#
5+
6+
import os
7+
import random
8+
9+
from sqlalchemy import create_engine
10+
from sqlalchemy import MetaData
11+
from sqlalchemy import select, insert, Table, Column
12+
from sqlalchemy import SMALLINT, Integer, BigInteger, Float, DECIMAL, BOOLEAN
13+
14+
from sqlalchemy.dialects.mysql.types import TINYINT, DOUBLE # borrow MySQL's impls
15+
from sqlalchemy import String, DATE, TIMESTAMP
16+
17+
# pickup settings from the env
18+
server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME")
19+
http_path = os.getenv("DATABRICKS_HTTP_PATH")
20+
access_token = os.getenv("DATABRICKS_TOKEN")
21+
default_schema = os.getenv("DATABRICKS_SCHEMA")
22+
23+
# use echo=True for verbose log
24+
engine = create_engine(f"databricks+thrift://token:{access_token}@{server_hostname}/{default_schema}?http_path={http_path}", echo=False, future=True)
25+
26+
metadata_obj = MetaData()
27+
28+
# NB: sample_numtypes is a pre-created/populated table
29+
numtypes = "sample_numtypes"
30+
31+
t1 = Table(
32+
numtypes,
33+
metadata_obj,
34+
Column('f_byte', TINYINT),
35+
Column('f_short', SMALLINT),
36+
Column('f_int', Integer),
37+
Column('f_long', BigInteger),
38+
Column('f_float', Float),
39+
Column('f_double', DOUBLE),
40+
Column('f_decimal', DECIMAL),
41+
Column('f_boolean', BOOLEAN)
42+
)
43+
44+
with engine.connect() as conn:
45+
stmt = insert(t1).values(f_byte=42,
46+
f_short=31415,
47+
f_int=random.randint(1,1001002003),
48+
f_long=4001002003004005006,
49+
f_float=1.41,
50+
f_double=1.6666,
51+
f_decimal=2.71828,
52+
f_boolean=False)
53+
54+
print(f"Attempting to execute: {stmt}\n")
55+
56+
print(f"Rows from table {numtypes}")
57+
for row in conn.execute(stmt):
58+
print(row)
59+
60+
61+
# NB: sample_strtypes is a pre-created/populated table
62+
strtypes = "sample_strtypes"
63+
64+
with engine.connect() as conn:
65+
t2 = Table(
66+
strtypes,
67+
metadata_obj,
68+
autoload_with=conn
69+
)
70+
71+
# stmt = insert(t2).values(f_string='Antarctic expedition', f_date='1911-12-14', f_timestamp='1911-12-14T15:00', f_interval='4 0:00:00' )
72+
stmt = insert(t2).values(f_string='Antarctic expedition', f_date='1911-12-14', f_timestamp='1911-12-14T15:00')
73+
print(f"Attempting to execute: {stmt}\n")
74+
75+
print(f"Rows from table {strtypes}")
76+
for row in conn.execute(stmt):
77+
print(row)

0 commit comments

Comments
 (0)