|
11 | 11 | from dbt.adapters.base.relation import BaseRelation
|
12 | 12 | from dbt.adapters.spark.impl import (
|
13 | 13 | SparkAdapter,
|
14 |
| - GET_COLUMNS_IN_RELATION_RAW_MACRO_NAME, |
15 | 14 | KEY_TABLE_OWNER,
|
16 | 15 | KEY_TABLE_STATISTICS,
|
17 | 16 | LIST_RELATIONS_MACRO_NAME,
|
@@ -163,48 +162,6 @@ def parse_describe_extended(
|
163 | 162 | for idx, column in enumerate(rows)
|
164 | 163 | ]
|
165 | 164 |
|
166 |
| - def get_columns_in_relation(self, relation: DatabricksRelation) -> List[DatabricksColumn]: |
167 |
| - cached_relations = self.cache.get_relations(relation.database, relation.schema) |
168 |
| - cached_relation = next( |
169 |
| - ( |
170 |
| - cached_relation |
171 |
| - for cached_relation in cached_relations |
172 |
| - if str(cached_relation) == str(relation) |
173 |
| - ), |
174 |
| - None, |
175 |
| - ) |
176 |
| - columns = [] |
177 |
| - if cached_relation and cached_relation.information: |
178 |
| - columns = self.parse_columns_from_information(cached_relation) |
179 |
| - if not columns: |
180 |
| - # in open source delta 'show table extended' query output doesnt |
181 |
| - # return relation's schema. if columns are empty from cache, |
182 |
| - # use get_columns_in_relation spark macro |
183 |
| - # which would execute 'describe extended tablename' query |
184 |
| - try: |
185 |
| - rows: List[Row] = self.execute_macro( |
186 |
| - GET_COLUMNS_IN_RELATION_RAW_MACRO_NAME, kwargs={"relation": relation} |
187 |
| - ) |
188 |
| - columns = self.parse_describe_extended(relation, rows) |
189 |
| - except dbt.exceptions.RuntimeException as e: |
190 |
| - # spark would throw error when table doesn't exist, where other |
191 |
| - # CDW would just return and empty list, normalizing the behavior here |
192 |
| - errmsg = getattr(e, "msg", "") |
193 |
| - if any( |
194 |
| - msg in errmsg |
195 |
| - for msg in ( |
196 |
| - "[TABLE_OR_VIEW_NOT_FOUND]", |
197 |
| - "Table or view not found", |
198 |
| - "NoSuchTableException", |
199 |
| - ) |
200 |
| - ): |
201 |
| - pass |
202 |
| - else: |
203 |
| - raise e |
204 |
| - |
205 |
| - # strip hudi metadata columns. |
206 |
| - return [x for x in columns if x.name not in self.HUDI_METADATA_COLUMNS] |
207 |
| - |
208 | 165 | def parse_columns_from_information(
|
209 | 166 | self, relation: DatabricksRelation
|
210 | 167 | ) -> List[DatabricksColumn]:
|
|
0 commit comments