Add support for Snowflake secure views. (#1017)

Author: BenBirt

core/adapters/bigquery.ts

@@ -80,7 +80,11 @@ export class BigQueryAdapter extends Adapter implements IAdapter {
     return tasks;
   }
 
-  public createOrReplace(table: dataform.ITable) {
+  public dropIfExists(target: dataform.ITarget, type: dataform.TableMetadata.Type) {
+    return `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`;
+  }
+
+  private createOrReplace(table: dataform.ITable) {
     return `create or replace ${this.tableTypeAsSql(
       this.baseTableType(table.type)
     )} ${this.resolveTarget(table.target)} ${
@@ -94,16 +98,12 @@ export class BigQueryAdapter extends Adapter implements IAdapter {
     }as ${table.query}`;
   }
 
-  public createOrReplaceView(target: dataform.ITarget, query: string) {
+  private createOrReplaceView(target: dataform.ITarget, query: string) {
     return `
       create or replace view ${this.resolveTarget(target)} as ${query}`;
   }
 
-  public dropIfExists(target: dataform.ITarget, type: dataform.TableMetadata.Type) {
-    return `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`;
-  }
-
-  public mergeInto(
+  private mergeInto(
     target: dataform.ITarget,
     columns: string[],
     query: string,

core/adapters/redshift.ts

@@ -38,7 +38,6 @@ export class RedshiftAdapter extends Adapter implements IAdapter {
           table.uniqueKey && table.uniqueKey.length > 0
             ? this.mergeInto(
                 table.target,
-                tableMetadata.fields.map(f => f.name),
                 this.where(table.incrementalQuery || table.query, table.where),
                 table.uniqueKey
               )
@@ -78,7 +77,15 @@ export class RedshiftAdapter extends Adapter implements IAdapter {
       .add(Task.assertion(`select sum(1) as row_count from ${this.resolveTarget(target)}`));
   }
 
-  public createOrReplaceView(target: dataform.ITarget, query: string, bind: boolean) {
+  public dropIfExists(target: dataform.ITarget, type: dataform.TableMetadata.Type) {
+    const query = `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`;
+    if (!this.isBindSupported()) {
+      return query;
+    }
+    return `${query} cascade`;
+  }
+
+  private createOrReplaceView(target: dataform.ITarget, query: string, bind: boolean) {
     const createQuery = `create or replace view ${this.resolveTarget(target)} as ${query}`;
     // Postgres doesn't support with no schema binding.
     if (bind || this.project.warehouse === "postgres") {
@@ -87,7 +94,7 @@ export class RedshiftAdapter extends Adapter implements IAdapter {
     return `${createQuery} with no schema binding`;
   }
 
-  public createOrReplace(table: dataform.ITable) {
+  private createOrReplace(table: dataform.ITable) {
     if (table.type === "view") {
       const isBindDefined = table.redshift && table.redshift.hasOwnProperty("bind");
       const bindDefaultValue = semver.gte(this.dataformCoreVersion, "1.4.1") ? false : true;
@@ -116,7 +123,7 @@ export class RedshiftAdapter extends Adapter implements IAdapter {
       );
   }
 
-  public createTable(table: dataform.ITable, target: dataform.ITarget) {
+  private createTable(table: dataform.ITable, target: dataform.ITarget) {
     if (table.redshift) {
       let query = `create table ${this.resolveTarget(target)}`;
 
@@ -135,20 +142,7 @@ export class RedshiftAdapter extends Adapter implements IAdapter {
     return `create table ${this.resolveTarget(target)} as ${table.query}`;
   }
 
-  public dropIfExists(target: dataform.ITarget, type: dataform.TableMetadata.Type) {
-    const query = `drop ${this.tableTypeAsSql(type)} if exists ${this.resolveTarget(target)}`;
-    if (!this.isBindSupported()) {
-      return query;
-    }
-    return `${query} cascade`;
-  }
-
-  public mergeInto(
-    target: dataform.ITarget,
-    columns: string[],
-    query: string,
-    uniqueKey: string[]
-  ) {
+  private mergeInto(target: dataform.ITarget, query: string, uniqueKey: string[]) {
     const finalTarget = this.resolveTarget(target);
     // Schema name not allowed for temporary tables.
     const tempTarget = `"${target.schema}__${target.name}_incremental_temp"`;

core/adapters/snowflake.ts

@@ -73,23 +73,25 @@ export class SnowflakeAdapter extends Adapter implements IAdapter {
         schema: projectConfig.assertionSchema,
         name: assertion.name
       });
-    tasks.add(Task.statement(this.createOrReplaceView(target, assertion.query)));
+    tasks.add(Task.statement(this.createOrReplaceView(target, assertion.query, false)));
     tasks.add(Task.assertion(`select sum(1) as row_count from ${this.resolveTarget(target)}`));
     return tasks;
   }
 
-  public createOrReplaceView(target: dataform.ITarget, query: string) {
-    return `
-      create or replace view ${this.resolveTarget(target)} as ${query}`;
+  private createOrReplaceView(target: dataform.ITarget, query: string, secure: boolean) {
+    return `create or replace ${secure ? "secure " : ""}view ${this.resolveTarget(
+      target
+    )} as ${query}`;
   }
 
-  public createOrReplace(table: dataform.ITable) {
-    return `create or replace ${this.tableTypeAsSql(
-      this.baseTableType(table.type || "table")
-    )} ${this.resolveTarget(table.target)} as ${table.query}`;
+  private createOrReplace(table: dataform.ITable) {
+    if (table.type === "view") {
+      return this.createOrReplaceView(table.target, table.query, table.snowflake?.secure);
+    }
+    return `create or replace table ${this.resolveTarget(table.target)} as ${table.query}`;
   }
 
-  public mergeInto(
+  private mergeInto(
     target: dataform.ITarget,
     columns: string[],
     query: string,

core/adapters/sqldatawarehouse.ts

@@ -90,7 +90,16 @@ export class SQLDataWarehouseAdapter extends Adapter implements IAdapter {
     )}','U') is not null drop table ${this.resolveTarget(target)}`;
   }
 
-  public createOrReplace(table: dataform.ITable, alreadyExists: boolean) {
+  public insertInto(target: dataform.ITarget, columns: string[], query: string) {
+    return `
+insert into ${this.resolveTarget(target)}
+(${columns.join(",")})
+select ${columns.join(",")}
+from (${query}
+) as insertions`;
+  }
+
+  private createOrReplace(table: dataform.ITable, alreadyExists: boolean) {
     if (table.type === "view") {
       return Tasks.create().add(
         Task.statement(
@@ -116,7 +125,7 @@ export class SQLDataWarehouseAdapter extends Adapter implements IAdapter {
       );
   }
 
-  public createTable(table: dataform.ITable, target: dataform.ITarget) {
+  private createTable(table: dataform.ITable, target: dataform.ITarget) {
     const distribution =
       table.sqlDataWarehouse && table.sqlDataWarehouse.distribution
         ? table.sqlDataWarehouse.distribution
@@ -127,13 +136,4 @@ export class SQLDataWarehouseAdapter extends Adapter implements IAdapter {
      ) 
      as ${table.query}`;
   }
-
-  public insertInto(target: dataform.ITarget, columns: string[], query: string) {
-    return `
-insert into ${this.resolveTarget(target)}
-(${columns.join(",")})
-select ${columns.join(",")}
-from (${query}
-) as insertions`;
-  }
 }

core/session.ts

@@ -572,6 +572,17 @@ export class Session {
         );
       }
 
+      // snowflake config
+      if (!!table.snowflake) {
+        if (table.snowflake.secure && table.type !== "view") {
+          this.compileError(
+            new Error(`The 'secure' option is only valid for Snowflake views`),
+            table.fileName,
+            table.name
+          );
+        }
+      }
+
       // sqldatawarehouse config
       if (!!table.sqlDataWarehouse) {
         if (!!table.uniqueKey && table.uniqueKey.length > 0) {

core/table.ts

@@ -59,7 +59,7 @@ export const SortStyleType = ["compound", "interleaved"] as const;
 export type SortStyleType = typeof SortStyleType[number];
 
 /**
- * Redshift specific warehouse options.
+ * Redshift-specific warehouse options.
  */
 export interface IRedshiftOptions {
   /**
@@ -95,7 +95,20 @@ const IRedshiftOptionsProperties = () =>
   strictKeysOf<IRedshiftOptions>()(["distKey", "distStyle", "sortKeys", "sortStyle"]);
 
 /**
- * Options for creating tables within Azure SQL Data Warehouse projects.
+ * Snowflake-specific warehouse options.
+ */
+export interface ISnowflakeOptions {
+  /**
+   * If set to true, a secure view will be created.
+   *
+   * For more information, read the [Snowflake Secure Views docs](https://docs.snowflake.com/en/user-guide/views-secure.html).
+   */
+  secure?: boolean;
+}
+const ISnowflakeOptionsProperties = () => strictKeysOf<ISnowflakeOptions>()(["secure"]);
+
+/**
+ * Azure SQL Data Warehouse-specific warehouse options.
  */
 export interface ISQLDataWarehouseOptions {
   /**
@@ -109,7 +122,7 @@ const ISQLDataWarehouseOptionsProperties = () =>
   strictKeysOf<ISQLDataWarehouseOptions>()(["distribution"]);
 
 /**
- * Options for creating tables within BigQuery projects.
+ * BigQuery-specific warehouse options.
  */
 export interface IBigQueryOptions {
   /**
@@ -206,6 +219,11 @@ export interface ITableConfig
    */
   bigquery?: IBigQueryOptions;
 
+  /**
+   * Snowflake-specific options.
+   */
+  snowflake?: ISnowflakeOptions;
+
   /**
    * Azure SQL Data Warehouse-specific options.
    */
@@ -235,6 +253,7 @@ export const ITableConfigProperties = () =>
     "name",
     "redshift",
     "bigquery",
+    "snowflake",
     "sqldatawarehouse",
     "tags",
     "uniqueKey",
@@ -274,6 +293,7 @@ export class Table {
     inline: [
       "bigquery",
       "redshift",
+      "snowflake",
       "sqlDataWarehouse",
       "preOps",
       "postOps",
@@ -326,6 +346,9 @@ export class Table {
     if (config.bigquery) {
       this.bigquery(config.bigquery);
     }
+    if (config.snowflake) {
+      this.snowflake(config.snowflake);
+    }
     if (config.sqldatawarehouse) {
       this.sqldatawarehouse(config.sqldatawarehouse);
     }
@@ -393,6 +416,17 @@ export class Table {
     this.proto.uniqueKey = uniqueKey;
   }
 
+  public snowflake(snowflake: dataform.ISnowflakeOptions) {
+    checkExcessProperties(
+      (e: Error) => this.session.compileError(e),
+      snowflake,
+      ISnowflakeOptionsProperties(),
+      "snowflake config"
+    );
+    this.proto.snowflake = dataform.SnowflakeOptions.create(snowflake);
+    return this;
+  }
+
   public sqldatawarehouse(sqlDataWarehouse: dataform.ISQLDataWarehouseOptions) {
     checkExcessProperties(
       (e: Error) => this.session.compileError(e),

protos/core.proto

@@ -164,6 +164,10 @@ message RedshiftOptions {
   bool bind = 5 [deprecated = true];
 }
 
+message SnowflakeOptions {
+  bool secure = 1;
+}
+
 message SQLDataWarehouseOptions {
   string distribution = 1;
 }
@@ -285,6 +289,7 @@ message Table {
   // Warehouse specific features.
   RedshiftOptions redshift = 21;
   BigQueryOptions bigquery = 22;
+  SnowflakeOptions snowflake = 33;
   SQLDataWarehouseOptions sql_data_warehouse = 25;
 
   // Generated.

tests/integration/snowflake_project/definitions/example_view.sqlx

@@ -3,6 +3,9 @@ config {
     description: "An example view",
     columns: {
         val: "val doc",
+    },
+    snowflake: {
+        secure: true
     }
 }
 select * from ${ref("sample_data")}

version.bzl

@@ -1,3 +1,3 @@
 # NOTE: If you change the format of this line, you must change the bash command
 # in /scripts/publish to extract the version string correctly.
-DF_VERSION = "1.11.1"
+DF_VERSION = "1.12.0"