Skip to content

Commit 51aff50

Browse files
obada-abobada-ab
authored
fix: add support for repeated record query parameters (#2698)
Add support for array of struct query parameters which are used to query repeated record fields Fixes #2485 ☕️
1 parent 499af3f commit 51aff50

File tree

7 files changed

+580
-5
lines changed

7 files changed

+580
-5
lines changed

‎README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-bigquery/tree
127127
| Copy Multiple Tables | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CopyMultipleTables.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CopyMultipleTables.java) |
128128
| Copy Table | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CopyTable.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CopyTable.java) |
129129
| Copy Table Cmek | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CopyTableCmek.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CopyTableCmek.java) |
130+
| Create And Query Repeated Record Field | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CreateAndQueryRepeatedRecordField.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CreateAndQueryRepeatedRecordField.java) |
130131
| Create Clustered Table | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CreateClusteredTable.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CreateClusteredTable.java) |
131132
| Create Dataset | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CreateDataset.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CreateDataset.java) |
132133
| Create Dataset Aws | [source code](https://github.com/googleapis/java-bigquery/blob/main/samples/snippets/src/main/java/com/example/bigquery/CreateDatasetAws.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-bigquery&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/bigquery/CreateDatasetAws.java) |

‎google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryParameterValue.java

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,11 @@ public static <T> QueryParameterValue array(T[] array, Class<T> clazz) {
349349
public static <T> QueryParameterValue array(T[] array, StandardSQLTypeName type) {
350350
List<QueryParameterValue> listValues = new ArrayList<>();
351351
for (T obj : array) {
352-
listValues.add(QueryParameterValue.of(obj, type));
352+
if (type == StandardSQLTypeName.STRUCT) {
353+
listValues.add((QueryParameterValue) obj);
354+
} else {
355+
listValues.add(QueryParameterValue.of(obj, type));
356+
}
353357
}
354358
return QueryParameterValue.newBuilder()
355359
.setArrayValues(listValues)
@@ -522,9 +526,15 @@ QueryParameterType toTypePb() {
522526
QueryParameterType typePb = new QueryParameterType();
523527
typePb.setType(getType().toString());
524528
if (getArrayType() != null) {
525-
QueryParameterType arrayTypePb = new QueryParameterType();
526-
arrayTypePb.setType(getArrayType().toString());
527-
typePb.setArrayType(arrayTypePb);
529+
List<QueryParameterValue> values = getArrayValues();
530+
if (getArrayType() == StandardSQLTypeName.STRUCT && values != null && values.size() != 0) {
531+
QueryParameterType structType = values.get(0).toTypePb();
532+
typePb.setArrayType(structType);
533+
} else {
534+
QueryParameterType arrayTypePb = new QueryParameterType();
535+
arrayTypePb.setType(getArrayType().toString());
536+
typePb.setArrayType(arrayTypePb);
537+
}
528538
}
529539
if (getStructTypes() != null) {
530540
List<QueryParameterType.StructTypes> structTypes = new ArrayList<>();

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryParameterValueTest.java

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.math.BigDecimal;
2929
import java.text.ParseException;
3030
import java.time.Period;
31+
import java.util.ArrayList;
3132
import java.util.Date;
3233
import java.util.HashMap;
3334
import java.util.List;
@@ -563,6 +564,48 @@ public void testNestedStruct() {
563564
assertThat(nestedRecordField.getStructValues().size()).isEqualTo(structValue.size());
564565
}
565566

567+
@Test
568+
public void testStructArray() {
569+
Boolean[] boolValues = new Boolean[] {true, false};
570+
Integer[] intValues = new Integer[] {15, 20};
571+
String[] stringValues = new String[] {"test-string", "test-string2"};
572+
List<ImmutableMap<String, QueryParameterValue>> fieldMaps = new ArrayList<>();
573+
List<QueryParameterValue> tuples = new ArrayList<>();
574+
for (int i = 0; i < 2; i++) {
575+
QueryParameterValue booleanField = QueryParameterValue.bool(boolValues[i]);
576+
QueryParameterValue integerField = QueryParameterValue.int64(intValues[i]);
577+
QueryParameterValue stringField = QueryParameterValue.string(stringValues[i]);
578+
ImmutableMap<String, QueryParameterValue> fieldMap =
579+
ImmutableMap.of(
580+
"booleanField",
581+
booleanField,
582+
"integerField",
583+
integerField,
584+
"stringField",
585+
stringField);
586+
fieldMaps.add(fieldMap);
587+
QueryParameterValue recordField = QueryParameterValue.struct(fieldMap);
588+
tuples.add(recordField);
589+
}
590+
QueryParameterValue repeatedRecordField =
591+
QueryParameterValue.array(tuples.toArray(), StandardSQLTypeName.STRUCT);
592+
com.google.api.services.bigquery.model.QueryParameterValue parameterValue =
593+
repeatedRecordField.toValuePb();
594+
QueryParameterType parameterType = repeatedRecordField.toTypePb();
595+
QueryParameterValue queryParameterValue =
596+
QueryParameterValue.fromPb(parameterValue, parameterType);
597+
assertThat(queryParameterValue.getValue()).isNull();
598+
assertThat(queryParameterValue.getType()).isEqualTo(StandardSQLTypeName.ARRAY);
599+
assertThat(queryParameterValue.getArrayType()).isEqualTo(StandardSQLTypeName.STRUCT);
600+
assertThat(queryParameterValue.getArrayValues().size()).isEqualTo(2);
601+
for (int i = 0; i < 2; i++) {
602+
QueryParameterValue record = queryParameterValue.getArrayValues().get(i);
603+
assertThat(record.getType()).isEqualTo(StandardSQLTypeName.STRUCT);
604+
assertThat(record.getStructTypes()).isNotNull();
605+
assertThat(record.getStructValues()).isEqualTo(fieldMaps.get(i));
606+
}
607+
}
608+
566609
private static void assertArrayDataEquals(
567610
String[] expectedValues,
568611
StandardSQLTypeName expectedType,

‎google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/it/ITBigQueryTest.java

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,7 @@
156156
import java.util.HashMap;
157157
import java.util.HashSet;
158158
import java.util.Iterator;
159+
import java.util.LinkedHashMap;
159160
import java.util.List;
160161
import java.util.Map;
161162
import java.util.Set;
@@ -231,6 +232,31 @@ public class ITBigQueryTest {
231232
.setMode(Field.Mode.REQUIRED)
232233
.setDescription("RecordDescription")
233234
.build();
235+
236+
private static final Field REPEATED_RECORD_FIELD_SCHEMA =
237+
Field.newBuilder(
238+
"Addresses",
239+
LegacySQLTypeName.RECORD,
240+
Field.newBuilder("Status", LegacySQLTypeName.STRING)
241+
.setMode(Field.Mode.NULLABLE)
242+
.build(),
243+
Field.newBuilder("Address", LegacySQLTypeName.STRING)
244+
.setMode(Field.Mode.NULLABLE)
245+
.build(),
246+
Field.newBuilder("City", LegacySQLTypeName.STRING)
247+
.setMode(Field.Mode.NULLABLE)
248+
.build(),
249+
Field.newBuilder("State", LegacySQLTypeName.STRING)
250+
.setMode(Field.Mode.NULLABLE)
251+
.build(),
252+
Field.newBuilder("Zip", LegacySQLTypeName.STRING)
253+
.setMode(Field.Mode.NULLABLE)
254+
.build(),
255+
Field.newBuilder("NumberOfYears", LegacySQLTypeName.STRING)
256+
.setMode(Field.Mode.NULLABLE)
257+
.build())
258+
.setMode(Field.Mode.REPEATED)
259+
.build();
234260
private static final Field INTEGER_FIELD_SCHEMA =
235261
Field.newBuilder("IntegerField", LegacySQLTypeName.INTEGER)
236262
.setMode(Field.Mode.NULLABLE)
@@ -422,6 +448,18 @@ public class ITBigQueryTest {
422448
.setMode(Field.Mode.NULLABLE)
423449
.build());
424450

451+
private static final Schema REPEATED_RECORD_TABLE_SCHEMA =
452+
Schema.of(
453+
Field.newBuilder("ID", LegacySQLTypeName.STRING).setMode(Field.Mode.NULLABLE).build(),
454+
Field.newBuilder("FirstName", LegacySQLTypeName.STRING)
455+
.setMode(Field.Mode.NULLABLE)
456+
.build(),
457+
Field.newBuilder("LastName", LegacySQLTypeName.STRING)
458+
.setMode(Field.Mode.NULLABLE)
459+
.build(),
460+
Field.newBuilder("DOB", LegacySQLTypeName.DATE).setMode(Field.Mode.NULLABLE).build(),
461+
REPEATED_RECORD_FIELD_SCHEMA);
462+
425463
private static final Schema SIMPLE_SCHEMA = Schema.of(STRING_FIELD_SCHEMA);
426464
private static final Schema QUERY_RESULT_SCHEMA =
427465
Schema.of(
@@ -4062,6 +4100,214 @@ public void testStructNamedQueryParameters() throws InterruptedException {
40624100
}
40634101
}
40644102

4103+
@Test
4104+
public void testRepeatedRecordNamedQueryParameters() throws InterruptedException {
4105+
String[] stringValues = new String[] {"test-stringField", "test-stringField2"};
4106+
List<QueryParameterValue> tuples = new ArrayList<>();
4107+
for (int i = 0; i < 2; i++) {
4108+
QueryParameterValue stringValue = QueryParameterValue.string(stringValues[i]);
4109+
Map<String, QueryParameterValue> struct = new HashMap<>();
4110+
struct.put("stringField", stringValue);
4111+
QueryParameterValue recordValue = QueryParameterValue.struct(struct);
4112+
tuples.add(recordValue);
4113+
}
4114+
4115+
QueryParameterValue repeatedRecord =
4116+
QueryParameterValue.array(tuples.toArray(), StandardSQLTypeName.STRUCT);
4117+
String query = "SELECT @repeatedRecordField AS repeatedRecord";
4118+
QueryJobConfiguration config =
4119+
QueryJobConfiguration.newBuilder(query)
4120+
.setDefaultDataset(DATASET)
4121+
.setUseLegacySql(false)
4122+
.addNamedParameter("repeatedRecordField", repeatedRecord)
4123+
.build();
4124+
TableResult result = bigquery.query(config);
4125+
assertEquals(1, Iterables.size(result.getValues()));
4126+
4127+
FieldList subSchema = result.getSchema().getFields().get("repeatedRecord").getSubFields();
4128+
for (FieldValueList values : result.iterateAll()) {
4129+
for (FieldValue value : values) {
4130+
assertEquals(FieldValue.Attribute.REPEATED, value.getAttribute());
4131+
assertEquals(2, value.getRepeatedValue().size());
4132+
for (int i = 0; i < 2; i++) {
4133+
FieldValue record = value.getRepeatedValue().get(i);
4134+
assertEquals(FieldValue.Attribute.RECORD, record.getAttribute());
4135+
FieldValueList recordValue = record.getRecordValue();
4136+
assertEquals(
4137+
stringValues[i],
4138+
FieldValueList.of(recordValue, subSchema).get("stringField").getValue());
4139+
}
4140+
}
4141+
}
4142+
}
4143+
4144+
@Test
4145+
public void testUnnestRepeatedRecordNamedQueryParameter() throws InterruptedException {
4146+
Boolean[] boolValues = new Boolean[] {true, false};
4147+
List<QueryParameterValue> tuples = new ArrayList<>();
4148+
for (int i = 0; i < 2; i++) {
4149+
QueryParameterValue boolValue = QueryParameterValue.bool(boolValues[i]);
4150+
Map<String, QueryParameterValue> struct = new HashMap<>();
4151+
struct.put("boolField", boolValue);
4152+
QueryParameterValue recordValue = QueryParameterValue.struct(struct);
4153+
tuples.add(recordValue);
4154+
}
4155+
4156+
QueryParameterValue repeatedRecord =
4157+
QueryParameterValue.array(tuples.toArray(), StandardSQLTypeName.STRUCT);
4158+
String query =
4159+
"SELECT * FROM (SELECT STRUCT("
4160+
+ boolValues[0]
4161+
+ " AS boolField) AS repeatedRecord) WHERE repeatedRecord IN UNNEST(@repeatedRecordField)";
4162+
QueryJobConfiguration config =
4163+
QueryJobConfiguration.newBuilder(query)
4164+
.setDefaultDataset(DATASET)
4165+
.setUseLegacySql(false)
4166+
.addNamedParameter("repeatedRecordField", repeatedRecord)
4167+
.build();
4168+
TableResult result = bigquery.query(config);
4169+
assertEquals(1, Iterables.size(result.getValues()));
4170+
4171+
FieldList subSchema = result.getSchema().getFields().get("repeatedRecord").getSubFields();
4172+
for (FieldValueList values : result.iterateAll()) {
4173+
for (FieldValue value : values) {
4174+
assertEquals(FieldValue.Attribute.RECORD, value.getAttribute());
4175+
FieldValueList recordValue = value.getRecordValue();
4176+
assertEquals(
4177+
boolValues[0],
4178+
FieldValueList.of(recordValue, subSchema).get("boolField").getBooleanValue());
4179+
}
4180+
}
4181+
}
4182+
4183+
@Test
4184+
public void testUnnestRepeatedRecordNamedQueryParameterFromDataset() throws InterruptedException {
4185+
TableId tableId = TableId.of(DATASET, "test_repeated_record_table");
4186+
setUpRepeatedRecordTable(tableId);
4187+
4188+
List<QueryParameterValue> tuples = new ArrayList<>();
4189+
QueryParameterValue statusValue = QueryParameterValue.string("single");
4190+
QueryParameterValue addressValue = QueryParameterValue.string("123 this lane");
4191+
QueryParameterValue cityValue = QueryParameterValue.string("Toronto");
4192+
QueryParameterValue stateValue = QueryParameterValue.string("ON");
4193+
QueryParameterValue zipValue = QueryParameterValue.string("1h2j34");
4194+
QueryParameterValue numberOfYearsValue = QueryParameterValue.string("3");
4195+
4196+
Map<String, QueryParameterValue> struct = new LinkedHashMap<>();
4197+
struct.put("statusValue", statusValue);
4198+
struct.put("addressValue", addressValue);
4199+
struct.put("cityValue", cityValue);
4200+
struct.put("stateValue", stateValue);
4201+
struct.put("zipValue", zipValue);
4202+
struct.put("numberOfYearsValue", numberOfYearsValue);
4203+
QueryParameterValue recordValue = QueryParameterValue.struct(struct);
4204+
tuples.add(recordValue);
4205+
4206+
QueryParameterValue repeatedRecord =
4207+
QueryParameterValue.array(tuples.toArray(), StandardSQLTypeName.STRUCT);
4208+
4209+
String query =
4210+
"SELECT * FROM "
4211+
+ tableId.getTable()
4212+
+ ", UNNEST(@repeatedRecord) AS TEMP where TEMP IN UNNEST(addresses);";
4213+
QueryJobConfiguration queryConfig =
4214+
QueryJobConfiguration.newBuilder(query)
4215+
.setDefaultDataset(DATASET)
4216+
.setUseLegacySql(false)
4217+
.addNamedParameter("repeatedRecord", repeatedRecord)
4218+
.build();
4219+
TableResult results = bigquery.query(queryConfig);
4220+
4221+
assertEquals(1, Iterables.size(results.getValues()));
4222+
for (FieldValueList values : results.iterateAll()) {
4223+
assertEquals("1", values.get("ID").getStringValue());
4224+
assertEquals("first_name1", values.get("FirstName").getStringValue());
4225+
assertEquals(2, values.get("Addresses").getRecordValue().size());
4226+
}
4227+
}
4228+
4229+
private void setUpRepeatedRecordTable(TableId tableId) {
4230+
StandardTableDefinition tableDefinition =
4231+
StandardTableDefinition.of(REPEATED_RECORD_TABLE_SCHEMA);
4232+
TableInfo tableInfo = TableInfo.of(tableId, tableDefinition);
4233+
bigquery.create(tableInfo);
4234+
4235+
ImmutableMap.Builder<String, Object> builder1 = ImmutableMap.builder();
4236+
builder1.put("ID", "1");
4237+
builder1.put("FirstName", "first_name1");
4238+
builder1.put("LastName", "last_name1");
4239+
builder1.put("DOB", "1995-08-09");
4240+
builder1.put(
4241+
"Addresses",
4242+
ImmutableList.of(
4243+
ImmutableMap.of(
4244+
"Status", "single",
4245+
"Address", "123 this lane",
4246+
"City", "Toronto",
4247+
"State", "ON",
4248+
"Zip", "1h2j34",
4249+
"NumberOfYears", "3"),
4250+
ImmutableMap.of(
4251+
"Status", "couple",
4252+
"Address", "345 that lane",
4253+
"City", "Maple",
4254+
"State", "ON",
4255+
"Zip", "1h2j34",
4256+
"NumberOfYears", "5")));
4257+
4258+
ImmutableMap.Builder<String, Object> builder2 = ImmutableMap.builder();
4259+
builder2.put("ID", "2");
4260+
builder2.put("FirstName", "first_name2");
4261+
builder2.put("LastName", "last_name2");
4262+
builder2.put("DOB", "1992-03-19");
4263+
builder2.put(
4264+
"Addresses",
4265+
ImmutableList.of(
4266+
ImmutableMap.of(
4267+
"Status", "single",
4268+
"Address", "97 Kota lane",
4269+
"City", "Ottawa",
4270+
"State", "ON",
4271+
"Zip", "1h2j34",
4272+
"NumberOfYears", "3"),
4273+
ImmutableMap.of(
4274+
"Status", "couple",
4275+
"Address", "75 Malta lane",
4276+
"City", "Victoria",
4277+
"State", "AL",
4278+
"Zip", "1h2j34",
4279+
"NumberOfYears", "5")));
4280+
4281+
InsertAllRequest request =
4282+
InsertAllRequest.newBuilder(tableInfo.getTableId())
4283+
.addRow(builder1.build())
4284+
.addRow(builder2.build())
4285+
.build();
4286+
bigquery.insertAll(request);
4287+
}
4288+
4289+
@Test
4290+
public void testEmptyRepeatedRecordNamedQueryParameters() throws InterruptedException {
4291+
QueryParameterValue[] tuples = {};
4292+
4293+
QueryParameterValue repeatedRecord =
4294+
QueryParameterValue.array(tuples, StandardSQLTypeName.STRUCT);
4295+
String query =
4296+
"SELECT * FROM (SELECT STRUCT(false AS boolField) AS repeatedRecord) WHERE repeatedRecord IN UNNEST(@repeatedRecordField)";
4297+
QueryJobConfiguration config =
4298+
QueryJobConfiguration.newBuilder(query)
4299+
.setDefaultDataset(DATASET)
4300+
.setUseLegacySql(false)
4301+
.addNamedParameter("repeatedRecordField", repeatedRecord)
4302+
.build();
4303+
try {
4304+
bigquery.query(config);
4305+
fail("an empty array of struct query parameter shouldn't work with 'IN UNNEST'");
4306+
} catch (BigQueryException e) {
4307+
// Nothing to do
4308+
}
4309+
}
4310+
40654311
@Test
40664312
public void testStructQuery() throws InterruptedException {
40674313
// query into a table

0 commit comments

Comments
 (0)