Skip to content

Commit e2fcfbf

Browse files
dgray16dgray16
committed
[bq] test BQ with Docker emulator
1 parent 6f3277a commit e2fcfbf

21 files changed

+340
-473
lines changed

‎spring-batch-bigquery/pom.xml

+9-3
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,12 @@
112112
<version>2.0.16</version>
113113
<scope>test</scope>
114114
</dependency>
115-
115+
<dependency>
116+
<groupId>org.testcontainers</groupId>
117+
<artifactId>junit-jupiter</artifactId>
118+
<version>1.20.1</version>
119+
<scope>test</scope>
120+
</dependency>
116121

117122
</dependencies>
118123

@@ -136,8 +141,9 @@
136141
<version>3.3.1</version>
137142
<configuration>
138143
<includes>
139-
<!-- Integration tests are omitted because they are designed to be run locally -->
140-
<include>/unit</include>
144+
<!-- Google cloud tests are omitted because they are designed to be run locally -->
145+
<include>**/unit/**</include>
146+
<include>**/emulator/**</include>
141147
</includes>
142148
</configuration>
143149
</plugin>

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/BigQueryDataLoader.java

-23
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,7 @@
2222
import com.google.cloud.bigquery.TableId;
2323
import com.google.cloud.bigquery.WriteChannelConfiguration;
2424
import org.springframework.batch.extensions.bigquery.writer.BigQueryCsvItemWriter;
25-
import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter;
2625
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryCsvItemWriterBuilder;
27-
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder;
2826
import org.springframework.batch.item.Chunk;
2927

3028
import java.util.Comparator;
@@ -69,25 +67,4 @@ public void loadCsvSample(String tableName) throws Exception {
6967
job.get().waitFor();
7068
}
7169

72-
public void loadJsonSample(String tableName) throws Exception {
73-
AtomicReference<Job> job = new AtomicReference<>();
74-
75-
WriteChannelConfiguration channelConfiguration = WriteChannelConfiguration
76-
.newBuilder(TableId.of(TestConstants.DATASET, tableName))
77-
.setSchema(PersonDto.getBigQuerySchema())
78-
.setAutodetect(false)
79-
.setFormatOptions(FormatOptions.json())
80-
.build();
81-
82-
BigQueryJsonItemWriter<PersonDto> writer = new BigQueryJsonItemWriterBuilder<PersonDto>()
83-
.bigQuery(bigQuery)
84-
.writeChannelConfig(channelConfiguration)
85-
.jobConsumer(job::set)
86-
.build();
87-
88-
writer.afterPropertiesSet();
89-
writer.write(CHUNK);
90-
job.get().waitFor();
91-
}
92-
9370
}

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/TestConstants.java

+2
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ private TestConstants() {}
2626
public static final String DATASET = "spring_batch_extensions";
2727
public static final String NAME = "name";
2828
public static final String AGE = "age";
29+
public static final String CSV = "csv";
30+
public static final String JSON = "json";
2931

3032
public static final Converter<FieldValueList, PersonDto> PERSON_MAPPER = res -> new PersonDto(
3133
res.get(NAME).getStringValue(), Long.valueOf(res.get(AGE).getLongValue()).intValue()

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/emulator/reader/BaseEmulatorItemReaderTest.java

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package org.springframework.batch.extensions.bigquery.emulator.reader;
2+
3+
import com.google.cloud.NoCredentials;
4+
import com.google.cloud.bigquery.BigQuery;
5+
import com.google.cloud.bigquery.BigQueryOptions;
6+
import org.junit.jupiter.api.BeforeAll;
7+
import org.testcontainers.containers.GenericContainer;
8+
import org.testcontainers.junit.jupiter.Container;
9+
import org.testcontainers.junit.jupiter.Testcontainers;
10+
import org.testcontainers.utility.MountableFile;
11+
12+
@Testcontainers
13+
abstract class BaseEmulatorItemReaderTest {
14+
private static final int PORT = 9050;
15+
16+
private static final String PROJECT = "batch-test";
17+
18+
@Container
19+
private static final GenericContainer<?> CONTAINER = new GenericContainer<>("ghcr.io/goccy/bigquery-emulator")
20+
.withExposedPorts(PORT)
21+
.withCommand("--project=" + PROJECT, "--data-from-yaml=/test-data.yaml")
22+
.withCopyFileToContainer(MountableFile.forClasspathResource("test-data.yaml"), "/test-data.yaml");
23+
24+
protected static BigQuery bigQuery;
25+
26+
@BeforeAll
27+
static void init() {
28+
bigQuery = BigQueryOptions
29+
.newBuilder()
30+
.setHost("http://%s:%d".formatted(CONTAINER.getHost(), CONTAINER.getMappedPort(PORT)))
31+
.setProjectId(PROJECT)
32+
.setCredentials(NoCredentials.getInstance())
33+
.build()
34+
.getService();
35+
}
36+
}

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/emulator/reader/BigQueryEmulatorItemReaderTest.java

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package org.springframework.batch.extensions.bigquery.emulator.reader;
2+
3+
import com.google.cloud.bigquery.*;
4+
import org.junit.jupiter.api.Assertions;
5+
import org.junit.jupiter.api.Test;
6+
import org.springframework.batch.extensions.bigquery.common.PersonDto;
7+
import org.springframework.batch.extensions.bigquery.common.TestConstants;
8+
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
9+
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
10+
11+
class BigQueryEmulatorItemReaderTest extends BaseEmulatorItemReaderTest {
12+
13+
@Test
14+
void testBatchReader() throws Exception {
15+
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
16+
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.csv p ORDER BY p.name LIMIT 2")
17+
.setDestinationTable(TableId.of(TestConstants.DATASET, TestConstants.CSV))
18+
.setPriority(QueryJobConfiguration.Priority.BATCH)
19+
.build();
20+
21+
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
22+
.bigQuery(bigQuery)
23+
.rowMapper(TestConstants.PERSON_MAPPER)
24+
.jobConfiguration(jobConfiguration)
25+
.build();
26+
27+
reader.afterPropertiesSet();
28+
29+
verifyResult(reader);
30+
}
31+
32+
@Test
33+
void testInteractiveReader() throws Exception {
34+
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
35+
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.csv p ORDER BY p.name LIMIT 2")
36+
.setDestinationTable(TableId.of(TestConstants.DATASET, TestConstants.CSV))
37+
.build();
38+
39+
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
40+
.bigQuery(bigQuery)
41+
.rowMapper(TestConstants.PERSON_MAPPER)
42+
.jobConfiguration(jobConfiguration)
43+
.build();
44+
45+
reader.afterPropertiesSet();
46+
47+
verifyResult(reader);
48+
}
49+
50+
private void verifyResult(BigQueryQueryItemReader<PersonDto> reader) throws Exception {
51+
PersonDto actual1 = reader.read();
52+
Assertions.assertEquals("Volodymyr", actual1.name());
53+
Assertions.assertEquals(27, actual1.age());
54+
55+
PersonDto actual2 = reader.read();
56+
Assertions.assertEquals("Oleksandra", actual2.name());
57+
Assertions.assertEquals(26, actual2.age());
58+
}
59+
}

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/emulator/writer/JsonWriterTest.java

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package org.springframework.batch.extensions.bigquery.emulator.writer;
2+
3+
// TODO
4+
public class JsonWriterTest {
5+
}

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/base/BaseBigQueryIntegrationTest.java renamed to ‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/base/BaseBigQueryGcloudIntegrationTest.java

+3-17
Original file line numberDiff line numberDiff line change
@@ -14,25 +14,11 @@
1414
* limitations under the License.
1515
*/
1616

17-
package org.springframework.batch.extensions.bigquery.integration.base;
17+
package org.springframework.batch.extensions.bigquery.gcloud.base;
1818

1919
import com.google.cloud.bigquery.BigQuery;
2020
import com.google.cloud.bigquery.BigQueryOptions;
21-
import org.junit.jupiter.api.TestInfo;
2221

23-
import java.lang.reflect.Method;
24-
25-
public abstract class BaseBigQueryIntegrationTest {
26-
27-
private static final String TABLE_PATTERN = "%s_%s";
28-
29-
public final BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService();
30-
31-
protected String getTableName(TestInfo testInfo) {
32-
return String.format(
33-
TABLE_PATTERN,
34-
testInfo.getTags().iterator().next(),
35-
testInfo.getTestMethod().map(Method::getName).orElseThrow()
36-
);
37-
}
22+
public abstract class BaseBigQueryGcloudIntegrationTest {
23+
protected static final BigQuery BIG_QUERY = BigQueryOptions.getDefaultInstance().getService();
3824
}

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/package-info.java renamed to ‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/package-info.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,4 @@
2525
*
2626
* @see <a href="https://cloud.google.com/bigquery/docs/quickstarts/quickstart-client-libraries#before-you-begin">Authentication</a>
2727
*/
28-
package org.springframework.batch.extensions.bigquery.integration;
28+
package org.springframework.batch.extensions.bigquery.gcloud;

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/batch/BigQueryBatchQueryCsvItemReaderTest.java renamed to ‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/reader/BigQueryGcloudItemReaderTest.java

+48-22
Original file line numberDiff line numberDiff line change
@@ -14,50 +14,76 @@
1414
* limitations under the License.
1515
*/
1616

17-
package org.springframework.batch.extensions.bigquery.integration.reader.batch;
18-
19-
import com.google.cloud.bigquery.QueryJobConfiguration;
20-
import com.google.cloud.bigquery.TableId;
21-
import org.junit.jupiter.api.Assertions;
22-
import org.junit.jupiter.api.Tag;
23-
import org.junit.jupiter.api.Test;
24-
import org.junit.jupiter.api.TestInfo;
17+
package org.springframework.batch.extensions.bigquery.gcloud.reader;
18+
19+
import com.google.cloud.bigquery.*;
20+
import org.junit.jupiter.api.*;
2521
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
2622
import org.springframework.batch.extensions.bigquery.common.PersonDto;
2723
import org.springframework.batch.extensions.bigquery.common.TestConstants;
28-
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
24+
import org.springframework.batch.extensions.bigquery.gcloud.base.BaseBigQueryGcloudIntegrationTest;
2925
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
3026
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
31-
import org.springframework.batch.item.Chunk;
3227

33-
@Tag("csv")
34-
class BigQueryBatchQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
28+
class BigQueryGcloudItemReaderTest extends BaseBigQueryGcloudIntegrationTest {
3529

36-
@Test
37-
void batchQueryTest1(TestInfo testInfo) throws Exception {
38-
String tableName = getTableName(testInfo);
39-
new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
40-
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
30+
@BeforeAll
31+
static void init() throws Exception {
32+
if (BIG_QUERY.getDataset(TestConstants.DATASET) == null) {
33+
BIG_QUERY.create(DatasetInfo.of(TestConstants.DATASET));
34+
}
35+
36+
if (BIG_QUERY.getTable(TestConstants.DATASET, TestConstants.CSV) == null) {
37+
TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema());
38+
BIG_QUERY.create(TableInfo.of(TableId.of(TestConstants.DATASET, TestConstants.CSV), tableDefinition));
39+
}
40+
41+
new BigQueryDataLoader(BIG_QUERY).loadCsvSample(TestConstants.CSV);
42+
}
4143

44+
@AfterAll
45+
static void cleanupTest() {
46+
BIG_QUERY.delete(TableId.of(TestConstants.DATASET, TestConstants.CSV));
47+
}
48+
49+
@Test
50+
void testBatchQuery() throws Exception {
4251
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
43-
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2")
44-
.setDestinationTable(TableId.of(TestConstants.DATASET, tableName))
52+
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2".formatted(TestConstants.CSV))
53+
.setDestinationTable(TableId.of(TestConstants.DATASET, TestConstants.CSV))
4554
.setPriority(QueryJobConfiguration.Priority.BATCH)
4655
.build();
4756

4857
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
49-
.bigQuery(bigQuery)
58+
.bigQuery(BIG_QUERY)
5059
.rowMapper(TestConstants.PERSON_MAPPER)
5160
.jobConfiguration(jobConfiguration)
5261
.build();
5362

5463
reader.afterPropertiesSet();
5564

65+
verifyResult(reader);
66+
}
67+
68+
@Test
69+
void testInteractiveQuery() throws Exception {
70+
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
71+
.bigQuery(BIG_QUERY)
72+
.query("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2".formatted(TestConstants.CSV))
73+
.rowMapper(TestConstants.PERSON_MAPPER)
74+
.build();
75+
76+
reader.afterPropertiesSet();
77+
78+
verifyResult(reader);
79+
}
80+
81+
private void verifyResult(BigQueryQueryItemReader<PersonDto> reader) throws Exception {
5682
PersonDto actualFirstPerson = reader.read();
57-
PersonDto expectedFirstPerson = chunk.getItems().get(0);
83+
PersonDto expectedFirstPerson = BigQueryDataLoader.CHUNK.getItems().get(0);
5884

5985
PersonDto actualSecondPerson = reader.read();
60-
PersonDto expectedSecondPerson = chunk.getItems().get(1);
86+
PersonDto expectedSecondPerson = BigQueryDataLoader.CHUNK.getItems().get(1);
6187

6288
PersonDto actualThirdPerson = reader.read();
6389

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryCsvItemWriterTest.java renamed to ‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/writer/BaseBigQueryGcloudItemWriterTest.java

+11-25
Original file line numberDiff line numberDiff line change
@@ -14,49 +14,35 @@
1414
* limitations under the License.
1515
*/
1616

17-
package org.springframework.batch.extensions.bigquery.integration.writer;
17+
package org.springframework.batch.extensions.bigquery.gcloud.writer;
1818

19-
import com.google.cloud.bigquery.BigQuery;
20-
import com.google.cloud.bigquery.Dataset;
21-
import com.google.cloud.bigquery.Table;
22-
import com.google.cloud.bigquery.TableId;
23-
import com.google.cloud.bigquery.TableResult;
19+
import com.google.cloud.bigquery.*;
2420
import org.junit.jupiter.api.Assertions;
25-
import org.junit.jupiter.api.Tag;
26-
import org.junit.jupiter.api.Test;
27-
import org.junit.jupiter.api.TestInfo;
2821
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
2922
import org.springframework.batch.extensions.bigquery.common.PersonDto;
3023
import org.springframework.batch.extensions.bigquery.common.TestConstants;
31-
import org.springframework.batch.extensions.bigquery.integration.writer.base.BaseBigQueryItemWriterTest;
32-
import org.springframework.batch.item.Chunk;
24+
import org.springframework.batch.extensions.bigquery.gcloud.base.BaseBigQueryGcloudIntegrationTest;
3325

34-
@Tag("csv")
35-
class BigQueryCsvItemWriterTest extends BaseBigQueryItemWriterTest {
26+
abstract class BaseBigQueryGcloudItemWriterTest extends BaseBigQueryGcloudIntegrationTest {
3627

37-
@Test
38-
void test1(TestInfo testInfo) throws Exception {
39-
String tableName = getTableName(testInfo);
40-
new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
41-
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
42-
43-
Dataset dataset = bigQuery.getDataset(TestConstants.DATASET);
44-
Table table = bigQuery.getTable(TableId.of(TestConstants.DATASET, tableName));
28+
protected void verifyResults(String tableName) {
29+
Dataset dataset = BIG_QUERY.getDataset(TestConstants.DATASET);
30+
Table table = BIG_QUERY.getTable(TableId.of(TestConstants.DATASET, tableName));
4531
TableId tableId = table.getTableId();
46-
TableResult tableResult = bigQuery.listTableData(tableId, BigQuery.TableDataListOption.pageSize(2L));
32+
TableResult tableResult = BIG_QUERY.listTableData(tableId, BigQuery.TableDataListOption.pageSize(2L));
4733

4834
Assertions.assertNotNull(dataset.getDatasetId());
4935
Assertions.assertNotNull(tableId);
50-
Assertions.assertEquals(chunk.size(), tableResult.getTotalRows());
36+
Assertions.assertEquals(BigQueryDataLoader.CHUNK.size(), tableResult.getTotalRows());
5137

5238
tableResult
5339
.getValues()
5440
.forEach(field -> {
5541
Assertions.assertTrue(
56-
chunk.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(0).getStringValue().equals(name))
42+
BigQueryDataLoader.CHUNK.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(0).getStringValue().equals(name))
5743
);
5844

59-
boolean ageCondition = chunk
45+
boolean ageCondition = BigQueryDataLoader.CHUNK
6046
.getItems()
6147
.stream()
6248
.map(PersonDto::age)

0 commit comments

Comments
 (0)