Skip to content

Commit 7695536

Browse files
dgray16dgray16
committed
[bq] WIP
1 parent 6f3277a commit 7695536

20 files changed

+349
-471
lines changed

‎spring-batch-bigquery/pom.xml

+9-3
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,12 @@
112112
<version>2.0.16</version>
113113
<scope>test</scope>
114114
</dependency>
115-
115+
<dependency>
116+
<groupId>org.testcontainers</groupId>
117+
<artifactId>junit-jupiter</artifactId>
118+
<version>1.20.1</version>
119+
<scope>test</scope>
120+
</dependency>
116121

117122
</dependencies>
118123

@@ -136,8 +141,9 @@
136141
<version>3.3.1</version>
137142
<configuration>
138143
<includes>
139-
<!-- Integration tests are omitted because they are designed to be run locally -->
140-
<include>/unit</include>
144+
<!-- Google cloud tests are omitted because they are designed to be run locally -->
145+
<include>**/unit/**</include>
146+
<include>**/emulator/**</include>
141147
</includes>
142148
</configuration>
143149
</plugin>

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/BigQueryDataLoader.java

-21
Original file line numberDiff line numberDiff line change
@@ -69,25 +69,4 @@ public void loadCsvSample(String tableName) throws Exception {
6969
job.get().waitFor();
7070
}
7171

72-
public void loadJsonSample(String tableName) throws Exception {
73-
AtomicReference<Job> job = new AtomicReference<>();
74-
75-
WriteChannelConfiguration channelConfiguration = WriteChannelConfiguration
76-
.newBuilder(TableId.of(TestConstants.DATASET, tableName))
77-
.setSchema(PersonDto.getBigQuerySchema())
78-
.setAutodetect(false)
79-
.setFormatOptions(FormatOptions.json())
80-
.build();
81-
82-
BigQueryJsonItemWriter<PersonDto> writer = new BigQueryJsonItemWriterBuilder<PersonDto>()
83-
.bigQuery(bigQuery)
84-
.writeChannelConfig(channelConfiguration)
85-
.jobConsumer(job::set)
86-
.build();
87-
88-
writer.afterPropertiesSet();
89-
writer.write(CHUNK);
90-
job.get().waitFor();
91-
}
92-
9372
}

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/emulator/reader/BaseEmulatorItemReaderTest.java

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
package org.springframework.batch.extensions.bigquery.emulator.reader;
2+
3+
import com.google.cloud.NoCredentials;
4+
import com.google.cloud.bigquery.BigQuery;
5+
import com.google.cloud.bigquery.BigQueryOptions;
6+
import org.junit.jupiter.api.BeforeAll;
7+
import org.testcontainers.containers.GenericContainer;
8+
import org.testcontainers.junit.jupiter.Container;
9+
import org.testcontainers.junit.jupiter.Testcontainers;
10+
import org.testcontainers.utility.MountableFile;
11+
12+
@Testcontainers
13+
abstract class BaseEmulatorItemReaderTest {
14+
private static final int PORT = 9050;
15+
16+
private static final String PROJECT = "batch-test";
17+
18+
@Container
19+
private static final GenericContainer<?> CONTAINER = new GenericContainer<>("ghcr.io/goccy/bigquery-emulator")
20+
.withExposedPorts(PORT)
21+
.withCommand("--project=" + PROJECT, "--data-from-yaml=/test-data.yaml")
22+
.withCopyFileToContainer(MountableFile.forClasspathResource("test-data.yaml"), "/test-data.yaml");
23+
24+
protected static BigQuery bigQuery;
25+
26+
@BeforeAll
27+
static void init() {
28+
bigQuery = BigQueryOptions
29+
.newBuilder()
30+
.setHost("http://%s:%d".formatted(CONTAINER.getHost(), CONTAINER.getMappedPort(PORT)))
31+
.setProjectId(PROJECT)
32+
.setCredentials(NoCredentials.getInstance())
33+
.build()
34+
.getService();
35+
}
36+
}

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/emulator/reader/BigQueryEmulatorItemReaderTest.java

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package org.springframework.batch.extensions.bigquery.emulator.reader;
2+
3+
import com.google.cloud.bigquery.*;
4+
import org.junit.jupiter.api.Assertions;
5+
import org.junit.jupiter.api.Test;
6+
import org.springframework.batch.extensions.bigquery.common.PersonDto;
7+
import org.springframework.batch.extensions.bigquery.common.TestConstants;
8+
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
9+
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
10+
11+
class BigQueryEmulatorItemReaderTest extends BaseEmulatorItemReaderTest {
12+
13+
@Test
14+
void testBatchReader() throws Exception {
15+
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
16+
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.csv p ORDER BY p.name LIMIT 2")
17+
.setDestinationTable(TableId.of(TestConstants.DATASET, "csv"))
18+
.setPriority(QueryJobConfiguration.Priority.BATCH)
19+
.build();
20+
21+
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
22+
.bigQuery(bigQuery)
23+
.rowMapper(TestConstants.PERSON_MAPPER)
24+
.jobConfiguration(jobConfiguration)
25+
.build();
26+
27+
reader.afterPropertiesSet();
28+
29+
PersonDto actual1 = reader.read();
30+
Assertions.assertEquals("Volodymyr", actual1.name());
31+
Assertions.assertEquals(27, actual1.age());
32+
33+
PersonDto actual2 = reader.read();
34+
Assertions.assertEquals("Oleksandra", actual2.name());
35+
Assertions.assertEquals(26, actual2.age());
36+
}
37+
38+
@Test
39+
void testInteractiveReader() throws Exception {
40+
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
41+
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.csv p ORDER BY p.name LIMIT 2")
42+
.setDestinationTable(TableId.of(TestConstants.DATASET, "csv"))
43+
.build();
44+
45+
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
46+
.bigQuery(bigQuery)
47+
.rowMapper(TestConstants.PERSON_MAPPER)
48+
.jobConfiguration(jobConfiguration)
49+
.build();
50+
51+
reader.afterPropertiesSet();
52+
53+
PersonDto actual1 = reader.read();
54+
Assertions.assertEquals("Volodymyr", actual1.name());
55+
Assertions.assertEquals(27, actual1.age());
56+
57+
PersonDto actual2 = reader.read();
58+
Assertions.assertEquals("Oleksandra", actual2.name());
59+
Assertions.assertEquals(26, actual2.age());
60+
}
61+
62+
}

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/emulator/writer/JsonWriterTest.java

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package org.springframework.batch.extensions.bigquery.emulator.writer;
2+
3+
// TODO
4+
public class JsonWriterTest {
5+
}

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/base/BaseBigQueryIntegrationTest.java renamed to ‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/base/BaseBigQueryGcloudIntegrationTest.java

+3-17
Original file line numberDiff line numberDiff line change
@@ -14,25 +14,11 @@
1414
* limitations under the License.
1515
*/
1616

17-
package org.springframework.batch.extensions.bigquery.integration.base;
17+
package org.springframework.batch.extensions.bigquery.gcloud.base;
1818

1919
import com.google.cloud.bigquery.BigQuery;
2020
import com.google.cloud.bigquery.BigQueryOptions;
21-
import org.junit.jupiter.api.TestInfo;
2221

23-
import java.lang.reflect.Method;
24-
25-
public abstract class BaseBigQueryIntegrationTest {
26-
27-
private static final String TABLE_PATTERN = "%s_%s";
28-
29-
public final BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService();
30-
31-
protected String getTableName(TestInfo testInfo) {
32-
return String.format(
33-
TABLE_PATTERN,
34-
testInfo.getTags().iterator().next(),
35-
testInfo.getTestMethod().map(Method::getName).orElseThrow()
36-
);
37-
}
22+
public abstract class BaseBigQueryGcloudIntegrationTest {
23+
protected static final BigQuery BIG_QUERY = BigQueryOptions.getDefaultInstance().getService();
3824
}

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/package-info.java renamed to ‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/package-info.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,4 @@
2525
*
2626
* @see <a href="https://cloud.google.com/bigquery/docs/quickstarts/quickstart-client-libraries#before-you-begin">Authentication</a>
2727
*/
28-
package org.springframework.batch.extensions.bigquery.integration;
28+
package org.springframework.batch.extensions.bigquery.gcloud;

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/batch/BigQueryBatchQueryCsvItemReaderTest.java renamed to ‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/reader/BigQueryGcloudItemReaderTest.java

+50-22
Original file line numberDiff line numberDiff line change
@@ -14,50 +14,78 @@
1414
* limitations under the License.
1515
*/
1616

17-
package org.springframework.batch.extensions.bigquery.integration.reader.batch;
18-
19-
import com.google.cloud.bigquery.QueryJobConfiguration;
20-
import com.google.cloud.bigquery.TableId;
21-
import org.junit.jupiter.api.Assertions;
22-
import org.junit.jupiter.api.Tag;
23-
import org.junit.jupiter.api.Test;
24-
import org.junit.jupiter.api.TestInfo;
17+
package org.springframework.batch.extensions.bigquery.gcloud.reader;
18+
19+
import com.google.cloud.bigquery.*;
20+
import org.junit.jupiter.api.*;
2521
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
2622
import org.springframework.batch.extensions.bigquery.common.PersonDto;
2723
import org.springframework.batch.extensions.bigquery.common.TestConstants;
28-
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
24+
import org.springframework.batch.extensions.bigquery.gcloud.base.BaseBigQueryGcloudIntegrationTest;
2925
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
3026
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
31-
import org.springframework.batch.item.Chunk;
3227

33-
@Tag("csv")
34-
class BigQueryBatchQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
28+
class BigQueryGcloudItemReaderTest extends BaseBigQueryGcloudIntegrationTest {
3529

36-
@Test
37-
void batchQueryTest1(TestInfo testInfo) throws Exception {
38-
String tableName = getTableName(testInfo);
39-
new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
40-
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
30+
private static final String TABLE = "csv";
31+
32+
@BeforeAll
33+
static void init() throws Exception {
34+
if (BIG_QUERY.getDataset(TestConstants.DATASET) == null) {
35+
BIG_QUERY.create(DatasetInfo.of(TestConstants.DATASET));
36+
}
37+
38+
if (BIG_QUERY.getTable(TestConstants.DATASET, TABLE) == null) {
39+
TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema());
40+
BIG_QUERY.create(TableInfo.of(TableId.of(TestConstants.DATASET, TABLE), tableDefinition));
41+
}
42+
43+
new BigQueryDataLoader(BIG_QUERY).loadCsvSample(TABLE);
44+
}
45+
46+
@AfterAll
47+
static void cleanupTest() {
48+
BIG_QUERY.delete(TableId.of(TestConstants.DATASET, TABLE));
49+
}
4150

51+
@Test
52+
void testBatchQuery() throws Exception {
4253
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
43-
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2")
44-
.setDestinationTable(TableId.of(TestConstants.DATASET, tableName))
54+
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2".formatted(TABLE))
55+
.setDestinationTable(TableId.of(TestConstants.DATASET, TABLE))
4556
.setPriority(QueryJobConfiguration.Priority.BATCH)
4657
.build();
4758

4859
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
49-
.bigQuery(bigQuery)
60+
.bigQuery(BIG_QUERY)
5061
.rowMapper(TestConstants.PERSON_MAPPER)
5162
.jobConfiguration(jobConfiguration)
5263
.build();
5364

5465
reader.afterPropertiesSet();
5566

67+
verifyResult(reader);
68+
}
69+
70+
@Test
71+
void testInteractiveQuery() throws Exception {
72+
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
73+
.bigQuery(BIG_QUERY)
74+
.query("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2".formatted(TABLE))
75+
.rowMapper(TestConstants.PERSON_MAPPER)
76+
.build();
77+
78+
reader.afterPropertiesSet();
79+
80+
verifyResult(reader);
81+
}
82+
83+
private void verifyResult(BigQueryQueryItemReader<PersonDto> reader) throws Exception {
5684
PersonDto actualFirstPerson = reader.read();
57-
PersonDto expectedFirstPerson = chunk.getItems().get(0);
85+
PersonDto expectedFirstPerson = BigQueryDataLoader.CHUNK.getItems().get(0);
5886

5987
PersonDto actualSecondPerson = reader.read();
60-
PersonDto expectedSecondPerson = chunk.getItems().get(1);
88+
PersonDto expectedSecondPerson = BigQueryDataLoader.CHUNK.getItems().get(1);
6189

6290
PersonDto actualThirdPerson = reader.read();
6391

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryCsvItemWriterTest.java renamed to ‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/writer/BaseBigQueryGcloudItemWriterTest.java

+11-25
Original file line numberDiff line numberDiff line change
@@ -14,49 +14,35 @@
1414
* limitations under the License.
1515
*/
1616

17-
package org.springframework.batch.extensions.bigquery.integration.writer;
17+
package org.springframework.batch.extensions.bigquery.gcloud.writer;
1818

19-
import com.google.cloud.bigquery.BigQuery;
20-
import com.google.cloud.bigquery.Dataset;
21-
import com.google.cloud.bigquery.Table;
22-
import com.google.cloud.bigquery.TableId;
23-
import com.google.cloud.bigquery.TableResult;
19+
import com.google.cloud.bigquery.*;
2420
import org.junit.jupiter.api.Assertions;
25-
import org.junit.jupiter.api.Tag;
26-
import org.junit.jupiter.api.Test;
27-
import org.junit.jupiter.api.TestInfo;
2821
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
2922
import org.springframework.batch.extensions.bigquery.common.PersonDto;
3023
import org.springframework.batch.extensions.bigquery.common.TestConstants;
31-
import org.springframework.batch.extensions.bigquery.integration.writer.base.BaseBigQueryItemWriterTest;
32-
import org.springframework.batch.item.Chunk;
24+
import org.springframework.batch.extensions.bigquery.gcloud.base.BaseBigQueryGcloudIntegrationTest;
3325

34-
@Tag("csv")
35-
class BigQueryCsvItemWriterTest extends BaseBigQueryItemWriterTest {
26+
abstract class BaseBigQueryGcloudItemWriterTest extends BaseBigQueryGcloudIntegrationTest {
3627

37-
@Test
38-
void test1(TestInfo testInfo) throws Exception {
39-
String tableName = getTableName(testInfo);
40-
new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
41-
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
42-
43-
Dataset dataset = bigQuery.getDataset(TestConstants.DATASET);
44-
Table table = bigQuery.getTable(TableId.of(TestConstants.DATASET, tableName));
28+
protected void verifyResults(String tableName) {
29+
Dataset dataset = BIG_QUERY.getDataset(TestConstants.DATASET);
30+
Table table = BIG_QUERY.getTable(TableId.of(TestConstants.DATASET, tableName));
4531
TableId tableId = table.getTableId();
46-
TableResult tableResult = bigQuery.listTableData(tableId, BigQuery.TableDataListOption.pageSize(2L));
32+
TableResult tableResult = BIG_QUERY.listTableData(tableId, BigQuery.TableDataListOption.pageSize(2L));
4733

4834
Assertions.assertNotNull(dataset.getDatasetId());
4935
Assertions.assertNotNull(tableId);
50-
Assertions.assertEquals(chunk.size(), tableResult.getTotalRows());
36+
Assertions.assertEquals(BigQueryDataLoader.CHUNK.size(), tableResult.getTotalRows());
5137

5238
tableResult
5339
.getValues()
5440
.forEach(field -> {
5541
Assertions.assertTrue(
56-
chunk.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(0).getStringValue().equals(name))
42+
BigQueryDataLoader.CHUNK.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(0).getStringValue().equals(name))
5743
);
5844

59-
boolean ageCondition = chunk
45+
boolean ageCondition = BigQueryDataLoader.CHUNK
6046
.getItems()
6147
.stream()
6248
.map(PersonDto::age)

0 commit comments

Comments
 (0)