Skip to content

Commit 4e31d04

Browse files
olavloitegcf-owl-bot[bot]burkedavison
authored
feat: support partitioned queries + data boost in Connection API (#2540)
* feat: support partitioned queries + data boost in Connection API Adds support for Partitioned Queries and Data Boost in the Connection API. This enables the use of these features in the JDBC driver and PGAdapter. * fix: match the correct group in regex * feat: add more SQL statements for partitioned queries * chore: refactor client side statements to accept the entire parsed statement Refactor the internal interface of client-side statements so these receive the entire parsed statement, including any query parameters in the statement. This allows us to create client-side statements that actually use the query parameters that have been specified by the user. * chore: simplify test * chore: cleanup differences * chore: cleanup unrelated changes * fix: update converter name * test: add more tests * chore: add missing license header * fix: handle empty partitioned queries correctly * fix: do not use any random staleness for partitioned queries * fix: only return false for next() if all have finished * chore: rename to autoPartitionMode * chore: rename sql statements + add tests for empty results * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * chore: address review comments * Batch read connection api native adjustments (#2569) * chore: add ClientSideStatementPartitionExecutor to SpannerFeature * chore: wrap AbstractStatementParser static initialization in try/catch * chore: add ClientSideStatementRunPartitionExecutor to SpannerFeature * chore: add ClientSideStatementRunPartitionedQueryExecutor to SpannerFeature * chore: lint formatting --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: Burke Davison <[email protected]>
1 parent b59940c commit 4e31d04

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+39843
-25929
lines changed

google-cloud-spanner/clirr-ignored-differences.xml

+56
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,62 @@
360360
<method>boolean isDelayTransactionStartUntilFirstWrite()</method>
361361
</difference>
362362

363+
<!-- Partitioned queries in Connection API -->
364+
<difference>
365+
<differenceType>7012</differenceType>
366+
<className>com/google/cloud/spanner/connection/Connection</className>
367+
<method>int getMaxPartitionedParallelism()</method>
368+
</difference>
369+
<difference>
370+
<differenceType>7012</differenceType>
371+
<className>com/google/cloud/spanner/connection/Connection</className>
372+
<method>int getMaxPartitions()</method>
373+
</difference>
374+
<difference>
375+
<differenceType>7012</differenceType>
376+
<className>com/google/cloud/spanner/connection/Connection</className>
377+
<method>boolean isAutoPartitionMode()</method>
378+
</difference>
379+
<difference>
380+
<differenceType>7012</differenceType>
381+
<className>com/google/cloud/spanner/connection/Connection</className>
382+
<method>boolean isDataBoostEnabled()</method>
383+
</difference>
384+
<difference>
385+
<differenceType>7012</differenceType>
386+
<className>com/google/cloud/spanner/connection/Connection</className>
387+
<method>com.google.cloud.spanner.ResultSet partitionQuery(com.google.cloud.spanner.Statement, com.google.cloud.spanner.PartitionOptions, com.google.cloud.spanner.Options$QueryOption[])</method>
388+
</difference>
389+
<difference>
390+
<differenceType>7012</differenceType>
391+
<className>com/google/cloud/spanner/connection/Connection</className>
392+
<method>com.google.cloud.spanner.ResultSet runPartition(java.lang.String)</method>
393+
</difference>
394+
<difference>
395+
<differenceType>7012</differenceType>
396+
<className>com/google/cloud/spanner/connection/Connection</className>
397+
<method>com.google.cloud.spanner.connection.PartitionedQueryResultSet runPartitionedQuery(com.google.cloud.spanner.Statement, com.google.cloud.spanner.PartitionOptions, com.google.cloud.spanner.Options$QueryOption[])</method>
398+
</difference>
399+
<difference>
400+
<differenceType>7012</differenceType>
401+
<className>com/google/cloud/spanner/connection/Connection</className>
402+
<method>void setAutoPartitionMode(boolean)</method>
403+
</difference>
404+
<difference>
405+
<differenceType>7012</differenceType>
406+
<className>com/google/cloud/spanner/connection/Connection</className>
407+
<method>void setDataBoostEnabled(boolean)</method>
408+
</difference>
409+
<difference>
410+
<differenceType>7012</differenceType>
411+
<className>com/google/cloud/spanner/connection/Connection</className>
412+
<method>void setMaxPartitionedParallelism(int)</method>
413+
</difference>
414+
<difference>
415+
<differenceType>7012</differenceType>
416+
<className>com/google/cloud/spanner/connection/Connection</className>
417+
<method>void setMaxPartitions(int)</method>
418+
</difference>
363419
<!-- (Internal change, use stream timeout) -->
364420
<difference>
365421
<differenceType>7012</differenceType>

google-cloud-spanner/src/main/java/com/google/cloud/spanner/Statement.java

+6
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,12 @@ private Builder(Statement statement) {
8686
statement.queryOptions == null ? null : statement.queryOptions.toBuilder().build();
8787
}
8888

89+
/** Replaces the current SQL of this builder with the given string. */
90+
public Builder replace(String sql) {
91+
sqlBuffer.replace(0, sqlBuffer.length(), sql);
92+
return this;
93+
}
94+
8995
/** Appends {@code sqlFragment} to the statement. */
9096
public Builder append(String sqlFragment) {
9197
sqlBuffer.append(checkNotNull(sqlFragment));

google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/AbstractBaseUnitOfWork.java

+44
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,22 @@
2121
import com.google.api.gax.grpc.GrpcCallContext;
2222
import com.google.api.gax.longrunning.OperationFuture;
2323
import com.google.api.gax.rpc.ApiCallContext;
24+
import com.google.cloud.spanner.BatchReadOnlyTransaction;
25+
import com.google.cloud.spanner.BatchTransactionId;
2426
import com.google.cloud.spanner.Dialect;
2527
import com.google.cloud.spanner.ErrorCode;
28+
import com.google.cloud.spanner.Options.QueryOption;
2629
import com.google.cloud.spanner.Options.RpcPriority;
30+
import com.google.cloud.spanner.Partition;
31+
import com.google.cloud.spanner.PartitionOptions;
32+
import com.google.cloud.spanner.ResultSet;
33+
import com.google.cloud.spanner.ResultSets;
2734
import com.google.cloud.spanner.SpannerException;
2835
import com.google.cloud.spanner.SpannerExceptionFactory;
2936
import com.google.cloud.spanner.SpannerOptions;
3037
import com.google.cloud.spanner.Statement;
38+
import com.google.cloud.spanner.Struct;
39+
import com.google.cloud.spanner.Type.StructField;
3140
import com.google.cloud.spanner.connection.AbstractStatementParser.ParsedStatement;
3241
import com.google.cloud.spanner.connection.StatementExecutor.StatementTimeout;
3342
import com.google.common.base.Preconditions;
@@ -39,13 +48,15 @@
3948
import java.util.Collection;
4049
import java.util.Collections;
4150
import java.util.HashSet;
51+
import java.util.List;
4252
import java.util.Set;
4353
import java.util.concurrent.Callable;
4454
import java.util.concurrent.CancellationException;
4555
import java.util.concurrent.ExecutionException;
4656
import java.util.concurrent.Future;
4757
import java.util.concurrent.TimeUnit;
4858
import java.util.concurrent.TimeoutException;
59+
import java.util.stream.Collectors;
4960
import javax.annotation.Nonnull;
5061
import javax.annotation.Nullable;
5162
import javax.annotation.concurrent.GuardedBy;
@@ -157,6 +168,39 @@ public void rollbackToSavepoint(
157168
"Rollback to savepoint is not supported for " + getUnitOfWorkName());
158169
}
159170

171+
@Override
172+
public ApiFuture<ResultSet> partitionQueryAsync(
173+
CallType callType,
174+
ParsedStatement query,
175+
PartitionOptions partitionOptions,
176+
QueryOption... options) {
177+
throw SpannerExceptionFactory.newSpannerException(
178+
ErrorCode.FAILED_PRECONDITION,
179+
"Partition query is not supported for " + getUnitOfWorkName());
180+
}
181+
182+
ResultSet partitionQuery(
183+
BatchReadOnlyTransaction transaction,
184+
PartitionOptions partitionOptions,
185+
ParsedStatement query,
186+
QueryOption... options) {
187+
final String partitionColumnName = "PARTITION";
188+
BatchTransactionId transactionId = transaction.getBatchTransactionId();
189+
List<Partition> partitions =
190+
transaction.partitionQuery(partitionOptions, query.getStatement(), options);
191+
return ResultSets.forRows(
192+
com.google.cloud.spanner.Type.struct(
193+
StructField.of(partitionColumnName, com.google.cloud.spanner.Type.string())),
194+
partitions.stream()
195+
.map(
196+
partition ->
197+
Struct.newBuilder()
198+
.set(partitionColumnName)
199+
.to(PartitionId.encodeToString(transactionId, partition))
200+
.build())
201+
.collect(Collectors.toList()));
202+
}
203+
160204
StatementExecutor getStatementExecutor() {
161205
return statementExecutor;
162206
}

google-cloud-spanner/src/main/java/com/google/cloud/spanner/connection/AbstractStatementParser.java

+21-11
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434
import java.util.Objects;
3535
import java.util.Set;
3636
import java.util.concurrent.Callable;
37+
import java.util.logging.Level;
38+
import java.util.logging.Logger;
3739

3840
/**
3941
* Internal class for the Spanner Connection API.
@@ -91,8 +93,7 @@ public static AbstractStatementParser getInstance(Dialect dialect) {
9193
*/
9294

9395
/** Begins a transaction. */
94-
static final ParsedStatement BEGIN_STATEMENT =
95-
AbstractStatementParser.getInstance(Dialect.GOOGLE_STANDARD_SQL).parse(Statement.of("BEGIN"));
96+
static final ParsedStatement BEGIN_STATEMENT;
9697

9798
/**
9899
* Create a COMMIT statement to use with the {@link #commit()} method to allow it to be cancelled,
@@ -104,14 +105,10 @@ public static AbstractStatementParser getInstance(Dialect dialect) {
104105
* #commit()} method is called directly, we do not have a {@link ParsedStatement}, and the method
105106
* uses this statement instead in order to use the same logic as the other statements.
106107
*/
107-
static final ParsedStatement COMMIT_STATEMENT =
108-
AbstractStatementParser.getInstance(Dialect.GOOGLE_STANDARD_SQL)
109-
.parse(Statement.of("COMMIT"));
108+
static final ParsedStatement COMMIT_STATEMENT;
110109

111110
/** The {@link Statement} and {@link Callable} for rollbacks */
112-
static final ParsedStatement ROLLBACK_STATEMENT =
113-
AbstractStatementParser.getInstance(Dialect.GOOGLE_STANDARD_SQL)
114-
.parse(Statement.of("ROLLBACK"));
111+
static final ParsedStatement ROLLBACK_STATEMENT;
115112

116113
/**
117114
* Create a RUN BATCH statement to use with the {@link #executeBatchUpdate(Iterable)} method to
@@ -124,9 +121,22 @@ public static AbstractStatementParser getInstance(Dialect dialect) {
124121
* and the method uses this statement instead in order to use the same logic as the other
125122
* statements.
126123
*/
127-
static final ParsedStatement RUN_BATCH_STATEMENT =
128-
AbstractStatementParser.getInstance(Dialect.GOOGLE_STANDARD_SQL)
129-
.parse(Statement.of("RUN BATCH"));
124+
static final ParsedStatement RUN_BATCH_STATEMENT;
125+
126+
static {
127+
try {
128+
BEGIN_STATEMENT = getInstance(Dialect.GOOGLE_STANDARD_SQL).parse(Statement.of("BEGIN"));
129+
COMMIT_STATEMENT = getInstance(Dialect.GOOGLE_STANDARD_SQL).parse(Statement.of("COMMIT"));
130+
ROLLBACK_STATEMENT = getInstance(Dialect.GOOGLE_STANDARD_SQL).parse(Statement.of("ROLLBACK"));
131+
RUN_BATCH_STATEMENT =
132+
getInstance(Dialect.GOOGLE_STANDARD_SQL).parse(Statement.of("RUN BATCH"));
133+
134+
} catch (Throwable ex) {
135+
Logger logger = Logger.getLogger(AbstractStatementParser.class.getName());
136+
logger.log(Level.SEVERE, "Static initialization failure.", ex);
137+
throw ex;
138+
}
139+
}
130140

131141
/** The type of statement that has been recognized by the parser. */
132142
@InternalApi
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner.connection;
18+
19+
import com.google.cloud.spanner.ErrorCode;
20+
import com.google.cloud.spanner.SpannerExceptionFactory;
21+
import com.google.cloud.spanner.Statement;
22+
import com.google.cloud.spanner.connection.AbstractStatementParser.ParsedStatement;
23+
import com.google.cloud.spanner.connection.ClientSideStatementImpl.CompileException;
24+
import java.lang.reflect.Method;
25+
import java.util.regex.Matcher;
26+
27+
/** Executor for <code>PARTITION &lt;sql&gt;</code> statements. */
28+
class ClientSideStatementPartitionExecutor implements ClientSideStatementExecutor {
29+
private final ClientSideStatementImpl statement;
30+
private final Method method;
31+
32+
ClientSideStatementPartitionExecutor(ClientSideStatementImpl statement) throws CompileException {
33+
try {
34+
this.statement = statement;
35+
this.method =
36+
ConnectionStatementExecutor.class.getDeclaredMethod(
37+
statement.getMethodName(), Statement.class);
38+
} catch (Exception e) {
39+
throw new CompileException(e, statement);
40+
}
41+
}
42+
43+
@Override
44+
public StatementResult execute(
45+
ConnectionStatementExecutor connection, ParsedStatement parsedStatement) throws Exception {
46+
String sql = getParameterValue(parsedStatement);
47+
return (StatementResult)
48+
method.invoke(connection, parsedStatement.getStatement().toBuilder().replace(sql).build());
49+
}
50+
51+
String getParameterValue(ParsedStatement parsedStatement) {
52+
Matcher matcher = statement.getPattern().matcher(parsedStatement.getSqlWithoutComments());
53+
if (matcher.find() && matcher.groupCount() >= 2) {
54+
String space = matcher.group(1);
55+
String value = matcher.group(2);
56+
return (space + value).trim();
57+
}
58+
throw SpannerExceptionFactory.newSpannerException(
59+
ErrorCode.INVALID_ARGUMENT,
60+
String.format(
61+
"Invalid argument for PARTITION: %s", parsedStatement.getStatement().getSql()));
62+
}
63+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner.connection;
18+
19+
import com.google.cloud.spanner.ErrorCode;
20+
import com.google.cloud.spanner.SpannerExceptionFactory;
21+
import com.google.cloud.spanner.Value;
22+
import com.google.cloud.spanner.connection.AbstractStatementParser.ParsedStatement;
23+
import com.google.cloud.spanner.connection.ClientSideStatementImpl.CompileException;
24+
import com.google.common.base.Strings;
25+
import java.lang.reflect.Method;
26+
import java.util.regex.Matcher;
27+
28+
/** Executor for <code>RUN PARTITION &lt;partition_id&gt;</code> statements. */
29+
class ClientSideStatementRunPartitionExecutor implements ClientSideStatementExecutor {
30+
private final ClientSideStatementImpl statement;
31+
private final Method method;
32+
33+
ClientSideStatementRunPartitionExecutor(ClientSideStatementImpl statement)
34+
throws CompileException {
35+
try {
36+
this.statement = statement;
37+
this.method =
38+
ConnectionStatementExecutor.class.getDeclaredMethod(
39+
statement.getMethodName(), String.class);
40+
} catch (Exception e) {
41+
throw new CompileException(e, statement);
42+
}
43+
}
44+
45+
@Override
46+
public StatementResult execute(
47+
ConnectionStatementExecutor connection, ParsedStatement parsedStatement) throws Exception {
48+
String partitionId = getParameterValue(parsedStatement);
49+
if (partitionId == null) {
50+
throw SpannerExceptionFactory.newSpannerException(
51+
ErrorCode.INVALID_ARGUMENT,
52+
"No valid partition id found in statement: " + parsedStatement.getStatement().getSql());
53+
}
54+
return (StatementResult) method.invoke(connection, partitionId);
55+
}
56+
57+
String getParameterValue(ParsedStatement parsedStatement) {
58+
// The statement has the form `RUN PARTITION ['partition-id']`.
59+
// The regex that is defined for this statement is (simplified) `run\s+partition(?:\s*'(.*)')?`
60+
// This regex has one capturing group, which captures the partition-id inside the single quotes.
61+
// That capturing group is however inside a non-capturing optional group.
62+
// That means that:
63+
// 1. If the matcher matches and returns one or more groups, we know that we have a partition-id
64+
// in the SQL statement itself, as that is the only thing that can be in a capturing group.
65+
// 2. If the matcher matches and returns zero groups, we know that the statement is valid, but
66+
// that it does not contain a partition-id in the SQL statement. The partition-id must then
67+
// be included in the statement as a query parameter.
68+
Matcher matcher = statement.getPattern().matcher(parsedStatement.getSqlWithoutComments());
69+
if (matcher.find() && matcher.groupCount() >= 1) {
70+
String value = matcher.group(1);
71+
if (!Strings.isNullOrEmpty(value)) {
72+
return value;
73+
}
74+
}
75+
if (parsedStatement.getStatement().getParameters().size() == 1) {
76+
Value value = parsedStatement.getStatement().getParameters().values().iterator().next();
77+
return value.getAsString();
78+
}
79+
return null;
80+
}
81+
}

0 commit comments

Comments
 (0)