Skip to content

Commit 05fca58

Browse files
tonytangerTony Tanggcf-owl-bot[bot]
authored
feat: change stream retention to create and update table (#1823)
* feat: add change stream retention to create table Change-Id: I6eec06f3e3178143150490aa5fd97b83b1878cd2 * Remove creating change stream enabled table in IT because we can't delete change stream enabled table Change-Id: Ia2f92ccae3a2582da771b68a26adc1ab5f9d516e * feat: add update table with change stream retention Change-Id: I485507dd224eb0a3a160f7e9b5c569e1ae13ed84 * Disable change stream at end of IT so the table can be deleted Change-Id: I9b64e208ba3bf47f39a7bac3e4a3eb851b2c9468 * Add change stream retention to create table IT Change-Id: I8076da5631aa4dc4f97bba88c8f799303e81a65f * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Tony Tang <[email protected]> Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent 1893be7 commit 05fca58

File tree

8 files changed

+382
-6
lines changed

8 files changed

+382
-6
lines changed

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClient.java

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import com.google.cloud.bigtable.admin.v2.models.RestoredTableResult;
5151
import com.google.cloud.bigtable.admin.v2.models.Table;
5252
import com.google.cloud.bigtable.admin.v2.models.UpdateBackupRequest;
53+
import com.google.cloud.bigtable.admin.v2.models.UpdateTableRequest;
5354
import com.google.cloud.bigtable.admin.v2.stub.EnhancedBigtableTableAdminStub;
5455
import com.google.common.base.Preconditions;
5556
import com.google.common.collect.ImmutableList;
@@ -258,6 +259,72 @@ public ApiFuture<Table> createTableAsync(CreateTableRequest request) {
258259
this.stub.createTableCallable().futureCall(request.toProto(projectId, instanceId)));
259260
}
260261

262+
/**
263+
* Update a table with the specified configuration.
264+
*
265+
* <p>Sample code:
266+
*
267+
* <pre>{@code
268+
* // Alter change stream retention period.
269+
* Table table = client.updateTable(
270+
* UpdateTableRequest.of("my-table")
271+
* .addChangeStreamRetention(Duration.ofHours(24))
272+
* );
273+
*
274+
* // Disable change stream
275+
* Table table = client.updateTable(
276+
* UpdateTableRequest.of("my-table")
277+
* .disableChangeStream()
278+
* );
279+
* }</pre>
280+
*
281+
* @see UpdateTableRequest for available options.
282+
*/
283+
public Table updateTable(UpdateTableRequest request) {
284+
return ApiExceptions.callAndTranslateApiException(updateTableAsync(request));
285+
}
286+
287+
/**
288+
* Asynchronously update a table with the specified configuration.
289+
*
290+
* <p>Sample code:
291+
*
292+
* <pre>{@code
293+
* // Update table to 1 day change stream retention.
294+
* ApiFuture<Table> tableFuture = client.createTableAsync(
295+
* UpdateTableRequest.of("my-table")
296+
* .addChangeStreamRetention(Duration.ofHours(24))
297+
* );
298+
*
299+
* ApiFutures.addCallback(
300+
* tableFuture,
301+
* new ApiFutureCallback<Table>() {
302+
* public void onSuccess(Table table) {
303+
* System.out.println("Updated table: " + table.getTableName());
304+
* }
305+
*
306+
* public void onFailure(Throwable t) {
307+
* t.printStackTrace();
308+
* }
309+
* },
310+
* MoreExecutors.directExecutor()
311+
* );
312+
* }</pre>
313+
*
314+
* @see UpdateTableRequest for available options.
315+
*/
316+
public ApiFuture<Table> updateTableAsync(UpdateTableRequest request) {
317+
return ApiFutures.transform(
318+
stub.updateTableOperationCallable().futureCall(request.toProto(projectId, instanceId)),
319+
new ApiFunction<com.google.bigtable.admin.v2.Table, Table>() {
320+
@Override
321+
public Table apply(com.google.bigtable.admin.v2.Table tableProto) {
322+
return Table.fromProto(tableProto);
323+
}
324+
},
325+
MoreExecutors.directExecutor());
326+
}
327+
261328
/**
262329
* Creates, updates and drops column families as specified in the request.
263330
*

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/CreateTableRequest.java

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,15 @@
1616
package com.google.cloud.bigtable.admin.v2.models;
1717

1818
import com.google.api.core.InternalApi;
19+
import com.google.bigtable.admin.v2.ChangeStreamConfig;
1920
import com.google.bigtable.admin.v2.ColumnFamily;
2021
import com.google.cloud.bigtable.admin.v2.internal.NameUtil;
2122
import com.google.cloud.bigtable.admin.v2.models.GCRules.GCRule;
2223
import com.google.common.base.Objects;
2324
import com.google.common.base.Preconditions;
2425
import com.google.protobuf.ByteString;
2526
import javax.annotation.Nonnull;
27+
import org.threeten.bp.Duration;
2628

2729
/**
2830
* Fluent wrapper for {@link com.google.bigtable.admin.v2.CreateTableRequest}
@@ -76,6 +78,22 @@ public CreateTableRequest addSplit(ByteString key) {
7678
return this;
7779
}
7880

81+
/** Add change stream retention period between 1 day and 7 days */
82+
public CreateTableRequest addChangeStreamRetention(Duration retention) {
83+
Preconditions.checkNotNull(retention);
84+
requestBuilder
85+
.getTableBuilder()
86+
.setChangeStreamConfig(
87+
ChangeStreamConfig.newBuilder()
88+
.setRetentionPeriod(
89+
com.google.protobuf.Duration.newBuilder()
90+
.setSeconds(retention.getSeconds())
91+
.setNanos(retention.getNano())
92+
.build())
93+
.build());
94+
return this;
95+
}
96+
7997
@Override
8098
public boolean equals(Object o) {
8199
if (this == o) {

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/Table.java

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.util.Map;
2626
import java.util.Map.Entry;
2727
import javax.annotation.Nonnull;
28+
import org.threeten.bp.Duration;
2829

2930
/** Wrapper for {@link Table} protocol buffer object */
3031
public final class Table {
@@ -103,6 +104,8 @@ public com.google.bigtable.admin.v2.Table.ClusterState.ReplicationState toProto(
103104
private final Map<String, ReplicationState> replicationStatesByClusterId;
104105
private final List<ColumnFamily> columnFamilies;
105106

107+
private final Duration changeStreamRetention;
108+
106109
@InternalApi
107110
public static Table fromProto(@Nonnull com.google.bigtable.admin.v2.Table proto) {
108111
ImmutableMap.Builder<String, ReplicationState> replicationStates = ImmutableMap.builder();
@@ -120,18 +123,31 @@ public static Table fromProto(@Nonnull com.google.bigtable.admin.v2.Table proto)
120123
columnFamilies.add(ColumnFamily.fromProto(entry.getKey(), entry.getValue()));
121124
}
122125

126+
Duration changeStreamConfig = null;
127+
if (proto.hasChangeStreamConfig()) {
128+
changeStreamConfig =
129+
Duration.ofSeconds(
130+
proto.getChangeStreamConfig().getRetentionPeriod().getSeconds(),
131+
proto.getChangeStreamConfig().getRetentionPeriod().getNanos());
132+
}
133+
123134
return new Table(
124-
TableName.parse(proto.getName()), replicationStates.build(), columnFamilies.build());
135+
TableName.parse(proto.getName()),
136+
replicationStates.build(),
137+
columnFamilies.build(),
138+
changeStreamConfig);
125139
}
126140

127141
private Table(
128142
TableName tableName,
129143
Map<String, ReplicationState> replicationStatesByClusterId,
130-
List<ColumnFamily> columnFamilies) {
144+
List<ColumnFamily> columnFamilies,
145+
Duration changeStreamRetention) {
131146
this.instanceId = tableName.getInstance();
132147
this.id = tableName.getTable();
133148
this.replicationStatesByClusterId = replicationStatesByClusterId;
134149
this.columnFamilies = columnFamilies;
150+
this.changeStreamRetention = changeStreamRetention;
135151
}
136152

137153
/** Gets the table's id. */
@@ -152,6 +168,10 @@ public List<ColumnFamily> getColumnFamilies() {
152168
return columnFamilies;
153169
}
154170

171+
public Duration getChangeStreamRetention() {
172+
return changeStreamRetention;
173+
}
174+
155175
@Override
156176
public boolean equals(Object o) {
157177
if (this == o) {
@@ -164,11 +184,13 @@ public boolean equals(Object o) {
164184
return Objects.equal(id, table.id)
165185
&& Objects.equal(instanceId, table.instanceId)
166186
&& Objects.equal(replicationStatesByClusterId, table.replicationStatesByClusterId)
167-
&& Objects.equal(columnFamilies, table.columnFamilies);
187+
&& Objects.equal(columnFamilies, table.columnFamilies)
188+
&& Objects.equal(changeStreamRetention, table.changeStreamRetention);
168189
}
169190

170191
@Override
171192
public int hashCode() {
172-
return Objects.hashCode(id, instanceId, replicationStatesByClusterId, columnFamilies);
193+
return Objects.hashCode(
194+
id, instanceId, replicationStatesByClusterId, columnFamilies, changeStreamRetention);
173195
}
174196
}

‎google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/admin/v2/models/UpdateTableRequest.java

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Copyright 2023 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.bigtable.admin.v2.models;
18+
19+
import com.google.api.core.InternalApi;
20+
import com.google.bigtable.admin.v2.ChangeStreamConfig;
21+
import com.google.cloud.bigtable.admin.v2.internal.NameUtil;
22+
import com.google.common.base.Preconditions;
23+
import java.util.Objects;
24+
import org.threeten.bp.Duration;
25+
26+
/**
27+
* Wrapper for {@link com.google.bigtable.admin.v2.UpdateTableRequest}
28+
*
29+
* <p>Allows for updating table:
30+
*
31+
* <ul>
32+
* <li>Change stream retention period.
33+
* </ul>
34+
*/
35+
public class UpdateTableRequest {
36+
37+
private final String tableId;
38+
39+
private final com.google.bigtable.admin.v2.UpdateTableRequest.Builder requestBuilder =
40+
com.google.bigtable.admin.v2.UpdateTableRequest.newBuilder();
41+
42+
public static UpdateTableRequest of(String tableId) {
43+
return new UpdateTableRequest(tableId);
44+
}
45+
46+
private UpdateTableRequest(String tableId) {
47+
this.tableId = tableId;
48+
}
49+
50+
/** Update change stream retention period between 1 day and 7 days. */
51+
public UpdateTableRequest addChangeStreamRetention(Duration retention) {
52+
Preconditions.checkNotNull(retention);
53+
if (!retention.isZero()) {
54+
requestBuilder
55+
.getTableBuilder()
56+
.setChangeStreamConfig(
57+
ChangeStreamConfig.newBuilder()
58+
.setRetentionPeriod(
59+
com.google.protobuf.Duration.newBuilder()
60+
.setSeconds(retention.getSeconds())
61+
.setNanos(retention.getNano())
62+
.build())
63+
.build());
64+
requestBuilder.getUpdateMaskBuilder().addPaths("change_stream_config.retention_period");
65+
} else {
66+
requestBuilder.getTableBuilder().clearChangeStreamConfig();
67+
requestBuilder.getUpdateMaskBuilder().addPaths("change_stream_config");
68+
}
69+
return this;
70+
}
71+
72+
/** Disable change stream for table */
73+
public UpdateTableRequest disableChangeStreamRetention() {
74+
return addChangeStreamRetention(Duration.ZERO);
75+
}
76+
77+
@InternalApi
78+
public com.google.bigtable.admin.v2.UpdateTableRequest toProto(
79+
String projectId, String instanceId) {
80+
requestBuilder
81+
.getTableBuilder()
82+
.setName(NameUtil.formatTableName(projectId, instanceId, tableId));
83+
return requestBuilder.build();
84+
}
85+
86+
@Override
87+
public boolean equals(Object o) {
88+
if (this == o) return true;
89+
if (!(o instanceof UpdateTableRequest)) return false;
90+
UpdateTableRequest that = (UpdateTableRequest) o;
91+
return Objects.equals(requestBuilder, that.requestBuilder);
92+
}
93+
94+
@Override
95+
public int hashCode() {
96+
return Objects.hash(requestBuilder);
97+
}
98+
}

‎google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/BigtableTableAdminClientTests.java

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.google.api.gax.rpc.testing.FakeOperationSnapshot;
3030
import com.google.bigtable.admin.v2.Backup.State;
3131
import com.google.bigtable.admin.v2.BackupInfo;
32+
import com.google.bigtable.admin.v2.ChangeStreamConfig;
3233
import com.google.bigtable.admin.v2.ColumnFamily;
3334
import com.google.bigtable.admin.v2.CreateBackupMetadata;
3435
import com.google.bigtable.admin.v2.DeleteBackupRequest;
@@ -45,6 +46,7 @@
4546
import com.google.bigtable.admin.v2.Table.ClusterState;
4647
import com.google.bigtable.admin.v2.Table.View;
4748
import com.google.bigtable.admin.v2.TableName;
49+
import com.google.bigtable.admin.v2.UpdateTableMetadata;
4850
import com.google.cloud.Identity;
4951
import com.google.cloud.Policy;
5052
import com.google.cloud.Role;
@@ -68,6 +70,7 @@
6870
import com.google.common.io.BaseEncoding;
6971
import com.google.longrunning.Operation;
7072
import com.google.protobuf.ByteString;
73+
import com.google.protobuf.Duration;
7174
import com.google.protobuf.Empty;
7275
import com.google.protobuf.Timestamp;
7376
import com.google.protobuf.util.Timestamps;
@@ -117,6 +120,13 @@ public class BigtableTableAdminClientTests {
117120
com.google.bigtable.admin.v2.CreateTableRequest, com.google.bigtable.admin.v2.Table>
118121
mockCreateTableCallable;
119122

123+
@Mock
124+
private OperationCallable<
125+
com.google.bigtable.admin.v2.UpdateTableRequest,
126+
com.google.bigtable.admin.v2.Table,
127+
UpdateTableMetadata>
128+
mockUpdateTableOperationCallable;
129+
120130
@Mock
121131
private UnaryCallable<
122132
com.google.bigtable.admin.v2.ModifyColumnFamiliesRequest,
@@ -204,6 +214,40 @@ public void testCreateTable() {
204214
assertThat(result).isEqualTo(Table.fromProto(expectedResponse));
205215
}
206216

217+
@Test
218+
public void testUpdateTable() {
219+
// Setup
220+
Mockito.when(mockStub.updateTableOperationCallable())
221+
.thenReturn(mockUpdateTableOperationCallable);
222+
223+
com.google.cloud.bigtable.admin.v2.models.UpdateTableRequest request =
224+
com.google.cloud.bigtable.admin.v2.models.UpdateTableRequest.of(TABLE_ID)
225+
.addChangeStreamRetention(org.threeten.bp.Duration.ofHours(24));
226+
227+
com.google.bigtable.admin.v2.Table expectedResponse =
228+
com.google.bigtable.admin.v2.Table.newBuilder()
229+
.setName(TABLE_NAME)
230+
.setChangeStreamConfig(
231+
ChangeStreamConfig.newBuilder()
232+
.setRetentionPeriod(Duration.newBuilder().setSeconds(86400).build())
233+
.build())
234+
.build();
235+
236+
mockOperationResult(
237+
mockUpdateTableOperationCallable,
238+
request.toProto(PROJECT_ID, INSTANCE_ID),
239+
expectedResponse,
240+
UpdateTableMetadata.newBuilder().setName(TABLE_NAME).build());
241+
242+
// Execute
243+
Table actualResult = adminClient.updateTable(request);
244+
245+
// Verify
246+
assertThat(actualResult.getId()).isEqualTo(TABLE_ID);
247+
assertThat(actualResult.getChangeStreamRetention())
248+
.isEqualTo(org.threeten.bp.Duration.ofHours(24));
249+
}
250+
207251
@Test
208252
public void testModifyFamilies() {
209253
// Setup

0 commit comments

Comments
 (0)