[server] snapshots for PVC workspaces

Author: svenefftinge

components/server/ee/src/workspace/gitpod-server-impl.ts

@@ -51,7 +51,6 @@ import {
 } from "@gitpod/gitpod-protocol";
 import { ResponseError } from "vscode-jsonrpc";
 import {
-    TakeSnapshotRequest,
     AdmissionLevel,
     ControlAdmissionRequest,
     StopWorkspacePolicy,
@@ -530,18 +529,7 @@ export class GitpodServerEEImpl extends GitpodServerImpl {
         }
         await this.guardAccess({ kind: "workspaceInstance", subject: instance, workspace }, "get");
 
-        const client = await this.workspaceManagerClientProvider.get(
-            instance.region,
-            this.config.installationShortname,
-        );
-        const request = new TakeSnapshotRequest();
-        request.setId(instance.id);
-        request.setReturnImmediately(true);
-
-        // this triggers the snapshots, but returns early! cmp. waitForSnapshot to wait for it's completion
-        const resp = await client.takeSnapshot(ctx, request);
-
-        const snapshot = await this.snapshotService.createSnapshot(options, resp.getUrl());
+        const snapshot = await this.snapshotService.createSnapshot(ctx, instance);
 
         // to be backwards compatible during rollout, we require new clients to explicitly pass "dontWait: true"
         const waitOpts = { workspaceOwner: workspace.ownerId, snapshot };

components/server/ee/src/workspace/snapshot-service.ts

@@ -7,11 +7,15 @@
 import { inject, injectable } from "inversify";
 import { v4 as uuidv4 } from "uuid";
 import { WorkspaceDB } from "@gitpod/gitpod-db/lib";
-import { Disposable, GitpodServer, Snapshot } from "@gitpod/gitpod-protocol";
+import { Disposable, Snapshot, WorkspaceInstance } from "@gitpod/gitpod-protocol";
 import { StorageClient } from "../../../src/storage/storage-client";
 import { ConsensusLeaderQorum } from "../../../src/consensus/consensus-leader-quorum";
 import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
 import { repeat } from "@gitpod/gitpod-protocol/lib/util/repeat";
+import { WorkspaceManagerClientProvider } from "@gitpod/ws-manager/lib/client-provider";
+import { GetVolumeSnapshotRequest, TakeSnapshotRequest } from "@gitpod/ws-manager/lib";
+import { Config } from "../../../src/config";
+import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
 
 const SNAPSHOT_TIMEOUT_SECONDS = 60 * 30;
 const SNAPSHOT_POLL_INTERVAL_SECONDS = 5;
@@ -31,6 +35,9 @@ export class SnapshotService {
     @inject(WorkspaceDB) protected readonly workspaceDb: WorkspaceDB;
     @inject(StorageClient) protected readonly storageClient: StorageClient;
     @inject(ConsensusLeaderQorum) protected readonly leaderQuorum: ConsensusLeaderQorum;
+    @inject(WorkspaceManagerClientProvider)
+    protected readonly workspaceManagerClientProvider: WorkspaceManagerClientProvider;
+    @inject(Config) protected readonly config: Config;
 
     protected readonly runningSnapshots: Map<string, Promise<void>> = new Map();
 
@@ -74,14 +81,24 @@ export class SnapshotService {
         }
     }
 
-    public async createSnapshot(options: GitpodServer.TakeSnapshotOptions, snapshotUrl: string): Promise<Snapshot> {
+    public async createSnapshot(ctx: TraceContext, instance: WorkspaceInstance): Promise<Snapshot> {
+        const client = await this.workspaceManagerClientProvider.get(
+            instance.region,
+            this.config.installationShortname,
+        );
+        const request = new TakeSnapshotRequest();
+        request.setId(instance.id);
+        request.setReturnImmediately(true);
+
+        // this triggers the snapshots, but returns early! cmp. waitForSnapshot to wait for it's completion
+        const resp = await client.takeSnapshot(ctx, request);
         const id = uuidv4();
         return await this.workspaceDb.storeSnapshot({
             id,
             creationTime: new Date().toISOString(),
             state: "pending",
-            bucketId: snapshotUrl,
-            originalWorkspaceId: options.workspaceId,
+            bucketId: resp.getUrl(),
+            originalWorkspaceId: instance.workspaceId,
         });
     }
 
@@ -112,6 +129,24 @@ export class SnapshotService {
 
         const { id: snapshotId, bucketId, originalWorkspaceId, creationTime } = opts.snapshot;
         const start = new Date(creationTime).getTime();
+        const workspace = await this.workspaceDb.findWorkspaceAndInstance(originalWorkspaceId);
+        if (!workspace) {
+            const message = `Couldn't find original workspace for snapshot.`;
+            await this.workspaceDb.updateSnapshot({
+                id: snapshotId,
+                state: "error",
+                message,
+            });
+            throw new Error(message);
+        }
+        const client = await this.workspaceManagerClientProvider.get(
+            workspace.region,
+            this.config.installationShortname,
+        );
+        const req = new GetVolumeSnapshotRequest();
+        req.setId(workspace.instanceId);
+
+        const isPVC = workspace?.config._featureFlags?.some((f) => f === "persistent_volume_claim");
         while (start + SNAPSHOT_TIMEOUT_SECONDS * 1000 > Date.now()) {
             await new Promise((resolve) => setTimeout(resolve, SNAPSHOT_POLL_INTERVAL_SECONDS * 1000));
 
@@ -126,13 +161,19 @@ export class SnapshotService {
             if (snapshot.state === "error") {
                 throw new Error(`snapshot error: ${snapshot.message}`);
             }
+            let exists = false;
+            if (isPVC) {
+                const response = await client.getVolumeSnapshot({}, req);
+                exists = response.getReady();
+            } else {
+                // pending: check if the snapshot is there
+                exists = await this.storageClient.workspaceSnapshotExists(
+                    opts.workspaceOwner,
+                    originalWorkspaceId,
+                    bucketId,
+                );
+            }
 
-            // pending: check if the snapshot is there
-            const exists = await this.storageClient.workspaceSnapshotExists(
-                opts.workspaceOwner,
-                originalWorkspaceId,
-                bucketId,
-            );
             if (exists) {
                 await this.workspaceDb.updateSnapshot({
                     id: snapshotId,

components/ws-manager-api/typescript/src/promisified-client.ts

@@ -30,6 +30,8 @@ import {
     TakeSnapshotResponse,
     UpdateSSHKeyRequest,
     UpdateSSHKeyResponse,
+    GetVolumeSnapshotResponse,
+    GetVolumeSnapshotRequest,
 } from "./core_pb";
 import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
 import * as opentracing from "opentracing";
@@ -120,7 +122,7 @@ export class PromisifiedWorkspaceManagerClient implements Disposable {
                         {
                             // Important!!!!: client timeout must be higher than ws-manager to be able to process any error
                             // https://github.com/gitpod-io/gitpod/blob/main/components/ws-manager/pkg/manager/manager.go#L171
-                            deadline: new Date(new Date().getTime() + 60000*11),
+                            deadline: new Date(new Date().getTime() + 60000 * 11),
                             interceptors: this.interceptor,
                         },
                         (err, resp) => {
@@ -306,6 +308,30 @@ export class PromisifiedWorkspaceManagerClient implements Disposable {
         );
     }
 
+    public getVolumeSnapshot(ctx: TraceContext, request: GetVolumeSnapshotRequest): Promise<GetVolumeSnapshotResponse> {
+        return this.retryIfUnavailable(
+            (attempt: number) =>
+                new Promise<GetVolumeSnapshotResponse>((resolve, reject) => {
+                    const span = TraceContext.startSpan(`/ws-manager/getVolumeSnapshot`, ctx);
+                    span.log({ attempt });
+                    this.client.getVolumeSnapshot(
+                        request,
+                        withTracing({ span }),
+                        this.getDefaultUnaryOptions(),
+                        (err, resp) => {
+                            span.finish();
+                            if (err) {
+                                TraceContext.setError(ctx, err);
+                                reject(err);
+                            } else {
+                                resolve(resp);
+                            }
+                        },
+                    );
+                }),
+        );
+    }
+
     public controlAdmission(ctx: TraceContext, request: ControlAdmissionRequest): Promise<ControlAdmissionResponse> {
         // we do not use the default options here as takeSnapshot can take a very long time - much longer than the default deadline allows
         return this.retryIfUnavailable(