[server] mark prebuild as failed when image build fails

Author: AlexTugarev

components/server/ee/src/prebuilds/prebuild-manager.ts

@@ -97,6 +97,20 @@ export class PrebuildManager {
         }
     }
 
+    protected async findNonFailedPrebuiltWorkspace(ctx: TraceContext, cloneURL: string, commitSHA: string) {
+        const existingPB = await this.workspaceDB.trace(ctx).findPrebuiltWorkspaceByCommit(cloneURL, commitSHA);
+
+        if (
+            !!existingPB &&
+            existingPB.state !== "aborted" &&
+            existingPB.state !== "failed" &&
+            existingPB.state !== "timeout"
+        ) {
+            return existingPB;
+        }
+        return undefined;
+    }
+
     async startPrebuild(
         ctx: TraceContext,
         { context, project, user, commitInfo }: StartPrebuildParams,
@@ -111,16 +125,10 @@ export class PrebuildManager {
             if (user.blocked) {
                 throw new Error(`Blocked users cannot start prebuilds (${user.name})`);
             }
-            const existingPB = await this.workspaceDB
-                .trace({ span })
-                .findPrebuiltWorkspaceByCommit(cloneURL, commitSHAIdentifier);
-            // If the existing prebuild is failed, we want to retrigger it.
-            if (
-                !!existingPB &&
-                existingPB.state !== "aborted" &&
-                existingPB.state !== "failed" &&
-                existingPB.state !== "timeout"
-            ) {
+            const existingPB = await this.findNonFailedPrebuiltWorkspace({ span }, cloneURL, commitSHAIdentifier);
+
+            // If the existing prebuild is failed, it will be retriggered in the afterwards
+            if (existingPB) {
                 // If the existing prebuild is based on an outdated project config, we also want to retrigger it.
                 const existingPBWS = await this.workspaceDB.trace({ span }).findById(existingPB.buildWorkspaceId);
                 const existingConfig = existingPBWS?.config;

components/server/src/workspace/workspace-starter.ts

@@ -51,7 +51,6 @@ import {
     WorkspaceInstanceStatus,
     WorkspaceProbeContext,
     Permission,
-    HeadlessWorkspaceEvent,
     HeadlessWorkspaceEventType,
     DisposableCollection,
     AdditionalContentContext,
@@ -718,28 +717,39 @@ export class WorkspaceStarter {
             await this.messageBus.notifyOnInstanceUpdate(workspace.ownerId, instance);
 
             // If we just attempted to start a workspace for a prebuild - and that failed, we have to fail the prebuild itself.
+            await this.failPrebuildWorkspace({ span }, err, workspace);
+        } catch (err) {
+            TraceContext.setError({ span }, err);
+            log.error(
+                { workspaceId: workspace.id, instanceId: instance.id, userId: workspace.ownerId },
+                "cannot properly fail workspace instance during start",
+                err,
+            );
+        } finally {
+            span.finish();
+        }
+    }
+
+    protected async failPrebuildWorkspace(ctx: TraceContext, err: Error, workspace: Workspace) {
+        const span = TraceContext.startSpan("failInstanceStart", ctx);
+        try {
             if (workspace.type === "prebuild") {
                 const prebuild = await this.workspaceDb.trace({ span }).findPrebuildByWorkspaceID(workspace.id);
                 if (prebuild && prebuild.state !== "failed") {
                     prebuild.state = "failed";
                     prebuild.error = err.toString();
 
                     await this.workspaceDb.trace({ span }).storePrebuiltWorkspace(prebuild);
-                    await this.messageBus.notifyHeadlessUpdate({ span }, workspace.ownerId, workspace.id, <
-                        HeadlessWorkspaceEvent
-                    >{
+                    await this.messageBus.notifyHeadlessUpdate({ span }, workspace.ownerId, workspace.id, {
                         type: HeadlessWorkspaceEventType.Failed,
-                        // TODO: `workspaceID: workspace.id` not needed here? (found in ee/src/prebuilds/prebuild-queue-maintainer.ts and ee/src/bridge.ts)
+                        workspaceID: workspace.id, // required in prebuild-queue-maintainer.ts
+                        text: "",
                     });
                 }
             }
         } catch (err) {
             TraceContext.setError({ span }, err);
-            log.error(
-                { workspaceId: workspace.id, instanceId: instance.id, userId: workspace.ownerId },
-                "cannot properly fail workspace instance during start",
-                err,
-            );
+            throw err;
         } finally {
             span.finish();
         }
@@ -1286,6 +1296,11 @@ export class WorkspaceStarter {
                 stoppedTime: now,
                 stoppingTime: now,
             });
+
+            // Mark the PrebuildWorkspace as failed
+            await this.failPrebuildWorkspace({ span }, err, workspace);
+
+            // Push updated workspace instance over messagebus
             await this.messageBus.notifyOnInstanceUpdate(workspace.ownerId, instance);
 
             TraceContext.setError({ span }, err);
@@ -1419,14 +1434,16 @@ export class WorkspaceStarter {
 
         if (workspace.config.coreDump?.enabled) {
             // default core dump size is 262144 blocks (if blocksize is 4096)
-            const defaultLimit:number=1073741824;
+            const defaultLimit: number = 1073741824;
 
             const rLimitCore = new EnvironmentVariable();
             rLimitCore.setName("GITPOD_RLIMIT_CORE");
-            rLimitCore.setValue(JSON.stringify({
-                softLimit: workspace.config.coreDump?.softLimit || defaultLimit,
-                hardLimit: workspace.config.coreDump?.hardLimit || defaultLimit,
-            }));
+            rLimitCore.setValue(
+                JSON.stringify({
+                    softLimit: workspace.config.coreDump?.softLimit || defaultLimit,
+                    hardLimit: workspace.config.coreDump?.hardLimit || defaultLimit,
+                }),
+            );
             envvars.push(rLimitCore);
         }
 

components/ws-manager-bridge/ee/src/prebuild-updater-db.ts

@@ -7,7 +7,6 @@
 import { inject, injectable } from "inversify";
 import { TraceContext } from "@gitpod/gitpod-protocol/lib/util/tracing";
 import { WorkspaceStatus, WorkspaceType } from "@gitpod/ws-manager/lib";
-import { HeadlessWorkspaceEvent } from "@gitpod/gitpod-protocol/lib/headless-workspace-log";
 import { WorkspaceInstance } from "@gitpod/gitpod-protocol";
 import { log, LogContext } from "@gitpod/gitpod-protocol/lib/util/logging";
 import { PrebuildStateMapper } from "../../src/prebuild-state-mapper";
@@ -98,9 +97,10 @@ export class PrebuildUpdaterDB implements PrebuildUpdater {
 
                 // notify updates
                 // headless update
-                await this.messagebus.notifyHeadlessUpdate({ span }, userId, workspaceId, <HeadlessWorkspaceEvent>{
+                await this.messagebus.notifyHeadlessUpdate({ span }, userId, workspaceId, {
                     type: update.type,
                     workspaceID: workspaceId,
+                    text: "",
                 });
 
                 // prebuild info