|
3 | 3 | package com.coder.gateway |
4 | 4 |
|
5 | 5 | import com.coder.gateway.models.WorkspaceProjectIDE |
| 6 | +import com.coder.gateway.models.toRawString |
6 | 7 | import com.coder.gateway.services.CoderRecentWorkspaceConnectionsService |
7 | 8 | import com.coder.gateway.services.CoderSettingsService |
8 | 9 | import com.coder.gateway.settings.CoderSettings |
|
21 | 22 | import com.intellij.openapi.rd.util.launchUnderBackgroundProgress |
22 | 23 | import com.intellij.openapi.ui.Messages |
23 | 24 | import com.intellij.openapi.ui.panel.ComponentPanelBuilder |
| 25 | +import com.intellij.remote.AuthType |
| 26 | +import com.intellij.remote.RemoteCredentialsHolder |
| 27 | +import com.intellij.remoteDev.hostStatus.UnattendedHostStatus |
24 | 28 | import com.intellij.ui.AppIcon |
25 | 29 | import com.intellij.ui.components.JBTextField |
26 | 30 | import com.intellij.ui.components.dialog |
27 | 31 | import com.intellij.ui.dsl.builder.RowLayout |
28 | 32 | import com.intellij.ui.dsl.builder.panel |
29 | 33 | import com.intellij.util.applyIf |
30 | 34 | import com.intellij.util.ui.UIUtil |
31 | | -import com.jetbrains.gateway.ssh.SshDeployFlowUtil |
32 | | -import com.jetbrains.gateway.ssh.SshMultistagePanelContext |
| 35 | +import com.jetbrains.gateway.ssh.ClientOverSshTunnelConnector |
| 36 | +import com.jetbrains.gateway.ssh.HighLevelHostAccessor |
| 37 | +import com.jetbrains.gateway.ssh.SshHostTunnelConnector |
33 | 38 | import com.jetbrains.gateway.ssh.deploy.DeployException |
| 39 | +import com.jetbrains.gateway.ssh.deploy.ShellArgument |
| 40 | +import com.jetbrains.gateway.ssh.deploy.TransferProgressTracker |
| 41 | +import com.jetbrains.gateway.ssh.util.validateIDEInstallPath |
34 | 42 | import com.jetbrains.rd.util.lifetime.LifetimeDefinition |
| 43 | +import com.jetbrains.rd.util.lifetime.LifetimeStatus |
| 44 | +import kotlinx.coroutines.delay |
| 45 | +import kotlinx.coroutines.isActive |
| 46 | +import kotlinx.coroutines.launch |
| 47 | +import kotlinx.coroutines.suspendCancellableCoroutine |
35 | 48 | import net.schmizz.sshj.common.SSHException |
36 | 49 | import net.schmizz.sshj.connection.ConnectionException |
| 50 | +import org.zeroturnaround.exec.ProcessExecutor |
37 | 51 | import java.awt.Dimension |
38 | 52 | import java.net.HttpURLConnection |
| 53 | +import java.net.URI |
39 | 54 | import java.net.URL |
40 | 55 | import java.time.Duration |
| 56 | +import java.time.LocalDateTime |
| 57 | +import java.time.format.DateTimeFormatter |
41 | 58 | import java.util.concurrent.TimeoutException |
42 | 59 | import javax.net.ssl.SSLHandshakeException |
| 60 | +import kotlin.coroutines.resume |
| 61 | +import kotlin.coroutines.resumeWithException |
43 | 62 |
|
44 | 63 | // CoderRemoteConnection uses the provided workspace SSH parameters to launch an |
45 | 64 | // IDE against the workspace. If successful the connection is added to recent |
46 | 65 | // connections. |
| 66 | +@Suppress("UnstableApiUsage") |
47 | 67 | class CoderRemoteConnectionHandle { |
48 | 68 | private val recentConnectionsService = service<CoderRecentWorkspaceConnectionsService>() |
49 | 69 | private val settings = service<CoderSettingsService>() |
50 | 70 |
|
| 71 | + private val localTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MMM-dd HH:mm") |
| 72 | + |
51 | 73 | fun connect(getParameters: (indicator: ProgressIndicator) -> WorkspaceProjectIDE) { |
52 | 74 | val clientLifetime = LifetimeDefinition() |
53 | 75 | clientLifetime.launchUnderBackgroundProgress(CoderGatewayBundle.message("gateway.connector.coder.connection.provider.title")) { |
54 | 76 | try { |
55 | 77 | val parameters = getParameters(indicator) |
56 | 78 | logger.debug("Creating connection handle", parameters) |
57 | 79 | indicator.text = CoderGatewayBundle.message("gateway.connector.coder.connecting") |
58 | | - val context = |
59 | | - suspendingRetryWithExponentialBackOff( |
60 | | - action = { attempt -> |
61 | | - logger.info("Connecting... (attempt $attempt)") |
62 | | - if (attempt > 1) { |
63 | | - // indicator.text is the text above the progress bar. |
64 | | - indicator.text = CoderGatewayBundle.message("gateway.connector.coder.connecting.retry", attempt) |
| 80 | + suspendingRetryWithExponentialBackOff( |
| 81 | + action = { attempt -> |
| 82 | + logger.info("Connecting... (attempt $attempt)") |
| 83 | + if (attempt > 1) { |
| 84 | + // indicator.text is the text above the progress bar. |
| 85 | + indicator.text = CoderGatewayBundle.message("gateway.connector.coder.connecting.retry", attempt) |
| 86 | + } |
| 87 | + doConnect( |
| 88 | + parameters, |
| 89 | + indicator, |
| 90 | + clientLifetime, |
| 91 | + settings.setupCommand, |
| 92 | + settings.ignoreSetupFailure, |
| 93 | + ) |
| 94 | + }, |
| 95 | + retryIf = { |
| 96 | + it is ConnectionException || it is TimeoutException || |
| 97 | + it is SSHException || it is DeployException |
| 98 | + }, |
| 99 | + onException = { attempt, nextMs, e -> |
| 100 | + logger.error("Failed to connect (attempt $attempt; will retry in $nextMs ms)") |
| 101 | + // indicator.text2 is the text below the progress bar. |
| 102 | + indicator.text2 = |
| 103 | + if (isWorkerTimeout(e)) { |
| 104 | + "Failed to upload worker binary...it may have timed out" |
| 105 | + } else { |
| 106 | + e.message ?: e.javaClass.simpleName |
65 | 107 | } |
66 | | - val deployInputs = |
67 | | - parameters.deploy( |
68 | | - indicator, |
69 | | - Duration.ofMinutes(10), |
70 | | - settings.setupCommand, |
71 | | - settings.ignoreSetupFailure, |
72 | | - ) |
73 | | - SshMultistagePanelContext(deployInputs) |
74 | | - }, |
75 | | - retryIf = { |
76 | | - it is ConnectionException || it is TimeoutException || |
77 | | - it is SSHException || it is DeployException |
78 | | - }, |
79 | | - onException = { attempt, nextMs, e -> |
80 | | - logger.error("Failed to connect (attempt $attempt; will retry in $nextMs ms)") |
81 | | - // indicator.text2 is the text below the progress bar. |
82 | | - indicator.text2 = |
83 | | - if (isWorkerTimeout(e)) { |
84 | | - "Failed to upload worker binary...it may have timed out" |
85 | | - } else { |
86 | | - e.message ?: e.javaClass.simpleName |
87 | | - } |
88 | | - }, |
89 | | - onCountdown = { remainingMs -> |
90 | | - indicator.text = |
91 | | - CoderGatewayBundle.message( |
92 | | - "gateway.connector.coder.connecting.failed.retry", |
93 | | - humanizeDuration(remainingMs), |
94 | | - ) |
95 | | - }, |
96 | | - ) |
97 | | - logger.info("Starting and connecting to ${parameters.ideName} with $context") |
98 | | - // At this point JetBrains takes over with their own UI. |
99 | | - @Suppress("UnstableApiUsage") |
100 | | - SshDeployFlowUtil.fullDeployCycle( |
101 | | - clientLifetime, |
102 | | - context, |
103 | | - Duration.ofMinutes(10), |
| 108 | + }, |
| 109 | + onCountdown = { remainingMs -> |
| 110 | + indicator.text = |
| 111 | + CoderGatewayBundle.message( |
| 112 | + "gateway.connector.coder.connecting.failed.retry", |
| 113 | + humanizeDuration(remainingMs), |
| 114 | + ) |
| 115 | + }, |
104 | 116 | ) |
105 | 117 | logger.info("Adding ${parameters.ideName} for ${parameters.hostname}:${parameters.projectPath} to recent connections") |
106 | 118 | recentConnectionsService.addRecentConnection(parameters.toRecentWorkspaceConnection()) |
|
123 | 135 | } |
124 | 136 | } |
125 | 137 |
|
| 138 | + /** |
| 139 | + * Deploy (if needed), connect to the IDE, and update the last opened date. |
| 140 | + */ |
| 141 | + private suspend fun doConnect( |
| 142 | + workspace: WorkspaceProjectIDE, |
| 143 | + indicator: ProgressIndicator, |
| 144 | + lifetime: LifetimeDefinition, |
| 145 | + setupCommand: String, |
| 146 | + ignoreSetupFailure: Boolean, |
| 147 | + timeout: Duration = Duration.ofMinutes(10), |
| 148 | + ): Unit { |
| 149 | + workspace.lastOpened = localTimeFormatter.format(LocalDateTime.now()) |
| 150 | + |
| 151 | + // This establishes an SSH connection to a remote worker binary. |
| 152 | + // TODO: Can/should accessors to the same host be shared? |
| 153 | + indicator.text = "Connecting to remote worker..." |
| 154 | + logger.info("Connecting to remote worker on ${workspace.hostname}") |
| 155 | + val credentials = RemoteCredentialsHolder().apply { |
| 156 | + setHost(workspace.hostname) |
| 157 | + userName = "coder" |
| 158 | + port = 22 |
| 159 | + authType = AuthType.OPEN_SSH |
| 160 | + } |
| 161 | + val accessor = HighLevelHostAccessor.create(credentials, true) |
| 162 | + |
| 163 | + // Deploy if we need to. |
| 164 | + val ideDir = this.deploy(workspace, accessor, indicator, timeout) |
| 165 | + workspace.idePathOnHost = ideDir.toRawString() |
| 166 | + |
| 167 | + // Run the setup command. |
| 168 | + this.setup(workspace, indicator, setupCommand, ignoreSetupFailure) |
| 169 | + |
| 170 | + // Wait for the IDE to come up. |
| 171 | + var status: UnattendedHostStatus? = null |
| 172 | + val remoteProjectPath = accessor.makeRemotePath(ShellArgument.PlainText(workspace.projectPath)) |
| 173 | + val logsDir = accessor.getLogsDir(workspace.ideProductCode.productCode, remoteProjectPath) |
| 174 | + while (lifetime.status == LifetimeStatus.Alive) { |
| 175 | + status = ensureIDEBackend(workspace, accessor, ideDir, remoteProjectPath, logsDir, lifetime, null) |
| 176 | + if (!status?.joinLink.isNullOrBlank()) { |
| 177 | + break |
| 178 | + } |
| 179 | + delay(5000) |
| 180 | + } |
| 181 | + |
| 182 | + // We wait for non-null, so this only happens on cancellation. |
| 183 | + val joinLink = status?.joinLink |
| 184 | + if (joinLink.isNullOrBlank()) { |
| 185 | + logger.info("Connection to ${workspace.ideName} on ${workspace.hostname} was canceled") |
| 186 | + return |
| 187 | + } |
| 188 | + |
| 189 | + // Make the initial connection. |
| 190 | + indicator.text = "Connecting ${workspace.ideName} client..." |
| 191 | + logger.info("Connecting ${workspace.ideName} client to coder@${workspace.hostname}:22") |
| 192 | + val client = ClientOverSshTunnelConnector(lifetime, SshHostTunnelConnector(credentials)) |
| 193 | + val handle = client.connect(URI(joinLink)) // Downloads the client too, if needed. |
| 194 | + |
| 195 | + // Reconnect if the join link changes. |
| 196 | + logger.info("Launched ${workspace.ideName} client; beginning backend monitoring") |
| 197 | + lifetime.coroutineScope.launch { |
| 198 | + while (isActive) { |
| 199 | + delay(5000) |
| 200 | + val newStatus = ensureIDEBackend(workspace, accessor, ideDir, remoteProjectPath, logsDir, lifetime, status) |
| 201 | + val newLink = newStatus?.joinLink |
| 202 | + if (newLink != null && newLink != status?.joinLink) { |
| 203 | + logger.info("${workspace.ideName} backend join link changed; updating") |
| 204 | + // Unfortunately, updating the link is not a smooth |
| 205 | + // reconnection. The client closes and is relaunched. |
| 206 | + // Trying to reconnect without updating the link results in |
| 207 | + // a fingerprint mismatch error. |
| 208 | + handle.updateJoinLink(URI(newLink), true) |
| 209 | + status = newStatus |
| 210 | + } |
| 211 | + } |
| 212 | + } |
| 213 | + |
| 214 | + // Tie the lifetime and client together, and wait for the initial open. |
| 215 | + suspendCancellableCoroutine { continuation -> |
| 216 | + // Close the client if the user cancels. |
| 217 | + lifetime.onTermination { |
| 218 | + logger.info("Connection to ${workspace.ideName} on ${workspace.hostname} canceled") |
| 219 | + if (continuation.isActive) { |
| 220 | + continuation.cancel() |
| 221 | + } |
| 222 | + handle.close() |
| 223 | + } |
| 224 | + // Kill the lifetime if the client is closed by the user. |
| 225 | + handle.clientClosed.advise(lifetime) { |
| 226 | + logger.info("${workspace.ideName} client ${workspace.hostname} closed") |
| 227 | + if (lifetime.status == LifetimeStatus.Alive) { |
| 228 | + if (continuation.isActive) { |
| 229 | + continuation.resumeWithException(Exception("${workspace.ideName} client was closed")) |
| 230 | + } |
| 231 | + lifetime.terminate() |
| 232 | + } |
| 233 | + } |
| 234 | + // Continue once the client is present. |
| 235 | + handle.onClientPresenceChanged.advise(lifetime) { |
| 236 | + if (handle.clientPresent && continuation.isActive) { |
| 237 | + continuation.resume(true) |
| 238 | + } |
| 239 | + } |
| 240 | + } |
| 241 | + |
| 242 | + // The presence handler runs a good deal earlier than the client |
| 243 | + // actually appears, which results in some dead space where it can look |
| 244 | + // like opening the client silently failed. This delay janks around |
| 245 | + // that, so we can keep the progress indicator open a bit longer. |
| 246 | + delay(5000) |
| 247 | + } |
| 248 | + |
| 249 | + /** |
| 250 | + * Deploy the IDE if necessary and return the path to its location on disk. |
| 251 | + */ |
| 252 | + private suspend fun deploy( |
| 253 | + workspace: WorkspaceProjectIDE, |
| 254 | + accessor: HighLevelHostAccessor, |
| 255 | + indicator: ProgressIndicator, |
| 256 | + timeout: Duration, |
| 257 | + ): ShellArgument.RemotePath { |
| 258 | + // The backend might already exist at the provided path. |
| 259 | + if (!workspace.idePathOnHost.isNullOrBlank()) { |
| 260 | + indicator.text = "Verifying ${workspace.ideName} installation..." |
| 261 | + logger.info("Verifying ${workspace.ideName} exists at ${workspace.hostname}:${workspace.idePathOnHost}") |
| 262 | + val validatedPath = validateIDEInstallPath(workspace.idePathOnHost, accessor).pathOrNull |
| 263 | + if (validatedPath != null) { |
| 264 | + logger.info("${workspace.ideName} exists at ${workspace.hostname}:${validatedPath.toRawString()}") |
| 265 | + return validatedPath |
| 266 | + } |
| 267 | + } |
| 268 | + |
| 269 | + // The backend might already be installed somewhere on the system. |
| 270 | + indicator.text = "Searching for ${workspace.ideName} installation..." |
| 271 | + logger.info("Searching for ${workspace.ideName} on ${workspace.hostname}") |
| 272 | + val installed = |
| 273 | + accessor.getInstalledIDEs().find { |
| 274 | + it.product == workspace.ideProductCode && it.buildNumber == workspace.ideBuildNumber |
| 275 | + } |
| 276 | + if (installed != null) { |
| 277 | + logger.info("${workspace.ideName} found at ${workspace.hostname}:${installed.pathToIde}") |
| 278 | + return accessor.makeRemotePath(ShellArgument.PlainText(installed.pathToIde)) |
| 279 | + } |
| 280 | + |
| 281 | + // Otherwise we have to download it. |
| 282 | + if (workspace.downloadSource.isNullOrBlank()) { |
| 283 | + throw Exception("${workspace.ideName} could not be found on the remote and no download source was provided") |
| 284 | + } |
| 285 | + |
| 286 | + // TODO: Should we download to idePathOnHost if set? That would require |
| 287 | + // symlinking instead of creating the sentinel file if the path is |
| 288 | + // outside the default dist directory. |
| 289 | + indicator.text = "Downloading ${workspace.ideName}..." |
| 290 | + indicator.text2 = workspace.downloadSource |
| 291 | + val distDir = accessor.getDefaultDistDir() |
| 292 | + |
| 293 | + // HighLevelHostAccessor.downloadFile does NOT create the directory. |
| 294 | + logger.info("Creating ${workspace.hostname}:${distDir.toRawString()}") |
| 295 | + accessor.createPathOnRemote(distDir) |
| 296 | + |
| 297 | + // Download the IDE. |
| 298 | + val fileName = workspace.downloadSource.split("/").last() |
| 299 | + val downloadPath = distDir.join(listOf(ShellArgument.PlainText(fileName))) |
| 300 | + logger.info("Downloading ${workspace.ideName} to ${workspace.hostname}:${downloadPath.toRawString()} from ${workspace.downloadSource}") |
| 301 | + accessor.downloadFile( |
| 302 | + indicator, |
| 303 | + URI(workspace.downloadSource), |
| 304 | + downloadPath, |
| 305 | + object : TransferProgressTracker { |
| 306 | + override var isCancelled: Boolean = false |
| 307 | + |
| 308 | + override fun updateProgress( |
| 309 | + transferred: Long, |
| 310 | + speed: Long?, |
| 311 | + ) { |
| 312 | + // Since there is no total size, this is useless. |
| 313 | + } |
| 314 | + }, |
| 315 | + ) |
| 316 | + |
| 317 | + // Extract the IDE to its final resting place. |
| 318 | + val ideDir = distDir.join(listOf(ShellArgument.PlainText(workspace.ideName))) |
| 319 | + indicator.text = "Extracting ${workspace.ideName}..." |
| 320 | + indicator.text2 = "" |
| 321 | + logger.info("Extracting ${workspace.ideName} to ${workspace.hostname}:${ideDir.toRawString()}") |
| 322 | + accessor.removePathOnRemote(ideDir) |
| 323 | + accessor.expandArchive(downloadPath, ideDir, timeout.toMillis()) |
| 324 | + accessor.removePathOnRemote(downloadPath) |
| 325 | + |
| 326 | + // Without this file it does not show up in the installed IDE list. |
| 327 | + val sentinelFile = ideDir.join(listOf(ShellArgument.PlainText(".expandSucceeded"))).toRawString() |
| 328 | + logger.info("Creating ${workspace.hostname}:$sentinelFile") |
| 329 | + accessor.fileAccessor.uploadFileFromLocalStream( |
| 330 | + sentinelFile, |
| 331 | + "".byteInputStream(), |
| 332 | + null, |
| 333 | + ) |
| 334 | + |
| 335 | + logger.info("Successfully installed ${workspace.ideName} on ${workspace.hostname}") |
| 336 | + return ideDir |
| 337 | + } |
| 338 | + |
| 339 | + /** |
| 340 | + * Run the setup command in the IDE's bin directory. |
| 341 | + */ |
| 342 | + private fun setup( |
| 343 | + workspace: WorkspaceProjectIDE, |
| 344 | + indicator: ProgressIndicator, |
| 345 | + setupCommand: String, |
| 346 | + ignoreSetupFailure: Boolean, |
| 347 | + ) { |
| 348 | + if (setupCommand.isNotBlank()) { |
| 349 | + indicator.text = "Running setup command..." |
| 350 | + try { |
| 351 | + exec(workspace, setupCommand) |
| 352 | + } catch (ex: Exception) { |
| 353 | + if (!ignoreSetupFailure) { |
| 354 | + throw ex |
| 355 | + } |
| 356 | + } |
| 357 | + } else { |
| 358 | + logger.info("No setup command to run on ${workspace.hostname}") |
| 359 | + } |
| 360 | + } |
| 361 | + |
| 362 | + /** |
| 363 | + * Execute a command in the IDE's bin directory. |
| 364 | + * This exists since the accessor does not provide a generic exec. |
| 365 | + */ |
| 366 | + private fun exec(workspace: WorkspaceProjectIDE, command: String): String { |
| 367 | + logger.info("Running command `$command` in ${workspace.hostname}:${workspace.idePathOnHost}/bin...") |
| 368 | + return ProcessExecutor() |
| 369 | + .command("ssh", "-t", workspace.hostname, "cd '${workspace.idePathOnHost}' ; cd bin ; $command") |
| 370 | + .exitValues(0) |
| 371 | + .readOutput(true) |
| 372 | + .execute() |
| 373 | + .outputUTF8() |
| 374 | + } |
| 375 | + |
| 376 | + /** |
| 377 | + * Ensure the backend is started. Link is null if not ready to join. |
| 378 | + */ |
| 379 | + private suspend fun ensureIDEBackend( |
| 380 | + workspace: WorkspaceProjectIDE, |
| 381 | + accessor: HighLevelHostAccessor, |
| 382 | + ideDir: ShellArgument.RemotePath, |
| 383 | + remoteProjectPath: ShellArgument.RemotePath, |
| 384 | + logsDir: ShellArgument.RemotePath, |
| 385 | + lifetime: LifetimeDefinition, |
| 386 | + currentStatus: UnattendedHostStatus?, |
| 387 | + ): UnattendedHostStatus? { |
| 388 | + return try { |
| 389 | + // Start the backend if not running. |
| 390 | + val currentPid = currentStatus?.appPid |
| 391 | + if (currentPid == null || !accessor.isPidAlive(currentPid.toInt())) { |
| 392 | + logger.info("Starting ${workspace.ideName} backend from ${workspace.hostname}:${ideDir.toRawString()}, project=${remoteProjectPath.toRawString()}, logs=${logsDir.toRawString()}") |
| 393 | + // This appears to be idempotent. |
| 394 | + accessor.startHostIdeInBackgroundAndDetach(lifetime, ideDir, remoteProjectPath, logsDir) |
| 395 | + } else if (!currentStatus.joinLink.isNullOrBlank()) { |
| 396 | + // We already have a valid join link. |
| 397 | + return currentStatus |
| 398 | + } |
| 399 | + // Get new PID and join link. |
| 400 | + val status = accessor.getHostIdeStatus(ideDir, remoteProjectPath) |
| 401 | + logger.info("Got ${workspace.ideName} status from ${workspace.hostname}:${ideDir.toRawString()}, pid=${status.appPid} project=${remoteProjectPath.toRawString()} joinLink=${status.joinLink}") |
| 402 | + status |
| 403 | + } catch (ex: Exception) { |
| 404 | + logger.info("Failed to get ${workspace.ideName} status from ${workspace.hostname}:${ideDir.toRawString()}, project=${remoteProjectPath.toRawString()}", ex) |
| 405 | + currentStatus |
| 406 | + } |
| 407 | + } |
| 408 | + |
126 | 409 | companion object { |
127 | 410 | val logger = Logger.getInstance(CoderRemoteConnectionHandle::class.java.simpleName) |
128 | 411 |
|
|
0 commit comments