Skip to content

Commit a408856

Browse files
VampireVampire
committed
Make workflow generation via Gradle able to run in parallel
1 parent c389573 commit a408856

File tree

6 files changed

+305
-82
lines changed

6 files changed

+305
-82
lines changed

Diff for: ‎gradle/build-logic/src/main/kotlin/net/kautler/github_actions.gradle.kts

+24-80
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2020-2024 Björn Kautler
2+
* Copyright 2020-2025 Björn Kautler
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -16,26 +16,18 @@
1616

1717
package net.kautler
1818

19+
import net.kautler.githubactions.DetermineImportedFiles
20+
import net.kautler.githubactions.PreprocessGithubWorkflow
1921
import org.gradle.accessors.dm.LibrariesForLibs
20-
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY
21-
import org.jetbrains.kotlin.cli.common.messages.MessageCollector.Companion.NONE
22-
import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler
23-
import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles.JVM_CONFIG_FILES
24-
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
25-
import org.jetbrains.kotlin.com.intellij.openapi.util.Disposer
26-
import org.jetbrains.kotlin.com.intellij.openapi.vfs.local.CoreLocalFileSystem
27-
import org.jetbrains.kotlin.com.intellij.openapi.vfs.local.CoreLocalVirtualFile
28-
import org.jetbrains.kotlin.com.intellij.psi.PsiManager
29-
import org.jetbrains.kotlin.config.CompilerConfiguration
30-
import org.jetbrains.kotlin.psi.KtFile
31-
import org.jetbrains.kotlin.psi.KtLiteralStringTemplateEntry
32-
import org.jetbrains.kotlin.psi.KtStringTemplateExpression
33-
import java.nio.file.Path
3422

3523
plugins {
3624
`java-base`
3725
}
3826

27+
val compilerEmbeddableClasspath by configurations.creating {
28+
isCanBeConsumed = false
29+
}
30+
3931
val compilerClasspath by configurations.creating {
4032
isCanBeConsumed = false
4133
}
@@ -47,6 +39,7 @@ val scriptClasspath by configurations.creating {
4739
val libs = the<LibrariesForLibs>()
4840

4941
dependencies {
42+
compilerEmbeddableClasspath(libs.workflows.kotlin.compiler.embeddable)
5043
compilerClasspath(libs.workflows.kotlin.compiler)
5144
compilerClasspath(libs.workflows.kotlin.scripting.compiler)
5245
scriptClasspath(libs.workflows.kotlin.main.kts) {
@@ -64,72 +57,23 @@ file(".github/workflows")
6457
val pascalCasedWorkflowName = workflowName.replace("""-\w""".toRegex()) {
6558
it.value.substring(1).replaceFirstChar(Char::uppercaseChar)
6659
}.replaceFirstChar(Char::uppercaseChar)
67-
val preprocessWorkflow = tasks.register<JavaExec>("preprocess${pascalCasedWorkflowName}Workflow") {
68-
group = "github actions"
69-
70-
inputs
71-
.file(workflowScript)
72-
.withPropertyName("workflowScript")
73-
inputs
74-
.files(file(workflowScript).importedFiles)
75-
.withPropertyName("importedFiles")
76-
outputs
77-
.file(workflowScript.resolveSibling("$workflowName.yaml"))
78-
.withPropertyName("workflowFile")
79-
80-
javaLauncher.set(javaToolchains.launcherFor {
81-
languageVersion.set(JavaLanguageVersion.of(17))
82-
})
83-
classpath(compilerClasspath)
84-
mainClass.set(K2JVMCompiler::class.qualifiedName)
85-
args("-no-stdlib", "-no-reflect")
86-
args("-classpath", scriptClasspath.asPath)
87-
args("-script", workflowScript.absolutePath)
88-
89-
// work-around for https://youtrack.jetbrains.com/issue/KT-42101
90-
systemProperty("kotlin.main.kts.compiled.scripts.cache.dir", "")
91-
}
60+
val determineImportedFiles =
61+
tasks.register<DetermineImportedFiles>("determineImportedFilesFor${pascalCasedWorkflowName}Workflow") {
62+
mainKtsFile.set(workflowScript)
63+
importedFiles.set(layout.buildDirectory.file("importedFilesFor${pascalCasedWorkflowName}Workflow.txt"))
64+
kotlinCompilerEmbeddableClasspath.from(compilerEmbeddableClasspath)
65+
}
66+
val preprocessWorkflow =
67+
tasks.register<PreprocessGithubWorkflow>("preprocess${pascalCasedWorkflowName}Workflow") {
68+
this.workflowScript.set(workflowScript)
69+
importedFiles.from(determineImportedFiles.flatMap { it.importedFiles }.map { it.asFile.readLines() })
70+
kotlinCompilerClasspath.from(compilerClasspath)
71+
mainKtsClasspath.from(scriptClasspath)
72+
javaLauncher.set(javaToolchains.launcherFor {
73+
languageVersion.set(JavaLanguageVersion.of(17))
74+
})
75+
}
9276
preprocessWorkflows {
9377
dependsOn(preprocessWorkflow)
9478
}
9579
}
96-
97-
val File.importedFiles: List<File>
98-
get() = if (!isFile) {
99-
emptyList()
100-
} else {
101-
PsiManager
102-
.getInstance(
103-
KotlinCoreEnvironment
104-
.createForProduction(
105-
Disposer.newDisposable(),
106-
CompilerConfiguration().apply {
107-
put(MESSAGE_COLLECTOR_KEY, NONE)
108-
},
109-
JVM_CONFIG_FILES
110-
)
111-
.project
112-
)
113-
.findFile(
114-
// work-around for API change between version we compile against and version we run against
115-
// after upgrading Gradle to a version that contains Kotlin 1.9 the embeddable compiler can
116-
// be upgraded to v2 also for compilation and then this can be removed
117-
CoreLocalVirtualFile::class
118-
.java
119-
.getConstructor(CoreLocalFileSystem::class.java, Path::class.java)
120-
.newInstance(CoreLocalFileSystem(), toPath())
121-
)
122-
.let { it as KtFile }
123-
.fileAnnotationList
124-
?.annotationEntries
125-
?.asSequence()
126-
?.filter { it.shortName?.asString() == "Import" }
127-
?.flatMap { it.valueArgumentList?.arguments ?: emptyList() }
128-
?.mapNotNull { it.getArgumentExpression() as? KtStringTemplateExpression }
129-
?.map { it.entries.first() }
130-
?.mapNotNull { it as? KtLiteralStringTemplateEntry }
131-
?.map { resolveSibling(it.text) }
132-
?.flatMap { it.importedFiles + it }
133-
?.toList()
134-
?: emptyList()
135-
}

Diff for: ‎gradle/build-logic/src/main/kotlin/net/kautler/githubactions/DetermineImportedFiles.kt

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright 2025 Björn Kautler
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package net.kautler.githubactions
18+
19+
import org.gradle.api.DefaultTask
20+
import org.gradle.api.file.ConfigurableFileCollection
21+
import org.gradle.api.file.ProjectLayout
22+
import org.gradle.api.file.RegularFileProperty
23+
import org.gradle.api.tasks.InputFile
24+
import org.gradle.api.tasks.InputFiles
25+
import org.gradle.api.tasks.OutputFile
26+
import org.gradle.api.tasks.TaskAction
27+
import org.gradle.api.tasks.UntrackedTask
28+
import org.gradle.kotlin.dsl.submit
29+
import org.gradle.workers.WorkerExecutor
30+
import javax.inject.Inject
31+
32+
@UntrackedTask(because = "imported files can import other files so inputs are not determinable upfront")
33+
abstract class DetermineImportedFiles : DefaultTask() {
34+
@get:InputFile
35+
abstract val mainKtsFile: RegularFileProperty
36+
37+
@get:InputFiles
38+
abstract val kotlinCompilerEmbeddableClasspath: ConfigurableFileCollection
39+
40+
@get:OutputFile
41+
abstract val importedFiles: RegularFileProperty
42+
43+
@get:Inject
44+
abstract val workerExecutor: WorkerExecutor
45+
46+
@get:Inject
47+
abstract val layout: ProjectLayout
48+
49+
@TaskAction
50+
fun determineImportedFiles() {
51+
workerExecutor.classLoaderIsolation {
52+
classpath.from(kotlinCompilerEmbeddableClasspath)
53+
}.submit(DetermineImportedFilesWorkAction::class) {
54+
projectDirectory.set(layout.projectDirectory)
55+
mainKtsFile.set(this@DetermineImportedFiles.mainKtsFile)
56+
importedFiles.set(this@DetermineImportedFiles.importedFiles)
57+
}
58+
}
59+
}

Diff for: ‎gradle/build-logic/src/main/kotlin/net/kautler/githubactions/DetermineImportedFilesWorkAction.kt

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
* Copyright 2025 Björn Kautler
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package net.kautler.githubactions
18+
19+
import org.gradle.api.file.DirectoryProperty
20+
import org.gradle.api.file.RegularFileProperty
21+
import org.gradle.workers.WorkAction
22+
import org.gradle.workers.WorkParameters
23+
import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY
24+
import org.jetbrains.kotlin.cli.common.messages.MessageCollector.Companion.NONE
25+
import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles.JVM_CONFIG_FILES
26+
import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
27+
import org.jetbrains.kotlin.com.intellij.openapi.util.Disposer
28+
import org.jetbrains.kotlin.com.intellij.openapi.vfs.local.CoreLocalFileSystem
29+
import org.jetbrains.kotlin.com.intellij.openapi.vfs.local.CoreLocalVirtualFile
30+
import org.jetbrains.kotlin.com.intellij.psi.PsiManager
31+
import org.jetbrains.kotlin.config.CompilerConfiguration
32+
import org.jetbrains.kotlin.psi.KtFile
33+
import org.jetbrains.kotlin.psi.KtLiteralStringTemplateEntry
34+
import org.jetbrains.kotlin.psi.KtStringTemplateExpression
35+
import java.io.File
36+
import java.nio.file.Path
37+
38+
abstract class DetermineImportedFilesWorkAction : WorkAction<DetermineImportedFilesWorkAction.Parameters> {
39+
override fun execute() {
40+
val projectDirectory = parameters.projectDirectory.get().asFile
41+
parameters
42+
.mainKtsFile
43+
.get()
44+
.asFile
45+
.importedFiles
46+
.map { it.relativeTo(projectDirectory).invariantSeparatorsPath }
47+
.distinct()
48+
.sorted()
49+
.joinToString("\n")
50+
.also(parameters.importedFiles.get().asFile::writeText)
51+
}
52+
53+
interface Parameters : WorkParameters {
54+
val projectDirectory: DirectoryProperty
55+
val mainKtsFile: RegularFileProperty
56+
val importedFiles: RegularFileProperty
57+
}
58+
}
59+
60+
private val File.importedFiles: List<File>
61+
get() = if (!isFile) {
62+
emptyList()
63+
} else {
64+
PsiManager
65+
.getInstance(
66+
KotlinCoreEnvironment
67+
.createForProduction(
68+
Disposer.newDisposable(),
69+
CompilerConfiguration().apply {
70+
put(MESSAGE_COLLECTOR_KEY, NONE)
71+
},
72+
JVM_CONFIG_FILES
73+
)
74+
.project
75+
)
76+
.findFile(CoreLocalVirtualFile::class.java.getConstructor(CoreLocalFileSystem::class.java, Path::class.java).newInstance(CoreLocalFileSystem(), toPath()))
77+
.let { it as KtFile }
78+
.fileAnnotationList
79+
?.annotationEntries
80+
?.asSequence()
81+
?.filter { it.shortName?.asString() == "Import" }
82+
?.flatMap { it.valueArgumentList?.arguments ?: emptyList() }
83+
?.mapNotNull { it.getArgumentExpression() as? KtStringTemplateExpression }
84+
?.map { it.entries.first() }
85+
?.mapNotNull { it as? KtLiteralStringTemplateEntry }
86+
?.map { resolveSibling(it.text) }
87+
?.flatMap { it.importedFiles + it }
88+
?.toList()
89+
?: emptyList()
90+
}

Diff for: ‎gradle/build-logic/src/main/kotlin/net/kautler/githubactions/PreprocessGithubWorkflow.kt

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Copyright 2025 Björn Kautler
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package net.kautler.githubactions
18+
19+
import org.gradle.api.DefaultTask
20+
import org.gradle.api.file.ConfigurableFileCollection
21+
import org.gradle.api.file.RegularFileProperty
22+
import org.gradle.api.provider.Property
23+
import org.gradle.api.provider.Provider
24+
import org.gradle.api.tasks.InputFile
25+
import org.gradle.api.tasks.InputFiles
26+
import org.gradle.api.tasks.Nested
27+
import org.gradle.api.tasks.OutputFile
28+
import org.gradle.api.tasks.TaskAction
29+
import org.gradle.jvm.toolchain.JavaLauncher
30+
import org.gradle.kotlin.dsl.submit
31+
import org.gradle.workers.WorkerExecutor
32+
import java.io.File
33+
import javax.inject.Inject
34+
35+
abstract class PreprocessGithubWorkflow : DefaultTask() {
36+
@get:InputFile
37+
abstract val workflowScript: RegularFileProperty
38+
39+
@get:InputFiles
40+
abstract val importedFiles: ConfigurableFileCollection
41+
42+
@get:InputFiles
43+
abstract val kotlinCompilerClasspath: ConfigurableFileCollection
44+
45+
@get:InputFiles
46+
abstract val mainKtsClasspath: ConfigurableFileCollection
47+
48+
@get:Nested
49+
abstract val javaLauncher: Property<JavaLauncher>
50+
51+
@get:OutputFile
52+
val workflowFile: Provider<File> = workflowScript.map {
53+
val workflowScript = it.asFile
54+
workflowScript.resolveSibling("${workflowScript.name.removeSuffix(".main.kts")}.yaml")
55+
}
56+
57+
@get:Inject
58+
abstract val workerExecutor: WorkerExecutor
59+
60+
init {
61+
group = "github workflows"
62+
}
63+
64+
@TaskAction
65+
fun determineImportedFiles() {
66+
workerExecutor.noIsolation().submit(PreprocessGithubWorkflowWorkAction::class) {
67+
workflowScript.set(this@PreprocessGithubWorkflow.workflowScript)
68+
kotlinCompilerClasspath.from(this@PreprocessGithubWorkflow.kotlinCompilerClasspath)
69+
mainKtsClasspath.from(this@PreprocessGithubWorkflow.mainKtsClasspath)
70+
javaExecutable.set(this@PreprocessGithubWorkflow.javaLauncher.map { it.executablePath })
71+
}
72+
}
73+
}

0 commit comments

Comments
 (0)