Skip to content

Commit 85b1a2b

Browse files
elizarovelizarov
authored
Support JetBrains cache-redirector service for stable CI builds (#2247)
On CI set CACHE_REDIRECTOR=true environment property.
1 parent 2d0686b commit 85b1a2b

File tree

3 files changed

+192
-3
lines changed

3 files changed

+192
-3
lines changed

‎build.gradle

+9
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@ buildscript {
8383
// JMH plugins
8484
classpath "com.github.jengelman.gradle.plugins:shadow:5.1.0"
8585
}
86+
87+
CacheRedirector.configureBuildScript(buildscript, rootProject)
8688
}
8789

8890
import org.jetbrains.kotlin.gradle.plugin.KotlinPlatformType
@@ -295,7 +297,14 @@ configure(subprojects.findAll { !unpublished.contains(it.name) }) {
295297
// Report Kotlin compiler version when building project
296298
println("Using Kotlin compiler version: $org.jetbrains.kotlin.config.KotlinCompilerVersion.VERSION")
297299

300+
// --------------- Cache redirector ---------------
301+
302+
allprojects {
303+
CacheRedirector.configure(project)
304+
}
305+
298306
// --------------- Configure sub-projects that are published ---------------
307+
299308
def publishTasks = getTasksByName("publish", true) + getTasksByName("publishNpm", true)
300309

301310
task deploy(dependsOn: publishTasks)

‎buildSrc/build.gradle.kts

+31-3
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,41 @@
1+
/*
2+
* Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
15
import java.util.*
26

7+
buildscript {
8+
val cacheRedirectorEnabled = System.getenv("CACHE_REDIRECTOR")?.toBoolean() == true
9+
10+
if (cacheRedirectorEnabled) {
11+
println("Redirecting repositories for buildSrc buildscript")
12+
}
13+
14+
repositories {
15+
if (cacheRedirectorEnabled) {
16+
maven("https://cache-redirector.jetbrains.com/plugins.gradle.org/m2")
17+
} else {
18+
maven("https://plugins.gradle.org/m2")
19+
}
20+
}
21+
}
22+
323
plugins {
424
`kotlin-dsl`
525
}
626

27+
val cacheRedirectorEnabled = System.getenv("CACHE_REDIRECTOR")?.toBoolean() == true
28+
729
repositories {
8-
gradlePluginPortal()
9-
maven("https://kotlin.bintray.com/kotlin-eap")
10-
maven("https://kotlin.bintray.com/kotlin-dev")
30+
if (cacheRedirectorEnabled) {
31+
maven("https://cache-redirector.jetbrains.com/plugins.gradle.org/m2")
32+
maven("https://cache-redirector.jetbrains.com/dl.bintray.com/kotlin/kotlin-eap")
33+
maven("https://cache-redirector.jetbrains.com/dl.bintray.com/kotlin/kotlin-dev")
34+
} else {
35+
maven("https://plugins.gradle.org/m2")
36+
maven("https://dl.bintray.com/kotlin/kotlin-eap")
37+
maven("https://dl.bintray.com/kotlin/kotlin-dev")
38+
}
1139
}
1240

1341
kotlinDslPluginOptions {

‎buildSrc/src/main/kotlin/CacheRedirector.kt

+152
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
/*
2+
* Copyright 2016-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
3+
*/
4+
5+
import org.gradle.api.*
6+
import org.gradle.api.artifacts.dsl.*
7+
import org.gradle.api.artifacts.repositories.*
8+
import org.gradle.api.initialization.dsl.*
9+
import java.net.*
10+
11+
/**
12+
* Enabled via environment variable, so that it can be reliably accessed from any piece of the build script,
13+
* including buildSrc within TeamCity CI.
14+
*/
15+
private val cacheRedirectorEnabled = System.getenv("CACHE_REDIRECTOR")?.toBoolean() == true
16+
17+
/**
18+
* The list of repositories supported by cache redirector should be synced with the list at https://cache-redirector.jetbrains.com/redirects_generated.html
19+
* To add a repository to the list create an issue in ADM project (example issue https://youtrack.jetbrains.com/issue/IJI-149)
20+
*/
21+
private val mirroredUrls = listOf(
22+
"https://cdn.azul.com/zulu/bin",
23+
"https://clojars.org/repo",
24+
"https://dl.bintray.com/d10xa/maven",
25+
"https://dl.bintray.com/groovy/maven",
26+
"https://dl.bintray.com/jetbrains/maven-patched",
27+
"https://dl.bintray.com/jetbrains/scala-plugin-deps",
28+
"https://dl.bintray.com/kodein-framework/Kodein-DI",
29+
"https://dl.bintray.com/konsoletyper/teavm",
30+
"https://dl.bintray.com/kotlin/kotlin-dev",
31+
"https://dl.bintray.com/kotlin/kotlin-eap",
32+
"https://dl.bintray.com/kotlin/kotlinx.html",
33+
"https://dl.bintray.com/kotlin/kotlinx",
34+
"https://dl.bintray.com/kotlin/ktor",
35+
"https://dl.bintray.com/scalacenter/releases",
36+
"https://dl.bintray.com/scalamacros/maven",
37+
"https://dl.bintray.com/kotlin/exposed",
38+
"https://dl.bintray.com/cy6ergn0m/maven",
39+
"https://dl.bintray.com/kotlin/kotlin-js-wrappers",
40+
"https://dl.google.com/android/repository",
41+
"https://dl.google.com/dl/android/maven2",
42+
"https://dl.google.com/dl/android/studio/ide-zips",
43+
"https://dl.google.com/go",
44+
"https://download.jetbrains.com",
45+
"https://jcenter.bintray.com",
46+
"https://jetbrains.bintray.com/dekaf",
47+
"https://jetbrains.bintray.com/intellij-jbr",
48+
"https://jetbrains.bintray.com/intellij-jdk",
49+
"https://jetbrains.bintray.com/intellij-plugin-service",
50+
"https://jetbrains.bintray.com/intellij-terraform",
51+
"https://jetbrains.bintray.com/intellij-third-party-dependencies",
52+
"https://jetbrains.bintray.com/jediterm",
53+
"https://jetbrains.bintray.com/kotlin-native-dependencies",
54+
"https://jetbrains.bintray.com/markdown",
55+
"https://jetbrains.bintray.com/teamcity-rest-client",
56+
"https://jetbrains.bintray.com/test-discovery",
57+
"https://jetbrains.bintray.com/wormhole",
58+
"https://jitpack.io",
59+
"https://maven.pkg.jetbrains.space/kotlin/p/kotlin/dev",
60+
"https://maven.pkg.jetbrains.space/kotlin/p/kotlin/bootstrap",
61+
"https://maven.pkg.jetbrains.space/kotlin/p/kotlin/eap",
62+
"https://kotlin.bintray.com/dukat",
63+
"https://kotlin.bintray.com/kotlin-dependencies",
64+
"https://oss.sonatype.org/content/repositories/releases",
65+
"https://oss.sonatype.org/content/repositories/snapshots",
66+
"https://oss.sonatype.org/content/repositories/staging",
67+
"https://packages.confluent.io/maven/",
68+
"https://plugins.gradle.org/m2",
69+
"https://plugins.jetbrains.com/maven",
70+
"https://repo1.maven.org/maven2",
71+
"https://repo.grails.org/grails/core",
72+
"https://repo.jenkins-ci.org/releases",
73+
"https://repo.maven.apache.org/maven2",
74+
"https://repo.spring.io/milestone",
75+
"https://repo.typesafe.com/typesafe/ivy-releases",
76+
"https://services.gradle.org",
77+
"https://www.exasol.com/artifactory/exasol-releases",
78+
"https://www.myget.org/F/intellij-go-snapshots/maven",
79+
"https://www.myget.org/F/rd-model-snapshots/maven",
80+
"https://www.myget.org/F/rd-snapshots/maven",
81+
"https://www.python.org/ftp",
82+
"https://www.jetbrains.com/intellij-repository/nightly",
83+
"https://www.jetbrains.com/intellij-repository/releases",
84+
"https://www.jetbrains.com/intellij-repository/snapshots"
85+
)
86+
87+
private val aliases = mapOf(
88+
"https://repo.maven.apache.org/maven2" to "https://repo1.maven.org/maven2", // Maven Central
89+
"https://kotlin.bintray.com/kotlin-dev" to "https://dl.bintray.com/kotlin/kotlin-dev",
90+
"https://kotlin.bintray.com/kotlin-eap" to "https://dl.bintray.com/kotlin/kotlin-eap",
91+
"https://kotlin.bintray.com/kotlinx" to "https://dl.bintray.com/kotlin/kotlinx"
92+
)
93+
94+
private fun URI.toCacheRedirectorUri() = URI("https://cache-redirector.jetbrains.com/$host/$path")
95+
96+
private fun URI.maybeRedirect(): URI? {
97+
val url = toString().trimEnd('/')
98+
val dealiasedUrl = aliases.getOrDefault(url, url)
99+
100+
return if (mirroredUrls.any { dealiasedUrl.startsWith(it) }) {
101+
URI(dealiasedUrl).toCacheRedirectorUri()
102+
} else {
103+
null
104+
}
105+
}
106+
107+
private fun URI.isCachedOrLocal() = scheme == "file" ||
108+
host == "cache-redirector.jetbrains.com" ||
109+
host == "teamcity.jetbrains.com" ||
110+
host == "buildserver.labs.intellij.net"
111+
112+
private fun Project.checkRedirectUrl(url: URI, containerName: String): URI {
113+
val redirected = url.maybeRedirect()
114+
if (redirected == null && !url.isCachedOrLocal()) {
115+
val msg = "Repository $url in $containerName should be cached with cache-redirector"
116+
val details = "Using non cached repository may lead to download failures in CI builds." +
117+
" Check buildSrc/src/main/kotlin/CacheRedirector.kt for details."
118+
logger.warn("WARNING - $msg\n$details")
119+
}
120+
return if (cacheRedirectorEnabled) redirected ?: url else url
121+
}
122+
123+
private fun Project.checkRedirect(repositories: RepositoryHandler, containerName: String) {
124+
if (cacheRedirectorEnabled) {
125+
logger.info("Redirecting repositories for $containerName")
126+
}
127+
for (repository in repositories) {
128+
when (repository) {
129+
is MavenArtifactRepository -> repository.url = checkRedirectUrl(repository.url, containerName)
130+
is IvyArtifactRepository -> repository.url = checkRedirectUrl(repository.url, containerName)
131+
}
132+
}
133+
}
134+
135+
// Used from Groovy scripts
136+
object CacheRedirector {
137+
/**
138+
* Substitutes repositories in buildScript { } block.
139+
*/
140+
@JvmStatic
141+
fun ScriptHandler.configureBuildScript(rootProject: Project) {
142+
rootProject.checkRedirect(repositories, "${rootProject.displayName} buildscript")
143+
}
144+
145+
/**
146+
* Substitutes repositories in a project.
147+
*/
148+
@JvmStatic
149+
fun Project.configure() {
150+
checkRedirect(repositories, displayName)
151+
}
152+
}

0 commit comments

Comments
 (0)