Skip to content

Commit eeb6372

Browse files
lrytzlrytz
authored
Merge pull request #186 from lrytz/merge-212
Merge 212
2 parents d3e4c73 + 78bd696 commit eeb6372

File tree

14 files changed

+349
-33
lines changed

14 files changed

+349
-33
lines changed

‎.travis.yml

Lines changed: 44 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,57 @@
1+
# The scala/scala travis job triggers a build of scala/scala-dist with `mode=release`. For
2+
# the other modes (see below), use the web UI to trigger a build.
3+
#
4+
# Additional env vars are defined using a `before_install` custom config(*), for example
5+
# before_install: export version=2.12.N scala_sha=abc123 mode=release
6+
#
7+
# Available modes:
8+
# - `release` to build native packages and upload them to S3
9+
# - `archive` to copy archives to chara (for scala-lang.org)
10+
# - `update-api` to update the scaladoc api symlinks
11+
#
12+
# (*) Using an `env: global: ...` section does not work because that overrides the default `env`
13+
# from .travis.yml. For releases triggered with the REST API we could use the "merge mode", but
14+
# not when triggering from the web UI. Anyway, `before_install` works well.
15+
116
# we need rpmbuild but it's unlikely to be whitelisted, according to
217
# https://github.com/travis-ci/apt-package-whitelist/pull/1700
318
sudo: required
4-
before_install:
19+
20+
install:
521
- sudo apt-get -qq update
622
- sudo apt-get install -y rpm
723

824
language: scala
925
jdk: oraclejdk8
1026

11-
# although this is the 2.13.x branch, we don't have any 2.13 scala-dist
12-
# artifacts to depend on yet. so for now, just keep using 2.12.0
27+
env:
28+
global:
29+
# PRIV_KEY_SECRET
30+
- secure: "NlnFqZs4mvCi63GqCdUNDsx9BoiyrjgoV0cUaKlhVYp49/qAMlEaKf6JABWy/oCHoWsTyysyLEtZLsWfjAyE4+FUHSzngNUHVFBbtyudA6aKQ7jMsqkcMM/K0h+ymnhHSv8bugc8jXKFgmfYk4YTi5dZgv8CAOSQFAFUaxVdYTY="
31+
# APPVEYOR_TOKEN (login with GitHub as lrytz, use the scala organization)
32+
- secure: "aHM0UWkbFN0bXQja2f9lUcSgqHzXYkYqqV8/nP5MTLKJG8BpuaLYxI4gHfkPmxSm9/TTniA3zUVN//lJTvhco5J5Dtxfo5DeCLWsjQcQxrsgAFD97FpMpGbtFAViZuiv7SdPGvzXQY6AwvteBpxOday4T224aH5OjZJ0DxySx8Q="
33+
# TRAVIS_TOKEN (login with GitHub as lrytz)
34+
- secure: "K6G3SeMRMgC8JUZADyd4RKvQpgHOZb8/8kk3AlDtRWZmrBPkZpTrtcuQLYXRiEAO/beQH956i3+uDeFryF079YTJZKtdqLTxwXS53gVurOv9VVbZFubu4HzXWMWeCIi9Np7S0eB1qc4NIKB+T5pbmIfEFGKThBGK179uwf2lqQ8="
35+
# AWS_ACCESS_KEY_ID
36+
- secure: "BrsLFLXLdJ3aesJGuoVXFdKluYl7QYBYjn5+ttoep2nc6/E+L3UDaR//ztxwex4a9yFi5q2jCs911FzVsk78ODamr5w8FBlOuu06RoVx642+U/agG7yUdAtvEzH2KVTBNjY0oYHZ+OlVEkeYgnkrsbfki+ujPWEnxkakWyrGtJw="
37+
# AWS_SECRET_ACCESS_KEY
38+
- secure: "dv0pR9uqnLyKVHu1L1nvl5TVpo3bOxh4icLNybwFyCFQd9XiM/RU8Vhx3svZqqnAG+GiCDSiCaBQkTLUS5u9e+9eVw3cUj5meNr9EW673f8D6H8Tr433jlvu54CynD9DsBjhNo/xIrECOKTq+0wu480OLSjOkuNoclG2dSm4Dis="
39+
# GITHUB_OAUTH_TOKEN (for GitHub user scala-jenkins, same token as scabot)
40+
- secure: "FtrGQs3tHwOPZOXi1xDcCo1KtP1rUfaiK+l5lrjzXhuCjtWkG+MHB0VI1D1QwbXh+BiRdYIyfUyKBa1I+lRrjKx5aDgxZ7EweML5CtzIu7hAuxbc6zchC3R0qZJe20DcM1BjokCXi2EnKtgABIBuCP0KY4WiH0NQkGjZsemoCAQ="
41+
1342
script:
14-
- sbt -Dproject.version=2.12.0 s3Upload::mappings
43+
- source scripts/common
44+
- postCommitStatus "pending"
45+
- admin/build.sh
46+
47+
after_script:
48+
- |
49+
if [[ "$TRAVIS_TEST_RESULT" == "0" ]]; then
50+
local state="success"
51+
else
52+
local state="failure"
53+
fi
54+
postCommitStatus $state
1555
1656
cache:
1757
directories:

‎admin/build.ps1

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
$ErrorActionPreference = "Stop"
2+
3+
Function checkExit() {
4+
if (-not $?) {
5+
echo "Last command failed."
6+
Exit 1
7+
}
8+
}
9+
10+
Function ensureVersion() {
11+
$verPat="^[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9-]+)?$"
12+
if ($env:version -notmatch $verPat) {
13+
echo "Not a valid Scala version: '$env:version'"
14+
Exit 1
15+
}
16+
}
17+
18+
Function clearIvyCache() {
19+
remove-item -erroraction ignore -force -verbose $homeDir\.ivy2\exclude_classifiers, $homeDir\.ivy2\exclude_classifiers.lock
20+
remove-item -erroraction ignore -force -verbose -recurse $homeDir\.ivy2\cache\org.scala-lang, $homeDir\.ivy2\cache\org.scala-lang.modules
21+
remove-item -erroraction ignore -force -verbose -recurse $homeDir\.ivy2\local\org.scala-lang, $homeDir\.ivy2\local\org.scala-lang.modules
22+
get-childitem -erroraction ignore -path $homeDir\.ivy2 -recurse -include "*compiler-interface*$env:version*" | remove-item -force -recurse -verbose
23+
get-childitem -erroraction ignore -path $homeDir\.sbt -recurse -include "*compiler-interface*$env:version*" | remove-item -force -recurse -verbose
24+
}
25+
26+
# oh boy. i don't (want to) fully understand, but executing commands and redirecting is difficult.
27+
# - https://mnaoumov.wordpress.com/2015/01/11/execution-of-external-commands-in-powershell-done-right/
28+
# - https://stackoverflow.com/a/35980675/248998
29+
# letting cmd do the redirect avoids confusing powershell.
30+
& cmd /c 'java -version' '2>&1'
31+
checkExit
32+
33+
if ($env:APPVEYOR_FORCED_BUILD -eq 'true') {
34+
ensureVersion
35+
clearIvyCache
36+
if ($env:mode -eq 'release') {
37+
echo "Running a release for $env:version"
38+
$repositoriesFile="$env:APPVEYOR_BUILD_FOLDER\conf\repositories"
39+
& cmd /c "sbt ""-Dsbt.override.build.repos=true"" ""-Dsbt.repository.config=$repositoriesFile"" ""-Dproject.version=$env:version"" ""show fullResolvers"" clean update s3Upload" '2>&1'
40+
checkExit
41+
} else {
42+
echo "Unknown mode: '$env:mode'"
43+
Exit 1
44+
}
45+
} else {
46+
# although this is the 2.13.x branch, we don't have any 2.13 scala-dist
47+
# artifacts to depend on yet. so for now, just keep using 2.12.4
48+
$env:version="2.12.4"
49+
clearIvyCache
50+
# By default, test building the packages (but don't uplaod)
51+
# Need to redirect stderr, otherwise any error output (like jvm warning) fails the build (ErrorActionPreference)
52+
& cmd /c "sbt ""-Dproject.version=$env:version"" ""show s3Upload::mappings""" '2>&1'
53+
checkExit
54+
}

‎admin/build.sh

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
# Encryping files (if you need to encrypt a new file but no longer have the secret, create a new
6+
# secret and re-encrypt all files):
7+
#
8+
# 1. Generate secret
9+
# cat /dev/urandom | head -c 10000 | openssl sha1 > ./secret
10+
# 2. Save the secret on travis
11+
# travis encrypt "PRIV_KEY_SECRET=$(cat ./secret)"
12+
# 3. Encrypt the file
13+
# openssl aes-256-cbc -pass "file:./secret" -in jenkins_lightbend_chara -out jenkins_lightbend_chara.enc
14+
# 4. Decrypt the file
15+
# openssl aes-256-cbc -d -pass "pass:$PRIV_KEY_SECRET" -in admin/files/jenkins_lightbend_chara.enc > ~/.ssh/jenkins_lightbend_chara 2>/dev/null
16+
17+
function ensureVersion() {
18+
local verPat="[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9-]+)?"
19+
[[ "$version" =~ $verPat ]] || {
20+
echo "Not a valid Scala version: '$version'"
21+
exit 1
22+
}
23+
}
24+
25+
function decrypt() {
26+
# Even though we're running bash with -x, travis hides the private key from the log
27+
openssl aes-256-cbc -d -pass "pass:$PRIV_KEY_SECRET" -in $1 > $2 2>/dev/null
28+
}
29+
30+
function setupSSH() {
31+
mkdir -p ~/.ssh
32+
cp admin/files/ssh_config ~/.ssh/config
33+
echo 'chara.epfl.ch,128.178.154.107 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBLER9rqy0iCfz24+8BBBObh7FXXQJCoLLE9UuyQHNFUU4SS5FSzNjEoKXwTj8nqNy+8l0rOkj3KG8p2cLxsqjY=' >> ~/.ssh/known_hosts
34+
decrypt admin/files/jenkins_lightbend_chara.enc ~/.ssh/jenkins_lightbend_chara
35+
chmod 700 ~/.ssh && $(cd ~/.ssh && chmod 600 config known_hosts jenkins_lightbend_chara)
36+
}
37+
38+
function triggerMsiRelease() {
39+
local jsonTemplate='{ "accountName": "scala", "projectSlug": "scala-dist", "branch": "%s", "commitId": "%s", "environmentVariables": { "mode": "%s", "version": "%s", "scala_sha": "%s" } }'
40+
local json=$(printf "$jsonTemplate" "$TRAVIS_BRANCH" "$TRAVIS_COMMIT" "$mode" "$version" "$scala_sha")
41+
42+
local curlStatus=$(curl \
43+
-s -o /dev/null -w "%{http_code}" \
44+
-H "Authorization: Bearer $APPVEYOR_TOKEN" \
45+
-H "Content-Type: application/json" \
46+
-d "$json" \
47+
https://ci.appveyor.com/api/builds)
48+
49+
[[ "$curlStatus" == "200" ]] || {
50+
echo "Failed to start AppVeyor build"
51+
exit 1
52+
}
53+
}
54+
55+
function triggerSmoketest() {
56+
local jsonTemplate='{ "request": { "branch": "%s", "message": "Smoketest %s", "config": { "before_install": "export version=%s scala_sha=%s" } } }'
57+
local json=$(printf "$jsonTemplate" "$TRAVIS_BRANCH" "$version" "$version" "$scala_sha")
58+
59+
local curlStatus=$(curl \
60+
-s -o /dev/null -w "%{http_code}" \
61+
-H "Travis-API-Version: 3" \
62+
-H "Authorization: token $TRAVIS_TOKEN" \
63+
-H "Content-Type: application/json" \
64+
-d "$json" \
65+
https://api.travis-ci.org/repo/scala%2Fscala-dist-smoketest/requests)
66+
67+
[[ "$curlStatus" == "202" ]] || {
68+
echo "Failed to start travis build"
69+
exit 1
70+
}
71+
}
72+
73+
clearIvyCache() {
74+
rm -fv $HOME/.ivy2/exclude_classifiers $HOME/.ivy2/exclude_classifiers.lock
75+
rm -rfv $HOME/.ivy2/cache/org.scala-lang $HOME/.ivy2/cache/org.scala-lang.modules
76+
rm -rfv $HOME/.ivy2/local/org.scala-lang $HOME/.ivy2/local/org.scala-lang.modules
77+
if [ -d $HOME/.ivy2 ]; then find $HOME/.ivy2 -name "*compiler-interface*$version*" | xargs rm -rfv; fi
78+
if [ -d $HOME/.sbt ]; then find $HOME/.sbt -name "*compiler-interface*$version*" | xargs rm -rfv; fi
79+
}
80+
81+
if [[ "$TRAVIS_EVENT_TYPE" == "api" ]]; then
82+
ensureVersion
83+
clearIvyCache
84+
if [[ "$mode" == "archives" ]]; then
85+
echo "Running 'archives' for $version"
86+
setupSSH
87+
. scripts/jobs/release/website/archives
88+
elif [[ "$mode" == "update-api" ]]; then
89+
echo "Running 'update-api' for $version"
90+
setupSSH
91+
. scripts/jobs/release/website/update-api
92+
elif [[ "$mode" == "release" ]]; then
93+
echo "Running a release for $version"
94+
triggerMsiRelease
95+
repositoriesFile="$TRAVIS_BUILD_DIR/conf/repositories"
96+
# The log is too long for the travis UI, so remove ANSI codes to have a clean raw version
97+
sbt -Dsbt.log.noformat=true \
98+
-Dsbt.override.build.repos=true -Dsbt.repository.config="$repositoriesFile" \
99+
-Dproject.version=$version \
100+
"show fullResolvers" clean update s3Upload
101+
triggerSmoketest
102+
else
103+
echo "Unknown build mode: '$mode'"
104+
exit 1
105+
fi
106+
else
107+
# although this is the 2.13.x branch, we don't have any 2.13 scala-dist
108+
# artifacts to depend on yet. so for now, just keep using 2.12.4
109+
version="2.12.4"
110+
clearIvyCache
111+
# By default, test building the packages (but don't uplaod)
112+
sbt -Dproject.version=$version "show s3Upload::mappings"
113+
fi

‎admin/files/jenkins_lightbend_chara.enc

3.19 KB
Binary file not shown.

‎admin/files/ssh_config

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Host chara
2+
HostName chara.epfl.ch
3+
IdentityFile /home/travis/.ssh/jenkins_lightbend_chara
4+
User scalatest

‎appveyor.yml

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
version: '{build}'
2+
3+
branches:
4+
only:
5+
- 2.11.x
6+
- 2.12.x
7+
- 2.13.x
8+
9+
image: Visual Studio 2015
10+
11+
environment:
12+
JAVA_HOME: C:\Program Files\Java\jdk1.8.0
13+
AWS_ACCESS_KEY_ID:
14+
secure: X1Ix1soRBDMtfbi8IFNPOggDP2XquhW+uKcJ+XC0kiM=
15+
AWS_SECRET_ACCESS_KEY:
16+
secure: woXwpM2+P3uZ1+LFM+SbJEKLfi7Kax5PvqJSN62nn1PQrJgEzVUAIoRGoXBx6X72
17+
GITHUB_OAUTH_TOKEN:
18+
secure: oji2LTrdEhQhJQOoRU3TFuCmCb92XD1BJKW7uw0/XJdEFkOrb4lKbHjPGeuOY6/5
19+
20+
install:
21+
- cmd: choco install sbt -ia "INSTALLDIR=""C:\sbt"""
22+
- cmd: SET PATH=C:\sbt\bin;%JAVA_HOME%\bin;%PATH%
23+
24+
build_script:
25+
- ps: . .\scripts\common.ps1
26+
- ps: postCommitStatus "pending"
27+
- ps: .\admin\build.ps1
28+
29+
on_success:
30+
- ps: postCommitStatus "success"
31+
32+
on_failure:
33+
- ps: postCommitStatus "failure"
34+
35+
cache:
36+
- C:\sbt
37+
- C:\Users\appveyor\.sbt
38+
- C:\Users\appveyor\.ivy2

‎build.sbt

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import com.typesafe.sbt.SbtGit._
2-
import S3._
2+
import ScalaDist.upload
33

44
// so we don't require a native git install
55
useJGit
@@ -16,11 +16,25 @@ Versioning.settings
1616
// are known/understood, at scala/scala-dist#171
1717
scalaVersion := version.value
1818

19-
s3Settings
19+
mappings in upload := Seq()
2020

21-
host in upload := "downloads.typesafe.com.s3.amazonaws.com"
21+
upload := {
22+
import com.amazonaws.{ClientConfiguration, Protocol}
23+
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain
24+
import com.amazonaws.services.s3.AmazonS3ClientBuilder
25+
import com.amazonaws.services.s3.model.PutObjectRequest
26+
import com.amazonaws.regions.Regions
2227

23-
credentials += Credentials(Path.userHome / ".s3credentials")
28+
// The standard client picks credentials from AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY env vars
29+
val client = AmazonS3ClientBuilder.standard.withRegion(Regions.US_EAST_1).build
30+
31+
val log = streams.value.log
32+
33+
(mappings in upload).value map { case (file, key) =>
34+
log.info("Uploading "+ file.getAbsolutePath() +" as "+ key)
35+
client.putObject(new PutObjectRequest("downloads.typesafe.com", key, file))
36+
}
37+
}
2438

2539
ScalaDist.settings
2640

@@ -32,4 +46,4 @@ enablePlugins(UniversalPlugin, RpmPlugin, JDebPackaging, WindowsPlugin)
3246

3347
// resolvers += "local" at "file:///e:/.m2/repository"
3448
// resolvers += Resolver.mavenLocal
35-
// to test, run e.g., stage, or windows:packageBin, show s3-upload::mappings
49+
// to test, run e.g., stage, or windows:packageBin, show s3Upload::mappings

‎project/ScalaDist.scala

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@ import com.typesafe.sbt.packager.MappingsHelper._
66
import com.typesafe.sbt.packager.universal.UniversalPlugin.autoImport.useNativeZip
77
import com.typesafe.sbt.packager.Keys._
88

9-
import com.typesafe.sbt.S3Plugin.S3.upload
9+
import com.amazonaws.services.s3.model.PutObjectResult
1010

1111
// can't call it Universal -- that's taken by the packager
1212
object ScalaDist {
13+
val upload=TaskKey[Seq[PutObjectResult]]("s3-upload","Uploads files to an S3 bucket.")
14+
1315
def createMappingsWith(deps: Seq[(String, ModuleID, Artifact, File)],
1416
distMappingGen: (ModuleID, Artifact, File) => Seq[(File, String)]): Seq[(File, String)] =
1517
deps flatMap {
@@ -88,13 +90,13 @@ object ScalaDist {
8890

8991
// create mappings from the unzip scala-dist zip
9092
contentOf(tmpdir) filter {
91-
case (file, dest) => !(dest.endsWith("MANIFEST.MF") || dest.endsWith("META-INF"))
93+
case (file, dest) => !(dest.endsWith("MANIFEST.MF") || dest.endsWith("META-INF"))
9294
} map {
9395
// make unix scripts executable (heuristically...)
94-
case (file, dest) if (dest startsWith "bin/") && !(dest endsWith ".bat") =>
96+
case (file, dest) if (dest startsWith "bin/") && !(dest endsWith ".bat") =>
9597
file.setExecutable(true, true)
96-
file -> dest
97-
case mapping => mapping
98+
file -> dest
99+
case mapping => mapping
98100
}
99101

100102
// core jars: use simple name for backwards compat

‎project/build.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
sbt.version=0.13.13
1+
sbt.version=0.13.17

‎project/plugins.sbt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@ libraryDependencies += "org.vafer" % "jdeb" % "1.3" artifacts (Artifact("jdeb",
55
libraryDependencies += "com.spotify" % "docker-client" % "8.9.0"
66
addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.3")
77

8-
addSbtPlugin("com.typesafe.sbt" % "sbt-s3" % "0.8")
8+
libraryDependencies += "com.amazonaws" % "aws-java-sdk-s3" % "1.11.277"
99

1010
// git plugin
1111
resolvers += "jgit-repo" at "http://download.eclipse.org/jgit/maven"
1212

1313
addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.6.4")
14+

‎scripts/common

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# https://github.com/travis-ci/docs-travis-ci-com/issues/949
2+
function travis_fold_start() {
3+
echo ""
4+
echo -e "travis_fold:start:$1\033[33;1m$2\033[0m"
5+
}
6+
7+
function travis_fold_end() {
8+
echo -e "\ntravis_fold:end:$1\r"
9+
echo ""
10+
}
11+
12+
# Params:
13+
# - state: pending / success / failed / error
14+
function postCommitStatus() {
15+
if [[ "$scala_sha" != "" ]]; then
16+
local jsonTemplate='{ "state": "%s", "target_url": "%s", "description": "%s", "context": "%s"}'
17+
local json=$(printf "$jsonTemplate" "$1" "https://travis-ci.org/scala/scala-dist/builds/$TRAVIS_BUILD_ID" "$1" "travis/scala-dist/$version/$mode")
18+
19+
local curlStatus=$(curl \
20+
-s -o /dev/null -w "%{http_code}" \
21+
-H "Accept: application/vnd.github.v3+json" \
22+
-H "Authorization: token $GITHUB_OAUTH_TOKEN" \
23+
-d "$json" \
24+
https://api.github.com/repos/scala/scala/statuses/$scala_sha)
25+
26+
[[ "$curlStatus" == "201" ]] || {
27+
echo "Failed to publish GitHub commit status"
28+
exit 1
29+
}
30+
fi
31+
}

0 commit comments

Comments
 (0)