Merge pull request #46 from adriaanm/master

Centralize strings in prep of private-repo move.

Author: adriaanm

attributes/master.rb

@@ -21,15 +21,33 @@
 default['master']['github']['authenticatedUserCreateJobPermission'] = 'false'
 
 default['master']['adminAddress'] = "[email protected]"
+default['master']['jenkinsHost']  = "scala-ci.typesafe.com" # duplicated because attributes can't refer to each other...
 default['master']['jenkinsUrl']   = "https://scala-ci.typesafe.com/"
 default['master']['jenkins']['notifyUrl'] = "http://scala-ci.typesafe.com:8888/jenkins"
 
+default['repos']['private']['realm']        = "Artifactory Realm"
+default['repos']['private']['host']         = "private-repo.typesafe.com"
+default['repos']['private']['pr-snap']      = "http://private-repo.typesafe.com/typesafe/scala-pr-validation-snapshots/",
+default['repos']['private']['release-temp'] = "http://private-repo.typesafe.com/typesafe/scala-release-temp/"
+
+default['s3']['downloads']['host'] = "downloads.typesafe.com.s3.amazonaws.com"
+
+# see below (note that default['master']['env'] can only indirect through node -- workerJavaOpts is not in scope)
+workerJavaOpts = "-Dfile.encoding=UTF-8 -server -XX:+AggressiveOpts -XX:+UseParNewGC -Xmx2G -Xss1M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=128M -Dpartest.threads=4"
+default['jenkinsEnv']['JAVA_OPTS']  = workerJavaOpts
+default['jenkinsEnv']['ANT_OPTS']   = workerJavaOpts
+default['jenkinsEnv']['MAVEN_OPTS'] = workerJavaOpts # doesn't technically need the -Dpartest one, but oh well
+
+# NOTE: This is a string that represents a closure that closes over the worker node for which it computes the environment.
+# (by convention -- see `environment((eval node["master"]["env"])...` in _master-config-workers
+# Since we can't marshall closures, while attributes need to be sent from master to workers, we must encode them as something that can be shipped...
 default['master']['env'] = <<-'EOH'.gsub(/^ {2}/, '')
   lambda{| node | Chef::Node::ImmutableMash.new({
-    "JAVA_OPTS"  => "-Dfile.encoding=UTF-8 -server -XX:+AggressiveOpts -XX:+UseParNewGC -Xmx2G -Xss1M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=128M -Dpartest.threads=4",
-    "ANT_OPTS"   => "-Dfile.encoding=UTF-8 -server -XX:+AggressiveOpts -XX:+UseParNewGC -Xmx2G -Xss1M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=128M -Dpartest.threads=4",
-    "MAVEN_OPTS" => "-Dfile.encoding=UTF-8 -server -XX:+AggressiveOpts -XX:+UseParNewGC -Xmx2G -Xss1M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=128M",
-    "prRepoUrl"  => "http://private-repo.typesafe.com/typesafe/scala-pr-validation-snapshots/"
+    "JAVA_HOME"  => node['java']['java_home'], # we get the jre if we don't do this
+    "JAVA_OPTS"  => node['jenkinsEnv']['JAVA_OPTS'],
+    "ANT_OPTS"   => node['jenkinsEnv']['ANT_OPTS'],
+    "MAVEN_OPTS" => node['jenkinsEnv']['MAVEN_OPTS'],
+    "prRepoUrl"  => node['repos']['private']['pr-snap']
   })}
   EOH
 

attributes/worker.rb

@@ -63,15 +63,17 @@
   default["jenkinsHomes"][jenkinsHome]["in_demand_delay"] = 1  # if builds are in queue for even one minute, launch this worker
   default["jenkinsHomes"][jenkinsHome]["idle_delay"]      = 15 # take worker off-line after 15 min of idling (we're charged by the hour, so no rush)
 
-  # can't marshall closures, but they sometimes need to be shipped, so encode as string, closing over `node`
   default["_jenkinsHome"] = jenkinsHome
   default["_jenkinsTmp"]  = jenkinsTmp
+
+  # Worker-specific env, the rest is defined in master's attribs.
+  # This needs to be a closure to get laziness so that we can refer to other attributes, but can't marshall closures,
+  # and they sometimes need to be shipped, so encode as string, closing over `node`...
   default["jenkinsHomes"][jenkinsHome]["env"]         = <<-'EOH'.gsub(/^ {4}/, '')
     lambda{| node | Chef::Node::ImmutableMash.new({
       "PATH"          => "/bin:/usr/bin:/cygdrive/c/java/jdk-1.6/bin:/cygdrive/c/Program Files (x86)/Git/Cmd", # TODO express in terms of attributes
       "sbtLauncher"   => "#{node['sbt']['launcher_path']}\\sbt-launch.jar", # from chef-sbt cookbook
       "WIX"           => node['wix']['home'],
-      "JAVA_HOME"     => node['java']['java_home'],
       "TMP"           => "#{node['_jenkinsTmp']}",
       "_JAVA_OPTIONS" => "-Duser.home=#{node['_jenkinsHome']}", # no other way to do this... sbt boot will fail pretty weirdly if it can't write to $HOME/.sbt and $TMP/...
       "SHELLOPTS"     => "igncr" # ignore line-ending issues in shell scripts
@@ -110,13 +112,15 @@
   default["jenkinsHomes"]["/home/jenkins"]["usage_mode"] = publisher ? "exclusive" : "normal"
   default["jenkinsHomes"]["/home/jenkins"]["labels"]     = ["linux", publisher ? "publish": "public"]
 
-  # can't marshall closures, and this one needs to be shipped from worker to master (note: sshCharaArgs only use on publisher, but doesn't contain any private date, so not bothering)
+  # Worker-specific env, the rest is defined in master's attribs.
+  # (note: sshCharaArgs only used on publisher, but doesn't contain any private date, so not bothering to split it out)
+  # This needs to be a closure to get laziness so that we can refer to other attributes, but can't marshall closures,
+  # and they sometimes need to be shipped, so encode as string, closing over `node`...
   default["jenkinsHomes"]["/home/jenkins"]["env"]         = <<-'EOH'.gsub(/^ {4}/, '')
     lambda{| node | Chef::Node::ImmutableMash.new({
       "sshCharaArgs" => '("[email protected]" "-i" "/home/jenkins/.ssh/for_chara")',
       "sbtLauncher"  => File.join(node['sbt']['launcher_path'], "sbt-launch.jar"), # from chef-sbt cookbook
-      "sbtCmd"       => File.join(node['sbt-extras']['setup_dir'], node['sbt-extras']['script_name']), # sbt-extras
-      "JAVA_HOME"    => node['java']['java_home'] # we get the jre if we don't do this
+      "sbtCmd"       => File.join(node['sbt-extras']['setup_dir'], node['sbt-extras']['script_name']) # sbt-extras
     })}
     EOH
 

recipes/_master-config-workers.rb

@@ -38,7 +38,7 @@
 
       # TODO: make retrying more robust
       max_num_retries  10  # how often to retry when the SSH connection is refused during initial connect
-      retry_wait_time  60 # seconds between retries
+      retry_wait_time  60  # seconds between retries
 
       remote_fs   jenkinsHome.dup
       jvm_options workerConfig["jvm_options"]
@@ -54,7 +54,7 @@
       in_demand_delay workerConfig["in_demand_delay"]
       idle_delay      workerConfig["idle_delay"]
 
-      environment((eval node["master"]["env"]).call(node).merge((eval workerConfig["env"]).call(worker)))
+      environment((eval node["master"]["env"]).call(worker).merge((eval workerConfig["env"]).call(worker)))
 
       action [:create] # we don't need to :connect, :online since the ec2 start/stop plugin will do that. Also, if connect fails, it may be that chef-client hasn't yet run on the client to initialize jenkins home with .ssh/authorized_keys
     end

templates/default/credentials-private-repo.erb

@@ -1,4 +1,4 @@
-realm=Artifactory Realm
-host=private-repo.typesafe.com
+realm=<%=node['repos']['private']['realm']%>
+host=<%=node['repos']['private']['host']%>
 user=<%= @privateRepoUser %>
 password=<%= @privateRepoPass %>

templates/default/m2-settings-public-jobs.xml.erb

@@ -21,12 +21,12 @@
         <repository>
           <id>pr-scala</id>
           <name>Scala PR validation snapshots</name>
-          <url>http://private-repo.typesafe.com/typesafe/scala-pr-validation-snapshots</url>
+          <url><%=node['repos']['private']['pr-snap']%></url>
         </repository>
         <repository>
           <id>scala-release-temp</id>
           <name>Scala Relese snapshots</name>
-          <url>http://private-repo.typesafe.com/typesafe/scala-release-temp/</url>
+          <url><%=node['repos']['private']['release-temp']%></url>
         </repository>
       </repositories>
     </profile>

templates/default/nginx-jenkins.conf

@@ -13,7 +13,7 @@ server {
 # based on https://gist.github.com/plentz/6737338, among others
 server {
   listen 443 ssl default deferred;
-  server_name scala-ci.typesafe.com;
+  server_name <%=node['master']['jenkinsHost']%>;
 
   ssl on;
   ssl_certificate /etc/nginx/ssl/scala-ci.crt;

templates/default/s3credentials.erb

@@ -1,4 +1,4 @@
 realm=Amazon S3
-host=downloads.typesafe.com.s3.amazonaws.com
-user=<%= @s3DownloadsUser %>
-password=<%= @s3DownloadsPass %>
+host=<%=node['s3']['downloads']['host']%>
+user=<%=@s3DownloadsUser%>
+password=<%=@s3DownloadsPass%>

templates/default/scabot.conf.erb

@@ -2,7 +2,7 @@ scala: {
   jenkins: {
     job:   "scala-2.11.x-validate-main"
     jobPrefix: "scala-2.11.x-" // this will be stripped when reporting to github TODO: remove once it's computed as s"${github.repo}-${pull.base.ref}-"
-    host:  "scala-ci.typesafe.com"
+    host:  "<%=node['master']['jenkinsHost']%>"
     user:  "<%=node['scabot']['jenkins']['user']%>"
     token: "<%=node['scabot']['jenkins']['token']%>"
   }