Hierarchical script/job config layout

Author: adriaanm

libraries/job_blurbs.rb

@@ -62,11 +62,19 @@ def scmBlurb
       EOH
     end
 
+    def versionedJob(version, name)
+      "scala-#{version}-#{name.gsub(/\//, '-')}"
+    end
+
+    def job(name)
+      versionedJob(@version, name)
+    end
+
     def scriptBuild
-      <<-EOH.gsub(/^ {4}/, '')
+      <<-EOH.gsub(/^      /, '')
       <hudson.tasks.Shell>
         <command>#!/bin/bash -ex
-        source scripts/jobs/$JOB_NAME
+      source scripts/#{@scriptName}
         </command>
       </hudson.tasks.Shell>
       EOH

recipes/_master-config-jobs.rb

@@ -15,20 +15,48 @@
   end
 end
 
-# TODO: auto-discover templates, organize them in views according to subdirs under templates/defaults
-# run_context.cookbook_collection[cookbook_name].manifest[:templates] is an array of
-# {"name"=>"scala-release-2.11.x-unix.xml.erb",
-# "path"=>"templates/default/scala-release-2.11.x/scala-release-2.11.x-unix.xml.erb", ...}
 
-%w(scala-release-2.11.x scala-release-2.11.x-build scala-release-2.11.x-unix scala-release-2.11.x-windows scala-release-2.11.x-smoketest scala-release-2.11.x-scala-lang scala-release-2.11.x-scala-lang-update-current).each do |name|
-  xml = File.join(Chef::Config[:file_cache_path], "#{name}.xml")
+# TODO: is there no shorter way??
+class Blurbs
+  include ScalaJenkinsInfra::JobBlurbs
+end
+
+# turn template path into jenkins job name
+def templDesc(version, path)
+  blurbs = Blurbs.new
 
-  template xml do
-    source "scala-release-2.11.x/#{name}.xml.erb"
-    helpers(ScalaJenkinsInfra::JobBlurbs)
+  m = path.match(/templates\/default\/jobs\/(.*)\.xml\.erb$/)
+  if m == nil
+    []
+  else
+    relativePath = m.captures.first
+
+    [ { :templatePath => "jobs/#{relativePath}.xml.erb",
+        :scriptName   => "jobs/#{relativePath}",
+        :jobName      => blurbs.versionedJob(version, relativePath),
+        :version      => version
+      }
+    ]
   end
+end
+
+# create scala-$version-$jobName for every template under jobs/
+%w{ 2.11.x }.each do | version |
+  node.run_context.cookbook_collection["scala-jenkins-infra"].manifest[:templates]
+    .flat_map { |mani| templDesc(version, mani['path']) }
+    .each do | desc |
+
+    xml = File.join(Chef::Config[:file_cache_path], "#{desc[:jobName]}.xml")
+
+    template xml do
+      variables(desc)
+
+      source desc[:templatePath]
+      helpers(ScalaJenkinsInfra::JobBlurbs)
+    end
 
-  jenkins_job name do
-    config xml
+    jenkins_job desc[:jobName] do
+      config xml
+    end
   end
 end

templates/default/scala-release-2.11.x/scala-release-2.11.x-build.xml.erb renamed to templates/default/jobs/integrate/bootstrap.xml.erb

@@ -3,20 +3,13 @@
 <%= githubProject(
   repoUser:    "scala",
   repoName:    "scala",
-  repoRef:     "2.11.x",
-  description: %{ This job publishes a Scala distribution to maven using
-    https://github.com/scala/jenkins-scripts/blob/master/job/scala-release-2.11.x. It
-    builds locker and uses it to build and publish the required modules as well as the
+  repoRef:     @version,
+  description: %{ Stage a Scala distribution on sonatype using scripts/job/bootstrap.
+    It builds locker and uses it to build and publish the required modules as well as the
     Scala compiler/library for this release, and generates an updated versions.properties
     that captures the modules/STARR used.
 
-    <p> This job is run nightly to build nightly releases (unless HEAD of scala/scala is
-    tagged, on which it builds a release and stages it on sonatype).
-
-    <p> On building a release, the remaining tasks are to close the staging repo (see
-    https://github.com/adriaanm/binfu/blob/master/sonafu.sh for scripts), possibly re-run
-    the downstream scala-release-2.11.x-dist job once all external artifacts are in place
-    (akka-actor), and send out some emails.
+    <p> This job is run on every merge to publish a "nightly" snapshot, and on every tag, to stage a full release on sonatype.
 
     <p> Use github.com/scala/make-release-notes to build the release notes.
    }.gsub(/^    /, ''),

templates/default/scala-release-2.11.x/scala-release-2.11.x.xml.erb renamed to templates/default/jobs/release/main.xml.erb

@@ -1,17 +1,13 @@
 <?xml version='1.0' encoding='UTF-8'?>
 <com.cloudbees.plugins.flow.BuildFlow plugin="[email protected]">
-  <description>Run as downstream job from scala-release-2.11.x.&#xd;
-&lt;p&gt;&#xd;
-Coordinates the build of a Scala distribution based on artifacts on maven (built by scala-release-2.11.x).&#xd;
-Downstream jobs: scala-release-2.11.x-[unix|windows|scala-lang].&#xd;
-&lt;p&gt;&#xd;
-Nightly downloads go to http://www.scala-lang.org/files/archive/nightly/2.11.x/ &lt;br&gt;&#xd;
+  <description>Build, package and stage a Scala release. TODO
+  Nightly downloads go to http://www.scala-lang.org/files/archive/nightly/<%=@version%>/ &lt;br&gt;&#xd;
   Regular ones to http://www.scala-lang.org/files/archive/</description>
   <properties>
     <hudson.model.ParametersDefinitionProperty>
       <parameterDefinitions>
         <%= scmUserParam("scala") %>
-        <%= scmRefParam("2.11.x") %>
+        <%= scmRefParam(@version) %>
         <hudson.model.StringParameterDefinition>
           <name>SCALA_VER_BASE</name>
           <description>When set, a release with version "$SCALA_VER_BASE$SCALA_VER_SUFFIX" will be built. Same effect as for tagging $repo_ref as "v$SCALA_VER_BASE$SCALA_VER_SUFFIX".</description>
@@ -38,10 +34,10 @@ build.setDisplayName(&quot;Scala distribution ${ver} [${build.number}]&quot;)
 if (params.repo_ref) buildParams = params
 else buildParams = params + [ &apos;repo_ref&apos; : &quot;v$ver&quot; ]
 
-buildRun = build(buildParams, &quot;scala-release-2.11.x-build&quot;)
+buildRun = build(buildParams, &quot;<%=job("integrate/bootstrap")%>&quot;)
 
 upstreamBuild = buildRun.build
-// for testing, to avoid re-running the main build: upstreamBuild = jenkins.model.Jenkins.instance.items.find{it.name == &quot;scala-release-2.11.x-build&quot;}.builds[0]
+// for testing, to avoid re-running the main build: upstreamBuild = jenkins.model.Jenkins.instance.items.find{it.name == &quot;<%=job("integrate/bootstrap")%>&quot;}.builds[0]
 // defines repo_ref, version, sbtDistVersionOverride
 upstreamPropsArtifact = upstreamBuild.artifacts.find{it.name == &quot;jenkins.properties&quot;}
 
@@ -51,13 +47,13 @@ buildVersionProps.load(new java.io.FileInputStream(upstreamPropsArtifact.file))
 testParams = buildParams + buildVersionProps
 
 parallel (
-    { build(testParams, &quot;scala-release-2.11.x-unix&quot;) },
-    { build(testParams, &quot;scala-release-2.11.x-windows&quot;) }
+    { build(testParams, &quot;<%=job("release/package/unix")%>&quot;) },
+    { build(testParams, &quot;<%=job("release/package/windows")%>&quot;) }
 )
 
-build(testParams, &quot;scala-release-2.11.x-smoketest&quot;)
+build(testParams, &quot;<%=job("release/smoketest")%>&quot;)
 
-build(testParams, &quot;scala-release-2.11.x-scala-lang&quot;)
+build(testParams, &quot;<%=job("release/website/archives")%>&quot;)
 </dsl>
   <publishers>
     <hudson.tasks.Mailer plugin="[email protected]">

templates/default/scala-release-2.11.x/scala-release-2.11.x-unix.xml.erb renamed to templates/default/jobs/release/package/unix.xml.erb

@@ -3,8 +3,8 @@
 <%= githubProject(
   repoUser:    "scala",
   repoName:    "scala-dist",
-  repoRef:     "2.11.x",
-  description: %{ Run as part of the scala-release-2.11.x-dist flow. <p> Builds and uploads the universal and unix-specific artifacts for a
+  repoRef:     @version,
+  description: "Run as part of the #{job("release/main")} flow.<p>" + %{Builds and uploads the universal and unix-specific artifacts for a
     Scala release.<br>
 
     Specify a tag that exists on <a href="https://github.com/scala/scala-dist">scala/scala-dist</a>,<br> and that points to a

templates/default/scala-release-2.11.x/scala-release-2.11.x-windows.xml.erb renamed to templates/default/jobs/release/package/windows.xml.erb

@@ -3,8 +3,8 @@
 <%= githubProject(
   repoUser:    "scala",
   repoName:    "scala-dist",
-  repoRef:     "2.11.x",
-  description: %{Run as part of the scala-release-2.11.x-dist flow. <p> Builds and uploads the windows-specific artifacts for a Scala
+  repoRef:     @version,
+  description: "Run as part of the #{job("release/main")} flow.<p>" + %{Builds and uploads the windows-specific artifacts for a Scala
     release.<br>
 
     Specify a tag that exists on <a href="https://github.com/scala/scala-dist">scala/scala-dist</a>,<br> and that points to a

templates/default/scala-release-2.11.x/scala-release-2.11.x-smoketest.xml.erb renamed to templates/default/jobs/release/smoketest.xml.erb

@@ -3,8 +3,8 @@
 <%= githubProject(
   repoUser:    "scala",
   repoName:    "scala-dist-smoketest",
-  repoRef:     "2.11.x",
-  description: %{ Run as part of the scala-release-2.11.x-dist flow. <br/> It runs the program in
+  repoRef:     @version,
+  description: "Run as part of the #{job("release/main")} flow." + %{ <br/> It runs the program in
     https://github.com/scala/scala-dist-smoketest/tree/${repo_ref} to make sure all JARs of the distribution
     contain classfiles. <br/> It downloads the archives for a Scala release from
     http://downloads.typesafe.com/scala/$version/index.html and makes sure the bundled scripts work.

templates/default/scala-release-2.11.x/scala-release-2.11.x-scala-lang.xml.erb renamed to templates/default/jobs/release/website/archives.xml.erb

@@ -3,14 +3,13 @@
 <%= githubProject(
   repoUser:    "scala",
   repoName:    "scala-dist",
-  repoRef:     "2.11.x",
-  description: %{ <H1>Run as part of the scala-release-2.11.x-dist flow.</H1>
-    <p>It downloads the archives for a Scala release
+  repoRef:     @version,
+  description: "<H1>Run as part of the #{job("release/main")} flow.</H1>" +
+   %{<p>It downloads the archives for a Scala release
        from http://downloads.typesafe.com/scala/$version/index.html and
        re-organizes them into the right hierarchy for scala-lang.org/files/archive.</p>
     <p>Also expands api docs for library, reflect, compiler and other modules
-       to http://www.scala-lang.org/api/2.11.x/$module (it expands under api/$version
-       and updates the api/2.11.x symlink to point there).</p>
+       to http://www.scala-lang.org/api/2.11.x/$module (it expands under api/$version).</p>
     <p>When version ends in -nightly, the root for the expansion process
        is http://www.scala-lang.org/files/archive/nightly/2.11.x/
        instead of http://www.scala-lang.org/files/archive/</p>

templates/default/jobs/release/website/update-api.xml.erb

@@ -0,0 +1,14 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<project>
+<%= githubProject(
+  repoUser:    "scala",
+  repoName:    "scala-dist",
+  repoRef:     @version,
+  description: "Update scala-lang's api/#{@version} symlink to point to api/$version",
+  nodeRestriction: "publish",
+  params: [
+    {:name => "version", :desc => "The version to be shown for the current API docs."}
+  ])
+%>
+  <builders><%= scriptBuild %></builders>
+</project>