Add Concourse CI pipeline

Add CI pipeline for builds and releases.

Fixes gh-9316

ci/README.adoc

@@ -0,0 +1,49 @@
+== Concourse pipeline
+
+To set the pipeline first create a file in this directory called `secrets.yml`:
+
+[source,yaml]
+.secrets.yml
+----
+docker-hub-username: <user>
+docker-hub-password: <secret>
+github-username: <user>
+github-password: <secret>
+artifactory-username: <user>
+artifactory-password: <secret>
+----
+
+NOTE: The file should be ignored by git, make sure that you don't commit it!
+
+Once the file has been created, the pipeline can be deployed:
+
+[source]
+----
+$ fly -t spring set-pipeline -p spring-boot -c ci/pipeline.yml -l ci/parameters.yml -l ci/secrets.yml
+----
+
+=== Release
+
+To release a milestone:
+
+[source]
+----
+$ fly -t spring trigger-job -j spring-boot/stage-milestone
+$ fly -t spring trigger-job -j spring-boot/promote-milestone
+----
+
+To release an RC:
+
+[source]
+----
+$ fly -t spring trigger-job -j spring-boot/stage-rc
+$ fly -t spring trigger-job -j spring-boot/promote-rc
+----
+
+To release a GA:
+
+[source]
+----
+$ fly -t spring trigger-job -j spring-boot/stage-release
+$ fly -t spring trigger-job -j spring-boot/promote-release
+----

ci/images/spring-boot-ci-image/Dockerfile

@@ -0,0 +1,8 @@
+FROM openjdk:8u141-jdk
+
+RUN apt-get update && \
+    apt-get install -y git && \
+    apt-get install -y libxml2-utils && \
+    apt-get install -y jq
+
+ADD https://raw.githubusercontent.com/spring-io/concourse-java-scripts/v0.0.1/concourse-java.sh /opt/

ci/images/spring-boot-jdk9-ci-image/Dockerfile

@@ -0,0 +1,8 @@
+FROM openjdk:9-b179-jdk
+
+RUN apt-get update && \
+    apt-get install -y git && \
+    apt-get install -y libxml2-utils && \
+    apt-get install -y jq
+
+ADD https://raw.githubusercontent.com/spring-io/concourse-java-scripts/v0.0.1/concourse-java.sh /opt/

ci/parameters.yml

@@ -0,0 +1,10 @@
+email-server: "smtp.svc.pivotal.io"
+email-from: "ci@spring.io"
+email-to: ["spring-boot-dev@pivotal.io"]
+github-repo: "https://github.com/spring-projects/spring-boot.git"
+docker-hub-organization: "springci"
+artifactory-server: "https://repo.spring.io/"
+branch: "master"
+build-name: "spring-boot"
+bintray-subject: "spring"
+bintray-repo: "jars"

ci/pipeline.yml

@@ -0,0 +1,259 @@
+resource_types:
+- name: artifactory-resource
+  type: docker-image
+  source:
+    repository: springio/artifactory-resource
+    tag: 0.0.1
+- name: email
+  type: docker-image
+  source:
+    repository: pcfseceng/email-resource
+resources:
+- name: git-repo
+  type: git
+  source:
+    uri: ((github-repo))
+    username: ((github-username))
+    password: ((github-password))
+    branch: ((branch))
+    ignore_paths: ["ci/images/*"]
+- name: ci-images-git-repo
+  type: git
+  source:
+    uri: ((github-repo))
+    branch: ((branch))
+    paths: ["ci/images/*"]
+- name: spring-boot-ci-image
+  type: docker-image
+  source:
+    repository: ((docker-hub-organization))/spring-boot-ci-image
+    username: ((docker-hub-username))
+    password: ((docker-hub-password))
+    tag: ((branch))
+- name: spring-boot-jdk9-ci-image
+  type: docker-image
+  source:
+    repository: ((docker-hub-organization))/spring-boot-jdk9-ci-image
+    username: ((docker-hub-username))
+    password: ((docker-hub-password))
+    tag: ((branch))
+- name: artifactory-repo
+  type: artifactory-resource
+  source:
+    uri: ((artifactory-server))
+    username: ((artifactory-username))
+    password: ((artifactory-password))
+    build_name: ((build-name))
+- name: email-notification
+  type: email
+  source:
+    smtp:
+      host: ((email-server))
+      port: "25"
+      anonymous: true
+      skip_ssl_validation: true
+    from: ((email-from))
+    to: ((email-to))
+jobs:
+- name: build-spring-boot-ci-images
+  plan:
+  - get: ci-images-git-repo
+    trigger: true
+  - put: spring-boot-ci-image
+    params:
+      build: ci-images-git-repo/ci/images/spring-boot-ci-image
+  - put: spring-boot-jdk9-ci-image
+    params:
+      build: ci-images-git-repo/ci/images/spring-boot-jdk9-ci-image
+- name: build
+  serial: true
+  plan:
+  - get: spring-boot-ci-image
+  - get: git-repo
+    trigger: true
+  - do:
+    - task: build-project
+      timeout: 1h30m
+      image: spring-boot-ci-image
+      file: git-repo/ci/tasks/build-project.yml
+    - aggregate:
+      - task: build-samples
+        timeout: 1h30m
+        image: spring-boot-ci-image
+        file: git-repo/ci/tasks/build-samples.yml
+      - task: build-integration-tests
+        timeout: 1h30m
+        image: spring-boot-ci-image
+        file: git-repo/ci/tasks/build-integration-tests.yml
+      - task: build-deployment-tests
+        timeout: 1h30m
+        image: spring-boot-ci-image
+        file: git-repo/ci/tasks/build-deployment-tests.yml
+    on_failure:
+      put: email-notification
+      params:
+        subject_text: "Build failure ${BUILD_PIPELINE_NAME} / ${BUILD_JOB_NAME} / ${BUILD_NAME}"
+        body_text: "Build ${ATC_EXTERNAL_URL}/teams/${BUILD_TEAM_NAME}/pipelines/${BUILD_PIPELINE_NAME}/jobs/${BUILD_JOB_NAME}/builds/${BUILD_NAME} has failed!"
+  - put: artifactory-repo
+    params: &artifactory-params
+      repo: libs-snapshot-local
+      build_number: "${BUILD_ID}"
+      folder: distribution-repository
+      build_uri: "https://ci.spring.io/teams/${BUILD_TEAM_NAME}/pipelines/${BUILD_PIPELINE_NAME}/jobs/${BUILD_JOB_NAME}/builds/${BUILD_NAME}"
+      build_number: "${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}"
+      artifact_set:
+      - include:
+        - "/**/spring-boot-docs-*.zip"
+        properties:
+          zip-type: "docs"
+          zip-deployed: "false"
+- name: jdk9-build
+  serial: true
+  plan:
+  - get: spring-boot-jdk9-ci-image
+  - get: git-repo
+  - do:
+    - task: build-project
+      timeout: 1h30m
+      image: spring-boot-jdk9-ci-image
+      file: git-repo/ci/tasks/build-project.yml
+    - aggregate:
+      - task: build-samples
+        timeout: 1h30m
+        image: spring-boot-jdk9-ci-image
+        file: git-repo/ci/tasks/build-samples.yml
+      - task: build-integration-tests
+        timeout: 1h30m
+        image: spring-boot-jdk9-ci-image
+        file: git-repo/ci/tasks/build-integration-tests.yml
+      - task: build-deployment-tests
+        timeout: 1h30m
+        image: spring-boot-jdk9-ci-image
+        file: git-repo/ci/tasks/build-deployment-tests.yml
+    on_failure:
+      put: email-notification
+      params:
+        subject_text: "JDK 9 Build failure ${BUILD_PIPELINE_NAME} / ${BUILD_JOB_NAME} / ${BUILD_NAME}"
+        body_text: "Build ${ATC_EXTERNAL_URL}/teams/${BUILD_TEAM_NAME}/pipelines/${BUILD_PIPELINE_NAME}/jobs/${BUILD_JOB_NAME}/builds/${BUILD_NAME} has failed!"
+- name: stage-milestone
+  serial: true
+  plan:
+  - get: spring-boot-ci-image
+  - get: git-repo
+    trigger: false
+  - task: stage
+    image: spring-boot-ci-image
+    file: git-repo/ci/tasks/stage.yml
+    params:
+      RELEASE_TYPE: M
+  - put: artifactory-repo
+    params:
+      <<: *artifactory-params
+      repo: libs-staging-local
+  - put: git-repo
+    params:
+      repository: stage-git-repo
+- name: stage-rc
+  serial: true
+  plan:
+  - get: spring-boot-ci-image
+  - get: git-repo
+    trigger: false
+  - task: stage
+    image: spring-boot-ci-image
+    file: git-repo/ci/tasks/stage.yml
+    params:
+      RELEASE_TYPE: RC
+  - put: artifactory-repo
+    params:
+      <<: *artifactory-params
+      repo: libs-staging-local
+  - put: git-repo
+    params:
+      repository: stage-git-repo
+- name: stage-release
+  serial: true
+  plan:
+  - get: spring-boot-ci-image
+  - get: git-repo
+    trigger: false
+  - task: stage
+    image: spring-boot-ci-image
+    file: git-repo/ci/tasks/stage.yml
+    params:
+      RELEASE_TYPE: RELEASE
+  - put: artifactory-repo
+    params:
+      <<: *artifactory-params
+      repo: libs-staging-local
+  - put: git-repo
+    params:
+      repository: stage-git-repo
+- name: promote-milestone
+  serial: true
+  plan:
+  - get: spring-boot-ci-image
+  - get: git-repo
+    trigger: false
+  - get: artifactory-repo
+    trigger: false
+    passed: [stage-milestone]
+    params:
+      save_build_info: true
+  - task: promote
+    image: spring-boot-ci-image
+    file: git-repo/ci/tasks/promote.yml
+    params:
+      RELEASE_TYPE: M
+      ARTIFACTORY_SERVER: ((artifactory-server))
+      ARTIFACTORY_USERNAME: ((artifactory-username))
+      ARTIFACTORY_PASSWORD: ((artifactory-password))
+- name: promote-rc
+  serial: true
+  plan:
+  - get: git-repo
+    trigger: false
+  - get: artifactory-repo
+    trigger: false
+    passed: [stage-rc]
+    params:
+      save_build_info: true
+  - task: promote
+    image: spring-boot-ci-image
+    file: git-repo/ci/tasks/promote.yml
+    params:
+      RELEASE_TYPE: RC
+      ARTIFACTORY_SERVER: ((artifactory-server))
+      ARTIFACTORY_USERNAME: ((artifactory-username))
+      ARTIFACTORY_PASSWORD: ((artifactory-password))
+- name: promote-release
+  serial: true
+  plan:
+  - get: git-repo
+    trigger: false
+  - get: artifactory-repo
+    trigger: false
+    passed: [stage-release]
+    params:
+      save_build_info: true
+  - task: promote
+    image: spring-boot-ci-image
+    file: git-repo/ci/tasks/promote.yml
+    params:
+      RELEASE_TYPE: RELEASE
+      ARTIFACTORY_SERVER: ((artifactory-server))
+      ARTIFACTORY_USERNAME: ((artifactory-username))
+      ARTIFACTORY_PASSWORD: ((artifactory-password))
+      BINTRAY_USERNAME: ((bintray-username))
+      BINTRAY_PASSWORD: ((bintray-password))
+      SONATYPE_USERNAME: ((sonattype-username))
+      SONATYPE_PASSWORD: ((sonattype-password))
+      BINTRAY_SUBJECT: ((bintray-subject))
+      BINTRAY_REPO: ((bintray-repo))
+groups:
+- name: "Build"
+  jobs: ["build", "jdk9-build"]
+- name: "Release"
+  jobs: ["stage-milestone", "stage-rc", "stage-release", "promote-milestone", "promote-rc", "promote-release"]
+- name: "CI Images"
+  jobs: ["build-spring-boot-ci-images"]

ci/scripts/build-deployment-tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/common.sh
+repository=$(pwd)/distribution-repository
+
+pushd git-repo > /dev/null
+run_maven -f spring-boot-tests/spring-boot-deployment-tests/pom.xml clean install -U -Dfull -Drepository=file://${repository}
+popd > /dev/null

ci/scripts/build-integration-tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/common.sh
+repository=$(pwd)/distribution-repository
+
+pushd git-repo > /dev/null
+run_maven -f spring-boot-tests/spring-boot-integration-tests/pom.xml clean install -U -Dfull -Drepository=file://${repository}
+popd > /dev/null

ci/scripts/build-project.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/common.sh
+repository=$(pwd)/distribution-repository
+
+pushd git-repo > /dev/null
+run_maven -f spring-boot-project/pom.xml clean deploy -U -Dfull -DaltDeploymentRepository=distribution::default::file://${repository}
+popd > /dev/null

ci/scripts/build-samples.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/common.sh
+repository=$(pwd)/distribution-repository
+
+pushd git-repo > /dev/null
+run_maven -f spring-boot-samples/pom.xml clean install -U -Dfull -Drepository=file://${repository}
+popd > /dev/null

ci/scripts/common.sh

@@ -0,0 +1,4 @@
+source /opt/concourse-java.sh
+
+setup_symlinks
+cleanup_maven_repo "org.springframework.boot"

ci/scripts/promote.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/common.sh
+
+buildName=$( cat artifactory-repo/build-info.json | jq -r '.buildInfo.name' )
+buildNumber=$( cat artifactory-repo/build-info.json | jq -r '.buildInfo.number' )
+groupId=$( cat artifactory-repo/build-info.json | jq -r '.buildInfo.modules[0].id' | sed 's/\(.*\):.*:.*/\1/' )
+version=$( cat artifactory-repo/build-info.json | jq -r '.buildInfo.modules[0].id' | sed 's/.*:.*:\(.*\)/\1/' )
+
+
+if [[ $RELEASE_TYPE = "M" ]]; then
+	targetRepo="libs-milestone-local"
+elif [[ $RELEASE_TYPE = "RC" ]]; then
+	targetRepo="libs-milestone-local"
+elif [[ $RELEASE_TYPE = "RELEASE" ]]; then
+	targetRepo="libs-release-local"
+else
+	echo "Unknown release type $RELEASE_TYPE" >&2; exit 1;
+fi
+
+echo "Promoting ${buildName}/${buildNumber} to ${targetRepo}"
+
+curl \
+	-u ${ARTIFACTORY_USERNAME}:${ARTIFACTORY_PASSWORD} \
+	-H"Content-type:application/json" \
+	-d "{\"status\": \"staged\", \"sourceRepo\": \"libs-staging-local\", \"targetRepo\": \"\"}"  \
+	-f \
+	-X \
+	POST "${ARTIFACTORY_SERVER}/api/build/promote/${buildName}/${buildNumber}" > /dev/null || { echo "Failed to promote" >&2; exit 1; }
+
+elif [[ $RELEASE_TYPE = "RELEASE" ]]; then
+	curl \
+		-u ${ARTIFACTORY_USERNAME}:${ARTIFACTORY_PASSWORD} \
+		-H"Content-type:application/json" \
+		-u ${ARTIFACTORY_USERNAME}:${ARTIFACTORY_PASSWORD} \
+		-d "{\"sourceRepos\": [\"libs-release-local\"], \"targetRepo\" : \"spring-distributions\"}"
+		-f \
+		-X \
+		POST "${ARTIFACTORY_SERVER}/api/build/distribute/${buildName}/${buildNumber}" > /dev/null || { echo "Failed to publish" >&2; exit 1; }
+
+	curl \
+		-u ${BINTRAY_USERNAME}:${BINTRAY_PASSWORD} \
+		-H "Content-Type: application/json" -d "{\"username\": \"${SONATYPE_USERNAME}\", \"password\": \"${SONATYPE_PASSWORD}\"}"
+		-f \
+		-X \
+		POST "https://api.bintray.com/maven_central_sync/${BINTRAY_SUBJECT}/${BINTRAY_REPO}/${groupId}/versions/${version}" > /dev/null || { echo "Failed to sync" >&2; exit 1; }
+fi
+
+
+echo "Promotion complete"

ci/scripts/stage.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/common.sh
+repository=$(pwd)/distribution-repository
+
+pushd git-repo > /dev/null
+git fetch --tags --all > /dev/null
+popd > /dev/null
+
+git clone git-repo stage-git-repo > /dev/null
+
+pushd stage-git-repo > /dev/null
+
+snapshotVersion=$( get_revision_from_pom )
+if [[ $RELEASE_TYPE = "M" ]]; then
+	stageVersion=$( get_next_milestone_release $snapshotVersion)
+	nextVersion=$snapshotVersion
+elif [[ $RELEASE_TYPE = "RC" ]]; then
+	stage=$( get_next_rc_release $snapshotVersion)
+	nextVersion=$snapshotVersion
+elif [[ $RELEASE_TYPE = "RELEASE" ]]; then
+	stage=$( strip_snapshot_suffix $snapshotVersion)
+	nextVersion=$( bump_version_number $snapshotVersion)
+else
+	echo "Unknown release type $RELEASE_TYPE" >&2; exit 1;
+fi
+
+echo "Staging $stageVersion (next version will be $nextVersion)"
+
+set_revision_to_pom "$stageVersion"
+git config user.name "Spring Buildmaster" > /dev/null
+git config user.email "buildmaster@springframework.org" > /dev/null
+git add pom.xml > /dev/null
+git commit -m"Release v$stageVersion" > /dev/null
+git tag -a "v$stageVersion" -m"Release v$stageVersion" > /dev/null
+
+run_maven -f spring-boot-project/pom.xml clean deploy -U -Dfull -DaltDeploymentRepository=distribution::default::file://${repository}
+run_maven -f spring-boot-tests/spring-boot-sample-tests/pom.xml clean install -U -Dfull -Drepository=file://${repository}
+run_maven -f spring-boot-tests/spring-boot-integration-tests/pom.xml clean install -U -Dfull -Drepository=file://${repository}
+run_maven -f spring-boot-tests/spring-boot-deployment-tests/pom.xml clean install -U -Dfull -Drepository=file://${repository}
+
+git reset --hard HEAD^ > /dev/null
+if [[ $nextVersion != $snapshotVersion ]]; then
+	echo "Setting next development version (v$nextVersion)"
+	set_revision_to_pom "$nextVersion"
+	git add pom.xml > /dev/null
+	git commit -m"Next development version (v$nextVersion)" > /dev/null
+fi;
+
+echo "DONE"
+
+popd > /dev/null

ci/tasks/build-deployment-tests.yml

@@ -0,0 +1,10 @@
+---
+platform: linux
+inputs:
+- name: git-repo
+- name: distribution-repository
+caches:
+- path: maven
+- path: gradle
+run:
+  path: git-repo/ci/scripts/build-deployment-tests.sh

ci/tasks/build-integration-tests.yml

@@ -0,0 +1,10 @@
+---
+platform: linux
+inputs:
+- name: git-repo
+- name: distribution-repository
+caches:
+- path: maven
+- path: gradle
+run:
+  path: git-repo/ci/scripts/build-integration-tests.sh

ci/tasks/build-project.yml

@@ -0,0 +1,11 @@
+---
+platform: linux
+inputs:
+- name: git-repo
+outputs:
+- name: distribution-repository
+caches:
+- path: maven
+- path: gradle
+run:
+  path: git-repo/ci/scripts/build-project.sh

ci/tasks/build-samples.yml

@@ -0,0 +1,10 @@
+---
+platform: linux
+inputs:
+- name: git-repo
+- name: distribution-repository
+caches:
+- path: maven
+- path: gradle
+run:
+  path: git-repo/ci/scripts/build-samples.sh

ci/tasks/promote.yml

@@ -0,0 +1,7 @@
+---
+platform: linux
+inputs:
+- name: git-repo
+- name: artifactory-repo
+run:
+  path: git-repo/ci/scripts/promote.sh

ci/tasks/stage.yml

@@ -0,0 +1,12 @@
+---
+platform: linux
+inputs:
+- name: git-repo
+outputs:
+- name: stage-git-repo
+- name: distribution-repository
+caches:
+- path: maven
+- path: gradle
+run:
+  path: git-repo/ci/scripts/stage.sh