From e9f94629175dfe309f955811ce77d5a6a612abd0 Mon Sep 17 00:00:00 2001 From: Phillip Webb Date: Thu, 28 Sep 2017 10:57:35 -0700 Subject: [PATCH] Add Concourse CI pipeline Add CI pipeline for builds and releases. Fixes gh-9316 --- ci/README.adoc | 49 ++++ ci/images/spring-boot-ci-image/Dockerfile | 8 + .../spring-boot-jdk9-ci-image/Dockerfile | 8 + ci/parameters.yml | 10 + ci/pipeline.yml | 259 ++++++++++++++++++ ci/scripts/build-deployment-tests.sh | 9 + ci/scripts/build-integration-tests.sh | 9 + ci/scripts/build-project.sh | 9 + ci/scripts/build-samples.sh | 9 + ci/scripts/common.sh | 4 + ci/scripts/promote.sh | 51 ++++ ci/scripts/stage.sh | 53 ++++ ci/tasks/build-deployment-tests.yml | 10 + ci/tasks/build-integration-tests.yml | 10 + ci/tasks/build-project.yml | 11 + ci/tasks/build-samples.yml | 10 + ci/tasks/promote.yml | 7 + ci/tasks/stage.yml | 12 + 18 files changed, 538 insertions(+) create mode 100644 ci/README.adoc create mode 100644 ci/images/spring-boot-ci-image/Dockerfile create mode 100644 ci/images/spring-boot-jdk9-ci-image/Dockerfile create mode 100644 ci/parameters.yml create mode 100644 ci/pipeline.yml create mode 100755 ci/scripts/build-deployment-tests.sh create mode 100755 ci/scripts/build-integration-tests.sh create mode 100755 ci/scripts/build-project.sh create mode 100755 ci/scripts/build-samples.sh create mode 100644 ci/scripts/common.sh create mode 100755 ci/scripts/promote.sh create mode 100755 ci/scripts/stage.sh create mode 100644 ci/tasks/build-deployment-tests.yml create mode 100644 ci/tasks/build-integration-tests.yml create mode 100644 ci/tasks/build-project.yml create mode 100644 ci/tasks/build-samples.yml create mode 100644 ci/tasks/promote.yml create mode 100644 ci/tasks/stage.yml diff --git a/ci/README.adoc b/ci/README.adoc new file mode 100644 index 0000000000..4b9dd163bf --- /dev/null +++ b/ci/README.adoc @@ -0,0 +1,49 @@ +== Concourse pipeline + +To set the pipeline first create a file in this directory called `secrets.yml`: + +[source,yaml] +.secrets.yml +---- +docker-hub-username: +docker-hub-password: +github-username: +github-password: +artifactory-username: +artifactory-password: +---- + +NOTE: The file should be ignored by git, make sure that you don't commit it! + +Once the file has been created, the pipeline can be deployed: + +[source] +---- +$ fly -t spring set-pipeline -p spring-boot -c ci/pipeline.yml -l ci/parameters.yml -l ci/secrets.yml +---- + +=== Release + +To release a milestone: + +[source] +---- +$ fly -t spring trigger-job -j spring-boot/stage-milestone +$ fly -t spring trigger-job -j spring-boot/promote-milestone +---- + +To release an RC: + +[source] +---- +$ fly -t spring trigger-job -j spring-boot/stage-rc +$ fly -t spring trigger-job -j spring-boot/promote-rc +---- + +To release a GA: + +[source] +---- +$ fly -t spring trigger-job -j spring-boot/stage-release +$ fly -t spring trigger-job -j spring-boot/promote-release +---- diff --git a/ci/images/spring-boot-ci-image/Dockerfile b/ci/images/spring-boot-ci-image/Dockerfile new file mode 100644 index 0000000000..da239ca060 --- /dev/null +++ b/ci/images/spring-boot-ci-image/Dockerfile @@ -0,0 +1,8 @@ +FROM openjdk:8u141-jdk + +RUN apt-get update && \ + apt-get install -y git && \ + apt-get install -y libxml2-utils && \ + apt-get install -y jq + +ADD https://raw.githubusercontent.com/spring-io/concourse-java-scripts/v0.0.1/concourse-java.sh /opt/ \ No newline at end of file diff --git a/ci/images/spring-boot-jdk9-ci-image/Dockerfile b/ci/images/spring-boot-jdk9-ci-image/Dockerfile new file mode 100644 index 0000000000..33bd2215f2 --- /dev/null +++ b/ci/images/spring-boot-jdk9-ci-image/Dockerfile @@ -0,0 +1,8 @@ +FROM openjdk:9-b179-jdk + +RUN apt-get update && \ + apt-get install -y git && \ + apt-get install -y libxml2-utils && \ + apt-get install -y jq + +ADD https://raw.githubusercontent.com/spring-io/concourse-java-scripts/v0.0.1/concourse-java.sh /opt/ \ No newline at end of file diff --git a/ci/parameters.yml b/ci/parameters.yml new file mode 100644 index 0000000000..bee807c5dd --- /dev/null +++ b/ci/parameters.yml @@ -0,0 +1,10 @@ +email-server: "smtp.svc.pivotal.io" +email-from: "ci@spring.io" +email-to: ["spring-boot-dev@pivotal.io"] +github-repo: "https://github.com/spring-projects/spring-boot.git" +docker-hub-organization: "springci" +artifactory-server: "https://repo.spring.io/" +branch: "master" +build-name: "spring-boot" +bintray-subject: "spring" +bintray-repo: "jars" \ No newline at end of file diff --git a/ci/pipeline.yml b/ci/pipeline.yml new file mode 100644 index 0000000000..b2cf489202 --- /dev/null +++ b/ci/pipeline.yml @@ -0,0 +1,259 @@ +resource_types: +- name: artifactory-resource + type: docker-image + source: + repository: springio/artifactory-resource + tag: 0.0.1 +- name: email + type: docker-image + source: + repository: pcfseceng/email-resource +resources: +- name: git-repo + type: git + source: + uri: ((github-repo)) + username: ((github-username)) + password: ((github-password)) + branch: ((branch)) + ignore_paths: ["ci/images/*"] +- name: ci-images-git-repo + type: git + source: + uri: ((github-repo)) + branch: ((branch)) + paths: ["ci/images/*"] +- name: spring-boot-ci-image + type: docker-image + source: + repository: ((docker-hub-organization))/spring-boot-ci-image + username: ((docker-hub-username)) + password: ((docker-hub-password)) + tag: ((branch)) +- name: spring-boot-jdk9-ci-image + type: docker-image + source: + repository: ((docker-hub-organization))/spring-boot-jdk9-ci-image + username: ((docker-hub-username)) + password: ((docker-hub-password)) + tag: ((branch)) +- name: artifactory-repo + type: artifactory-resource + source: + uri: ((artifactory-server)) + username: ((artifactory-username)) + password: ((artifactory-password)) + build_name: ((build-name)) +- name: email-notification + type: email + source: + smtp: + host: ((email-server)) + port: "25" + anonymous: true + skip_ssl_validation: true + from: ((email-from)) + to: ((email-to)) +jobs: +- name: build-spring-boot-ci-images + plan: + - get: ci-images-git-repo + trigger: true + - put: spring-boot-ci-image + params: + build: ci-images-git-repo/ci/images/spring-boot-ci-image + - put: spring-boot-jdk9-ci-image + params: + build: ci-images-git-repo/ci/images/spring-boot-jdk9-ci-image +- name: build + serial: true + plan: + - get: spring-boot-ci-image + - get: git-repo + trigger: true + - do: + - task: build-project + timeout: 1h30m + image: spring-boot-ci-image + file: git-repo/ci/tasks/build-project.yml + - aggregate: + - task: build-samples + timeout: 1h30m + image: spring-boot-ci-image + file: git-repo/ci/tasks/build-samples.yml + - task: build-integration-tests + timeout: 1h30m + image: spring-boot-ci-image + file: git-repo/ci/tasks/build-integration-tests.yml + - task: build-deployment-tests + timeout: 1h30m + image: spring-boot-ci-image + file: git-repo/ci/tasks/build-deployment-tests.yml + on_failure: + put: email-notification + params: + subject_text: "Build failure ${BUILD_PIPELINE_NAME} / ${BUILD_JOB_NAME} / ${BUILD_NAME}" + body_text: "Build ${ATC_EXTERNAL_URL}/teams/${BUILD_TEAM_NAME}/pipelines/${BUILD_PIPELINE_NAME}/jobs/${BUILD_JOB_NAME}/builds/${BUILD_NAME} has failed!" + - put: artifactory-repo + params: &artifactory-params + repo: libs-snapshot-local + build_number: "${BUILD_ID}" + folder: distribution-repository + build_uri: "https://ci.spring.io/teams/${BUILD_TEAM_NAME}/pipelines/${BUILD_PIPELINE_NAME}/jobs/${BUILD_JOB_NAME}/builds/${BUILD_NAME}" + build_number: "${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" + artifact_set: + - include: + - "/**/spring-boot-docs-*.zip" + properties: + zip-type: "docs" + zip-deployed: "false" +- name: jdk9-build + serial: true + plan: + - get: spring-boot-jdk9-ci-image + - get: git-repo + - do: + - task: build-project + timeout: 1h30m + image: spring-boot-jdk9-ci-image + file: git-repo/ci/tasks/build-project.yml + - aggregate: + - task: build-samples + timeout: 1h30m + image: spring-boot-jdk9-ci-image + file: git-repo/ci/tasks/build-samples.yml + - task: build-integration-tests + timeout: 1h30m + image: spring-boot-jdk9-ci-image + file: git-repo/ci/tasks/build-integration-tests.yml + - task: build-deployment-tests + timeout: 1h30m + image: spring-boot-jdk9-ci-image + file: git-repo/ci/tasks/build-deployment-tests.yml + on_failure: + put: email-notification + params: + subject_text: "JDK 9 Build failure ${BUILD_PIPELINE_NAME} / ${BUILD_JOB_NAME} / ${BUILD_NAME}" + body_text: "Build ${ATC_EXTERNAL_URL}/teams/${BUILD_TEAM_NAME}/pipelines/${BUILD_PIPELINE_NAME}/jobs/${BUILD_JOB_NAME}/builds/${BUILD_NAME} has failed!" +- name: stage-milestone + serial: true + plan: + - get: spring-boot-ci-image + - get: git-repo + trigger: false + - task: stage + image: spring-boot-ci-image + file: git-repo/ci/tasks/stage.yml + params: + RELEASE_TYPE: M + - put: artifactory-repo + params: + <<: *artifactory-params + repo: libs-staging-local + - put: git-repo + params: + repository: stage-git-repo +- name: stage-rc + serial: true + plan: + - get: spring-boot-ci-image + - get: git-repo + trigger: false + - task: stage + image: spring-boot-ci-image + file: git-repo/ci/tasks/stage.yml + params: + RELEASE_TYPE: RC + - put: artifactory-repo + params: + <<: *artifactory-params + repo: libs-staging-local + - put: git-repo + params: + repository: stage-git-repo +- name: stage-release + serial: true + plan: + - get: spring-boot-ci-image + - get: git-repo + trigger: false + - task: stage + image: spring-boot-ci-image + file: git-repo/ci/tasks/stage.yml + params: + RELEASE_TYPE: RELEASE + - put: artifactory-repo + params: + <<: *artifactory-params + repo: libs-staging-local + - put: git-repo + params: + repository: stage-git-repo +- name: promote-milestone + serial: true + plan: + - get: spring-boot-ci-image + - get: git-repo + trigger: false + - get: artifactory-repo + trigger: false + passed: [stage-milestone] + params: + save_build_info: true + - task: promote + image: spring-boot-ci-image + file: git-repo/ci/tasks/promote.yml + params: + RELEASE_TYPE: M + ARTIFACTORY_SERVER: ((artifactory-server)) + ARTIFACTORY_USERNAME: ((artifactory-username)) + ARTIFACTORY_PASSWORD: ((artifactory-password)) +- name: promote-rc + serial: true + plan: + - get: git-repo + trigger: false + - get: artifactory-repo + trigger: false + passed: [stage-rc] + params: + save_build_info: true + - task: promote + image: spring-boot-ci-image + file: git-repo/ci/tasks/promote.yml + params: + RELEASE_TYPE: RC + ARTIFACTORY_SERVER: ((artifactory-server)) + ARTIFACTORY_USERNAME: ((artifactory-username)) + ARTIFACTORY_PASSWORD: ((artifactory-password)) +- name: promote-release + serial: true + plan: + - get: git-repo + trigger: false + - get: artifactory-repo + trigger: false + passed: [stage-release] + params: + save_build_info: true + - task: promote + image: spring-boot-ci-image + file: git-repo/ci/tasks/promote.yml + params: + RELEASE_TYPE: RELEASE + ARTIFACTORY_SERVER: ((artifactory-server)) + ARTIFACTORY_USERNAME: ((artifactory-username)) + ARTIFACTORY_PASSWORD: ((artifactory-password)) + BINTRAY_USERNAME: ((bintray-username)) + BINTRAY_PASSWORD: ((bintray-password)) + SONATYPE_USERNAME: ((sonattype-username)) + SONATYPE_PASSWORD: ((sonattype-password)) + BINTRAY_SUBJECT: ((bintray-subject)) + BINTRAY_REPO: ((bintray-repo)) +groups: +- name: "Build" + jobs: ["build", "jdk9-build"] +- name: "Release" + jobs: ["stage-milestone", "stage-rc", "stage-release", "promote-milestone", "promote-rc", "promote-release"] +- name: "CI Images" + jobs: ["build-spring-boot-ci-images"] diff --git a/ci/scripts/build-deployment-tests.sh b/ci/scripts/build-deployment-tests.sh new file mode 100755 index 0000000000..c93fd09008 --- /dev/null +++ b/ci/scripts/build-deployment-tests.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +source $(dirname $0)/common.sh +repository=$(pwd)/distribution-repository + +pushd git-repo > /dev/null +run_maven -f spring-boot-tests/spring-boot-deployment-tests/pom.xml clean install -U -Dfull -Drepository=file://${repository} +popd > /dev/null diff --git a/ci/scripts/build-integration-tests.sh b/ci/scripts/build-integration-tests.sh new file mode 100755 index 0000000000..953eaebb1c --- /dev/null +++ b/ci/scripts/build-integration-tests.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +source $(dirname $0)/common.sh +repository=$(pwd)/distribution-repository + +pushd git-repo > /dev/null +run_maven -f spring-boot-tests/spring-boot-integration-tests/pom.xml clean install -U -Dfull -Drepository=file://${repository} +popd > /dev/null diff --git a/ci/scripts/build-project.sh b/ci/scripts/build-project.sh new file mode 100755 index 0000000000..08ee4e5c8d --- /dev/null +++ b/ci/scripts/build-project.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +source $(dirname $0)/common.sh +repository=$(pwd)/distribution-repository + +pushd git-repo > /dev/null +run_maven -f spring-boot-project/pom.xml clean deploy -U -Dfull -DaltDeploymentRepository=distribution::default::file://${repository} +popd > /dev/null diff --git a/ci/scripts/build-samples.sh b/ci/scripts/build-samples.sh new file mode 100755 index 0000000000..bf27164251 --- /dev/null +++ b/ci/scripts/build-samples.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +source $(dirname $0)/common.sh +repository=$(pwd)/distribution-repository + +pushd git-repo > /dev/null +run_maven -f spring-boot-samples/pom.xml clean install -U -Dfull -Drepository=file://${repository} +popd > /dev/null diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh new file mode 100644 index 0000000000..836e8923a1 --- /dev/null +++ b/ci/scripts/common.sh @@ -0,0 +1,4 @@ +source /opt/concourse-java.sh + +setup_symlinks +cleanup_maven_repo "org.springframework.boot" diff --git a/ci/scripts/promote.sh b/ci/scripts/promote.sh new file mode 100755 index 0000000000..95ed1e2a52 --- /dev/null +++ b/ci/scripts/promote.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -e + +source $(dirname $0)/common.sh + +buildName=$( cat artifactory-repo/build-info.json | jq -r '.buildInfo.name' ) +buildNumber=$( cat artifactory-repo/build-info.json | jq -r '.buildInfo.number' ) +groupId=$( cat artifactory-repo/build-info.json | jq -r '.buildInfo.modules[0].id' | sed 's/\(.*\):.*:.*/\1/' ) +version=$( cat artifactory-repo/build-info.json | jq -r '.buildInfo.modules[0].id' | sed 's/.*:.*:\(.*\)/\1/' ) + + +if [[ $RELEASE_TYPE = "M" ]]; then + targetRepo="libs-milestone-local" +elif [[ $RELEASE_TYPE = "RC" ]]; then + targetRepo="libs-milestone-local" +elif [[ $RELEASE_TYPE = "RELEASE" ]]; then + targetRepo="libs-release-local" +else + echo "Unknown release type $RELEASE_TYPE" >&2; exit 1; +fi + +echo "Promoting ${buildName}/${buildNumber} to ${targetRepo}" + +curl \ + -u ${ARTIFACTORY_USERNAME}:${ARTIFACTORY_PASSWORD} \ + -H"Content-type:application/json" \ + -d "{\"status\": \"staged\", \"sourceRepo\": \"libs-staging-local\", \"targetRepo\": \"\"}" \ + -f \ + -X \ + POST "${ARTIFACTORY_SERVER}/api/build/promote/${buildName}/${buildNumber}" > /dev/null || { echo "Failed to promote" >&2; exit 1; } + +elif [[ $RELEASE_TYPE = "RELEASE" ]]; then + curl \ + -u ${ARTIFACTORY_USERNAME}:${ARTIFACTORY_PASSWORD} \ + -H"Content-type:application/json" \ + -u ${ARTIFACTORY_USERNAME}:${ARTIFACTORY_PASSWORD} \ + -d "{\"sourceRepos\": [\"libs-release-local\"], \"targetRepo\" : \"spring-distributions\"}" + -f \ + -X \ + POST "${ARTIFACTORY_SERVER}/api/build/distribute/${buildName}/${buildNumber}" > /dev/null || { echo "Failed to publish" >&2; exit 1; } + + curl \ + -u ${BINTRAY_USERNAME}:${BINTRAY_PASSWORD} \ + -H "Content-Type: application/json" -d "{\"username\": \"${SONATYPE_USERNAME}\", \"password\": \"${SONATYPE_PASSWORD}\"}" + -f \ + -X \ + POST "https://api.bintray.com/maven_central_sync/${BINTRAY_SUBJECT}/${BINTRAY_REPO}/${groupId}/versions/${version}" > /dev/null || { echo "Failed to sync" >&2; exit 1; } +fi + + +echo "Promotion complete" diff --git a/ci/scripts/stage.sh b/ci/scripts/stage.sh new file mode 100755 index 0000000000..3d3973789f --- /dev/null +++ b/ci/scripts/stage.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -e + +source $(dirname $0)/common.sh +repository=$(pwd)/distribution-repository + +pushd git-repo > /dev/null +git fetch --tags --all > /dev/null +popd > /dev/null + +git clone git-repo stage-git-repo > /dev/null + +pushd stage-git-repo > /dev/null + +snapshotVersion=$( get_revision_from_pom ) +if [[ $RELEASE_TYPE = "M" ]]; then + stageVersion=$( get_next_milestone_release $snapshotVersion) + nextVersion=$snapshotVersion +elif [[ $RELEASE_TYPE = "RC" ]]; then + stage=$( get_next_rc_release $snapshotVersion) + nextVersion=$snapshotVersion +elif [[ $RELEASE_TYPE = "RELEASE" ]]; then + stage=$( strip_snapshot_suffix $snapshotVersion) + nextVersion=$( bump_version_number $snapshotVersion) +else + echo "Unknown release type $RELEASE_TYPE" >&2; exit 1; +fi + +echo "Staging $stageVersion (next version will be $nextVersion)" + +set_revision_to_pom "$stageVersion" +git config user.name "Spring Buildmaster" > /dev/null +git config user.email "buildmaster@springframework.org" > /dev/null +git add pom.xml > /dev/null +git commit -m"Release v$stageVersion" > /dev/null +git tag -a "v$stageVersion" -m"Release v$stageVersion" > /dev/null + +run_maven -f spring-boot-project/pom.xml clean deploy -U -Dfull -DaltDeploymentRepository=distribution::default::file://${repository} +run_maven -f spring-boot-tests/spring-boot-sample-tests/pom.xml clean install -U -Dfull -Drepository=file://${repository} +run_maven -f spring-boot-tests/spring-boot-integration-tests/pom.xml clean install -U -Dfull -Drepository=file://${repository} +run_maven -f spring-boot-tests/spring-boot-deployment-tests/pom.xml clean install -U -Dfull -Drepository=file://${repository} + +git reset --hard HEAD^ > /dev/null +if [[ $nextVersion != $snapshotVersion ]]; then + echo "Setting next development version (v$nextVersion)" + set_revision_to_pom "$nextVersion" + git add pom.xml > /dev/null + git commit -m"Next development version (v$nextVersion)" > /dev/null +fi; + +echo "DONE" + +popd > /dev/null diff --git a/ci/tasks/build-deployment-tests.yml b/ci/tasks/build-deployment-tests.yml new file mode 100644 index 0000000000..e45a4c2a30 --- /dev/null +++ b/ci/tasks/build-deployment-tests.yml @@ -0,0 +1,10 @@ +--- +platform: linux +inputs: +- name: git-repo +- name: distribution-repository +caches: +- path: maven +- path: gradle +run: + path: git-repo/ci/scripts/build-deployment-tests.sh diff --git a/ci/tasks/build-integration-tests.yml b/ci/tasks/build-integration-tests.yml new file mode 100644 index 0000000000..bc96aa6cef --- /dev/null +++ b/ci/tasks/build-integration-tests.yml @@ -0,0 +1,10 @@ +--- +platform: linux +inputs: +- name: git-repo +- name: distribution-repository +caches: +- path: maven +- path: gradle +run: + path: git-repo/ci/scripts/build-integration-tests.sh diff --git a/ci/tasks/build-project.yml b/ci/tasks/build-project.yml new file mode 100644 index 0000000000..b4576a4849 --- /dev/null +++ b/ci/tasks/build-project.yml @@ -0,0 +1,11 @@ +--- +platform: linux +inputs: +- name: git-repo +outputs: +- name: distribution-repository +caches: +- path: maven +- path: gradle +run: + path: git-repo/ci/scripts/build-project.sh diff --git a/ci/tasks/build-samples.yml b/ci/tasks/build-samples.yml new file mode 100644 index 0000000000..e045473650 --- /dev/null +++ b/ci/tasks/build-samples.yml @@ -0,0 +1,10 @@ +--- +platform: linux +inputs: +- name: git-repo +- name: distribution-repository +caches: +- path: maven +- path: gradle +run: + path: git-repo/ci/scripts/build-samples.sh diff --git a/ci/tasks/promote.yml b/ci/tasks/promote.yml new file mode 100644 index 0000000000..92b92c3446 --- /dev/null +++ b/ci/tasks/promote.yml @@ -0,0 +1,7 @@ +--- +platform: linux +inputs: +- name: git-repo +- name: artifactory-repo +run: + path: git-repo/ci/scripts/promote.sh diff --git a/ci/tasks/stage.yml b/ci/tasks/stage.yml new file mode 100644 index 0000000000..1cbcc1fa09 --- /dev/null +++ b/ci/tasks/stage.yml @@ -0,0 +1,12 @@ +--- +platform: linux +inputs: +- name: git-repo +outputs: +- name: stage-git-repo +- name: distribution-repository +caches: +- path: maven +- path: gradle +run: + path: git-repo/ci/scripts/stage.sh