From 4752d7e91a89def038fe1b86d47a77a67ac65683 Mon Sep 17 00:00:00 2001 From: dreis2211 Date: Sun, 15 Mar 2020 15:27:08 +0100 Subject: [PATCH 1/2] Automate Docker updates in CI See gh-20530 --- ci/images/get-docker-url.sh | 5 ++++ ci/images/setup.sh | 3 ++- ci/parameters.yml | 1 + ci/pipeline.yml | 41 ++++++++++++++++++++++++++++- ci/scripts/create-pull-request.sh | 11 ++++++++ ci/scripts/detect-docker-updates.sh | 37 ++++++++++++++++++++++++++ ci/tasks/create-pull-request.yml | 14 ++++++++++ ci/tasks/detect-docker-updates.yml | 14 ++++++++++ 8 files changed, 124 insertions(+), 2 deletions(-) create mode 100755 ci/images/get-docker-url.sh create mode 100755 ci/scripts/create-pull-request.sh create mode 100755 ci/scripts/detect-docker-updates.sh create mode 100644 ci/tasks/create-pull-request.yml create mode 100644 ci/tasks/detect-docker-updates.yml diff --git a/ci/images/get-docker-url.sh b/ci/images/get-docker-url.sh new file mode 100755 index 0000000000..f3540567a7 --- /dev/null +++ b/ci/images/get-docker-url.sh @@ -0,0 +1,5 @@ +#!/bin/bash +set -e + +version="19.03.7" +echo "https://download.docker.com/linux/static/stable/x86_64/docker-$version.tgz"; diff --git a/ci/images/setup.sh b/ci/images/setup.sh index bd91949680..b2309333d3 100755 --- a/ci/images/setup.sh +++ b/ci/images/setup.sh @@ -27,9 +27,10 @@ test -f /opt/openjdk/bin/javac ########################################################### # DOCKER ########################################################### +DOCKER_URL=$( ./get-docker-url.sh ) cd / -curl -L https://download.docker.com/linux/static/stable/x86_64/docker-19.03.7.tgz | tar zx +curl -L ${DOCKER_URL} | tar zx mv /docker/* /bin/ chmod +x /bin/docker* diff --git a/ci/parameters.yml b/ci/parameters.yml index bdde8a1bf6..e39ea64a72 100644 --- a/ci/parameters.yml +++ b/ci/parameters.yml @@ -1,3 +1,4 @@ +docker-upgrade-issue-title: "Upgrade Docker version in CI" email-server: "smtp.svc.pivotal.io" email-from: "ci@spring.io" email-to: ["spring-boot-dev@pivotal.io"] diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 27a5036974..aaeaf19d02 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -27,6 +27,14 @@ resources: username: ((github-username)) password: ((github-password)) branch: ((branch)) +- name: git-repo-ci-docker + type: git + icon: github-circle + source: + uri: ((github-repo)) + username: ((github-username)) + password: ((github-password)) + branch: ci-docker-((branch)) - name: git-repo-windows type: git source: @@ -178,6 +186,37 @@ jobs: GITHUB_PASSWORD: ((github-password)) GITHUB_USERNAME: ((github-username)) image: spring-boot-ci-image +- name: detect-docker-updates + plan: + - get: git-repo + resource: git-repo-ci-docker + - get: every-wednesday + trigger: true + - get: spring-boot-ci-image + - do: + - task: detect-docker-updates + file: git-repo/ci/tasks/detect-docker-updates.yml + params: + GITHUB_REPO: spring-boot + GITHUB_ORGANIZATION: spring-projects + GITHUB_PASSWORD: ((github-password)) + GITHUB_USERNAME: ((github-username)) + ISSUE_TITLE: ((docker-upgrade-issue-title)) + image: spring-boot-ci-image + - put: git-repo-ci-docker + params: + repository: docker-updates-git-repo + force: true + - task: create-pull-request + file: git-repo/ci/tasks/create-pull-request.yml + params: + BASE_BRANCH: ((branch)) + BRANCH: ci-docker-((branch)) + GITHUB_REPO: spring-boot + GITHUB_ORGANIZATION: spring-projects + GITHUB_PASSWORD: ((github-password)) + GITHUB_USERNAME: ((github-username)) + ISSUE_TITLE: ((docker-upgrade-issue-title)) - name: build serial: true public: true @@ -540,6 +579,6 @@ groups: - name: "Release" jobs: ["stage-milestone", "stage-rc", "stage-release", "promote-milestone", "promote-rc", "promote-release", "sync-to-maven-central"] - name: "CI Images" - jobs: ["build-spring-boot-ci-images", "detect-jdk-updates", "detect-ubuntu-image-updates"] + jobs: ["build-spring-boot-ci-images", "detect-docker-updates", "detect-jdk-updates", "detect-ubuntu-image-updates"] - name: "Build Pull Requests" jobs: ["build-pull-requests"] diff --git a/ci/scripts/create-pull-request.sh b/ci/scripts/create-pull-request.sh new file mode 100755 index 0000000000..ad02a4dbc6 --- /dev/null +++ b/ci/scripts/create-pull-request.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +curl \ + -s \ + -u ${GITHUB_USERNAME}:${GITHUB_PASSWORD} \ + -H "Content-type:application/json" \ + -d "{\"head\":\"${BRANCH}\",\"base\":\"${BASE_BRANCH}\",\"title\":\"${ISSUE_TITLE}\",\"body\":\"\",\"labels\":[\"status: waiting-for-triage\",\"type: task\"]}" \ + -f \ + -X \ + POST "https://api.github.com/repos/${GITHUB_ORGANIZATION}/${GITHUB_REPO}/pulls" > /dev/null || { echo "Failed to create pull request" >&2; exit 1; } diff --git a/ci/scripts/detect-docker-updates.sh b/ci/scripts/detect-docker-updates.sh new file mode 100755 index 0000000000..6e0758e0bf --- /dev/null +++ b/ci/scripts/detect-docker-updates.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +latest_version=$(curl -I -s https://github.com/docker/docker-ce/releases/latest | grep "location:" | awk '{n=split($0, parts, "/"); print substr(parts[n],2);}' | awk '{$1=$1;print}' | tr -d '\r' | tr -d '\n' ) + +if [[ $latest_version =~ (beta|rc) ]]; then + echo "Skip pre-release versions" + exit 0; +fi + +latest="https://download.docker.com/linux/static/stable/x86_64/docker-$latest_version.tgz" +current=$( git-repo/ci/images/get-docker-url.sh ) + +if [[ $current = $latest ]]; then + echo "Already up-to-date" + exit 0; +fi + +existing_tasks=$( curl -s https://api.github.com/repos/${GITHUB_ORGANIZATION}/${GITHUB_REPO}/pulls\?labels\=type:%20task\&state\=open\&creator\=spring-buildmaster ) +existing_upgrade_issues=$( echo "$existing_tasks" | jq -c --arg TITLE "$ISSUE_TITLE" '.[] | select(.title==$TITLE)' ) + +if [[ ${existing_upgrade_issues} = "" ]]; then + pushd git-repo > /dev/null + popd > /dev/null + git clone git-repo docker-updates-git-repo > /dev/null + pushd docker-updates-git-repo > /dev/null + # Create changes in dedicated branch + branch="ci-docker-$latest_version" + git config user.name "Spring Buildmaster" > /dev/null + git config user.email "buildmaster@springframework.org" > /dev/null + git checkout -b "$branch" origin/master > /dev/null + sed -i "s/version=.*/version=\"$latest_version\"/" ci/images/get-docker-url.sh + git add ci/images/get-docker-url.sh > /dev/null + commit_message="Upgrade to Docker $latest_version in CI" + git commit -m "$commit_message" > /dev/null +else + echo "Pull request already exists." +fi diff --git a/ci/tasks/create-pull-request.yml b/ci/tasks/create-pull-request.yml new file mode 100644 index 0000000000..0af302c4f1 --- /dev/null +++ b/ci/tasks/create-pull-request.yml @@ -0,0 +1,14 @@ +--- +platform: linux +inputs: + - name: git-repo +params: + BASE_BRANCH: + BRANCH: + GITHUB_REPO: + GITHUB_ORGANIZATION: + GITHUB_PASSWORD: + GITHUB_USERNAME: + ISSUE_TITLE: +run: + path: git-repo/ci/scripts/create-pull-request.sh diff --git a/ci/tasks/detect-docker-updates.yml b/ci/tasks/detect-docker-updates.yml new file mode 100644 index 0000000000..fd76ac4301 --- /dev/null +++ b/ci/tasks/detect-docker-updates.yml @@ -0,0 +1,14 @@ +--- +platform: linux +inputs: + - name: git-repo +outputs: + - name: docker-updates-git-repo +params: + GITHUB_REPO: + GITHUB_ORGANIZATION: + GITHUB_PASSWORD: + GITHUB_USERNAME: + ISSUE_TITLE: +run: + path: git-repo/ci/scripts/detect-docker-updates.sh From e174231d150524ac65794a85f4f969b1cd3a1305 Mon Sep 17 00:00:00 2001 From: Madhura Bhave Date: Mon, 30 Mar 2020 17:26:09 -0700 Subject: [PATCH 2/2] Polish "Automate Docker update in CI" See gh-20530 --- ci/pipeline.yml | 5 +---- ci/scripts/create-pull-request.sh | 22 ++++++++++++++-------- ci/scripts/detect-docker-updates.sh | 11 ++++------- ci/tasks/create-pull-request.yml | 1 + ci/tasks/detect-docker-updates.yml | 5 ++--- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index aaeaf19d02..aae5a0bebc 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -189,7 +189,6 @@ jobs: - name: detect-docker-updates plan: - get: git-repo - resource: git-repo-ci-docker - get: every-wednesday trigger: true - get: spring-boot-ci-image @@ -199,13 +198,11 @@ jobs: params: GITHUB_REPO: spring-boot GITHUB_ORGANIZATION: spring-projects - GITHUB_PASSWORD: ((github-password)) - GITHUB_USERNAME: ((github-username)) ISSUE_TITLE: ((docker-upgrade-issue-title)) image: spring-boot-ci-image - put: git-repo-ci-docker params: - repository: docker-updates-git-repo + repository: git-repo-updated force: true - task: create-pull-request file: git-repo/ci/tasks/create-pull-request.yml diff --git a/ci/scripts/create-pull-request.sh b/ci/scripts/create-pull-request.sh index ad02a4dbc6..3ea42d5870 100755 --- a/ci/scripts/create-pull-request.sh +++ b/ci/scripts/create-pull-request.sh @@ -1,11 +1,17 @@ #!/bin/bash set -e -curl \ - -s \ - -u ${GITHUB_USERNAME}:${GITHUB_PASSWORD} \ - -H "Content-type:application/json" \ - -d "{\"head\":\"${BRANCH}\",\"base\":\"${BASE_BRANCH}\",\"title\":\"${ISSUE_TITLE}\",\"body\":\"\",\"labels\":[\"status: waiting-for-triage\",\"type: task\"]}" \ - -f \ - -X \ - POST "https://api.github.com/repos/${GITHUB_ORGANIZATION}/${GITHUB_REPO}/pulls" > /dev/null || { echo "Failed to create pull request" >&2; exit 1; } +commit_message=$( cat commit-details/message ) + +if [[ ${commit_message} != "" ]]; then + curl \ + -s \ + -u ${GITHUB_USERNAME}:${GITHUB_PASSWORD} \ + -H "Content-type:application/json" \ + -d "{\"head\":\"${BRANCH}\",\"base\":\"${BASE_BRANCH}\",\"title\":\"${ISSUE_TITLE}\",\"body\":\"\",\"labels\":[\"status: waiting-for-triage\",\"type: task\"]}" \ + -f \ + -X \ + POST "https://api.github.com/repos/${GITHUB_ORGANIZATION}/${GITHUB_REPO}/pulls" > /dev/null || { echo "Failed to create pull request" >&2; exit 1; } +else + echo "Already up-to-date" +fi \ No newline at end of file diff --git a/ci/scripts/detect-docker-updates.sh b/ci/scripts/detect-docker-updates.sh index 6e0758e0bf..52f3d1d3cf 100755 --- a/ci/scripts/detect-docker-updates.sh +++ b/ci/scripts/detect-docker-updates.sh @@ -18,20 +18,17 @@ fi existing_tasks=$( curl -s https://api.github.com/repos/${GITHUB_ORGANIZATION}/${GITHUB_REPO}/pulls\?labels\=type:%20task\&state\=open\&creator\=spring-buildmaster ) existing_upgrade_issues=$( echo "$existing_tasks" | jq -c --arg TITLE "$ISSUE_TITLE" '.[] | select(.title==$TITLE)' ) +git clone git-repo git-repo-updated > /dev/null + if [[ ${existing_upgrade_issues} = "" ]]; then - pushd git-repo > /dev/null - popd > /dev/null - git clone git-repo docker-updates-git-repo > /dev/null - pushd docker-updates-git-repo > /dev/null - # Create changes in dedicated branch - branch="ci-docker-$latest_version" + pushd git-repo-updated > /dev/null git config user.name "Spring Buildmaster" > /dev/null git config user.email "buildmaster@springframework.org" > /dev/null - git checkout -b "$branch" origin/master > /dev/null sed -i "s/version=.*/version=\"$latest_version\"/" ci/images/get-docker-url.sh git add ci/images/get-docker-url.sh > /dev/null commit_message="Upgrade to Docker $latest_version in CI" git commit -m "$commit_message" > /dev/null + echo ${commit_message} > commit-details/message else echo "Pull request already exists." fi diff --git a/ci/tasks/create-pull-request.yml b/ci/tasks/create-pull-request.yml index 0af302c4f1..f726d32d0c 100644 --- a/ci/tasks/create-pull-request.yml +++ b/ci/tasks/create-pull-request.yml @@ -2,6 +2,7 @@ platform: linux inputs: - name: git-repo + - name: commit-details params: BASE_BRANCH: BRANCH: diff --git a/ci/tasks/detect-docker-updates.yml b/ci/tasks/detect-docker-updates.yml index fd76ac4301..bfced31c98 100644 --- a/ci/tasks/detect-docker-updates.yml +++ b/ci/tasks/detect-docker-updates.yml @@ -3,12 +3,11 @@ platform: linux inputs: - name: git-repo outputs: - - name: docker-updates-git-repo + - name: git-repo-updated + - name: commit-details params: GITHUB_REPO: GITHUB_ORGANIZATION: - GITHUB_PASSWORD: - GITHUB_USERNAME: ISSUE_TITLE: run: path: git-repo/ci/scripts/detect-docker-updates.sh