From ab416a19ff3b1547e0f13f430d1216213db1f816 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 15 Apr 2021 14:10:44 +0200 Subject: [PATCH 01/26] Bump dokka from 1.4.10.2 to 1.4.30 Remove workaround for Kotlin/dokka#1644 --- .ci/README.md | 8 ------- .ci/build-coveralls.sh | 5 ---- .ci/build-pr-win-macos.sh | 5 ---- .ci/build-sonar.sh | 5 ---- .ci/build.sh | 5 ---- .ci/inc/maven-dependencies.inc | 42 ---------------------------------- pom.xml | 2 +- 7 files changed, 1 insertion(+), 71 deletions(-) delete mode 100644 .ci/inc/maven-dependencies.inc diff --git a/.ci/README.md b/.ci/README.md index 7614027b70..c3711495ca 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -49,14 +49,6 @@ established. More information about configuring this can be found at [wagon-http](https://maven.apache.org/wagon/wagon-providers/wagon-http/). -However, this doesn't work when [dokka-maven-plugin](https://github.com/Kotlin/dokka) is used: This plugin -downloads additional dokka plugins at runtime and reconfigures somehow Maven. After this plugin is loaded, -the above system properties have no effect anymore. -See [dokka/dokka-maven-plugin#1625](https://github.com/Kotlin/dokka/issues/1625) and -[dokka/dokka-maven-plugin#1626](https://github.com/Kotlin/dokka/issues/1626). - -The workaround now in place is, to download all the dependencies first, see `inc/maven-dependencies.inc`. - ## Hints ### Remote debugging diff --git a/.ci/build-coveralls.sh b/.ci/build-coveralls.sh index 89a0743fb2..43031a03ec 100755 --- a/.ci/build-coveralls.sh +++ b/.ci/build-coveralls.sh @@ -3,7 +3,6 @@ source $(dirname $0)/inc/logger.inc source $(dirname $0)/inc/setup-secrets.inc source $(dirname $0)/inc/install-openjdk.inc -source $(dirname $0)/inc/maven-dependencies.inc set -e @@ -15,10 +14,6 @@ log_group_start "Setup private env and OpenJDK" export CI_BRANCH="${PMD_CI_GIT_REF##refs/heads/}" log_group_end -log_group_start "Downloading maven dependencies" - maven_dependencies_resolve -log_group_end - log_group_start "Executing build with coveralls" ./mvnw \ -Dmaven.javadoc.skip=true \ diff --git a/.ci/build-pr-win-macos.sh b/.ci/build-pr-win-macos.sh index 6fadee2b26..e88508510a 100755 --- a/.ci/build-pr-win-macos.sh +++ b/.ci/build-pr-win-macos.sh @@ -3,7 +3,6 @@ source $(dirname $0)/inc/logger.inc source $(dirname $0)/inc/install-openjdk.inc source $(dirname $0)/inc/regression-tester.inc -source $(dirname $0)/inc/maven-dependencies.inc set -e @@ -19,10 +18,6 @@ log_group_start "Installing Java" fi log_group_end -log_group_start "Downloading maven dependencies" - maven_dependencies_resolve -log_group_end - log_group_start "Building with maven" ./mvnw -e -V -B clean verify ${PMD_EXTRA_OPT} log_group_end diff --git a/.ci/build-sonar.sh b/.ci/build-sonar.sh index 461fd317a0..24b5b895da 100755 --- a/.ci/build-sonar.sh +++ b/.ci/build-sonar.sh @@ -3,7 +3,6 @@ source $(dirname $0)/inc/logger.inc source $(dirname $0)/inc/setup-secrets.inc source $(dirname $0)/inc/install-openjdk.inc -source $(dirname $0)/inc/maven-dependencies.inc set -e @@ -12,10 +11,6 @@ log_group_start "Setup private env and OpenJDK" install_openjdk_setdefault 11 log_group_end -log_group_start "Downloading maven dependencies" - maven_dependencies_resolve -log_group_end - log_group_start "Executing build with sonar" ./mvnw \ -Dmaven.javadoc.skip=true \ diff --git a/.ci/build.sh b/.ci/build.sh index 6dcecba98f..08ff81ddf1 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -7,7 +7,6 @@ source $(dirname $0)/inc/pmd-doc.inc source $(dirname $0)/inc/pmd-code-api.inc source $(dirname $0)/inc/regression-tester.inc source $(dirname $0)/inc/github-releases-api.inc -source $(dirname $0)/inc/maven-dependencies.inc source $(dirname $0)/inc/install-openjdk.inc set -e @@ -27,10 +26,6 @@ function pmd_ci_build_main() { pmd_ci_build_setup_env log_group_end - log_group_start "Downloading maven dependencies" - maven_dependencies_resolve - log_group_end - log_group_start "Build and Deploy" pmd_ci_build_run pmd_ci_deploy_build_artifacts diff --git a/.ci/inc/maven-dependencies.inc b/.ci/inc/maven-dependencies.inc deleted file mode 100644 index 7e6105ce88..0000000000 --- a/.ci/inc/maven-dependencies.inc +++ /dev/null @@ -1,42 +0,0 @@ - -# -# needs "inc/logger.inc" -# - -# -# On azure, outgoing idle connection are dropped after 4 minutes. -# Usually, you can configure wagon with ttl. But these settings are -# ignored, as soon as dokka-maven-plugin is loaded. -# dokka-maven-plugin tries to load additional dependencies at runtime -# and injects a different http client, which is not configured correctly -# and thus maven fails if it tries to download later in the build process -# further dependencies. -# -# The workaround applied here is: first resolve all dependencies, -# then explicitly get dokka-maven-plugin and then resolve all plugins -# execpt for dokka-maven-plugin, as it does not play well with dependency-plugin. -# -function maven_dependencies_resolve() { - dokka_version=$(./mvnw -q -Dexec.executable="echo" -Dexec.args='${dokka.version}' \ - --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) - - # build first the modules, that have dependencies between themselves - # first build pmd-lang-test, pmd-test and pmd-core - used by all modules - ./mvnw clean install -pl pmd-core,pmd-test,pmd-lang-test -DskipTests -Dpmd.skip=true \ - -B -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true - # then build dependencies for pmd-visualforce needs: pmd-apex->pmd-apex-jorje+pmd-test+pmd-core - ./mvnw clean install -pl pmd-core,pmd-test,pmd-lang-test,pmd-apex-jorje,pmd-apex -DskipTests -Dpmd.skip=true \ - -B -Dcheckstyle.skip=true -Dmaven.javadoc.skip=true -Dmaven.source.skip=true - - # the resolve most other projects. The excluded projects depend on other projects in the reactor, which is not - # completely built yet, so these are excluded. - ./mvnw dependency:resolve -pl '!pmd-dist,!pmd-java8,!pmd-doc,!pmd-scala' -Dsilent -B - - ./mvnw dependency:get -B -Dsilent \ - -DgroupId=org.jetbrains.dokka \ - -DartifactId=dokka-maven-plugin \ - -Dversion=${dokka_version} \ - -Dpackaging=jar \ - -DremoteRepositories=jcenter::default::https://jcenter.bintray.com/ - ./mvnw dependency:resolve-plugins -B -Dsilent -DexcludeGroupIds=org.jetbrains.dokka -Psign -} diff --git a/pom.xml b/pom.xml index 1110216a52..bbfe147f7c 100644 --- a/pom.xml +++ b/pom.xml @@ -87,7 +87,7 @@ ${maven.compiler.test.target} 1.4.10 4.3.1 - 1.4.10.2 + 1.4.30 5.0 From 10c6906f0fd15f996101b865510fb96ac027bcac Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 15 Apr 2021 15:26:33 +0200 Subject: [PATCH 02/26] [ci] First round of migration to new build scripts --- .ci/build.sh | 228 ++++++++---------- .ci/check-environment.sh | 60 ----- .ci/files/id_rsa.gpg | Bin 2557 -> 0 bytes .ci/files/id_rsa.pub | 1 - .ci/files/maven-settings.xml | 31 --- .ci/files/private-env.gpg | Bin 611 -> 0 bytes ...lease-signing-key-D0BF1D737C9A1C22.gpg.gpg | Bin 8226 -> 0 bytes .ci/inc/fetch_ci_scripts.bash | 19 ++ .ci/inc/github-releases-api.inc | 214 ---------------- .ci/inc/install-openjdk.inc | 103 -------- .ci/inc/logger.inc | 30 --- .ci/inc/pmd-code-api.inc | 44 ++-- .ci/inc/pmd-doc.inc | 56 +++-- .ci/inc/regression-tester.inc | 33 +-- .ci/inc/setup-secrets.inc | 62 ----- .ci/inc/sourceforge-api.inc | 166 ------------- .github/workflows/build.yml | 55 +++++ 17 files changed, 244 insertions(+), 858 deletions(-) delete mode 100755 .ci/check-environment.sh delete mode 100644 .ci/files/id_rsa.gpg delete mode 100644 .ci/files/id_rsa.pub delete mode 100644 .ci/files/maven-settings.xml delete mode 100644 .ci/files/private-env.gpg delete mode 100644 .ci/files/release-signing-key-D0BF1D737C9A1C22.gpg.gpg create mode 100644 .ci/inc/fetch_ci_scripts.bash delete mode 100644 .ci/inc/github-releases-api.inc delete mode 100644 .ci/inc/install-openjdk.inc delete mode 100644 .ci/inc/logger.inc delete mode 100644 .ci/inc/setup-secrets.inc delete mode 100644 .ci/inc/sourceforge-api.inc create mode 100644 .github/workflows/build.yml diff --git a/.ci/build.sh b/.ci/build.sh index 08ff81ddf1..fd073f04c3 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -1,66 +1,88 @@ #!/usr/bin/env bash -source $(dirname $0)/inc/logger.inc -source $(dirname $0)/inc/setup-secrets.inc -source $(dirname $0)/inc/sourceforge-api.inc -source $(dirname $0)/inc/pmd-doc.inc -source $(dirname $0)/inc/pmd-code-api.inc -source $(dirname $0)/inc/regression-tester.inc -source $(dirname $0)/inc/github-releases-api.inc -source $(dirname $0)/inc/install-openjdk.inc - +# Exit this script immediately if a command/function exits with a non-zero status. set -e -function pmd_ci_build_main() { - log_group_start "Setting up private secrets" - pmd_ci_setup_private_env - pmd_ci_setup_gpg_key - pmd_ci_setup_ssh - log_group_end +SCRIPT_INCLUDES="log.bash utils.bash setup-secrets.bash openjdk.bash maven.bash github-releases-api.bash + sourceforge-api.bash pmd-doc.inc pmd-code-api.inc regression-tester.inc" +# shellcheck source=inc/fetch_ci_scripts.bash +source "$(dirname "$0")/inc/fetch_ci_scripts.bash" && fetch_ci_scripts - log_group_start "Prepare Java 7+11, Maven, Bundler" - install_openjdk_setdefault 11 - install_oraclejdk7 - pmd_ci_build_setup_maven +function build() { + pmd_ci_log_group_start "Prepare Java 7+11, Bundler" + pmd_ci_openjdk_install_adoptopenjdk 11 + pmd_ci_openjdk_setdefault 11 + PMD_MAVEN_EXTRA_OPTS="" + if [ "$(pmd_ci_utils_get_os)" = "linux" ]; then + log_info "Install openjdk7 for integration tests" + pmd_ci_openjdk_install_zuluopenjdk 7 + PMD_MAVEN_EXTRA_OPTS="-Djava7.home=${HOME}/oraclejdk7" + fi pmd_ci_build_setup_bundler - pmd_ci_build_setup_env - log_group_end + pmd_ci_log_group_end - log_group_start "Build and Deploy" - pmd_ci_build_run - pmd_ci_deploy_build_artifacts - log_group_end + echo + pmd_ci_maven_display_info_banner + pmd_ci_utils_determine_build_env pmd/pmd + echo - log_group_start "Build and Upload documentation" - pmd_ci_build_and_upload_doc - log_group_end + if pmd_ci_utils_is_fork_or_pull_request; then + pmd_ci_log_group_start "Build with mvnw" + ./mvnw clean verify --show-version --errors --batch-mode --no-transfer-progress ${PMD_MAVEN_EXTRA_OPTS} + pmd_ci_log_group_end - if pmd_ci_build_isRelease; then - log_group_start "Publishing Release" - gh_release_publishRelease "$GH_RELEASE" - sourceforge_selectDefault "${VERSION}" - log_group_end + # Danger is executed only on the linux runner + if [ "$(pmd_ci_utils_get_os)" = "linux" ]; then + pmd_ci_log_group_start "Executing danger" + regression_tester_setup_ci + regression_tester_executeDanger + pmd_ci_log_group_end + fi + + exit 0 fi - log_group_start "Creating new baseline for regression tester" + + if [ "$(pmd_ci_utils_get_os)" != "linux" ]; then + pmd_ci_log_group_start "Build with mvnw" + ./mvnw clean verify --show-version --errors --batch-mode --no-transfer-progress ${PMD_MAVEN_EXTRA_OPTS} + pmd_ci_log_group_end + + pmd_ci_log_info "Stopping build here, because os is not linux" + exit 0 + fi + + # only builds on pmd/pmd continue here + pmd_ci_log_group_start "Setup environment" + pmd_ci_setup_secrets_private_env + pmd_ci_setup_secrets_gpg_key + pmd_ci_setup_secrets_ssh + pmd_ci_maven_setup_settings + pmd_ci_log_group_end + + pmd_ci_log_group_start "Build and Deploy" + pmd_ci_build_run + pmd_ci_deploy_build_artifacts + pmd_ci_log_group_end + + pmd_ci_log_group_start "Build and Upload documentation" + pmd_ci_build_and_upload_doc + pmd_ci_log_group_end + + if pmd_ci_maven_isReleaseBuild; then + pmd_ci_log_group_start "Publishing Release" + pmd_ci_gh_releases_publishRelease "$GH_RELEASE" + pmd_ci_sourceforge_selectDefault "${PMD_CI_MAVEN_PROJECT_VERSION}" + pmd_ci_log_group_end + fi + + pmd_ci_log_group_start "Creating new baseline for regression tester" regression_tester_setup_ci regression_tester_uploadBaseline - log_group_end - - exit 0 + pmd_ci_log_group_end } -# -# Configures maven. -# Needed for deploy to central (both snapshots and releases) -# and for signing the artifacts. -# -function pmd_ci_build_setup_maven() { - mkdir -p ${HOME}/.m2 - cp .ci/files/maven-settings.xml ${HOME}/.m2/settings.xml -} - # # Installs bundler, which is needed for doc generation and regression tester # @@ -69,54 +91,22 @@ function pmd_ci_build_setup_bundler() { gem install bundler } -# -# Setups common build parameters: -# * Determines the VERSION of PMD, that is being built -# * Determines the PMD_CI_BRANCH or PMD_CI_TAG, that is being built -# -function pmd_ci_build_setup_env() { - VERSION=$(pmd_ci_build_get_pom_version) - - if [[ "${PMD_CI_GIT_REF}" == refs/heads/* ]]; then - PMD_CI_BRANCH=${PMD_CI_GIT_REF##refs/heads/} - unset PMD_CI_TAG - log_info "Building PMD ${VERSION} on branch ${PMD_CI_BRANCH}" - elif [[ "${PMD_CI_GIT_REF}" == refs/tags/* ]]; then - unset PMD_CI_BRANCH - PMD_CI_TAG=${PMD_CI_GIT_REF##refs/tags/} - log_info "Building PMD ${VERSION} on tag ${PMD_CI_TAG}" - else - log_error "Unknown branch/tag: PMD_CI_GIT_REF=${PMD_CI_GIT_REF}" - exit 1 - fi - - if [[ "${VERSION}" == *-SNAPSHOT && -z "$PMD_CI_BRANCH" ]]; then - log_error "Invalid combination: snapshot version ${VERSION} but no branch in PMD_CI_GIT_REF=${PMD_CI_GIT_REF}" - exit 1 - fi - - if [[ "${VERSION}" != *-SNAPSHOT && -z "$PMD_CI_TAG" ]]; then - log_error "Invalid combination: non-snapshot version ${VERSION} but no tag in PMD_CI_GIT_REF=${PMD_CI_GIT_REF}" - exit 1 - fi -} - # # Performs the actual build. # Deploys the artifacts to maven central. # Also generates rule documentation. # function pmd_ci_build_run() { - local mvn_profiles="ossrh,sign,generate-rule-docs" + local mvn_profiles="sign,generate-rule-docs" - if pmd_ci_build_isRelease; then - log_info "This is a release build" + if pmd_ci_maven_isReleaseBuild; then + pmd_ci_log_info "This is a release build" mvn_profiles="${mvn_profiles},pmd-release" else - log_info "This is a snapshot build" + pmd_ci_log_info "This is a snapshot build" fi - ./mvnw clean deploy -P${mvn_profiles} -e -B -V -Djava7.home=${HOME}/oraclejdk7 + ./mvnw clean deploy -P${mvn_profiles} --show-version --errors --batch-mode --no-transfer-progress ${PMD_MAVEN_EXTRA_OPTS} } # @@ -124,17 +114,17 @@ function pmd_ci_build_run() { # function pmd_ci_deploy_build_artifacts() { # Deploy to sourceforge files - sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip" - sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip" + pmd_ci_sourceforge_uploadFile "${PMD_CI_MAVEN_PROJECT_VERSION}" "pmd-dist/target/pmd-bin-${PMD_CI_MAVEN_PROJECT_VERSION}.zip" + pmd_ci_sourceforge_uploadFile "${PMD_CI_MAVEN_PROJECT_VERSION}" "pmd-dist/target/pmd-src-${PMD_CI_MAVEN_PROJECT_VERSION}.zip" - if pmd_ci_build_isRelease; then + if pmd_ci_maven_isReleaseBuild; then # create a draft github release - gh_releases_createDraftRelease "${PMD_CI_TAG}" "$(git rev-list -n 1 ${PMD_CI_TAG})" + pmd_ci_gh_releases_createDraftRelease "${PMD_CI_TAG}" "$(git rev-list -n 1 ${PMD_CI_TAG})" GH_RELEASE="$RESULT" # Deploy to github releases - gh_release_uploadAsset "$GH_RELEASE" "pmd-dist/target/pmd-bin-${VERSION}.zip" - gh_release_uploadAsset "$GH_RELEASE" "pmd-dist/target/pmd-src-${VERSION}.zip" + pmd_ci_gh_releases_uploadAsset "$GH_RELEASE" "pmd-dist/target/pmd-bin-${PMD_CI_MAVEN_PROJECT_VERSION}.zip" + pmd_ci_gh_releases_uploadAsset "$GH_RELEASE" "pmd-dist/target/pmd-src-${PMD_CI_MAVEN_PROJECT_VERSION}.zip" fi } @@ -145,35 +135,35 @@ function pmd_ci_build_and_upload_doc() { pmd_doc_generate_jekyll_site pmd_doc_create_archive - sourceforge_uploadFile "${VERSION}" "docs/pmd-doc-${VERSION}.zip" - if pmd_ci_build_isRelease; then - gh_release_uploadAsset "$GH_RELEASE" "docs/pmd-doc-${VERSION}.zip" + pmd_ci_sourceforge_uploadFile "${PMD_CI_MAVEN_PROJECT_VERSION}" "docs/pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}.zip" + if pmd_ci_maven_isReleaseBuild; then + pmd_ci_gh_releases_uploadAsset "$GH_RELEASE" "docs/pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}.zip" fi - # Deploy doc to https://docs.pmd-code.org/pmd-doc-${VERSION}/ - pmd_code_uploadDocumentation "${VERSION}" "docs/pmd-doc-${VERSION}.zip" - # Deploy javadoc to https://docs.pmd-code.org/apidocs/*/${VERSION}/ - pmd_code_uploadJavadoc "${VERSION}" "$(pwd)" + # Deploy doc to https://docs.pmd-code.org/pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}/ + pmd_code_uploadDocumentation "${PMD_CI_MAVEN_PROJECT_VERSION}" "docs/pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}.zip" + # Deploy javadoc to https://docs.pmd-code.org/apidocs/*/${PMD_CI_MAVEN_PROJECT_VERSION}/ + pmd_code_uploadJavadoc "${PMD_CI_MAVEN_PROJECT_VERSION}" "$(pwd)" - if [[ "${VERSION}" == *-SNAPSHOT && "${PMD_CI_BRANCH}" == "master" ]]; then + if [ pmd_ci_maven_isSnapshotBuild && "${PMD_CI_BRANCH}" = "master" ]; then # only for snapshot builds from branch master - pmd_code_createSymlink "${VERSION}" "snapshot" + pmd_code_createSymlink "${PMD_CI_MAVEN_PROJECT_VERSION}" "snapshot" # update github pages https://pmd.github.io/pmd/ pmd_doc_publish_to_github_pages # rsync site to https://pmd.sourceforge.io/snapshot - sourceforge_rsyncSnapshotDocumentation "${VERSION}" "snapshot" + pmd_ci_sourceforge_rsyncSnapshotDocumentation "${PMD_CI_MAVEN_PROJECT_VERSION}" "snapshot" fi - if pmd_ci_build_isRelease; then - # documentation is already uploaded to https://docs.pmd-code.org/pmd-doc-${VERSION} + if pmd_ci_maven_isReleaseBuild; then + # documentation is already uploaded to https://docs.pmd-code.org/pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION} # we only need to setup symlinks for the released version - pmd_code_createSymlink "${VERSION}" "latest" + pmd_code_createSymlink "${PMD_CI_MAVEN_PROJECT_VERSION}" "latest" # remove old doc and point to the new version - pmd_code_removeDocumentation "${VERSION}-SNAPSHOT" - pmd_code_createSymlink "${VERSION}" "${VERSION}-SNAPSHOT" + pmd_code_removeDocumentation "${PMD_CI_MAVEN_PROJECT_VERSION}-SNAPSHOT" + pmd_code_createSymlink "${PMD_CI_MAVEN_PROJECT_VERSION}" "${PMD_CI_MAVEN_PROJECT_VERSION}-SNAPSHOT" # remove old javadoc - pmd_code_removeJavadoc "${VERSION}-SNAPSHOT" + pmd_code_removeJavadoc "${PMD_CI_MAVEN_PROJECT_VERSION}-SNAPSHOT" # updating github release text rm -f .bundle/config @@ -182,28 +172,16 @@ function pmd_ci_build_and_upload_doc() { bundle install # renders, and skips the first 6 lines - the Jekyll front-matter local rendered_release_notes=$(bundle exec .ci/render_release_notes.rb docs/pages/release_notes.md | tail -n +6) - local release_name="PMD ${VERSION} ($(date -u +%d-%B-%Y))" - gh_release_updateRelease "$GH_RELEASE" "$release_name" "$rendered_release_notes" - sourceforge_uploadReleaseNotes "${VERSION}" "${rendered_release_notes}" + local release_name="PMD ${PMD_CI_MAVEN_PROJECT_VERSION} ($(date -u +%d-%B-%Y))" + pmd_ci_gh_releases_updateRelease "$GH_RELEASE" "$release_name" "$rendered_release_notes" + pmd_ci_sourceforge_uploadReleaseNotes "${PMD_CI_MAVEN_PROJECT_VERSION}" "${rendered_release_notes}" - # updates https://pmd.github.io/latest/ and https://pmd.github.io/pmd-${VERSION} + # updates https://pmd.github.io/latest/ and https://pmd.github.io/pmd-${PMD_CI_MAVEN_PROJECT_VERSION} publish_release_documentation_github - sourceforge_rsyncSnapshotDocumentation "${VERSION}" "pmd-${VERSION}" + pmd_ci_sourceforge_rsyncSnapshotDocumentation "${PMD_CI_MAVEN_PROJECT_VERSION}" "pmd-${PMD_CI_MAVEN_PROJECT_VERSION}" fi } +build -function pmd_ci_build_isRelease() { - if [[ "${VERSION}" != *-SNAPSHOT && -n "${PMD_CI_TAG}" && -z "${PMD_CI_BRANCH}" ]]; then - return 0 - else - return 1 - fi -} - -function pmd_ci_build_get_pom_version() { - echo $(./mvnw -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) -} - - -pmd_ci_build_main +exit 0 diff --git a/.ci/check-environment.sh b/.ci/check-environment.sh deleted file mode 100755 index d164569a19..0000000000 --- a/.ci/check-environment.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -# -# This script should check, that all needed commands are available -# and are in the correct version. -# - -source $(dirname $0)/inc/logger.inc - -set -e - -function check() { - local CMD=$1 - local VERSION_CMD=$2 - local VERSION_STRING=$3 - - echo -n "Checking ${CMD}..." - - if hash "$CMD" 2>/dev/null; then - local VERSION_FULL=$(${VERSION_CMD} 2>&1) - local VERSION=$(echo "${VERSION_FULL}" | grep "${VERSION_STRING}" 2>&1) - if [ -n "${VERSION}" ]; then - echo -e "${COL_GREEN}OK${COL_RESET}" - echo " ${VERSION}" - else - echo -e "${COL_RED}wrong version${COL_RESET}. Expected: ${VERSION_STRING}" - echo " ${VERSION_FULL}" - fi - else - echo -e "${COL_RED}not found!${COL_RESET}" - fi -} - -# every OS: -check "curl" "curl --version" "curl" -check "jq" "jq --version" "jq" - -case "$(uname)" in - Linux*) - check "ruby" "ruby --version" "ruby 2.7" - check "gpg" "gpg --version" "gpg (GnuPG) 2." - check "printenv" "printenv --version" "printenv (GNU coreutils)" - check "rsync" "rsync --version" "version" - check "ssh" "ssh -V" "OpenSSH" - check "git" "git --version" "git version" - check "mvn" "mvn --version" "Apache Maven" - check "unzip" "unzip --version" "UnZip" - check "zip" "zip --version" "This is Zip" - #check "7z" "7z -version" "7-Zip" - ;; - Darwin*) - ;; - CYGWIN*|MINGW*) - check "7z" "7z -version" "7-Zip" - ;; - *) - log_error "Unknown OS: $(uname)" - exit 1 - ;; -esac diff --git a/.ci/files/id_rsa.gpg b/.ci/files/id_rsa.gpg deleted file mode 100644 index 8e68ae0f100435fca4d9c8066e5dbd2117687a73..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2557 zcmV!5$``#6|RLZhUVdBKN+V=w;z99mr9QTC3Z zAe@zNZ~6_Iv1b1cQpVX-o>T!pH@*jfk$6tXh9Wh*$=k5_`Fl|GDKoavxUlA!Ep07e zbb9d9VnagZPnaUm4;v})+Z)46v3E*u(`&M;vpOLWnYt&%)2H5TCj8XzsRN0D^IMAw=(g%pB^cO7PUO%`!e!52(8Nd? zVuj~0yrP1dtqtl9hyRoeu7%6NBUq|=nZqQ-^~|h9u`@~xSU&Ujj&)A_wK{uA4lt-x z*Ljfk9r8IhuM=Z4FI!%LW3g>6e!eysMg^$5=1777L2OlUFJk`RfTZZy8@epgoB6qr4 zg6zy~EgI7tW@Of%RRQGC4XWOiq$gsU9t!~$(^yY!HE5a;?OPhNy%9h~wOFA4!Gh29 z7G)0(Oj;sOh`QS~7UU=0=ub*btn(Fw6UHM^F!+iRgHT9iniv7S@Q?`JJ?frq2h zrxl>dbgVs2U;x*1trEKjhRR$4H$K~689Y4^?Bkdewfd17j8>d=2Jc9ye{-WBsPXpr zV>3_FpkF?H#rXjDT^KrRlBjqD=j?%87OA%z=T{BuWgOSspCqEQwb|@$fH3gd>vPA} z5G=?$0Dz@^OJ}u6@5zaH;nNRWZ%zy88w`)5a0DN`l8HUNU8b{6(>BTJku~T`3AZKM zeliav!l&y`PI!4`>YZic^zDGFq?_d8=#t@@Xx^!|;L{>n^Rj?Yet$Rq?bSr70#_hn zhheYlKFfj{kr><`*e0ulkSK#rRkp=!SlU{nG5Z~+*ft%H=uj)`8#tFfFnP?|KaGjn zfS2~wZfwWFRePmIBF8`tY2@H?q99{ScKV(ueyN9QP1wSQa>co`B0BLKZ~XUil>||m zm+7T@z<0B;avhxJMjcgHLQXQ*czHn*R2BfRN(;WHyZ!XmG_CJdo@}$iV5*lK9BA20 ziLEUTjklY+UNui}ojS1JnY>)FPa@&{E#t316u#mFTy6MZ6a@dTYl#VVien4atgXRCAG){bB3Fh2w?T(wGN zLTDl6oM?R%3?@VH`>pHmoo&dSj^~i*bU$reMC-s z<>o|GB~v>O;}Pd=EM*8DqiJx0KM_Wq2o%9S`g*<-QH}KQ9A~;GyJc2*H)n+y@Jrpo z3E&a*dAlgjaWX7GS&Vdid$-Cce}zMK^~_tuzki+x;$cZyC16Sc`ArfZIWztzhyt>( z($=aY)Z`BwFHp(#?t6s>!x-wS&CpBgCgoxyC1U)mosnZ)>Jn&wDmLnO#P>g>1$R|> zmU?jY>kefOXlJIXWTy?!X5o-WYxEXQAa2=~(H9ko92s6S>p?vS#+OiiW-& z;{lQ4e_IUbR{yyn@sKXLqq-q&Pr}+_T);i#ZC6FRQmMt)$%TM+E;w4DQ64m2+qFP? zBxvMdbc%;d$m0rZ{gCOp_NSPZ{*F?)qc7@D@~$p(61hpEMaL!zOO$ z#nbqs+K8lQ`U(s`FjH%Q+r;z6`0pVbhf+m*oz6HTV4h8&D(c^dw!MNF)djhghFm3n zjm_==z4T5vWr8ItC0zai%4R^|iY$qpbtL9(PVlI#3#P_iDdSZgt4!;hOgLFwObztq zvT9Pn#DoDl)XcM}Voo$3H)t*aVIsHVR;b&%E~AOI4yXU@9D~5dGkKY|qe+l{+1*4pZ94ub;)NC}d9;>@qDJ#R?)dF+T;-qeWdAc?t1qm^k%ZW2DWpS|>o zluFTk1+G2jz2o$v_T?R*eOu?Ers)?3M#qh^vR=8L373S~i<03u+#DCo&`F3=o^gT{N49K|#e$)HI9(>woA zm3Lc8cHs$&YuBh$B|3C(YgLG*3d~X~G}tHOv9r`L63tE~^N! zjXhBCW}kE0AE)q*J0L1tIYMph#ABztuX}J$D0sQ65m|fWOMp)Onq*9N*`jpi}kciBI%~TaH{2H6EV^!t>ISI%8o$t?8>dk)e{BQ!-@l2;;{{oxQHU Ttgvvhf~HU5@82)qpB@Z;=9uuZ diff --git a/.ci/files/id_rsa.pub b/.ci/files/id_rsa.pub deleted file mode 100644 index 7e1b51e198..0000000000 --- a/.ci/files/id_rsa.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC8yHVLHYDsKui8tYg/sFvkDqcs97pEZz0BzK9HtBF4O+/It1drRYRdUAFfjoImfprpKSxkJCTglHixGRp24eNaZ6woWVJ4/bmiMkEqEZAjr1NZ3qw7zIruMJMSkCV+YTtmL4cYcZlvMRPzzOZOnFbV05oi79oy41MUFHYjolK9QxMFNsVNN5iyzFxM3HqSFozz+ylKbFBtDk6ZHZQNRL/Xl2V9DJ69fVzjG4OZfcWNGmmKHHARmsnJyUOMeeKpLjDOe1M6ZdI8HkXWac8yCr9JTETNZZwemZAcS/RKoKCDqfIUOzkZfIPmyaznfVetTGsMi7yQrJhAyjznuNGF4+3lfgTcmRF8wz5FCeUkdYTmy2wNSFi5HiLPfC5OgRtjKzC6yb8rbRjDx6XQ2ph15PKOaXwzk49TaMc0xJvoiGDMZaTU0iTm3Y1/QUtfLvo3/jGMbtUdV3soWpuBAV2JUI4aB5xdLX9iNmcrVzoUe3y9DWuuTX46eoCvpUNXv/DXKhQw1D7xd7J67db5qUck/Akiqi0JR+e0SoBJvZFtYwVNLGC2bIJ/s8SR8X5Zp+1+ypf3WYjIylxQTkO1r4NfI0Cd9qXg7nmUrHAU7Z6xtJmUK8ZWzSST4wul8WkRJURtODLxt5firtlKhyZ93t9Mjuk6mATIPxr/b3x20T+IH463kw== ssh key for pmd. used for travis accessing sourceforge and github. diff --git a/.ci/files/maven-settings.xml b/.ci/files/maven-settings.xml deleted file mode 100644 index a8d61b8f1a..0000000000 --- a/.ci/files/maven-settings.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - org.sonarsource.scanner.maven - - - - ossrh - ${env.CI_DEPLOY_USER} - ${env.CI_DEPLOY_PASSWORD} - - - - - - - ossrh - - ${env.CI_SIGN_KEY} - ${env.CI_SIGN_PASSPHRASE} - - - - - diff --git a/.ci/files/private-env.gpg b/.ci/files/private-env.gpg deleted file mode 100644 index af292ee8b509f5ac8706b4c430886d4bf45507f3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 611 zcmV-p0-XJf4Fm}T0%ntHiUuqoGyl@*0Y|e$-<{CT>g4Z@)yY&AH!x4vsIy9TIu$+|)SqI&@_<4&5V*PO{5l?sq# zYawHZwi}Vc)ubf2zv~rN>^Pn8{Rbqo$_DA6Qdz4 zy#IFXfgOOBku`edUjWI|=}j7V7_1aPD=3qDqLFzIXAG2^0!xh0JFjdMHFu%IrM%-P zZy~Z#fuH*#Q&?H8F*5GeAv?@lIB}*uLfql4gVWzGOb29EHzOOg54s$_u3Y5d3}9&m6IiyrQ%@T~;ZR|?3b zq0UP{rRBIhS*?=v;!Bp95gWLDbke@BZj&_%2PkQ1ooWJZd5y}Jc#@oR?le|8gXQcS z$7jW>1i^}<%s#K8OB7I_ow!FR#K4}ZbHbbXG1wG1s)9ZtbIcFPA)o=?ZroS^-c%{t zD;9U&zx*tAdQ)b(vM=)dPiQ$Lw+-CfH*yNQ#NZmm7J7nz6%ccATq1fQdv_(zmznC< z$8#u>Tf;6zU4I*g>B_>1u}ZhQ7@C&p?3jGbM~wX~b=mo|w@?@hro@KEtXKt#`BS&; zaX2-v{-KI{T;r1GD0k((l>>ft`3Mvq44=;7)w$FHUI?unpE3aGSp9p<{n|$pumhBza>#;^G&s~u*`hFL0#De6Wetn+D;(CoAXSkNx z@oW9`ad(oSQ+_ebZ(Urs(*eHVz*hjb%RMbYzAP2SB-qs{p~fUcxjj2ur8Kt#1LbgW zsd=B5qDcG6?+B-1W%?}Iv}{?Fqa4_vrcd-as_$84kEDmru?hP~_VbcHWLVy`yx}&C zxVfdv&dAzT2U%kbI)vs5b2~kHdY?3ao^Se`MQhCsTmovB5-q2k>wS;?hcA zN)tM4i5M258d1PPkf{UFT%Ic0m?HCS$BiWqcCA$JF#L>se=vMxmtcj~$j$2mfU7+D zhX=@fsUFn0N#?DyNyp$Zd{tuLx;zJYA>n@KL1Q2fV~P*5LuuUK(VUN4t?L>doTgv% ztjkssq>D65$=0{+$2#h~80mH6Q5!>0otkOp!(1AF1u7}0i~%z>)^G>q1s{b!L{cw* zm=!`X?%5c%WgWavfN@)~Cvce-TyN_$vmeaOoXhb5{x|fv57{tNt_^10Lpi@cVuiQ9 z>BrSiJD}$4UVeoX+2fMxJr;XllCST;RMor9Pp1fD^5MH{%;O!fF38Vfw$UB@=#FJG`yR`k-b@AOrR zQM*4?;}7?Ti>i=w*qCwW#_KN4VF+TbtcG76ki-158y>9NFcNlRcu*0?tSne##Q{v} zv^*zxgnp41c-(|0@E>fu^eR+VL_(w0FI{OvbICm6fbEQ0%Df=3RR8oDa)O|TeJZw< zl`@Tq^SjGlc7%-G;5C!k?h2Qo8TvT|2C(muJTt`L zIz$~1S}f?HqS5oGwa-p76jBdCvL-tsaJs3vLXe*RFF52}1rq4^G4Ec4za$dRU5dX( zHbJbwdOcU%-%^P{z|!!GhY*n;lwBD1yqh=Hsh3h;{^@J0P`-{!4*)Bn8H&rPo+cf0 z3ET(`{H;Op8HaFkPHV0hckQpX{ONikjQSl@L_o_ZD@3Rb`|NNX+@rM6ZANI)k})DB zC&p*8Ytta27)v@%*T@BkJWO2b)FiTkX%g33-R1j?8lK8uIDKC`XEdH*{o^O!9X#D2 z)!yf$L6lw)y2LB;;cc!SK-ojA`Ad~)988^2)!p>em`vU<)`6TS>GKHa(Ex)pxZ=Ta zgp@E!u3TxJT=#o=W9AeeEG0nPdPpmwG4GTnuIbA};_aj=@LM zd7SOSG)gbWMKRq;l%E&y#ROJvi|mwG$UfH?lEC+PbBb+i;E;oN!Brn5Tdb>N;LX$E zW%7O6E~#xJz0lL5g&U1ZmEGoEGk6+1=E>On>r+YuFLy92Bn?TmxuyA-CFxjR`&RA``eriESPctLnURkf zdR~h2*P=T!y^W;5EE61tZ8m@=c!FcYO$RayspPmXi5e&Ds}a91yR(DXfwkc4B9NJQ z(H|?^mZF%|<-Ohiu>Jkkrf7^GIjD4miB~y!OcM5+N}TOG_717#s8|?+pD5ld%+d3- zcPQRDH5M3K6y6kZqwKmD(}y=KDFn=)8F07AK|e)mFFnbcLdlyyGS$4iDb%dk2 zFFxc)%Rk}uw?1{DZq>N41!#NDCjds=FvRHObeBTHqfGwPo_tPBG@?CuJMmX%O`bvK z8OXxaM@jWo4ZJ<7faY+SfGP1-%SJZNg!H_3@p2Z6H4gt9&R3u#4=eCF93YTrtVswa zm}F_rj(P-kD*eOli%xRuOUfbR*dMlrYpsQ z6oLZGbp!Pp>go+4RpE~oMiVp5AT(d5nk)8nAtDEG>`{U&GMwWfuIzJErTSBKv0Jx` zayLwG`;ZhEbt^bV(q}^aN6BBzo}dlHZ_Kk^9r$X2|5wNsP>Qdq^I7 zbZ5)qwJ+vMX&5nA9j4ecru*QB7G2Oc?MO1kHl-9+R|BEh(T*gv8mLN{Z zzrTA6%DjGxoFiIB4Ta!wz=WG$f5BI!7OW4AV!3ez;7-Xl8+qE)EmV1<%TKqQ2+pGU z8FW7o;*0U-gIjsvf!TN2YMOW7XMZCat5Qct0Fu~~B?MJ8szJPNaxDhGKoS8was>NA z$mvNjWPfn6w}Ug-BQ5|Zn5JJIE%J1NV-wV4ZUC)rmhZ6EL2ecMO(T$Rt`Y>K8v!u> z(+yDt7r5uQBZw)g5KOCtU2otEaAaWn$6y z{f=kvP9ghHU{k&>mHpSL1j&nF3d!7GgS1DRK<;pcl{?>FX-pzFc96dsA1gD{$5&#b zA}f{~5+bBuQOy>qA8u*EN>*CzTvX^~{Au@0Hk+Tg6q@{_EI2n*$$&(>A^wmOGAAY6 z19crvS~&MKxNS@?l>i;RWt*NV@+} zJ1l!B(G|r;KZtv~5a(kBL0eBEJQu1o4qOQx4`#TtJJ>cXoW2H(#>B%gC^~(dqVdMo z=d!KN^cTB7n{NyVGu&EYsC+26vAB_>6-_ipN9~SE2=%o+On3U7?FZhk0q<&jP-Hw{ z{pErXY&JA5o~;OFZJq;5D`eG=6v(Tq62#vcFRW^m1P*~aj$lR8uCQ0PZ4$)ue#HS; zg7qBE;BsaUa({zT3>ks0Wf448WdN{9tegk(dbkDq?hT!F=@M%|V2(Z~P7@|7$`}S9 zhmjXV93=A9dV8^dP z3PAS15{6DpZKEJ&JXHkJcC}&*V?@^YLOP}iIHAyX25|7Ep;>C9c`N`UEAZ%f{~UoJ zg(y-`aT6|Gi|{<|a>U;Ypi-}QDGU9t-ujQ)_fQWaamgO0ae$D!bn5HOjzNTfA|2r} z45UEy8Kq*62DY(Hv)$An=Dx6yq4_Q*4@Mr_F#h*QzY&ftId;j~@Qd{iu!O4AbrS-j zcC7eQHfgb!?EIds`AX2mTft5xE%k7aa@mKE@MQnhKXl@KHx&(F292|_P}_EHq3Ua` z1|CxJ3ShGK?4%e(R;`jp&85^eK+pG*E6&*66UNXd#8M3(v>wUf#y|abL_>6?rR2#O zO^&e|A$ZRcFzGH{nu

if&`bg6%RXx?HNtc)s0>xNVA#Mrf7Z3^whw#AUj5g+ zI-UA8J%EVf)LO$$^xX8S7jfQR4%5>&i}_&!8qtejerqsGG)pqNgTWF#Ox3@Vw&(i~ zjh&Wsus&h;^y}oXHQvSCC+7a>-L(5;C$rb8HMN9Z75P4^9xI3GD(4<`NWX0&p+u^sJih4+ zy$Sy%aYE~?xZA9x;6n;<#Y|Eum(@*iZi zo{+-Bn<4QxgNCPr#%$b+fQFZ}EbM~eBS6j)(Fyksy}_R#y?D4s^(%_AqMv8%u*(kq zp7~dXjn^)Zc6Q=_6|k;YDi(z!!dr@U#-{nqZE%#RrV1HAD1FRQwP{i1*s7kQgE1lf zwsFCn#R(*oIH6pHNh|AZBhrHkF-Cf!)S_?fyk7rH1@m%oWlx%bx&`#^T1jpaofJK){x6# z!P-IeS?xjO1tVEWfgsZHxjPrdVb&*9$&{#h;1`Z^a}&mTkDiBD`OPv&iDUjKCXgBi z-QaPmiZs!j5i&O&o^x$?Blb!FnGbS-S@vb|y0BD`StkhO5zB`3FQlV&Vq-70MhRG%eJrLWr0#ed(vkVw3Z(S)r>3eX#>>yjpDwrDG+xaGk6Kx48SC z?kQzQ$QLx9tG_ZuTITru!Y1q9Q-pZb9FH`>@MygNYadj!JP@Ty5NAMJHl-hV=PpvE zA`&b8qm4dW_*Pf5@WY%E>_H@^`P;__|%QV(|-st!^3u_P~4 zSqDujK8@=$Y-nnl`}7f4!i=g8e;mu5>UCCN$R#1ETE-lI4UUsI+GQ_j)&lJ~Ify>@ zwH5^%9jfIS&Dd`8_l$VkS8z-+;?u*Fdd#d*IdNo9*n9bRX1cqxHG;%p(?VTME16$D zGa$6KP}#nLk8KTBF5f8d2g1ohZ7msyG>{|tw2Ud_R=w94=)Arfbin|caVcSAbRw49 zE>o9pKzIo28c%%|V-*F@Jlk1y)P}=Bat5SREpBBKQ0Q#A{d){Nehe`U!$A|5I6n7~ zQALnB6wi>TZGhn0QhO0_7-gds7Rq+YNj5b3`0ue%-JX;7Z?V;j^NQ{+l-?qwfxk92 zuc8YGh=bW#=Fp&dpdJ7L7^$8Chefual|M(4aqjP|0+gzly%3%xN5r|o%G-$r#SJYo zi|-a>U^}=6HmrYq!4c4VPU)l0M`w4x4 zdQ6{=s*YDx;8W7>z2F-_#xmy(qFNDX=^2N*+`%7%JA^%BPb%{FcoY$dziWNQ(j&bQ zyKXCQ_t15gZiS6$kk9y~mRa4RjrP{y`gv50>B=p9$qqDpI4_u2y(MuX^X@zSMD?3BzUJ{?@O>5B zYC&49vHc8hNx9^Bzd1%*$nn)@#R58<{Zm0{3d6#p zq{4&Dq0e$O)3AT7d3-%fj~jcB*xqiL0p5x8ld8!LisY}E!BUgbOESW8%=o|?=albj zY7y7Q)?xc3xnsSzgH zG}>uo=XKxtNUb`u@>V%ZRH3ad!}#yZ*NMP`&aP_y6Pwv@EqdZjaEH-3tG0|`BI@aB z-Dozju1Onk{X_|Kz6pL= zpG|SxK!wuUq1tj6GD`;bTB`n-owvve)R*ht^DNpwqg3vfh#3MMPdr8NcO&ySWFwaI zsNbx*Qz&`hjos2#^r@PHk|P(;{90BEk^mZ2+ZQ2CUI)tm%|$gmsc{YQNN zN=F?b_~zTG*0w&JQW0gyjQ`~US3t#gS6Wy@_aWMTwY~?_GnuW7SwUq6*_)eU zXY|G7wTkXaw-q>9^Wq*lg6s7wV0;9bMXk7$bY+Wbz*1d~@<2&q%?RrSp$AvG9PL@% z&-Cl<;|+5-Zgp@R@f~i#t7;Z*(bo9j=aOdB-m#4s;0T0ZyYq3jW&l<~YI4`k6NmxP`5-)9 zr9>%2)5&7+fOH$XTbp-Kjq;u1qzahMIKMB=n=BAzS`s z@~xin?17W%f?KPK&YMn%1&79g@9oQo>By3xo3ovD4nDsOL46?2XF+x*Cj!Hi2htl) z5g72pZN{E9(>yK69>Io2sJGMB)zDM-Wa$L~&p`j|Cg32~4$~t0UUWogQgQ_U%0t_v zwkGpmKrcgRmHOA{h?*Kl)}=j% zza;|Tj&qF_jw>rd+V}`K0MrT`nmVGfX5LO0to2o{6CwCDq7El80-r}68p>`3JfG3! zR^BgCqypr2_7~L7-Az&je+Ttz(<=AL&VpA?en*2$J&{&>XsyMnWGbYD#I(zFSIqR6 z3P;aa8~~T3QNOf`Y2#U>u6JcnNkHZm?!a(YK`yb};j*R*KLI{kfz#yCzm4=|x4|ng z3aWruc_exdIDs5@d<@7aZuRhdQMVYz00)U=VT=L%$+=GjC@BfT_^hN>2reC9Cw5Ue z$u)z1=YU&!V1J31a7Csbb(OC@7cxLydKP~Pa_=NV-14OX`&z<7Gb$nk_$0Ac`1t~F zdq&w&5npyH4Jaf+vvR})%?I*5>NpHd-JfE@H-ky1({weUreMnu9&x5gcvyN+YE}G* zt$c$ZrZeDQs6Cq8`%FO1t+~ya{+;`Sk-s>-%~) z5&&ijbBJpEla;`$>FNGvC+@Xh#JogA4Ln;N_;WIIXW5|`^sLr<=5p8>|JB`k1?+)U z(l&S&MOwYmjxMAE&{bFYEq$QCLb%_x+Hur!a8uysNC_1t`j(gDhTGFk3Ujma%CM!7 zhZa+sYKILkL4FKr*bM&D{(Nj@=FXjOHfie2wRz-R4TJn*pLS63V^Qz+amOHf$RB%& zSlp^#p_4|1cp~eMQFdq_KoCVg z)huOF9|2|#mD?(^)rO}(4E6dHWmxN&zQ2%P(E@TrT=r-B6O4^Q18oiUNP<3-gr>vj zFwlYHfo6P&ES zv4A__56A8#d5!+a0t@thz{+eTD5E3db~&6IR20#%TW1Dz6JC7kZC9zz<|L79DAMb# zF&LC*E7~BX-qW{QLqxp}IPeTx!>Ak7RcWdc#YF1Uvi|RGKV##o>5Gdt*M%*y>Xu&(O`LWyj|?ksNkU_yNz?Q^Sh^{2tqCdJnDz zc7)~(VZQidgj^H+SZ=nP3~bJn=QiI1`Di|Ng2MUvbxjQxtygd4q$bnk2cE*d#@DVmyFyuk=Xfzu#(}V4yF(Dt+2o zWj-sz9lLY;GTW6xl6H&mUF<4L6cT61$C9<0BMmvl7)+V_2s}05kkW0?PMctnA6t`v zv{b(L+&GWuEEG-6mrWcNJ}ij)WU?LZ8{-e=g3H4orCkX$@n}F2*C6ME`rb551~&j0ECumAu6 diff --git a/.ci/inc/fetch_ci_scripts.bash b/.ci/inc/fetch_ci_scripts.bash new file mode 100644 index 0000000000..d608e40531 --- /dev/null +++ b/.ci/inc/fetch_ci_scripts.bash @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +function fetch_ci_scripts() { + local inc_dir + local inc_url + inc_dir="$(dirname "$0")/inc" + inc_url="${PMD_CI_SCRIPTS_URL:-https://raw.githubusercontent.com/pmd/build-tools/master/scripts}/inc" + + mkdir -p "${inc_dir}" + + for f in ${SCRIPT_INCLUDES}; do + if [ ! -e "${inc_dir}/$f" ]; then + curl -sSL "${inc_url}/$f" > "${inc_dir}/$f" + fi + [ "$PMD_CI_DEBUG" = "true" ] && echo "loading ${inc_dir}/$f in ${MODULE:-$0}" + # shellcheck source=/dev/null + source "${inc_dir}/$f" || exit 1 + done +} diff --git a/.ci/inc/github-releases-api.inc b/.ci/inc/github-releases-api.inc deleted file mode 100644 index e71e67f941..0000000000 --- a/.ci/inc/github-releases-api.inc +++ /dev/null @@ -1,214 +0,0 @@ -# -# The functions here require the following scripts: -# logger.inc -# -# The functions here require the following environment variables: -# GITHUB_OAUTH_TOKEN -# GITHUB_BASE_URL -# - -# -# Creates a new release on github with the given tag and target_commit. -# The release is draft and not published. -# -# $RESULT = release json string -# -# See: https://developer.github.com/v3/repos/releases/#create-a-release -# -function gh_releases_createDraftRelease() { - local tagName="$1" - local targetCommitish="$2" - - log_debug "$FUNCNAME: Creating new draft release for tag=$tagName and commit=$targetCommitish" - - local request=$(cat <<-EOF - { - "tag_name": "${tagName}", - "target_commitish": "${targetCommitish}", - "name": "${tagName}", - "draft": true - } - EOF - ) - - log_debug "POST $GITHUB_BASE_URL/releases" - log_info "Creating github draft release" - RESULT=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ - -H "Content-Type: application/json" \ - -X POST \ - --data "${request}" \ - "$GITHUB_BASE_URL/releases") - log_debug " -> response: $RESULT" - - log_success "Created draft release with id $(echo $RESULT | jq --raw-output ".url")" -} - -# -# Gets the latest release, if it is a draft and returns with 0. -# Returns with 1, if the latest release is not a draft - meaning, there is no -# draft release (yet?). -# -# RESULT = release json string -# -# See: https://developer.github.com/v3/repos/releases/#list-releases-for-a-repository -# -function gh_releases_getLatestDraftRelease() { - log_debug "$FUNCNAME" - log_debug "GET $GITHUB_BASE_URL/releases?per_page=1" - RESULT=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ - "$GITHUB_BASE_URL/releases?per_page=1" | jq ".[0]") - log_debug " -> response: $RESULT" - local draft=$(echo $RESULT | jq ".draft") - if [ "$draft" != "true" ]; then - RESULT="" - log_error "Could not find draft release!" - return 1 - fi - log_info "Found draft release: $(echo $RESULT | jq --raw-output ".url")" -} - -# -# Deletes a release. -# -# See: https://developer.github.com/v3/repos/releases/#delete-a-release -# -function gh_release_deleteRelease() { - local release="$1" - - gh_release_getIdFromData "$release" - local releaseId="$RESULT" - log_debug "$FUNCNAME id=$releaseId" - log_debug "DELETE $GITHUB_BASE_URL/releases/$releaseId" - log_info "Deleting github release $releaseId" - local response - response=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ - -X DELETE \ - "$GITHUB_BASE_URL/releases/$releaseId") - log_debug " -> response: $response" - log_success "Deleted release with id $releaseId" -} - -# -# Determines the release id from the given JSON release data. -# -# RESULT = "the release id" -# -function gh_release_getIdFromData() { - local release="$1" - - RESULT=$(echo $release | jq --raw-output ".id") -} - -# -# Determines the tag_name from the given JSON release data. -# -# RESULT = "the tag name" -# -function gh_release_getTagNameFromData() { - local release="$1" - - RESULT=$(echo $release | jq --raw-output ".tag_name") -} - -# -# Uploads a asset to an existing release. -# -# See: https://developer.github.com/v3/repos/releases/#upload-a-release-asset -# -function gh_release_uploadAsset() { - local release="$1" - local filename="$2" - local name=$(basename $filename) - - gh_release_getIdFromData "$release" - local releaseId="$RESULT" - log_debug "$FUNCNAME: releaseId=$releaseId file=$filename name=$name" - - local uploadUrl=$(echo "$release" | jq --raw-output ".upload_url") - uploadUrl="${uploadUrl%%\{\?name,label\}}" - uploadUrl="${uploadUrl}?name=${name}" - log_debug "POST $uploadUrl" - log_info "Uploading $filename to github release $releaseId" - local response - response=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ - -H "Content-Type: application/zip" \ - --data-binary "@$filename" \ - -X POST \ - "${uploadUrl}") - log_debug " -> response: $response" - log_success "Uploaded release asset $filename for release $releaseId" -} - -# -# Updates the release info: name and body. -# The body is escaped to fit into JSON, so it is allowed for the body to be -# a multi-line string. -# -# See: https://developer.github.com/v3/repos/releases/#edit-a-release -# -function gh_release_updateRelease() { - local release="$1" - local name="$2" - local body="$3" - - gh_release_getIdFromData "$release" - local releaseId="$RESULT" - gh_release_getTagNameFromData "$release" - local tagName="$RESULT" - log_debug "$FUNCNAME releaseId=$releaseId name=$name tag_name=$tagName" - - body="${body//'\'/\\\\}" - body="${body//$'\r'/}" - body="${body//$'\n'/\\r\\n}" - body="${body//'"'/\\\"}" - - local request=$(cat <<-EOF - { - "tag_name": "${tagName}", - "name": "${name}", - "body": "${body}" - } - EOF - ) - - log_debug "PATCH $GITHUB_BASE_URL/releases/${releaseId}" - log_debug " -> request: $request" - log_info "Updating github release $releaseId" - local response - response=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "${request}" \ - -X PATCH \ - "$GITHUB_BASE_URL/releases/${releaseId}") - log_debug " -> response: $response" - log_success "Updated release with id=$releaseId" -} - -# -# Publish a release by setting draft="false". -# Note: This will send out the notification emails if somebody -# watched the releases. -# -# See: https://developer.github.com/v3/repos/releases/#edit-a-release -# -function gh_release_publishRelease() { - local release="$1" - - gh_release_getIdFromData "$release" - local releaseId="$RESULT" - log_debug "$FUNCNAME releaseId=$releaseId" - - local request='{"draft":false}' - log_debug "PATCH $GITHUB_BASE_URL/releases/${releaseId}" - log_debug " -> request: $request" - log_info "Publishing github release $releaseId" - local response - response=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "${request}" \ - -X PATCH \ - "$GITHUB_BASE_URL/releases/${releaseId}") - log_debug " -> response: $response" - local htmlUrl=$(echo "$response" | jq --raw-output ".html_url") - log_success "Published release with id=$releaseId at $htmlUrl" -} diff --git a/.ci/inc/install-openjdk.inc b/.ci/inc/install-openjdk.inc deleted file mode 100644 index c9a963242b..0000000000 --- a/.ci/inc/install-openjdk.inc +++ /dev/null @@ -1,103 +0,0 @@ - -# needs: -# inc/logger - -# -# Downloads openjdk from AdoptOpenJDK by accessing the API. -# The API is documented at https://api.adoptopenjdk.net/swagger-ui/ -# - -function install_openjdk() { - OPENJDK_VERSION=$1 - - case "$(uname)" in - Linux*) - JDK_OS=linux - COMPONENTS_TO_STRIP=1 # e.g. openjdk-11.0.3+7/bin/java - ;; - Darwin*) - JDK_OS=mac - COMPONENTS_TO_STRIP=3 # e.g. jdk-11.0.3+7/Contents/Home/bin/java - ;; - CYGWIN*|MINGW*) - JDK_OS=windows - ;; - *) - log_error "Unknown OS: $(uname)" - exit 1 - ;; - esac - - - DOWNLOAD_URL=$(curl --silent -X GET "https://api.adoptopenjdk.net/v3/assets/feature_releases/${OPENJDK_VERSION}/ga?architecture=x64&heap_size=normal&image_type=jdk&jvm_impl=hotspot&os=${JDK_OS}&page=0&page_size=1&project=jdk&sort_method=DEFAULT&sort_order=DESC&vendor=adoptopenjdk" \ - -H "accept: application/json" \ - | jq -r ".[0].binaries[0].package.link") - - OPENJDK_ARCHIVE=$(basename ${DOWNLOAD_URL}) - log_debug "Archive name: ${OPENJDK_ARCHIVE}" - - CACHE_DIR=${HOME}/.cache/openjdk - TARGET_DIR=${HOME}/openjdk${OPENJDK_VERSION} - - mkdir -p ${CACHE_DIR} - mkdir -p ${TARGET_DIR} - - if [ ! -e ${CACHE_DIR}/${OPENJDK_ARCHIVE} ]; then - log_info "Downloading from ${DOWNLOAD_URL} to ${CACHE_DIR}" - curl --location --output ${CACHE_DIR}/${OPENJDK_ARCHIVE} "${DOWNLOAD_URL}" - else - log_info "Skipped download, file ${CACHE_DIR}/${OPENJDK_ARCHIVE} already exists" - fi - - log_info "Extracting to ${TARGET_DIR}" - - case "$OPENJDK_ARCHIVE" in - *.zip) - 7z x ${CACHE_DIR}/${OPENJDK_ARCHIVE} -o${TARGET_DIR} - mv ${TARGET_DIR}/*/* ${TARGET_DIR}/ - ;; - *.tar.gz) - tar --extract --file ${CACHE_DIR}/${OPENJDK_ARCHIVE} -C ${TARGET_DIR} --strip-components=${COMPONENTS_TO_STRIP} - ;; - *) - log_error "Unknown filetype: ${OPENJDK_ARCHIVE}" - exit 1 - ;; - esac -} - -function install_openjdk_setdefault() { - OPENJDK_VERSION=$1 - - install_openjdk $OPENJDK_VERSION - - log_info "Using OpenJDK ${OPENJDK_VERSION} as default" - TARGET_DIR=${HOME}/openjdk${OPENJDK_VERSION} - export JAVA_HOME="${TARGET_DIR}" - export PATH="${TARGET_DIR}/bin:${PATH}" - - java -version -} - -# -# Installs jdk7 for integration test -# -function install_oraclejdk7() { - local local_dir="${HOME}/.cache/jdk7" - local target_dir="${HOME}/oraclejdk7" - local download_url="https://pmd-code.org/oraclejdk/jdk-7u80-linux-x64.tar.gz" - local archive=$(basename $download_url) - - mkdir -p ${local_dir} - mkdir -p ${target_dir} - if [ ! -e ${local_dir}/${archive} ]; then - log_info "Downloading from ${download_url} to ${local_dir}" - curl --location --output ${local_dir}/${archive} ${download_url} - else - log_info "Skipped download, file ${local_dir}/${archive} already exists" - fi - log_info "Extracting to ${target_dir}" - tar --extract --file ${local_dir}/${archive} -C ${target_dir} --strip-components=1 - - log_info "OracleJDK7 can be used via -Djava7.home=${HOME}/oraclejdk7" -} diff --git a/.ci/inc/logger.inc b/.ci/inc/logger.inc deleted file mode 100644 index cf002de9a1..0000000000 --- a/.ci/inc/logger.inc +++ /dev/null @@ -1,30 +0,0 @@ -COL_GREEN="\e[32m" -COL_RED="\e[31m" -COL_RESET="\e[0m" -COL_YELLOW="\e[33;1m" - -function log_error() { - echo -e "${COL_RED}[ERROR ] $*${COL_RESET}" -} - -function log_info() { - echo -e "${COL_YELLOW}[INFO ] $*${COL_RESET}" -} - -function log_success() { - echo -e "${COL_GREEN}[SUCCESS] $*${COL_RESET}" -} - -function log_debug() { - true - #echo -e "[DEBUG ] $*" -} - -function log_group_start() { - echo "::group::$*" - log_info $* -} - -function log_group_end() { - echo "::endgroup::" -} \ No newline at end of file diff --git a/.ci/inc/pmd-code-api.inc b/.ci/inc/pmd-code-api.inc index d9e75beb7c..122f7be59b 100644 --- a/.ci/inc/pmd-code-api.inc +++ b/.ci/inc/pmd-code-api.inc @@ -1,7 +1,9 @@ -# -# The functions here require the following scripts: -# inc/logger.inc -# +#!/usr/bin/env bash + +MODULE="pmd-code-api" +SCRIPT_INCLUDES="log.bash" +# shellcheck source=inc/fetch_ci_scripts.bash +source "$(dirname "$0")/inc/fetch_ci_scripts.bash" && fetch_ci_scripts PMD_CODE_SSH_USER=pmd PMD_CODE_DOCS_PATH=/docs.pmd-code.org/ @@ -11,53 +13,51 @@ function pmd_code_uploadDocumentation() { local filename="$2" local basefilename="$(basename $filename)" - log_debug "$FUNCNAME pmdVersion=$pmdVersion filename=$filename" + pmd_ci_log_debug "$FUNCNAME pmdVersion=$pmdVersion filename=$filename" scp "${filename}" ${PMD_CODE_SSH_USER}@pmd-code.org:${PMD_CODE_DOCS_PATH} ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ unzip -qo ${basefilename} && \ rm ${basefilename}" - log_info "Docs updated: https://docs.pmd-code.org/pmd-doc-${pmdVersion}/" + pmd_ci_log_info "Docs updated: https://docs.pmd-code.org/pmd-doc-${pmdVersion}/" } function pmd_code_removeDocumentation() { local pmdVersion="$1" - log_debug "$FUNCNAME pmdVersion=$pmdVersion" + pmd_ci_log_debug "$FUNCNAME pmdVersion=$pmdVersion" ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ rm -rf pmd-doc-${pmdVersion}/" - log_info "Removed docs: https://docs.pmd-code.org/pmd-doc-${pmdVersion}/" + pmd_ci_log_info "Removed docs: https://docs.pmd-code.org/pmd-doc-${pmdVersion}/" } function pmd_code_createSymlink() { local pmdVersion="$1" local name="$2" - log_debug "$FUNCNAME pmdVersion=$pmdVersion name=$name" + pmd_ci_log_debug "$FUNCNAME pmdVersion=$pmdVersion name=$name" ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ rm -f $name && \ ln -s pmd-doc-${pmdVersion} $name" - log_info "Symlink created: https://docs.pmd-code.org/$name/ -> https://docs.pmd-code.org/pmd-doc-${pmdVersion}/" + pmd_ci_log_info "Symlink created: https://docs.pmd-code.org/$name/ -> https://docs.pmd-code.org/pmd-doc-${pmdVersion}/" } function pmd_code_uploadJavadoc() { local pmdVersion="$1" local basePath="$2" - log_debug "$FUNCNAME pmdVersion=$pmdVersion basePath=$basePath" + pmd_ci_log_debug "$FUNCNAME pmdVersion=$pmdVersion basePath=$basePath" for i in ${basePath}/*/target/*-javadoc.jar */*/target/*-javadoc.jar; do pmd_code_uploadJavadocModule "$pmdVersion" "$i" done - pmd_code_fixPmdLangTestStyle "${basePath}" - # make sure https://docs.pmd-code.org/apidocs/ shows directory index ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH}/apidocs && \ echo 'Options +Indexes' > .htaccess" - log_info "Directory index enabled for https://docs.pmd-code.org/apidocs/" + pmd_ci_log_info "Directory index enabled for https://docs.pmd-code.org/apidocs/" } function pmd_code_uploadJavadocModule() { @@ -66,29 +66,21 @@ function pmd_code_uploadJavadocModule() { local moduleJavadocJarBasename="$(basename $moduleJavadocJar)" local module=${moduleJavadocJarBasename%%-${pmdVersion}-javadoc.jar} - log_debug "$FUNCNAME pmdVersion=$pmdVersion moduleJavadocJar=$moduleJavadocJar module=$module" + pmd_ci_log_debug "$FUNCNAME pmdVersion=$pmdVersion moduleJavadocJar=$moduleJavadocJar module=$module" scp "$moduleJavadocJar" ${PMD_CODE_SSH_USER}@pmd-code.org:${PMD_CODE_DOCS_PATH} ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ mkdir -p apidocs/${module}/${pmdVersion} && \ unzip -qo -d apidocs/${module}/${pmdVersion} ${moduleJavadocJarBasename} && \ rm ${moduleJavadocJarBasename}" - log_info "JavaDoc for $module uploaded: https://docs.pmd-code.org/apidocs/${module}/${pmdVersion}/" -} - -function pmd_code_fixPmdLangTestStyle { - local basePath="$1" - - log_debug "$FUNCNAME basePath=$basePath" - scp "${basePath}/pmd-lang-test/target/dokka/style.css" ${PMD_CODE_SSH_USER}@pmd-code.org:${PMD_CODE_DOCS_PATH}/apidocs/pmd-lang-test/ - log_info "Fixed style for https://docs.pmd-code.org/apidocs/pmd-lang-test/*/" + pmd_ci_log_info "JavaDoc for $module uploaded: https://docs.pmd-code.org/apidocs/${module}/${pmdVersion}/" } function pmd_code_removeJavadoc() { local pmdVersion="$1" - log_debug "$FUNCNAME pmdVersion=$pmdVersion" + pmd_ci_log_debug "$FUNCNAME pmdVersion=$pmdVersion" ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ rm -rf apidocs/*/${pmdVersion}" - log_info "Removed Javadoc: https://docs.pmd-code.org/apidocs/*/${pmdVersion}/ is gone" + pmd_ci_log_info "Removed Javadoc: https://docs.pmd-code.org/apidocs/*/${pmdVersion}/ is gone" } diff --git a/.ci/inc/pmd-doc.inc b/.ci/inc/pmd-doc.inc index 6054baca46..370b22d3fe 100644 --- a/.ci/inc/pmd-doc.inc +++ b/.ci/inc/pmd-doc.inc @@ -1,3 +1,9 @@ +#!/usr/bin/env bash + +MODULE="pmd-doc" +SCRIPT_INCLUDES="log.bash" +# shellcheck source=inc/fetch_ci_scripts.bash +source "$(dirname "$0")/inc/fetch_ci_scripts.bash" && fetch_ci_scripts # Used env vars: # PMD_CI_JOB_URL @@ -12,7 +18,7 @@ function pmd_doc_generate_jekyll_site() { pushd docs echo -e "\n\n" - log_info "Building documentation using jekyll..." + pmd_ci_log_info "Building documentation using jekyll..." bundle config set --local path vendor/bundle bundle install bundle exec jekyll build @@ -27,21 +33,21 @@ function pmd_doc_create_archive() { pushd docs echo -e "\n\n" - log_info "Creating pmd-doc archive..." - mv _site pmd-doc-${VERSION} - zip -qr pmd-doc-${VERSION}.zip pmd-doc-${VERSION}/ - log_success "Successfully created pmd-doc-${VERSION}.zip" + pmd_ci_log_info "Creating pmd-doc archive..." + mv _site pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION} + zip -qr pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}.zip pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}/ + pmd_ci_log_success "Successfully created pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}.zip" popd } # -# Publishes the site to https://pmd.github.io/pmd-${VERSION} and +# Publishes the site to https://pmd.github.io/pmd-${PMD_CI_MAVEN_PROJECT_VERSION} and # https://pmd.github.io/latest/ # function publish_release_documentation_github() { echo -e "\n\n" - log_info "Adding the new doc to pmd.github.io..." + pmd_ci_log_info "Adding the new doc to pmd.github.io..." # clone pmd.github.io. Note: This uses the ssh key setup earlier # In order to speed things up, we use a sparse checkout - no need to checkout all directories here mkdir pmd.github.io @@ -55,30 +61,30 @@ function publish_release_documentation_github() { echo "/latest/" > .git/info/sparse-checkout echo "/sitemap.xml" >> .git/info/sparse-checkout git pull --depth=1 origin master - log_info "Copying documentation from ../docs/pmd-doc-${VERSION}/ to pmd-${VERSION}/ ..." - rsync -ah --stats ../docs/pmd-doc-${VERSION}/ pmd-${VERSION}/ + pmd_ci_log_info "Copying documentation from ../docs/pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}/ to pmd-${PMD_CI_MAVEN_PROJECT_VERSION}/ ..." + rsync -ah --stats ../docs/pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}/ pmd-${PMD_CI_MAVEN_PROJECT_VERSION}/ git status - echo "Executing: git add pmd-${VERSION}" - git add pmd-${VERSION} - echo "Executing: git commit..." - git commit -q -m "Added pmd-${VERSION}" + pmd_ci_log_debug "Executing: git add pmd-${PMD_CI_MAVEN_PROJECT_VERSION}" + git add pmd-${PMD_CI_MAVEN_PROJECT_VERSION} + pmd_ci_log_debug "Executing: git commit..." + git commit -q -m "Added pmd-${PMD_CI_MAVEN_PROJECT_VERSION}" - log_info "Copying pmd-${VERSION} to latest ..." + pmd_ci_log_info "Copying pmd-${PMD_CI_MAVEN_PROJECT_VERSION} to latest ..." git rm -qr latest - cp -a pmd-${VERSION} latest - echo "Executing: git add latest" + cp -a pmd-${PMD_CI_MAVEN_PROJECT_VERSION} latest + pmd_ci_log_debug "Executing: git add latest" git add latest - echo "Executing: git commit..." - git commit -q -m "Copying pmd-${VERSION} to latest" + pmd_ci_log_debug "Executing: git commit..." + git commit -q -m "Copying pmd-${PMD_CI_MAVEN_PROJECT_VERSION} to latest" - log_info "Generating sitemap.xml" + pmd_ci_log_info "Generating sitemap.xml" ../docs/sitemap_generator.sh > sitemap.xml - echo "Executing: git add sitemap.xml" + pmd_ci_log_debug "Executing: git add sitemap.xml" git add sitemap.xml - echo "Executing: git commit..." + pmd_ci_log_debug "Executing: git commit..." git commit -q -m "Generated sitemap.xml" - echo "Executing: git push origin master" + pmd_ci_log_info "Executing: git push origin master" git push origin master ) } @@ -89,12 +95,12 @@ function publish_release_documentation_github() { # function pmd_doc_publish_to_github_pages() { echo -e "\n\n" - log_info "Pushing the new site to github pages..." + pmd_ci_log_info "Pushing the new site to github pages..." git clone --branch gh-pages --depth 1 git@github.com:pmd/pmd.git pmd-gh-pages # clear the files first rm -rf pmd-gh-pages/* # copy the new site - cp -a docs/pmd-doc-${VERSION}/* pmd-gh-pages/ + cp -a docs/pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}/* pmd-gh-pages/ ( cd pmd-gh-pages git config user.name "PMD CI (pmd-bot)" @@ -106,6 +112,6 @@ ${PMD_CI_JOB_URL} ${PMD_CI_PUSH_COMMIT_COMPARE}" git commit -q -m "$MSG" git push git@github.com:pmd/pmd.git HEAD:gh-pages - log_success "Successfully pushed site to https://pmd.github.io/pmd/" + pmd_ci_log_success "Successfully pushed site to https://pmd.github.io/pmd/" ) } diff --git a/.ci/inc/regression-tester.inc b/.ci/inc/regression-tester.inc index 3bc72ccd81..65d6e1e6b2 100644 --- a/.ci/inc/regression-tester.inc +++ b/.ci/inc/regression-tester.inc @@ -1,7 +1,10 @@ -# -# The functions here require the following scripts: -# inc/logger.inc -# inc/install-openjdk.inc +#!/usr/bin/env bash + +MODULE="pmd-doc" +SCRIPT_INCLUDES="log.bash openjdk.bash" +# shellcheck source=inc/fetch_ci_scripts.bash +source "$(dirname "$0")/inc/fetch_ci_scripts.bash" && fetch_ci_scripts + # # The functions here require the following environment variables: # PMD_SF_USER @@ -11,8 +14,8 @@ # PMD_CI_CHUNK_TOKEN function regression_tester_setup_ci() { - log_info "Install openjdk8 for pmd-regression-tests" - install_openjdk 8 + pmd_ci_log_info "Install openjdk8 for pmd-regression-tests" + pmd_ci_openjdk_install_adoptopenjdk 8 gpg --batch --yes --decrypt --passphrase="GnxdjywUEPveyCD1RLiTd7t8CImnefYr" \ --output .ci/files/public-env .ci/files/public-env.gpg @@ -20,9 +23,9 @@ function regression_tester_setup_ci() { rm .ci/files/public-env if hash "bundler" 2>/dev/null; then - log_debug "Bundler is already installed" + pmd_ci_log_debug "Bundler is already installed" else - log_info "Installing bundler..." + pmd_ci_log_info "Installing bundler..." gem install bundler fi @@ -38,9 +41,9 @@ function regression_tester_setup_ci() { function regression_tester_uploadBaseline() { local pmdcodeUrl="https://pmd-code.org/pmd-regression-tester/" local baseline_branch="${PMD_CI_BRANCH:-$PMD_CI_TAG}" - log_debug "$FUNCNAME branch=${baseline_branch}" + pmd_ci_log_debug "$FUNCNAME branch=${baseline_branch}" - log_info "Generating and uploading baseline for pmdtester (${baseline_branch})..." + pmd_ci_log_info "Generating and uploading baseline for pmdtester (${baseline_branch})..." pushd .. rm -f .bundle/config bundle config set --local gemfile pmd/Gemfile @@ -54,9 +57,9 @@ function regression_tester_uploadBaseline() { pushd target/reports BRANCH_FILENAME="${baseline_branch/\//_}" zip -q -r ${BRANCH_FILENAME}-baseline.zip ${BRANCH_FILENAME}/ - # ssh-key for pmd-code.org is setup already by pmd_ci_setup_ssh + # ssh-key for pmd-code.org is setup already by pmd_ci_setup_secrets_ssh scp ${BRANCH_FILENAME}-baseline.zip pmd@pmd-code.org:/httpdocs/pmd-regression-tester/ - log_success "Successfully uploaded ${BRANCH_FILENAME}-baseline.zip to ${pmdcodeUrl}" + pmd_ci_log_success "Successfully uploaded ${BRANCH_FILENAME}-baseline.zip to ${pmdcodeUrl}" popd popd } @@ -65,7 +68,7 @@ function regression_tester_uploadBaseline() { # Execute danger, which executes pmd-regression-tester (via Dangerfile). # function regression_tester_executeDanger() { - log_debug "$FUNCNAME" + pmd_ci_log_debug "$FUNCNAME" # Create a corresponding remote branch locally if ! git show-ref --verify --quiet refs/heads/${PMD_CI_BRANCH}; then @@ -81,7 +84,7 @@ function regression_tester_executeDanger() { git branch master origin/master fi - log_info "Running danger on branch ${PMD_CI_BRANCH}" + pmd_ci_log_info "Running danger on branch ${PMD_CI_BRANCH}" bundle exec danger --verbose - log_success "Executed danger successfully" + pmd_ci_log_success "Executed danger successfully" } diff --git a/.ci/inc/setup-secrets.inc b/.ci/inc/setup-secrets.inc deleted file mode 100644 index 6cd3ba953b..0000000000 --- a/.ci/inc/setup-secrets.inc +++ /dev/null @@ -1,62 +0,0 @@ -function pmd_ci_setup_private_env() { - log_info "Setting up secrets as environment variables..." - local -r ENV_FILE=.ci/files/private-env - - printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \ - --passphrase-fd 0 \ - --output ${ENV_FILE} ${ENV_FILE}.gpg - - source ${ENV_FILE} >/dev/null 2>&1 - rm ${ENV_FILE} -} - -function pmd_ci_setup_gpg_key() { - log_info "Setting up GPG release signing key..." - local -r GPG_FILE=.ci/files/release-signing-key-D0BF1D737C9A1C22.gpg - - mkdir -p "${HOME}/.gpg" - printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \ - --passphrase-fd 0 \ - --output ${GPG_FILE} ${GPG_FILE}.gpg - gpg --batch --import ${GPG_FILE} - rm ${GPG_FILE} -} - -function pmd_ci_setup_ssh() { - log_info "Setting up .ssh/id_rsa..." - local -r SSH_KEY_FILE=.ci/files/id_rsa - - printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \ - --passphrase-fd 0 \ - --output ${SSH_KEY_FILE} ${SSH_KEY_FILE}.gpg - chmod 600 ${SSH_KEY_FILE} - - mkdir -p ${HOME}/.ssh - chmod 700 "${HOME}/.ssh" - mv ${SSH_KEY_FILE} "${HOME}/.ssh/id_rsa" - - log_info "Setting up .ssh/known_hosts..." - # - # https://sourceforge.net/p/forge/documentation/SSH%20Key%20Fingerprints/ - # - # run locally: - # ssh-keyscan web.sourceforge.net | tee -a known_hosts - # - # verify fingerprints: - # ssh-keygen -F web.sourceforge.net -l -f known_hosts - # # Host web.sourceforge.net found: line 1 - # web.sourceforge.net RSA SHA256:xB2rnn0NUjZ/E0IXQp4gyPqc7U7gjcw7G26RhkDyk90 - # # Host web.sourceforge.net found: line 2 - # web.sourceforge.net ECDSA SHA256:QAAxYkf0iI/tc9oGa0xSsVOAzJBZstcO8HqGKfjpxcY - # # Host web.sourceforge.net found: line 3 - # web.sourceforge.net ED25519 SHA256:209BDmH3jsRyO9UeGPPgLWPSegKmYCBIya0nR/AWWCY - # - # then add output of `ssh-keygen -F web.sourceforge.net -f known_hosts` - # - echo 'web.sourceforge.net ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2uifHZbNexw6cXbyg1JnzDitL5VhYs0E65Hk/tLAPmcmm5GuiGeUoI/B0eUSNFsbqzwgwrttjnzKMKiGLN5CWVmlN1IXGGAfLYsQwK6wAu7kYFzkqP4jcwc5Jr9UPRpJdYIK733tSEmzab4qc5Oq8izKQKIaxXNe7FgmL15HjSpatFt9w/ot/CHS78FUAr3j3RwekHCm/jhPeqhlMAgC+jUgNJbFt3DlhDaRMa0NYamVzmX8D47rtmBbEDU3ld6AezWBPUR5Lh7ODOwlfVI58NAf/aYNlmvl2TZiauBCTa7OPYSyXJnIPbQXg6YQlDknNCr0K769EjeIlAfY87Z4tw==' >> "$HOME/.ssh/known_hosts" - echo 'web.sourceforge.net ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCwsY6sZT4MTTkHfpRzYjxG7mnXrGL74RCT2cO/NFvRrZVNB5XNwKNn7G5fHbYLdJ6UzpURDRae1eMg92JG0+yo=' >> "$HOME/.ssh/known_hosts" - echo 'web.sourceforge.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOQD35Ujalhh+JJkPvMckDlhu4dS7WH6NsOJ15iGCJLC' >> "$HOME/.ssh/known_hosts" - - # add pmd-code.org (ssh-keyscan pmd-code.org) - echo 'pmd-code.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVsIeF6xU0oPb/bMbxG1nU1NDyBpR/cBEPZcm/PuJwdI9B0ydPHA6FysqAnt32fNFznC2SWisnWyY3iNsP3pa8RQJVwmnnv9OboGFlW2/61o3iRyydcpPbgl+ADdt8iU9fmMI7dC04UqgHGBoqOwVNna9VylTjp5709cK2qHnwU450F6YcOEiOKeZfJvV4PmpJCz/JcsUVqft6StviR31jKnqbnkZdP8qNoTbds6WmGKyXkhHdLSZE7X1CFQH28tk8XFqditX93ezeCiThFL7EleDexV/3+2+cs5878sDMUMzHS5KShTjkxzhHaodhtIEdNesinq/hOPbxAGkQ0FbD' >> $HOME/.ssh/known_hosts -} diff --git a/.ci/inc/sourceforge-api.inc b/.ci/inc/sourceforge-api.inc deleted file mode 100644 index 24f165cf6f..0000000000 --- a/.ci/inc/sourceforge-api.inc +++ /dev/null @@ -1,166 +0,0 @@ -# -# The functions here require the following scripts: -# logger.inc -# -# The functions here require the following environment variables: -# PMD_SF_USER -# PMD_SF_APIKEY -# - -# -# Uploads the release notes to sourceforge files as "ReadMe.md". -# -# Note: this function always succeeds, even if the upload fails. -# In that case, just a error logging is provided. -# -function sourceforge_uploadReleaseNotes() { - local pmdVersion="$1" - local releaseNotes="$2" - - log_debug "$FUNCNAME pmdVersion=$pmdVersion" - local targetUrl="https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}" - - local errexitstate="$(shopt -po errexit)" - set +e # disable errexit - ( - # This handler is called if any command fails - function release_notes_fail() { - log_error "Error while uploading release notes as ReadMe.md to sourceforge!" - log_error "Please upload manually: ${targetUrl}" - cleanup_temp_dir - } - - function cleanup_temp_dir() { - log_debug "Cleanup tempdir $releaseNotesTempDir" - rm "${releaseNotesTempDir}/${pmdVersion}/ReadMe.md" || true - rmdir "${releaseNotesTempDir}/${pmdVersion}" || true - rmdir "${releaseNotesTempDir}" || true - } - - # exit subshell after trap - set -e - trap release_notes_fail ERR - - local releaseNotesTempDir=$(mktemp -d) - log_debug "Tempdir: $releaseNotesTempDir" - mkdir -p "${releaseNotesTempDir}/${pmdVersion}" - echo "$releaseNotes" > "${releaseNotesTempDir}/${pmdVersion}/ReadMe.md" - - log_info "Uploading release notes to sourceforge for version $pmdVersion" - rsync -avz \ - "${releaseNotesTempDir}/" \ - "${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/" - - log_success "Successfully uploaded release notes as ReadMe.md to sourceforge: ${targetUrl}" - - cleanup_temp_dir - ) - # restore errexit state - eval "$errexitstate" -} - -# -# Uploads the given file to sourceforge. -# -# Note: This function always succeeds, even if the upload fails. -# In that case, just a error logging is provided. -# -function sourceforge_uploadFile() { - local pmdVersion="$1" - local filename="$2" - - log_debug "$FUNCNAME pmdVersion=$pmdVersion filename=$filename" - local targetUrl="https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}" - - local errexitstate="$(shopt -po errexit)" - set +e # disable errexit - ( - # This handler is called if any command fails - function upload_failed() { - log_error "Error while uploading ${filename} to sourceforge!" - log_error "Please upload manually: ${targetUrl}" - } - - # exit subshell after trap - set -e - trap upload_failed ERR - - log_info "Uploading $filename to sourceforge..." - .ci/travis_wait "rsync -avh ${filename} ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/${pmdVersion}/" - log_success "Successfully uploaded ${filename} to sourceforge: ${targetUrl}" - ) - # restore errexit state - eval "$errexitstate" -} - -# -# Select the given version as the new default download. -# -# Note: This function always succeeds, even if the request fails. -# In that case, just a error logging is provided. -# -function sourceforge_selectDefault() { - local pmdVersion="$1" - - log_debug "$FUNCNAME pmdVersion=$pmdVersion" - local targetUrl="https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}" - - local errexitstate="$(shopt -po errexit)" - set +e # disable errexit - ( - # This handler is called if any command fails - function request_failed() { - log_error "Error while selecting ${pmdVersion} as new default download on sourceforge!" - log_error "Please do it manually: ${targetUrl}" - } - - # exit subshell after trap - set -e - trap request_failed ERR - - log_info "Selecting $pmdVersion as new default on sourceforge..." - local response - response=$(curl --fail -s -H "Accept: application/json" \ - -X PUT \ - -d "api_key=${PMD_SF_APIKEY}" \ - -d "default=windows&default=mac&default=linux&default=bsd&default=solaris&default=others" \ - "https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}/pmd-bin-${pmdVersion}.zip") - log_debug " -> response: $response" - log_success "Successfully selected $pmdVersion as new default on sourceforge: ${targetUrl}" - ) - # restore errexit state - eval "$errexitstate" -} - -# -# Rsyncs the complete documentation to sourceforge. -# -# Note: This function always succeeds, even if the upload fails. -# In that case, just a error logging is provided. -# -function sourceforge_rsyncSnapshotDocumentation() { - local pmdVersion="$1" - local targetPath="$2" - - log_debug "$FUNCNAME pmdVersion=$pmdVersion targetPath=$targetPath" - local targetUrl="https://pmd.sourceforge.io/${targetPath}/" - - local errexitstate="$(shopt -po errexit)" - set +e # disable errexit - ( - # This handler is called if any command fails - function upload_failed() { - log_error "Couldn't upload the documentation. It won't be current on ${targetUrl}" - } - - # exit subshell after trap - set -e - trap upload_failed ERR - - log_info "Uploading documentation to ${targetUrl}..." - .ci/travis_wait "rsync -ah --stats --delete docs/pmd-doc-${VERSION}/ ${PMD_SF_USER}@web.sourceforge.net:/home/project-web/pmd/htdocs/snapshot/" - log_success "Successfully uploaded documentation: ${targetUrl}" - ) - # restore errexit state - eval "$errexitstate" -} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..33a35ec101 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,55 @@ +name: build + +on: + push: + branches: + - main + - master + tags: + - '**' + pull_request: + schedule: + # build it monthly: At 04:00 on day-of-month 1. + - cron: '0 4 1 * *' + +jobs: + build: + runs-on: ${{ matrix.os }} + continue-on-error: false + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }} + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}- + - name: Set up Ruby 2.7 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.7 + - name: Setup Environment + shell: bash + run: | + echo "LANG=en_US.UTF-8" >> $GITHUB_ENV + echo "MAVEN_OPTS=-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" >> $GITHUB_ENV + echo "PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/master/scripts" >> $GITHUB_ENV + - name: Check Environment + shell: bash + run: | + f=check-environment.sh; \ + mkdir -p .ci && \ + ( [ -e .ci/$f ] || curl -sSL "${PMD_CI_SCRIPTS_URL}/$f" > ".ci/$f" ) && \ + chmod 755 .ci/$f && \ + .ci/$f + - name: Build + run: .ci/build.sh + shell: bash + env: + PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} From 417cc7fcbeceb9da41d08e88a56bfa34f76f19fb Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 15 Apr 2021 16:20:22 +0200 Subject: [PATCH 03/26] [ci] Finish scripts migration, integration sonar+coveralls --- .ci/build-coveralls.sh | 32 --- .ci/build-pr-win-macos.sh | 34 --- .ci/build-sonar.sh | 28 --- .ci/build.sh | 31 +++ .ci/travis_wait | 324 -------------------------- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/build.yml | 4 +- .github/workflows/pull-requests.yml | 40 ---- .github/workflows/pushes.yml | 118 ---------- .github/workflows/releases.yml | 28 --- .github/workflows/troubleshooting.yml | 25 +- BUILDING.md | 4 +- Gemfile | 4 +- README.md | 2 +- 14 files changed, 61 insertions(+), 615 deletions(-) delete mode 100755 .ci/build-coveralls.sh delete mode 100755 .ci/build-pr-win-macos.sh delete mode 100755 .ci/build-sonar.sh delete mode 100755 .ci/travis_wait delete mode 100644 .github/workflows/pull-requests.yml delete mode 100644 .github/workflows/pushes.yml delete mode 100644 .github/workflows/releases.yml diff --git a/.ci/build-coveralls.sh b/.ci/build-coveralls.sh deleted file mode 100755 index 43031a03ec..0000000000 --- a/.ci/build-coveralls.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -source $(dirname $0)/inc/logger.inc -source $(dirname $0)/inc/setup-secrets.inc -source $(dirname $0)/inc/install-openjdk.inc - -set -e - -log_group_start "Setup private env and OpenJDK" - pmd_ci_setup_private_env - install_openjdk_setdefault 11 - export CI_NAME="github actions" - export CI_BUILD_URL="${PMD_CI_JOB_URL}" - export CI_BRANCH="${PMD_CI_GIT_REF##refs/heads/}" -log_group_end - -log_group_start "Executing build with coveralls" - ./mvnw \ - -Dmaven.javadoc.skip=true \ - -Dmaven.source.skip \ - -Dcheckstyle.skip \ - -DrepoToken=${COVERALLS_REPO_TOKEN} \ - -B -V -e \ - clean package jacoco:report \ - coveralls:report -Pcoveralls - - if [ $? -ne 0 ]; then - log_error "Error creating coveralls report" - else - log_success "New coveralls result: https://coveralls.io/github/pmd/pmd" - fi -log_group_end diff --git a/.ci/build-pr-win-macos.sh b/.ci/build-pr-win-macos.sh deleted file mode 100755 index e88508510a..0000000000 --- a/.ci/build-pr-win-macos.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -source $(dirname $0)/inc/logger.inc -source $(dirname $0)/inc/install-openjdk.inc -source $(dirname $0)/inc/regression-tester.inc - -set -e - -log_group_start "Installing Java" - log_info "Install openjdk11 as default" - install_openjdk_setdefault 11 - - PMD_EXTRA_OPT="" - if [[ "$(uname)" == Linux* ]]; then - log_info "Install oracle7 for integration tests" - install_oraclejdk7 - PMD_EXTRA_OPT="-Djava7.home=${HOME}/oraclejdk7" - fi -log_group_end - -log_group_start "Building with maven" - ./mvnw -e -V -B clean verify ${PMD_EXTRA_OPT} -log_group_end - - -# Danger is executed only on the linux runner -case "$(uname)" in - Linux*) - log_group_start "Executing danger" - regression_tester_setup_ci - regression_tester_executeDanger - log_group_end - ;; -esac diff --git a/.ci/build-sonar.sh b/.ci/build-sonar.sh deleted file mode 100755 index 24b5b895da..0000000000 --- a/.ci/build-sonar.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -source $(dirname $0)/inc/logger.inc -source $(dirname $0)/inc/setup-secrets.inc -source $(dirname $0)/inc/install-openjdk.inc - -set -e - -log_group_start "Setup private env and OpenJDK" - pmd_ci_setup_private_env - install_openjdk_setdefault 11 -log_group_end - -log_group_start "Executing build with sonar" - ./mvnw \ - -Dmaven.javadoc.skip=true \ - -Dmaven.source.skip \ - -Dcheckstyle.skip \ - -B -V -e \ - clean package \ - sonar:sonar -Dsonar.login=${SONAR_TOKEN} -Psonar - - if [ $? -ne 0 ]; then - log_error "Error updating sonar..." - else - log_success "New sonar results: https://sonarcloud.io/dashboard?id=net.sourceforge.pmd%3Apmd" - fi -log_group_end diff --git a/.ci/build.sh b/.ci/build.sh index fd073f04c3..043b7ab6c8 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -80,6 +80,37 @@ function build() { regression_tester_setup_ci regression_tester_uploadBaseline pmd_ci_log_group_end + + if pmd_ci_maven_isSnapshotBuild; then + pmd_ci_log_group_start "Executing build with sonar" + # Note: Sonar also needs GITHUB_TOKEN (!) + ./mvnw \ + -Dmaven.javadoc.skip=true \ + -Dmaven.source.skip \ + -Dcheckstyle.skip \ + -Dpmd.skip \ + --show-version --errors --batch-mode --no-transfer-progress \ + clean package \ + sonar:sonar -Dsonar.login=${SONAR_TOKEN} -Psonar + pmd_ci_log_success "New sonar results: https://sonarcloud.io/dashboard?id=net.sourceforge.pmd%3Apmd" + pmd_ci_log_group_end + + pmd_ci_log_group_start "Executing build with coveralls" + export CI_NAME="github actions" + export CI_BUILD_URL="${PMD_CI_JOB_URL}" + export CI_BRANCH="${PMD_CI_BRANCH}" + ./mvnw \ + -Dmaven.javadoc.skip=true \ + -Dmaven.source.skip \ + -Dcheckstyle.skip \ + -Dpmd.skip \ + -DrepoToken=${COVERALLS_REPO_TOKEN} \ + --show-version --errors --batch-mode --no-transfer-progress \ + clean package jacoco:report \ + coveralls:report -Pcoveralls + pmd_ci_log_success "New coveralls result: https://coveralls.io/github/pmd/pmd" + pmd_ci_log_group_end + fi } diff --git a/.ci/travis_wait b/.ci/travis_wait deleted file mode 100755 index d81dd41089..0000000000 --- a/.ci/travis_wait +++ /dev/null @@ -1,324 +0,0 @@ -#!/usr/bin/env bash - -# The MIT License (MIT) - -# Copyright (c) 2015 -# m3t (96bd6c8bb869fe632b3650fb7156c797ef8c2a055d31dde634565f3edda485b) - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -# Available from https://github.com/m3t/travis_wait -# Please report bugs at https://github.com/m3t/travis_wait/issues - -# Coding (Style) Guidelines: -# https://www.chromium.org/chromium-os/shell-style-guidelines -# http://mywiki.wooledge.org/BashGuide/Practices -# http://wiki.bash-hackers.org/scripting/style - - -# bash available? -if [ -z "$BASH_VERSINFO" ]; then - echo "Please make sure you're using bash!" - exit 1 -fi - - -# INITIALIZE CONSTANTS AND GLOBALS -# Only lower case, esp. for export! -# That ensures that system vars stay untouched in any case -readonly prog_name=$(basename "$0") - - -is_writeable() { - local var="$1" - - is_writeable_empty "${var}" 0 -} - -is_writeable_empty() { - local var="$1" - local empty="$2" - [[ -z "${empty}" ]] && empty=1 - - # http://mywiki.wooledge.org/BashGuide/TestsAndConditionals - # "touch" creates file, if it doesn't exist, - # so further tests won't fail at the beginning - if { touch -a "${var}" >/dev/null 2>&1; }; then - if [[ ! -s "${var}" ]]; then - if [[ ! -w "${var}" ]]; then - #show_warning "${var} is not writeable" - return 1 - fi - else - #show_warning "${var} is not empty" - [[ "${empty}" -eq 1 ]] && return 1 - fi - else - #show_warning "Destination for ${var} is not accessible at all" - return 1 - fi - - return 0 -} - -is_number() { - local int="$1" - # http://mywiki.wooledge.org/BashFAQ/054 - [[ "$int" != *[!0-9]* ]] -} - -is_empty() { - local var="$1" - - [[ -z "$var" ]] -} - -show_error() { - printf "\n%s\n" "${prog_name}: error: $*" >&2 - exit 1 -} - -show_warning() { - printf "\n%s\n" "${prog_name}: $*" >&2 -} - -show_help() { - # http://wiki.bash-hackers.org/syntax/redirection#here_documents - cat <<- EOF - - Usage: ${prog_name} [options] [] - - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor - incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, - quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo - consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse - cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat - non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. - - Arguments: - Slowpoke command - Where 's output will be saved - Default: \$RANDOM-output.log - - Options: - -i, --interval Refresh interval in sec. - Default: 30 - - -l, --limit Limit execution time in sec. - Default: 0 (Off) - - -x, --exit-code Force the exit code - Default: -1 (Off) - - -a, --append PRN append output to existing logfile - Off: 0 (Default) - On: 1 - - -h This help screen - - - Copyright (C) 2015 m3t - The MIT License (MIT) - -EOF - - exit 0 -} - -cleanup() { - kill -0 ${pid_slowpoke} >/dev/null 2>&1 && kill ${pid_slowpoke} >/dev/null 2>&1 -} - -main() { - - # INITIALIZE LOCAL VARIABLES - # Variables to be evaluated as shell arithmetic should be initialized - # to a default or validated beforehand. - # CAUTION: Arguments' (not options) default values will be overwritten here anyway - # So they are set in VALIDATE INPUT - local i=0 - local msg="" - local time_passed=0 - local pid_slowpoke=0 - local exit_slowpoke=0 - # Options: - local interval=30 - local time_limit=0 - local exit_force=-1 - local append=0 - # Arguments: - local cmd_slowpoke="" - local file_log="" - - - # SIGNAL HANDLING - # http://mywiki.wooledge.org/SignalTrap - # http://mywiki.wooledge.org/SignalTrap#Special_Note_On_SIGINT - trap 'cleanup; trap - INT; kill -INT $$' INT QUIT # CTRL+C OR CTRL+\ - trap 'cleanup; exit 1' TERM # kill's default signal - - - # COMMAND-LINE ARGUMENTS AND OPTIONS - # http://mywiki.wooledge.org/BashFAQ/035 - msg="requires a non-empty option argument." - while :; do - case "$1" in - -h|-\?|--help) - show_help - exit - ;; - -l|--limit) - if [ -n "$2" ]; then - time_limit="$2" - shift 2 - continue - else - show_error "--limit ${msg}" - fi - ;; - --limit=?*) - time_limit="${1#*=}" - ;; - --limit=) - show_error "--limit ${msg}" - ;; - -i|--interval) - if [ -n "$2" ]; then - interval="$2" - shift 2 - continue - else - show_error "--interval ${msg}" - fi - ;; - --interval=?*) - interval="${1#*=}" - ;; - --interval=) - show_error "--interval ${msg}" - ;; - -x|--exit-code) - if [ -n "$2" ]; then - exit_force="$2" - shift 2 - continue - else - show_error "--exit-code ${msg}" - fi - ;; - --exit-code=?*) - exit_force="${1#*=}" - ;; - --exit-code=) - show_error "--exit-code ${msg}" - ;; - -a|--append) - if [ -n "$2" ]; then - append="$2" - shift 2 - continue - else - show_error "--append ${msg}" - fi - ;; - --append=?*) - append="${1#*=}" - ;; - --append=) - show_error "--append ${msg}" - ;; - --) # End of all options. - shift - break - ;; - -?*) - show_warning "Unknown option (ignored): $1" - ;; - *) # Default case: If no more options then break out of the loop. - break - esac - - shift - done - # Arguments following the options - # will remain in the "$@" positional parameters. - cmd_slowpoke="$1" - file_log="$2" - - - # VALIDATE INPUT - is_number "${interval}" || show_error "Interval is not a valid number" - is_number "${time_limit}" || show_error "Limit is not a valid number" - is_empty "${cmd_slowpoke}" && show_error "Command to execute is not given. See --help." - is_empty "${file_log}" && file_log="$RANDOM-output.log" # http://mywiki.wooledge.org/BashFAQ/062 - - # START CMD - # http://mywiki.wooledge.org/ProcessManagement - if [[ "${append}" -ne 1 ]]; then - is_writeable_empty "${file_log}" || show_error "${file_log} is not writeable or not empty." - ${cmd_slowpoke} > "${file_log}" & pid_slowpoke=$! - else - is_writeable "${file_log}" || show_error "${file_log} is not writeable." - ${cmd_slowpoke} >> "${file_log}" & pid_slowpoke=$! - fi - - - # WAIT - # Terminates when $cmd_slowpoke is finished - # OR - # $time_limit has reached - i=0 - while kill -0 ${pid_slowpoke} >/dev/null 2>&1; do - : $(( time_passed = i * interval )) - - printf "%s\n" \ - "Still waiting for about ${time_passed} seconds" \ - "Used disk space: $(du -sh .)" - - # Output last line from $file_log - tail -1 "${file_log}" - - # $time_limit - if [[ "${time_limit}" -ne 0 ]] && [[ "${time_passed}" -ge "${time_limit}" ]]; then - cleanup - break - fi - - sleep ${interval} - - : $(( i += 1 )) - done - - - # FINISHED - # Shall I fake the exit code? - if ! is_number "${exit_force}"; then - # Get exit code from child process that is terminated already, see above - wait ${pid_slowpoke}; exit_slowpoke=$? - else - exit_slowpoke=${exit_force} - fi - # Output last couple of lines from $file_log - tail -5 "${file_log}" - show_warning "Your given command has terminated with exit code $exit_slowpoke. So do I." - exit ${exit_slowpoke} - -} - -main "$@" diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a1c9a7bd37..41efb1e92c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -14,6 +14,6 @@ - [ ] Added unit tests for fixed bug/feature - [ ] Passing all unit tests -- [ ] Complete build `./mvnw clean verify` passes (checked automatically by travis) +- [ ] Complete build `./mvnw clean verify` passes (checked automatically by github actions) - [ ] Added (in-code) documentation (if needed) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 33a35ec101..dc43e3bf58 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,6 +27,7 @@ jobs: path: | ~/.m2/repository ~/.cache + vendor/bundle key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}- @@ -38,7 +39,7 @@ jobs: shell: bash run: | echo "LANG=en_US.UTF-8" >> $GITHUB_ENV - echo "MAVEN_OPTS=-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" >> $GITHUB_ENV + echo "MAVEN_OPTS=-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -DstagingProgressTimeoutMinutes=30" >> $GITHUB_ENV echo "PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/master/scripts" >> $GITHUB_ENV - name: Check Environment shell: bash @@ -53,3 +54,4 @@ jobs: shell: bash env: PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml deleted file mode 100644 index 9fd0c2fcbe..0000000000 --- a/.github/workflows/pull-requests.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Pull Requests - -on: pull_request - -jobs: - build: - runs-on: ${{ matrix.os }} - continue-on-error: false - timeout-minutes: 60 - strategy: - matrix: - os: [ ubuntu-latest, windows-latest, macos-latest ] - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 2 - - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository - ~/.cache - vendor/bundle - key: pr-${{ runner.os }}-${{ hashFiles('**/pom.xml') }} - restore-keys: | - pr-${{ runner.os }}- - - name: Set up Ruby 2.7 - uses: actions/setup-ruby@v1 - with: - ruby-version: 2.7 - - name: Check Environment - run: .ci/check-environment.sh - shell: bash - - name: Build - run: .ci/build-pr-win-macos.sh - shell: bash - env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 - PMD_CI_BRANCH: ${{ github.event.pull_request.base.ref }} - PMD_CI_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/pushes.yml b/.github/workflows/pushes.yml deleted file mode 100644 index 6f8956bf5f..0000000000 --- a/.github/workflows/pushes.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: Pushes -on: - push: - branches: - - main - - master - schedule: - # build it monthly: At 04:00 on day-of-month 1. - - cron: '0 4 1 * *' - -jobs: - linux: - runs-on: ubuntu-latest - continue-on-error: false - if: "!contains(github.event.head_commit.message, '[skip ci]')" - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository - ~/.cache - vendor/bundle - key: push-${{ runner.os }}-${{ hashFiles('**/pom.xml') }} - restore-keys: | - push-${{ runner.os }}- - - name: Set up Ruby 2.7 - uses: actions/setup-ruby@v1 - with: - ruby-version: 2.7 - - name: Check Environment - run: .ci/check-environment.sh - shell: bash - - name: Build - run: .ci/build.sh - shell: bash - env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -DstagingProgressTimeoutMinutes=30 - PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} - PMD_CI_JOB_URL: "https://github.com/pmd/pmd/runs/${{ github.run_id }}" - PMD_CI_PUSH_COMMIT_COMPARE: ${{ github.event.compare }} - PMD_CI_GIT_REF: ${{ github.ref }} - - windows-macos: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ windows-latest, macos-latest ] - needs: linux - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository - ~/.cache - vendor/bundle - key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}- - - name: Check Environment - run: .ci/check-environment.sh - shell: bash - - name: Build - run: .ci/build-pr-win-macos.sh - shell: bash - env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 - - coveralls: - runs-on: ubuntu-latest - needs: [linux] - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository - ~/.cache - vendor/bundle - key: coveralls-${{ runner.os }}-${{ hashFiles('**/pom.xml') }} - restore-keys: | - coveralls-${{ runner.os }}- - - name: Check Environment - run: .ci/check-environment.sh - shell: bash - - name: Build - run: .ci/build-coveralls.sh - shell: bash - env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 - PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} - PMD_CI_JOB_URL: "https://github.com/pmd/pmd/runs/${{ github.run_id }}" - - sonar: - runs-on: ubuntu-latest - needs: [linux] - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository - ~/.cache - vendor/bundle - key: sonar-${{ runner.os }}-${{ hashFiles('**/pom.xml') }} - restore-keys: | - sonar-${{ runner.os }}- - - name: Check Environment - run: .ci/check-environment.sh - shell: bash - - name: Build - run: .ci/build-sonar.sh - shell: bash - env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 - PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml deleted file mode 100644 index 9e96a7f1ac..0000000000 --- a/.github/workflows/releases.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Release Builds -on: - push: - tags: - - 'pmd_releases/*' - -jobs: - release: - runs-on: ubuntu-latest - continue-on-error: false - steps: - - uses: actions/checkout@v2 - - name: Set up Ruby 2.7 - uses: actions/setup-ruby@v1 - with: - ruby-version: 2.7 - - name: Check Environment - run: .ci/check-environment.sh - shell: bash - - name: Build - run: .ci/build.sh - shell: bash - env: - MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 - PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} - PMD_CI_JOB_URL: "https://github.com/pmd/pmd/runs/${{ github.run_id }}" - PMD_CI_PUSH_COMMIT_COMPARE: ${{ github.event.compare }} - PMD_CI_GIT_REF: ${{ github.ref }} diff --git a/.github/workflows/troubleshooting.yml b/.github/workflows/troubleshooting.yml index dd79aa8c3e..d895a6cf2f 100644 --- a/.github/workflows/troubleshooting.yml +++ b/.github/workflows/troubleshooting.yml @@ -26,11 +26,28 @@ jobs: uses: actions/setup-ruby@v1 with: ruby-version: 2.7 - - name: Install OpenJDK 11 + - name: Setup Environment + shell: bash run: | - source .ci/inc/logger.inc - source .ci/inc/install-openjdk.inc - install_openjdk 11 + echo "LANG=en_US.UTF-8" >> $GITHUB_ENV + echo "MAVEN_OPTS=-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -DstagingProgressTimeoutMinutes=30" >> $GITHUB_ENV + echo "PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/master/scripts" >> $GITHUB_ENV + - name: Check Environment + shell: bash + run: | + f=check-environment.sh; \ + mkdir -p .ci && \ + ( [ -e .ci/$f ] || curl -sSL "${PMD_CI_SCRIPTS_URL}/$f" > ".ci/$f" ) && \ + chmod 755 .ci/$f && \ + .ci/$f + - name: Build + run: | + f=openjdk.bash; \ + mkdir -p .ci/inc && \ + ( [ -e .ci/inc/$f ] || curl -sSL "${PMD_CI_SCRIPTS_URL}/inc/$f" > ".ci/inc/$f" ) && \ + source .ci/inc/$f ; \ + pmd_ci_openjdk_install_adoptopenjdk 11 ; \ + pmd_ci_openjdk_setdefault 11 shell: bash - name: Setup tmate session uses: mxschmitt/action-tmate@v3 diff --git a/BUILDING.md b/BUILDING.md index bfa7c16245..8654ec8953 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -25,12 +25,12 @@ limitations apply: * Generally give **different results on Windows and Unix** because of different newlines. (carriage return linefeed on Windows, linefeed on Unixes). - We build our releases under **Linux** on [Travis CI](https://travis-ci.com/pmd/pmd). + We build our releases under **Linux** on [Github Actions](https://github.com/pmd/pmd/actions). * Generally depend on the **major version of the JDK** used to compile. (Even with source/target defined, each major JDK version changes the generated bytecode.). - We build our releases using OpenJDK 11 (see "before_install" step in `.travis.yml`). + We build our releases using OpenJDK 11. ## How to build the documentation? diff --git a/Gemfile b/Gemfile index 67b22b628d..6d66b0fb74 100644 --- a/Gemfile +++ b/Gemfile @@ -7,8 +7,8 @@ gem 'pmdtester', '~> 1' gem 'danger', '~> 5.6', '>= 5.6' # This group is only needed for rendering release notes -# this happens during release (.travis/release.sh and do-release.sh) -# but also during regular builds (.travis/build-deploy.sh) +# this happens during release (.ci/build.sh and do-release.sh) +# but also during regular builds (.ci/build.sh) group :release_notes_preprocessing do gem 'liquid', '>=4.0.0' gem 'safe_yaml', '>=1.0' diff --git a/README.md b/README.md index f639615e8a..0ee521b467 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # PMD [![Join the chat at https://gitter.im/pmd/pmd](https://badges.gitter.im/pmd/pmd.svg)](https://gitter.im/pmd/pmd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![Build Status](https://github.com/pmd/pmd/workflows/Pushes/badge.svg?branch=master)](https://github.com/pmd/pmd/actions) +[![Build Status](https://github.com/pmd/pmd/workflows/build/badge.svg?branch=master)](https://github.com/pmd/pmd/actions) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/net.sourceforge.pmd/pmd/badge.svg)](https://maven-badges.herokuapp.com/maven-central/net.sourceforge.pmd/pmd) [![Reproducible Builds](https://img.shields.io/badge/Reproducible_Builds-ok-green?labelColor=blue)](https://github.com/jvm-repo-rebuild/reproducible-central#net.sourceforge.pmd:pmd) [![Coverage Status](https://coveralls.io/repos/github/pmd/pmd/badge.svg)](https://coveralls.io/github/pmd/pmd) From 640b3fcab3c26458e189e49429637be5a7590dd4 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 15 Apr 2021 16:44:17 +0200 Subject: [PATCH 04/26] [ci] Fix shellcheck issues --- .ci/build.sh | 28 +++++++------ .ci/inc/pmd-code-api.inc | 74 +++++++++++++++++++---------------- .ci/inc/pmd-doc.inc | 24 ++++++------ .ci/inc/regression-tester.inc | 2 +- 4 files changed, 68 insertions(+), 60 deletions(-) diff --git a/.ci/build.sh b/.ci/build.sh index 043b7ab6c8..e97f909715 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -12,11 +12,11 @@ function build() { pmd_ci_log_group_start "Prepare Java 7+11, Bundler" pmd_ci_openjdk_install_adoptopenjdk 11 pmd_ci_openjdk_setdefault 11 - PMD_MAVEN_EXTRA_OPTS="" + PMD_MAVEN_EXTRA_OPTS=() if [ "$(pmd_ci_utils_get_os)" = "linux" ]; then - log_info "Install openjdk7 for integration tests" + pmd_ci_log_info "Install openjdk7 for integration tests" pmd_ci_openjdk_install_zuluopenjdk 7 - PMD_MAVEN_EXTRA_OPTS="-Djava7.home=${HOME}/oraclejdk7" + PMD_MAVEN_EXTRA_OPTS=(-Djava7.home="${HOME}/oraclejdk7") fi pmd_ci_build_setup_bundler pmd_ci_log_group_end @@ -28,7 +28,7 @@ function build() { if pmd_ci_utils_is_fork_or_pull_request; then pmd_ci_log_group_start "Build with mvnw" - ./mvnw clean verify --show-version --errors --batch-mode --no-transfer-progress ${PMD_MAVEN_EXTRA_OPTS} + ./mvnw clean verify --show-version --errors --batch-mode --no-transfer-progress "${PMD_MAVEN_EXTRA_OPTS[@]}" pmd_ci_log_group_end # Danger is executed only on the linux runner @@ -45,7 +45,7 @@ function build() { if [ "$(pmd_ci_utils_get_os)" != "linux" ]; then pmd_ci_log_group_start "Build with mvnw" - ./mvnw clean verify --show-version --errors --batch-mode --no-transfer-progress ${PMD_MAVEN_EXTRA_OPTS} + ./mvnw clean verify --show-version --errors --batch-mode --no-transfer-progress "${PMD_MAVEN_EXTRA_OPTS[@]}" pmd_ci_log_group_end pmd_ci_log_info "Stopping build here, because os is not linux" @@ -91,7 +91,7 @@ function build() { -Dpmd.skip \ --show-version --errors --batch-mode --no-transfer-progress \ clean package \ - sonar:sonar -Dsonar.login=${SONAR_TOKEN} -Psonar + sonar:sonar -Dsonar.login="${SONAR_TOKEN}" -Psonar pmd_ci_log_success "New sonar results: https://sonarcloud.io/dashboard?id=net.sourceforge.pmd%3Apmd" pmd_ci_log_group_end @@ -104,7 +104,7 @@ function build() { -Dmaven.source.skip \ -Dcheckstyle.skip \ -Dpmd.skip \ - -DrepoToken=${COVERALLS_REPO_TOKEN} \ + -DrepoToken="${COVERALLS_REPO_TOKEN}" \ --show-version --errors --batch-mode --no-transfer-progress \ clean package jacoco:report \ coveralls:report -Pcoveralls @@ -118,7 +118,7 @@ function build() { # Installs bundler, which is needed for doc generation and regression tester # function pmd_ci_build_setup_bundler() { - log_info "Installing bundler..." + pmd_ci_log_info "Installing bundler..." gem install bundler } @@ -137,7 +137,7 @@ function pmd_ci_build_run() { pmd_ci_log_info "This is a snapshot build" fi - ./mvnw clean deploy -P${mvn_profiles} --show-version --errors --batch-mode --no-transfer-progress ${PMD_MAVEN_EXTRA_OPTS} + ./mvnw clean deploy -P${mvn_profiles} --show-version --errors --batch-mode --no-transfer-progress "${PMD_MAVEN_EXTRA_OPTS[@]}" } # @@ -150,7 +150,7 @@ function pmd_ci_deploy_build_artifacts() { if pmd_ci_maven_isReleaseBuild; then # create a draft github release - pmd_ci_gh_releases_createDraftRelease "${PMD_CI_TAG}" "$(git rev-list -n 1 ${PMD_CI_TAG})" + pmd_ci_gh_releases_createDraftRelease "${PMD_CI_TAG}" "$(git rev-list -n 1 "${PMD_CI_TAG}")" GH_RELEASE="$RESULT" # Deploy to github releases @@ -176,7 +176,7 @@ function pmd_ci_build_and_upload_doc() { # Deploy javadoc to https://docs.pmd-code.org/apidocs/*/${PMD_CI_MAVEN_PROJECT_VERSION}/ pmd_code_uploadJavadoc "${PMD_CI_MAVEN_PROJECT_VERSION}" "$(pwd)" - if [ pmd_ci_maven_isSnapshotBuild && "${PMD_CI_BRANCH}" = "master" ]; then + if pmd_ci_maven_isSnapshotBuild && [ "${PMD_CI_BRANCH}" = "master" ]; then # only for snapshot builds from branch master pmd_code_createSymlink "${PMD_CI_MAVEN_PROJECT_VERSION}" "snapshot" @@ -202,8 +202,10 @@ function pmd_ci_build_and_upload_doc() { bundle config set --local with release_notes_preprocessing bundle install # renders, and skips the first 6 lines - the Jekyll front-matter - local rendered_release_notes=$(bundle exec .ci/render_release_notes.rb docs/pages/release_notes.md | tail -n +6) - local release_name="PMD ${PMD_CI_MAVEN_PROJECT_VERSION} ($(date -u +%d-%B-%Y))" + local rendered_release_notes + rendered_release_notes=$(bundle exec .ci/render_release_notes.rb docs/pages/release_notes.md | tail -n +6) + local release_name + release_name="PMD ${PMD_CI_MAVEN_PROJECT_VERSION} ($(date -u +%d-%B-%Y))" pmd_ci_gh_releases_updateRelease "$GH_RELEASE" "$release_name" "$rendered_release_notes" pmd_ci_sourceforge_uploadReleaseNotes "${PMD_CI_MAVEN_PROJECT_VERSION}" "${rendered_release_notes}" diff --git a/.ci/inc/pmd-code-api.inc b/.ci/inc/pmd-code-api.inc index 122f7be59b..913e5919d7 100644 --- a/.ci/inc/pmd-code-api.inc +++ b/.ci/inc/pmd-code-api.inc @@ -9,78 +9,84 @@ PMD_CODE_SSH_USER=pmd PMD_CODE_DOCS_PATH=/docs.pmd-code.org/ function pmd_code_uploadDocumentation() { - local pmdVersion="$1" - local filename="$2" - local basefilename="$(basename $filename)" + local -r pmdVersion="$1" + local -r filename="$2" + local -r basefilename="$(basename "$filename")" - pmd_ci_log_debug "$FUNCNAME pmdVersion=$pmdVersion filename=$filename" + pmd_ci_log_debug "${FUNCNAME[0]} pmdVersion=$pmdVersion filename=$filename" scp "${filename}" ${PMD_CODE_SSH_USER}@pmd-code.org:${PMD_CODE_DOCS_PATH} - ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ - unzip -qo ${basefilename} && \ - rm ${basefilename}" + # shellcheck disable=SC2029 + ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd \"${PMD_CODE_DOCS_PATH}\" && \ + unzip -qo \"${basefilename}\" && \ + rm \"${basefilename}\"" pmd_ci_log_info "Docs updated: https://docs.pmd-code.org/pmd-doc-${pmdVersion}/" } function pmd_code_removeDocumentation() { local pmdVersion="$1" - pmd_ci_log_debug "$FUNCNAME pmdVersion=$pmdVersion" + pmd_ci_log_debug "${FUNCNAME[0]} pmdVersion=$pmdVersion" - ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ - rm -rf pmd-doc-${pmdVersion}/" + # shellcheck disable=SC2029 + ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd \"${PMD_CODE_DOCS_PATH}\" && \ + rm -rf \"pmd-doc-${pmdVersion}/\"" pmd_ci_log_info "Removed docs: https://docs.pmd-code.org/pmd-doc-${pmdVersion}/" } function pmd_code_createSymlink() { - local pmdVersion="$1" - local name="$2" + local -r pmdVersion="$1" + local -r name="$2" - pmd_ci_log_debug "$FUNCNAME pmdVersion=$pmdVersion name=$name" + pmd_ci_log_debug "${FUNCNAME[0]} pmdVersion=$pmdVersion name=$name" - ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ - rm -f $name && \ - ln -s pmd-doc-${pmdVersion} $name" + # shellcheck disable=SC2029 + ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd \"${PMD_CODE_DOCS_PATH}\" && \ + rm -f \"$name\" && \ + ln -s \"pmd-doc-${pmdVersion}\" \"$name\"" pmd_ci_log_info "Symlink created: https://docs.pmd-code.org/$name/ -> https://docs.pmd-code.org/pmd-doc-${pmdVersion}/" } function pmd_code_uploadJavadoc() { - local pmdVersion="$1" - local basePath="$2" + local -r pmdVersion="$1" + local -r basePath="$2" - pmd_ci_log_debug "$FUNCNAME pmdVersion=$pmdVersion basePath=$basePath" + pmd_ci_log_debug "${FUNCNAME[0]} pmdVersion=$pmdVersion basePath=$basePath" - for i in ${basePath}/*/target/*-javadoc.jar */*/target/*-javadoc.jar; do + for i in "${basePath}"/*/target/*-javadoc.jar */*/target/*-javadoc.jar; do pmd_code_uploadJavadocModule "$pmdVersion" "$i" done # make sure https://docs.pmd-code.org/apidocs/ shows directory index - ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH}/apidocs && \ + # shellcheck disable=SC2029 + ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd \"${PMD_CODE_DOCS_PATH}/apidocs\" && \ echo 'Options +Indexes' > .htaccess" pmd_ci_log_info "Directory index enabled for https://docs.pmd-code.org/apidocs/" } function pmd_code_uploadJavadocModule() { - local pmdVersion="$1" - local moduleJavadocJar="$2" - local moduleJavadocJarBasename="$(basename $moduleJavadocJar)" - local module=${moduleJavadocJarBasename%%-${pmdVersion}-javadoc.jar} + local -r pmdVersion="$1" + local -r moduleJavadocJar="$2" + local -r moduleJavadocJarBasename="$(basename "$moduleJavadocJar")" + local -r module=${moduleJavadocJarBasename%%-${pmdVersion}-javadoc.jar} - pmd_ci_log_debug "$FUNCNAME pmdVersion=$pmdVersion moduleJavadocJar=$moduleJavadocJar module=$module" + pmd_ci_log_debug "${FUNCNAME[0]} pmdVersion=$pmdVersion moduleJavadocJar=$moduleJavadocJar module=$module" scp "$moduleJavadocJar" ${PMD_CODE_SSH_USER}@pmd-code.org:${PMD_CODE_DOCS_PATH} - ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ - mkdir -p apidocs/${module}/${pmdVersion} && \ - unzip -qo -d apidocs/${module}/${pmdVersion} ${moduleJavadocJarBasename} && \ - rm ${moduleJavadocJarBasename}" + # shellcheck disable=SC2029 + ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd \"${PMD_CODE_DOCS_PATH}\" && \ + mkdir -p \"apidocs/${module}/${pmdVersion}\" && \ + unzip -qo -d \"apidocs/${module}/${pmdVersion}\" \"${moduleJavadocJarBasename}\" && \ + rm \"${moduleJavadocJarBasename}\"" pmd_ci_log_info "JavaDoc for $module uploaded: https://docs.pmd-code.org/apidocs/${module}/${pmdVersion}/" } function pmd_code_removeJavadoc() { - local pmdVersion="$1" + local -r pmdVersion="$1" - pmd_ci_log_debug "$FUNCNAME pmdVersion=$pmdVersion" - ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ - rm -rf apidocs/*/${pmdVersion}" + pmd_ci_log_debug "${FUNCNAME[0]} pmdVersion=$pmdVersion" + # shellcheck disable=SC2029 + ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd \"${PMD_CODE_DOCS_PATH}\" && \ + rm -rf apidocs/*/\"${pmdVersion}\"" pmd_ci_log_info "Removed Javadoc: https://docs.pmd-code.org/apidocs/*/${pmdVersion}/ is gone" } diff --git a/.ci/inc/pmd-doc.inc b/.ci/inc/pmd-doc.inc index 370b22d3fe..aa0fa1e045 100644 --- a/.ci/inc/pmd-doc.inc +++ b/.ci/inc/pmd-doc.inc @@ -15,7 +15,7 @@ source "$(dirname "$0")/inc/fetch_ci_scripts.bash" && fetch_ci_scripts # The documentation will be generated in the directory "docs/_site". # function pmd_doc_generate_jekyll_site() { - pushd docs + pushd docs || { echo "Directory 'docs' doesn't exist"; exit 1; } echo -e "\n\n" pmd_ci_log_info "Building documentation using jekyll..." @@ -23,22 +23,22 @@ function pmd_doc_generate_jekyll_site() { bundle install bundle exec jekyll build - popd + popd || exit 1 } # # Creates the pmd-doc.zip archive. It will be placed in "docs/". # function pmd_doc_create_archive() { - pushd docs + pushd docs || { echo "Directory 'docs' doesn't exist"; exit 1; } echo -e "\n\n" pmd_ci_log_info "Creating pmd-doc archive..." - mv _site pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION} - zip -qr pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}.zip pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}/ + mv _site "pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}" + zip -qr "pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}.zip" "pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}/" pmd_ci_log_success "Successfully created pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}.zip" - popd + popd || exit 1 } # @@ -52,7 +52,7 @@ function publish_release_documentation_github() { # In order to speed things up, we use a sparse checkout - no need to checkout all directories here mkdir pmd.github.io ( - cd pmd.github.io + cd pmd.github.io || { echo "Directory 'pmd.github.io' doesn't exist"; exit 1; } git init git config user.name "PMD CI (pmd-bot)" git config user.email "andreas.dangel+pmd-bot@adangel.org" @@ -62,16 +62,16 @@ function publish_release_documentation_github() { echo "/sitemap.xml" >> .git/info/sparse-checkout git pull --depth=1 origin master pmd_ci_log_info "Copying documentation from ../docs/pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}/ to pmd-${PMD_CI_MAVEN_PROJECT_VERSION}/ ..." - rsync -ah --stats ../docs/pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}/ pmd-${PMD_CI_MAVEN_PROJECT_VERSION}/ + rsync -ah --stats "../docs/pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}/" "pmd-${PMD_CI_MAVEN_PROJECT_VERSION}/" git status pmd_ci_log_debug "Executing: git add pmd-${PMD_CI_MAVEN_PROJECT_VERSION}" - git add pmd-${PMD_CI_MAVEN_PROJECT_VERSION} + git add "pmd-${PMD_CI_MAVEN_PROJECT_VERSION}" pmd_ci_log_debug "Executing: git commit..." git commit -q -m "Added pmd-${PMD_CI_MAVEN_PROJECT_VERSION}" pmd_ci_log_info "Copying pmd-${PMD_CI_MAVEN_PROJECT_VERSION} to latest ..." git rm -qr latest - cp -a pmd-${PMD_CI_MAVEN_PROJECT_VERSION} latest + cp -a "pmd-${PMD_CI_MAVEN_PROJECT_VERSION}" latest pmd_ci_log_debug "Executing: git add latest" git add latest pmd_ci_log_debug "Executing: git commit..." @@ -100,9 +100,9 @@ function pmd_doc_publish_to_github_pages() { # clear the files first rm -rf pmd-gh-pages/* # copy the new site - cp -a docs/pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}/* pmd-gh-pages/ + cp -a "docs/pmd-doc-${PMD_CI_MAVEN_PROJECT_VERSION}"/* pmd-gh-pages/ ( - cd pmd-gh-pages + cd pmd-gh-pages || { echo "Directory 'pmd-gh-pages' doesn't exist"; exit 1; } git config user.name "PMD CI (pmd-bot)" git config user.email "andreas.dangel+pmd-bot@adangel.org" git add -A diff --git a/.ci/inc/regression-tester.inc b/.ci/inc/regression-tester.inc index 65d6e1e6b2..127713cf71 100644 --- a/.ci/inc/regression-tester.inc +++ b/.ci/inc/regression-tester.inc @@ -74,7 +74,7 @@ function regression_tester_executeDanger() { if ! git show-ref --verify --quiet refs/heads/${PMD_CI_BRANCH}; then git fetch --no-tags --depth=1 origin +refs/heads/${PMD_CI_BRANCH}:refs/remotes/origin/${PMD_CI_BRANCH} git branch ${PMD_CI_BRANCH} origin/${PMD_CI_BRANCH} - log_debug "Created local branch ${PMD_CI_BRANCH}" + pmd_ci_log_debug "Created local branch ${PMD_CI_BRANCH}" fi # Fetch more commits of the PR for danger and regression tester git fetch --no-tags --depth=50 origin +$(git rev-parse HEAD^2): From c384bb8732338dbb592974f5dd2d854b6793c158 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 15 Apr 2021 18:16:32 +0200 Subject: [PATCH 05/26] [ci] Fix shellcheck issues --- .ci/inc/regression-tester.inc | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.ci/inc/regression-tester.inc b/.ci/inc/regression-tester.inc index 127713cf71..093e096e95 100644 --- a/.ci/inc/regression-tester.inc +++ b/.ci/inc/regression-tester.inc @@ -19,6 +19,7 @@ function regression_tester_setup_ci() { gpg --batch --yes --decrypt --passphrase="GnxdjywUEPveyCD1RLiTd7t8CImnefYr" \ --output .ci/files/public-env .ci/files/public-env.gpg + # shellcheck disable=SC1091 source .ci/files/public-env >/dev/null 2>&1 rm .ci/files/public-env @@ -41,7 +42,7 @@ function regression_tester_setup_ci() { function regression_tester_uploadBaseline() { local pmdcodeUrl="https://pmd-code.org/pmd-regression-tester/" local baseline_branch="${PMD_CI_BRANCH:-$PMD_CI_TAG}" - pmd_ci_log_debug "$FUNCNAME branch=${baseline_branch}" + pmd_ci_log_debug "${FUNCNAME[0]} branch=${baseline_branch}" pmd_ci_log_info "Generating and uploading baseline for pmdtester (${baseline_branch})..." pushd .. @@ -50,34 +51,34 @@ function regression_tester_uploadBaseline() { bundle exec pmdtester \ --mode single \ --local-git-repo ./pmd \ - --patch-branch ${baseline_branch} \ + --patch-branch "${baseline_branch}" \ --patch-config ./pmd/.ci/files/all-java.xml \ --list-of-project ./pmd/.ci/files/project-list.xml --html-flag \ --error-recovery - pushd target/reports + pushd target/reports || { echo "Directory 'target/reports' doesn't exist"; exit 1; } BRANCH_FILENAME="${baseline_branch/\//_}" - zip -q -r ${BRANCH_FILENAME}-baseline.zip ${BRANCH_FILENAME}/ + zip -q -r "${BRANCH_FILENAME}-baseline.zip" "${BRANCH_FILENAME}/" # ssh-key for pmd-code.org is setup already by pmd_ci_setup_secrets_ssh - scp ${BRANCH_FILENAME}-baseline.zip pmd@pmd-code.org:/httpdocs/pmd-regression-tester/ + scp "${BRANCH_FILENAME}-baseline.zip" pmd@pmd-code.org:/httpdocs/pmd-regression-tester/ pmd_ci_log_success "Successfully uploaded ${BRANCH_FILENAME}-baseline.zip to ${pmdcodeUrl}" - popd - popd + popd || exit 1 + popd || exit 1 } # # Execute danger, which executes pmd-regression-tester (via Dangerfile). # function regression_tester_executeDanger() { - pmd_ci_log_debug "$FUNCNAME" + pmd_ci_log_debug "${FUNCNAME[0]}" # Create a corresponding remote branch locally - if ! git show-ref --verify --quiet refs/heads/${PMD_CI_BRANCH}; then - git fetch --no-tags --depth=1 origin +refs/heads/${PMD_CI_BRANCH}:refs/remotes/origin/${PMD_CI_BRANCH} - git branch ${PMD_CI_BRANCH} origin/${PMD_CI_BRANCH} + if ! git show-ref --verify --quiet "refs/heads/${PMD_CI_BRANCH}"; then + git fetch --no-tags --depth=1 origin "+refs/heads/${PMD_CI_BRANCH}:refs/remotes/origin/${PMD_CI_BRANCH}" + git branch "${PMD_CI_BRANCH}" "origin/${PMD_CI_BRANCH}" pmd_ci_log_debug "Created local branch ${PMD_CI_BRANCH}" fi # Fetch more commits of the PR for danger and regression tester - git fetch --no-tags --depth=50 origin +$(git rev-parse HEAD^2): + git fetch --no-tags --depth=50 origin "+$(git rev-parse HEAD^2):" # Fetch more commits from master branch for regression tester if [[ "${PMD_CI_BRANCH}" != "master" ]]; then git fetch --no-tags --depth=50 origin +master: From b896aacabf01ddb3bc9d8fa64643e063c2821145 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 15 Apr 2021 18:19:47 +0200 Subject: [PATCH 06/26] [ci] Move render_release_notes.rb to docs --- .ci/build.sh | 2 +- Gemfile | 2 +- do-release.sh | 2 +- docs/_config.yml | 2 ++ {.ci => docs}/render_release_notes.rb | 0 5 files changed, 5 insertions(+), 3 deletions(-) rename {.ci => docs}/render_release_notes.rb (100%) diff --git a/.ci/build.sh b/.ci/build.sh index e97f909715..7a7962befc 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -203,7 +203,7 @@ function pmd_ci_build_and_upload_doc() { bundle install # renders, and skips the first 6 lines - the Jekyll front-matter local rendered_release_notes - rendered_release_notes=$(bundle exec .ci/render_release_notes.rb docs/pages/release_notes.md | tail -n +6) + rendered_release_notes=$(bundle exec docs/render_release_notes.rb docs/pages/release_notes.md | tail -n +6) local release_name release_name="PMD ${PMD_CI_MAVEN_PROJECT_VERSION} ($(date -u +%d-%B-%Y))" pmd_ci_gh_releases_updateRelease "$GH_RELEASE" "$release_name" "$rendered_release_notes" diff --git a/Gemfile b/Gemfile index 6d66b0fb74..57359b1785 100644 --- a/Gemfile +++ b/Gemfile @@ -6,7 +6,7 @@ source 'https://rubygems.org/' gem 'pmdtester', '~> 1' gem 'danger', '~> 5.6', '>= 5.6' -# This group is only needed for rendering release notes +# This group is only needed for rendering release notes (docs/render_release_notes.rb) # this happens during release (.ci/build.sh and do-release.sh) # but also during regular builds (.ci/build.sh) group :release_notes_preprocessing do diff --git a/do-release.sh b/do-release.sh index bff8d1f6aa..a191f84799 100755 --- a/do-release.sh +++ b/do-release.sh @@ -127,7 +127,7 @@ bundle install export RELEASE_NOTES_POST="_posts/$(date -u +%Y-%m-%d)-PMD-${RELEASE_VERSION}.md" echo "Generating ../pmd.github.io/${RELEASE_NOTES_POST}..." -NEW_RELEASE_NOTES=$(bundle exec .ci/render_release_notes.rb docs/pages/release_notes.md | tail -n +6) +NEW_RELEASE_NOTES=$(bundle exec docs/render_release_notes.rb docs/pages/release_notes.md | tail -n +6) cat > ../pmd.github.io/${RELEASE_NOTES_POST} < Date: Thu, 15 Apr 2021 18:47:44 +0200 Subject: [PATCH 07/26] [ci] Update doc about pmd build --- .ci/README.md | 65 ++++++++++++---------------------- .ci/files/public-env.gpg | Bin 365 -> 630 bytes .ci/inc/regression-tester.inc | 1 - 3 files changed, 23 insertions(+), 43 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index c3711495ca..c541ba8c3e 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -1,31 +1,28 @@ -## PMD CI Scripts +# PMD CI Scripts -This folder contains scripts used for CI. +This folder contains scripts used for CI, that are PMD specific. +It uses the common scripts from [build-tools](https://github.com/pmd/build-tools). -## Secrets +## .ci/files/public-env.gpg -One secret is required for decrypting the GPG Key with which the PMD Releases are signed and -for a ssh key, which is used to copy files to sourceforge. +This files contains the following environment variables: -## Environment variables +* DANGER_GITHUB_API_TOKEN: Token for danger to add comments to PRs as +* PMD_CI_CHUNK_TOKEN: Token for uploading reports to chunk.io -* PMD_CI_SECRET_PASSPHRASE -* CI_DEPLOY_USER -* CI_DEPLOY_PASSWORD -* CI_SIGN_KEY -* CI_SIGN_PASSPHRASE -* PMD_SF_USER -* PMD_SF_APIKEY -* GITHUB_OAUTH_TOKEN -* GITHUB_BASE_URL -* COVERALLS_REPO_TOKEN -* SONAR_TOKEN -* DANGER_GITHUB_API_TOKEN -* PMD_CI_CHUNK_TOKEN +The file is encrypted, so that the tokens are not automatically disabled when github detects them +in clear text. -## Encrypting +**Decrypting**: - gpg --batch --symmetric --cipher-algo AES256 --passphrase="$PMD_CI_SECRET_PASSPHRASE" file.txt + gpg --batch --yes --decrypt --passphrase="GnxdjywUEPveyCD1RLiTd7t8CImnefYr" \ + --output .ci/files/public-env .ci/files/public-env.gpg + +**Encrypting**: + + gpg --batch --symmetric --cipher-algo AES256 \ + --armor --passphrase="GnxdjywUEPveyCD1RLiTd7t8CImnefYr" \ + --output .ci/files/public-env.gpg .ci/files/public-env ## Known Issues @@ -40,7 +37,7 @@ and [WAGON-486](https://issues.apache.org/jira/browse/WAGON-486): The setting `-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3` makes sure, that Maven doesn't try to use pooled connections that have been unused for more than 180 seconds. -These settings are placed as environment variable `MAVEN_OPTS` in all workflows, so that they are active for +These settings are placed as environment variable `MAVEN_OPTS` in the workflow, so that they are active for all Maven executions (including builds done by regression tester). Alternatively, pooling could be disabled completely via `-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false`. @@ -49,9 +46,7 @@ established. More information about configuring this can be found at [wagon-http](https://maven.apache.org/wagon/wagon-providers/wagon-http/). -## Hints - -### Remote debugging +## Remote debugging Debugging remotely is possible with . @@ -67,27 +62,13 @@ The workflow `troubleshooting` can be started manually, which already contains t **Note**: This is dangerous for push/pull builds on pmd/pmd, because these have access to the secrets and the SSH session is not protected. Builds triggered by pull requests from forked repositories don't have access to the secrets. -### Local tests with docker +## Local tests with docker -Create a local docker container: +Using the same docker container as described in [build-env @ build-tools](https://github.com/pmd/build-tools). -``` -cd .ci/docker_ubuntu18.04 -docker build -t pmd-ci . -``` +### Testing a push build (snapshot) -This container is based on Ubuntu 18.04, which is used for `ubuntu-latest` github actions runner, -see [Virtual Environment](https://github.com/actions/virtual-environments). -You can run a local instance with docker: - -``` -docker run -it pmd-ci -``` - -You'll be dropped into a bash. - -#### Testing a push build (snapshot) Start docker without binding to local directory, so that we can do a fresh checkout: `docker run -it pmd-ci`. You'll be dropped into a bash. Use the following script, to setup and start the build: diff --git a/.ci/files/public-env.gpg b/.ci/files/public-env.gpg index 8cae0c6b6aeddac3ac14268b7bbecd73d3d891fa..4504a0ef26466ec3cdab5ea64dff77971a5ac03a 100644 GIT binary patch literal 630 zcmZXSx00hk00nctVsmN>5HLWn0p5&+Hxe0En2~T22)_RIF1-$4()FtP`}-G3)lK`$ z)!na)8U&)Q{vE$DtP@F9_AYY&D4qZ}*^yTWa!~>?0XBG|Hth$hcgg294))RwtN-`!LM`W!Lk- z`g(u8nLH-$SEA3OsJ=z`s2}~L3Z3h@eXK}plB8jA%pDjp zH%|7quWA_?mSb@mcMd1=fItgQSy43u{c~Gvgu^ynU{57 zIaazaY>J;3l9Fj|9!62-t?&r($hZ(I84uoLfQ|G4c-0vFbZkl$J@OdyEXKHKOQP?g zy8|YW(>!1@C&|1IIgz%eMI3yTZ^rlRiF||1QBpxb>RQTl!v;>IO=s>X-2D{Cy1|)95YFGO^(Q)pd-hzILUm;)? zVK|~P0wC+?$wctbmD6bN_2Vu;w(7pk5g3oU&x58DIM>;ufwS*#7~ILRZS>;2+$hQ} z{F+Odn%^!@_T5;3z_M{g!MV}^Sgl~&bR~38*)7C7oEu>Ml-VSQ{xJW=>_iw8{D%sm zhTnBDWK7W9_^HVcFEWAe+|X6SM-ym5h<8?9RLu+bDxC0w`H?3Mn1m(ybrjjrw?eW8 zIOAB7ISoIGH!B`aNHD+tsW0BNq)%Dp9+^C8aD5#g0Gk72g6Nf5&LnYIe< Date: Thu, 15 Apr 2021 19:29:22 +0200 Subject: [PATCH 08/26] [ci] Update docs for testing the builds --- .ci/README.md | 97 +++++++++++++++++++++------------------------------ 1 file changed, 39 insertions(+), 58 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index c541ba8c3e..68d552fab7 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -68,19 +68,25 @@ Using the same docker container as described in [build-env @ build-tools](https: ### Testing a push build (snapshot) +Start docker without binding to local directory, so that we can do a fresh checkout + $ docker run \ + --interactive \ + --tty \ + --name pmd-build-env_pmd \ + pmd-build-env:latest -Start docker without binding to local directory, so that we can do a fresh checkout: `docker run -it pmd-ci`. -You'll be dropped into a bash. Use the following script, to setup and start the build: ``` -MAIN_BRANCH="master" +export LANG=en_US.UTF-8 export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" -export PMD_CI_JOB_URL="manual job execution in docker" -export PMD_CI_PUSH_COMMIT_COMPARE="" -export PMD_CI_GIT_REF="refs/heads/${MAIN_BRANCH}" +export PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/master/scripts export PMD_CI_SECRET_PASSPHRASE="xyz" +export PMD_CI_DEBUG=true + +MAIN_BRANCH="master" +eval $(~/create-gh-actions-env.sh push pmd/pmd $MAIN_BRANCH) cd /workspaces/pmd rmdir pmd && mkdir pmd @@ -90,69 +96,44 @@ git remote add origin https://github.com/pmd/pmd git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/heads/${MAIN_BRANCH}:refs/remotes/origin/${MAIN_BRANCH} git checkout --progress --force -B ${MAIN_BRANCH} refs/remotes/origin/${MAIN_BRANCH} -.ci/check-environment.sh + +f=check-environment.sh; \ + mkdir -p .ci && \ + ( [ -e .ci/$f ] || curl -sSL "${PMD_CI_SCRIPTS_URL}/$f" > ".ci/$f" ) && \ + chmod 755 .ci/$f && \ + .ci/$f .ci/build.sh ``` -#### Performing a release (push) build +### Testing a pull request -Start docker without binding to local directory, so that we can do a fresh checkout: `docker run -it pmd-ci`. -You'll be dropped into a bash. Use the following script, to setup and start the build: +Same as the above, but this line changes: ``` -TAG_NAME="pmd_releases/0.0.0_release_test" -export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" -export PMD_CI_JOB_URL="manual job execution in docker" -export PMD_CI_PUSH_COMMIT_COMPARE="" -export PMD_CI_GIT_REF="refs/tags/${TAG_NAME}" - -export PMD_CI_SECRET_PASSPHRASE="xyz" - -cd /workspace/pmd -rmdir pmd && mkdir pmd -cd pmd -git init -git remote add origin https://github.com/pmd/pmd -git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/tags/${TAG_NAME}:refs/tags/${TAG_NAME} -git checkout --progress --force refs/tags/${TAG_NAME} - -.ci/check-environment.sh - -.ci/build.sh +eval $(~/create-gh-actions-env.sh pull_request adangel/pmd $MAIN_BRANCH) ``` -**Warning:** This will build and upload to maven central! +And the checkout could be different... +### Forked build -#### Testing a pull request - -Start docker without binding to local directory, so that we can do a fresh checkout: `docker run -it pmd-ci`. -You'll be dropped into a bash. Use the following script, to setup and start the build: +A build executing on a forked repository. ``` -export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" -export PMD_CI_BRANCH="master" # base branch of the pull request -export PMD_CI_PULL_REQUEST_NUMBER=2913 - -unset PMD_CI_SECRET_PASSPHRASE - -# these are used by danger -export GITHUB_EVENT_PATH=/workspaces/event.json -export GITHUB_REPOSITORY=pmd/pmd -export GITHUB_ACTION=run1 -export GITHUB_EVENT_NAME=pull_request -/home/pmd-ci/create-gh-pull-request-event.sh - -cd /workspace/pmd -rmdir pmd && mkdir pmd -cd pmd -git init -git remote add origin https://github.com/pmd/pmd -git fetch --no-tags --prune --progress --no-recurse-submodules --depth=2 origin +refs/pull/${PMD_CI_PULL_REQUEST_NUMBER}/merge:refs/remotes/pull/${PMD_CI_PULL_REQUEST_NUMBER}/merge -git checkout --progress --force refs/remotes/pull/${PMD_CI_PULL_REQUEST_NUMBER}/merge - -.ci/check-environment.sh - -.ci/build-pr-win-macos.sh +$(~/create-gh-actions-env.sh push adangel/pmd $MAIN_BRANCH) ``` + + +### Performing a release (push) build + +``` +eval $(~/create-gh-actions-env.sh push pmd/pmd refs/tags/v1.0.0_release_test) +``` + +Make sure, that `MAVEN_OPTS` contains `-DskipRemoteStaging=true`, so that no maven artifacts are not deployed +to maven central. + +And the checkout could be different... + + From 6af30afb01db5bf78dee18fb66fb0fbd6236a81d Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 16 Apr 2021 09:53:38 +0200 Subject: [PATCH 09/26] [ci] Fix java7.home path --- .ci/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/build.sh b/.ci/build.sh index 7a7962befc..b6af53af70 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -16,7 +16,7 @@ function build() { if [ "$(pmd_ci_utils_get_os)" = "linux" ]; then pmd_ci_log_info "Install openjdk7 for integration tests" pmd_ci_openjdk_install_zuluopenjdk 7 - PMD_MAVEN_EXTRA_OPTS=(-Djava7.home="${HOME}/oraclejdk7") + PMD_MAVEN_EXTRA_OPTS=(-Djava7.home="${HOME}/openjdk7") fi pmd_ci_build_setup_bundler pmd_ci_log_group_end From f2e20ec96317926ecf7cc85bf723f4a6bea9dc8e Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 16 Apr 2021 15:56:36 +0200 Subject: [PATCH 10/26] [ci] Update docs for testing the builds --- .ci/README.md | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index 68d552fab7..4f3e986443 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -111,10 +111,24 @@ f=check-environment.sh; \ Same as the above, but this line changes: ``` -eval $(~/create-gh-actions-env.sh pull_request adangel/pmd $MAIN_BRANCH) +eval $(~/create-gh-actions-env.sh pull_request pmd/pmd $MAIN_BRANCH) ``` -And the checkout could be different... +Maybe update `/workspaces/event.json` to fill in a real pull request number, so that +danger can comment the correct PR. + +And the checkout must be different. Example for PR 3220: + +``` +PMD_CI_PULL_REQUEST_NUMBER=3220 +cd /workspace/pmd +rmdir pmd && mkdir pmd +cd pmd +git init +git remote add origin https://github.com/pmd/pmd +git fetch --no-tags --prune --progress --no-recurse-submodules --depth=2 origin +refs/pull/${PMD_CI_PULL_REQUEST_NUMBER}/merge:refs/remotes/pull/${PMD_CI_PULL_REQUEST_NUMBER}/merge +git checkout --progress --force refs/remotes/pull/${PMD_CI_PULL_REQUEST_NUMBER}/merge +``` ### Forked build @@ -131,8 +145,8 @@ $(~/create-gh-actions-env.sh push adangel/pmd $MAIN_BRANCH) eval $(~/create-gh-actions-env.sh push pmd/pmd refs/tags/v1.0.0_release_test) ``` -Make sure, that `MAVEN_OPTS` contains `-DskipRemoteStaging=true`, so that no maven artifacts are not deployed -to maven central. +Make sure, that `MAVEN_OPTS` contains `-DskipRemoteStaging=true`, so that no maven artifacts are deployed +to maven central (this is set by `create-gh-actions-env.sh`). And the checkout could be different... From 5fd66c0e77005972c5391d0ad613bacf371d7b5f Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 16 Apr 2021 15:59:57 +0200 Subject: [PATCH 11/26] [ci] Remove old docker config The docker from pmd/build-tools should be used instead. --- .ci/docker_ubuntu18.04/Dockerfile | 31 ------------------- .../create-gh-pull-request-event.sh | 10 ------ .ci/docker_ubuntu18.04/install-ruby.sh | 8 ----- 3 files changed, 49 deletions(-) delete mode 100644 .ci/docker_ubuntu18.04/Dockerfile delete mode 100755 .ci/docker_ubuntu18.04/create-gh-pull-request-event.sh delete mode 100755 .ci/docker_ubuntu18.04/install-ruby.sh diff --git a/.ci/docker_ubuntu18.04/Dockerfile b/.ci/docker_ubuntu18.04/Dockerfile deleted file mode 100644 index 8317beb799..0000000000 --- a/.ci/docker_ubuntu18.04/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -# https://hub.docker.com/_/ubuntu/ -FROM ubuntu:18.04 - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get upgrade --yes && \ - apt-get install --yes curl jq gpg rsync ssh git p7zip-full openjdk-8-jdk \ - libgdbm-dev libncurses5-dev automake libtool bison libffi-dev \ - sudo nano bash tzdata unzip zip && \ - apt-get clean -RUN cd opt && \ - curl https://mirror.checkdomain.de/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz | tar xz && \ - ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/local/bin/mvn && \ - cd .. -RUN groupadd --gid 1000 pmd-ci && useradd --gid 1000 --uid 1000 --groups sudo \ - --shell /bin/bash --create-home --password "" \ - pmd-ci - -RUN mkdir -p /workspaces/pmd/pmd && chown -R pmd-ci:pmd-ci /workspaces && ln -sf /workspaces /home/pmd-ci/workspaces - -USER pmd-ci:pmd-ci -WORKDIR /home/pmd-ci -COPY create-gh-pull-request-event.sh . -COPY install-ruby.sh . -RUN ./install-ruby.sh - -CMD ["/bin/bash", "--login"] - -# -# build with: docker build -t pmd-ci . -# run with: docker run -it pmd-ci -# \ No newline at end of file diff --git a/.ci/docker_ubuntu18.04/create-gh-pull-request-event.sh b/.ci/docker_ubuntu18.04/create-gh-pull-request-event.sh deleted file mode 100755 index 7f3576196e..0000000000 --- a/.ci/docker_ubuntu18.04/create-gh-pull-request-event.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -cat > /workspaces/event.json < Date: Fri, 16 Apr 2021 16:00:59 +0200 Subject: [PATCH 12/26] [ci] Remove unnecessary MAVEN_OPTS --- .ci/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.ci/README.md b/.ci/README.md index 4f3e986443..88b9183424 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -79,7 +79,6 @@ Start docker without binding to local directory, so that we can do a fresh check ``` export LANG=en_US.UTF-8 -export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" export PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/master/scripts export PMD_CI_SECRET_PASSPHRASE="xyz" From ff68dd7bef8fb7911f390ebc64769993b915e4ea Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 16 Apr 2021 16:35:52 +0200 Subject: [PATCH 13/26] [ci] Enable debug mode for check-environment.sh --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dc43e3bf58..68e4463434 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,7 +48,7 @@ jobs: mkdir -p .ci && \ ( [ -e .ci/$f ] || curl -sSL "${PMD_CI_SCRIPTS_URL}/$f" > ".ci/$f" ) && \ chmod 755 .ci/$f && \ - .ci/$f + bash -ex .ci/$f - name: Build run: .ci/build.sh shell: bash From 081edd5213446e3abd8a19cc08061e108e04c7f0 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 16 Apr 2021 16:43:28 +0200 Subject: [PATCH 14/26] Revert "[ci] Enable debug mode for check-environment.sh" This reverts commit ff68dd7bef8fb7911f390ebc64769993b915e4ea. --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 68e4463434..dc43e3bf58 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,7 +48,7 @@ jobs: mkdir -p .ci && \ ( [ -e .ci/$f ] || curl -sSL "${PMD_CI_SCRIPTS_URL}/$f" > ".ci/$f" ) && \ chmod 755 .ci/$f && \ - bash -ex .ci/$f + .ci/$f - name: Build run: .ci/build.sh shell: bash From 953a9999b43eff8b63526938732bf0442d53de26 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 16 Apr 2021 20:06:56 +0200 Subject: [PATCH 15/26] [ci] Checkout with fetch-depth 2 Otherwise we can't fetch later more commits for regression-tester and danger. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dc43e3bf58..058a6904d0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,8 @@ jobs: if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }} steps: - uses: actions/checkout@v2 + with: + fetch-depth: 2 - uses: actions/cache@v2 with: path: | From 4241c3713ca818cd448525ca2e8b7bf38ccc380f Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sun, 18 Apr 2021 16:54:43 +0200 Subject: [PATCH 16/26] [ci] Document how to test a release push build --- .ci/README.md | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index 88b9183424..71b1a31d47 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -92,7 +92,7 @@ rmdir pmd && mkdir pmd cd pmd git init git remote add origin https://github.com/pmd/pmd -git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/heads/${MAIN_BRANCH}:refs/remotes/origin/${MAIN_BRANCH} +git fetch --no-tags --prune --progress --no-recurse-submodules --depth=2 origin +refs/heads/${MAIN_BRANCH}:refs/remotes/origin/${MAIN_BRANCH} git checkout --progress --force -B ${MAIN_BRANCH} refs/remotes/origin/${MAIN_BRANCH} @@ -141,12 +141,47 @@ $(~/create-gh-actions-env.sh push adangel/pmd $MAIN_BRANCH) ### Performing a release (push) build ``` -eval $(~/create-gh-actions-env.sh push pmd/pmd refs/tags/v1.0.0_release_test) +export LANG=en_US.UTF-8 +export PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/master/scripts + +export PMD_CI_SECRET_PASSPHRASE="xyz" +export PMD_CI_DEBUG=true + +TAG_NAME=pmd_releases/6.33.0 + +eval $(~/create-gh-actions-env.sh push pmd/pmd refs/tags/$TAG_NAME) + +cd /workspaces/pmd +rmdir pmd && mkdir pmd +cd pmd +git init +git remote add origin https://github.com/pmd/pmd +git fetch --no-tags --prune --progress --no-recurse-submodules --depth=2 origin +refs/tags/$TAG_NAME:refs/tags/$TAG_NAME +git checkout --progress --force refs/tags/$TAG_NAME + +f=check-environment.sh; \ + mkdir -p .ci && \ + ( [ -e .ci/$f ] || curl -sSL "${PMD_CI_SCRIPTS_URL}/$f" > ".ci/$f" ) && \ + chmod 755 .ci/$f && \ + .ci/$f + +# +# .ci/build.sh +# ``` -Make sure, that `MAVEN_OPTS` contains `-DskipRemoteStaging=true`, so that no maven artifacts are deployed +Calling `.ci/build.sh` directly would re-release the tag $TAG_NAME - that's why it is commented out. +All the side-effects of a release would be carried out like creating and publishing a release on github, +uploading the release to sourceforge, uploading the docs to pmd.github.io/docs.pmd-code.org, uploading a +new baseline for the regression tester and so on. While the release should be reproducible and therefore should +produce exactly the same artifacts, re-uploading artifacts is not desired just for testing. + +Note that maven-central would not be changed, since this is skipped via MAVEN_OPTS: +`MAVEN_OPTS` contains `-DskipRemoteStaging=true`, so that no maven artifacts are deployed to maven central (this is set by `create-gh-actions-env.sh`). -And the checkout could be different... - +So for now in order to test the build script, you need to manually edit the script and comment out the +critical lines... (like publish github releases, uploading files to sourceforge ...). Later a +"dry-run" mode could be added. +Make sure to cleanup after the test, e.g. discard the draft github release. From dc56e6eed62f8bcfaf62f3f57041f26ed87c3932 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sun, 18 Apr 2021 16:57:52 +0200 Subject: [PATCH 17/26] [ci] Move doc to pmd/build-tools --- .ci/README.md | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index 71b1a31d47..ef88d0ef90 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -24,44 +24,6 @@ in clear text. --armor --passphrase="GnxdjywUEPveyCD1RLiTd7t8CImnefYr" \ --output .ci/files/public-env.gpg .ci/files/public-env -## Known Issues - -### Intermittent connection resets or timeouts while downloading dependencies from maven central - -Root issue seems to be SNAT Configs in Azure, which closes long running [idle TCP connections -after 4 minutes](https://docs.microsoft.com/en-us/azure/load-balancer/troubleshoot-outbound-connection#idletimeout). - -The workaround is described in [actions/virtual-environments#1499](https://github.com/actions/virtual-environments/issues/1499) -and [WAGON-545](https://issues.apache.org/jira/browse/WAGON-545) -and [WAGON-486](https://issues.apache.org/jira/browse/WAGON-486): - -The setting `-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3` -makes sure, that Maven doesn't try to use pooled connections that have been unused for more than 180 seconds. -These settings are placed as environment variable `MAVEN_OPTS` in the workflow, so that they are active for -all Maven executions (including builds done by regression tester). - -Alternatively, pooling could be disabled completely via `-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false`. -This has the consequence, that for each dependency, that is being downloaded, a new https connection is -established. - -More information about configuring this can be found at [wagon-http](https://maven.apache.org/wagon/wagon-providers/wagon-http/). - -## Remote debugging - -Debugging remotely is possible with . - -Just add the following step into the job: - -``` - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 -``` - -The workflow `troubleshooting` can be started manually, which already contains the tmate action. - -**Note**: This is dangerous for push/pull builds on pmd/pmd, because these have access to the secrets and the SSH session -is not protected. Builds triggered by pull requests from forked repositories don't have access to the secrets. - ## Local tests with docker Using the same docker container as described in [build-env @ build-tools](https://github.com/pmd/build-tools). From f413328653ba7f2cecf21475a7fdf71cf3621150 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sun, 18 Apr 2021 18:49:14 +0200 Subject: [PATCH 18/26] [ci] Set autoReleaseAfterClose=true in workflow --- .github/workflows/build.yml | 2 +- pom.xml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 058a6904d0..c81693983d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,7 +41,7 @@ jobs: shell: bash run: | echo "LANG=en_US.UTF-8" >> $GITHUB_ENV - echo "MAVEN_OPTS=-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -DstagingProgressTimeoutMinutes=30" >> $GITHUB_ENV + echo "MAVEN_OPTS=-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -DautoReleaseAfterClose=true -DstagingProgressTimeoutMinutes=30" >> $GITHUB_ENV echo "PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/master/scripts" >> $GITHUB_ENV - name: Check Environment shell: bash diff --git a/pom.xml b/pom.xml index bbfe147f7c..d01f5c8f35 100644 --- a/pom.xml +++ b/pom.xml @@ -527,9 +527,6 @@ ossrh https://oss.sonatype.org/ - - true From 381395d13f3dd882518172e5f016dd28d3136c8f Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sun, 18 Apr 2021 19:08:51 +0200 Subject: [PATCH 19/26] [ci] Create a news entry on sourceforge when releasing --- .ci/build.sh | 12 +++++++++++- do-release.sh | 10 +++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.ci/build.sh b/.ci/build.sh index b6af53af70..22b6f81d92 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -73,6 +73,7 @@ function build() { pmd_ci_log_group_start "Publishing Release" pmd_ci_gh_releases_publishRelease "$GH_RELEASE" pmd_ci_sourceforge_selectDefault "${PMD_CI_MAVEN_PROJECT_VERSION}" + pmd_ci_sourceforge_publishBlogPost "$SF_BLOG_URL" pmd_ci_log_group_end fi @@ -206,9 +207,18 @@ function pmd_ci_build_and_upload_doc() { rendered_release_notes=$(bundle exec docs/render_release_notes.rb docs/pages/release_notes.md | tail -n +6) local release_name release_name="PMD ${PMD_CI_MAVEN_PROJECT_VERSION} ($(date -u +%d-%B-%Y))" - pmd_ci_gh_releases_updateRelease "$GH_RELEASE" "$release_name" "$rendered_release_notes" + pmd_ci_gh_releases_updateRelease "$GH_RELEASE" "$release_name" "${rendered_release_notes}" pmd_ci_sourceforge_uploadReleaseNotes "${PMD_CI_MAVEN_PROJECT_VERSION}" "${rendered_release_notes}" + local rendered_release_notes_with_links + rendered_release_notes_with_links=" +* Downloads: https://github.com/pmd/pmd/releases/tag/pmd_releases%2F${PMD_CI_MAVEN_PROJECT_VERSION} +* Documentation: https://pmd.github.io/pmd-${PMD_CI_MAVEN_PROJECT_VERSION}/ + +${rendered_release_notes}" + pmd_ci_sourceforge_createDraftBlogPost "${release_name} released" "${rendered_release_notes_with_links}" "pmd,release" + SF_BLOG_URL="${RESULT}" + # updates https://pmd.github.io/latest/ and https://pmd.github.io/pmd-${PMD_CI_MAVEN_PROJECT_VERSION} publish_release_documentation_github pmd_ci_sourceforge_rsyncSnapshotDocumentation "${PMD_CI_MAVEN_PROJECT_VERSION}" "pmd-${PMD_CI_MAVEN_PROJECT_VERSION}" diff --git a/do-release.sh b/do-release.sh index a191f84799..12bd13fb81 100755 --- a/do-release.sh +++ b/do-release.sh @@ -226,20 +226,16 @@ echo echo echo echo "Verify the new release on github: " +echo "and the news entry at " echo echo "* Wait until the new version is synced to maven central and appears as latest version in" echo " ." -echo "* Submit news to SF on page. Use same text as in the email below." echo "* Send out an announcement mail to the mailing list:" echo echo "To: PMD Developers List " -echo "Subject: [ANNOUNCE] PMD ${RELEASE_VERSION} Released" -echo -echo "* Downloads: https://github.com/pmd/pmd/releases/tag/pmd_releases%2F${RELEASE_VERSION}" -echo "* Documentation: https://pmd.github.io/pmd-${RELEASE_VERSION}/" -echo -echo "$NEW_RELEASE_NOTES" +echo "Subject: [ANNOUNCE] PMD ${RELEASE_VERSION} released" echo +echo " You can copy the same text from ." echo echo tweet="PMD ${RELEASE_VERSION} released: https://github.com/pmd/pmd/releases/tag/pmd_releases/${RELEASE_VERSION} #PMD" From 5f0dc5baaaa585349835f44ffc424974d5b6e0b1 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sun, 18 Apr 2021 19:25:28 +0200 Subject: [PATCH 20/26] [ci] Fix shellcheck issues in do-release.sh --- do-release.sh | 61 ++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/do-release.sh b/do-release.sh index 12bd13fb81..00efa30a5e 100755 --- a/do-release.sh +++ b/do-release.sh @@ -5,7 +5,7 @@ set -e export LANG=C.UTF-8 # verify the current directory -if [ ! -f pom.xml -o ! -d ../pmd.github.io ]; then +if [ ! -f pom.xml ] || [ ! -d ../pmd.github.io ]; then echo "You seem to be in the wrong working directory or you don't have pmd.github.io checked out..." echo echo "Expected:" @@ -26,20 +26,20 @@ echo "-------------------------------------------" CURRENT_VERSION=$(./mvnw org.apache.maven.plugins:maven-help-plugin:3.2.0:evaluate -Dexpression=project.version -q -DforceStdout) RELEASE_VERSION=${CURRENT_VERSION%-SNAPSHOT} -MAJOR=$(echo $RELEASE_VERSION | cut -d . -f 1) -MINOR=$(echo $RELEASE_VERSION | cut -d . -f 2) -PATCH=$(echo $RELEASE_VERSION | cut -d . -f 3) +MAJOR=$(echo "$RELEASE_VERSION" | cut -d . -f 1) +MINOR=$(echo "$RELEASE_VERSION" | cut -d . -f 2) +PATCH=$(echo "$RELEASE_VERSION" | cut -d . -f 3) if [ "$PATCH" == "0" ]; then - NEXT_MINOR=$(expr ${MINOR} + 1) + NEXT_MINOR=$(("${MINOR}" + 1)) NEXT_PATCH="0" - LAST_MINOR=$(expr ${MINOR} - 1) + LAST_MINOR=$(("${MINOR}" - 1)) LAST_PATCH="0" else # this is a bugfixing release NEXT_MINOR="${MINOR}" - NEXT_PATCH=$(expr ${PATCH} + 1) + NEXT_PATCH=$(("${PATCH}" + 1)) LAST_MINOR="${MINOR}" - LAST_PATCH=$(expr ${PATCH} - 1) + LAST_PATCH=$(("${PATCH}" - 1)) fi LAST_VERSION="$MAJOR.$LAST_MINOR.$LAST_PATCH" DEVELOPMENT_VERSION="$MAJOR.$NEXT_MINOR.$NEXT_PATCH" @@ -65,7 +65,7 @@ echo echo "Is this correct?" echo echo "Press enter to continue... (or CTRL+C to cancel)" -read +read -r export LAST_VERSION export RELEASE_VERSION @@ -95,16 +95,16 @@ echo "* Update property \`pmd-designer.version\` in **pom.xml** to reference t echo " See for the available releases." echo echo "Press enter to continue..." -read +read -r # calculating stats for release notes STATS=$( echo "### Stats" -echo "* $(git log pmd_releases/${LAST_VERSION}..HEAD --oneline --no-merges |wc -l) commits" +echo "* $(git log pmd_releases/"${LAST_VERSION}"..HEAD --oneline --no-merges |wc -l) commits" echo "* $(curl -s https://api.github.com/repos/pmd/pmd/milestones|jq ".[] | select(.title == \"$RELEASE_VERSION\") | .closed_issues") closed tickets & PRs" -echo "* Days since last release: $(( ( $(date +%s) - $(git log --max-count=1 --format="%at" pmd_releases/${LAST_VERSION}) ) / 86400))" +echo "* Days since last release: $(( ( $(date +%s) - $(git log --max-count=1 --format="%at" pmd_releases/"${LAST_VERSION}") ) / 86400))" ) TEMP_RELEASE_NOTES=$(cat docs/pages/release_notes.md) @@ -118,17 +118,18 @@ echo echo "Please verify docs/pages/release_notes.md" echo echo "Press enter to continue..." -read +read -r # install bundles needed for rendering release notes bundle config set --local path vendor/bundle bundle config set --local with release_notes_preprocessing bundle install -export RELEASE_NOTES_POST="_posts/$(date -u +%Y-%m-%d)-PMD-${RELEASE_VERSION}.md" +RELEASE_NOTES_POST="_posts/$(date -u +%Y-%m-%d)-PMD-${RELEASE_VERSION}.md" +export RELEASE_NOTES_POST echo "Generating ../pmd.github.io/${RELEASE_NOTES_POST}..." NEW_RELEASE_NOTES=$(bundle exec docs/render_release_notes.rb docs/pages/release_notes.md | tail -n +6) -cat > ../pmd.github.io/${RELEASE_NOTES_POST} < "../pmd.github.io/${RELEASE_NOTES_POST}" </dev/null| egrep "^[AMDRC]" | wc -l) - if [ $changes -gt 0 ]; then + git add "${RELEASE_NOTES_POST}" + changes=$(git status --porcelain 2>/dev/null| grep -c -E "^[AMDRC]") + if [ "$changes" -gt 0 ]; then echo "Committing current changes (pmd.github.io)" git commit -a -m "Prepare pmd release ${RELEASE_VERSION}" && git push fi ) ./mvnw -B release:clean release:prepare \ - -Dtag=pmd_releases/${RELEASE_VERSION} \ - -DreleaseVersion=${RELEASE_VERSION} \ - -DdevelopmentVersion=${DEVELOPMENT_VERSION} \ + -Dtag="pmd_releases/${RELEASE_VERSION}" \ + -DreleaseVersion="${RELEASE_VERSION}" \ + -DdevelopmentVersion="${DEVELOPMENT_VERSION}" \ -Pgenerate-rule-docs @@ -166,7 +167,7 @@ echo "Tag has been pushed.... now check github actions: docs/pages/release_notes_old.md -echo "$NEW_RELEASE_NOTES" >> docs/pages/release_notes_old.md -echo >> docs/pages/release_notes_old.md -echo "$OLD_RELEASE_NOTES" >> docs/pages/release_notes_old.md +OLD_RELEASE_NOTES_HEADER=$(head -n 7 docs/pages/release_notes_old.md) +echo "${OLD_RELEASE_NOTES_HEADER} +${NEW_RELEASE_NOTES} +${OLD_RELEASE_NOTES}" > docs/pages/release_notes_old.md # reset release notes template cat > docs/pages/release_notes.md < Date: Sun, 18 Apr 2021 19:50:52 +0200 Subject: [PATCH 21/26] [ci] Add git-repo-sync --- .ci/README.md | 4 +++ .ci/git-repo-sync.sh | 44 ++++++++++++++++++++++++++ .github/workflows/git-repo-sync.yml | 49 +++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100755 .ci/git-repo-sync.sh create mode 100644 .github/workflows/git-repo-sync.yml diff --git a/.ci/README.md b/.ci/README.md index ef88d0ef90..719d931872 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -147,3 +147,7 @@ critical lines... (like publish github releases, uploading files to sourceforge "dry-run" mode could be added. Make sure to cleanup after the test, e.g. discard the draft github release. + +## Workflow git-repo-sync + +Synchronizes the github git repository pmd/pmd on every push to sourceforge. diff --git a/.ci/git-repo-sync.sh b/.ci/git-repo-sync.sh new file mode 100755 index 0000000000..a32d6c9b03 --- /dev/null +++ b/.ci/git-repo-sync.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Exit this script immediately if a command/function exits with a non-zero status. +set -e + +SCRIPT_INCLUDES="log.bash utils.bash setup-secrets.bash" +# shellcheck source=inc/fetch_ci_scripts.bash +source "$(dirname "$0")/inc/fetch_ci_scripts.bash" && fetch_ci_scripts + +function git_repo_sync() { + echo + pmd_ci_utils_determine_build_env pmd/pmd + echo + + if pmd_ci_utils_is_fork_or_pull_request; then + pmd_ci_log_error "This should not run on forked repositories or pull requests" + exit 0 + fi + + # only builds on pmd/pmd continue here + pmd_ci_log_group_start "Setup environment" + pmd_ci_setup_secrets_private_env + pmd_ci_setup_secrets_gpg_key + pmd_ci_setup_secrets_ssh + pmd_ci_log_group_end + + pmd_ci_log_group_start "Git Sync" + git remote add pmd-sf "${PMD_SF_USER}@git.code.sf.net:/p/pmd/code" + if [ -n "${PMD_CI_BRANCH}" ]; then + git push pmd-sf "${PMD_CI_BRANCH}:${PMD_CI_BRANCH}" + pmd_ci_log_success "Successfully pushed ${PMD_CI_BRANCH} to sourceforge" + elif [ -n "${PMD_CI_TAG}" ]; then + git push pmd-sf tag "${PMD_CI_TAG}" + pmd_ci_log_success "Successfully pushed tag ${PMD_CI_TAG} to sourceforge" + else + pmd_ci_log_error "Don't know what to do: neither PMD_CI_BRANCH nor PMD_CI_TAG is set" + exit 1 + fi + pmd_ci_log_group_end +} + +git_repo_sync + +exit 0 diff --git a/.github/workflows/git-repo-sync.yml b/.github/workflows/git-repo-sync.yml new file mode 100644 index 0000000000..18981e5f7f --- /dev/null +++ b/.github/workflows/git-repo-sync.yml @@ -0,0 +1,49 @@ +name: git-repo-sync + +on: + push: + branches: + - main + - master + tags: + - '**' + pull_request: + schedule: + # build it monthly: At 04:00 on day-of-month 1. + - cron: '0 4 1 * *' + +jobs: + build: + runs-on: ${{ matrix.os }} + continue-on-error: false + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + vendor/bundle + key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}- + - name: Set up Ruby 2.7 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.7 + - name: Setup Environment + shell: bash + run: | + echo "LANG=en_US.UTF-8" >> $GITHUB_ENV + echo "PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/master/scripts" >> $GITHUB_ENV + - name: Sync + run: .ci/git-repo-sync.sh + shell: bash + env: + PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} From a09e38471fe971f5e95917c2c5b3a15183c604df Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sun, 18 Apr 2021 19:57:50 +0200 Subject: [PATCH 22/26] [ci] Run git-repo-sync only on pushes, not on pull requests Also remove unneeded actions --- .github/workflows/git-repo-sync.yml | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/.github/workflows/git-repo-sync.yml b/.github/workflows/git-repo-sync.yml index 18981e5f7f..198d10e8fb 100644 --- a/.github/workflows/git-repo-sync.yml +++ b/.github/workflows/git-repo-sync.yml @@ -7,36 +7,15 @@ on: - master tags: - '**' - pull_request: - schedule: - # build it monthly: At 04:00 on day-of-month 1. - - cron: '0 4 1 * *' jobs: build: - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest continue-on-error: false - strategy: - matrix: - os: [ ubuntu-latest, windows-latest, macos-latest ] - if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }} steps: - uses: actions/checkout@v2 with: fetch-depth: 2 - - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository - ~/.cache - vendor/bundle - key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}- - - name: Set up Ruby 2.7 - uses: actions/setup-ruby@v1 - with: - ruby-version: 2.7 - name: Setup Environment shell: bash run: | From 27f1ca47a357276902c1c6fba2258062fe349ff6 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Tue, 20 Apr 2021 19:18:14 +0200 Subject: [PATCH 23/26] [doc] Add new infrastructure page --- docs/_data/sidebars/pmd_sidebar.yml | 3 + .../projectdocs/committers/infrastructure.md | 68 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 docs/pages/pmd/projectdocs/committers/infrastructure.md diff --git a/docs/_data/sidebars/pmd_sidebar.yml b/docs/_data/sidebars/pmd_sidebar.yml index 683a02cb16..5244fdbdf8 100644 --- a/docs/_data/sidebars/pmd_sidebar.yml +++ b/docs/_data/sidebars/pmd_sidebar.yml @@ -439,6 +439,9 @@ entries: - title: Project management output: web, pdf subfolderitems: + - title: Infrastructure + url: /pmd_projectdocs_committers_infrastructure.html + output: web, pdf - title: Release process url: /pmd_projectdocs_committers_releasing.html output: web, pdf diff --git a/docs/pages/pmd/projectdocs/committers/infrastructure.md b/docs/pages/pmd/projectdocs/committers/infrastructure.md new file mode 100644 index 0000000000..10bec8bd67 --- /dev/null +++ b/docs/pages/pmd/projectdocs/committers/infrastructure.md @@ -0,0 +1,68 @@ +--- +title: Infrastructure +permalink: pmd_projectdocs_committers_infrastructure.html +author: Andreas Dangel +last_updated: April 2021 +--- + +This page describes, which infrastructure and services is used by the pmd project. + +## github + +The main repository is hosted on . We own the organization "pmd". + +* source code in git repositories +* releases +* issue tracker +* discussions +* pull requests +* github actions for CI + +Also the [main landing page](pmd_projectdocs_committers_main_landing_page.html) () +is hosted using github pages. + +## sourceforge + +Before moving to github, sourceforge was the main place. It is still there: . + +Nowadays it is used for: + +* hosting archive of binaries: https://sourceforge.net/projects/pmd/files/ +* hosting an archive of documentation: https://pmd.sourceforge.io/archive.html +* mailing lists: + * + * +* discussion forum + +It also contains the old issue tracker. + +## domain, email and homepage + +We are using a webhosting package by [Netcup](https://www.netcup.de/). + +The following domains are registered for us: + +* pmd-code.org +* pmd-code.io +* pmd-code.com + +The webhosting package provides these services: + +* email service (including mailbox via IMAP) +* web pages + +The homepage redirects to . + +Some docs are hosted at . + + +## other services + +* Deployment to maven central via and +* Hosting eclipse plugin update site via +* Hosting result reports from pmd-regression-tester via +* Twitter: +* Rubygems for pmd-regression-tester: +* sonarcloud: +* travis ci was used before github actions: + From d4a605d0991e176c58c69b381c9b83f45f987ae0 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 22 Apr 2021 10:26:21 +0200 Subject: [PATCH 24/26] [doc] Update release process documentation --- .../pmd/projectdocs/committers/releasing.md | 186 ++++++++++++------ 1 file changed, 122 insertions(+), 64 deletions(-) diff --git a/docs/pages/pmd/projectdocs/committers/releasing.md b/docs/pages/pmd/projectdocs/committers/releasing.md index f64b93b896..3d33dd2932 100644 --- a/docs/pages/pmd/projectdocs/committers/releasing.md +++ b/docs/pages/pmd/projectdocs/committers/releasing.md @@ -1,43 +1,75 @@ --- title: Release process permalink: pmd_projectdocs_committers_releasing.html -author: Romain Pelisse , Andreas Dangel +author: Romain Pelisse , Andreas Dangel +last_updated: April 2021 --- This page describes the current status of the release process. Since versions 5.4.5 / 5.5.4 there is an automated release process using [travis-ci](https://travis-ci.com) -in place. However, there are still a few steps, that need manual examination. +in place. Since 6.30.0, the automated release process is using [Github Actions](https://github.com/pmd/pmd/actions). -Note: You can find a small shell script in the root of the repo: `do-release.sh`. This script guides you -through the release process. +However, there are still a few steps, that need manual examination. +## Overview + +This page gives an overview which tasks are automated to do a full release of PMD. This knowledge is +required in order to verify that the release was successful or in case the automated process fails for +some reason. Then individual steps need to be executed manually. Because the build is reproducible, these +steps can be repeated again if the same tag is used. + +The three main steps are: + +* Preparations (which creates the tag) - use `do-release.sh` for that +* The actual release (which is automated) +* Prepare the next release (make sure the current main branch is ready for further development) ## Preparations +This is the first step. It is always manual and is executed locally. It creates in the end the tag from which +the release is created. + Make sure code is up to date and everything is committed and pushed with git: $ ./mvnw clean $ git pull $ git status +As a help for the preparation task, the script `do-release.sh` guides you through the preparation tasks +and the whole release process. The script requires a specific source code folder and additional checkouts locally, +e.g. it requires that the repo `pmd.github.io` is checked out aside the main pmd repo: +* ➡️ `/home/joe/source/pmd` +* ➡️ `/home/joe/source/pmd.github.io` + +The script `do-release.sh` is called in the directory `/home/joe/source/pmd` and searches for `../pmd.github.io`. + +Also make sure, that the repo "pmd.github.io" is locally up to date and has no local changes. ### The Release Notes and docs +Before the release, you need to verify the release notes: Does it contain all the relevant changes for the +release? Is it formatted properly? Are there any typos? Does it render properly? + +As the release notes are part of the source code, it is not simple to change it afterwards. While the source +code for a tag cannot be changed anymore, the published release notes on the github releases pages or the +new posts can be changed afterwards (although that's an entirely manual process). + You can find the release notes here: `docs/pages/release_notes.md`. The date (`date +%d-%B-%Y`) and the version (remove the SNAPSHOT) must be updated in `docs/_config.yml`, e.g. +in order to release version "6.34.0", the configuration should look like this: -``` +```yaml pmd: - version: 6.22.0 - previous_version: 6.21.0 - date: 12-March-2020 + version: 6.34.0 + previous_version: 6.33.0 + date: 24-April-2021 release_type: minor ``` -The release type could be one of "bugfix", "minor", or "major". +The release type could be one of "bugfix" (e.g. 6.34.x), "minor" (6.x.0), or "major" (x.0.0). The release notes usual mention any new rules that have been added since the last release. Please double check the file `pmd-core/src/main/resources/rulesets/releases/.xml`, so @@ -61,8 +93,8 @@ on . The following snippet will create the numbers, that can be attached to the release notes as a last section: ```shell -LAST_VERSION=6.22.0 -NEW_VERSION=6.23.0 +LAST_VERSION=6.33.0 +NEW_VERSION=6.34.0 NEW_VERSION_COMMITISH=HEAD echo "### Stats" @@ -82,18 +114,20 @@ Check in all (version) changes to branch master or any other branch, from which ### The Homepage The github repo `pmd.github.io` hosts the homepage for [https://pmd.github.io](https://pmd.github.io). +All the following tasks are to be done in this repo. The new version needs to be entered into `_config.yml`, e.g.: -``` +```yaml pmd: - latestVersion: 6.22.0 - latestVersionDate: 12-March-2020 + latestVersion: 6.34.0 + latestVersionDate: 24-April-2021 ``` -Also move the previous version down into the "downloads" section. +Also move the previous version down into the "downloads" section. We usually keep only the last 3 versions +in this list, so remove the oldest version. -Then create a new page for the new release, e.g. `_posts/2020-03-12-PMD-6.22.0.md` and copy +Then create a new page for the new release, e.g. `_posts/2021-04-24-PMD-6.34.0.md` and copy the release notes into this page. This will appear under the news section. Check in all (version) changes to branch master: @@ -102,56 +136,73 @@ Check in all (version) changes to branch master: $ git push +## The actual release -## Creating the release +The actual release starts with one last local command: calling **maven-release-plugin**. -The release is created using the **maven-release-plugin**. This plugin changes the version by basically -removing the "-SNAPSHOT" suffix, builds the changed project locally, commits the version change, creates +This plugin changes the version by basically removing the "-SNAPSHOT" suffix, builds the changed project +locally, commits the version change, creates a new tag from this commit, changes the version of the project to the next snapshot, commits this change and pushes everything. `RELEASE_VERSION` is the version of the release. It is reused for the tag. `DEVELOPMENT_VERSION` is the next snapshot version after the release. - mvn -B release:clean release:prepare \ - -Dtag=pmd_releases/${RELEASE_VERSION} \ - -DreleaseVersion=${RELEASE_VERSION} \ - -DdevelopmentVersion=${DEVELOPMENT_VERSION} +```shell +RELEASE_VERSION=6.34.0 +DEVELOPMENT_VERSION=6.35.0-SNAPSHOT +./mvnw -B release:clean release:prepare \ + -Dtag=pmd_releases/${RELEASE_VERSION} \ + -DreleaseVersion=${RELEASE_VERSION} \ + -DdevelopmentVersion=${DEVELOPMENT_VERSION} +``` +Once the maven plugin has pushed the tag, github actions will start and build a new version from this tag. Since +it is a tag build and a released version build, the build script will do a couple of additional stuff. +This is all automated in `.ci/build.sh`. -Once the maven plugin has pushed the tag, travis-ci will start and build a new version from this tag. Since -it is a tag build and a released version build, travis-ci will do a couple of additional stuff: +Here is, what happens: * Deploy and release the build to maven central, so that it can be downloaded from . This is done automatically, if - all unit tests pass on travis-ci. The plugin [nexus-staging-maven-plugin](https://github.com/sonatype/nexus-maven-plugins/tree/master/staging/maven-plugin) is used for that. + all unit tests pass and the build doesn't fail for any other reason. + The plugin [nexus-staging-maven-plugin](https://github.com/sonatype/nexus-maven-plugins/tree/master/staging/maven-plugin) is used for that. * Upload the new binaries to github releases under . It also uploads - the release notes. + the release notes from `docs/pages/release_notes.md`. + Note: The during the process, the release is a draft mode and not visible yet. + At the end of the process, the release will be published. * Upload the new binaries additionally to sourceforge, so that they can be downloaded from - , including the release notes. The new binaries are - selected as the new default downloads for PMD. + , including the release notes. +* Render the documentation in `docs/` with `bundle exec jekyll build` and create a zip file from it. +* Upload the doc zip file to the current github release under and + to . +* Upload the documentation to , e.g. and + create a symlink, so that points to the new version. +* Remove the old snapshot documention, e.g. so that is gone. + Also create a symlink from pmd-doc-6.34.0-SNAPSHOT to pmd-doc-6.34.0, so that old references still work, e.g. + points to the released version. +* Deploy javadoc to "https://docs.pmd-code.org/apidocs/*/RELEASE_VERSION/", e.g. + . This is done for all modules. +* Remove old javadoc for the SNAPSHOT version, e.g. delete . +* Create a draft news post on for the new release. This contains the + rendered release notes. * Add the documentation of the new release to a subfolder on , also make - this folder available as `latest`. -* Upload the documentation to . + this folder available as `latest`, so that shows the new + version and is the URL for the specific release. +* Also copy the documentation to sourceforge's web space, so that it is available as + . All previously copied version are listed + under . +* After all this is done, the release on github () is published + and the news post on sourceforge (https://sourceforge.net/p/pmd/news/> is publishes as well. +* The new binary at is + selected as the new default for PMD. +* As a last step, a new baseline for the [regression tester](https://github.com/pmd/pmd-regression-tester) + is created and uploaded to . +The release on github actions currently takes about 30-45 minutes. Once this is done, you can spread additional +news: -## After the release - -The release on travis currently takes about 30 minutes. Once this is done, you can spread the news: - -### Submit a news on SF - -Submit news to SF on the [PMD Project News](https://sourceforge.net/p/pmd/news/) page. You can use -the following template: - - PMD released - - * Downloads: https://github.com/pmd/pmd/releases/tag/pmd_releases%2F - * Documentation: https://pmd.github.io/pmd-/ - - And Copy-Paste the release notes - -### Write an email to the mailing list +* Write an email to the mailing list To: PMD Developers List Subject: [ANNOUNCE] PMD released @@ -162,25 +213,36 @@ the following template: And Copy-Paste the release notes +* Tweet about the new release + +Tweet on , eg.: + + PMD 6.34.0 released: https://github.com/pmd/pmd/releases/tag/pmd_releases/6.34.0 #PMD ## Prepare the next release -### Prepare the new release notes +There are a couple of manual steps needed to prepare the current main branch for further development. -* Update version in **docs/_config.yml**. Note - the next version needs to have a SNAPSHOT in it. +* Move any open issues to the next milestone, close the current milestone + on and create a new one for the next + version (if one doesn't exist already). +* Update version in **docs/_config.yml**. Note - the next version needs to have a SNAPSHOT + in it otherwise the javadoc links won't work during development. - ``` + ```yaml pmd: - version: 6.23.0-SNAPSHOT - previous_version: 6.22.0 - date: ??-??-2020 + version: 6.35.0-SNAPSHOT + previous_version: 6.34.0 + date: ??-??-2021 release_type: minor ``` -* Move version/release info from **docs/pages/release_notes.md** to **docs/pages/release_notes_old.md**. -* Update version/release info in **docs/pages/release_notes.md**. Use the following template: +* Prepare a new empty release notes. Note, this is done by `do-release.sh` already. + * Move version/release info from **docs/pages/release_notes.md** to **docs/pages/release_notes_old.md**. + * Update version/release info in **docs/pages/release_notes.md**. Use the following template: {%raw%} + ``` --- title: PMD Release Notes @@ -210,19 +272,12 @@ This is a {{ site.pmd.release_type }} release. {%endraw%} -Commit and push +Finally commit and push the changes: $ git commit -m "Prepare next development version" $ git push origin master -### Close / Create new milestones - -Manage the milestones under . -Maybe there are some milestones on sourceforge, too: . - - - ## Branches ### Merging @@ -251,3 +306,6 @@ the `master` branch. Here are the steps: * Update the version in both the new branch and master, e.g. `mvn versions:set -DnewVersion=5.6.1-SNAPSHOT` and `mvn versions:set -DnewVersion=5.7.0-SNAPSHOT`. * Update the release notes on both the new branch and master + +The maintenance or bugfix branch could also be created later when needed from the actual tag. Then only the version on +the maintenance branch needs to be set. From 074127550544420f21cab78c7e14e4df74cb9202 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 22 Apr 2021 10:45:45 +0200 Subject: [PATCH 25/26] [doc] Add checklist for release process --- .../pmd/projectdocs/committers/releasing.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/pages/pmd/projectdocs/committers/releasing.md b/docs/pages/pmd/projectdocs/committers/releasing.md index 3d33dd2932..49990b22e7 100644 --- a/docs/pages/pmd/projectdocs/committers/releasing.md +++ b/docs/pages/pmd/projectdocs/committers/releasing.md @@ -219,6 +219,24 @@ Tweet on , eg.: PMD 6.34.0 released: https://github.com/pmd/pmd/releases/tag/pmd_releases/6.34.0 #PMD + +### Checklist + +| Task | Description | URL | ☐ / ✔ | +|------|-------------|-----|-------| +| maven central | The new version of all artifacts are available in maven central | | | +| github releases | A new release with 3 assets (bin, src, doc) is created | | | +| sourceforge files | The 3 assets (bin, src, doc) are uploaded, the new version is pre-selected as latest | | | +| homepage | Main landing page points to new version, doc for new version is available | | | +| homepage2 | New blogpost for the new release is posted | | | +| docs | New docs are uploaded | | | +| docs-archive | New docs are also on archive site | | | +| javadoc | New javadocs are uploaded | | | +| news | New blogpost on sourceforge is posted | | | +| regression-tester | New release baseline is uploaded | | | +| mailing list | announcement on mailing list is sent | | | +| twitter | tweet about the new release | | | + ## Prepare the next release There are a couple of manual steps needed to prepare the current main branch for further development. From fe9bf3c352a4c53afd415e67d23c038a2ebfe9f2 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 22 Apr 2021 10:59:51 +0200 Subject: [PATCH 26/26] Bump build-tools from 10 to 11 --- .github/workflows/build.yml | 2 +- .github/workflows/git-repo-sync.yml | 2 +- .github/workflows/troubleshooting.yml | 2 +- pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c81693983d..cfd907df51 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,7 +42,7 @@ jobs: run: | echo "LANG=en_US.UTF-8" >> $GITHUB_ENV echo "MAVEN_OPTS=-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -DautoReleaseAfterClose=true -DstagingProgressTimeoutMinutes=30" >> $GITHUB_ENV - echo "PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/master/scripts" >> $GITHUB_ENV + echo "PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/11/scripts" >> $GITHUB_ENV - name: Check Environment shell: bash run: | diff --git a/.github/workflows/git-repo-sync.yml b/.github/workflows/git-repo-sync.yml index 198d10e8fb..0d53106b36 100644 --- a/.github/workflows/git-repo-sync.yml +++ b/.github/workflows/git-repo-sync.yml @@ -20,7 +20,7 @@ jobs: shell: bash run: | echo "LANG=en_US.UTF-8" >> $GITHUB_ENV - echo "PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/master/scripts" >> $GITHUB_ENV + echo "PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/11/scripts" >> $GITHUB_ENV - name: Sync run: .ci/git-repo-sync.sh shell: bash diff --git a/.github/workflows/troubleshooting.yml b/.github/workflows/troubleshooting.yml index d895a6cf2f..d5a8965dad 100644 --- a/.github/workflows/troubleshooting.yml +++ b/.github/workflows/troubleshooting.yml @@ -31,7 +31,7 @@ jobs: run: | echo "LANG=en_US.UTF-8" >> $GITHUB_ENV echo "MAVEN_OPTS=-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -DstagingProgressTimeoutMinutes=30" >> $GITHUB_ENV - echo "PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/master/scripts" >> $GITHUB_ENV + echo "PMD_CI_SCRIPTS_URL=https://raw.githubusercontent.com/pmd/build-tools/11/scripts" >> $GITHUB_ENV - name: Check Environment shell: bash run: | diff --git a/pom.xml b/pom.xml index d01f5c8f35..f30f4ab18f 100644 --- a/pom.xml +++ b/pom.xml @@ -105,7 +105,7 @@ -Xmx512m -Dfile.encoding=${project.build.sourceEncoding} - 10 + 11 6.27.0