diff --git a/.ci/README.md b/.ci/README.md new file mode 100644 index 0000000000..8c1afbeab3 --- /dev/null +++ b/.ci/README.md @@ -0,0 +1,183 @@ +## PMD CI Scripts + +This folder contains scripts used for CI. + +## Secrets + +One secret is required for decrypting the GPG Key with which the PMD Releases are signed and +for a ssh key, which is used to copy files to sourceforge. + +## Environment variables + +* PMD_CI_SECRET_PASSPHRASE +* CI_DEPLOY_USER +* CI_DEPLOY_PASSWORD +* CI_SIGN_KEY +* CI_SIGN_PASSPHRASE +* PMD_SF_USER +* PMD_SF_APIKEY +* GITHUB_OAUTH_TOKEN +* GITHUB_BASE_URL +* COVERALLS_REPO_TOKEN +* SONAR_TOKEN +* DANGER_GITHUB_API_TOKEN +* PMD_CI_CHUNK_TOKEN + +## Encrypting + + gpg --batch --symmetric --cipher-algo AES256 --passphrase="$PMD_CI_SECRET_PASSPHRASE" file.txt + +## Known Issues + +### Intermittent connection resets or timeouts while downloading dependencies from maven central + +Root issue seems to be SNAT Configs in Azure, which closes long running [idle TCP connections +after 4 minutes](https://docs.microsoft.com/en-us/azure/load-balancer/troubleshoot-outbound-connection#idletimeout). + +The workaround is described in [actions/virtual-environments#1499](https://github.com/actions/virtual-environments/issues/1499) +and [WAGON-545](https://issues.apache.org/jira/browse/WAGON-545) +and [WAGON-486](https://issues.apache.org/jira/browse/WAGON-486): + +The setting `-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3` +makes sure, that Maven doesn't try to use pooled connections that have been unused for more than 180 seconds. +These settings are placed as environment variable `MAVEN_OPTS` in all workflows, so that they are active for +all Maven executions (including builds done by regression tester). + +Alternatively, pooling could be disabled completely via `-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false`. +This has the consequence, that for each dependency, that is being downloaded, a new https connection is +established. + +More information about configuring this can be found at [wagon-http](https://maven.apache.org/wagon/wagon-providers/wagon-http/). + +However, this doesn't work when [dokka-maven-plugin](https://github.com/Kotlin/dokka) is used: This plugin +downloads additional dokka plugins at runtime and reconfigures somehow Maven. After this plugin is loaded, +the above system properties have no effect anymore. +See [dokka/dokka-maven-plugin#1625](https://github.com/Kotlin/dokka/issues/1625) and +[dokka/dokka-maven-plugin#1626](https://github.com/Kotlin/dokka/issues/1626). + +The workaround now in place is, to download all the dependencies first, see `inc/maven-dependencies.inc`. + +## Hints + +### Remote debugging + +Debugging remotely is possible with . + +Just add the following step into the job: + +``` + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 +``` + +The workflow `troubleshooting` can be started manually, which already contains the tmate action. + +**Note**: This is dangerous for push/pull builds on pmd/pmd, because these have access to the secrets and the SSH session +is not protected. Builds triggered by pull requests from forked repositories don't have access to the secrets. + +### Local tests with docker + +Create a local docker container: + +``` +cd .ci/docker_ubuntu18.04 +docker build -t pmd-ci . +``` + +This container is based on Ubuntu 18.04, which is used for `ubuntu-latest` github actions runner, +see [Virtual Environment](https://github.com/actions/virtual-environments). + +You can run a local instance with docker: + +``` +docker run -it pmd-ci +``` + +You'll be dropped into a bash. + +#### Testing a push build (snapshot) + +Start docker without binding to local directory, so that we can do a fresh checkout: `docker run -it pmd-ci`. +You'll be dropped into a bash. Use the following script, to setup and start the build: + +``` +MAIN_BRANCH="master" +export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" +export PMD_CI_JOB_URL="manual job execution in docker" +export PMD_CI_PUSH_COMMIT_COMPARE="" +export PMD_CI_GIT_REF="refs/heads/${MAIN_BRANCH}" + +export PMD_CI_SECRET_PASSPHRASE="xyz" + +cd /workspaces/pmd +rmdir pmd && mkdir pmd +cd pmd +git init +git remote add origin https://github.com/pmd/pmd +git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/heads/${MAIN_BRANCH}:refs/remotes/origin/${MAIN_BRANCH} +git checkout --progress --force -B master refs/remotes/origin/${MAIN_BRANCH} + +.ci/check-environment.sh + +.ci/build.sh +``` + +#### Performing a release (push) build + +Start docker without binding to local directory, so that we can do a fresh checkout: `docker run -it pmd-ci`. +You'll be dropped into a bash. Use the following script, to setup and start the build: + +``` +TAG_NAME="pmd_releases/0.0.0_release_test" +export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" +export PMD_CI_JOB_URL="manual job execution in docker" +export PMD_CI_PUSH_COMMIT_COMPARE="" +export PMD_CI_GIT_REF="refs/tags/${TAG_NAME}" + +export PMD_CI_SECRET_PASSPHRASE="xyz" + +cd /workspace/pmd +rmdir pmd && mkdir pmd +cd pmd +git init +git remote add origin https://github.com/pmd/pmd +git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/tags/${TAG_NAME}:refs/tags/${TAG_NAME} +git checkout --progress --force refs/tags/${TAG_NAME} + +.ci/check-environment.sh + +.ci/build.sh +``` + +**Warning:** This will build and upload to maven central! + + +#### Testing a pull request + +Start docker without binding to local directory, so that we can do a fresh checkout: `docker run -it pmd-ci`. +You'll be dropped into a bash. Use the following script, to setup and start the build: + +``` +export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" +export PMD_CI_BRANCH="master" # base branch +export PMD_CI_PULL_REQUEST_NUMBER=2913 + +# these are used by danger +export GITHUB_EVENT_PATH=/workspaces/event.json +export GITHUB_REPOSITORY=pmd/pmd +export GITHUB_ACTION=run1 +export GITHUB_EVENT_NAME=pull_request +/home/pmd-ci/create-gh-pull-request-event.sh + +cd /workspace/pmd +rmdir pmd && mkdir pmd +cd pmd +git init +git remote add origin https://github.com/pmd/pmd +git fetch --no-tags --prune --progress --no-recurse-submodules --depth=2 origin +refs/pull/${PMD_CI_PULL_REQUEST_NUMBER}/merge:refs/remotes/pull/${PMD_CI_PULL_REQUEST_NUMBER}/merge +git checkout --progress --force refs/remotes/pull/${PMD_CI_PULL_REQUEST_NUMBER}/merge + +.ci/check-environment.sh + +.ci/build-pr.sh +``` diff --git a/.ci/build-coveralls.sh b/.ci/build-coveralls.sh new file mode 100755 index 0000000000..89a0743fb2 --- /dev/null +++ b/.ci/build-coveralls.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +source $(dirname $0)/inc/logger.inc +source $(dirname $0)/inc/setup-secrets.inc +source $(dirname $0)/inc/install-openjdk.inc +source $(dirname $0)/inc/maven-dependencies.inc + +set -e + +log_group_start "Setup private env and OpenJDK" + pmd_ci_setup_private_env + install_openjdk_setdefault 11 + export CI_NAME="github actions" + export CI_BUILD_URL="${PMD_CI_JOB_URL}" + export CI_BRANCH="${PMD_CI_GIT_REF##refs/heads/}" +log_group_end + +log_group_start "Downloading maven dependencies" + maven_dependencies_resolve +log_group_end + +log_group_start "Executing build with coveralls" + ./mvnw \ + -Dmaven.javadoc.skip=true \ + -Dmaven.source.skip \ + -Dcheckstyle.skip \ + -DrepoToken=${COVERALLS_REPO_TOKEN} \ + -B -V -e \ + clean package jacoco:report \ + coveralls:report -Pcoveralls + + if [ $? -ne 0 ]; then + log_error "Error creating coveralls report" + else + log_success "New coveralls result: https://coveralls.io/github/pmd/pmd" + fi +log_group_end diff --git a/.ci/build-pr-win-macos.sh b/.ci/build-pr-win-macos.sh new file mode 100755 index 0000000000..a6dca3172b --- /dev/null +++ b/.ci/build-pr-win-macos.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +source $(dirname $0)/inc/logger.inc +source $(dirname $0)/inc/install-openjdk.inc +source $(dirname $0)/inc/regression-tester.inc +source $(dirname $0)/inc/maven-dependencies.inc + +set -e + +log_group_start "Installing Java" + log_info "Install openjdk11 as default" + install_openjdk_setdefault 11 + + PMD_EXTRA_OPT="" + if [[ "$(uname)" == Linux* ]]; then + log_info "Install oracle7 for integration tests" + install_oraclejdk7 + PMD_EXTRA_OPT="-Djava7.home=${HOME}/oraclejdk7" + fi +log_group_end + +log_group_start "Downloading maven dependencies" + maven_dependencies_resolve +log_group_end + +log_group_start "Building with maven" + ./mvnw -e -V clean verify ${PMD_EXTRA_OPT} +log_group_end + + +# Danger is executed only on the linux runner +case "$(uname)" in + Linux*) + log_group_start "Executing danger" + regression_tester_setup_ci + regression_tester_executeDanger + log_group_end + ;; +esac diff --git a/.ci/build-sonar.sh b/.ci/build-sonar.sh new file mode 100755 index 0000000000..461fd317a0 --- /dev/null +++ b/.ci/build-sonar.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +source $(dirname $0)/inc/logger.inc +source $(dirname $0)/inc/setup-secrets.inc +source $(dirname $0)/inc/install-openjdk.inc +source $(dirname $0)/inc/maven-dependencies.inc + +set -e + +log_group_start "Setup private env and OpenJDK" + pmd_ci_setup_private_env + install_openjdk_setdefault 11 +log_group_end + +log_group_start "Downloading maven dependencies" + maven_dependencies_resolve +log_group_end + +log_group_start "Executing build with sonar" + ./mvnw \ + -Dmaven.javadoc.skip=true \ + -Dmaven.source.skip \ + -Dcheckstyle.skip \ + -B -V -e \ + clean package \ + sonar:sonar -Dsonar.login=${SONAR_TOKEN} -Psonar + + if [ $? -ne 0 ]; then + log_error "Error updating sonar..." + else + log_success "New sonar results: https://sonarcloud.io/dashboard?id=net.sourceforge.pmd%3Apmd" + fi +log_group_end diff --git a/.ci/build.sh b/.ci/build.sh new file mode 100755 index 0000000000..138f4041de --- /dev/null +++ b/.ci/build.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash + +source $(dirname $0)/inc/logger.inc +source $(dirname $0)/inc/setup-secrets.inc +source $(dirname $0)/inc/sourceforge-api.inc +source $(dirname $0)/inc/pmd-doc.inc +source $(dirname $0)/inc/pmd-code-api.inc +source $(dirname $0)/inc/regression-tester.inc +source $(dirname $0)/inc/github-releases-api.inc +source $(dirname $0)/inc/maven-dependencies.inc +source $(dirname $0)/inc/install-openjdk.inc + +set -e + +function pmd_ci_build_main() { + log_group_start "Setting up private secrets" + pmd_ci_setup_private_env + pmd_ci_setup_gpg_key + pmd_ci_setup_ssh + log_group_end + + log_group_start "Prepare Java 7+11, Maven, Bundler" + install_openjdk_setdefault 11 + install_oraclejdk7 + pmd_ci_build_setup_maven + pmd_ci_build_setup_bundler + pmd_ci_build_setup_env + log_group_end + + log_group_start "Downloading maven dependencies" + maven_dependencies_resolve + log_group_end + + log_group_start "Build and Deploy" + pmd_ci_build_run + pmd_ci_deploy_build_artifacts + log_group_end + + log_group_start "Build and Upload documentation" + pmd_ci_build_and_upload_doc + log_group_end + + if pmd_ci_build_isRelease; then + log_group_start "Publishing Release" + gh_release_publishRelease "$GH_RELEASE" + sourceforge_selectDefault "${VERSION}" + log_group_end + fi + + log_group_start "Creating new baseline for regression tester" + regression_tester_setup_ci + regression_tester_uploadBaseline + log_group_end + + exit 0 +} + + +# +# Configures maven. +# Needed for deploy to central (both snapshots and releases) +# and for signing the artifacts. +# +function pmd_ci_build_setup_maven() { + mkdir -p ${HOME}/.m2 + cp .ci/files/maven-settings.xml ${HOME}/.m2/settings.xml +} + +# +# Installs bundler, which is needed for doc generation and regression tester +# +function pmd_ci_build_setup_bundler() { + log_info "Installing bundler..." + gem install bundler +} + +# +# Setups common build parameters: +# * Determines the VERSION of PMD, that is being built +# * Determines the PMD_CI_BRANCH or PMD_CI_TAG, that is being built +# +function pmd_ci_build_setup_env() { + VERSION=$(pmd_ci_build_get_pom_version) + + if [[ "${PMD_CI_GIT_REF}" == refs/heads/* ]]; then + PMD_CI_BRANCH=${PMD_CI_GIT_REF##refs/heads/} + unset PMD_CI_TAG + log_info "Building PMD ${VERSION} on branch ${PMD_CI_BRANCH}" + elif [[ "${PMD_CI_GIT_REF}" == refs/tags/* ]]; then + unset PMD_CI_BRANCH + PMD_CI_TAG=${PMD_CI_GIT_REF##refs/tags/} + log_info "Building PMD ${VERSION} on tag ${PMD_CI_TAG}" + else + log_error "Unknown branch/tag: PMD_CI_GIT_REF=${PMD_CI_GIT_REF}" + exit 1 + fi + + if [[ "${VERSION}" == *-SNAPSHOT && -z "$PMD_CI_BRANCH" ]]; then + log_error "Invalid combination: snapshot version ${VERSION} but no branch in PMD_CI_GIT_REF=${PMD_CI_GIT_REF}" + exit 1 + fi + + if [[ "${VERSION}" != *-SNAPSHOT && -z "$PMD_CI_TAG" ]]; then + log_error "Invalid combination: non-snapshot version ${VERSION} but no tag in PMD_CI_GIT_REF=${PMD_CI_GIT_REF}" + exit 1 + fi +} + +# +# Performs the actual build. +# Deploys the artifacts to maven central. +# Also generates rule documentation. +# +function pmd_ci_build_run() { + local mvn_profiles="ossrh,sign,generate-rule-docs" + + if pmd_ci_build_isRelease; then + log_info "This is a release build" + mvn_profiles="${mvn_profiles},pmd-release" + else + log_info "This is a snapshot build" + fi + + ./mvnw clean deploy -P${mvn_profiles} -e -V -Djava7.home=${HOME}/oraclejdk7 +} + +# +# Deploys the binary distribution +# +function pmd_ci_deploy_build_artifacts() { + # Deploy to sourceforge files + sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip" + sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip" + + if pmd_ci_build_isRelease; then + # create a draft github release + gh_releases_createDraftRelease "${PMD_CI_TAG}" "$(git rev-list -n 1 ${PMD_CI_TAG})" + GH_RELEASE="$RESULT" + + # Deploy to github releases + gh_release_uploadAsset "$GH_RELEASE" "pmd-dist/target/pmd-bin-${VERSION}.zip" + gh_release_uploadAsset "$GH_RELEASE" "pmd-dist/target/pmd-src-${VERSION}.zip" + fi +} + +# +# Builds and uploads the documentation site +# +function pmd_ci_build_and_upload_doc() { + pmd_doc_generate_jekyll_site + pmd_doc_create_archive + + sourceforge_uploadFile "${VERSION}" "docs/pmd-doc-${VERSION}.zip" + if pmd_ci_build_isRelease; then + gh_release_uploadAsset "$GH_RELEASE" "docs/pmd-doc-${VERSION}.zip" + fi + + # Deploy doc to https://docs.pmd-code.org/pmd-doc-${VERSION}/ + pmd_code_uploadDocumentation "${VERSION}" "docs/pmd-doc-${VERSION}.zip" + # Deploy javadoc to https://docs.pmd-code.org/apidocs/*/${VERSION}/ + pmd_code_uploadJavadoc "${VERSION}" "$(pwd)" + + if [[ "${VERSION}" == *-SNAPSHOT && "${PMD_CI_BRANCH}" == "master" ]]; then + # only for snapshot builds from branch master + pmd_code_createSymlink "${VERSION}" "snapshot" + + # update github pages https://pmd.github.io/pmd/ + pmd_doc_publish_to_github_pages + # rsync site to https://pmd.sourceforge.io/snapshot + sourceforge_rsyncSnapshotDocumentation "${VERSION}" "snapshot" + fi + + if pmd_ci_build_isRelease; then + # documentation is already uploaded to https://docs.pmd-code.org/pmd-doc-${VERSION} + # we only need to setup symlinks for the released version + pmd_code_createSymlink "${VERSION}" "latest" + # remove old doc and point to the new version + pmd_code_removeDocumentation "${VERSION}-SNAPSHOT" + pmd_code_createSymlink "${VERSION}" "${VERSION}-SNAPSHOT" + # remove old javadoc + pmd_code_removeJavadoc "${VERSION}-SNAPSHOT" + + # updating github release text + # renders, and skips the first 6 lines - the Jekyll front-matter + local rendered_release_notes=$(bundle exec .ci/render_release_notes.rb docs/pages/release_notes.md | tail -n +6) + local release_name="PMD ${VERSION} ($(date -u +%d-%B-%Y))" + gh_release_updateRelease "$GH_RELEASE" "$release_name" "$rendered_release_notes" + sourceforge_uploadReleaseNotes "${VERSION}" "${rendered_release_notes}" + + publish_release_documentation_github + sourceforge_rsyncSnapshotDocumentation "${VERSION}" "pmd-${VERSION}" + fi +} + + +function pmd_ci_build_isRelease() { + if [[ "${VERSION}" != *-SNAPSHOT && -n "${PMD_CI_TAG}" && -z "${PMD_CI_BRANCH}" ]]; then + return 0 + else + return 1 + fi +} + +function pmd_ci_build_get_pom_version() { + echo $(./mvnw -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) +} + + +pmd_ci_build_main diff --git a/.ci/check-environment.sh b/.ci/check-environment.sh new file mode 100755 index 0000000000..d164569a19 --- /dev/null +++ b/.ci/check-environment.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# +# This script should check, that all needed commands are available +# and are in the correct version. +# + +source $(dirname $0)/inc/logger.inc + +set -e + +function check() { + local CMD=$1 + local VERSION_CMD=$2 + local VERSION_STRING=$3 + + echo -n "Checking ${CMD}..." + + if hash "$CMD" 2>/dev/null; then + local VERSION_FULL=$(${VERSION_CMD} 2>&1) + local VERSION=$(echo "${VERSION_FULL}" | grep "${VERSION_STRING}" 2>&1) + if [ -n "${VERSION}" ]; then + echo -e "${COL_GREEN}OK${COL_RESET}" + echo " ${VERSION}" + else + echo -e "${COL_RED}wrong version${COL_RESET}. Expected: ${VERSION_STRING}" + echo " ${VERSION_FULL}" + fi + else + echo -e "${COL_RED}not found!${COL_RESET}" + fi +} + +# every OS: +check "curl" "curl --version" "curl" +check "jq" "jq --version" "jq" + +case "$(uname)" in + Linux*) + check "ruby" "ruby --version" "ruby 2.7" + check "gpg" "gpg --version" "gpg (GnuPG) 2." + check "printenv" "printenv --version" "printenv (GNU coreutils)" + check "rsync" "rsync --version" "version" + check "ssh" "ssh -V" "OpenSSH" + check "git" "git --version" "git version" + check "mvn" "mvn --version" "Apache Maven" + check "unzip" "unzip --version" "UnZip" + check "zip" "zip --version" "This is Zip" + #check "7z" "7z -version" "7-Zip" + ;; + Darwin*) + ;; + CYGWIN*|MINGW*) + check "7z" "7z -version" "7-Zip" + ;; + *) + log_error "Unknown OS: $(uname)" + exit 1 + ;; +esac diff --git a/.ci/docker_ubuntu18.04/Dockerfile b/.ci/docker_ubuntu18.04/Dockerfile new file mode 100644 index 0000000000..8317beb799 --- /dev/null +++ b/.ci/docker_ubuntu18.04/Dockerfile @@ -0,0 +1,31 @@ +# https://hub.docker.com/_/ubuntu/ +FROM ubuntu:18.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get upgrade --yes && \ + apt-get install --yes curl jq gpg rsync ssh git p7zip-full openjdk-8-jdk \ + libgdbm-dev libncurses5-dev automake libtool bison libffi-dev \ + sudo nano bash tzdata unzip zip && \ + apt-get clean +RUN cd opt && \ + curl https://mirror.checkdomain.de/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz | tar xz && \ + ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/local/bin/mvn && \ + cd .. +RUN groupadd --gid 1000 pmd-ci && useradd --gid 1000 --uid 1000 --groups sudo \ + --shell /bin/bash --create-home --password "" \ + pmd-ci + +RUN mkdir -p /workspaces/pmd/pmd && chown -R pmd-ci:pmd-ci /workspaces && ln -sf /workspaces /home/pmd-ci/workspaces + +USER pmd-ci:pmd-ci +WORKDIR /home/pmd-ci +COPY create-gh-pull-request-event.sh . +COPY install-ruby.sh . +RUN ./install-ruby.sh + +CMD ["/bin/bash", "--login"] + +# +# build with: docker build -t pmd-ci . +# run with: docker run -it pmd-ci +# \ No newline at end of file diff --git a/.ci/docker_ubuntu18.04/create-gh-pull-request-event.sh b/.ci/docker_ubuntu18.04/create-gh-pull-request-event.sh new file mode 100755 index 0000000000..7f3576196e --- /dev/null +++ b/.ci/docker_ubuntu18.04/create-gh-pull-request-event.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +cat > /workspaces/event.json < ossrh - ${env.CI_DEPLOY_USERNAME} + ${env.CI_DEPLOY_USER} ${env.CI_DEPLOY_PASSWORD} @@ -22,7 +22,7 @@ ossrh - ${env.CI_SIGN_KEYNAME} + ${env.CI_SIGN_KEY} ${env.CI_SIGN_PASSPHRASE} diff --git a/.ci/files/private-env.gpg b/.ci/files/private-env.gpg new file mode 100644 index 0000000000..af292ee8b5 Binary files /dev/null and b/.ci/files/private-env.gpg differ diff --git a/.travis/project-list.xml b/.ci/files/project-list.xml similarity index 56% rename from .travis/project-list.xml rename to .ci/files/project-list.xml index 600061f674..d150ae3443 100644 --- a/.travis/project-list.xml +++ b/.ci/files/project-list.xml @@ -18,6 +18,8 @@ if test -e classpath.txt; then exit fi +set -e + mvn test-compile mvn dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=classpath.txt ]]> @@ -35,30 +37,56 @@ if test -e classpath.txt; then exit fi +set -e + # Note: openjdk8 will be installed by "before_install.sh" -JAVA_HOME=${HOME}/openjdk8 -PATH=$JAVA_HOME/bin:$PATH +export JAVA_HOME=${HOME}/openjdk8 +export PATH=$JAVA_HOME/bin:$PATH + +# keep the tabs!! +(cat <> build.gradle < classpath.txt ]]> cat classpath.txt diff --git a/.ci/files/public-env.gpg b/.ci/files/public-env.gpg new file mode 100644 index 0000000000..8cae0c6b6a Binary files /dev/null and b/.ci/files/public-env.gpg differ diff --git a/.ci/files/release-signing-key-D0BF1D737C9A1C22.gpg.gpg b/.ci/files/release-signing-key-D0BF1D737C9A1C22.gpg.gpg new file mode 100644 index 0000000000..d0b2babc1c Binary files /dev/null and b/.ci/files/release-signing-key-D0BF1D737C9A1C22.gpg.gpg differ diff --git a/.travis/github-releases-api.sh b/.ci/inc/github-releases-api.inc similarity index 99% rename from .travis/github-releases-api.sh rename to .ci/inc/github-releases-api.inc index c30c223f75..e71e67f941 100644 --- a/.travis/github-releases-api.sh +++ b/.ci/inc/github-releases-api.inc @@ -1,6 +1,6 @@ # # The functions here require the following scripts: -# logger.sh +# logger.inc # # The functions here require the following environment variables: # GITHUB_OAUTH_TOKEN diff --git a/.ci/inc/install-openjdk.inc b/.ci/inc/install-openjdk.inc new file mode 100644 index 0000000000..c9a963242b --- /dev/null +++ b/.ci/inc/install-openjdk.inc @@ -0,0 +1,103 @@ + +# needs: +# inc/logger + +# +# Downloads openjdk from AdoptOpenJDK by accessing the API. +# The API is documented at https://api.adoptopenjdk.net/swagger-ui/ +# + +function install_openjdk() { + OPENJDK_VERSION=$1 + + case "$(uname)" in + Linux*) + JDK_OS=linux + COMPONENTS_TO_STRIP=1 # e.g. openjdk-11.0.3+7/bin/java + ;; + Darwin*) + JDK_OS=mac + COMPONENTS_TO_STRIP=3 # e.g. jdk-11.0.3+7/Contents/Home/bin/java + ;; + CYGWIN*|MINGW*) + JDK_OS=windows + ;; + *) + log_error "Unknown OS: $(uname)" + exit 1 + ;; + esac + + + DOWNLOAD_URL=$(curl --silent -X GET "https://api.adoptopenjdk.net/v3/assets/feature_releases/${OPENJDK_VERSION}/ga?architecture=x64&heap_size=normal&image_type=jdk&jvm_impl=hotspot&os=${JDK_OS}&page=0&page_size=1&project=jdk&sort_method=DEFAULT&sort_order=DESC&vendor=adoptopenjdk" \ + -H "accept: application/json" \ + | jq -r ".[0].binaries[0].package.link") + + OPENJDK_ARCHIVE=$(basename ${DOWNLOAD_URL}) + log_debug "Archive name: ${OPENJDK_ARCHIVE}" + + CACHE_DIR=${HOME}/.cache/openjdk + TARGET_DIR=${HOME}/openjdk${OPENJDK_VERSION} + + mkdir -p ${CACHE_DIR} + mkdir -p ${TARGET_DIR} + + if [ ! -e ${CACHE_DIR}/${OPENJDK_ARCHIVE} ]; then + log_info "Downloading from ${DOWNLOAD_URL} to ${CACHE_DIR}" + curl --location --output ${CACHE_DIR}/${OPENJDK_ARCHIVE} "${DOWNLOAD_URL}" + else + log_info "Skipped download, file ${CACHE_DIR}/${OPENJDK_ARCHIVE} already exists" + fi + + log_info "Extracting to ${TARGET_DIR}" + + case "$OPENJDK_ARCHIVE" in + *.zip) + 7z x ${CACHE_DIR}/${OPENJDK_ARCHIVE} -o${TARGET_DIR} + mv ${TARGET_DIR}/*/* ${TARGET_DIR}/ + ;; + *.tar.gz) + tar --extract --file ${CACHE_DIR}/${OPENJDK_ARCHIVE} -C ${TARGET_DIR} --strip-components=${COMPONENTS_TO_STRIP} + ;; + *) + log_error "Unknown filetype: ${OPENJDK_ARCHIVE}" + exit 1 + ;; + esac +} + +function install_openjdk_setdefault() { + OPENJDK_VERSION=$1 + + install_openjdk $OPENJDK_VERSION + + log_info "Using OpenJDK ${OPENJDK_VERSION} as default" + TARGET_DIR=${HOME}/openjdk${OPENJDK_VERSION} + export JAVA_HOME="${TARGET_DIR}" + export PATH="${TARGET_DIR}/bin:${PATH}" + + java -version +} + +# +# Installs jdk7 for integration test +# +function install_oraclejdk7() { + local local_dir="${HOME}/.cache/jdk7" + local target_dir="${HOME}/oraclejdk7" + local download_url="https://pmd-code.org/oraclejdk/jdk-7u80-linux-x64.tar.gz" + local archive=$(basename $download_url) + + mkdir -p ${local_dir} + mkdir -p ${target_dir} + if [ ! -e ${local_dir}/${archive} ]; then + log_info "Downloading from ${download_url} to ${local_dir}" + curl --location --output ${local_dir}/${archive} ${download_url} + else + log_info "Skipped download, file ${local_dir}/${archive} already exists" + fi + log_info "Extracting to ${target_dir}" + tar --extract --file ${local_dir}/${archive} -C ${target_dir} --strip-components=1 + + log_info "OracleJDK7 can be used via -Djava7.home=${HOME}/oraclejdk7" +} diff --git a/.travis/logger.sh b/.ci/inc/logger.inc similarity index 75% rename from .travis/logger.sh rename to .ci/inc/logger.inc index cd76d5bd3a..cf002de9a1 100644 --- a/.travis/logger.sh +++ b/.ci/inc/logger.inc @@ -1,5 +1,3 @@ -#!/bin/bash - COL_GREEN="\e[32m" COL_RED="\e[31m" COL_RESET="\e[0m" @@ -21,3 +19,12 @@ function log_debug() { true #echo -e "[DEBUG ] $*" } + +function log_group_start() { + echo "::group::$*" + log_info $* +} + +function log_group_end() { + echo "::endgroup::" +} \ No newline at end of file diff --git a/.ci/inc/maven-dependencies.inc b/.ci/inc/maven-dependencies.inc new file mode 100644 index 0000000000..0aaa020b81 --- /dev/null +++ b/.ci/inc/maven-dependencies.inc @@ -0,0 +1,30 @@ + +# +# needs "inc/logger.inc" +# + +# +# On azure, outgoing idle connection are dropped after 4 minutes. +# Usually, you can configure wagon with ttl. But these settings are +# ignored, as soon as dokka-maven-plugin is loaded. +# dokka-maven-plugin tries to load additional dependencies at runtime +# and injects a different http client, which is not configured correctly +# and thus maven fails if it tries to download later in the build process +# further dependencies. +# +# The workaround applied here is: first resolve all dependencies, +# then explicitly get dokka-maven-plugin and then resolve all plugins +# execpt for dokka-maven-plugin, as it does not play well with dependency-plugin. +# +function maven_dependencies_resolve() { + dokka_version=$(./mvnw -q -Dexec.executable="echo" -Dexec.args='${dokka.version}' \ + --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) + + ./mvnw dependency:resolve + ./mvnw dependency:get -DgroupId=org.jetbrains.dokka \ + -DartifactId=dokka-maven-plugin \ + -Dversion=${dokka_version} \ + -Dpackaging=jar \ + -DremoteRepositories=jcenter::default::https://jcenter.bintray.com/ + ./mvnw dependency:resolve-plugins -DexcludeGroupIds=org.jetbrains.dokka -Psign +} diff --git a/.travis/pmd-code-api.sh b/.ci/inc/pmd-code-api.inc similarity index 99% rename from .travis/pmd-code-api.sh rename to .ci/inc/pmd-code-api.inc index 5f30afb09d..d9e75beb7c 100644 --- a/.travis/pmd-code-api.sh +++ b/.ci/inc/pmd-code-api.inc @@ -1,6 +1,6 @@ # # The functions here require the following scripts: -# .travis/logger.sh +# inc/logger.inc # PMD_CODE_SSH_USER=pmd diff --git a/.ci/inc/pmd-doc.inc b/.ci/inc/pmd-doc.inc new file mode 100644 index 0000000000..6054baca46 --- /dev/null +++ b/.ci/inc/pmd-doc.inc @@ -0,0 +1,111 @@ + +# Used env vars: +# PMD_CI_JOB_URL +# PMD_CI_PUSH_COMMIT_COMPARE + + +# +# Executes jekyll and generates the documentation +# The documentation will be generated in the directory "docs/_site". +# +function pmd_doc_generate_jekyll_site() { + pushd docs + + echo -e "\n\n" + log_info "Building documentation using jekyll..." + bundle config set --local path vendor/bundle + bundle install + bundle exec jekyll build + + popd +} + +# +# Creates the pmd-doc.zip archive. It will be placed in "docs/". +# +function pmd_doc_create_archive() { + pushd docs + + echo -e "\n\n" + log_info "Creating pmd-doc archive..." + mv _site pmd-doc-${VERSION} + zip -qr pmd-doc-${VERSION}.zip pmd-doc-${VERSION}/ + log_success "Successfully created pmd-doc-${VERSION}.zip" + + popd +} + +# +# Publishes the site to https://pmd.github.io/pmd-${VERSION} and +# https://pmd.github.io/latest/ +# +function publish_release_documentation_github() { + echo -e "\n\n" + log_info "Adding the new doc to pmd.github.io..." + # clone pmd.github.io. Note: This uses the ssh key setup earlier + # In order to speed things up, we use a sparse checkout - no need to checkout all directories here + mkdir pmd.github.io + ( + cd pmd.github.io + git init + git config user.name "PMD CI (pmd-bot)" + git config user.email "andreas.dangel+pmd-bot@adangel.org" + git config core.sparsecheckout true + git remote add origin git@github.com:pmd/pmd.github.io.git + echo "/latest/" > .git/info/sparse-checkout + echo "/sitemap.xml" >> .git/info/sparse-checkout + git pull --depth=1 origin master + log_info "Copying documentation from ../docs/pmd-doc-${VERSION}/ to pmd-${VERSION}/ ..." + rsync -ah --stats ../docs/pmd-doc-${VERSION}/ pmd-${VERSION}/ + git status + echo "Executing: git add pmd-${VERSION}" + git add pmd-${VERSION} + echo "Executing: git commit..." + git commit -q -m "Added pmd-${VERSION}" + + log_info "Copying pmd-${VERSION} to latest ..." + git rm -qr latest + cp -a pmd-${VERSION} latest + echo "Executing: git add latest" + git add latest + echo "Executing: git commit..." + git commit -q -m "Copying pmd-${VERSION} to latest" + + log_info "Generating sitemap.xml" + ../docs/sitemap_generator.sh > sitemap.xml + echo "Executing: git add sitemap.xml" + git add sitemap.xml + echo "Executing: git commit..." + git commit -q -m "Generated sitemap.xml" + + echo "Executing: git push origin master" + git push origin master + ) +} + +# +# Updates github pages of the main repository, +# so that https://pmd.github.io/pmd/ has the latest (snapshot) content +# +function pmd_doc_publish_to_github_pages() { + echo -e "\n\n" + log_info "Pushing the new site to github pages..." + git clone --branch gh-pages --depth 1 git@github.com:pmd/pmd.git pmd-gh-pages + # clear the files first + rm -rf pmd-gh-pages/* + # copy the new site + cp -a docs/pmd-doc-${VERSION}/* pmd-gh-pages/ + ( + cd pmd-gh-pages + git config user.name "PMD CI (pmd-bot)" + git config user.email "andreas.dangel+pmd-bot@adangel.org" + git add -A + MSG="Update documentation + +${PMD_CI_JOB_URL} +${PMD_CI_PUSH_COMMIT_COMPARE}" + git commit -q -m "$MSG" + git push git@github.com:pmd/pmd.git HEAD:gh-pages + log_success "Successfully pushed site to https://pmd.github.io/pmd/" + ) +} diff --git a/.travis/regression-tester.sh b/.ci/inc/regression-tester.inc similarity index 51% rename from .travis/regression-tester.sh rename to .ci/inc/regression-tester.inc index 6d48090dbd..1083c81e39 100644 --- a/.travis/regression-tester.sh +++ b/.ci/inc/regression-tester.inc @@ -1,10 +1,36 @@ # # The functions here require the following scripts: -# .travis/logger.sh -# .travis/common-functions.sh +# inc/logger.inc +# inc/install-openjdk.inc # # The functions here require the following environment variables: # PMD_SF_USER +# PMD_CI_BRANCH +# +# DANGER_GITHUB_API_TOKEN +# PMD_CI_CHUNK_TOKEN + +function regression_tester_setup_ci() { + log_info "Install openjdk8 for pmd-regression-tests" + install_openjdk 8 + + gpg --batch --yes --decrypt --passphrase="GnxdjywUEPveyCD1RLiTd7t8CImnefYr" \ + --output .ci/files/public-env .ci/files/public-env.gpg + source .ci/files/public-env >/dev/null 2>&1 + rm .ci/files/public-env + + if hash "bundler" 2>/dev/null; then + log_debug "Bundler is already installed" + else + log_info "Installing bundler..." + gem install bundler + fi + + rm -f .bundle/config + bundle config set --local path vendor/bundle + bundle config set --local with release_notes_preprocessing + bundle install +} # # Generate a new baseline and upload it to sourceforge @@ -12,9 +38,8 @@ # Note: this function always succeeds, even if the upload fails. # In that case, just a error logging is provided. # -function regression-tester_uploadBaseline() { - change_ruby_version - log_debug "$FUNCNAME branch=${TRAVIS_BRANCH}" +function regression_tester_uploadBaseline() { + log_debug "$FUNCNAME branch=${PMD_CI_BRANCH}" local targetUrl="https://sourceforge.net/projects/pmd/files/pmd-regression-tester/" local errexitstate="$(shopt -po errexit)" @@ -33,11 +58,16 @@ function regression-tester_uploadBaseline() { log_info "Generating and uploading baseline for pmdtester..." cd .. bundle config --local gemfile pmd/Gemfile - pmd/.travis/travis_wait "bundle exec pmdtester --mode single --local-git-repo ./pmd --patch-branch ${TRAVIS_BRANCH} --patch-config ./pmd/.travis/all-java.xml --list-of-project ./pmd/.travis/project-list.xml --html-flag" + pmd/.ci/travis_wait "bundle exec pmdtester + --mode single + --local-git-repo ./pmd + --patch-branch ${PMD_CI_BRANCH} + --patch-config ./pmd/.ci/files/all-java.xml + --list-of-project ./pmd/.ci/files/project-list.xml --html-flag" cd target/reports - BRANCH_FILENAME="${TRAVIS_BRANCH/\//_}" + BRANCH_FILENAME="${PMD_CI_BRANCH/\//_}" zip -q -r ${BRANCH_FILENAME}-baseline.zip ${BRANCH_FILENAME}/ - ../../pmd/.travis/travis_wait "rsync -avh ${BRANCH_FILENAME}-baseline.zip ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd-regression-tester/" + ../../pmd/.ci/travis_wait "rsync -avh ${BRANCH_FILENAME}-baseline.zip ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd-regression-tester/" log_success "Successfully uploaded ${BRANCH_FILENAME}-baseline.zip to ${targetUrl}" ) # restore errexit state @@ -50,8 +80,7 @@ function regression-tester_uploadBaseline() { # Note: this function always succeeds, even if the danger fails. # In that case, just a error logging is provided. # -function regression-tester_executeDanger() { - change_ruby_version +function regression_tester_executeDanger() { log_debug "$FUNCNAME" local errexitstate="$(shopt -po errexit)" @@ -67,13 +96,15 @@ function regression-tester_executeDanger() { trap danger_failed ERR # Create a corresponding remote branch locally - if ! git show-ref --verify --quiet refs/heads/${TRAVIS_BRANCH}; then - git fetch --no-tags origin +refs/heads/${TRAVIS_BRANCH}:refs/remotes/origin/${TRAVIS_BRANCH} - git branch ${TRAVIS_BRANCH} origin/${TRAVIS_BRANCH} - log_debug "Created local branch ${TRAVIS_BRANCH}" + if ! git show-ref --verify --quiet refs/heads/${PMD_CI_BRANCH}; then + git fetch --no-tags --depth=1 origin +refs/heads/${PMD_CI_BRANCH}:refs/remotes/origin/${PMD_CI_BRANCH} + git branch ${PMD_CI_BRANCH} origin/${PMD_CI_BRANCH} + log_debug "Created local branch ${PMD_CI_BRANCH}" fi + # Fetch more commits of the PR for danger + git fetch --no-tags --depth=50 origin +$(git rev-parse HEAD^2): - log_info "Running danger on branch ${TRAVIS_BRANCH}" + log_info "Running danger on branch ${PMD_CI_BRANCH}" bundle exec danger --verbose log_success "Executing danger successfully" ) diff --git a/.ci/inc/setup-secrets.inc b/.ci/inc/setup-secrets.inc new file mode 100644 index 0000000000..6cd3ba953b --- /dev/null +++ b/.ci/inc/setup-secrets.inc @@ -0,0 +1,62 @@ +function pmd_ci_setup_private_env() { + log_info "Setting up secrets as environment variables..." + local -r ENV_FILE=.ci/files/private-env + + printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \ + --passphrase-fd 0 \ + --output ${ENV_FILE} ${ENV_FILE}.gpg + + source ${ENV_FILE} >/dev/null 2>&1 + rm ${ENV_FILE} +} + +function pmd_ci_setup_gpg_key() { + log_info "Setting up GPG release signing key..." + local -r GPG_FILE=.ci/files/release-signing-key-D0BF1D737C9A1C22.gpg + + mkdir -p "${HOME}/.gpg" + printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \ + --passphrase-fd 0 \ + --output ${GPG_FILE} ${GPG_FILE}.gpg + gpg --batch --import ${GPG_FILE} + rm ${GPG_FILE} +} + +function pmd_ci_setup_ssh() { + log_info "Setting up .ssh/id_rsa..." + local -r SSH_KEY_FILE=.ci/files/id_rsa + + printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \ + --passphrase-fd 0 \ + --output ${SSH_KEY_FILE} ${SSH_KEY_FILE}.gpg + chmod 600 ${SSH_KEY_FILE} + + mkdir -p ${HOME}/.ssh + chmod 700 "${HOME}/.ssh" + mv ${SSH_KEY_FILE} "${HOME}/.ssh/id_rsa" + + log_info "Setting up .ssh/known_hosts..." + # + # https://sourceforge.net/p/forge/documentation/SSH%20Key%20Fingerprints/ + # + # run locally: + # ssh-keyscan web.sourceforge.net | tee -a known_hosts + # + # verify fingerprints: + # ssh-keygen -F web.sourceforge.net -l -f known_hosts + # # Host web.sourceforge.net found: line 1 + # web.sourceforge.net RSA SHA256:xB2rnn0NUjZ/E0IXQp4gyPqc7U7gjcw7G26RhkDyk90 + # # Host web.sourceforge.net found: line 2 + # web.sourceforge.net ECDSA SHA256:QAAxYkf0iI/tc9oGa0xSsVOAzJBZstcO8HqGKfjpxcY + # # Host web.sourceforge.net found: line 3 + # web.sourceforge.net ED25519 SHA256:209BDmH3jsRyO9UeGPPgLWPSegKmYCBIya0nR/AWWCY + # + # then add output of `ssh-keygen -F web.sourceforge.net -f known_hosts` + # + echo 'web.sourceforge.net ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2uifHZbNexw6cXbyg1JnzDitL5VhYs0E65Hk/tLAPmcmm5GuiGeUoI/B0eUSNFsbqzwgwrttjnzKMKiGLN5CWVmlN1IXGGAfLYsQwK6wAu7kYFzkqP4jcwc5Jr9UPRpJdYIK733tSEmzab4qc5Oq8izKQKIaxXNe7FgmL15HjSpatFt9w/ot/CHS78FUAr3j3RwekHCm/jhPeqhlMAgC+jUgNJbFt3DlhDaRMa0NYamVzmX8D47rtmBbEDU3ld6AezWBPUR5Lh7ODOwlfVI58NAf/aYNlmvl2TZiauBCTa7OPYSyXJnIPbQXg6YQlDknNCr0K769EjeIlAfY87Z4tw==' >> "$HOME/.ssh/known_hosts" + echo 'web.sourceforge.net ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCwsY6sZT4MTTkHfpRzYjxG7mnXrGL74RCT2cO/NFvRrZVNB5XNwKNn7G5fHbYLdJ6UzpURDRae1eMg92JG0+yo=' >> "$HOME/.ssh/known_hosts" + echo 'web.sourceforge.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOQD35Ujalhh+JJkPvMckDlhu4dS7WH6NsOJ15iGCJLC' >> "$HOME/.ssh/known_hosts" + + # add pmd-code.org (ssh-keyscan pmd-code.org) + echo 'pmd-code.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVsIeF6xU0oPb/bMbxG1nU1NDyBpR/cBEPZcm/PuJwdI9B0ydPHA6FysqAnt32fNFznC2SWisnWyY3iNsP3pa8RQJVwmnnv9OboGFlW2/61o3iRyydcpPbgl+ADdt8iU9fmMI7dC04UqgHGBoqOwVNna9VylTjp5709cK2qHnwU450F6YcOEiOKeZfJvV4PmpJCz/JcsUVqft6StviR31jKnqbnkZdP8qNoTbds6WmGKyXkhHdLSZE7X1CFQH28tk8XFqditX93ezeCiThFL7EleDexV/3+2+cs5878sDMUMzHS5KShTjkxzhHaodhtIEdNesinq/hOPbxAGkQ0FbD' >> $HOME/.ssh/known_hosts +} diff --git a/.travis/sourceforge-api.sh b/.ci/inc/sourceforge-api.inc similarity index 94% rename from .travis/sourceforge-api.sh rename to .ci/inc/sourceforge-api.inc index 41933c9a21..24f165cf6f 100644 --- a/.travis/sourceforge-api.sh +++ b/.ci/inc/sourceforge-api.inc @@ -1,6 +1,6 @@ # # The functions here require the following scripts: -# .travis/logger.sh +# logger.inc # # The functions here require the following environment variables: # PMD_SF_USER @@ -86,7 +86,7 @@ function sourceforge_uploadFile() { trap upload_failed ERR log_info "Uploading $filename to sourceforge..." - .travis/travis_wait "rsync -avh ${filename} ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/${pmdVersion}/" + .ci/travis_wait "rsync -avh ${filename} ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/${pmdVersion}/" log_success "Successfully uploaded ${filename} to sourceforge: ${targetUrl}" ) # restore errexit state @@ -158,7 +158,7 @@ function sourceforge_rsyncSnapshotDocumentation() { trap upload_failed ERR log_info "Uploading documentation to ${targetUrl}..." - .travis/travis_wait "rsync -ah --stats --delete docs/pmd-doc-${VERSION}/ ${PMD_SF_USER}@web.sourceforge.net:/home/project-web/pmd/htdocs/snapshot/" + .ci/travis_wait "rsync -ah --stats --delete docs/pmd-doc-${VERSION}/ ${PMD_SF_USER}@web.sourceforge.net:/home/project-web/pmd/htdocs/snapshot/" log_success "Successfully uploaded documentation: ${targetUrl}" ) # restore errexit state diff --git a/.travis/render_release_notes.rb b/.ci/render_release_notes.rb similarity index 100% rename from .travis/render_release_notes.rb rename to .ci/render_release_notes.rb diff --git a/.travis/travis_wait b/.ci/travis_wait similarity index 100% rename from .travis/travis_wait rename to .ci/travis_wait diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 3222a0467a..0000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Java CI - -on: [push, pull_request] - -jobs: - build: - runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.experimental }} - if: "!contains(github.event.head_commit.message, '[skip ci]')" - strategy: - matrix: - os: [ ubuntu-latest , windows-latest , macos-latest ] - java: [ 11 ] - experimental: [ false ] - - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - name: Build with mvnw - run: | - ./mvnw -V clean install diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml new file mode 100644 index 0000000000..a07af8d1e5 --- /dev/null +++ b/.github/workflows/pull-requests.yml @@ -0,0 +1,40 @@ +name: Pull Requests + +on: pull_request + +jobs: + build: + runs-on: ${{ matrix.os }} + continue-on-error: false + timeout-minutes: 30 + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + vendor/bundle + key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}- + - name: Set up Ruby 2.7 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.7 + - name: Check Environment + run: .ci/check-environment.sh + shell: bash + - name: Build + run: .ci/build-pr-win-macos.sh + shell: bash + env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 + PMD_CI_BRANCH: ${{ github.event.pull_request.base.ref }} + PMD_CI_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/pushes.yml b/.github/workflows/pushes.yml new file mode 100644 index 0000000000..a47ef7d51a --- /dev/null +++ b/.github/workflows/pushes.yml @@ -0,0 +1,118 @@ +name: Pushes +on: + push: + branches: + - main + - master + schedule: + # build it monthly: At 04:00 on day-of-month 1. + - cron: '0 4 1 * *' + +jobs: + linux: + runs-on: ubuntu-latest + continue-on-error: false + if: "!contains(github.event.head_commit.message, '[skip ci]')" + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + vendor/bundle + key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}- + - name: Set up Ruby 2.7 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.7 + - name: Check Environment + run: .ci/check-environment.sh + shell: bash + - name: Build + run: .ci/build.sh + shell: bash + env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 + PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} + PMD_CI_JOB_URL: "https://github.com/pmd/pmd/runs/${{ github.run_id }}" + PMD_CI_PUSH_COMMIT_COMPARE: ${{ github.event.compare }} + PMD_CI_GIT_REF: ${{ github.ref }} + + windows-macos: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ windows-latest, macos-latest ] + needs: linux + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + vendor/bundle + key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}- + - name: Check Environment + run: .ci/check-environment.sh + shell: bash + - name: Build + run: .ci/build-pr-win-macos.sh + shell: bash + env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 + + coveralls: + runs-on: ubuntu-latest + needs: [linux] + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + vendor/bundle + key: ${{ runner.os }}-coveralls-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-coveralls- + - name: Check Environment + run: .ci/check-environment.sh + shell: bash + - name: Build + run: .ci/build-coveralls.sh + shell: bash + env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 + PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} + PMD_CI_JOB_URL: "https://github.com/pmd/pmd/runs/${{ github.run_id }}" + + sonar: + runs-on: ubuntu-latest + needs: [linux] + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + vendor/bundle + key: ${{ runner.os }}-sonar-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-sonar- + - name: Check Environment + run: .ci/check-environment.sh + shell: bash + - name: Build + run: .ci/build-sonar.sh + shell: bash + env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 + PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml new file mode 100644 index 0000000000..2948cb6653 --- /dev/null +++ b/.github/workflows/releases.yml @@ -0,0 +1,24 @@ +name: Release Builds +on: + push: + tags: + - 'pmd_releases/*' + +jobs: + release: + runs-on: ubuntu-latest + continue-on-error: false + steps: + - uses: actions/checkout@v2 + - name: Check Environment + run: .ci/check-environment.sh + shell: bash + - name: Build + run: .ci/build.sh + shell: bash + env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 + PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} + PMD_CI_JOB_URL: "https://github.com/pmd/pmd/runs/${{ github.run_id }}" + PMD_CI_PUSH_COMMIT_COMPARE: ${{ github.event.compare }} + PMD_CI_GIT_REF: ${{ github.ref }} diff --git a/.github/workflows/troubleshooting.yml b/.github/workflows/troubleshooting.yml new file mode 100644 index 0000000000..8065541981 --- /dev/null +++ b/.github/workflows/troubleshooting.yml @@ -0,0 +1,36 @@ +name: troubleshooting + +on: workflow_dispatch + +jobs: + build: + runs-on: ${{ matrix.os }} + continue-on-error: false + strategy: + matrix: + #os: [ ubuntu-latest, windows-latest, macos-latest ] + os: [ ubuntu-latest ] + + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + vendor/bundle + key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}- + - name: Set up Ruby 2.7 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.7 + - name: Install OpenJDK 11 + run: | + source .ci/inc/logger.inc + source .ci/inc/install-openjdk.inc + install_openjdk 11 + shell: bash + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 34758b8d66..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,94 +0,0 @@ -language: java -os: linux -env: - global: - - secure: "KBEuB6U1p5RQXSYe157AwydFr/zpXQPA0IChVCgZV+X1mMyy9ZtrjH1J1AXuviseDDXDbaT25sRnsvpl82rfRw2xOkMGXHy4N95/ylTSr8DjHxTao71BhXsvFycNobFva5y2EGNWqDvpS8I2oSZo7Qk4la3yep3rcJQvcy6RDbbhpDTbL1QMFyadunIBm0WtqbunrMqtjSqaoPsXz8TiQuxHvX4vEXzVbaxV1QQt79Vi+daa6wAV3mRQAugnx+UffsC8JqMxgm06usWeJgCJzxgm8E7clZCLmf53B2TL8dK6bIYbqyvOY3uFxitsTG0d8Z0GOJwXBgZNgbniTRO8ZJSty5eZP8LBybbjVLSL25DNTWtCjADUL/uySnXIEidlMt2N/3QmH7zrGAfAk/tIwKpdRca2GLLydeXf6PSkiahnPEkIY/QupcsOLELhdifpdOjb8QW1OenA+vUbNM9dccLwKnX6Fj9cu4VQG601AcYDr2eyhq8WYkr3wYdw/6KdUa3hmplowTBs+qguppP+eOSgGuEsy38KLtqnvm6WlHy6tcLmcVYKG3DmR1b7TWXsOXC6/VMH8BHBkvsF1QdRg9+Cgx07vX3Hw7roPiYzmaO9Ajs20ATsUfRskMuWCTeTSK5pN8X27veRCZlhFjeKQMDdmfVwzpAfRgKsl3TEn1I=" - - secure: "U1DfAv6acUUWe+dao/ZSDUX64JRadNJY16rITsdrM4ZNAJSuXpEY3p/LWcYjN7D49YmyutbXH9+L3KKQUQGrGXj9QTarfYvd8ZsKt4FK8yv7AFy+RQNIbAjNEHBzHx15p+srMheTaetl7aLwY0qhF+D/RtGapxHKyY4dBHrb0lp8VGyiCiL7Aop8GGskosi1mtirPBp/BStPZ2bEyxG0QzU5SsVWkJWwV9aWLPVAR/n7Xgx/6Gjl6Fed2c/WSrWi4vchm3Ny8pfTweOax3PGYYjBVxIfuX0mqmwuJsY7gNfXCfN3dPiPKGJPFy1pC+LGyGkklO5ReKFLd4O1ME6fU0dlIGfD6n+Q4H6/w9FHXegcKTfWIJm/MFa6vA/tJM5R6zJQuiTQJboHm/UmS/iQj76z0p6sK15Xp5vFId+/dHKqa8xY+Bt6HiXy6z401HOc8QcYBAf7TqhqUt/ZE7HN4be46uR90KmzIrWz6wEoDW7HfwQ9ZMbs55zoOXrvekyE9/gXskypO0p2JT3Y0vlvO27KQvIrSwI480kOVOrlyrYA+LZqlcKaayOuCuZh4lITQUYinUoZZict5joYthH+Cyh2zovoBpxsntDJdMnaZNLtSC7hlhpbMBYaT2y1O6vZH5Yix0mxuDvs/x6ogP5CNBeUYlXhaL+g4GnwKyr0ZA0=" - - secure: "gxI7W4V4fUPQLMCvecXXdet1/mCh6m0RIQMtErVVOnwxEEjeDmko/3rHSl+wk6IeFaaQKcmHJJSJEj+e5TgWRH1uuzCnbFHZhuH28ce/H0EqLJ3GTXeDEgFLzhh//T3ySOZChZELgJ67cKxZJNFMhBVOR2/QIGjZjdzvl36ugsu80Ak3XJb8HCm8D2P8Vuezz/OlCkSOGXaiqZXfflV/cuaLWuueGfQGW9x/UyDTNsuBdB7YnEcxOWt8RgB4JrcPGV7/etxLHXTw9IMEeUhQ/RsxLZHKNYt9cx/QOZkpXemhdT9L2pAIi77eO98x+yAfB1qV6T6IeDd/OiABUzfb7asNBwE+bYRuVkCiWLo3x3hs8FcgsPe4L10guxfRMJPxax9E7uYb9TFxtvc42dDWNyp1BVCtEtmjErO4CJeW97ukhgkUkPhzUZgQ2Gj5Tn4VRjQOQyzh/S/YsHyfxoAQrscc/3bq5PBmtdowyGQd5dsvShdpE6glf+HnBZ8TCw5BaxQuTjapQWjsGuJVCVsrNDtwOFy5UTfxK9OeTUsjwfO5JjkoIeqGyLHPnTMWyZ6EYa//nWPOU1KgVE6Hrzfr3zxEzL1nN/TYahzhKOxXE701W9YhTmhCVljMlSYioTUfTaXHgGLpojvSWlhsTbc3r0gj4NglqakIy7MKgQLgES8=" - - secure: "otZkFNhApMofp2Jl2baZdpNEPcTa79Xc9wyOw1gl1+ubCLtOGr32FDUSr0HQWKrGXcYZR4bc0A80hx0/ykZ/twJaRAabcNYtjwRMDKKrp3plvp8RTZlB1QW85nsdmk3qmHACzc94wKnmcaDohM5FPxnWspZG5Y4ejYX4HEK6r1fwL4FjMPSLd5QW8yCaAc7dpBSm2UyIXuuDjkekXblJFe8ydbf9aF97WKJMCwqIs9Zo5oJa2pnWCw47ixwjVdkaH5x5727I+YmFqn5rjdEideUFEAlggqMaxuOtfYhuG/8k5W3Q/+WX+G7rEPTcemjmaosbbpoOUE0YUHV3Wi3R5D66NuWIpQwX6kw4rTwL0YIh//0aNb8jv4Z8KwkiTMrYuj0we5tXzARp3yYn56P0JBAm6BihNhfEBv06655Tg5LUsWqUTZCl3KJhwtO1N+OuOkE5qoZm59knq6DeZOsYWAVb3MpzklP06kXGYTjb44PVMUc6ynuuZjgPJDv/0906feA3REHBJVxd3PvJp1c8VvQSeW3ndghEz0akJHY/jxV1rJZAZC1ZeshYUpBNmh0sFDty/UE43IvIB4EInypQHyTtQvMc6LuLjTeu2E5ZMsA3udj66EMrYm4Y2eheJycSfttDqzhsKXIVJ4LmA/xtDR01cp1jsqbbZBhVI67rA40=" - - secure: "mmHRnK/8CHfgtwaTN3MgjFD1R8EVQhIb/a0Aag3gPaHWW52Ex6xWG7GsF4kRcdzHdIdrQxHZTiDa2oVMWbnQ16CAu4dZJadwTxhBW4JmE1tsi2C6i+uwMBPvnKBXHwt+QOAqNB0L/ByYwWLkucSLoyRs6qU0i7sJ3ZC/EPCuLI8VfBlFUVegSbp2WmclmFnxOi5MKr15Zwd8BGgyWUsvd1OHxhvO3Xr/ZPO4R+1kIgyojloTMu6oGM/shYgsrUXgmI/WvZRTq+25SRCjUmATJtuDXXgVeJtYabi/+Zju5Z7K06oaKvFYM+BgZWPmGQkVCxKVjSlug0rx6+qgRKIf/B/cPxmQDS0ZZVDmDcbxeibBL9JhRqIDwDb0XS9t1qBXvIJ5E4ZmdX/XuvLCnC4xJptu/vHyKl1wbRM3dcXEBk8J0knJc4JvY2yupF3LS32sfEHftLwzr63U3/0dOQvsCSJRBy5zgI8ieABUZAeZmmaQ6RCGOTEYrGO2G4CCF2uVq4YdrCOsfmrb0DzH8llNMkXKzXlITRvmkuBlTRRKxGnPzS3vf0iicdAkmTiU5u7lArFFc7IKcW4ypO+RKDTUpq74K+UZBI9eBjV7Po368EO8epJ5aIx32a2kyS/hYWkS/dZh7I0fNZa2qBUceZO/KZGRjH4dQFW9t0J9Q7LLXJY=" - - secure: "B2quYqla3mqu3QHqeRGMde66IFAluUHhzY87q/giPvONm/V7xmIcxzC6JNQ7X4pHotC7MUbsMf7vdcL1tyuPP5nqyfaxk0nnGnVoJZIQhMaBQ6Lkyt2Ac5u5/QBeHujrj8gqGjT7vzgPIYMYiSLV09VcPgMaYS6WxCrFmXOEJOeVGOGjG9J2wApAmlf6m/hBogLEbdbzRheqJwX5gMxppKX5XRaiqvKGR7hHFiW5HBaAiQ6caQEw6DZL9GnjMsqd5VMVL7DmbETncNE1utLVDO7mFwDMbkeonmwjsmQsJAo5Be+WFl/a/yRQLrqhhTFwag+V7g7RjcBKAdJ5IZJkN0EC9GjY0ZGOjE2cmcrX03U1g1voO7oreH6+25VqPTTGWHv5K8wDWM5QnhBJGtH2yH76p5XYK8iNOXPTqvjubN9sG3INsnTjxBHuQw+aY2vgwBGCvW+Asqe73BV8SDrmqcCqgmm/BUy0kZRd8mhaQdFL0SAO3suV7uf+fzIsmRtndFx24J5Cm6LE6NnTmtbhHhs4mUEqfngMq0XZBXj5z4LDhzOH4g3rf/Y7jHBjAAa6sKeh1sINdHrMdB843dDfNQPp/7g3hfKi31U9ZMczndsr2QYqm2OC/3lPiRHC+8Grj5PxXmiBplzTppyhl+1pjZWy5SCoQi6ulBOyaZcwfYM=" - - secure: "Kmh31kozw9NBbwtkucpf+QslinJIclB7rcktHz4RokVS5xiMD7njWvD8ptZhXqZG/tNcNiXkWskgM9EV/OvLTTavJToHE1gMhJel5ODxKM0b5QZDy6tkW/wI0Kdl0lck90EAJ7bmsOmF+oGczGsauiLPSlBOZV0tn1zdeJKwwmg46C9UyIhTDcFqUD1ehxExkDHnbOk6HmEPGCsZQCGNXY2t++dO7fRodIgjZAKcyTwgrh6WEZDfQDjvC51yQlVzL2mrcFqHi8qYAx3OHnuidRoMZ4huaoxLimw/1bGii/txTZwJ/pQdF90kGqSqLpRiO2lHLq5l8WAd4Z1XPSs1ZzbKUEhWO8VGgfR0Cp+Bkkd6P/vcuTq7bdhpvdnZn9FRUI/bfgZi827Skj/WoUh0RR7ns6RuDfQ5uGD9NiWDsbfoKLbvvYjyQycQlQTKglYQVcU/pWsuqVlx+27vM6JltR0Pct4uAJxAN+EFA4sTT/u4HiZ0NQyZWisZauu7CyMRN29fkKVVyYHHCr6QcNMUpvp1vvKYaUN7GeykGNVfoHB7CWAKcc6banh33c+QqG5DSTE++ZNa712gxEizQnO5MmLECUxNDpOGZnLsTlgbABwDxxe4PxFxXvBLm9pkJftq5cp9T9Syzz/yyT7z5weSGvRes94eUtSt0agoeX2MKsE=" - - secure: "VezxV+VdmbmtrQYT8AZIyg41WBROxuxpumerkcubADF7V4wV6lwx9Rd2G6yAr0VuHCNUUhS4m+gPFIsuiQbAhyupiEkwhzUYqk1tF+LITlLLPegLypjiLmhJMwGUNuDSSsih1Icmg9FzrP4VyzgGn9pBjoG9QYj1civBZeGwg++e/XDYlHMXrpd/UEfMKVB71JwB0tle4fKJZSvblIqP62yvbBaKHx6A4+ZWzJV5Vps0DoIeNtKCNmNNloKZVHfjbsvqSjnMYUJzkOzyPkM822q41N/D+3IAufO16+jH/W0vAZeN0e4GXiN5W+CVkr2Gbh0FwkVQcI3bekaOIn45XLUMLKdf+JsWDPKz9RraHelR9YxL5GoJ7ntwvmucxw0p8EVyJ/xLk/pBCP8iHq0Jb8//js25XHgxzzAWI37MErPAAGgTKZAVdAN0mGXbe63tWmwaBlEbK8h2A8di6abW5x6YHTkTo2BRlHUSTU8dE3VqTnpSkne5n1SlEa4g1Bci3J45M0/pLmHV6yCxCM5BrVXS5ByaB61py/umSbpmdIBFV6TM1MaKK3lAucQrR+8To/vCbm8XqPyujJdOR+ENIuuDgEU/Yh5Hv5SAODekUYaCp4pjfGzFADHQWVNDxIOXrwBN4OfSiAvRc1x6HXndOmNI4QtOxheuCRFFthq8VZI=" - - GITHUB_BASE_URL=https://api.github.com/repos/pmd/pmd - -jobs: - fast_finish: true - include: - - stage: build - name: "linux - mvn deploy" - os: linux - dist: bionic - env: BUILD=deploy - - name: "linux - build documentation" - os: linux - dist: bionic - env: BUILD=doc - - name: "linux - run sonar" - os: linux - dist: bionic - env: BUILD=sonar - if: "type = push && repo = pmd/pmd && fork = false" - - name: "linux - run coveralls" - os: linux - dist: bionic - env: BUILD=coveralls - if: "type = push && repo = pmd/pmd && fork = false" - - stage: publish - name: "Release Build - Publish" - os: linux - dist: bionic - if: "type = push && repo = pmd/pmd && fork = false && tag is present" - env: BUILD=publish - -before_install: - - bash .travis/before_install.sh "11" - - source ${HOME}/java.env -install: true -before_script: true -script: source .travis/build-$BUILD.sh -after_success: true - - -notifications: - webhooks: - urls: - - https://webhooks.gitter.im/e/0e6dd310db54b96c2653 - - https://webhooks.gitter.im/e/65efc1ea65175ad23ab6 - on_success: always # options: [always|never|change] default: always - on_failure: always # options: [always|never|change] default: always - on_start: never # options: [always|never|change] default: always - email: - recipients: - - andreas.dangel@adangel.org - on_success: change - on_failure: always -cache: - directories: - - "$HOME/.m2" - - "$HOME/.cache" - - vendor/bundle - - "$HOME/.rvm/" - -# Secure Keys, that need to be set for snapshot builds -# -# CI_DEPLOY_USERNAME - the user which can upload net.sourceforge.pmd:* to https://oss.sonatype.org/ -# CI_DEPLOY_PASSWORD -# -# SONAR_TOKEN - the token used to update https://sonarqube.com/dashboard/?id=393826 -# -# PMD_SF_USER - the sourceforge user, which is used to upload created binaries to sf files section. Note: an ssh key is -# required. See "before_install". -# -# Secure Keys, that need to be set for releases: -# -# PMD_SF_APIKEY - used to make the new release the default file in the files section. See https://sourceforge.net/auth/preferences/ -# under "Release API Key" -# -# CI_SIGN_KEYNAME -# CI_SIGN_PASSPHRASE -# add the encrypted GPG keyring file to repo (https://docs.travis-ci.com/user/encrypting-files/#Automated-Encryption), decrypt it and install it at the beginning of .travis-deploy.sh -# -# GITHUB_OAUTH_TOKEN - the token used to upload the binaries to github releases -# GITHUB_BASE_URL - the api url to use for github releases - does not need to be secure (https://api.github.com/repos/pmd/pmd) -# diff --git a/.travis/before_install.sh b/.travis/before_install.sh deleted file mode 100644 index b1234a3909..0000000000 --- a/.travis/before_install.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -set -e - -source .travis/logger.sh -source .travis/common-functions.sh - -travis_debug - -OPENJDK_VERSION=$1 - -bash .travis/setup-secrets.sh -bash .travis/configure-maven.sh -bash .travis/install-openjdk.sh $OPENJDK_VERSION - - -function install_jdk() { - LOCAL_DIR=$1 - TARGET_DIR=$2 - DOWNLOAD_URL=$3 - ARCHIVE=$(basename $DOWNLOAD_URL) - - mkdir -p ${LOCAL_DIR} - mkdir -p ${TARGET_DIR} - if [ ! -e ${LOCAL_DIR}/${ARCHIVE} ]; then - log_info "Downloading from ${DOWNLOAD_URL} to ${LOCAL_DIR}" - wget --directory-prefix ${LOCAL_DIR} --timestamping --continue ${DOWNLOAD_URL} - else - log_info "Skipped download, file ${LOCAL_DIR}/${ARCHIVE} already exists" - fi - log_info "Extracting to ${TARGET_DIR}" - tar --extract --file ${LOCAL_DIR}/${ARCHIVE} -C ${TARGET_DIR} --strip-components=1 -} - -if travis_isLinux; then - change_ruby_version - gem install bundler - bundle config set --local path vendor/bundle - bundle config set --local with release_notes_preprocessing - bundle install - - # install jdk7 for integration test - install_jdk "${HOME}/.cache/jdk7" "${HOME}/oraclejdk7" "https://pmd-code.org/oraclejdk/jdk-7u80-linux-x64.tar.gz" - log_info "OracleJDK7 can be used via -Djava7.home=${HOME}/oraclejdk7" - - # install openjdk8 for pmd-regression-tests - install_jdk "${HOME}/.cache/openjdk" "${HOME}/openjdk8" "https://pmd-code.org/openjdk/latest/jdk-8-linux64.tar.gz" - log_info "OpenJDK8 can be used from ${HOME}/openjdk8" -else - log_info "Not setting up ruby and additional jvms for ${TRAVIS_OS_NAME}." - exit 0 -fi diff --git a/.travis/build-coveralls.sh b/.travis/build-coveralls.sh deleted file mode 100755 index c3834512a7..0000000000 --- a/.travis/build-coveralls.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -e - -source .travis/logger.sh -source .travis/common-functions.sh - -VERSION=$(get_pom_version) -log_info "Building PMD Coveralls.io report ${VERSION} on branch ${TRAVIS_BRANCH}" - -if ! travis_isPush; then - echo "Not proceeding, since this is not a push!" - exit 0 -fi - -( - # disable fast fail, exit immediately, in this subshell - set +e - - ./mvnw clean install -DskipTests=true -Dmaven.javadoc.skip=true -B -V - ./mvnw package jacoco:report coveralls:report -Pcoveralls -B -V - - if [ $? -ne 0 ]; then - log_error "Error creating coveralls report" - else - log_success "New coveralls result: https://coveralls.io/github/pmd/pmd" - fi - true -) diff --git a/.travis/build-deploy.sh b/.travis/build-deploy.sh deleted file mode 100755 index e9250ad7c4..0000000000 --- a/.travis/build-deploy.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -set -e - -source .travis/logger.sh -source .travis/common-functions.sh -source .travis/github-releases-api.sh -source .travis/sourceforge-api.sh -source .travis/regression-tester.sh - -VERSION=$(get_pom_version) -log_info "Building PMD ${VERSION} on branch ${TRAVIS_BRANCH}" - -if travis_isLinux; then - MVN_BUILD_FLAGS="-B -V -Djava7.home=${HOME}/oraclejdk7" -else - MVN_BUILD_FLAGS="-B -V" -fi - -if travis_isOSX; then - - log_info "The build is running on OSX" - ./mvnw verify $MVN_BUILD_FLAGS - -elif travis_isWindows; then - - log_info "The build is running on Windows" - ./mvnw verify $MVN_BUILD_FLAGS - -elif travis_isPullRequest; then - - log_info "This is a pull-request build" - ./mvnw verify $MVN_BUILD_FLAGS - - regression-tester_executeDanger - -elif travis_isPush; then - - if [[ "${VERSION}" != *-SNAPSHOT && "${TRAVIS_TAG}" != "" ]]; then - echo -e "\n\n" - log_info "This is a release build for tag ${TRAVIS_TAG}" - echo -e "\n\n" - - # create a draft github release - gh_releases_createDraftRelease "${TRAVIS_TAG}" "$(git rev-list -n 1 ${TRAVIS_TAG})" - GH_RELEASE="$RESULT" - - # Build and deploy to ossrh / maven-central - ./mvnw deploy -Possrh,sign,pmd-release $MVN_BUILD_FLAGS - echo -e "\n\n" - - # Deploy to github releases - gh_release_uploadAsset "$GH_RELEASE" "pmd-dist/target/pmd-bin-${VERSION}.zip" - gh_release_uploadAsset "$GH_RELEASE" "pmd-dist/target/pmd-src-${VERSION}.zip" - - # Deploy to sourceforge files - sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip" - sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip" - - regression-tester_uploadBaseline - - elif [[ "${VERSION}" == *-SNAPSHOT ]]; then - log_info "This is a snapshot build" - ./mvnw deploy -Possrh,sign $MVN_BUILD_FLAGS - - # Deploy to sourceforge files - sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip" - sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip" - - if [ "${TRAVIS_BRANCH}" != "java-grammar" ]; then - regression-tester_uploadBaseline - else - log_info "Skipping regression tester for branch ${TRAVIS_BRANCH}" - fi - - else - # other build. Can happen during release: the commit with a non snapshot version is built, but not from the tag. - log_info "This is some other build, probably during release: commit with a non-snapshot version on branch master..." - ./mvnw verify $MVN_BUILD_FLAGS - # we stop here - no need to execute further steps - exit 0 - fi - -else - log_info "This is neither a pull request nor a push. Not executing any build." - exit 1 -fi diff --git a/.travis/build-doc.sh b/.travis/build-doc.sh deleted file mode 100755 index 4ac8028b1d..0000000000 --- a/.travis/build-doc.sh +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env bash -set -e - -source .travis/logger.sh -source .travis/common-functions.sh -source .travis/github-releases-api.sh -source .travis/sourceforge-api.sh -source .travis/pmd-code-api.sh - -function main() { - VERSION=$(get_pom_version) - log_info "Building PMD Documentation ${VERSION} on branch ${TRAVIS_BRANCH}" - - # - # First step: build pmd with profile "generate-rule-docs" - # The docs should appear under "docs/pages/rules/..." for each language - # With this profile, also the checks are executed (e.g. DeadLinksChecker). - # - ./mvnw clean verify -Dmaven.test.skip=true -P generate-rule-docs - - if ! travis_isPush; then - log_info "Not publishing site, since this is not a push!" - exit 0 - fi - - # - # For pushes, we'll update the online documentation - # - generate_jekyll_doc - create_pmd-doc_archive - - # Deploy to sourceforge files - sourceforge_uploadFile "${VERSION}" "docs/pmd-doc-${VERSION}.zip" - - # Deploy doc to https://docs.pmd-code.org/pmd-doc-${VERSION}/ - pmd_code_uploadDocumentation "${VERSION}" "docs/pmd-doc-${VERSION}.zip" - # Deploy javadoc to https://docs.pmd-code.org/apidocs/*/${VERSION}/ - pmd_code_uploadJavadoc "${VERSION}" "$(pwd)" - - - if [[ "${VERSION}" == *-SNAPSHOT && "${TRAVIS_BRANCH}" == "master" ]]; then - # only for snapshot builds from branch master - - pmd_code_createSymlink "${VERSION}" "snapshot" - - # update github pages https://pmd.github.io/pmd/ - publish_to_github_pages - # rsync site to https://pmd.sourceforge.io/snapshot - sourceforge_rsyncSnapshotDocumentation "${VERSION}" "snapshot" - fi - - - if [[ "${VERSION}" != *-SNAPSHOT && "${TRAVIS_TAG}" != "" ]]; then - log_info "This is a release documentation build for pmd ${VERSION}" - - # documentation is already uploaded to https://docs.pmd-code.org/pmd-doc-${VERSION} - pmd_code_createSymlink "${VERSION}" "latest" - # remove old doc and point to the new version - pmd_code_removeDocumentation "${VERSION}-SNAPSHOT" - pmd_code_createSymlink "${VERSION}" "${VERSION}-SNAPSHOT" - # remove old javadoc - pmd_code_removeJavadoc "${VERSION}-SNAPSHOT" - - # Deploy to github releases - gh_releases_getLatestDraftRelease - GH_RELEASE="$RESULT" - - gh_release_uploadAsset "$GH_RELEASE" "docs/pmd-doc-${VERSION}.zip" - - # updating github release text - # renders, and skips the first 6 lines - the Jekyll front-matter - RENDERED_RELEASE_NOTES=$(bundle exec .travis/render_release_notes.rb docs/pages/release_notes.md | tail -n +6) - RELEASE_NAME="PMD ${VERSION} ($(date -u +%d-%B-%Y))" - gh_release_updateRelease "$GH_RELEASE" "$RELEASE_NAME" "$RENDERED_RELEASE_NOTES" - sourceforge_uploadReleaseNotes "${VERSION}" "${RENDERED_RELEASE_NOTES}" - - publish_release_documentation_github - sourceforge_rsyncSnapshotDocumentation "${VERSION}" "pmd-${VERSION}" - fi -} - -# -# Executes jekyll and generates the documentation -# The documentation will be generated in the directory "docs/_site". -# -function generate_jekyll_doc() { - pushd docs - - echo -e "\n\n" - log_info "Building documentation using jekyll..." - bundle config set --local path vendor/bundle - bundle install - bundle exec jekyll build - - popd -} - -# -# Creates the pmd-doc.zip archive. It will be placed in "docs/". -# -function create_pmd-doc_archive() { - pushd docs - - echo -e "\n\n" - log_info "Creating pmd-doc archive..." - mv _site pmd-doc-${VERSION} - zip -qr pmd-doc-${VERSION}.zip pmd-doc-${VERSION}/ - log_success "Successfully created pmd-doc-${VERSION}.zip" - - popd -} - -# -# Publishes the site to https://pmd.github.io/pmd-${VERSION} and -# https://pmd.github.io/latest/ -# -function publish_release_documentation_github() { - echo -e "\n\n" - log_info "Adding the new doc to pmd.github.io..." - # clone pmd.github.io. Note: This uses the ssh key setup earlier - # In order to speed things up, we use a sparse checkout - no need to checkout all directories here - mkdir pmd.github.io - ( - cd pmd.github.io - git init - git config user.name "Travis CI (pmd-bot)" - git config user.email "andreas.dangel+pmd-bot@adangel.org" - git config core.sparsecheckout true - git remote add origin git@github.com:pmd/pmd.github.io.git - echo "/latest/" > .git/info/sparse-checkout - echo "/sitemap.xml" >> .git/info/sparse-checkout - git pull --depth=1 origin master - log_info "Copying documentation from ../docs/pmd-doc-${VERSION}/ to pmd-${VERSION}/ ..." - rsync -ah --stats ../docs/pmd-doc-${VERSION}/ pmd-${VERSION}/ - git status - echo "Executing: git add pmd-${VERSION}" - git add pmd-${VERSION} - echo "Executing: git commit..." - git commit -q -m "Added pmd-${VERSION}" - - log_info "Copying pmd-${VERSION} to latest ..." - git rm -qr latest - cp -a pmd-${VERSION} latest - echo "Executing: git add latest" - git add latest - echo "Executing: git commit..." - git commit -q -m "Copying pmd-${VERSION} to latest" - - log_info "Generating sitemap.xml" - ../.travis/sitemap_generator.sh > sitemap.xml - echo "Executing: git add sitemap.xml" - git add sitemap.xml - echo "Executing: git commit..." - git commit -q -m "Generated sitemap.xml" - - echo "Executing: git push origin master" - git push origin master - ) -} - -# -# Updates github pages of the main repository, -# so that https://pmd.github.io/pmd/ has the latest (snapshot) content -# -function publish_to_github_pages() { - echo -e "\n\n" - log_info "Pushing the new site to github pages..." - git clone --branch gh-pages --depth 1 git@github.com:pmd/pmd.git pmd-gh-pages - # clear the files first - rm -rf pmd-gh-pages/* - # copy the new site - cp -a docs/pmd-doc-${VERSION}/* pmd-gh-pages/ - ( - cd pmd-gh-pages - git config user.name "Travis CI (pmd-bot)" - git config user.email "andreas.dangel+pmd-bot@adangel.org" - git add -A - MSG="Update documentation - -TRAVIS_JOB_NUMBER=${TRAVIS_JOB_NUMBER} -TRAVIS_COMMIT_RANGE=${TRAVIS_COMMIT_RANGE}" - git commit -q -m "$MSG" - git push git@github.com:pmd/pmd.git HEAD:gh-pages - log_success "Successfully pushed site to https://pmd.github.io/pmd/" - ) -} - -main diff --git a/.travis/build-publish.sh b/.travis/build-publish.sh deleted file mode 100644 index 5599103585..0000000000 --- a/.travis/build-publish.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -e - -source .travis/logger.sh -source .travis/common-functions.sh -source .travis/github-releases-api.sh -source .travis/sourceforge-api.sh - -VERSION=$(get_pom_version) -log_info "PMD Release ${VERSION}" - -if ! travis_isPush; then - log_info "Not publishing release, since this is not a push!" - exit 0 -fi - -if [[ "${VERSION}" == *-SNAPSHOT || "${TRAVIS_TAG}" == "" ]]; then - log_info "Not publishing release, since this is not a release!" - exit 0 -fi - - -# Publish github release -gh_releases_getLatestDraftRelease -GH_RELEASE="$RESULT" - -gh_release_publishRelease "$GH_RELEASE" -sourceforge_selectDefault "${VERSION}" diff --git a/.travis/build-sonar.sh b/.travis/build-sonar.sh deleted file mode 100755 index 36cc2cc6b4..0000000000 --- a/.travis/build-sonar.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -set -e - -source .travis/logger.sh -source .travis/common-functions.sh - -VERSION=$(get_pom_version) -log_info "Building PMD Sonar ${VERSION} on branch ${TRAVIS_BRANCH}" - -if ! travis_isPush; then - echo "Not updating sonar, since this is not a push!" - exit 0 -fi - -( - # for sonar, we need to use java10, until sonarjava 5.8.0 is released (Sept. 2018) - JAVA_HOME=$(bash ./install-jdk.sh -F 10 -L GPL -W $HOME/jdk --emit-java-home | tail --lines 1) - export JAVA_HOME - export PATH=${JAVA_HOME}/bin:$PATH - - # disable fast fail, exit immediately, in this subshell - set +e - - # Run the build - ./mvnw clean package sonar:sonar -Dsonar.login=${SONAR_TOKEN} -Psonar -B -V - - if [ $? -ne 0 ]; then - log_error "Error updating sonar..." - else - log_success "New sonar results: https://sonarcloud.io/dashboard?id=net.sourceforge.pmd%3Apmd" - fi - true -) diff --git a/.travis/common-functions.sh b/.travis/common-functions.sh deleted file mode 100755 index c217bb3c6f..0000000000 --- a/.travis/common-functions.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -set -e - - -function travis_debug() { - echo "TRAVIS_REPO_SLUG: ${TRAVIS_REPO_SLUG}" - echo "TRAVIS_PULL_REQUEST_SLUG: ${TRAVIS_PULL_REQUEST_SLUG}" - echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}" - echo "TRAVIS_PULL_REQUEST: ${TRAVIS_PULL_REQUEST}" - echo "TRAVIS_SECURE_ENV_VARS: ${TRAVIS_SECURE_ENV_VARS}" - echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}" - echo "TRAVIS_TAG: ${TRAVIS_TAG}" - echo "TRAVIS_ALLOW_FAILURE: ${TRAVIS_ALLOW_FAILURE}" - echo "TRAVIS_OS_NAME: ${TRAVIS_OS_NAME}" -} - -function travis_isPullRequest() { - if [ "${TRAVIS_REPO_SLUG}" != "pmd/pmd" ] || [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then - return 0 - else - return 1 - fi -} - -function travis_isPush() { - if [ "${TRAVIS_REPO_SLUG}" = "pmd/pmd" ] && [ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ]; then - return 0 - else - return 1 - fi -} - -function travis_isOSX() { - if [[ $TRAVIS_OS_NAME == 'osx' ]]; then - return 0 - else - return 1 - fi -} - -function travis_isLinux() { - if [[ $TRAVIS_OS_NAME == 'linux' ]]; then - return 0 - else - return 1 - fi -} - -function travis_isWindows() { - if [[ $TRAVIS_OS_NAME == 'windows' ]]; then - return 0 - else - return 1 - fi -} - -function get_pom_version() { - echo $(./mvnw -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) -} - -function change_ruby_version() { - source "$HOME/.rvm/scripts/rvm" - rvm use ruby-2.7 -} diff --git a/.travis/configure-maven.sh b/.travis/configure-maven.sh deleted file mode 100755 index 0b8f10a17b..0000000000 --- a/.travis/configure-maven.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -e - -echo "MAVEN_OPTS='-Xms1g -Xmx1g'" > ${HOME}/.mavenrc -mkdir -p ${HOME}/.m2 -cp .travis/travis-settings.xml ${HOME}/.m2/settings.xml diff --git a/.travis/install-openjdk.sh b/.travis/install-openjdk.sh deleted file mode 100644 index af59870c50..0000000000 --- a/.travis/install-openjdk.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -set -e - -# -# Downloads AdoptOpenJDK Builds from -# https://pmd-code.org/openjdk/latest/jdk-11-linux64.tar.gz -# https://pmd-code.org/openjdk/latest/jdk-11-windows64.zip -# https://pmd-code.org/openjdk/latest/jdk-11-mac64.tar.gz -# -# The Build are originally from: -# https://github.com/AdoptOpenJDK/openjdk11-binaries/releases -# - -source .travis/logger.sh -source .travis/common-functions.sh - -# OPENJDK_VERSION e.g. "11" -OPENJDK_VERSION=$1 -BASE_URL=https://pmd-code.org/openjdk/latest/jdk-${OPENJDK_VERSION}- - -log_info "Installing OpenJDK${OPENJDK_VERSION} for ${TRAVIS_OS_NAME}" - -if travis_isOSX; then - DOWNLOAD_URL=${BASE_URL}mac64.tar.gz - COMPONENTS_TO_STRIP=3 # e.g. jdk-11.0.3+7/Contents/Home/bin/java -elif travis_isWindows; then - DOWNLOAD_URL=${BASE_URL}windows64.zip -else - DOWNLOAD_URL=${BASE_URL}linux64.tar.gz - COMPONENTS_TO_STRIP=1 # e.g. openjdk-11.0.3+7/bin/java -fi - -OPENJDK_ARCHIVE=$(basename $DOWNLOAD_URL) - -LOCAL_DIR=${HOME}/.cache/openjdk -TARGET_DIR=${HOME}/openjdk${OPENJDK_VERSION} - -mkdir -p ${LOCAL_DIR} -mkdir -p ${TARGET_DIR} -if [ ! -e ${LOCAL_DIR}/${OPENJDK_ARCHIVE} ]; then - log_info "Downloading from ${DOWNLOAD_URL} to ${LOCAL_DIR}" - wget --directory-prefix ${LOCAL_DIR} --timestamping --continue ${DOWNLOAD_URL} -else - log_info "Skipped download, file ${LOCAL_DIR}/${OPENJDK_ARCHIVE} already exists" -fi - -log_info "Extracting to ${TARGET_DIR}" -if travis_isWindows; then - 7z x ${LOCAL_DIR}/${OPENJDK_ARCHIVE} -o${TARGET_DIR} - mv ${TARGET_DIR}/*/* ${TARGET_DIR}/ -else - tar --extract --file ${LOCAL_DIR}/${OPENJDK_ARCHIVE} -C ${TARGET_DIR} --strip-components=${COMPONENTS_TO_STRIP} -fi - -cat > ${HOME}/java.env <> "$HOME/.ssh/known_hosts" -echo 'web.sourceforge.net ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCwsY6sZT4MTTkHfpRzYjxG7mnXrGL74RCT2cO/NFvRrZVNB5XNwKNn7G5fHbYLdJ6UzpURDRae1eMg92JG0+yo=' >> "$HOME/.ssh/known_hosts" -echo 'web.sourceforge.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOQD35Ujalhh+JJkPvMckDlhu4dS7WH6NsOJ15iGCJLC' >> "$HOME/.ssh/known_hosts" - -# add pmd-code.org (ssh-keyscan pmd-code.org) -echo 'pmd-code.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVsIeF6xU0oPb/bMbxG1nU1NDyBpR/cBEPZcm/PuJwdI9B0ydPHA6FysqAnt32fNFznC2SWisnWyY3iNsP3pa8RQJVwmnnv9OboGFlW2/61o3iRyydcpPbgl+ADdt8iU9fmMI7dC04UqgHGBoqOwVNna9VylTjp5709cK2qHnwU450F6YcOEiOKeZfJvV4PmpJCz/JcsUVqft6StviR31jKnqbnkZdP8qNoTbds6WmGKyXkhHdLSZE7X1CFQH28tk8XFqditX93ezeCiThFL7EleDexV/3+2+cs5878sDMUMzHS5KShTjkxzhHaodhtIEdNesinq/hOPbxAGkQ0FbD' >> $HOME/.ssh/known_hosts diff --git a/Dangerfile b/Dangerfile index f57f762c11..f26f7a528e 100644 --- a/Dangerfile +++ b/Dangerfile @@ -7,10 +7,10 @@ require 'logger' def run_pmdtester Dir.chdir('..') do argv = ['--local-git-repo', './pmd', - '--list-of-project', './pmd/.travis/project-list.xml', - '--base-branch', "#{ENV['TRAVIS_BRANCH']}", + '--list-of-project', './pmd/.ci/files/project-list.xml', + '--base-branch', "#{ENV['PMD_CI_BRANCH']}", '--patch-branch', 'HEAD', - '--patch-config', './pmd/.travis/all-java.xml', + '--patch-config', './pmd/.ci/files/all-java.xml', '--mode', 'online', '--auto-gen-config', # '--debug', @@ -28,14 +28,14 @@ end def upload_report Dir.chdir('target/reports') do - tar_filename = "pr-#{ENV['TRAVIS_PULL_REQUEST']}-diff-report-#{Time.now.strftime("%Y-%m-%dT%H-%M-%SZ")}.tar" + tar_filename = "pr-#{ENV['PMD_CI_PULL_REQUEST_NUMBER']}-diff-report-#{Time.now.strftime("%Y-%m-%dT%H-%M-%SZ")}.tar" unless Dir.exist?('diff/') message("No java rules are changed!", sticky: true) return end `tar -cf #{tar_filename} diff/` - report_url = `curl -u #{ENV['CHUNK_TOKEN']} -T #{tar_filename} https://chunk.io` + report_url = `curl -u #{ENV['PMD_CI_CHUNK_TOKEN']} -T #{tar_filename} https://chunk.io` if $?.success? @logger.info "Successfully uploaded #{tar_filename} to chunk.io" @@ -53,11 +53,6 @@ def upload_report end # Perform regression testing -can_merge = github.pr_json['mergeable'] -if can_merge - run_pmdtester -else - warn("This PR cannot be merged yet.", sticky: false) -end +run_pmdtester # vim: syntax=ruby diff --git a/README.md b/README.md index a1bed340d3..d07be46f1f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # PMD [![Join the chat at https://gitter.im/pmd/pmd](https://badges.gitter.im/pmd/pmd.svg)](https://gitter.im/pmd/pmd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![Build Status](https://travis-ci.com/pmd/pmd.svg?branch=master)](https://travis-ci.com/pmd/pmd) +[![Build Status](https://github.com/pmd/pmd/workflows/.github/workflows/pushes.yml/badge.svg?branch=master)](https://github.com/pmd/pmd/actions) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/net.sourceforge.pmd/pmd/badge.svg)](https://maven-badges.herokuapp.com/maven-central/net.sourceforge.pmd/pmd) [![Reproducible Builds](https://img.shields.io/badge/Reproducible_Builds-ok-green?labelColor=blue)](https://github.com/jvm-repo-rebuild/reproducible-central#net.sourceforge.pmd:pmd) [![Coverage Status](https://coveralls.io/repos/github/pmd/pmd/badge.svg)](https://coveralls.io/github/pmd/pmd) diff --git a/docs/pages/pmd/userdocs/pmd_report_formats.md b/docs/pages/pmd/userdocs/pmd_report_formats.md index 4bbfce41ec..5ca846ac69 100644 --- a/docs/pages/pmd/userdocs/pmd_report_formats.md +++ b/docs/pages/pmd/userdocs/pmd_report_formats.md @@ -169,8 +169,8 @@ and configuration errors are reported. Example: ``` -/home/pmd/source/pmd-core/src/main/java/net/sourceforge/pmd/RuleContext.java:124: Logger calls should be surrounded by log level guards. -/home/pmd/source/pmd-core/src/main/java/net/sourceforge/pmd/benchmark/Benchmarker.java:58: This for loop can be replaced by a foreach loop +/home/pmd/source/pmd-core/src/main/java/net/sourceforge/pmd/RuleContext.java:124: GuardLogStatement: Logger calls should be surrounded by log level guards. +/home/pmd/source/pmd-core/src/main/java/net/sourceforge/pmd/benchmark/Benchmarker.java:58: ForLoopCanBeForeach: This for loop can be replaced by a foreach loop /home/pmd/source/pmd-core/src/test/resources/net/sourceforge/pmd/cpd/files/file_with_ISO-8859-1_encoding.java - PMDException: Error while parsing /home/pmd/source/pmd-core/src/test/resources/net/sourceforge/pmd/cpd/files/file_with_ISO-8859-1_encoding.java CloseResource rule violation suppressed by Annotation in /home/pmd/source/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java LoosePackageCoupling - No packages or classes specified diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 2f8ddf80cb..5a2156f2b0 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -16,10 +16,16 @@ This is a {{ site.pmd.release_type }} release. ### Fixed Issues -* pmd-core +* core * [#1939](https://github.com/pmd/pmd/issues/1939): \[core] XPath expressions return handling -* pmd-java + * [#1961](https://github.com/pmd/pmd/issues/1961): \[core] Text renderer should include name of violated rule + * [#2874](https://github.com/pmd/pmd/pull/2874): \[core] Fix XMLRenderer with UTF-16 +* cs + * [#2938](https://github.com/pmd/pmd/pull/2938): \[cs] CPD: ignoring using directives could not be disabled +* java * [#2911](https://github.com/pmd/pmd/issues/2911): \[java] `ClassTypeResolver#searchNodeNameForClass` leaks memory +* scala + * [#2480](https://github.com/pmd/pmd/issues/2480): \[scala] Support CPD suppressions ### API Changes @@ -50,4 +56,10 @@ You can identify them with the `@InternalApi` annotation. You'll also get a depr ### External Contributions +* [#2914](https://github.com/pmd/pmd/pull/2914): \[core] Include rule name in text renderer - [Gunther Schrijvers](https://github.com/GuntherSchrijvers) +* [#2925](https://github.com/pmd/pmd/pull/2925): Cleanup: Correct annotation array initializer indents from checkstyle #8083 - [Abhishek Kumar](https://github.com/Abhishek-kumar09) +* [#2929](https://github.com/pmd/pmd/pull/2929): \[scala] Add support for CPD-ON and CPD-OFF special comments - [Andy Robinson](https://github.com/andyrobinson) +* [#2936](https://github.com/pmd/pmd/pull/2936): \[java] (doc) Fix typo: "an accessor" not "a" - [Igor Moreno](https://github.com/igormoreno) +* [#2938](https://github.com/pmd/pmd/pull/2938): \[cs] CPD: fix issue where ignoring using directives could not be disabled - [Maikel Steneker](https://github.com/maikelsteneker) + {% endtocmaker %} diff --git a/.travis/sitemap_generator.sh b/docs/sitemap_generator.sh similarity index 85% rename from .travis/sitemap_generator.sh rename to docs/sitemap_generator.sh index 1d38e8ee96..03e25a0bcf 100755 --- a/.travis/sitemap_generator.sh +++ b/docs/sitemap_generator.sh @@ -1,11 +1,11 @@ -#!/bin/bash +#!/usr/bin/env bash -# Sitemap generator -# Assumes we have the latest version of the site under "latest" and "pmd-${RELEASE_VERSION}" +# Sitemap generator for pmd.github.io main landing page. +# Assumes we have the latest version of the site under "latest" # https://www.sitemaps.org/protocol.html WEBSITE_PREFIX="https://pmd.github.io/" -DOC_PREFIX="latest/" # "pmd-${RELEASE_VERSION}/" +DOC_PREFIX="latest/" DATE=`date +%Y-%m-%d` # Priority is relative to the website, can be chosen in {0.1, 0.2, ..., 1} # Default priority is 0.5 diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/renderers/TextRenderer.java b/pmd-core/src/main/java/net/sourceforge/pmd/renderers/TextRenderer.java index 6830c97b7c..afce5e0740 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/renderers/TextRenderer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/renderers/TextRenderer.java @@ -16,6 +16,10 @@ import net.sourceforge.pmd.RuleViolation; */ public class TextRenderer extends AbstractIncrementingRenderer { + private static final char SMALL_SEPARATOR = ':'; + private static final String MEDIUM_SEPARATOR = ":\t"; + private static final String LARGE_SEPARATOR = "\t-\t"; + public static final String NAME = "text"; public TextRenderer() { @@ -35,8 +39,9 @@ public class TextRenderer extends AbstractIncrementingRenderer { buf.setLength(0); RuleViolation rv = violations.next(); buf.append(determineFileName(rv.getFilename())); - buf.append(':').append(Integer.toString(rv.getBeginLine())); - buf.append(":\t").append(rv.getDescription()).append(PMD.EOL); + buf.append(SMALL_SEPARATOR).append(rv.getBeginLine()); + buf.append(MEDIUM_SEPARATOR).append(rv.getRule().getName()); + buf.append(MEDIUM_SEPARATOR).append(rv.getDescription()).append(PMD.EOL); writer.write(buf.toString()); } } @@ -48,7 +53,7 @@ public class TextRenderer extends AbstractIncrementingRenderer { for (Report.ProcessingError error : errors) { buf.setLength(0); buf.append(determineFileName(error.getFile())); - buf.append("\t-\t").append(error.getMsg()).append(PMD.EOL); + buf.append(LARGE_SEPARATOR).append(error.getMsg()).append(PMD.EOL); writer.write(buf.toString()); } @@ -64,7 +69,7 @@ public class TextRenderer extends AbstractIncrementingRenderer { for (Report.ConfigurationError error : configErrors) { buf.setLength(0); buf.append(error.rule().getName()); - buf.append("\t-\t").append(error.issue()).append(PMD.EOL); + buf.append(LARGE_SEPARATOR).append(error.issue()).append(PMD.EOL); writer.write(buf.toString()); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/renderers/XMLRenderer.java b/pmd-core/src/main/java/net/sourceforge/pmd/renderers/XMLRenderer.java index 840b786ca6..204fc4f31a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/renderers/XMLRenderer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/renderers/XMLRenderer.java @@ -10,6 +10,8 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; import java.io.Writer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.text.SimpleDateFormat; import java.util.Date; @@ -66,7 +68,8 @@ public class XMLRenderer extends AbstractIncrementingRenderer { @Override public void start() throws IOException { String encoding = getProperty(ENCODING); - lineSeparator = PMD.EOL.getBytes(encoding); + String unmarkedEncoding = toUnmarkedEncoding(encoding); + lineSeparator = PMD.EOL.getBytes(unmarkedEncoding); try { xmlWriter.writeStartDocument(encoding, "1.0"); @@ -86,6 +89,26 @@ public class XMLRenderer extends AbstractIncrementingRenderer { } } + /** + * Return a encoding, which doesn't write a BOM (byte order mark). + * Only UTF-16 encoders might write a BOM, see {@link Charset}. + * + *

This is needed, so that we don't accidentally add BOMs whenever + * we insert a newline. + * + * @return + */ + private static String toUnmarkedEncoding(String encoding) { + if (StandardCharsets.UTF_16.name().equalsIgnoreCase(encoding)) { + return StandardCharsets.UTF_16BE.name(); + } + // edge case: UTF-16LE with BOM + if ("UTF-16LE_BOM".equalsIgnoreCase(encoding)) { + return StandardCharsets.UTF_16LE.name(); + } + return encoding; + } + /** * Outputs a platform dependent line separator. * diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/ant/PMDTaskTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/ant/PMDTaskTest.java index 396bda7876..0bdf85b5ad 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/ant/PMDTaskTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/ant/PMDTaskTest.java @@ -78,7 +78,7 @@ public class PMDTaskTest { String actual = IOUtils.toString(in, StandardCharsets.UTF_8); // remove any trailing newline actual = actual.replaceAll("\n|\r", ""); - Assert.assertEquals("sample.dummy:0:\tTest Rule 2", actual); + Assert.assertEquals("sample.dummy:0:\tSampleXPathRule:\tTest Rule 2", actual); } } } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/RenderersTests.java b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/RenderersTests.java index dcb68112b9..b97d81e0e2 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/RenderersTests.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/RenderersTests.java @@ -15,18 +15,18 @@ import org.junit.runners.Suite.SuiteClasses; */ @RunWith(Suite.class) @SuiteClasses({ - CodeClimateRendererTest.class, - CSVRendererTest.class, - EmacsRendererTest.class, - HTMLRendererTest.class, - IDEAJRendererTest.class, - PapariTextRendererTest.class, - SummaryHTMLRendererTest.class, - TextPadRendererTest.class, - VBHTMLRendererTest.class, - XMLRendererTest.class, - XSLTRendererTest.class, - YAHTMLRendererTest.class + CodeClimateRendererTest.class, + CSVRendererTest.class, + EmacsRendererTest.class, + HTMLRendererTest.class, + IDEAJRendererTest.class, + PapariTextRendererTest.class, + SummaryHTMLRendererTest.class, + TextPadRendererTest.class, + VBHTMLRendererTest.class, + XMLRendererTest.class, + XSLTRendererTest.class, + YAHTMLRendererTest.class }) public class RenderersTests { } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/TextRendererTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/TextRendererTest.java index e5118ab038..69983b5f85 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/TextRendererTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/TextRendererTest.java @@ -17,7 +17,7 @@ public class TextRendererTest extends AbstractRendererTest { @Override public String getExpected() { - return getSourceCodeFilename() + ":1:\tblah" + PMD.EOL; + return getSourceCodeFilename() + ":1:\tFoo:\tblah" + PMD.EOL; } @Override @@ -27,8 +27,8 @@ public class TextRendererTest extends AbstractRendererTest { @Override public String getExpectedMultiple() { - return getSourceCodeFilename() + ":1:\tblah" + PMD.EOL - + getSourceCodeFilename() + ":1:\tblah" + PMD.EOL; + return getSourceCodeFilename() + ":1:\tFoo:\tblah" + PMD.EOL + + getSourceCodeFilename() + ":1:\tFoo:\tblah" + PMD.EOL; } @Override diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/XMLRendererTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/XMLRendererTest.java index e72e44561f..62e628c106 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/XMLRendererTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/XMLRendererTest.java @@ -123,6 +123,12 @@ public class XMLRendererTest extends AbstractRendererTest { verifyXmlEscaping(renderer, "\ud801\udc1c", StandardCharsets.UTF_8); } + @Test + public void testXMLEscapingWithUTF16() throws Exception { + Renderer renderer = getRenderer(); + verifyXmlEscaping(renderer, "𐐜", StandardCharsets.UTF_16); + } + @Test public void testXMLEscapingWithoutUTF8() throws Exception { Renderer renderer = getRenderer(); diff --git a/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java b/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java index f56ee3d1a4..10fb5aed89 100644 --- a/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java +++ b/pmd-cs/src/main/java/net/sourceforge/pmd/cpd/CsTokenizer.java @@ -21,9 +21,7 @@ public class CsTokenizer extends AntlrTokenizer { private boolean ignoreUsings = false; public void setProperties(Properties properties) { - if (properties.containsKey(IGNORE_USINGS)) { - ignoreUsings = Boolean.parseBoolean(properties.getProperty(IGNORE_USINGS, "false")); - } + ignoreUsings = Boolean.parseBoolean(properties.getProperty(IGNORE_USINGS, "false")); } public void setIgnoreUsings(boolean ignoreUsings) { diff --git a/pmd-dist/src/main/resources/assemblies/pmd-src.xml b/pmd-dist/src/main/resources/assemblies/pmd-src.xml index 1513da9f56..a364b8f88f 100644 --- a/pmd-dist/src/main/resources/assemblies/pmd-src.xml +++ b/pmd-dist/src/main/resources/assemblies/pmd-src.xml @@ -29,9 +29,10 @@ **/.idea/** **/*.iml - .travis/secrets.tar - .travis/id_rsa - .travis/*.gpg + .ci/files/id_rsa + .ci/files/private-env + .ci/files/public-env + .ci/files/release-signing-key-D0BF1D737C9A1C22.gpg.gpg .bundle/** vendor/** diff --git a/pmd-dist/src/test/java/net/sourceforge/pmd/it/SourceDistributionIT.java b/pmd-dist/src/test/java/net/sourceforge/pmd/it/SourceDistributionIT.java index d483644813..acfdb2106d 100644 --- a/pmd-dist/src/test/java/net/sourceforge/pmd/it/SourceDistributionIT.java +++ b/pmd-dist/src/test/java/net/sourceforge/pmd/it/SourceDistributionIT.java @@ -10,7 +10,6 @@ import java.io.File; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.regex.Pattern; import org.junit.Assert; import org.junit.Test; @@ -19,7 +18,6 @@ import net.sourceforge.pmd.PMDVersion; public class SourceDistributionIT { private static final String BASE_PATH = "pmd-src-" + PMDVersion.VERSION; - private static final Pattern GPG_PATTERN = Pattern.compile(Pattern.quote(BASE_PATH + "/.travis/") + ".+\\.[gG][pP][gG]"); private File getSourceDistribution() { return new File(".", "target/" + BASE_PATH + ".zip"); @@ -33,13 +31,14 @@ public class SourceDistributionIT { @Test public void verifyExclusions() throws Exception { Set exclusions = new HashSet<>(); - exclusions.add(BASE_PATH + "/.travis/secrets.tar"); - exclusions.add(BASE_PATH + "/.travis/id_rsa"); + exclusions.add(BASE_PATH + "/.ci/files/id_rsa"); + exclusions.add(BASE_PATH + "/.ci/files/private-env"); + exclusions.add(BASE_PATH + "/.ci/files/public-env"); + exclusions.add(BASE_PATH + "/.ci/files/release-signing-key-D0BF1D737C9A1C22.gpg.gpg"); List files = ZipFileExtractor.readZipFile(getSourceDistribution().toPath()); for (String file : files) { - Assert.assertFalse("File " + file + " must not be included", exclusions.contains(file) - || GPG_PATTERN.matcher(file).matches()); + Assert.assertFalse("File " + file + " must not be included", exclusions.contains(file)); } } } diff --git a/pmd-java/src/main/resources/category/java/bestpractices.xml b/pmd-java/src/main/resources/category/java/bestpractices.xml index cf37657286..59e5e56c72 100644 --- a/pmd-java/src/main/resources/category/java/bestpractices.xml +++ b/pmd-java/src/main/resources/category/java/bestpractices.xml @@ -71,7 +71,7 @@ public class Outer { class="net.sourceforge.pmd.lang.java.rule.bestpractices.AccessorMethodGenerationRule" externalInfoUrl="${pmd.website.baseurl}/pmd_rules_java_bestpractices.html#accessormethodgeneration"> -When accessing a private field / method from another class, the Java compiler will generate a accessor methods +When accessing private fields / methods from another class, the Java compiler will generate accessor methods with package-private visibility. This adds overhead, and to the dex method count on Android. This situation can be avoided by changing the visibility of the field / method from private to package-private. diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/cpd/testdata/discardedElements.java b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/cpd/testdata/discardedElements.java index 54858f8423..93869d1726 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/cpd/testdata/discardedElements.java +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/cpd/testdata/discardedElements.java @@ -29,7 +29,7 @@ public class Foo { // class Bar @AnnotationWithParams({@Nested(1) , @Nested(2) , @Nested - }) + }) public void foo() { } diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/cpd/testdata/discardedElements_no_ignore_annots.txt b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/cpd/testdata/discardedElements_no_ignore_annots.txt index 705523dcca..40d68ba293 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/cpd/testdata/discardedElements_no_ignore_annots.txt +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/cpd/testdata/discardedElements_no_ignore_annots.txt @@ -49,8 +49,8 @@ L31 [@] 28 28 [Nested] 29 34 L32 - [}] 14 14 - [)] 15 15 + [}] 9 9 + [)] 10 10 L33 [public] 5 10 [void] 12 15 diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/cpd/testdata/specialComments.java b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/cpd/testdata/specialComments.java index 7b985c8bde..93852a818c 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/cpd/testdata/specialComments.java +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/cpd/testdata/specialComments.java @@ -4,8 +4,8 @@ package foo.bar.baz; // another irrelevant comment @ MyAnnotation ("ugh") @NamedQueries({ - @NamedQuery( - )}) + @NamedQuery( + )}) public class Foo {// CPD-ON diff --git a/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenAdapter.java b/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenAdapter.java new file mode 100644 index 0000000000..c4842eccf9 --- /dev/null +++ b/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenAdapter.java @@ -0,0 +1,66 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd; + +import net.sourceforge.pmd.lang.ast.GenericToken; + +import scala.meta.tokens.Token; + +/** + * Adapts the scala.meta.tokens.Token so that it can be used with the generic BaseTokenFilter + */ +public class ScalaTokenAdapter implements GenericToken { + + private Token token; + private GenericToken previousComment; + + ScalaTokenAdapter(Token token, GenericToken comment) { + this.token = token; + this.previousComment = comment; + } + + @Override + public GenericToken getNext() { + throw new UnsupportedOperationException(); + } + + @Override + public GenericToken getPreviousComment() { + return previousComment; + } + + @Override + public String getImage() { + return token.text(); + } + + @Override + public int getBeginLine() { + return token.pos().startLine() + 1; + } + + @Override + public int getEndLine() { + return token.pos().endLine() + 1; + } + + @Override + public int getBeginColumn() { + return token.pos().startColumn() + 1; + } + + @Override + public int getEndColumn() { + return token.pos().endColumn() + 1; + } + + @Override + public String toString() { + return "ScalaTokenAdapter{" + + "token=" + token + + ", previousComment=" + previousComment + + "}"; + } +} diff --git a/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenizer.java b/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenizer.java index 4685f63bd9..c545b26701 100644 --- a/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenizer.java +++ b/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenizer.java @@ -9,8 +9,11 @@ import java.util.Properties; import org.apache.commons.lang3.StringUtils; +import net.sourceforge.pmd.cpd.token.internal.BaseTokenFilter; import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.lang.LanguageVersion; +import net.sourceforge.pmd.lang.TokenManager; +import net.sourceforge.pmd.lang.ast.GenericToken; import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.scala.ScalaLanguageHandler; import net.sourceforge.pmd.lang.scala.ScalaLanguageModule; @@ -24,7 +27,7 @@ import scala.meta.tokenizers.TokenizeException; import scala.meta.tokens.Token; /** - * Scala Tokenizer class. Uses the Scala Meta Tokenizer. + * Scala Tokenizer class. Uses the Scala Meta Tokenizer, but adapts it for use with generic filtering */ public class ScalaTokenizer implements Tokenizer { @@ -72,19 +75,21 @@ public class ScalaTokenizer implements Tokenizer { // tokenize with a filter try { scala.meta.tokens.Tokens tokens = tokenizer.tokenize(); - ScalaTokenFilter filter = new ScalaTokenFilter(tokens.iterator()); - Token token; + // use extensions to the standard PMD TokenManager and Filter + ScalaTokenManager scalaTokenManager = new ScalaTokenManager(tokens.iterator()); + ScalaTokenFilter filter = new ScalaTokenFilter(scalaTokenManager); + + GenericToken token; while ((token = filter.getNextToken()) != null) { - if (StringUtils.isEmpty(token.text())) { + if (StringUtils.isEmpty(token.getImage())) { continue; } - Position pos = token.pos(); - TokenEntry cpdToken = new TokenEntry(token.text(), + TokenEntry cpdToken = new TokenEntry(token.getImage(), filename, - pos.startLine() + 1, - pos.startColumn() + 1, - pos.endColumn() + 1); + token.getBeginLine(), + token.getBeginColumn(), + token.getEndColumn()); tokenEntries.add(cpdToken); } } catch (Exception e) { @@ -103,19 +108,25 @@ public class ScalaTokenizer implements Tokenizer { } /** - * Token Filter skips un-helpful tokens to only register important tokens + * Implementation of the generic Token Manager, also skips un-helpful tokens and comments to only register important tokens * and patterns. + * + * Keeps track of comments, for special comment processing */ - private static class ScalaTokenFilter { + private static class ScalaTokenManager implements TokenManager { + Iterator tokenIter; Class[] skippableTokens = new Class[] { Token.Space.class, Token.Tab.class, Token.CR.class, - Token.LF.class, Token.FF.class, Token.LFLF.class, Token.EOF.class }; + Token.LF.class, Token.FF.class, Token.LFLF.class, Token.EOF.class, Token.Comment.class }; - ScalaTokenFilter(Iterator iterator) { + GenericToken previousComment = null; + + ScalaTokenManager(Iterator iterator) { this.tokenIter = iterator; } - Token getNextToken() { + @Override + public GenericToken getNextToken() { if (!tokenIter.hasNext()) { return null; } @@ -123,9 +134,12 @@ public class ScalaTokenizer implements Tokenizer { Token token; do { token = tokenIter.next(); + if (isComment(token)) { + previousComment = new ScalaTokenAdapter(token, previousComment); + } } while (token != null && skipToken(token) && tokenIter.hasNext()); - return token; + return new ScalaTokenAdapter(token, previousComment); } private boolean skipToken(Token token) { @@ -137,5 +151,27 @@ public class ScalaTokenizer implements Tokenizer { } return skip; } + + private boolean isComment(Token token) { + return token instanceof Token.Comment; + } + + @Override + public void setFileName(String fileName) { + throw new UnsupportedOperationException("setFileName deprecated"); + } } + + private static class ScalaTokenFilter extends BaseTokenFilter { + ScalaTokenFilter(TokenManager tokenManager) { + super(tokenManager); + } + + @Override + protected boolean shouldStopProcessing(ScalaTokenAdapter currentToken) { + return currentToken == null; + } + + } + } diff --git a/pmd-scala-modules/pmd-scala-common/src/test/java/net/sourceforge/pmd/cpd/ScalaTokenizerTest.java b/pmd-scala-modules/pmd-scala-common/src/test/java/net/sourceforge/pmd/cpd/ScalaTokenizerTest.java index 36865795c5..d09b6c84d1 100644 --- a/pmd-scala-modules/pmd-scala-common/src/test/java/net/sourceforge/pmd/cpd/ScalaTokenizerTest.java +++ b/pmd-scala-modules/pmd-scala-common/src/test/java/net/sourceforge/pmd/cpd/ScalaTokenizerTest.java @@ -36,6 +36,11 @@ public class ScalaTokenizerTest extends CpdTextComparisonTest { doTest("sample-LiftActor"); } + @Test + public void testSuppressionComments() { + doTest("special_comments"); + } + @Test public void tokenizeFailTest() { ex.expect(TokenMgrError.class); diff --git a/pmd-scala-modules/pmd-scala-common/src/test/resources/net/sourceforge/pmd/lang/scala/cpd/testdata/sample-LiftActor.txt b/pmd-scala-modules/pmd-scala-common/src/test/resources/net/sourceforge/pmd/lang/scala/cpd/testdata/sample-LiftActor.txt index 0534d3c538..6c4b243eb2 100644 --- a/pmd-scala-modules/pmd-scala-common/src/test/resources/net/sourceforge/pmd/lang/scala/cpd/testdata/sample-LiftActor.txt +++ b/pmd-scala-modules/pmd-scala-common/src/test/resources/net/sourceforge/pmd/lang/scala/cpd/testdata/sample-LiftActor.txt @@ -1,6 +1,4 @@ [Image] or [Truncated image[ Bcol Ecol -L1 - [/* Example source code copied from[ 1 4 L19 [package] 1 8 [net] 9 12 @@ -40,14 +38,10 @@ L26 [Unit] 19 23 L27 [}] 1 2 -L29 - [/**\n * The definition of a schedu[ 1 4 L32 [trait] 1 6 [LAScheduler] 7 18 [{] 19 20 -L33 - [/**\n * Execute some code on ano[ 3 6 L38 [def] 3 6 [execute] 7 14 @@ -79,8 +73,6 @@ L43 [onSameThread] 7 19 [=] 20 21 [false] 22 27 -L45 - [/**\n * Set this variable to the[ 3 6 L48 [@] 3 4 [volatile] 4 12 @@ -88,7 +80,6 @@ L48 [threadPoolSize] 17 31 [=] 32 33 [16] 34 36 - [// issue 194] 37 49 L50 [@] 3 4 [volatile] 4 12 @@ -98,8 +89,6 @@ L50 [threadPoolSize] 37 51 [*] 52 53 [25] 54 56 -L52 - [/**\n * If it's Full, then creat[ 3 6 L57 [@] 3 4 [volatile] 4 12 @@ -149,7 +138,6 @@ L64 [val] 15 18 [es] 19 21 [=] 22 23 - [// Executors.newFixedThreadPool(th[ 24 71 L65 [new] 9 12 [ThreadPoolExecutor] 13 31 @@ -282,8 +270,6 @@ L91 [ILAExecute] 13 23 [=] 24 25 [_] 26 27 -L93 - [/**\n * Execute some code on ano[ 3 6 L98 [def] 3 6 [execute] 7 14 @@ -464,8 +450,6 @@ L127 [MailboxItem] 15 26 [=] 27 28 [_] 29 30 -L129 - [/*\n def find(f: MailboxItem =>[ 5 7 L134 [def] 5 8 [remove] 9 15 @@ -588,8 +572,6 @@ L157 [\]] 71 72 [)] 72 73 [{] 74 75 -L158 - [// override def find(f: MailboxIte[ 5 78 L159 [next] 5 9 [=] 10 11 @@ -659,8 +641,6 @@ L167 [)] 42 43 L168 [}] 5 6 -L170 - [/**\n * Send a message to the Ac[ 3 6 L175 [def] 3 6 [send] 7 11 @@ -675,8 +655,6 @@ L175 [this] 28 32 [!] 33 34 [msg] 35 38 -L177 - [/**\n * Send a message to the Ac[ 3 6 L182 [def] 3 6 [!] 7 8 @@ -793,8 +771,6 @@ L199 [)] 10 11 L200 [}] 3 4 -L202 - [/**\n * This method inserts the [ 3 6 L207 [protected] 3 12 [def] 13 16 @@ -936,8 +912,6 @@ L230 [}] 5 6 L231 [}] 3 4 -L233 - [/**\n * A list of LoanWrappers t[ 3 6 L236 [protected] 3 12 [def] 13 16 @@ -949,8 +923,6 @@ L236 [\]] 52 53 [=] 54 55 [Nil] 56 59 -L238 - [/**\n * You can wrap calls aroun[ 3 6 L242 [protected] 3 12 [def] 13 16 @@ -1574,8 +1546,6 @@ L349 [}] 3 4 L350 [}] 1 2 -L352 - [/**\n * A SpecializedLiftActor des[ 1 4 L362 [class] 1 6 [MockSpecializedLiftActor] 7 31 @@ -1602,8 +1572,6 @@ L363 [\]] 45 46 [=] 47 48 [Nil] 49 52 -L365 - [/**\n * Send a message to the mo[ 3 6 L369 [override] 3 11 [def] 12 15 @@ -1630,10 +1598,6 @@ L372 [}] 5 6 L373 [}] 3 4 -L375 - [// We aren't required to implement[ 3 79 -L376 - [// since the message handler never[ 3 43 L377 [override] 3 11 [def] 12 15 @@ -1653,8 +1617,6 @@ L378 [=>] 12 14 L379 [}] 3 4 -L381 - [/**\n * Test to see if this acto[ 3 6 L384 [def] 3 6 [hasReceivedMessage_?] 7 27 @@ -1672,8 +1634,6 @@ L384 [(] 72 73 [msg] 73 76 [)] 76 77 -L386 - [/**\n * Returns the list of mess[ 3 6 L389 [def] 3 6 [messages] 7 15 @@ -1684,8 +1644,6 @@ L389 [\]] 23 24 [=] 25 26 [messagesReceived] 27 43 -L391 - [/**\n * Return the number of mes[ 3 6 L394 [def] 3 6 [messageCount] 7 19 @@ -1830,8 +1788,6 @@ L417 [msg] 24 27 L418 [}] 3 4 -L420 - [/**\n * Send a message to the Act[ 3 5 L425 [def] 3 6 [sendAndGetFuture] 7 23 @@ -1849,8 +1805,6 @@ L425 [this] 51 55 [!<] 56 58 [msg] 59 62 -L427 - [/**\n * Send a message to the Act[ 3 5 L431 [def] 3 6 [!<] 7 9 @@ -1888,8 +1842,6 @@ L434 [future] 5 11 L435 [}] 3 4 -L437 - [/**\n * Send a message to the Act[ 3 5 L442 [def] 3 6 [sendAndGetReply] 7 22 @@ -1904,8 +1856,6 @@ L442 [this] 40 44 [!?] 45 47 [msg] 48 51 -L444 - [/**\n * Send a message to the Act[ 3 5 L448 [def] 3 6 [!?] 7 9 @@ -1942,8 +1892,6 @@ L451 [get] 12 15 L452 [}] 3 4 -L455 - [/**\n * Send a message to the Act[ 3 5 L461 [def] 3 6 [sendAndGetReply] 7 22 @@ -1967,8 +1915,6 @@ L461 [,] 70 71 [msg] 72 75 [)] 75 76 -L463 - [/**\n * Send a message to the Act[ 3 5 L468 [def] 3 6 [!?] 7 9 @@ -1995,8 +1941,6 @@ L469 [,] 21 22 [timeout] 23 30 [)] 30 31 -L472 - [/**\n * Send a message to the A[ 5 7 L477 [def] 3 6 [!!] 7 9 @@ -2043,8 +1987,6 @@ L480 [)] 23 24 L481 [}] 3 4 -L483 - [/**\n * Send a message to the Act[ 3 5 L487 [def] 3 6 [!!] 7 9 @@ -2193,8 +2135,6 @@ L506 [)] 18 19 L507 [}] 3 4 -L509 - [/**\n * The Actor should call thi[ 3 5 L513 [protected] 3 12 [def] 13 16 @@ -2226,8 +2166,6 @@ L517 [}] 3 4 L518 [}] 1 2 -L520 - [/**\n * A MockLiftActor for use in[ 1 4 L529 [class] 1 6 [MockLiftActor] 7 20 @@ -2503,7 +2441,6 @@ L565 [(] 24 25 [true] 25 29 [)] 29 30 - [// access private and protected me[ 31 70 L566 [m] 9 10 [.] 10 11 @@ -2826,8 +2763,6 @@ L604 [}] 5 6 L605 [}] 1 2 -L607 - [/**\n * Java versions of Actors sh[ 1 4 L612 [class] 1 6 [LiftActorJ] 7 17 diff --git a/pmd-scala-modules/pmd-scala-common/src/test/resources/net/sourceforge/pmd/lang/scala/cpd/testdata/special_comments.scala b/pmd-scala-modules/pmd-scala-common/src/test/resources/net/sourceforge/pmd/lang/scala/cpd/testdata/special_comments.scala new file mode 100644 index 0000000000..655a06a4a5 --- /dev/null +++ b/pmd-scala-modules/pmd-scala-common/src/test/resources/net/sourceforge/pmd/lang/scala/cpd/testdata/special_comments.scala @@ -0,0 +1,24 @@ +// Testing CPD suppression + +// Irrelevant comment +// CPD-OFF +// CPD-ON + +case class Foo() { // special multiline comments + + /* CPD-OFF + * + * + * */ val hi = "Hello" /* This is a comment ending in CPD-ON */ + + private def bar(i: Int) : Int = { + val CPD = 40 + val OFF = 60 + CPD-OFF // This should tokenize + } + + /* CPD-OFF */ + def bar2(s: String): String = "bar2" + /* CPD-ON */ + +} diff --git a/pmd-scala-modules/pmd-scala-common/src/test/resources/net/sourceforge/pmd/lang/scala/cpd/testdata/special_comments.txt b/pmd-scala-modules/pmd-scala-common/src/test/resources/net/sourceforge/pmd/lang/scala/cpd/testdata/special_comments.txt new file mode 100644 index 0000000000..174012eb14 --- /dev/null +++ b/pmd-scala-modules/pmd-scala-common/src/test/resources/net/sourceforge/pmd/lang/scala/cpd/testdata/special_comments.txt @@ -0,0 +1,40 @@ + [Image] or [Truncated image[ Bcol Ecol +L7 + [case] 1 5 + [class] 6 11 + [Foo] 12 15 + [(] 15 16 + [)] 16 17 + [{] 18 19 +L14 + [private] 3 10 + [def] 11 14 + [bar] 15 18 + [(] 18 19 + [i] 19 20 + [:] 20 21 + [Int] 22 25 + [)] 25 26 + [:] 27 28 + [Int] 29 32 + [=] 33 34 + [{] 35 36 +L15 + [val] 5 8 + [CPD] 9 12 + [=] 13 14 + [40] 15 17 +L16 + [val] 5 8 + [OFF] 9 12 + [=] 13 14 + [60] 15 17 +L17 + [CPD] 5 8 + [-] 8 9 + [OFF] 9 12 +L18 + [}] 3 4 +L24 + [}] 1 2 +EOF diff --git a/pom.xml b/pom.xml index 2312e8579f..6cc41bf1d9 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ https://pmd.github.io/ - https://travis-ci.com/pmd/pmd + https://github.com/pmd/pmd/actions 2002 @@ -94,7 +94,7 @@ 3.0.0-M5 8.30 3.1.1 - 3.13.0 + 3.14.0 1.10.8 3.2.0 4.7.2 @@ -310,7 +310,7 @@ false - ${project.basedir}/../pmd-lang-test/target/dokka/pmd-lang-test + ${project.basedir}/../pmd-lang-test/target/dokkaJavadocJar ../../pmd-lang-test/${project.version} @@ -1028,6 +1028,7 @@ https://sonarcloud.io pmd + 1.8