From 3d4928dfbb9fefbd4e7b883710d81f424b5c66f4 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Tue, 10 Nov 2020 22:26:26 +0100 Subject: [PATCH 01/52] Initial version of new ci scripts --- .ci/README.md | 16 ++++ .ci/build.sh | 26 ++++++ .ci/check-environment.sh | 12 +++ .ci/id_rsa.gpg | Bin 0 -> 2557 bytes .ci/id_rsa.pub | 1 + .ci/install-openjdk.sh | 67 ++++++++++++++ .ci/logger.inc | 21 +++++ .ci/maven-settings.xml | 31 +++++++ ...lease-signing-key-D0BF1D737C9A1C22.gpg.gpg | Bin 0 -> 8226 bytes .ci/setup-secrets.sh | 42 +++++++++ .github/workflows/build.yml | 30 ------- .github/workflows/pull-requests.yml | 30 +++++++ .github/workflows/pushes.yml | 85 ++++++++++++++++++ .github/workflows/releases.yml | 17 ++++ 14 files changed, 348 insertions(+), 30 deletions(-) create mode 100644 .ci/README.md create mode 100755 .ci/build.sh create mode 100755 .ci/check-environment.sh create mode 100644 .ci/id_rsa.gpg create mode 100644 .ci/id_rsa.pub create mode 100755 .ci/install-openjdk.sh create mode 100644 .ci/logger.inc create mode 100644 .ci/maven-settings.xml create mode 100644 .ci/release-signing-key-D0BF1D737C9A1C22.gpg.gpg create mode 100755 .ci/setup-secrets.sh delete mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/pull-requests.yml create mode 100644 .github/workflows/pushes.yml create mode 100644 .github/workflows/releases.yml diff --git a/.ci/README.md b/.ci/README.md new file mode 100644 index 0000000000..bca56da5cf --- /dev/null +++ b/.ci/README.md @@ -0,0 +1,16 @@ +## PMD CI Scripts + +This folder contains scripts used for CI. + +## Secrets + +One secret is required for decrypting the GPG Key with which the PMD Releases are signed and +for a ssh key, which is used to copy files to sourceforge. + +## Environment variables + +* `PMD_CI_SECRET_PASSPHRASE` +* `CI_DEPLOY_PASSWORD` +* `CI_SIGN_PASSPHRASE` + + diff --git a/.ci/build.sh b/.ci/build.sh new file mode 100755 index 0000000000..295344ded5 --- /dev/null +++ b/.ci/build.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +source $(dirname $0)/logger.inc +source ${HOME}/java.env + +set -e + +# configure maven +# probably not needed? echo "MAVEN_OPTS='-Xms1g -Xmx1g'" > ${HOME}/.mavenrc +mkdir -p ${HOME}/.m2 +cp .ci/maven-settings.xml ${HOME}/.m2/settings.xml + + +#MVN_BUILD_FLAGS="-B -V -Djava7.home=${HOME}/oraclejdk7" +MVN_BUILD_FLAGS="-B -V" + +log_info "This is a snapshot build" +./mvnw deploy -Possrh,sign $MVN_BUILD_FLAGS + +# Deploy to sourceforge files +#sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip" +#sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip" + +#regression-tester_uploadBaseline + +#build and upload doc diff --git a/.ci/check-environment.sh b/.ci/check-environment.sh new file mode 100755 index 0000000000..524e3247cf --- /dev/null +++ b/.ci/check-environment.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# +# This script should check, that all needed commands are available +# and are in the correct version. +# + +source logger.inc + +set -e + +ruby --version | grep "ruby 2.7" || (log_error "Ruby is missing"; exit 1) diff --git a/.ci/id_rsa.gpg b/.ci/id_rsa.gpg new file mode 100644 index 0000000000000000000000000000000000000000..8e68ae0f100435fca4d9c8066e5dbd2117687a73 GIT binary patch literal 2557 zcmV!5$``#6|RLZhUVdBKN+V=w;z99mr9QTC3Z zAe@zNZ~6_Iv1b1cQpVX-o>T!pH@*jfk$6tXh9Wh*$=k5_`Fl|GDKoavxUlA!Ep07e zbb9d9VnagZPnaUm4;v})+Z)46v3E*u(`&M;vpOLWnYt&%)2H5TCj8XzsRN0D^IMAw=(g%pB^cO7PUO%`!e!52(8Nd? zVuj~0yrP1dtqtl9hyRoeu7%6NBUq|=nZqQ-^~|h9u`@~xSU&Ujj&)A_wK{uA4lt-x z*Ljfk9r8IhuM=Z4FI!%LW3g>6e!eysMg^$5=1777L2OlUFJk`RfTZZy8@epgoB6qr4 zg6zy~EgI7tW@Of%RRQGC4XWOiq$gsU9t!~$(^yY!HE5a;?OPhNy%9h~wOFA4!Gh29 z7G)0(Oj;sOh`QS~7UU=0=ub*btn(Fw6UHM^F!+iRgHT9iniv7S@Q?`JJ?frq2h zrxl>dbgVs2U;x*1trEKjhRR$4H$K~689Y4^?Bkdewfd17j8>d=2Jc9ye{-WBsPXpr zV>3_FpkF?H#rXjDT^KrRlBjqD=j?%87OA%z=T{BuWgOSspCqEQwb|@$fH3gd>vPA} z5G=?$0Dz@^OJ}u6@5zaH;nNRWZ%zy88w`)5a0DN`l8HUNU8b{6(>BTJku~T`3AZKM zeliav!l&y`PI!4`>YZic^zDGFq?_d8=#t@@Xx^!|;L{>n^Rj?Yet$Rq?bSr70#_hn zhheYlKFfj{kr><`*e0ulkSK#rRkp=!SlU{nG5Z~+*ft%H=uj)`8#tFfFnP?|KaGjn zfS2~wZfwWFRePmIBF8`tY2@H?q99{ScKV(ueyN9QP1wSQa>co`B0BLKZ~XUil>||m zm+7T@z<0B;avhxJMjcgHLQXQ*czHn*R2BfRN(;WHyZ!XmG_CJdo@}$iV5*lK9BA20 ziLEUTjklY+UNui}ojS1JnY>)FPa@&{E#t316u#mFTy6MZ6a@dTYl#VVien4atgXRCAG){bB3Fh2w?T(wGN zLTDl6oM?R%3?@VH`>pHmoo&dSj^~i*bU$reMC-s z<>o|GB~v>O;}Pd=EM*8DqiJx0KM_Wq2o%9S`g*<-QH}KQ9A~;GyJc2*H)n+y@Jrpo z3E&a*dAlgjaWX7GS&Vdid$-Cce}zMK^~_tuzki+x;$cZyC16Sc`ArfZIWztzhyt>( z($=aY)Z`BwFHp(#?t6s>!x-wS&CpBgCgoxyC1U)mosnZ)>Jn&wDmLnO#P>g>1$R|> zmU?jY>kefOXlJIXWTy?!X5o-WYxEXQAa2=~(H9ko92s6S>p?vS#+OiiW-& z;{lQ4e_IUbR{yyn@sKXLqq-q&Pr}+_T);i#ZC6FRQmMt)$%TM+E;w4DQ64m2+qFP? zBxvMdbc%;d$m0rZ{gCOp_NSPZ{*F?)qc7@D@~$p(61hpEMaL!zOO$ z#nbqs+K8lQ`U(s`FjH%Q+r;z6`0pVbhf+m*oz6HTV4h8&D(c^dw!MNF)djhghFm3n zjm_==z4T5vWr8ItC0zai%4R^|iY$qpbtL9(PVlI#3#P_iDdSZgt4!;hOgLFwObztq zvT9Pn#DoDl)XcM}Voo$3H)t*aVIsHVR;b&%E~AOI4yXU@9D~5dGkKY|qe+l{+1*4pZ94ub;)NC}d9;>@qDJ#R?)dF+T;-qeWdAc?t1qm^k%ZW2DWpS|>o zluFTk1+G2jz2o$v_T?R*eOu?Ers)?3M#qh^vR=8L373S~i<03u+#DCo&`F3=o^gT{N49K|#e$)HI9(>woA zm3Lc8cHs$&YuBh$B|3C(YgLG*3d~X~G}tHOv9r`L63tE~^N! zjXhBCW}kE0AE)q*J0L1tIYMph#ABztuX}J$D0sQ65m|fWOMp)Onq*9N*`jpi}kciBI%~TaH{2H6EV^!t>ISI%8o$t?8>dk)e{BQ!-@l2;;{{oxQHU Ttgvvhf~HU5@82)qpB@Z;=9uuZ literal 0 HcmV?d00001 diff --git a/.ci/id_rsa.pub b/.ci/id_rsa.pub new file mode 100644 index 0000000000..7e1b51e198 --- /dev/null +++ b/.ci/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC8yHVLHYDsKui8tYg/sFvkDqcs97pEZz0BzK9HtBF4O+/It1drRYRdUAFfjoImfprpKSxkJCTglHixGRp24eNaZ6woWVJ4/bmiMkEqEZAjr1NZ3qw7zIruMJMSkCV+YTtmL4cYcZlvMRPzzOZOnFbV05oi79oy41MUFHYjolK9QxMFNsVNN5iyzFxM3HqSFozz+ylKbFBtDk6ZHZQNRL/Xl2V9DJ69fVzjG4OZfcWNGmmKHHARmsnJyUOMeeKpLjDOe1M6ZdI8HkXWac8yCr9JTETNZZwemZAcS/RKoKCDqfIUOzkZfIPmyaznfVetTGsMi7yQrJhAyjznuNGF4+3lfgTcmRF8wz5FCeUkdYTmy2wNSFi5HiLPfC5OgRtjKzC6yb8rbRjDx6XQ2ph15PKOaXwzk49TaMc0xJvoiGDMZaTU0iTm3Y1/QUtfLvo3/jGMbtUdV3soWpuBAV2JUI4aB5xdLX9iNmcrVzoUe3y9DWuuTX46eoCvpUNXv/DXKhQw1D7xd7J67db5qUck/Akiqi0JR+e0SoBJvZFtYwVNLGC2bIJ/s8SR8X5Zp+1+ypf3WYjIylxQTkO1r4NfI0Cd9qXg7nmUrHAU7Z6xtJmUK8ZWzSST4wul8WkRJURtODLxt5firtlKhyZ93t9Mjuk6mATIPxr/b3x20T+IH463kw== ssh key for pmd. used for travis accessing sourceforge and github. diff --git a/.ci/install-openjdk.sh b/.ci/install-openjdk.sh new file mode 100755 index 0000000000..924fb7e025 --- /dev/null +++ b/.ci/install-openjdk.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +# +# Downloads openjdk from AdoptOpenJDK by accessing the API. +# The API is documented at https://api.adoptopenjdk.net/swagger-ui/ +# + +source $(dirname $0)/logger.inc + + +case "$(uname)" in + Linux*) + JDK_OS=linux + JDK_EXT=tar.gz + COMPONENTS_TO_STRIP=1 # e.g. openjdk-11.0.3+7/bin/java + ;; + Darwin*) + JDK_OS=mac + JDK_EXT=tar.gz + COMPONENTS_TO_STRIP=3 # e.g. jdk-11.0.3+7/Contents/Home/bin/java + ;; + CYGWIN*|MINGW*) + JDK_OS=windows + JDK_EXT=zip + ;; + *) + + ;; +esac + + +JDK_VERSION=11 +DOWNLOAD_URL=https://api.adoptopenjdk.net/v3/binary/latest/${JDK_VERSION}/ga/${JDK_OS}/x64/jdk/hotspot/normal/adoptopenjdk?project=jdk +OPENJDK_ARCHIVE=openjdk-${JDK_VERSION}-${JDK_OS}.${JDK_EXT} + +CACHE_DIR=${HOME}/.cache/openjdk +TARGET_DIR=${HOME}/openjdk${OPENJDK_VERSION} + +mkdir -p ${CACHE_DIR} +mkdir -p ${TARGET_DIR} + +if [ ! -e ${CACHE_DIR}/${OPENJDK_ARCHIVE} ]; then + log_info "Downloading from ${DOWNLOAD_URL} to ${CACHE_DIR}" + wget --directory-prefix=${CACHE_DIR} --timestamping --continue --output-document=${OPENJDK_ARCHIVE} ${DOWNLOAD_URL} +else + log_info "Skipped download, file ${CACHE_DIR}/${OPENJDK_ARCHIVE} already exists" +fi + +log_info "Extracting to ${TARGET_DIR}" + +if [ "${JDK_EXT}" = "zip" ]; then + 7z x ${CACHE_DIR}/${OPENJDK_ARCHIVE} -o${TARGET_DIR} + mv ${TARGET_DIR}/*/* ${TARGET_DIR}/ +else + tar --extract --file ${CACHE_DIR}/${OPENJDK_ARCHIVE} -C ${TARGET_DIR} --strip-components=${COMPONENTS_TO_STRIP} +fi + +cat > ${HOME}/java.env < + + + + + + org.sonarsource.scanner.maven + + + + ossrh + adangel + ${env.CI_DEPLOY_PASSWORD} + + + + + + + ossrh + + 0xD0BF1D737C9A1C22 + ${env.CI_SIGN_PASSPHRASE} + + + + + diff --git a/.ci/release-signing-key-D0BF1D737C9A1C22.gpg.gpg b/.ci/release-signing-key-D0BF1D737C9A1C22.gpg.gpg new file mode 100644 index 0000000000000000000000000000000000000000..d0b2babc1cc2d12c305e57e27255353877381c8c GIT binary patch literal 8226 zcmV+-Al=`L4Fm}T0uIfVeaeWS$p6yq0mYpTNUZYl9!8YYfmS!?DL1(H#ZUAes$_u3Y5d3}9&m6IiyrQ%@T~;ZR|?3b zq0UP{rRBIhS*?=v;!Bp95gWLDbke@BZj&_%2PkQ1ooWJZd5y}Jc#@oR?le|8gXQcS z$7jW>1i^}<%s#K8OB7I_ow!FR#K4}ZbHbbXG1wG1s)9ZtbIcFPA)o=?ZroS^-c%{t zD;9U&zx*tAdQ)b(vM=)dPiQ$Lw+-CfH*yNQ#NZmm7J7nz6%ccATq1fQdv_(zmznC< z$8#u>Tf;6zU4I*g>B_>1u}ZhQ7@C&p?3jGbM~wX~b=mo|w@?@hro@KEtXKt#`BS&; zaX2-v{-KI{T;r1GD0k((l>>ft`3Mvq44=;7)w$FHUI?unpE3aGSp9p<{n|$pumhBza>#;^G&s~u*`hFL0#De6Wetn+D;(CoAXSkNx z@oW9`ad(oSQ+_ebZ(Urs(*eHVz*hjb%RMbYzAP2SB-qs{p~fUcxjj2ur8Kt#1LbgW zsd=B5qDcG6?+B-1W%?}Iv}{?Fqa4_vrcd-as_$84kEDmru?hP~_VbcHWLVy`yx}&C zxVfdv&dAzT2U%kbI)vs5b2~kHdY?3ao^Se`MQhCsTmovB5-q2k>wS;?hcA zN)tM4i5M258d1PPkf{UFT%Ic0m?HCS$BiWqcCA$JF#L>se=vMxmtcj~$j$2mfU7+D zhX=@fsUFn0N#?DyNyp$Zd{tuLx;zJYA>n@KL1Q2fV~P*5LuuUK(VUN4t?L>doTgv% ztjkssq>D65$=0{+$2#h~80mH6Q5!>0otkOp!(1AF1u7}0i~%z>)^G>q1s{b!L{cw* zm=!`X?%5c%WgWavfN@)~Cvce-TyN_$vmeaOoXhb5{x|fv57{tNt_^10Lpi@cVuiQ9 z>BrSiJD}$4UVeoX+2fMxJr;XllCST;RMor9Pp1fD^5MH{%;O!fF38Vfw$UB@=#FJG`yR`k-b@AOrR zQM*4?;}7?Ti>i=w*qCwW#_KN4VF+TbtcG76ki-158y>9NFcNlRcu*0?tSne##Q{v} zv^*zxgnp41c-(|0@E>fu^eR+VL_(w0FI{OvbICm6fbEQ0%Df=3RR8oDa)O|TeJZw< zl`@Tq^SjGlc7%-G;5C!k?h2Qo8TvT|2C(muJTt`L zIz$~1S}f?HqS5oGwa-p76jBdCvL-tsaJs3vLXe*RFF52}1rq4^G4Ec4za$dRU5dX( zHbJbwdOcU%-%^P{z|!!GhY*n;lwBD1yqh=Hsh3h;{^@J0P`-{!4*)Bn8H&rPo+cf0 z3ET(`{H;Op8HaFkPHV0hckQpX{ONikjQSl@L_o_ZD@3Rb`|NNX+@rM6ZANI)k})DB zC&p*8Ytta27)v@%*T@BkJWO2b)FiTkX%g33-R1j?8lK8uIDKC`XEdH*{o^O!9X#D2 z)!yf$L6lw)y2LB;;cc!SK-ojA`Ad~)988^2)!p>em`vU<)`6TS>GKHa(Ex)pxZ=Ta zgp@E!u3TxJT=#o=W9AeeEG0nPdPpmwG4GTnuIbA};_aj=@LM zd7SOSG)gbWMKRq;l%E&y#ROJvi|mwG$UfH?lEC+PbBb+i;E;oN!Brn5Tdb>N;LX$E zW%7O6E~#xJz0lL5g&U1ZmEGoEGk6+1=E>On>r+YuFLy92Bn?TmxuyA-CFxjR`&RA``eriESPctLnURkf zdR~h2*P=T!y^W;5EE61tZ8m@=c!FcYO$RayspPmXi5e&Ds}a91yR(DXfwkc4B9NJQ z(H|?^mZF%|<-Ohiu>Jkkrf7^GIjD4miB~y!OcM5+N}TOG_717#s8|?+pD5ld%+d3- zcPQRDH5M3K6y6kZqwKmD(}y=KDFn=)8F07AK|e)mFFnbcLdlyyGS$4iDb%dk2 zFFxc)%Rk}uw?1{DZq>N41!#NDCjds=FvRHObeBTHqfGwPo_tPBG@?CuJMmX%O`bvK z8OXxaM@jWo4ZJ<7faY+SfGP1-%SJZNg!H_3@p2Z6H4gt9&R3u#4=eCF93YTrtVswa zm}F_rj(P-kD*eOli%xRuOUfbR*dMlrYpsQ z6oLZGbp!Pp>go+4RpE~oMiVp5AT(d5nk)8nAtDEG>`{U&GMwWfuIzJErTSBKv0Jx` zayLwG`;ZhEbt^bV(q}^aN6BBzo}dlHZ_Kk^9r$X2|5wNsP>Qdq^I7 zbZ5)qwJ+vMX&5nA9j4ecru*QB7G2Oc?MO1kHl-9+R|BEh(T*gv8mLN{Z zzrTA6%DjGxoFiIB4Ta!wz=WG$f5BI!7OW4AV!3ez;7-Xl8+qE)EmV1<%TKqQ2+pGU z8FW7o;*0U-gIjsvf!TN2YMOW7XMZCat5Qct0Fu~~B?MJ8szJPNaxDhGKoS8was>NA z$mvNjWPfn6w}Ug-BQ5|Zn5JJIE%J1NV-wV4ZUC)rmhZ6EL2ecMO(T$Rt`Y>K8v!u> z(+yDt7r5uQBZw)g5KOCtU2otEaAaWn$6y z{f=kvP9ghHU{k&>mHpSL1j&nF3d!7GgS1DRK<;pcl{?>FX-pzFc96dsA1gD{$5&#b zA}f{~5+bBuQOy>qA8u*EN>*CzTvX^~{Au@0Hk+Tg6q@{_EI2n*$$&(>A^wmOGAAY6 z19crvS~&MKxNS@?l>i;RWt*NV@+} zJ1l!B(G|r;KZtv~5a(kBL0eBEJQu1o4qOQx4`#TtJJ>cXoW2H(#>B%gC^~(dqVdMo z=d!KN^cTB7n{NyVGu&EYsC+26vAB_>6-_ipN9~SE2=%o+On3U7?FZhk0q<&jP-Hw{ z{pErXY&JA5o~;OFZJq;5D`eG=6v(Tq62#vcFRW^m1P*~aj$lR8uCQ0PZ4$)ue#HS; zg7qBE;BsaUa({zT3>ks0Wf448WdN{9tegk(dbkDq?hT!F=@M%|V2(Z~P7@|7$`}S9 zhmjXV93=A9dV8^dP z3PAS15{6DpZKEJ&JXHkJcC}&*V?@^YLOP}iIHAyX25|7Ep;>C9c`N`UEAZ%f{~UoJ zg(y-`aT6|Gi|{<|a>U;Ypi-}QDGU9t-ujQ)_fQWaamgO0ae$D!bn5HOjzNTfA|2r} z45UEy8Kq*62DY(Hv)$An=Dx6yq4_Q*4@Mr_F#h*QzY&ftId;j~@Qd{iu!O4AbrS-j zcC7eQHfgb!?EIds`AX2mTft5xE%k7aa@mKE@MQnhKXl@KHx&(F292|_P}_EHq3Ua` z1|CxJ3ShGK?4%e(R;`jp&85^eK+pG*E6&*66UNXd#8M3(v>wUf#y|abL_>6?rR2#O zO^&e|A$ZRcFzGH{nu

if&`bg6%RXx?HNtc)s0>xNVA#Mrf7Z3^whw#AUj5g+ zI-UA8J%EVf)LO$$^xX8S7jfQR4%5>&i}_&!8qtejerqsGG)pqNgTWF#Ox3@Vw&(i~ zjh&Wsus&h;^y}oXHQvSCC+7a>-L(5;C$rb8HMN9Z75P4^9xI3GD(4<`NWX0&p+u^sJih4+ zy$Sy%aYE~?xZA9x;6n;<#Y|Eum(@*iZi zo{+-Bn<4QxgNCPr#%$b+fQFZ}EbM~eBS6j)(Fyksy}_R#y?D4s^(%_AqMv8%u*(kq zp7~dXjn^)Zc6Q=_6|k;YDi(z!!dr@U#-{nqZE%#RrV1HAD1FRQwP{i1*s7kQgE1lf zwsFCn#R(*oIH6pHNh|AZBhrHkF-Cf!)S_?fyk7rH1@m%oWlx%bx&`#^T1jpaofJK){x6# z!P-IeS?xjO1tVEWfgsZHxjPrdVb&*9$&{#h;1`Z^a}&mTkDiBD`OPv&iDUjKCXgBi z-QaPmiZs!j5i&O&o^x$?Blb!FnGbS-S@vb|y0BD`StkhO5zB`3FQlV&Vq-70MhRG%eJrLWr0#ed(vkVw3Z(S)r>3eX#>>yjpDwrDG+xaGk6Kx48SC z?kQzQ$QLx9tG_ZuTITru!Y1q9Q-pZb9FH`>@MygNYadj!JP@Ty5NAMJHl-hV=PpvE zA`&b8qm4dW_*Pf5@WY%E>_H@^`P;__|%QV(|-st!^3u_P~4 zSqDujK8@=$Y-nnl`}7f4!i=g8e;mu5>UCCN$R#1ETE-lI4UUsI+GQ_j)&lJ~Ify>@ zwH5^%9jfIS&Dd`8_l$VkS8z-+;?u*Fdd#d*IdNo9*n9bRX1cqxHG;%p(?VTME16$D zGa$6KP}#nLk8KTBF5f8d2g1ohZ7msyG>{|tw2Ud_R=w94=)Arfbin|caVcSAbRw49 zE>o9pKzIo28c%%|V-*F@Jlk1y)P}=Bat5SREpBBKQ0Q#A{d){Nehe`U!$A|5I6n7~ zQALnB6wi>TZGhn0QhO0_7-gds7Rq+YNj5b3`0ue%-JX;7Z?V;j^NQ{+l-?qwfxk92 zuc8YGh=bW#=Fp&dpdJ7L7^$8Chefual|M(4aqjP|0+gzly%3%xN5r|o%G-$r#SJYo zi|-a>U^}=6HmrYq!4c4VPU)l0M`w4x4 zdQ6{=s*YDx;8W7>z2F-_#xmy(qFNDX=^2N*+`%7%JA^%BPb%{FcoY$dziWNQ(j&bQ zyKXCQ_t15gZiS6$kk9y~mRa4RjrP{y`gv50>B=p9$qqDpI4_u2y(MuX^X@zSMD?3BzUJ{?@O>5B zYC&49vHc8hNx9^Bzd1%*$nn)@#R58<{Zm0{3d6#p zq{4&Dq0e$O)3AT7d3-%fj~jcB*xqiL0p5x8ld8!LisY}E!BUgbOESW8%=o|?=albj zY7y7Q)?xc3xnsSzgH zG}>uo=XKxtNUb`u@>V%ZRH3ad!}#yZ*NMP`&aP_y6Pwv@EqdZjaEH-3tG0|`BI@aB z-Dozju1Onk{X_|Kz6pL= zpG|SxK!wuUq1tj6GD`;bTB`n-owvve)R*ht^DNpwqg3vfh#3MMPdr8NcO&ySWFwaI zsNbx*Qz&`hjos2#^r@PHk|P(;{90BEk^mZ2+ZQ2CUI)tm%|$gmsc{YQNN zN=F?b_~zTG*0w&JQW0gyjQ`~US3t#gS6Wy@_aWMTwY~?_GnuW7SwUq6*_)eU zXY|G7wTkXaw-q>9^Wq*lg6s7wV0;9bMXk7$bY+Wbz*1d~@<2&q%?RrSp$AvG9PL@% z&-Cl<;|+5-Zgp@R@f~i#t7;Z*(bo9j=aOdB-m#4s;0T0ZyYq3jW&l<~YI4`k6NmxP`5-)9 zr9>%2)5&7+fOH$XTbp-Kjq;u1qzahMIKMB=n=BAzS`s z@~xin?17W%f?KPK&YMn%1&79g@9oQo>By3xo3ovD4nDsOL46?2XF+x*Cj!Hi2htl) z5g72pZN{E9(>yK69>Io2sJGMB)zDM-Wa$L~&p`j|Cg32~4$~t0UUWogQgQ_U%0t_v zwkGpmKrcgRmHOA{h?*Kl)}=j% zza;|Tj&qF_jw>rd+V}`K0MrT`nmVGfX5LO0to2o{6CwCDq7El80-r}68p>`3JfG3! zR^BgCqypr2_7~L7-Az&je+Ttz(<=AL&VpA?en*2$J&{&>XsyMnWGbYD#I(zFSIqR6 z3P;aa8~~T3QNOf`Y2#U>u6JcnNkHZm?!a(YK`yb};j*R*KLI{kfz#yCzm4=|x4|ng z3aWruc_exdIDs5@d<@7aZuRhdQMVYz00)U=VT=L%$+=GjC@BfT_^hN>2reC9Cw5Ue z$u)z1=YU&!V1J31a7Csbb(OC@7cxLydKP~Pa_=NV-14OX`&z<7Gb$nk_$0Ac`1t~F zdq&w&5npyH4Jaf+vvR})%?I*5>NpHd-JfE@H-ky1({weUreMnu9&x5gcvyN+YE}G* zt$c$ZrZeDQs6Cq8`%FO1t+~ya{+;`Sk-s>-%~) z5&&ijbBJpEla;`$>FNGvC+@Xh#JogA4Ln;N_;WIIXW5|`^sLr<=5p8>|JB`k1?+)U z(l&S&MOwYmjxMAE&{bFYEq$QCLb%_x+Hur!a8uysNC_1t`j(gDhTGFk3Ujma%CM!7 zhZa+sYKILkL4FKr*bM&D{(Nj@=FXjOHfie2wRz-R4TJn*pLS63V^Qz+amOHf$RB%& zSlp^#p_4|1cp~eMQFdq_KoCVg z)huOF9|2|#mD?(^)rO}(4E6dHWmxN&zQ2%P(E@TrT=r-B6O4^Q18oiUNP<3-gr>vj zFwlYHfo6P&ES zv4A__56A8#d5!+a0t@thz{+eTD5E3db~&6IR20#%TW1Dz6JC7kZC9zz<|L79DAMb# zF&LC*E7~BX-qW{QLqxp}IPeTx!>Ak7RcWdc#YF1Uvi|RGKV##o>5Gdt*M%*y>Xu&(O`LWyj|?ksNkU_yNz?Q^Sh^{2tqCdJnDz zc7)~(VZQidgj^H+SZ=nP3~bJn=QiI1`Di|Ng2MUvbxjQxtygd4q$bnk2cE*d#@DVmyFyuk=Xfzu#(}V4yF(Dt+2o zWj-sz9lLY;GTW6xl6H&mUF<4L6cT61$C9<0BMmvl7)+V_2s}05kkW0?PMctnA6t`v zv{b(L+&GWuEEG-6mrWcNJ}ij)WU?LZ8{-e=g3H4orCkX$@n}F2*C6ME`rb551~&j0ECumAu6 literal 0 HcmV?d00001 diff --git a/.ci/setup-secrets.sh b/.ci/setup-secrets.sh new file mode 100755 index 0000000000..de19cb0b26 --- /dev/null +++ b/.ci/setup-secrets.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +source $(dirname $0)/logger.inc + +log_info "Setting up secrets..." + +mkdir -p ${HOME}/.ssh +chmod 700 "${HOME}/.ssh" +gpg --symmetric --cipher-algo AES256 --batch --passphrase="$PMD_CI_SECRET_PASSPHRASE" \ + --decrypt --output ${HOME}/id_rsa .ci/id_rsa.gpg +chmod 600 "${HOME}/.ssh/id_rsa" + +mkdir -p "${HOME}/.gpg" +gpg --symmetric --cipher-algo AES256 --batch --passphrase="$PMD_CI_SECRET_PASSPHRASE" \ + --decrypt --output .ci/release-signing-key-D0BF1D737C9A1C22.gpg .ci/release-signing-key-D0BF1D737C9A1C22.gpg.gpg +gpg --batch --import .ci/release-signing-key-D0BF1D737C9A1C22.gpg +rm .ci/release-signing-key-D0BF1D737C9A1C22.gpg + +log_info "Setting up .ssh/known_hosts..." +# +# https://sourceforge.net/p/forge/documentation/SSH%20Key%20Fingerprints/ +# +# run locally: +# ssh-keyscan web.sourceforge.net | tee -a known_hosts +# +# verify fingerprints: +# ssh-keygen -F web.sourceforge.net -l -f known_hosts +# # Host web.sourceforge.net found: line 1 +# web.sourceforge.net RSA SHA256:xB2rnn0NUjZ/E0IXQp4gyPqc7U7gjcw7G26RhkDyk90 +# # Host web.sourceforge.net found: line 2 +# web.sourceforge.net ECDSA SHA256:QAAxYkf0iI/tc9oGa0xSsVOAzJBZstcO8HqGKfjpxcY +# # Host web.sourceforge.net found: line 3 +# web.sourceforge.net ED25519 SHA256:209BDmH3jsRyO9UeGPPgLWPSegKmYCBIya0nR/AWWCY +# +# then add output of `ssh-keygen -F web.sourceforge.net -f known_hosts` +# +echo 'web.sourceforge.net ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2uifHZbNexw6cXbyg1JnzDitL5VhYs0E65Hk/tLAPmcmm5GuiGeUoI/B0eUSNFsbqzwgwrttjnzKMKiGLN5CWVmlN1IXGGAfLYsQwK6wAu7kYFzkqP4jcwc5Jr9UPRpJdYIK733tSEmzab4qc5Oq8izKQKIaxXNe7FgmL15HjSpatFt9w/ot/CHS78FUAr3j3RwekHCm/jhPeqhlMAgC+jUgNJbFt3DlhDaRMa0NYamVzmX8D47rtmBbEDU3ld6AezWBPUR5Lh7ODOwlfVI58NAf/aYNlmvl2TZiauBCTa7OPYSyXJnIPbQXg6YQlDknNCr0K769EjeIlAfY87Z4tw==' >> "$HOME/.ssh/known_hosts" +echo 'web.sourceforge.net ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCwsY6sZT4MTTkHfpRzYjxG7mnXrGL74RCT2cO/NFvRrZVNB5XNwKNn7G5fHbYLdJ6UzpURDRae1eMg92JG0+yo=' >> "$HOME/.ssh/known_hosts" +echo 'web.sourceforge.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOQD35Ujalhh+JJkPvMckDlhu4dS7WH6NsOJ15iGCJLC' >> "$HOME/.ssh/known_hosts" + +# add pmd-code.org (ssh-keyscan pmd-code.org) +echo 'pmd-code.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVsIeF6xU0oPb/bMbxG1nU1NDyBpR/cBEPZcm/PuJwdI9B0ydPHA6FysqAnt32fNFznC2SWisnWyY3iNsP3pa8RQJVwmnnv9OboGFlW2/61o3iRyydcpPbgl+ADdt8iU9fmMI7dC04UqgHGBoqOwVNna9VylTjp5709cK2qHnwU450F6YcOEiOKeZfJvV4PmpJCz/JcsUVqft6StviR31jKnqbnkZdP8qNoTbds6WmGKyXkhHdLSZE7X1CFQH28tk8XFqditX93ezeCiThFL7EleDexV/3+2+cs5878sDMUMzHS5KShTjkxzhHaodhtIEdNesinq/hOPbxAGkQ0FbD' >> $HOME/.ssh/known_hosts diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 3222a0467a..0000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Java CI - -on: [push, pull_request] - -jobs: - build: - runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.experimental }} - if: "!contains(github.event.head_commit.message, '[skip ci]')" - strategy: - matrix: - os: [ ubuntu-latest , windows-latest , macos-latest ] - java: [ 11 ] - experimental: [ false ] - - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}-maven- - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v1 - with: - java-version: ${{ matrix.java }} - - name: Build with mvnw - run: | - ./mvnw -V clean install diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml new file mode 100644 index 0000000000..2aeb9d2fc2 --- /dev/null +++ b/.github/workflows/pull-requests.yml @@ -0,0 +1,30 @@ +name: Pull Requests + +on: pull_request + +jobs: + build: + runs-on: ${{ matrix.os }} + continue-on-error: false + if: "!contains(github.event.head_commit.message, '[skip ci]')" + strategy: + matrix: + os: [ ubuntu-latest, windows-latest, macos-latest ] + + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}- + - name: Install OpenJDK + run: .ci/install-openjdk.sh + shell: bash + - name: Build with mvnw + run: | + source ${HOME}/java.env + ./mvnw -V clean install diff --git a/.github/workflows/pushes.yml b/.github/workflows/pushes.yml new file mode 100644 index 0000000000..6218846648 --- /dev/null +++ b/.github/workflows/pushes.yml @@ -0,0 +1,85 @@ +name: Pushes +on: + push: + branches: + - main + - master + schedule: + # build it monthly: At 04:00 on day-of-month 1. + - cron: '0 4 1 * *' + +jobs: + linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}- + - name: Set up Ruby 2.7 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.7 + - name: Check Environment + run: .ci/check-environment.sh + shell: bash + - name: Setup Secrets + run: .ci/setup-secrets.sh + shell: bash + env: + PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} + - name: Install OpenJDK + run: .ci/install-openjdk.sh + shell: bash + - name: build + run: .ci/build.sh + shell: bash + env: + PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} + CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} + CI_SIGN_PASSPHRASE: ${{ secrets.CI_SIGN_PASSPHRASE }} + + windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}- + - name: Install OpenJDK + run: .ci/install-openjdk.sh + shell: bash + - name: Build with mvnw + run: | + source ${HOME}/java.env + ./mvnw -V clean install + + macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}- + - name: Install OpenJDK + run: .ci/install-openjdk.sh + shell: bash + - name: Build with mvnw + run: | + source ${HOME}/java.env + ./mvnw -V clean install diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml new file mode 100644 index 0000000000..620bc7f6c2 --- /dev/null +++ b/.github/workflows/releases.yml @@ -0,0 +1,17 @@ +name: Release Builds +on: + push: + tags: + - 'pmd_releases/*' + +jobs: + release: + runs-on: ubuntu-latest + continue-on-error: false + steps: + - uses: actions/checkout@v2 + - name: Run Release Script + run: .ci/release.sh + shell: bash + env: + PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} From 2e1615d7ab49a6be86b772b04761858dff694be3 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Tue, 10 Nov 2020 22:34:14 +0100 Subject: [PATCH 02/52] Fix wget download of openjdk --- .ci/install-openjdk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/install-openjdk.sh b/.ci/install-openjdk.sh index 924fb7e025..90c42288d9 100755 --- a/.ci/install-openjdk.sh +++ b/.ci/install-openjdk.sh @@ -41,7 +41,7 @@ mkdir -p ${TARGET_DIR} if [ ! -e ${CACHE_DIR}/${OPENJDK_ARCHIVE} ]; then log_info "Downloading from ${DOWNLOAD_URL} to ${CACHE_DIR}" - wget --directory-prefix=${CACHE_DIR} --timestamping --continue --output-document=${OPENJDK_ARCHIVE} ${DOWNLOAD_URL} + wget --continue --output-document=${CACHE_DIR}/${OPENJDK_ARCHIVE} ${DOWNLOAD_URL} else log_info "Skipped download, file ${CACHE_DIR}/${OPENJDK_ARCHIVE} already exists" fi From a089bbc14351f8b1e6cf675d475744eeba6980c6 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Tue, 10 Nov 2020 22:37:04 +0100 Subject: [PATCH 03/52] Use bash --- .github/workflows/pull-requests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 2aeb9d2fc2..64ab3c558b 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -28,3 +28,4 @@ jobs: run: | source ${HOME}/java.env ./mvnw -V clean install + shell: bash From 8970b7fcd8f93d95cc1315469939886631b150b8 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 11 Nov 2020 21:15:04 +0100 Subject: [PATCH 04/52] Fix install-openjdk.sh --- .ci/check-environment.sh | 9 +++++++++ .ci/install-openjdk.sh | 39 ++++++++++++++++++++++++--------------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/.ci/check-environment.sh b/.ci/check-environment.sh index 524e3247cf..8786f03ee4 100755 --- a/.ci/check-environment.sh +++ b/.ci/check-environment.sh @@ -9,4 +9,13 @@ source logger.inc set -e + +# every OS: +# curl +# jq + +# linux only ruby --version | grep "ruby 2.7" || (log_error "Ruby is missing"; exit 1) + +# windows only +# 7zip diff --git a/.ci/install-openjdk.sh b/.ci/install-openjdk.sh index 90c42288d9..f521062bfc 100755 --- a/.ci/install-openjdk.sh +++ b/.ci/install-openjdk.sh @@ -7,31 +7,34 @@ source $(dirname $0)/logger.inc +set -e case "$(uname)" in Linux*) JDK_OS=linux - JDK_EXT=tar.gz COMPONENTS_TO_STRIP=1 # e.g. openjdk-11.0.3+7/bin/java ;; Darwin*) JDK_OS=mac - JDK_EXT=tar.gz COMPONENTS_TO_STRIP=3 # e.g. jdk-11.0.3+7/Contents/Home/bin/java ;; CYGWIN*|MINGW*) JDK_OS=windows - JDK_EXT=zip ;; *) - + log_error "Unknown OS: $(uname)" + exit 1 ;; esac -JDK_VERSION=11 -DOWNLOAD_URL=https://api.adoptopenjdk.net/v3/binary/latest/${JDK_VERSION}/ga/${JDK_OS}/x64/jdk/hotspot/normal/adoptopenjdk?project=jdk -OPENJDK_ARCHIVE=openjdk-${JDK_VERSION}-${JDK_OS}.${JDK_EXT} +OPENJDK_VERSION=11 +DOWNLOAD_URL=$(curl --silent -X GET "https://api.adoptopenjdk.net/v3/assets/feature_releases/${OPENJDK_VERSION}/ga?architecture=x64&heap_size=normal&image_type=jdk&jvm_impl=hotspot&os=${JDK_OS}&page=0&page_size=1&project=jdk&sort_method=DEFAULT&sort_order=DESC&vendor=adoptopenjdk" \ + -H "accept: application/json" \ + | jq -r ".[0].binaries[0].package.link") + +OPENJDK_ARCHIVE=$(basename ${DOWNLOAD_URL}) +log_debug "Archive name: ${OPENJDK_ARCHIVE}" CACHE_DIR=${HOME}/.cache/openjdk TARGET_DIR=${HOME}/openjdk${OPENJDK_VERSION} @@ -41,19 +44,26 @@ mkdir -p ${TARGET_DIR} if [ ! -e ${CACHE_DIR}/${OPENJDK_ARCHIVE} ]; then log_info "Downloading from ${DOWNLOAD_URL} to ${CACHE_DIR}" - wget --continue --output-document=${CACHE_DIR}/${OPENJDK_ARCHIVE} ${DOWNLOAD_URL} + curl --location --output ${CACHE_DIR}/${OPENJDK_ARCHIVE} "${DOWNLOAD_URL}" else log_info "Skipped download, file ${CACHE_DIR}/${OPENJDK_ARCHIVE} already exists" fi log_info "Extracting to ${TARGET_DIR}" -if [ "${JDK_EXT}" = "zip" ]; then - 7z x ${CACHE_DIR}/${OPENJDK_ARCHIVE} -o${TARGET_DIR} - mv ${TARGET_DIR}/*/* ${TARGET_DIR}/ -else - tar --extract --file ${CACHE_DIR}/${OPENJDK_ARCHIVE} -C ${TARGET_DIR} --strip-components=${COMPONENTS_TO_STRIP} -fi +case "$OPENJDK_ARCHIVE" in + *.zip) + 7z x ${CACHE_DIR}/${OPENJDK_ARCHIVE} -o${TARGET_DIR} + mv ${TARGET_DIR}/*/* ${TARGET_DIR}/ + ;; + *.tar.gz) + tar --extract --file ${CACHE_DIR}/${OPENJDK_ARCHIVE} -C ${TARGET_DIR} --strip-components=${COMPONENTS_TO_STRIP} + ;; + *) + log_error "Unknown filetype: ${OPENJDK_ARCHIVE}" + exit 1 + ;; +esac cat > ${HOME}/java.env < Date: Wed, 11 Nov 2020 22:56:02 +0100 Subject: [PATCH 05/52] Integrate setup secrets into build.sh, add env.gpg, add sourceforge upload --- .ci/README.md | 4 + .ci/build.sh | 83 +++++++++++--- .ci/check-environment.sh | 22 +++- .ci/env.gpg | Bin 0 -> 406 bytes .ci/install-openjdk.sh | 18 ++- .ci/maven-settings.xml | 4 +- .ci/setup-secrets.inc | 56 ++++++++++ .ci/setup-secrets.sh | 42 ------- .ci/sourceforge-api.inc | 166 ++++++++++++++++++++++++++++ .github/workflows/pull-requests.yml | 8 +- .github/workflows/pushes.yml | 11 +- .github/workflows/releases.yml | 6 + 12 files changed, 345 insertions(+), 75 deletions(-) create mode 100644 .ci/env.gpg create mode 100644 .ci/setup-secrets.inc delete mode 100755 .ci/setup-secrets.sh create mode 100644 .ci/sourceforge-api.inc diff --git a/.ci/README.md b/.ci/README.md index bca56da5cf..3f8fc97477 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -12,5 +12,9 @@ for a ssh key, which is used to copy files to sourceforge. * `PMD_CI_SECRET_PASSPHRASE` * `CI_DEPLOY_PASSWORD` * `CI_SIGN_PASSPHRASE` +* ... +## Encrypting + + gpg --batch --symmetric --cipher-algo AES256 --passphrase="$PMD_CI_SECRET_PASSPHRASE" file.txt diff --git a/.ci/build.sh b/.ci/build.sh index 295344ded5..7ed79c10ec 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -1,26 +1,83 @@ #!/usr/bin/env bash source $(dirname $0)/logger.inc +source $(dirname $0)/setup-secrets.inc +source $(dirname $0)/sourceforge-api.inc source ${HOME}/java.env set -e +#set -x -# configure maven -# probably not needed? echo "MAVEN_OPTS='-Xms1g -Xmx1g'" > ${HOME}/.mavenrc -mkdir -p ${HOME}/.m2 -cp .ci/maven-settings.xml ${HOME}/.m2/settings.xml +function pmd_ci_build_main() { + pmd_ci_setup_env + pmd_ci_setup_gpg_key + pmd_ci_setup_ssh + + pmd_ci_build_setup_maven + pmd_ci_build_setup_oraclejdk7 + pmd_ci_build_run + + # Deploy to sourceforge files + VERSION=$(pmd_ci_build_get_pom_version) + sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip" + sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip" + + #build and upload doc + + pmd_ci_build_setup_regression-tester + #regression-tester_uploadBaseline -#MVN_BUILD_FLAGS="-B -V -Djava7.home=${HOME}/oraclejdk7" -MVN_BUILD_FLAGS="-B -V" + exit 0 +} -log_info "This is a snapshot build" -./mvnw deploy -Possrh,sign $MVN_BUILD_FLAGS +function pmd_ci_build_get_pom_version() { + echo $(./mvnw -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) +} -# Deploy to sourceforge files -#sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip" -#sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip" +function pmd_ci_build_setup_maven() { + # configure maven + echo "MAVEN_OPTS='-Xms1g -Xmx1g'" > ${HOME}/.mavenrc + mkdir -p ${HOME}/.m2 + cp .ci/maven-settings.xml ${HOME}/.m2/settings.xml +} -#regression-tester_uploadBaseline +function pmd_ci_build_setup_oraclejdk7() { + # install jdk7 for integration test -#build and upload doc + LOCAL_DIR="${HOME}/.cache/jdk7" + TARGET_DIR="${HOME}/oraclejdk7" + DOWNLOAD_URL="https://pmd-code.org/oraclejdk/jdk-7u80-linux-x64.tar.gz" + ARCHIVE=$(basename $DOWNLOAD_URL) + + mkdir -p ${LOCAL_DIR} + mkdir -p ${TARGET_DIR} + if [ ! -e ${LOCAL_DIR}/${ARCHIVE} ]; then + log_info "Downloading from ${DOWNLOAD_URL} to ${LOCAL_DIR}" + curl --location --output ${LOCAL_DIR}/${ARCHIVE} ${DOWNLOAD_URL} + else + log_info "Skipped download, file ${LOCAL_DIR}/${ARCHIVE} already exists" + fi + log_info "Extracting to ${TARGET_DIR}" + tar --extract --file ${LOCAL_DIR}/${ARCHIVE} -C ${TARGET_DIR} --strip-components=1 + + log_info "OracleJDK7 can be used via -Djava7.home=${HOME}/oraclejdk7" +} + +function pmd_ci_build_run() { + MVN_BUILD_FLAGS="-B -V -Djava7.home=${HOME}/oraclejdk7" + + log_info "This is a snapshot build" + ./mvnw deploy -Possrh,sign $MVN_BUILD_FLAGS +} + +function pmd_ci_build_setup_regression-tester() { + # install openjdk8 for pmd-regression-tests + .ci/install-openjdk.sh 8 + gem install bundler + bundle config set --local path vendor/bundle + bundle config set --local with release_notes_preprocessing + bundle install +} + +pmd_ci_build_main diff --git a/.ci/check-environment.sh b/.ci/check-environment.sh index 8786f03ee4..a93b36cda8 100755 --- a/.ci/check-environment.sh +++ b/.ci/check-environment.sh @@ -5,17 +5,37 @@ # and are in the correct version. # -source logger.inc +source $(dirname $0)/logger.inc set -e +function check() { + local CMD=$1 + local VERSION_CMD=$2 + + echo -n "Checking ${CMD}..." + + if [ hash "$CMD" 2>/dev/null ]; then + echo -e "${COL_GREEN}OK${COL_RESET}" + else + echo -e "${COL_RED}failure${COL_RESET}" + fi +} + # every OS: # curl # jq # linux only ruby --version | grep "ruby 2.7" || (log_error "Ruby is missing"; exit 1) +check "ruby" "ruby --version" +# gpg +# printenv +# rsync +# ssh + + # windows only # 7zip diff --git a/.ci/env.gpg b/.ci/env.gpg new file mode 100644 index 0000000000000000000000000000000000000000..07dd553e73881351482bd373f034f76deaa5ae9d GIT binary patch literal 406 zcmV;H0crk>4Fm}T0tI+nP!WOFH2>1T!~qzymt0!>gktO?WM#Q@d;(HxHf_*1C%OQ! zr;Oj!oJjy_gIV%u7XHd*ew+R_m`?yR`MLG>oF%i>B-~oMzL(BHW54kGh~T1wMANn? z?G<>kUDb>)TExoqJ`L(jdjq=6O8^XT+qA@VyeV?anBP0U2zjBeQtrWJ&A)u|b~T7_ z4dID=w8GZH)(W)c;vI$uCzZz`)nFqm{|Yg0zd2t;-i4$TMlB4l|4@#60qTH2oL+eB z?l*R$K#el$!|Qxfkby#}LP{_;}-UV#wSx{(Ma}=tGvI4Dg~HK8o52Lyh6#% zb$6QrFGhtec`k377R+SbV)qub^o&Brbw*5=OJ$H{(wSK+2Mze3bvK+w{KNtn>xI>_ z+l_!@72xtT0+Lq!)b>x4A*aA`H5eZyrSGGoAy7iFn)Zr|a#kD|(bnjf=ExpztO&-> AB>(^b literal 0 HcmV?d00001 diff --git a/.ci/install-openjdk.sh b/.ci/install-openjdk.sh index f521062bfc..bb54b798e4 100755 --- a/.ci/install-openjdk.sh +++ b/.ci/install-openjdk.sh @@ -28,7 +28,7 @@ case "$(uname)" in esac -OPENJDK_VERSION=11 +OPENJDK_VERSION=$1 DOWNLOAD_URL=$(curl --silent -X GET "https://api.adoptopenjdk.net/v3/assets/feature_releases/${OPENJDK_VERSION}/ga?architecture=x64&heap_size=normal&image_type=jdk&jvm_impl=hotspot&os=${JDK_OS}&page=0&page_size=1&project=jdk&sort_method=DEFAULT&sort_order=DESC&vendor=adoptopenjdk" \ -H "accept: application/json" \ | jq -r ".[0].binaries[0].package.link") @@ -65,12 +65,18 @@ case "$OPENJDK_ARCHIVE" in ;; esac -cat > ${HOME}/java.env < ${HOME}/java.env < ossrh - adangel + ${env.CI_DEPLY_USER} ${env.CI_DEPLOY_PASSWORD} @@ -22,7 +22,7 @@ ossrh - 0xD0BF1D737C9A1C22 + ${env.CI_SIGN_KEY} ${env.CI_SIGN_PASSPHRASE} diff --git a/.ci/setup-secrets.inc b/.ci/setup-secrets.inc new file mode 100644 index 0000000000..df1dbfe364 --- /dev/null +++ b/.ci/setup-secrets.inc @@ -0,0 +1,56 @@ +function pmd_ci_setup_env() { + log_info "Setting up secrets as environment variables..." + + printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \ + --passphrase-fd 0 \ + --output .ci/env .ci/env.gpg + + source .ci/env + rm .ci/env +} + +function pmd_ci_setup_gpg_key() { + log_info "Setting up GPG release signing key..." + + mkdir -p "${HOME}/.gpg" + printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \ + --passphrase-fd 0 \ + --output .ci/release-signing-key-D0BF1D737C9A1C22.gpg .ci/release-signing-key-D0BF1D737C9A1C22.gpg.gpg + gpg --batch --import .ci/release-signing-key-D0BF1D737C9A1C22.gpg + rm .ci/release-signing-key-D0BF1D737C9A1C22.gpg +} + +function pmd_ci_setup_ssh() { + log_info "Setting up .ssh/id_rsa..." + mkdir -p ${HOME}/.ssh + chmod 700 "${HOME}/.ssh" + printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \ + --passphrase-fd 0 \ + --output ${HOME}/.ssh/id_rsa .ci/id_rsa.gpg + chmod 600 "${HOME}/.ssh/id_rsa" + + log_info "Setting up .ssh/known_hosts..." + # + # https://sourceforge.net/p/forge/documentation/SSH%20Key%20Fingerprints/ + # + # run locally: + # ssh-keyscan web.sourceforge.net | tee -a known_hosts + # + # verify fingerprints: + # ssh-keygen -F web.sourceforge.net -l -f known_hosts + # # Host web.sourceforge.net found: line 1 + # web.sourceforge.net RSA SHA256:xB2rnn0NUjZ/E0IXQp4gyPqc7U7gjcw7G26RhkDyk90 + # # Host web.sourceforge.net found: line 2 + # web.sourceforge.net ECDSA SHA256:QAAxYkf0iI/tc9oGa0xSsVOAzJBZstcO8HqGKfjpxcY + # # Host web.sourceforge.net found: line 3 + # web.sourceforge.net ED25519 SHA256:209BDmH3jsRyO9UeGPPgLWPSegKmYCBIya0nR/AWWCY + # + # then add output of `ssh-keygen -F web.sourceforge.net -f known_hosts` + # + echo 'web.sourceforge.net ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2uifHZbNexw6cXbyg1JnzDitL5VhYs0E65Hk/tLAPmcmm5GuiGeUoI/B0eUSNFsbqzwgwrttjnzKMKiGLN5CWVmlN1IXGGAfLYsQwK6wAu7kYFzkqP4jcwc5Jr9UPRpJdYIK733tSEmzab4qc5Oq8izKQKIaxXNe7FgmL15HjSpatFt9w/ot/CHS78FUAr3j3RwekHCm/jhPeqhlMAgC+jUgNJbFt3DlhDaRMa0NYamVzmX8D47rtmBbEDU3ld6AezWBPUR5Lh7ODOwlfVI58NAf/aYNlmvl2TZiauBCTa7OPYSyXJnIPbQXg6YQlDknNCr0K769EjeIlAfY87Z4tw==' >> "$HOME/.ssh/known_hosts" + echo 'web.sourceforge.net ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCwsY6sZT4MTTkHfpRzYjxG7mnXrGL74RCT2cO/NFvRrZVNB5XNwKNn7G5fHbYLdJ6UzpURDRae1eMg92JG0+yo=' >> "$HOME/.ssh/known_hosts" + echo 'web.sourceforge.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOQD35Ujalhh+JJkPvMckDlhu4dS7WH6NsOJ15iGCJLC' >> "$HOME/.ssh/known_hosts" + + # add pmd-code.org (ssh-keyscan pmd-code.org) + echo 'pmd-code.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVsIeF6xU0oPb/bMbxG1nU1NDyBpR/cBEPZcm/PuJwdI9B0ydPHA6FysqAnt32fNFznC2SWisnWyY3iNsP3pa8RQJVwmnnv9OboGFlW2/61o3iRyydcpPbgl+ADdt8iU9fmMI7dC04UqgHGBoqOwVNna9VylTjp5709cK2qHnwU450F6YcOEiOKeZfJvV4PmpJCz/JcsUVqft6StviR31jKnqbnkZdP8qNoTbds6WmGKyXkhHdLSZE7X1CFQH28tk8XFqditX93ezeCiThFL7EleDexV/3+2+cs5878sDMUMzHS5KShTjkxzhHaodhtIEdNesinq/hOPbxAGkQ0FbD' >> $HOME/.ssh/known_hosts +} diff --git a/.ci/setup-secrets.sh b/.ci/setup-secrets.sh deleted file mode 100755 index de19cb0b26..0000000000 --- a/.ci/setup-secrets.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -source $(dirname $0)/logger.inc - -log_info "Setting up secrets..." - -mkdir -p ${HOME}/.ssh -chmod 700 "${HOME}/.ssh" -gpg --symmetric --cipher-algo AES256 --batch --passphrase="$PMD_CI_SECRET_PASSPHRASE" \ - --decrypt --output ${HOME}/id_rsa .ci/id_rsa.gpg -chmod 600 "${HOME}/.ssh/id_rsa" - -mkdir -p "${HOME}/.gpg" -gpg --symmetric --cipher-algo AES256 --batch --passphrase="$PMD_CI_SECRET_PASSPHRASE" \ - --decrypt --output .ci/release-signing-key-D0BF1D737C9A1C22.gpg .ci/release-signing-key-D0BF1D737C9A1C22.gpg.gpg -gpg --batch --import .ci/release-signing-key-D0BF1D737C9A1C22.gpg -rm .ci/release-signing-key-D0BF1D737C9A1C22.gpg - -log_info "Setting up .ssh/known_hosts..." -# -# https://sourceforge.net/p/forge/documentation/SSH%20Key%20Fingerprints/ -# -# run locally: -# ssh-keyscan web.sourceforge.net | tee -a known_hosts -# -# verify fingerprints: -# ssh-keygen -F web.sourceforge.net -l -f known_hosts -# # Host web.sourceforge.net found: line 1 -# web.sourceforge.net RSA SHA256:xB2rnn0NUjZ/E0IXQp4gyPqc7U7gjcw7G26RhkDyk90 -# # Host web.sourceforge.net found: line 2 -# web.sourceforge.net ECDSA SHA256:QAAxYkf0iI/tc9oGa0xSsVOAzJBZstcO8HqGKfjpxcY -# # Host web.sourceforge.net found: line 3 -# web.sourceforge.net ED25519 SHA256:209BDmH3jsRyO9UeGPPgLWPSegKmYCBIya0nR/AWWCY -# -# then add output of `ssh-keygen -F web.sourceforge.net -f known_hosts` -# -echo 'web.sourceforge.net ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2uifHZbNexw6cXbyg1JnzDitL5VhYs0E65Hk/tLAPmcmm5GuiGeUoI/B0eUSNFsbqzwgwrttjnzKMKiGLN5CWVmlN1IXGGAfLYsQwK6wAu7kYFzkqP4jcwc5Jr9UPRpJdYIK733tSEmzab4qc5Oq8izKQKIaxXNe7FgmL15HjSpatFt9w/ot/CHS78FUAr3j3RwekHCm/jhPeqhlMAgC+jUgNJbFt3DlhDaRMa0NYamVzmX8D47rtmBbEDU3ld6AezWBPUR5Lh7ODOwlfVI58NAf/aYNlmvl2TZiauBCTa7OPYSyXJnIPbQXg6YQlDknNCr0K769EjeIlAfY87Z4tw==' >> "$HOME/.ssh/known_hosts" -echo 'web.sourceforge.net ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCwsY6sZT4MTTkHfpRzYjxG7mnXrGL74RCT2cO/NFvRrZVNB5XNwKNn7G5fHbYLdJ6UzpURDRae1eMg92JG0+yo=' >> "$HOME/.ssh/known_hosts" -echo 'web.sourceforge.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOQD35Ujalhh+JJkPvMckDlhu4dS7WH6NsOJ15iGCJLC' >> "$HOME/.ssh/known_hosts" - -# add pmd-code.org (ssh-keyscan pmd-code.org) -echo 'pmd-code.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVsIeF6xU0oPb/bMbxG1nU1NDyBpR/cBEPZcm/PuJwdI9B0ydPHA6FysqAnt32fNFznC2SWisnWyY3iNsP3pa8RQJVwmnnv9OboGFlW2/61o3iRyydcpPbgl+ADdt8iU9fmMI7dC04UqgHGBoqOwVNna9VylTjp5709cK2qHnwU450F6YcOEiOKeZfJvV4PmpJCz/JcsUVqft6StviR31jKnqbnkZdP8qNoTbds6WmGKyXkhHdLSZE7X1CFQH28tk8XFqditX93ezeCiThFL7EleDexV/3+2+cs5878sDMUMzHS5KShTjkxzhHaodhtIEdNesinq/hOPbxAGkQ0FbD' >> $HOME/.ssh/known_hosts diff --git a/.ci/sourceforge-api.inc b/.ci/sourceforge-api.inc new file mode 100644 index 0000000000..860769d612 --- /dev/null +++ b/.ci/sourceforge-api.inc @@ -0,0 +1,166 @@ +# +# The functions here require the following scripts: +# logger.inc +# +# The functions here require the following environment variables: +# PMD_SF_USER +# PMD_SF_APIKEY +# + +# +# Uploads the release notes to sourceforge files as "ReadMe.md". +# +# Note: this function always succeeds, even if the upload fails. +# In that case, just a error logging is provided. +# +function sourceforge_uploadReleaseNotes() { + local pmdVersion="$1" + local releaseNotes="$2" + + log_debug "$FUNCNAME pmdVersion=$pmdVersion" + local targetUrl="https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}" + + local errexitstate="$(shopt -po errexit)" + set +e # disable errexit + ( + # This handler is called if any command fails + function release_notes_fail() { + log_error "Error while uploading release notes as ReadMe.md to sourceforge!" + log_error "Please upload manually: ${targetUrl}" + cleanup_temp_dir + } + + function cleanup_temp_dir() { + log_debug "Cleanup tempdir $releaseNotesTempDir" + rm "${releaseNotesTempDir}/${pmdVersion}/ReadMe.md" || true + rmdir "${releaseNotesTempDir}/${pmdVersion}" || true + rmdir "${releaseNotesTempDir}" || true + } + + # exit subshell after trap + set -e + trap release_notes_fail ERR + + local releaseNotesTempDir=$(mktemp -d) + log_debug "Tempdir: $releaseNotesTempDir" + mkdir -p "${releaseNotesTempDir}/${pmdVersion}" + echo "$releaseNotes" > "${releaseNotesTempDir}/${pmdVersion}/ReadMe.md" + + log_info "Uploading release notes to sourceforge for version $pmdVersion" + rsync -avz \ + "${releaseNotesTempDir}/" \ + "${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/" + + log_success "Successfully uploaded release notes as ReadMe.md to sourceforge: ${targetUrl}" + + cleanup_temp_dir + ) + # restore errexit state + eval "$errexitstate" +} + +# +# Uploads the given file to sourceforge. +# +# Note: This function always succeeds, even if the upload fails. +# In that case, just a error logging is provided. +# +function sourceforge_uploadFile() { + local pmdVersion="$1" + local filename="$2" + + log_debug "$FUNCNAME pmdVersion=$pmdVersion filename=$filename" + local targetUrl="https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}" + + local errexitstate="$(shopt -po errexit)" + set +e # disable errexit + ( + # This handler is called if any command fails + function upload_failed() { + log_error "Error while uploading ${filename} to sourceforge!" + log_error "Please upload manually: ${targetUrl}" + } + + # exit subshell after trap + set -e + trap upload_failed ERR + + log_info "Uploading $filename to sourceforge..." + .travis/travis_wait "rsync -avh ${filename} ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/${pmdVersion}/" + log_success "Successfully uploaded ${filename} to sourceforge: ${targetUrl}" + ) + # restore errexit state + eval "$errexitstate" +} + +# +# Select the given version as the new default download. +# +# Note: This function always succeeds, even if the request fails. +# In that case, just a error logging is provided. +# +function sourceforge_selectDefault() { + local pmdVersion="$1" + + log_debug "$FUNCNAME pmdVersion=$pmdVersion" + local targetUrl="https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}" + + local errexitstate="$(shopt -po errexit)" + set +e # disable errexit + ( + # This handler is called if any command fails + function request_failed() { + log_error "Error while selecting ${pmdVersion} as new default download on sourceforge!" + log_error "Please do it manually: ${targetUrl}" + } + + # exit subshell after trap + set -e + trap request_failed ERR + + log_info "Selecting $pmdVersion as new default on sourceforge..." + local response + response=$(curl --fail -s -H "Accept: application/json" \ + -X PUT \ + -d "api_key=${PMD_SF_APIKEY}" \ + -d "default=windows&default=mac&default=linux&default=bsd&default=solaris&default=others" \ + "https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}/pmd-bin-${pmdVersion}.zip") + log_debug " -> response: $response" + log_success "Successfully selected $pmdVersion as new default on sourceforge: ${targetUrl}" + ) + # restore errexit state + eval "$errexitstate" +} + +# +# Rsyncs the complete documentation to sourceforge. +# +# Note: This function always succeeds, even if the upload fails. +# In that case, just a error logging is provided. +# +function sourceforge_rsyncSnapshotDocumentation() { + local pmdVersion="$1" + local targetPath="$2" + + log_debug "$FUNCNAME pmdVersion=$pmdVersion targetPath=$targetPath" + local targetUrl="https://pmd.sourceforge.io/${targetPath}/" + + local errexitstate="$(shopt -po errexit)" + set +e # disable errexit + ( + # This handler is called if any command fails + function upload_failed() { + log_error "Couldn't upload the documentation. It won't be current on ${targetUrl}" + } + + # exit subshell after trap + set -e + trap upload_failed ERR + + log_info "Uploading documentation to ${targetUrl}..." + .travis/travis_wait "rsync -ah --stats --delete docs/pmd-doc-${VERSION}/ ${PMD_SF_USER}@web.sourceforge.net:/home/project-web/pmd/htdocs/snapshot/" + log_success "Successfully uploaded documentation: ${targetUrl}" + ) + # restore errexit state + eval "$errexitstate" +} diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 64ab3c558b..8f75346792 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -21,11 +21,15 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}- - - name: Install OpenJDK - run: .ci/install-openjdk.sh + - name: Install OpenJDK 11 + run: .ci/install-openjdk.sh 11 shell: bash - name: Build with mvnw run: | source ${HOME}/java.env ./mvnw -V clean install shell: bash + - name: Execute Danger + if: ${{ runner.os == 'Linux' }} + run: echo "TODO" + shell: bash diff --git a/.github/workflows/pushes.yml b/.github/workflows/pushes.yml index 6218846648..e145ab8296 100644 --- a/.github/workflows/pushes.yml +++ b/.github/workflows/pushes.yml @@ -28,21 +28,14 @@ jobs: - name: Check Environment run: .ci/check-environment.sh shell: bash - - name: Setup Secrets - run: .ci/setup-secrets.sh - shell: bash - env: - PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} - - name: Install OpenJDK - run: .ci/install-openjdk.sh + - name: Install OpenJDK 11 + run: .ci/install-openjdk.sh 11 shell: bash - name: build run: .ci/build.sh shell: bash env: PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} - CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} - CI_SIGN_PASSPHRASE: ${{ secrets.CI_SIGN_PASSPHRASE }} windows: runs-on: windows-latest diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 620bc7f6c2..cc031398c6 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -10,6 +10,12 @@ jobs: continue-on-error: false steps: - uses: actions/checkout@v2 + - name: Check Environment + run: .ci/check-environment.sh + shell: bash + - name: Install OpenJDK 11 + run: .ci/install-openjdk.sh 11 + shell: bash - name: Run Release Script run: .ci/release.sh shell: bash From b42a3588445ee8b434a386fc8da55ca7cd548267 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 09:44:17 +0100 Subject: [PATCH 06/52] Add workaround for connection timeouts --- .ci/README.md | 11 +++++++++++ .ci/build.sh | 3 +-- .github/workflows/pull-requests.yml | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index 3f8fc97477..505b59bff9 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -18,3 +18,14 @@ for a ssh key, which is used to copy files to sourceforge. gpg --batch --symmetric --cipher-algo AES256 --passphrase="$PMD_CI_SECRET_PASSPHRASE" file.txt +## Known Issues + +Intermittent build failures while downloading dependencies from maven central. +Root issue seems to be SNAT configs in Azure, which closes long running TCP connections +only on one side: https://docs.microsoft.com/en-us/azure/load-balancer/troubleshoot-outbound-connection#idletimeout +The default timeout is 4 minutes. + +Workaround as described in https://github.com/actions/virtual-environments/issues/1499 and +https://issues.apache.org/jira/browse/WAGON-545 is applied: + +`-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3` diff --git a/.ci/build.sh b/.ci/build.sh index 7ed79c10ec..712da3223c 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -37,7 +37,6 @@ function pmd_ci_build_get_pom_version() { function pmd_ci_build_setup_maven() { # configure maven - echo "MAVEN_OPTS='-Xms1g -Xmx1g'" > ${HOME}/.mavenrc mkdir -p ${HOME}/.m2 cp .ci/maven-settings.xml ${HOME}/.m2/settings.xml } @@ -65,7 +64,7 @@ function pmd_ci_build_setup_oraclejdk7() { } function pmd_ci_build_run() { - MVN_BUILD_FLAGS="-B -V -Djava7.home=${HOME}/oraclejdk7" + MVN_BUILD_FLAGS="-e -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -V -Djava7.home=${HOME}/oraclejdk7" log_info "This is a snapshot build" ./mvnw deploy -Possrh,sign $MVN_BUILD_FLAGS diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 8f75346792..023f1ac239 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -27,7 +27,7 @@ jobs: - name: Build with mvnw run: | source ${HOME}/java.env - ./mvnw -V clean install + ./mvnw -e -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -V clean install shell: bash - name: Execute Danger if: ${{ runner.os == 'Linux' }} From e0a4fafcf4a5373376c4ef4faa1e0df17cdab6c2 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 09:47:55 +0100 Subject: [PATCH 07/52] Make sure to not log secrets while loading --- .ci/setup-secrets.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/setup-secrets.inc b/.ci/setup-secrets.inc index df1dbfe364..fc49d40962 100644 --- a/.ci/setup-secrets.inc +++ b/.ci/setup-secrets.inc @@ -5,7 +5,7 @@ function pmd_ci_setup_env() { --passphrase-fd 0 \ --output .ci/env .ci/env.gpg - source .ci/env + source .ci/env >/dev/null 2>&1 rm .ci/env } From 328d2ad8280cf021969a6dea334514d9df908a9a Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 10:17:44 +0100 Subject: [PATCH 08/52] Move includes into inc/ and files into files/, add pmd_doc support --- .ci/build.sh | 45 ++++++-- .ci/check-environment.sh | 2 +- .ci/{ => files}/env.gpg | Bin .ci/{ => files}/id_rsa.gpg | Bin .ci/{ => files}/id_rsa.pub | 0 .ci/{ => files}/maven-settings.xml | 0 ...lease-signing-key-D0BF1D737C9A1C22.gpg.gpg | Bin .ci/{ => inc}/logger.inc | 0 .ci/inc/pmd-code-api.inc | 94 ++++++++++++++++ .ci/inc/pmd-doc.inc | 105 ++++++++++++++++++ .ci/{ => inc}/setup-secrets.inc | 26 +++-- .ci/{ => inc}/sourceforge-api.inc | 0 .ci/install-openjdk.sh | 2 +- 13 files changed, 253 insertions(+), 21 deletions(-) rename .ci/{ => files}/env.gpg (100%) rename .ci/{ => files}/id_rsa.gpg (100%) rename .ci/{ => files}/id_rsa.pub (100%) rename .ci/{ => files}/maven-settings.xml (100%) rename .ci/{ => files}/release-signing-key-D0BF1D737C9A1C22.gpg.gpg (100%) rename .ci/{ => inc}/logger.inc (100%) create mode 100644 .ci/inc/pmd-code-api.inc create mode 100644 .ci/inc/pmd-doc.inc rename .ci/{ => inc}/setup-secrets.inc (84%) rename .ci/{ => inc}/sourceforge-api.inc (100%) diff --git a/.ci/build.sh b/.ci/build.sh index 712da3223c..7e285ee4d1 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash -source $(dirname $0)/logger.inc -source $(dirname $0)/setup-secrets.inc -source $(dirname $0)/sourceforge-api.inc +source $(dirname $0)/inc/logger.inc +source $(dirname $0)/inc/setup-secrets.inc +source $(dirname $0)/inc/sourceforge-api.inc +source $(dirname $0)/inc/pmd-doc.inc +source $(dirname $0)/inc/pmd-code-api.inc source ${HOME}/java.env set -e @@ -15,22 +17,44 @@ function pmd_ci_build_main() { pmd_ci_build_setup_maven pmd_ci_build_setup_oraclejdk7 + pmd_ci_build_setup_bundler + + VERSION=$(pmd_ci_build_get_pom_version) + log_info "Building PMD ${VERSION}..." + pmd_ci_build_run # Deploy to sourceforge files - VERSION=$(pmd_ci_build_get_pom_version) sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip" sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip" - #build and upload doc + pmd_ci_build_and_upload_doc - pmd_ci_build_setup_regression-tester + #pmd_ci_build_setup_regression-tester #regression-tester_uploadBaseline - exit 0 } +function pmd_ci_build_and_upload_doc() { + pmd_doc_generate_jekyll_site + pmd_doc_create_archive + + sourceforge_uploadFile "${VERSION}" "docs/pmd-doc-${VERSION}.zip" + + # Deploy doc to https://docs.pmd-code.org/pmd-doc-${VERSION}/ + pmd_code_uploadDocumentation "${VERSION}" "docs/pmd-doc-${VERSION}.zip" + # Deploy javadoc to https://docs.pmd-code.org/apidocs/*/${VERSION}/ + pmd_code_uploadJavadoc "${VERSION}" "$(pwd)" + + pmd_code_createSymlink "${VERSION}" "snapshot" + + # update github pages https://pmd.github.io/pmd/ + pmd_doc_publish_to_github_pages + # rsync site to https://pmd.sourceforge.io/snapshot + sourceforge_rsyncSnapshotDocumentation "${VERSION}" "snapshot" +} + function pmd_ci_build_get_pom_version() { echo $(./mvnw -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) } @@ -67,16 +91,19 @@ function pmd_ci_build_run() { MVN_BUILD_FLAGS="-e -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -V -Djava7.home=${HOME}/oraclejdk7" log_info "This is a snapshot build" - ./mvnw deploy -Possrh,sign $MVN_BUILD_FLAGS + ./mvnw deploy -Possrh,sign,generate-rule-docs $MVN_BUILD_FLAGS } function pmd_ci_build_setup_regression-tester() { # install openjdk8 for pmd-regression-tests .ci/install-openjdk.sh 8 - gem install bundler bundle config set --local path vendor/bundle bundle config set --local with release_notes_preprocessing bundle install } +function pmd_ci_build_setup_bundler() { + gem install bundler +} + pmd_ci_build_main diff --git a/.ci/check-environment.sh b/.ci/check-environment.sh index a93b36cda8..12a835cf26 100755 --- a/.ci/check-environment.sh +++ b/.ci/check-environment.sh @@ -5,7 +5,7 @@ # and are in the correct version. # -source $(dirname $0)/logger.inc +source $(dirname $0)/inc/logger.inc set -e diff --git a/.ci/env.gpg b/.ci/files/env.gpg similarity index 100% rename from .ci/env.gpg rename to .ci/files/env.gpg diff --git a/.ci/id_rsa.gpg b/.ci/files/id_rsa.gpg similarity index 100% rename from .ci/id_rsa.gpg rename to .ci/files/id_rsa.gpg diff --git a/.ci/id_rsa.pub b/.ci/files/id_rsa.pub similarity index 100% rename from .ci/id_rsa.pub rename to .ci/files/id_rsa.pub diff --git a/.ci/maven-settings.xml b/.ci/files/maven-settings.xml similarity index 100% rename from .ci/maven-settings.xml rename to .ci/files/maven-settings.xml diff --git a/.ci/release-signing-key-D0BF1D737C9A1C22.gpg.gpg b/.ci/files/release-signing-key-D0BF1D737C9A1C22.gpg.gpg similarity index 100% rename from .ci/release-signing-key-D0BF1D737C9A1C22.gpg.gpg rename to .ci/files/release-signing-key-D0BF1D737C9A1C22.gpg.gpg diff --git a/.ci/logger.inc b/.ci/inc/logger.inc similarity index 100% rename from .ci/logger.inc rename to .ci/inc/logger.inc diff --git a/.ci/inc/pmd-code-api.inc b/.ci/inc/pmd-code-api.inc new file mode 100644 index 0000000000..d9e75beb7c --- /dev/null +++ b/.ci/inc/pmd-code-api.inc @@ -0,0 +1,94 @@ +# +# The functions here require the following scripts: +# inc/logger.inc +# + +PMD_CODE_SSH_USER=pmd +PMD_CODE_DOCS_PATH=/docs.pmd-code.org/ + +function pmd_code_uploadDocumentation() { + local pmdVersion="$1" + local filename="$2" + local basefilename="$(basename $filename)" + + log_debug "$FUNCNAME pmdVersion=$pmdVersion filename=$filename" + + scp "${filename}" ${PMD_CODE_SSH_USER}@pmd-code.org:${PMD_CODE_DOCS_PATH} + ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ + unzip -qo ${basefilename} && \ + rm ${basefilename}" + log_info "Docs updated: https://docs.pmd-code.org/pmd-doc-${pmdVersion}/" +} + +function pmd_code_removeDocumentation() { + local pmdVersion="$1" + + log_debug "$FUNCNAME pmdVersion=$pmdVersion" + + ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ + rm -rf pmd-doc-${pmdVersion}/" + log_info "Removed docs: https://docs.pmd-code.org/pmd-doc-${pmdVersion}/" +} + +function pmd_code_createSymlink() { + local pmdVersion="$1" + local name="$2" + + log_debug "$FUNCNAME pmdVersion=$pmdVersion name=$name" + + ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ + rm -f $name && \ + ln -s pmd-doc-${pmdVersion} $name" + log_info "Symlink created: https://docs.pmd-code.org/$name/ -> https://docs.pmd-code.org/pmd-doc-${pmdVersion}/" +} + +function pmd_code_uploadJavadoc() { + local pmdVersion="$1" + local basePath="$2" + + log_debug "$FUNCNAME pmdVersion=$pmdVersion basePath=$basePath" + + for i in ${basePath}/*/target/*-javadoc.jar */*/target/*-javadoc.jar; do + pmd_code_uploadJavadocModule "$pmdVersion" "$i" + done + + pmd_code_fixPmdLangTestStyle "${basePath}" + + # make sure https://docs.pmd-code.org/apidocs/ shows directory index + ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH}/apidocs && \ + echo 'Options +Indexes' > .htaccess" + log_info "Directory index enabled for https://docs.pmd-code.org/apidocs/" +} + +function pmd_code_uploadJavadocModule() { + local pmdVersion="$1" + local moduleJavadocJar="$2" + local moduleJavadocJarBasename="$(basename $moduleJavadocJar)" + local module=${moduleJavadocJarBasename%%-${pmdVersion}-javadoc.jar} + + log_debug "$FUNCNAME pmdVersion=$pmdVersion moduleJavadocJar=$moduleJavadocJar module=$module" + + scp "$moduleJavadocJar" ${PMD_CODE_SSH_USER}@pmd-code.org:${PMD_CODE_DOCS_PATH} + ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ + mkdir -p apidocs/${module}/${pmdVersion} && \ + unzip -qo -d apidocs/${module}/${pmdVersion} ${moduleJavadocJarBasename} && \ + rm ${moduleJavadocJarBasename}" + log_info "JavaDoc for $module uploaded: https://docs.pmd-code.org/apidocs/${module}/${pmdVersion}/" +} + +function pmd_code_fixPmdLangTestStyle { + local basePath="$1" + + log_debug "$FUNCNAME basePath=$basePath" + scp "${basePath}/pmd-lang-test/target/dokka/style.css" ${PMD_CODE_SSH_USER}@pmd-code.org:${PMD_CODE_DOCS_PATH}/apidocs/pmd-lang-test/ + log_info "Fixed style for https://docs.pmd-code.org/apidocs/pmd-lang-test/*/" +} + +function pmd_code_removeJavadoc() { + local pmdVersion="$1" + + log_debug "$FUNCNAME pmdVersion=$pmdVersion" + ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ + rm -rf apidocs/*/${pmdVersion}" + log_info "Removed Javadoc: https://docs.pmd-code.org/apidocs/*/${pmdVersion}/ is gone" +} diff --git a/.ci/inc/pmd-doc.inc b/.ci/inc/pmd-doc.inc new file mode 100644 index 0000000000..2e8f6ad649 --- /dev/null +++ b/.ci/inc/pmd-doc.inc @@ -0,0 +1,105 @@ +# +# Executes jekyll and generates the documentation +# The documentation will be generated in the directory "docs/_site". +# +function pmd_doc_generate_jekyll_site() { + pushd docs + + echo -e "\n\n" + log_info "Building documentation using jekyll..." + bundle config set --local path vendor/bundle + bundle install + bundle exec jekyll build + + popd +} + +# +# Creates the pmd-doc.zip archive. It will be placed in "docs/". +# +function pmd_doc_create_archive() { + pushd docs + + echo -e "\n\n" + log_info "Creating pmd-doc archive..." + mv _site pmd-doc-${VERSION} + zip -qr pmd-doc-${VERSION}.zip pmd-doc-${VERSION}/ + log_success "Successfully created pmd-doc-${VERSION}.zip" + + popd +} + +# +# Publishes the site to https://pmd.github.io/pmd-${VERSION} and +# https://pmd.github.io/latest/ +# +function publish_release_documentation_github() { + echo -e "\n\n" + log_info "Adding the new doc to pmd.github.io..." + # clone pmd.github.io. Note: This uses the ssh key setup earlier + # In order to speed things up, we use a sparse checkout - no need to checkout all directories here + mkdir pmd.github.io + ( + cd pmd.github.io + git init + git config user.name "Travis CI (pmd-bot)" + git config user.email "andreas.dangel+pmd-bot@adangel.org" + git config core.sparsecheckout true + git remote add origin git@github.com:pmd/pmd.github.io.git + echo "/latest/" > .git/info/sparse-checkout + echo "/sitemap.xml" >> .git/info/sparse-checkout + git pull --depth=1 origin master + log_info "Copying documentation from ../docs/pmd-doc-${VERSION}/ to pmd-${VERSION}/ ..." + rsync -ah --stats ../docs/pmd-doc-${VERSION}/ pmd-${VERSION}/ + git status + echo "Executing: git add pmd-${VERSION}" + git add pmd-${VERSION} + echo "Executing: git commit..." + git commit -q -m "Added pmd-${VERSION}" + + log_info "Copying pmd-${VERSION} to latest ..." + git rm -qr latest + cp -a pmd-${VERSION} latest + echo "Executing: git add latest" + git add latest + echo "Executing: git commit..." + git commit -q -m "Copying pmd-${VERSION} to latest" + + log_info "Generating sitemap.xml" + ../.travis/sitemap_generator.sh > sitemap.xml + echo "Executing: git add sitemap.xml" + git add sitemap.xml + echo "Executing: git commit..." + git commit -q -m "Generated sitemap.xml" + + echo "Executing: git push origin master" + git push origin master + ) +} + +# +# Updates github pages of the main repository, +# so that https://pmd.github.io/pmd/ has the latest (snapshot) content +# +function pmd_doc_publish_to_github_pages() { + echo -e "\n\n" + log_info "Pushing the new site to github pages..." + git clone --branch gh-pages --depth 1 git@github.com:pmd/pmd.git pmd-gh-pages + # clear the files first + rm -rf pmd-gh-pages/* + # copy the new site + cp -a docs/pmd-doc-${VERSION}/* pmd-gh-pages/ + ( + cd pmd-gh-pages + git config user.name "Travis CI (pmd-bot)" + git config user.email "andreas.dangel+pmd-bot@adangel.org" + git add -A + MSG="Update documentation + +TRAVIS_JOB_NUMBER=${TRAVIS_JOB_NUMBER} +TRAVIS_COMMIT_RANGE=${TRAVIS_COMMIT_RANGE}" + git commit -q -m "$MSG" + git push git@github.com:pmd/pmd.git HEAD:gh-pages + log_success "Successfully pushed site to https://pmd.github.io/pmd/" + ) +} diff --git a/.ci/setup-secrets.inc b/.ci/inc/setup-secrets.inc similarity index 84% rename from .ci/setup-secrets.inc rename to .ci/inc/setup-secrets.inc index fc49d40962..849e0f9acd 100644 --- a/.ci/setup-secrets.inc +++ b/.ci/inc/setup-secrets.inc @@ -1,33 +1,39 @@ function pmd_ci_setup_env() { log_info "Setting up secrets as environment variables..." + local -r ENV_FILE=.ci/files/env printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \ --passphrase-fd 0 \ - --output .ci/env .ci/env.gpg + --output ${ENV_FILE} ${ENV_FILE}.gpg - source .ci/env >/dev/null 2>&1 - rm .ci/env + source ${ENV_FILE} >/dev/null 2>&1 + rm ${ENV_FILE} } function pmd_ci_setup_gpg_key() { log_info "Setting up GPG release signing key..." + local -r GPG_FILE=.ci/files/release-signing-key-D0BF1D737C9A1C22.gpg mkdir -p "${HOME}/.gpg" printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \ --passphrase-fd 0 \ - --output .ci/release-signing-key-D0BF1D737C9A1C22.gpg .ci/release-signing-key-D0BF1D737C9A1C22.gpg.gpg - gpg --batch --import .ci/release-signing-key-D0BF1D737C9A1C22.gpg - rm .ci/release-signing-key-D0BF1D737C9A1C22.gpg + --output ${GPG_FILE} ${GPG_FILE}.gpg + gpg --batch --import ${GPG_FILE} + rm ${GPG_FILE} } function pmd_ci_setup_ssh() { log_info "Setting up .ssh/id_rsa..." - mkdir -p ${HOME}/.ssh - chmod 700 "${HOME}/.ssh" + local -r SSH_KEY_FILE=.ci/files/id_rsa + printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \ --passphrase-fd 0 \ - --output ${HOME}/.ssh/id_rsa .ci/id_rsa.gpg - chmod 600 "${HOME}/.ssh/id_rsa" + --output ${SSH_KEY_FILE} ${SSH_KEY_FILE}.gpg + chmod 600 ${SSH_KEY_FILE} + + mkdir -p ${HOME}/.ssh + chmod 700 "${HOME}/.ssh" + mv ${SSH_KEY_FILE} "${HOME}/.ssh/id_rsa" log_info "Setting up .ssh/known_hosts..." # diff --git a/.ci/sourceforge-api.inc b/.ci/inc/sourceforge-api.inc similarity index 100% rename from .ci/sourceforge-api.inc rename to .ci/inc/sourceforge-api.inc diff --git a/.ci/install-openjdk.sh b/.ci/install-openjdk.sh index bb54b798e4..e1359675a2 100755 --- a/.ci/install-openjdk.sh +++ b/.ci/install-openjdk.sh @@ -5,7 +5,7 @@ # The API is documented at https://api.adoptopenjdk.net/swagger-ui/ # -source $(dirname $0)/logger.inc +source $(dirname $0)/inc/logger.inc set -e From 0f14066adc274f3571ff0ddf4533da5e63108b75 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 10:42:19 +0100 Subject: [PATCH 09/52] Use github job infos --- .ci/check-environment.sh | 1 + .ci/inc/pmd-doc.inc | 14 ++++++++++---- .github/workflows/pushes.yml | 2 ++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.ci/check-environment.sh b/.ci/check-environment.sh index 12a835cf26..bc4a1ed86b 100755 --- a/.ci/check-environment.sh +++ b/.ci/check-environment.sh @@ -34,6 +34,7 @@ check "ruby" "ruby --version" # printenv # rsync # ssh +# git diff --git a/.ci/inc/pmd-doc.inc b/.ci/inc/pmd-doc.inc index 2e8f6ad649..64640df6c2 100644 --- a/.ci/inc/pmd-doc.inc +++ b/.ci/inc/pmd-doc.inc @@ -1,3 +1,9 @@ + +# Used env vars: +# PMD_CI_JOB_URL +# PMD_CI_PUSH_COMMIT_COMPARE + + # # Executes jekyll and generates the documentation # The documentation will be generated in the directory "docs/_site". @@ -42,7 +48,7 @@ function publish_release_documentation_github() { ( cd pmd.github.io git init - git config user.name "Travis CI (pmd-bot)" + git config user.name "PMD CI (pmd-bot)" git config user.email "andreas.dangel+pmd-bot@adangel.org" git config core.sparsecheckout true git remote add origin git@github.com:pmd/pmd.github.io.git @@ -91,13 +97,13 @@ function pmd_doc_publish_to_github_pages() { cp -a docs/pmd-doc-${VERSION}/* pmd-gh-pages/ ( cd pmd-gh-pages - git config user.name "Travis CI (pmd-bot)" + git config user.name "PMD CI (pmd-bot)" git config user.email "andreas.dangel+pmd-bot@adangel.org" git add -A MSG="Update documentation -TRAVIS_JOB_NUMBER=${TRAVIS_JOB_NUMBER} -TRAVIS_COMMIT_RANGE=${TRAVIS_COMMIT_RANGE}" +${PMD_CI_JOB_URL} +${PMD_CI_PUSH_COMMIT_COMPARE} git commit -q -m "$MSG" git push git@github.com:pmd/pmd.git HEAD:gh-pages log_success "Successfully pushed site to https://pmd.github.io/pmd/" diff --git a/.github/workflows/pushes.yml b/.github/workflows/pushes.yml index e145ab8296..09690c5bea 100644 --- a/.github/workflows/pushes.yml +++ b/.github/workflows/pushes.yml @@ -36,6 +36,8 @@ jobs: shell: bash env: PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} + PMD_CI_JOB_URL: "https://github.com/pmd/pmd/runs/${{ github.run_id }}" + PMD_CI_PUSH_COMMIT_COMPARE: ${{ github.event.compare }} windows: runs-on: windows-latest From cf656cc5588b3f060680f7a0ec384fddcddc67f5 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 11:42:50 +0100 Subject: [PATCH 10/52] Fix path for maven settings, remove progress-meter while downloading --- .ci/build.sh | 4 ++-- .ci/inc/pmd-doc.inc | 2 +- .ci/install-openjdk.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/build.sh b/.ci/build.sh index 7e285ee4d1..11bcf524fe 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -62,7 +62,7 @@ function pmd_ci_build_get_pom_version() { function pmd_ci_build_setup_maven() { # configure maven mkdir -p ${HOME}/.m2 - cp .ci/maven-settings.xml ${HOME}/.m2/settings.xml + cp .ci/files/maven-settings.xml ${HOME}/.m2/settings.xml } function pmd_ci_build_setup_oraclejdk7() { @@ -77,7 +77,7 @@ function pmd_ci_build_setup_oraclejdk7() { mkdir -p ${TARGET_DIR} if [ ! -e ${LOCAL_DIR}/${ARCHIVE} ]; then log_info "Downloading from ${DOWNLOAD_URL} to ${LOCAL_DIR}" - curl --location --output ${LOCAL_DIR}/${ARCHIVE} ${DOWNLOAD_URL} + curl --location --no-progress-meter --output ${LOCAL_DIR}/${ARCHIVE} ${DOWNLOAD_URL} else log_info "Skipped download, file ${LOCAL_DIR}/${ARCHIVE} already exists" fi diff --git a/.ci/inc/pmd-doc.inc b/.ci/inc/pmd-doc.inc index 64640df6c2..c55f18fd92 100644 --- a/.ci/inc/pmd-doc.inc +++ b/.ci/inc/pmd-doc.inc @@ -103,7 +103,7 @@ function pmd_doc_publish_to_github_pages() { MSG="Update documentation ${PMD_CI_JOB_URL} -${PMD_CI_PUSH_COMMIT_COMPARE} +${PMD_CI_PUSH_COMMIT_COMPARE}" git commit -q -m "$MSG" git push git@github.com:pmd/pmd.git HEAD:gh-pages log_success "Successfully pushed site to https://pmd.github.io/pmd/" diff --git a/.ci/install-openjdk.sh b/.ci/install-openjdk.sh index e1359675a2..79ca912d08 100755 --- a/.ci/install-openjdk.sh +++ b/.ci/install-openjdk.sh @@ -44,7 +44,7 @@ mkdir -p ${TARGET_DIR} if [ ! -e ${CACHE_DIR}/${OPENJDK_ARCHIVE} ]; then log_info "Downloading from ${DOWNLOAD_URL} to ${CACHE_DIR}" - curl --location --output ${CACHE_DIR}/${OPENJDK_ARCHIVE} "${DOWNLOAD_URL}" + curl --location --no-progress-meter --output ${CACHE_DIR}/${OPENJDK_ARCHIVE} "${DOWNLOAD_URL}" else log_info "Skipped download, file ${CACHE_DIR}/${OPENJDK_ARCHIVE} already exists" fi From 865e35439b82be74bc0b6f34d8c99f7d8a474a36 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 11:43:16 +0100 Subject: [PATCH 11/52] Implement check-environment --- .ci/check-environment.sh | 54 ++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/.ci/check-environment.sh b/.ci/check-environment.sh index bc4a1ed86b..850b82a621 100755 --- a/.ci/check-environment.sh +++ b/.ci/check-environment.sh @@ -9,34 +9,50 @@ source $(dirname $0)/inc/logger.inc set -e - function check() { local CMD=$1 local VERSION_CMD=$2 + local VERSION_STRING=$3 echo -n "Checking ${CMD}..." - if [ hash "$CMD" 2>/dev/null ]; then - echo -e "${COL_GREEN}OK${COL_RESET}" + if hash "$CMD" 2>/dev/null; then + local VERSION_FULL=$(${VERSION_CMD} 2>&1) + local VERSION=$(echo "${VERSION_FULL}" | grep "${VERSION_STRING}" 2>&1) + if [ -n "${VERSION}" ]; then + echo -e "${COL_GREEN}OK${COL_RESET}" + echo " ${VERSION}" + else + echo -e "${COL_RED}wrong version${COL_RESET}. Expected: ${VERSION_STRING}" + echo " ${VERSION_FULL}" + fi else - echo -e "${COL_RED}failure${COL_RESET}" + echo -e "${COL_RED}not found!${COL_RESET}" fi } # every OS: -# curl -# jq +check "curl" "curl --version" "curl" +check "jq" "jq --version" "jq" -# linux only -ruby --version | grep "ruby 2.7" || (log_error "Ruby is missing"; exit 1) -check "ruby" "ruby --version" -# gpg -# printenv -# rsync -# ssh -# git - - - -# windows only -# 7zip +case "$(uname)" in + Linux*) + check "ruby" "ruby --version" "ruby 2.7" + check "gpg" "gpg --version" "gpg (GnuPG) 2." + check "printenv" "printenv --version" "printenv (GNU coreutils)" + check "rsync" "rsync --version" "version" + check "ssh" "ssh -V" "OpenSSH" + check "git" "git --version" "git version" + check "mvn" "mvn --version" "Apache Maven" + #check "7z" "7z -version" "7-Zip" + ;; + Darwin*) + ;; + CYGWIN*|MINGW*) + check "7z" "7z -version" "7-Zip" + ;; + *) + log_error "Unknown OS: $(uname)" + exit 1 + ;; +esac From 46286645450481224d5eb6a6a7767f0ff110de84 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 11:54:51 +0100 Subject: [PATCH 12/52] Add regression-tester for updating baseline --- .ci/build.sh | 9 ++-- .ci/inc/regression-tester.inc | 89 +++++++++++++++++++++++++++++++++++ .github/workflows/pushes.yml | 1 + 3 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 .ci/inc/regression-tester.inc diff --git a/.ci/build.sh b/.ci/build.sh index 11bcf524fe..53a3ba4c05 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -5,6 +5,7 @@ source $(dirname $0)/inc/setup-secrets.inc source $(dirname $0)/inc/sourceforge-api.inc source $(dirname $0)/inc/pmd-doc.inc source $(dirname $0)/inc/pmd-code-api.inc +source $(dirname $0)/inc/regression-tester.inc source ${HOME}/java.env set -e @@ -30,8 +31,8 @@ function pmd_ci_build_main() { pmd_ci_build_and_upload_doc - #pmd_ci_build_setup_regression-tester - #regression-tester_uploadBaseline + pmd_ci_build_setup_regression_tester + regression_tester_uploadBaseline exit 0 } @@ -94,15 +95,17 @@ function pmd_ci_build_run() { ./mvnw deploy -Possrh,sign,generate-rule-docs $MVN_BUILD_FLAGS } -function pmd_ci_build_setup_regression-tester() { +function pmd_ci_build_setup_regression_tester() { # install openjdk8 for pmd-regression-tests .ci/install-openjdk.sh 8 bundle config set --local path vendor/bundle bundle config set --local with release_notes_preprocessing + bundle config --local gemfile Gemfile bundle install } function pmd_ci_build_setup_bundler() { + log_info "Installing bundler..." gem install bundler } diff --git a/.ci/inc/regression-tester.inc b/.ci/inc/regression-tester.inc new file mode 100644 index 0000000000..1a33be11b3 --- /dev/null +++ b/.ci/inc/regression-tester.inc @@ -0,0 +1,89 @@ +# +# The functions here require the following scripts: +# inc/logger +# +# The functions here require the following environment variables: +# PMD_SF_USER +# PMD_CI_BRANCH + +# +# Generate a new baseline and upload it to sourceforge +# +# Note: this function always succeeds, even if the upload fails. +# In that case, just a error logging is provided. +# +function regression_tester_uploadBaseline() { + log_debug "$FUNCNAME branch=${PMD_CI_BRANCH}" + local targetUrl="https://sourceforge.net/projects/pmd/files/pmd-regression-tester/" + + PMD_CI_BRANCH=${PMD_CI_BRANCH##refs/heads/} + log_debug "Using modified branch name: ${PMD_CI_BRANCH}" + + local errexitstate="$(shopt -po errexit)" + set +e # disable errexit + ( + # This handler is called if any command fails + function upload_failed() { + log_error "Error while uploading ${BRANCH_FILENAME}-baseline.zip to sourceforge!" + log_error "Please upload manually: ${targetUrl}" + } + + # exit subshell after trap + set -e + trap upload_failed ERR + + log_info "Generating and uploading baseline for pmdtester..." + cd .. + bundle config --local gemfile pmd/Gemfile + pmd/.travis/travis_wait "bundle exec pmdtester + --mode single + --local-git-repo ./pmd + --patch-branch ${PMD_CI_BRANCH} + --patch-config ./pmd/.travis/all-java.xml + --list-of-project ./pmd/.travis/project-list.xml --html-flag" + cd target/reports + BRANCH_FILENAME="${PMD_CI_BRANCH/\//_}" + zip -q -r ${BRANCH_FILENAME}-baseline.zip ${BRANCH_FILENAME}/ + ../../pmd/.travis/travis_wait "rsync -avh ${BRANCH_FILENAME}-baseline.zip ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd-regression-tester/" + log_success "Successfully uploaded ${BRANCH_FILENAME}-baseline.zip to ${targetUrl}" + ) + # restore errexit state + eval "$errexitstate" +} + +# +# Execute danger, which executes pmd-regression-tester (via Dangerfile). +# +# Note: this function always succeeds, even if the danger fails. +# In that case, just a error logging is provided. +# +function regression_tester_executeDanger() { + change_ruby_version + log_debug "$FUNCNAME" + + local errexitstate="$(shopt -po errexit)" + set +e # disable errexit + ( + # This handler is called if any command fails + function danger_failed() { + log_error "Error while executing danger/pmd-regression-tester" + } + + # exit subshell after trap + set -e + trap danger_failed ERR + + # Create a corresponding remote branch locally + if ! git show-ref --verify --quiet refs/heads/${PMD_CI_BRANCH}; then + git fetch --no-tags origin +refs/heads/${PMD_CI_BRANCH}:refs/remotes/origin/${PMD_CI_BRANCH} + git branch ${PMD_CI_BRANCH} origin/${PMD_CI_BRANCH} + log_debug "Created local branch ${PMD_CI_BRANCH}" + fi + + log_info "Running danger on branch ${PMD_CI_BRANCH}" + bundle exec danger --verbose + log_success "Executing danger successfully" + ) + # restore errexit state + eval "$errexitstate" +} diff --git a/.github/workflows/pushes.yml b/.github/workflows/pushes.yml index 09690c5bea..972db01d47 100644 --- a/.github/workflows/pushes.yml +++ b/.github/workflows/pushes.yml @@ -38,6 +38,7 @@ jobs: PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} PMD_CI_JOB_URL: "https://github.com/pmd/pmd/runs/${{ github.run_id }}" PMD_CI_PUSH_COMMIT_COMPARE: ${{ github.event.compare }} + PMD_CI_BRANCH: ${{ github.ref }} windows: runs-on: windows-latest From 7691f8ff24fb56892904f1e130c56b90e82e10b6 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 11:55:05 +0100 Subject: [PATCH 13/52] Add debugging hints --- .ci/README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.ci/README.md b/.ci/README.md index 505b59bff9..20302d08ea 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -29,3 +29,30 @@ Workaround as described in https://github.com/actions/virtual-environments/issue https://issues.apache.org/jira/browse/WAGON-545 is applied: `-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3` + +## Hints + +### Remote debugging + +Debugging remotely is possible with . + +Just add the following step into the job: + +``` + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 +``` + +Note: This is dangerous for push builds, because these have access to the secrets and the SSH session +is not protected. + +### Local tests + +You can run a local instance with docker: + +``` +docker run -it --mount type=bind,source=path/to/pmd,target=/workspaces/pmd/pmd ubuntu:latest +``` + +You'll need to install a few packages before you can start... You can verify with `check-environment.sh` +if everything is prepared. From 7b3a1beeae9dacce1d775a0314b232f41453c910 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 11:57:00 +0100 Subject: [PATCH 14/52] Remove potential old stale bundle config --- .ci/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/build.sh b/.ci/build.sh index 53a3ba4c05..2b4413b156 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -98,9 +98,9 @@ function pmd_ci_build_run() { function pmd_ci_build_setup_regression_tester() { # install openjdk8 for pmd-regression-tests .ci/install-openjdk.sh 8 + rm -f .bundle/config bundle config set --local path vendor/bundle bundle config set --local with release_notes_preprocessing - bundle config --local gemfile Gemfile bundle install } From 55dcd77458ff180ed7938774d6ac129318d2ad80 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 12:29:19 +0100 Subject: [PATCH 15/52] Copy files from old .travis directory --- .ci/files/all-java.xml | 18 ++ .ci/files/project-list.xml | 73 ++++++++ .ci/inc/regression-tester.inc | 8 +- .ci/inc/sourceforge-api.inc | 4 +- .ci/travis_wait | 324 ++++++++++++++++++++++++++++++++++ 5 files changed, 421 insertions(+), 6 deletions(-) create mode 100644 .ci/files/all-java.xml create mode 100644 .ci/files/project-list.xml create mode 100755 .ci/travis_wait diff --git a/.ci/files/all-java.xml b/.ci/files/all-java.xml new file mode 100644 index 0000000000..2a4f1345dc --- /dev/null +++ b/.ci/files/all-java.xml @@ -0,0 +1,18 @@ + + + + Every java rule in PMD which is used for the regression tests with pmdtester + + + + + + + + + + + diff --git a/.ci/files/project-list.xml b/.ci/files/project-list.xml new file mode 100644 index 0000000000..600061f674 --- /dev/null +++ b/.ci/files/project-list.xml @@ -0,0 +1,73 @@ + + + + Standard Projects + + + checkstyle + git + https://github.com/checkstyle/checkstyle + checkstyle-8.10 + + .*/target/test-classes/com/puppycrawl/tools/checkstyle/.* + .*/target/generated-sources/.* + + + echo -n "$(pwd)/target/classes:$(pwd)/target/test-classes:"; cat classpath.txt + + + + spring-framework + git + https://github.com/spring-projects/spring-framework + v5.0.6.RELEASE + + > build.gradle < classpath.txt +]]> + cat classpath.txt + + + + diff --git a/.ci/inc/regression-tester.inc b/.ci/inc/regression-tester.inc index 1a33be11b3..7a509ae1d7 100644 --- a/.ci/inc/regression-tester.inc +++ b/.ci/inc/regression-tester.inc @@ -35,16 +35,16 @@ function regression_tester_uploadBaseline() { log_info "Generating and uploading baseline for pmdtester..." cd .. bundle config --local gemfile pmd/Gemfile - pmd/.travis/travis_wait "bundle exec pmdtester + pmd/.ci/travis_wait "bundle exec pmdtester --mode single --local-git-repo ./pmd --patch-branch ${PMD_CI_BRANCH} - --patch-config ./pmd/.travis/all-java.xml - --list-of-project ./pmd/.travis/project-list.xml --html-flag" + --patch-config ./pmd/.ci/files/all-java.xml + --list-of-project ./pmd/.ci/files/project-list.xml --html-flag" cd target/reports BRANCH_FILENAME="${PMD_CI_BRANCH/\//_}" zip -q -r ${BRANCH_FILENAME}-baseline.zip ${BRANCH_FILENAME}/ - ../../pmd/.travis/travis_wait "rsync -avh ${BRANCH_FILENAME}-baseline.zip ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd-regression-tester/" + ../../pmd/.ci/travis_wait "rsync -avh ${BRANCH_FILENAME}-baseline.zip ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd-regression-tester/" log_success "Successfully uploaded ${BRANCH_FILENAME}-baseline.zip to ${targetUrl}" ) # restore errexit state diff --git a/.ci/inc/sourceforge-api.inc b/.ci/inc/sourceforge-api.inc index 860769d612..24f165cf6f 100644 --- a/.ci/inc/sourceforge-api.inc +++ b/.ci/inc/sourceforge-api.inc @@ -86,7 +86,7 @@ function sourceforge_uploadFile() { trap upload_failed ERR log_info "Uploading $filename to sourceforge..." - .travis/travis_wait "rsync -avh ${filename} ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/${pmdVersion}/" + .ci/travis_wait "rsync -avh ${filename} ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/${pmdVersion}/" log_success "Successfully uploaded ${filename} to sourceforge: ${targetUrl}" ) # restore errexit state @@ -158,7 +158,7 @@ function sourceforge_rsyncSnapshotDocumentation() { trap upload_failed ERR log_info "Uploading documentation to ${targetUrl}..." - .travis/travis_wait "rsync -ah --stats --delete docs/pmd-doc-${VERSION}/ ${PMD_SF_USER}@web.sourceforge.net:/home/project-web/pmd/htdocs/snapshot/" + .ci/travis_wait "rsync -ah --stats --delete docs/pmd-doc-${VERSION}/ ${PMD_SF_USER}@web.sourceforge.net:/home/project-web/pmd/htdocs/snapshot/" log_success "Successfully uploaded documentation: ${targetUrl}" ) # restore errexit state diff --git a/.ci/travis_wait b/.ci/travis_wait new file mode 100755 index 0000000000..d81dd41089 --- /dev/null +++ b/.ci/travis_wait @@ -0,0 +1,324 @@ +#!/usr/bin/env bash + +# The MIT License (MIT) + +# Copyright (c) 2015 +# m3t (96bd6c8bb869fe632b3650fb7156c797ef8c2a055d31dde634565f3edda485b) + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# Available from https://github.com/m3t/travis_wait +# Please report bugs at https://github.com/m3t/travis_wait/issues + +# Coding (Style) Guidelines: +# https://www.chromium.org/chromium-os/shell-style-guidelines +# http://mywiki.wooledge.org/BashGuide/Practices +# http://wiki.bash-hackers.org/scripting/style + + +# bash available? +if [ -z "$BASH_VERSINFO" ]; then + echo "Please make sure you're using bash!" + exit 1 +fi + + +# INITIALIZE CONSTANTS AND GLOBALS +# Only lower case, esp. for export! +# That ensures that system vars stay untouched in any case +readonly prog_name=$(basename "$0") + + +is_writeable() { + local var="$1" + + is_writeable_empty "${var}" 0 +} + +is_writeable_empty() { + local var="$1" + local empty="$2" + [[ -z "${empty}" ]] && empty=1 + + # http://mywiki.wooledge.org/BashGuide/TestsAndConditionals + # "touch" creates file, if it doesn't exist, + # so further tests won't fail at the beginning + if { touch -a "${var}" >/dev/null 2>&1; }; then + if [[ ! -s "${var}" ]]; then + if [[ ! -w "${var}" ]]; then + #show_warning "${var} is not writeable" + return 1 + fi + else + #show_warning "${var} is not empty" + [[ "${empty}" -eq 1 ]] && return 1 + fi + else + #show_warning "Destination for ${var} is not accessible at all" + return 1 + fi + + return 0 +} + +is_number() { + local int="$1" + # http://mywiki.wooledge.org/BashFAQ/054 + [[ "$int" != *[!0-9]* ]] +} + +is_empty() { + local var="$1" + + [[ -z "$var" ]] +} + +show_error() { + printf "\n%s\n" "${prog_name}: error: $*" >&2 + exit 1 +} + +show_warning() { + printf "\n%s\n" "${prog_name}: $*" >&2 +} + +show_help() { + # http://wiki.bash-hackers.org/syntax/redirection#here_documents + cat <<- EOF + + Usage: ${prog_name} [options] [] + + Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor + incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, + quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo + consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse + cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat + non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. + + Arguments: + Slowpoke command + Where 's output will be saved + Default: \$RANDOM-output.log + + Options: + -i, --interval Refresh interval in sec. + Default: 30 + + -l, --limit Limit execution time in sec. + Default: 0 (Off) + + -x, --exit-code Force the exit code + Default: -1 (Off) + + -a, --append PRN append output to existing logfile + Off: 0 (Default) + On: 1 + + -h This help screen + + + Copyright (C) 2015 m3t + The MIT License (MIT) + +EOF + + exit 0 +} + +cleanup() { + kill -0 ${pid_slowpoke} >/dev/null 2>&1 && kill ${pid_slowpoke} >/dev/null 2>&1 +} + +main() { + + # INITIALIZE LOCAL VARIABLES + # Variables to be evaluated as shell arithmetic should be initialized + # to a default or validated beforehand. + # CAUTION: Arguments' (not options) default values will be overwritten here anyway + # So they are set in VALIDATE INPUT + local i=0 + local msg="" + local time_passed=0 + local pid_slowpoke=0 + local exit_slowpoke=0 + # Options: + local interval=30 + local time_limit=0 + local exit_force=-1 + local append=0 + # Arguments: + local cmd_slowpoke="" + local file_log="" + + + # SIGNAL HANDLING + # http://mywiki.wooledge.org/SignalTrap + # http://mywiki.wooledge.org/SignalTrap#Special_Note_On_SIGINT + trap 'cleanup; trap - INT; kill -INT $$' INT QUIT # CTRL+C OR CTRL+\ + trap 'cleanup; exit 1' TERM # kill's default signal + + + # COMMAND-LINE ARGUMENTS AND OPTIONS + # http://mywiki.wooledge.org/BashFAQ/035 + msg="requires a non-empty option argument." + while :; do + case "$1" in + -h|-\?|--help) + show_help + exit + ;; + -l|--limit) + if [ -n "$2" ]; then + time_limit="$2" + shift 2 + continue + else + show_error "--limit ${msg}" + fi + ;; + --limit=?*) + time_limit="${1#*=}" + ;; + --limit=) + show_error "--limit ${msg}" + ;; + -i|--interval) + if [ -n "$2" ]; then + interval="$2" + shift 2 + continue + else + show_error "--interval ${msg}" + fi + ;; + --interval=?*) + interval="${1#*=}" + ;; + --interval=) + show_error "--interval ${msg}" + ;; + -x|--exit-code) + if [ -n "$2" ]; then + exit_force="$2" + shift 2 + continue + else + show_error "--exit-code ${msg}" + fi + ;; + --exit-code=?*) + exit_force="${1#*=}" + ;; + --exit-code=) + show_error "--exit-code ${msg}" + ;; + -a|--append) + if [ -n "$2" ]; then + append="$2" + shift 2 + continue + else + show_error "--append ${msg}" + fi + ;; + --append=?*) + append="${1#*=}" + ;; + --append=) + show_error "--append ${msg}" + ;; + --) # End of all options. + shift + break + ;; + -?*) + show_warning "Unknown option (ignored): $1" + ;; + *) # Default case: If no more options then break out of the loop. + break + esac + + shift + done + # Arguments following the options + # will remain in the "$@" positional parameters. + cmd_slowpoke="$1" + file_log="$2" + + + # VALIDATE INPUT + is_number "${interval}" || show_error "Interval is not a valid number" + is_number "${time_limit}" || show_error "Limit is not a valid number" + is_empty "${cmd_slowpoke}" && show_error "Command to execute is not given. See --help." + is_empty "${file_log}" && file_log="$RANDOM-output.log" # http://mywiki.wooledge.org/BashFAQ/062 + + # START CMD + # http://mywiki.wooledge.org/ProcessManagement + if [[ "${append}" -ne 1 ]]; then + is_writeable_empty "${file_log}" || show_error "${file_log} is not writeable or not empty." + ${cmd_slowpoke} > "${file_log}" & pid_slowpoke=$! + else + is_writeable "${file_log}" || show_error "${file_log} is not writeable." + ${cmd_slowpoke} >> "${file_log}" & pid_slowpoke=$! + fi + + + # WAIT + # Terminates when $cmd_slowpoke is finished + # OR + # $time_limit has reached + i=0 + while kill -0 ${pid_slowpoke} >/dev/null 2>&1; do + : $(( time_passed = i * interval )) + + printf "%s\n" \ + "Still waiting for about ${time_passed} seconds" \ + "Used disk space: $(du -sh .)" + + # Output last line from $file_log + tail -1 "${file_log}" + + # $time_limit + if [[ "${time_limit}" -ne 0 ]] && [[ "${time_passed}" -ge "${time_limit}" ]]; then + cleanup + break + fi + + sleep ${interval} + + : $(( i += 1 )) + done + + + # FINISHED + # Shall I fake the exit code? + if ! is_number "${exit_force}"; then + # Get exit code from child process that is terminated already, see above + wait ${pid_slowpoke}; exit_slowpoke=$? + else + exit_slowpoke=${exit_force} + fi + # Output last couple of lines from $file_log + tail -5 "${file_log}" + show_warning "Your given command has terminated with exit code $exit_slowpoke. So do I." + exit ${exit_slowpoke} + +} + +main "$@" From 295505af5b73b5aab72fffefcd5784d885eb460d Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 13:11:56 +0100 Subject: [PATCH 16/52] Integrate danger for pull requests --- .ci/README.md | 15 +++++++++++---- .ci/build-pr.sh | 21 +++++++++++++++++++++ .ci/build.sh | 12 ++---------- .ci/files/env.gpg | Bin 406 -> 489 bytes .ci/inc/regression-tester.inc | 22 +++++++++++++++++++++- .github/workflows/pull-requests.yml | 18 ++++++++++-------- Dangerfile | 17 ++++++----------- 7 files changed, 71 insertions(+), 34 deletions(-) create mode 100755 .ci/build-pr.sh diff --git a/.ci/README.md b/.ci/README.md index 20302d08ea..96d6091bd4 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -9,10 +9,17 @@ for a ssh key, which is used to copy files to sourceforge. ## Environment variables -* `PMD_CI_SECRET_PASSPHRASE` -* `CI_DEPLOY_PASSWORD` -* `CI_SIGN_PASSPHRASE` -* ... +* PMD_CI_SECRET_PASSPHRASE +* CI_DEPLOY_USER +* CI_DEPLOY_PASSWORD +* CI_SIGN_KEY +* CI_SIGN_PASSPHRASE +* PMD_SF_USER +* PMD_SF_APIKEY +* GITHUB_OAUTH_TOKEN +* GITHUB_BASE_URL +* DANGER_GITHUB_API_TOKEN +* PMD_CI_CHUNK_TOKEN ## Encrypting diff --git a/.ci/build-pr.sh b/.ci/build-pr.sh new file mode 100755 index 0000000000..2d44e5a4f0 --- /dev/null +++ b/.ci/build-pr.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +source $(dirname $0)/inc/logger.inc +source $(dirname $0)/inc/setup-secrets.inc +source $(dirname $0)/inc/regression-tester.inc +source ${HOME}/java.env + +set -e +#set -x + +./mvnw -e -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -V clean verify + +# Danger is executed only on the linux runner +case "$(uname)" in + Linux*) + log_info "Executing danger..." + pmd_ci_setup_env + regression_tester_setup_ci + regression_tester_executeDanger + ;; +esac diff --git a/.ci/build.sh b/.ci/build.sh index 2b4413b156..7148a73949 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -31,7 +31,7 @@ function pmd_ci_build_main() { pmd_ci_build_and_upload_doc - pmd_ci_build_setup_regression_tester + regression_tester_setup_ci regression_tester_uploadBaseline exit 0 @@ -95,15 +95,7 @@ function pmd_ci_build_run() { ./mvnw deploy -Possrh,sign,generate-rule-docs $MVN_BUILD_FLAGS } -function pmd_ci_build_setup_regression_tester() { - # install openjdk8 for pmd-regression-tests - .ci/install-openjdk.sh 8 - rm -f .bundle/config - bundle config set --local path vendor/bundle - bundle config set --local with release_notes_preprocessing - bundle install -} - +# Needed for doc generation and regression tester function pmd_ci_build_setup_bundler() { log_info "Installing bundler..." gem install bundler diff --git a/.ci/files/env.gpg b/.ci/files/env.gpg index 07dd553e73881351482bd373f034f76deaa5ae9d..d379ecefe2134949dedee3142a0c33d14132ec39 100644 GIT binary patch literal 489 zcmVh5Cn{jb|5u*%km&lrrHM0V=1e}D~hVy zAG7-Frz5RB#yp-qo18`ev(X~ugDL7O7sP;m%1VtoXxdz)pgxvlo58TmWBC2D0avCHP}hki}6v|m%G`;+b^D()-~4XdW9E7MLn2nU_A z3CUHcX{K3#&9{F--Q0+8i74jTb265brVy{r$hla^^P3OS#`@MsM7ZYVHnQkyvYq+w z0qNCjo#cKsSL=XQT85iudD9STL`wbZR^5hzOMp9)$5_siz;DBP;dA?4f%u#A4@b1% z&*Bn3>`Rd^N$kmDP(9`;HFLHlF%G7K^eGmrg0GY((v!hSo<_3- z-FNlJDllWjrQQL&uCQswy@8;ihV+kNPWo22dD`{?j(oUS6Qo^YEA1u6rhzZuZ7i)0 zyn+oj&Fk+YxtIo9#@vz@uBI2Db^DAFs>WULK>j*n>JMhtN*Xdm&W^C6B-%A5N{l7m z2=;fA@al{Zdw&QJLl}!}51$w9BhQgQGJW^~@Ys{1^`!h_Hn$W0ZF0R@CKs-b;c(nE flam{g#_A<1HUX=ast~esrY2$33)nqFF1+0U*~I${ literal 406 zcmV;H0crk>4Fm}T0tI+nP!WOFH2>1T!~qzymt0!>gktO?WM#Q@d;(HxHf_*1C%OQ! zr;Oj!oJjy_gIV%u7XHd*ew+R_m`?yR`MLG>oF%i>B-~oMzL(BHW54kGh~T1wMANn? z?G<>kUDb>)TExoqJ`L(jdjq=6O8^XT+qA@VyeV?anBP0U2zjBeQtrWJ&A)u|b~T7_ z4dID=w8GZH)(W)c;vI$uCzZz`)nFqm{|Yg0zd2t;-i4$TMlB4l|4@#60qTH2oL+eB z?l*R$K#el$!|Qxfkby#}LP{_;}-UV#wSx{(Ma}=tGvI4Dg~HK8o52Lyh6#% zb$6QrFGhtec`k377R+SbV)qub^o&Brbw*5=OJ$H{(wSK+2Mze3bvK+w{KNtn>xI>_ z+l_!@72xtT0+Lq!)b>x4A*aA`H5eZyrSGGoAy7iFn)Zr|a#kD|(bnjf=ExpztO&-> AB>(^b diff --git a/.ci/inc/regression-tester.inc b/.ci/inc/regression-tester.inc index 7a509ae1d7..9b66372071 100644 --- a/.ci/inc/regression-tester.inc +++ b/.ci/inc/regression-tester.inc @@ -6,6 +6,23 @@ # PMD_SF_USER # PMD_CI_BRANCH +function regression_tester_setup_ci() { + # install openjdk8 for pmd-regression-tests + .ci/install-openjdk.sh 8 + + if hash "bundler" 2>/dev/null; then + log_debug "Bundler is already installed" + else + log_info "Installing bundler..." + gem install bundler + fi + + rm -f .bundle/config + bundle config set --local path vendor/bundle + bundle config set --local with release_notes_preprocessing + bundle install +} + # # Generate a new baseline and upload it to sourceforge # @@ -58,7 +75,6 @@ function regression_tester_uploadBaseline() { # In that case, just a error logging is provided. # function regression_tester_executeDanger() { - change_ruby_version log_debug "$FUNCNAME" local errexitstate="$(shopt -po errexit)" @@ -73,6 +89,10 @@ function regression_tester_executeDanger() { set -e trap danger_failed ERR + PMD_CI_BRANCH=${PMD_CI_BRANCH##refs/heads/} + export PMD_CI_BRANCH + log_debug "Using modified branch name: ${PMD_CI_BRANCH}" + # Create a corresponding remote branch locally if ! git show-ref --verify --quiet refs/heads/${PMD_CI_BRANCH}; then git fetch --no-tags origin +refs/heads/${PMD_CI_BRANCH}:refs/remotes/origin/${PMD_CI_BRANCH} diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 023f1ac239..26109a0253 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -21,15 +21,17 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}- + - name: Set up Ruby 2.7 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.7 - name: Install OpenJDK 11 run: .ci/install-openjdk.sh 11 shell: bash - - name: Build with mvnw - run: | - source ${HOME}/java.env - ./mvnw -e -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -V clean install - shell: bash - - name: Execute Danger - if: ${{ runner.os == 'Linux' }} - run: echo "TODO" + - name: Build + run: .ci/build-pr.sh shell: bash + env: + PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} + PMD_CI_BRANCH: ${{ github.ref }} + PMD_CI_PULL_REQUEST_NUMBER: ${{ github.event.number }} diff --git a/Dangerfile b/Dangerfile index f57f762c11..f26f7a528e 100644 --- a/Dangerfile +++ b/Dangerfile @@ -7,10 +7,10 @@ require 'logger' def run_pmdtester Dir.chdir('..') do argv = ['--local-git-repo', './pmd', - '--list-of-project', './pmd/.travis/project-list.xml', - '--base-branch', "#{ENV['TRAVIS_BRANCH']}", + '--list-of-project', './pmd/.ci/files/project-list.xml', + '--base-branch', "#{ENV['PMD_CI_BRANCH']}", '--patch-branch', 'HEAD', - '--patch-config', './pmd/.travis/all-java.xml', + '--patch-config', './pmd/.ci/files/all-java.xml', '--mode', 'online', '--auto-gen-config', # '--debug', @@ -28,14 +28,14 @@ end def upload_report Dir.chdir('target/reports') do - tar_filename = "pr-#{ENV['TRAVIS_PULL_REQUEST']}-diff-report-#{Time.now.strftime("%Y-%m-%dT%H-%M-%SZ")}.tar" + tar_filename = "pr-#{ENV['PMD_CI_PULL_REQUEST_NUMBER']}-diff-report-#{Time.now.strftime("%Y-%m-%dT%H-%M-%SZ")}.tar" unless Dir.exist?('diff/') message("No java rules are changed!", sticky: true) return end `tar -cf #{tar_filename} diff/` - report_url = `curl -u #{ENV['CHUNK_TOKEN']} -T #{tar_filename} https://chunk.io` + report_url = `curl -u #{ENV['PMD_CI_CHUNK_TOKEN']} -T #{tar_filename} https://chunk.io` if $?.success? @logger.info "Successfully uploaded #{tar_filename} to chunk.io" @@ -53,11 +53,6 @@ def upload_report end # Perform regression testing -can_merge = github.pr_json['mergeable'] -if can_merge - run_pmdtester -else - warn("This PR cannot be merged yet.", sticky: false) -end +run_pmdtester # vim: syntax=ruby From 6b2b33bdece6ad2a863bafd3941836b25584f5c0 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 16:13:02 +0100 Subject: [PATCH 17/52] Add docker for testing --- .ci/README.md | 22 +++++++++-- .ci/docker_ubuntu18.04/Dockerfile | 37 +++++++++++++++++++ .../create-gh-pull-request-event.sh | 10 +++++ .ci/docker_ubuntu18.04/install-ruby.sh | 8 ++++ 4 files changed, 73 insertions(+), 4 deletions(-) create mode 100644 .ci/docker_ubuntu18.04/Dockerfile create mode 100755 .ci/docker_ubuntu18.04/create-gh-pull-request-event.sh create mode 100755 .ci/docker_ubuntu18.04/install-ruby.sh diff --git a/.ci/README.md b/.ci/README.md index 96d6091bd4..13ec72eba0 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -55,11 +55,25 @@ is not protected. ### Local tests -You can run a local instance with docker: +Create a local docker container: ``` -docker run -it --mount type=bind,source=path/to/pmd,target=/workspaces/pmd/pmd ubuntu:latest +cd .ci/docker_ubuntu18.04 +docker build -t pmd-ci . ``` -You'll need to install a few packages before you can start... You can verify with `check-environment.sh` -if everything is prepared. +This container is based on Ubuntu 18.04, which is used for `ubuntu-latest` github actions runner +(see ). + +You can run a local instance with docker and mount your local pmd checkout into the container: + +``` +docker run -it --mount type=bind,source=path/to/pmd,target=/workspaces/pmd/pmd pmd-ci +``` + +You'll be dropped into a bash. Start e.g. with + +``` +cd workspaces/pmd/pmd +.ci/check-environment.sh +``` diff --git a/.ci/docker_ubuntu18.04/Dockerfile b/.ci/docker_ubuntu18.04/Dockerfile new file mode 100644 index 0000000000..ae761efa59 --- /dev/null +++ b/.ci/docker_ubuntu18.04/Dockerfile @@ -0,0 +1,37 @@ +# https://hub.docker.com/_/ubuntu/ +FROM ubuntu:18.04 + +RUN apt-get update && apt-get upgrade --yes && \ + apt-get install --yes curl jq gpg rsync ssh git p7zip-full openjdk-8-jdk \ + libgdbm-dev libncurses5-dev automake libtool bison libffi-dev \ + sudo nano bash && \ + apt-get clean +RUN cd opt && \ + curl https://mirror.checkdomain.de/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz | tar xz && \ + ln -sf /opt/apache-maven-3.6.3/bin/mvn /usr/local/bin/mvn && \ + cd .. +RUN groupadd --gid 1000 pmd-ci && useradd --gid 1000 --uid 1000 --groups sudo \ + --shell /bin/bash --create-home --password "" \ + pmd-ci + +RUN mkdir /workspaces && chown pmd-ci:pmd-ci /workspaces && ln -sf /workspaces /home/pmd-ci/workspaces + +USER pmd-ci:pmd-ci +WORKDIR /home/pmd-ci +ENV GITHUB_EVENT_PATH=/workspaces/event.json +ENV GITHUB_REPOSITORY=pmd/pmd +ENV GITHUB_ACTION=run1 +ENV GITHUB_EVENT_NAME=pull_request +ENV PMD_CI_BRANCH=refs/heads/master +ENV PMD_CI_PULL_REQUEST_NUMBER=2913 +COPY create-gh-pull-request-event.sh . +COPY install-ruby.sh . +RUN ./install-ruby.sh +RUN ./create-gh-pull-request-event.sh + +CMD ["/bin/bash", "--login"] + +# +# build with: docker build -t pmd-ci . +# run with: docker run -it pmd-ci +# \ No newline at end of file diff --git a/.ci/docker_ubuntu18.04/create-gh-pull-request-event.sh b/.ci/docker_ubuntu18.04/create-gh-pull-request-event.sh new file mode 100755 index 0000000000..7f3576196e --- /dev/null +++ b/.ci/docker_ubuntu18.04/create-gh-pull-request-event.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +cat > /workspaces/event.json < Date: Thu, 12 Nov 2020 16:14:09 +0100 Subject: [PATCH 18/52] Remove unsupported --no-progress-meter for curl --- .ci/build.sh | 2 +- .ci/install-openjdk.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/build.sh b/.ci/build.sh index 7148a73949..9141f0f260 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -78,7 +78,7 @@ function pmd_ci_build_setup_oraclejdk7() { mkdir -p ${TARGET_DIR} if [ ! -e ${LOCAL_DIR}/${ARCHIVE} ]; then log_info "Downloading from ${DOWNLOAD_URL} to ${LOCAL_DIR}" - curl --location --no-progress-meter --output ${LOCAL_DIR}/${ARCHIVE} ${DOWNLOAD_URL} + curl --location --output ${LOCAL_DIR}/${ARCHIVE} ${DOWNLOAD_URL} else log_info "Skipped download, file ${LOCAL_DIR}/${ARCHIVE} already exists" fi diff --git a/.ci/install-openjdk.sh b/.ci/install-openjdk.sh index 79ca912d08..e1359675a2 100755 --- a/.ci/install-openjdk.sh +++ b/.ci/install-openjdk.sh @@ -44,7 +44,7 @@ mkdir -p ${TARGET_DIR} if [ ! -e ${CACHE_DIR}/${OPENJDK_ARCHIVE} ]; then log_info "Downloading from ${DOWNLOAD_URL} to ${CACHE_DIR}" - curl --location --no-progress-meter --output ${CACHE_DIR}/${OPENJDK_ARCHIVE} "${DOWNLOAD_URL}" + curl --location --output ${CACHE_DIR}/${OPENJDK_ARCHIVE} "${DOWNLOAD_URL}" else log_info "Skipped download, file ${CACHE_DIR}/${OPENJDK_ARCHIVE} already exists" fi From 623213e528b89654c6ecc867afd2c51d7756f306 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 16:46:35 +0100 Subject: [PATCH 19/52] Add unzip and timezone --- .ci/check-environment.sh | 1 + .ci/docker_ubuntu18.04/Dockerfile | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/check-environment.sh b/.ci/check-environment.sh index 850b82a621..51dfc28791 100755 --- a/.ci/check-environment.sh +++ b/.ci/check-environment.sh @@ -44,6 +44,7 @@ case "$(uname)" in check "ssh" "ssh -V" "OpenSSH" check "git" "git --version" "git version" check "mvn" "mvn --version" "Apache Maven" + check "unzip" "unzip --version" "UnZip" #check "7z" "7z -version" "7-Zip" ;; Darwin*) diff --git a/.ci/docker_ubuntu18.04/Dockerfile b/.ci/docker_ubuntu18.04/Dockerfile index ae761efa59..6999125846 100644 --- a/.ci/docker_ubuntu18.04/Dockerfile +++ b/.ci/docker_ubuntu18.04/Dockerfile @@ -1,10 +1,11 @@ # https://hub.docker.com/_/ubuntu/ FROM ubuntu:18.04 +ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get upgrade --yes && \ apt-get install --yes curl jq gpg rsync ssh git p7zip-full openjdk-8-jdk \ libgdbm-dev libncurses5-dev automake libtool bison libffi-dev \ - sudo nano bash && \ + sudo nano bash tzdata unzip && \ apt-get clean RUN cd opt && \ curl https://mirror.checkdomain.de/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz | tar xz && \ @@ -14,7 +15,7 @@ RUN groupadd --gid 1000 pmd-ci && useradd --gid 1000 --uid 1000 --groups sudo \ --shell /bin/bash --create-home --password "" \ pmd-ci -RUN mkdir /workspaces && chown pmd-ci:pmd-ci /workspaces && ln -sf /workspaces /home/pmd-ci/workspaces +RUN mkdir -p /workspaces/pmd/pmd && chown -R pmd-ci:pmd-ci /workspaces && ln -sf /workspaces /home/pmd-ci/workspaces USER pmd-ci:pmd-ci WORKDIR /home/pmd-ci From 3d0f5b0a6cf12be3556574a1b8d308fe3d53ddbf Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 17:59:43 +0100 Subject: [PATCH 20/52] Try disable http pooling --- .ci/README.md | 6 +++++- .ci/build-pr.sh | 2 +- .ci/build.sh | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index 13ec72eba0..b18d9c2ca6 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -35,7 +35,11 @@ The default timeout is 4 minutes. Workaround as described in https://github.com/actions/virtual-environments/issues/1499 and https://issues.apache.org/jira/browse/WAGON-545 is applied: -`-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3` +The setting `-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3` +doesn't seem to work. + +Now we disable pooling completeley, so that for downloading a artifact/dependency, always new, fresh +connections are sued: `-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false`. ## Hints diff --git a/.ci/build-pr.sh b/.ci/build-pr.sh index 2d44e5a4f0..3705831421 100755 --- a/.ci/build-pr.sh +++ b/.ci/build-pr.sh @@ -8,7 +8,7 @@ source ${HOME}/java.env set -e #set -x -./mvnw -e -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -V clean verify +./mvnw -e -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -V clean verify # Danger is executed only on the linux runner case "$(uname)" in diff --git a/.ci/build.sh b/.ci/build.sh index 9141f0f260..bc147351b9 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -89,7 +89,7 @@ function pmd_ci_build_setup_oraclejdk7() { } function pmd_ci_build_run() { - MVN_BUILD_FLAGS="-e -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 -V -Djava7.home=${HOME}/oraclejdk7" + MVN_BUILD_FLAGS="-e -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -V -Djava7.home=${HOME}/oraclejdk7" log_info "This is a snapshot build" ./mvnw deploy -Possrh,sign,generate-rule-docs $MVN_BUILD_FLAGS From 11932a140d21a45c373862f401df58b4d46be50d Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 18:35:04 +0100 Subject: [PATCH 21/52] Try to set wagon options via MAVEN_OPTS --- .ci/build-pr.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/build-pr.sh b/.ci/build-pr.sh index 3705831421..efec2e2a93 100755 --- a/.ci/build-pr.sh +++ b/.ci/build-pr.sh @@ -8,7 +8,8 @@ source ${HOME}/java.env set -e #set -x -./mvnw -e -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -V clean verify +export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" +./mvnw -e -V clean verify # Danger is executed only on the linux runner case "$(uname)" in From df7b75087add3b3600f7d094fee80da4844cee8b Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 18:57:35 +0100 Subject: [PATCH 22/52] Another try with maven.wagon.http.pool=false --- .ci/build-pr.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/build-pr.sh b/.ci/build-pr.sh index efec2e2a93..4c7473b2c0 100755 --- a/.ci/build-pr.sh +++ b/.ci/build-pr.sh @@ -8,7 +8,8 @@ source ${HOME}/java.env set -e #set -x -export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" +#export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" +export MAVEN_OPTS="-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false" ./mvnw -e -V clean verify # Danger is executed only on the linux runner From 1edc65945660ec1f2b6e0a6d22853519d56eb54f Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 19:15:29 +0100 Subject: [PATCH 23/52] Take out windows build It's always failing to download dependencies --- .ci/README.md | 10 ++++++---- .ci/build-pr.sh | 2 +- .ci/build.sh | 6 +++--- .github/workflows/pull-requests.yml | 3 ++- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index b18d9c2ca6..0d4a7b5627 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -38,8 +38,10 @@ https://issues.apache.org/jira/browse/WAGON-545 is applied: The setting `-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3` doesn't seem to work. -Now we disable pooling completeley, so that for downloading a artifact/dependency, always new, fresh -connections are sued: `-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false`. +Now we disable pooling completely, so that for downloading a artifact/dependency, always new, fresh +connections are used: `-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false`. + +Not working either. ## Hints @@ -54,8 +56,8 @@ Just add the following step into the job: uses: mxschmitt/action-tmate@v3 ``` -Note: This is dangerous for push builds, because these have access to the secrets and the SSH session -is not protected. +**Note**: This is dangerous for push/pull builds, because these have access to the secrets and the SSH session +is not protected... ### Local tests diff --git a/.ci/build-pr.sh b/.ci/build-pr.sh index 4c7473b2c0..a044b66aaa 100755 --- a/.ci/build-pr.sh +++ b/.ci/build-pr.sh @@ -9,7 +9,7 @@ set -e #set -x #export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" -export MAVEN_OPTS="-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false" +#export MAVEN_OPTS="-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false" ./mvnw -e -V clean verify # Danger is executed only on the linux runner diff --git a/.ci/build.sh b/.ci/build.sh index bc147351b9..f7ced21cec 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -89,10 +89,10 @@ function pmd_ci_build_setup_oraclejdk7() { } function pmd_ci_build_run() { - MVN_BUILD_FLAGS="-e -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -V -Djava7.home=${HOME}/oraclejdk7" - log_info "This is a snapshot build" - ./mvnw deploy -Possrh,sign,generate-rule-docs $MVN_BUILD_FLAGS + #export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" + #export MAVEN_OPTS="-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false" + ./mvnw deploy -Possrh,sign,generate-rule-docs -e -V -Djava7.home=${HOME}/oraclejdk7 } # Needed for doc generation and regression tester diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 26109a0253..2aeec4475e 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -9,7 +9,8 @@ jobs: if: "!contains(github.event.head_commit.message, '[skip ci]')" strategy: matrix: - os: [ ubuntu-latest, windows-latest, macos-latest ] + #os: [ ubuntu-latest, windows-latest, macos-latest ] + os: [ ubuntu-latest, macos-latest ] steps: - uses: actions/checkout@v2 From 15fc3f8fc41daffe38c689caf6eb14c163f8d0a0 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 19:47:38 +0100 Subject: [PATCH 24/52] Try again with wagon options --- .ci/build-pr.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/build-pr.sh b/.ci/build-pr.sh index a044b66aaa..f90221d9dc 100755 --- a/.ci/build-pr.sh +++ b/.ci/build-pr.sh @@ -8,7 +8,7 @@ source ${HOME}/java.env set -e #set -x -#export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" +export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" #export MAVEN_OPTS="-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false" ./mvnw -e -V clean verify From 1790684fff7ad7ce5b313315d37f698b8f48e38c Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 12 Nov 2020 20:12:07 +0100 Subject: [PATCH 25/52] Add troubleshooting --- .github/workflows/pull-requests.yml | 3 ++- .github/workflows/troubleshooting.yml | 33 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/troubleshooting.yml diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 2aeec4475e..764eebb453 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -1,6 +1,7 @@ name: Pull Requests -on: pull_request +#on: pull_request +on: workflow_dispatch jobs: build: diff --git a/.github/workflows/troubleshooting.yml b/.github/workflows/troubleshooting.yml new file mode 100644 index 0000000000..7a4af43a7c --- /dev/null +++ b/.github/workflows/troubleshooting.yml @@ -0,0 +1,33 @@ +name: troubleshooting + +on: pull_request + +jobs: + build: + runs-on: ${{ matrix.os }} + continue-on-error: false + if: "!contains(github.event.head_commit.message, '[skip ci]')" + strategy: + matrix: + #os: [ ubuntu-latest, windows-latest, macos-latest ] + os: [ ubuntu-latest ] + + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}- + - name: Set up Ruby 2.7 + uses: actions/setup-ruby@v1 + with: + ruby-version: 2.7 + - name: Install OpenJDK 11 + run: .ci/install-openjdk.sh 11 + shell: bash + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 From 70f2c56888fbd78e09144cde75f091c7cc9b7fe3 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 11:37:05 +0100 Subject: [PATCH 26/52] Resolve maven dependencies before building This tries to solve build timeouts while downloading dependencies. Also the job timeout for PRs is 30 minutes - if it takes longer, something is wrong. --- .ci/build-pr.sh | 5 +++-- .ci/inc/maven-dependencies.inc | 25 +++++++++++++++++++++++++ .github/workflows/pull-requests.yml | 2 ++ .github/workflows/troubleshooting.yml | 3 +-- 4 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 .ci/inc/maven-dependencies.inc diff --git a/.ci/build-pr.sh b/.ci/build-pr.sh index f90221d9dc..084974bd05 100755 --- a/.ci/build-pr.sh +++ b/.ci/build-pr.sh @@ -3,13 +3,14 @@ source $(dirname $0)/inc/logger.inc source $(dirname $0)/inc/setup-secrets.inc source $(dirname $0)/inc/regression-tester.inc +source $(dirname $0)/inc/maven-dependencies.inc source ${HOME}/java.env set -e #set -x -export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" -#export MAVEN_OPTS="-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false" +maven_dependencies_resolve + ./mvnw -e -V clean verify # Danger is executed only on the linux runner diff --git a/.ci/inc/maven-dependencies.inc b/.ci/inc/maven-dependencies.inc new file mode 100644 index 0000000000..2c48bc39a4 --- /dev/null +++ b/.ci/inc/maven-dependencies.inc @@ -0,0 +1,25 @@ + +# +# On azure, outgoing idle connection are dropped after 4 minutes. +# Usually, you can configure wagon with ttl. But these settings are +# ignored, as soon as dokka-maven-plugin is loaded. +# dokka-maven-plugin tries to load additional dependencies at runtime +# and injects a different http client, which is not configured correctly +# and thus maven fails if it tries to download later in the build process +# further dependencies. +# +# The workaround applied here is: first resolve all dependencies, +# then explicitly get dokka-maven-plugin and then resolve all plugins +# execpt for dokka-maven-plugin, as it does not play well with dependency-plugin. +# +function maven_dependencies_resolve() { + dokka_version=$(./mvnw -q -Dexec.executable="echo" -Dexec.args='${dokka.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) + + ./mvnw dependency:resolve + ./mvnw dependency:get -DgroupId=org.jetbrains.dokka \ + -DartifactId=dokka-maven-plugin \ + -Dversion=${dokka_version} \ + -Dpackaging=jar \ + -DremoteRepositories=jcenter::default::https://jcenter.bintray.com/ + ./mvnw dependency:resolve-plugins -DexcludeGroupIds=org.jetbrains.dokka +} diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 764eebb453..e278d7ba3a 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -7,6 +7,7 @@ jobs: build: runs-on: ${{ matrix.os }} continue-on-error: false + timeout-minutes: 30 if: "!contains(github.event.head_commit.message, '[skip ci]')" strategy: matrix: @@ -34,6 +35,7 @@ jobs: run: .ci/build-pr.sh shell: bash env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} PMD_CI_BRANCH: ${{ github.ref }} PMD_CI_PULL_REQUEST_NUMBER: ${{ github.event.number }} diff --git a/.github/workflows/troubleshooting.yml b/.github/workflows/troubleshooting.yml index 7a4af43a7c..dcdeeb9338 100644 --- a/.github/workflows/troubleshooting.yml +++ b/.github/workflows/troubleshooting.yml @@ -1,12 +1,11 @@ name: troubleshooting -on: pull_request +on: workflow_dispatch jobs: build: runs-on: ${{ matrix.os }} continue-on-error: false - if: "!contains(github.event.head_commit.message, '[skip ci]')" strategy: matrix: #os: [ ubuntu-latest, windows-latest, macos-latest ] From 9d531cba4d269c8b72cec708eaf31e668cae2547 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 11:39:30 +0100 Subject: [PATCH 27/52] Enable pull-requests workflow --- .github/workflows/pull-requests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index e278d7ba3a..cecc0c1c54 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -1,7 +1,6 @@ name: Pull Requests -#on: pull_request -on: workflow_dispatch +on: pull_request jobs: build: From 6e8175407897f8a89d42b88aace74a75d8719a25 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 12:01:28 +0100 Subject: [PATCH 28/52] There are no secrets in pull requests --- .ci/build-pr.sh | 2 -- .ci/inc/regression-tester.inc | 5 +++++ .github/workflows/pull-requests.yml | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.ci/build-pr.sh b/.ci/build-pr.sh index 084974bd05..14a39f9fd5 100755 --- a/.ci/build-pr.sh +++ b/.ci/build-pr.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash source $(dirname $0)/inc/logger.inc -source $(dirname $0)/inc/setup-secrets.inc source $(dirname $0)/inc/regression-tester.inc source $(dirname $0)/inc/maven-dependencies.inc source ${HOME}/java.env @@ -17,7 +16,6 @@ maven_dependencies_resolve case "$(uname)" in Linux*) log_info "Executing danger..." - pmd_ci_setup_env regression_tester_setup_ci regression_tester_executeDanger ;; diff --git a/.ci/inc/regression-tester.inc b/.ci/inc/regression-tester.inc index 9b66372071..1cd532716a 100644 --- a/.ci/inc/regression-tester.inc +++ b/.ci/inc/regression-tester.inc @@ -17,6 +17,11 @@ function regression_tester_setup_ci() { gem install bundler fi + # Token for danger to add comments to PRs + export DANGER_GITHUB_API_TOKEN=f2f658fd5e72410e465104b65805dd7245479666 + # Token for uploading reports to chunk.io + export PMD_CI_CHUNK_TOKEN="pmd:dMvJ1TH3oOl76QHf04EJ" + rm -f .bundle/config bundle config set --local path vendor/bundle bundle config set --local with release_notes_preprocessing diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index cecc0c1c54..b4ddafff2a 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -35,6 +35,5 @@ jobs: shell: bash env: MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 - PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} PMD_CI_BRANCH: ${{ github.ref }} PMD_CI_PULL_REQUEST_NUMBER: ${{ github.event.number }} From be07f5b8c81bf30aab4ad2ecfa944515c29e6edb Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 12:01:53 +0100 Subject: [PATCH 29/52] Enable windows build again --- .github/workflows/pull-requests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index b4ddafff2a..80fbad8ea6 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -10,8 +10,7 @@ jobs: if: "!contains(github.event.head_commit.message, '[skip ci]')" strategy: matrix: - #os: [ ubuntu-latest, windows-latest, macos-latest ] - os: [ ubuntu-latest, macos-latest ] + os: [ ubuntu-latest, windows-latest, macos-latest ] steps: - uses: actions/checkout@v2 From 7e35165c01fa259113a1b641c3c5681b86579cfe Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 14:27:26 +0100 Subject: [PATCH 30/52] Externalize tokens for Danger, use correct base branch ref --- .ci/build.sh | 2 +- .ci/files/maven-settings.xml | 2 +- .ci/files/{env.gpg => private-env.gpg} | Bin .ci/files/public-env.gpg | Bin 0 -> 365 bytes .ci/inc/regression-tester.inc | 20 ++++++++------------ .ci/inc/setup-secrets.inc | 4 ++-- .github/workflows/pull-requests.yml | 2 +- 7 files changed, 13 insertions(+), 17 deletions(-) rename .ci/files/{env.gpg => private-env.gpg} (100%) create mode 100644 .ci/files/public-env.gpg diff --git a/.ci/build.sh b/.ci/build.sh index f7ced21cec..e4b8169a07 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -12,7 +12,7 @@ set -e #set -x function pmd_ci_build_main() { - pmd_ci_setup_env + pmd_ci_setup_private_env pmd_ci_setup_gpg_key pmd_ci_setup_ssh diff --git a/.ci/files/maven-settings.xml b/.ci/files/maven-settings.xml index 7e5fc8f539..a8d61b8f1a 100644 --- a/.ci/files/maven-settings.xml +++ b/.ci/files/maven-settings.xml @@ -12,7 +12,7 @@ ossrh - ${env.CI_DEPLY_USER} + ${env.CI_DEPLOY_USER} ${env.CI_DEPLOY_PASSWORD} diff --git a/.ci/files/env.gpg b/.ci/files/private-env.gpg similarity index 100% rename from .ci/files/env.gpg rename to .ci/files/private-env.gpg diff --git a/.ci/files/public-env.gpg b/.ci/files/public-env.gpg new file mode 100644 index 0000000000000000000000000000000000000000..8cae0c6b6aeddac3ac14268b7bbecd73d3d891fa GIT binary patch literal 365 zcmV-z0h0cV4Fm}T0$RwwO_uU9D*w{Jn*sl}ZilHm;mEMu&J;4aJr)3JVp8_#Y*TcB zI*hN)V3G7&xoKeR_&#mlga5+P_n{Q+Ykxp)QfLFRS!|3X({H|y2mu!FqK^mXVI3xJ zZ3ORd4%kTq2Jz6fV{|yX&a>l!v;>IO=s>X-2D{Cy1|)95YFGO^(Q)pd-hzILUm;)? zVK|~P0wC+?$wctbmD6bN_2Vu;w(7pk5g3oU&x58DIM>;ufwS*#7~ILRZS>;2+$hQ} z{F+Odn%^!@_T5;3z_M{g!MV}^Sgl~&bR~38*)7C7oEu>Ml-VSQ{xJW=>_iw8{D%sm zhTnBDWK7W9_^HVcFEWAe+|X6SM-ym5h<8?9RLu+bDxC0w`H?3Mn1m(ybrjjrw?eW8 zIOAB7ISoIGH!B`aNHD+tsW0BNq)%Dp9+^C8aD5#g0Gk72g6Nf5&LnYIe</dev/null 2>&1 + rm .ci/files/public-env + if hash "bundler" 2>/dev/null; then log_debug "Bundler is already installed" else @@ -17,11 +25,6 @@ function regression_tester_setup_ci() { gem install bundler fi - # Token for danger to add comments to PRs - export DANGER_GITHUB_API_TOKEN=f2f658fd5e72410e465104b65805dd7245479666 - # Token for uploading reports to chunk.io - export PMD_CI_CHUNK_TOKEN="pmd:dMvJ1TH3oOl76QHf04EJ" - rm -f .bundle/config bundle config set --local path vendor/bundle bundle config set --local with release_notes_preprocessing @@ -38,9 +41,6 @@ function regression_tester_uploadBaseline() { log_debug "$FUNCNAME branch=${PMD_CI_BRANCH}" local targetUrl="https://sourceforge.net/projects/pmd/files/pmd-regression-tester/" - PMD_CI_BRANCH=${PMD_CI_BRANCH##refs/heads/} - log_debug "Using modified branch name: ${PMD_CI_BRANCH}" - local errexitstate="$(shopt -po errexit)" set +e # disable errexit ( @@ -94,10 +94,6 @@ function regression_tester_executeDanger() { set -e trap danger_failed ERR - PMD_CI_BRANCH=${PMD_CI_BRANCH##refs/heads/} - export PMD_CI_BRANCH - log_debug "Using modified branch name: ${PMD_CI_BRANCH}" - # Create a corresponding remote branch locally if ! git show-ref --verify --quiet refs/heads/${PMD_CI_BRANCH}; then git fetch --no-tags origin +refs/heads/${PMD_CI_BRANCH}:refs/remotes/origin/${PMD_CI_BRANCH} diff --git a/.ci/inc/setup-secrets.inc b/.ci/inc/setup-secrets.inc index 849e0f9acd..6cd3ba953b 100644 --- a/.ci/inc/setup-secrets.inc +++ b/.ci/inc/setup-secrets.inc @@ -1,6 +1,6 @@ -function pmd_ci_setup_env() { +function pmd_ci_setup_private_env() { log_info "Setting up secrets as environment variables..." - local -r ENV_FILE=.ci/files/env + local -r ENV_FILE=.ci/files/private-env printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \ --passphrase-fd 0 \ diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 80fbad8ea6..2b73e3e647 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -34,5 +34,5 @@ jobs: shell: bash env: MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 - PMD_CI_BRANCH: ${{ github.ref }} + PMD_CI_BRANCH: ${{ github.event.base.ref }} PMD_CI_PULL_REQUEST_NUMBER: ${{ github.event.number }} From 9877ea7f96afd3a71b5e8d645eda4242ab46b47b Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 14:44:39 +0100 Subject: [PATCH 31/52] Add logging groups --- .ci/build-pr.sh | 6 +++++- .ci/inc/logger.inc | 9 +++++++++ .ci/inc/maven-dependencies.inc | 8 ++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.ci/build-pr.sh b/.ci/build-pr.sh index 14a39f9fd5..8aaea771d1 100755 --- a/.ci/build-pr.sh +++ b/.ci/build-pr.sh @@ -10,13 +10,17 @@ set -e maven_dependencies_resolve +log_group_start "Building with maven" ./mvnw -e -V clean verify +log_group_end + # Danger is executed only on the linux runner case "$(uname)" in Linux*) - log_info "Executing danger..." + log_group_start "Executing danger" regression_tester_setup_ci regression_tester_executeDanger + log_group_end ;; esac diff --git a/.ci/inc/logger.inc b/.ci/inc/logger.inc index 0ea7231b60..0ed1f96ce3 100644 --- a/.ci/inc/logger.inc +++ b/.ci/inc/logger.inc @@ -19,3 +19,12 @@ function log_debug() { #true echo -e "[DEBUG ] $*" } + +function log_group_start() { + echo "::group::$*" + log_info $* +} + +function log_group_end() { + echo "::endgroup::" +} \ No newline at end of file diff --git a/.ci/inc/maven-dependencies.inc b/.ci/inc/maven-dependencies.inc index 2c48bc39a4..64afe8b95c 100644 --- a/.ci/inc/maven-dependencies.inc +++ b/.ci/inc/maven-dependencies.inc @@ -1,4 +1,8 @@ +# +# needs "inc/logger.inc" +# + # # On azure, outgoing idle connection are dropped after 4 minutes. # Usually, you can configure wagon with ttl. But these settings are @@ -13,6 +17,8 @@ # execpt for dokka-maven-plugin, as it does not play well with dependency-plugin. # function maven_dependencies_resolve() { + log_group_start "Downloading maven dependencies" + dokka_version=$(./mvnw -q -Dexec.executable="echo" -Dexec.args='${dokka.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) ./mvnw dependency:resolve @@ -22,4 +28,6 @@ function maven_dependencies_resolve() { -Dpackaging=jar \ -DremoteRepositories=jcenter::default::https://jcenter.bintray.com/ ./mvnw dependency:resolve-plugins -DexcludeGroupIds=org.jetbrains.dokka + + log_group_end } From 412ddb3018f57995fa559f13441577cc098d5c52 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 14:58:57 +0100 Subject: [PATCH 32/52] Fix pull requests build - correctly extract data from event --- .github/workflows/pull-requests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 2b73e3e647..c8199cd8b2 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -34,5 +34,5 @@ jobs: shell: bash env: MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 - PMD_CI_BRANCH: ${{ github.event.base.ref }} - PMD_CI_PULL_REQUEST_NUMBER: ${{ github.event.number }} + PMD_CI_BRANCH: ${{ github.event.pull_request.base.ref }} + PMD_CI_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} From e8525008b2376dbd24d516a02a6cb54c758cb76b Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 15:35:16 +0100 Subject: [PATCH 33/52] Move install-openjdk into main script, add check-environment --- .ci/build-pr.sh | 3 +- .ci/build.sh | 2 +- .ci/inc/install-openjdk.inc | 83 +++++++++++++++++++++++++++++ .ci/inc/regression-tester.inc | 5 +- .ci/install-openjdk.sh | 82 ---------------------------- .github/workflows/pull-requests.yml | 4 +- .github/workflows/pushes.yml | 3 -- 7 files changed, 91 insertions(+), 91 deletions(-) create mode 100644 .ci/inc/install-openjdk.inc delete mode 100755 .ci/install-openjdk.sh diff --git a/.ci/build-pr.sh b/.ci/build-pr.sh index 8aaea771d1..38ea0c6f41 100755 --- a/.ci/build-pr.sh +++ b/.ci/build-pr.sh @@ -1,13 +1,14 @@ #!/usr/bin/env bash source $(dirname $0)/inc/logger.inc +source $(dirname $0)/inc/install-openjdk.inc source $(dirname $0)/inc/regression-tester.inc source $(dirname $0)/inc/maven-dependencies.inc -source ${HOME}/java.env set -e #set -x +install_openjdk_setdefault 11 maven_dependencies_resolve log_group_start "Building with maven" diff --git a/.ci/build.sh b/.ci/build.sh index e4b8169a07..6b68291bc1 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -6,7 +6,6 @@ source $(dirname $0)/inc/sourceforge-api.inc source $(dirname $0)/inc/pmd-doc.inc source $(dirname $0)/inc/pmd-code-api.inc source $(dirname $0)/inc/regression-tester.inc -source ${HOME}/java.env set -e #set -x @@ -16,6 +15,7 @@ function pmd_ci_build_main() { pmd_ci_setup_gpg_key pmd_ci_setup_ssh + install_openjdk_setdefault 11 pmd_ci_build_setup_maven pmd_ci_build_setup_oraclejdk7 pmd_ci_build_setup_bundler diff --git a/.ci/inc/install-openjdk.inc b/.ci/inc/install-openjdk.inc new file mode 100644 index 0000000000..2a69ba6382 --- /dev/null +++ b/.ci/inc/install-openjdk.inc @@ -0,0 +1,83 @@ + +# needs: +# inc/logger + +# +# Downloads openjdk from AdoptOpenJDK by accessing the API. +# The API is documented at https://api.adoptopenjdk.net/swagger-ui/ +# + +function install_openjdk() { + OPENJDK_VERSION=$1 + log_group_start "Installing OpenJDK ${OPENJDK_VERSION}" + + case "$(uname)" in + Linux*) + JDK_OS=linux + COMPONENTS_TO_STRIP=1 # e.g. openjdk-11.0.3+7/bin/java + ;; + Darwin*) + JDK_OS=mac + COMPONENTS_TO_STRIP=3 # e.g. jdk-11.0.3+7/Contents/Home/bin/java + ;; + CYGWIN*|MINGW*) + JDK_OS=windows + ;; + *) + log_error "Unknown OS: $(uname)" + exit 1 + ;; + esac + + + DOWNLOAD_URL=$(curl --silent -X GET "https://api.adoptopenjdk.net/v3/assets/feature_releases/${OPENJDK_VERSION}/ga?architecture=x64&heap_size=normal&image_type=jdk&jvm_impl=hotspot&os=${JDK_OS}&page=0&page_size=1&project=jdk&sort_method=DEFAULT&sort_order=DESC&vendor=adoptopenjdk" \ + -H "accept: application/json" \ + | jq -r ".[0].binaries[0].package.link") + + OPENJDK_ARCHIVE=$(basename ${DOWNLOAD_URL}) + log_debug "Archive name: ${OPENJDK_ARCHIVE}" + + CACHE_DIR=${HOME}/.cache/openjdk + TARGET_DIR=${HOME}/openjdk${OPENJDK_VERSION} + + mkdir -p ${CACHE_DIR} + mkdir -p ${TARGET_DIR} + + if [ ! -e ${CACHE_DIR}/${OPENJDK_ARCHIVE} ]; then + log_info "Downloading from ${DOWNLOAD_URL} to ${CACHE_DIR}" + curl --location --output ${CACHE_DIR}/${OPENJDK_ARCHIVE} "${DOWNLOAD_URL}" + else + log_info "Skipped download, file ${CACHE_DIR}/${OPENJDK_ARCHIVE} already exists" + fi + + log_info "Extracting to ${TARGET_DIR}" + + case "$OPENJDK_ARCHIVE" in + *.zip) + 7z x ${CACHE_DIR}/${OPENJDK_ARCHIVE} -o${TARGET_DIR} + mv ${TARGET_DIR}/*/* ${TARGET_DIR}/ + ;; + *.tar.gz) + tar --extract --file ${CACHE_DIR}/${OPENJDK_ARCHIVE} -C ${TARGET_DIR} --strip-components=${COMPONENTS_TO_STRIP} + ;; + *) + log_error "Unknown filetype: ${OPENJDK_ARCHIVE}" + exit 1 + ;; + esac + + log_group_end +} + +function install_openjdk_setdefault() { + OPENJDK_VERSION=$1 + + install_openjdk $OPENJDK_VERSION + + log_info "Using OpenJDK ${OPENJDK_VERSION} as default" + TARGET_DIR=${HOME}/openjdk${OPENJDK_VERSION} + export JAVA_HOME="${TARGET_DIR}" + export PATH="${TARGET_DIR}/bin:${PATH}" + + java -version +} \ No newline at end of file diff --git a/.ci/inc/regression-tester.inc b/.ci/inc/regression-tester.inc index cb854ab421..2ac9290fb9 100644 --- a/.ci/inc/regression-tester.inc +++ b/.ci/inc/regression-tester.inc @@ -1,6 +1,7 @@ # # The functions here require the following scripts: -# inc/logger +# inc/logger.inc +# inc/install-openjdk.inc # # The functions here require the following environment variables: # PMD_SF_USER @@ -11,7 +12,7 @@ function regression_tester_setup_ci() { # install openjdk8 for pmd-regression-tests - .ci/install-openjdk.sh 8 + install_openjdk 8 gpg --batch --yes --decrypt --passphrase="GnxdjywUEPveyCD1RLiTd7t8CImnefYr" \ --output .ci/files/public-env .ci/files/public-env.gpg diff --git a/.ci/install-openjdk.sh b/.ci/install-openjdk.sh deleted file mode 100755 index e1359675a2..0000000000 --- a/.ci/install-openjdk.sh +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env bash - -# -# Downloads openjdk from AdoptOpenJDK by accessing the API. -# The API is documented at https://api.adoptopenjdk.net/swagger-ui/ -# - -source $(dirname $0)/inc/logger.inc - -set -e - -case "$(uname)" in - Linux*) - JDK_OS=linux - COMPONENTS_TO_STRIP=1 # e.g. openjdk-11.0.3+7/bin/java - ;; - Darwin*) - JDK_OS=mac - COMPONENTS_TO_STRIP=3 # e.g. jdk-11.0.3+7/Contents/Home/bin/java - ;; - CYGWIN*|MINGW*) - JDK_OS=windows - ;; - *) - log_error "Unknown OS: $(uname)" - exit 1 - ;; -esac - - -OPENJDK_VERSION=$1 -DOWNLOAD_URL=$(curl --silent -X GET "https://api.adoptopenjdk.net/v3/assets/feature_releases/${OPENJDK_VERSION}/ga?architecture=x64&heap_size=normal&image_type=jdk&jvm_impl=hotspot&os=${JDK_OS}&page=0&page_size=1&project=jdk&sort_method=DEFAULT&sort_order=DESC&vendor=adoptopenjdk" \ - -H "accept: application/json" \ - | jq -r ".[0].binaries[0].package.link") - -OPENJDK_ARCHIVE=$(basename ${DOWNLOAD_URL}) -log_debug "Archive name: ${OPENJDK_ARCHIVE}" - -CACHE_DIR=${HOME}/.cache/openjdk -TARGET_DIR=${HOME}/openjdk${OPENJDK_VERSION} - -mkdir -p ${CACHE_DIR} -mkdir -p ${TARGET_DIR} - -if [ ! -e ${CACHE_DIR}/${OPENJDK_ARCHIVE} ]; then - log_info "Downloading from ${DOWNLOAD_URL} to ${CACHE_DIR}" - curl --location --output ${CACHE_DIR}/${OPENJDK_ARCHIVE} "${DOWNLOAD_URL}" -else - log_info "Skipped download, file ${CACHE_DIR}/${OPENJDK_ARCHIVE} already exists" -fi - -log_info "Extracting to ${TARGET_DIR}" - -case "$OPENJDK_ARCHIVE" in - *.zip) - 7z x ${CACHE_DIR}/${OPENJDK_ARCHIVE} -o${TARGET_DIR} - mv ${TARGET_DIR}/*/* ${TARGET_DIR}/ - ;; - *.tar.gz) - tar --extract --file ${CACHE_DIR}/${OPENJDK_ARCHIVE} -C ${TARGET_DIR} --strip-components=${COMPONENTS_TO_STRIP} - ;; - *) - log_error "Unknown filetype: ${OPENJDK_ARCHIVE}" - exit 1 - ;; -esac - -if [ ! -e ${HOME}/java.env ]; then - cat > ${HOME}/java.env < Date: Fri, 13 Nov 2020 15:56:05 +0100 Subject: [PATCH 34/52] Increase fetch depth so that danger finds the HEAD commit of the PR --- .ci/inc/regression-tester.inc | 2 +- .github/workflows/pull-requests.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.ci/inc/regression-tester.inc b/.ci/inc/regression-tester.inc index 2ac9290fb9..555252790f 100644 --- a/.ci/inc/regression-tester.inc +++ b/.ci/inc/regression-tester.inc @@ -97,7 +97,7 @@ function regression_tester_executeDanger() { # Create a corresponding remote branch locally if ! git show-ref --verify --quiet refs/heads/${PMD_CI_BRANCH}; then - git fetch --no-tags origin +refs/heads/${PMD_CI_BRANCH}:refs/remotes/origin/${PMD_CI_BRANCH} + git fetch --no-tags --depth=5 origin +refs/heads/${PMD_CI_BRANCH}:refs/remotes/origin/${PMD_CI_BRANCH} git branch ${PMD_CI_BRANCH} origin/${PMD_CI_BRANCH} log_debug "Created local branch ${PMD_CI_BRANCH}" fi diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index 88fcab5f7a..dd32ee7856 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -14,6 +14,8 @@ jobs: steps: - uses: actions/checkout@v2 + with: + fetch-depth: 5 - uses: actions/cache@v2 with: path: | From 825e3426d7593e771d62ce6fe0cdff5151204d7f Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 17:15:53 +0100 Subject: [PATCH 35/52] Fetch more commits of the PR for danger --- .ci/inc/regression-tester.inc | 4 +++- .github/workflows/pull-requests.yml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/inc/regression-tester.inc b/.ci/inc/regression-tester.inc index 555252790f..abbd15586f 100644 --- a/.ci/inc/regression-tester.inc +++ b/.ci/inc/regression-tester.inc @@ -97,10 +97,12 @@ function regression_tester_executeDanger() { # Create a corresponding remote branch locally if ! git show-ref --verify --quiet refs/heads/${PMD_CI_BRANCH}; then - git fetch --no-tags --depth=5 origin +refs/heads/${PMD_CI_BRANCH}:refs/remotes/origin/${PMD_CI_BRANCH} + git fetch --no-tags --depth=1 origin +refs/heads/${PMD_CI_BRANCH}:refs/remotes/origin/${PMD_CI_BRANCH} git branch ${PMD_CI_BRANCH} origin/${PMD_CI_BRANCH} log_debug "Created local branch ${PMD_CI_BRANCH}" fi + # Fetch more commits of the PR for danger + git fetch --no-tags --depth=50 origin +$(git rev-parse HEAD^2): log_info "Running danger on branch ${PMD_CI_BRANCH}" bundle exec danger --verbose diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index dd32ee7856..bfa9e6eea3 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v2 with: - fetch-depth: 5 + fetch-depth: 2 - uses: actions/cache@v2 with: path: | From 8c087e9357a052b23267fc947078d037e7254ba8 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 19:43:55 +0100 Subject: [PATCH 36/52] Describe workaround for failing downloads --- .ci/README.md | 45 ++++++++++++++++++--------- .github/workflows/troubleshooting.yml | 4 ++- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index 0d4a7b5627..a84fbe0a4a 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -27,21 +27,33 @@ for a ssh key, which is used to copy files to sourceforge. ## Known Issues -Intermittent build failures while downloading dependencies from maven central. -Root issue seems to be SNAT configs in Azure, which closes long running TCP connections -only on one side: https://docs.microsoft.com/en-us/azure/load-balancer/troubleshoot-outbound-connection#idletimeout -The default timeout is 4 minutes. +### Intermittent connection resets or timeouts while downloading dependencies from maven central -Workaround as described in https://github.com/actions/virtual-environments/issues/1499 and -https://issues.apache.org/jira/browse/WAGON-545 is applied: +Root issue seems to be SNAT Configs in Azure, which closes long running [idle TCP connections +after 4 minutes](https://docs.microsoft.com/en-us/azure/load-balancer/troubleshoot-outbound-connection#idletimeout). + +The workaround is described in [actions/virtual-environments#1499](https://github.com/actions/virtual-environments/issues/1499) +and [WAGON-545](https://issues.apache.org/jira/browse/WAGON-545) +and [WAGON-486](https://issues.apache.org/jira/browse/WAGON-486): The setting `-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3` -doesn't seem to work. +makes sure, that Maven doesn't try to use pooled connections that have been unused for more than 180 seconds. +These settings are placed as environment variable `MAVEN_OPTS` in all workflows, so that they are active for +all Maven executions (including builds done by regression tester). -Now we disable pooling completely, so that for downloading a artifact/dependency, always new, fresh -connections are used: `-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false`. +Alternatively, pooling could be disabled completely via `-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false`. +This has the consequence, that for each dependency, that is being downloaded, a new https connection is +established. -Not working either. +More information about configuring this can be found at [wagon-http](https://maven.apache.org/wagon/wagon-providers/wagon-http/). + +However, this doesn't work when [dokka-maven-plugin](https://github.com/Kotlin/dokka) is used: This plugin +downloads additional dokka plugins at runtime and reconfigures somehow Maven. After this plugin is loaded, +the above system properties have no effect anymore. +See [dokka/dokka-maven-plugin#1625](https://github.com/Kotlin/dokka/issues/1625) and +[dokka/dokka-maven-plugin#1626](https://github.com/Kotlin/dokka/issues/1626). + +The workaround now in place is, to download all the dependencies first, see `inc/maven-dependencies.inc`. ## Hints @@ -56,10 +68,12 @@ Just add the following step into the job: uses: mxschmitt/action-tmate@v3 ``` -**Note**: This is dangerous for push/pull builds, because these have access to the secrets and the SSH session -is not protected... +The workflow `troubleshooting` can be started manually, which already contains the tmate action. -### Local tests +**Note**: This is dangerous for push/pull builds on pmd/pmd, because these have access to the secrets and the SSH session +is not protected. Builds triggered by pull requests from forked repositories don't have access to the secrets. + +### Local tests with docker Create a local docker container: @@ -68,8 +82,8 @@ cd .ci/docker_ubuntu18.04 docker build -t pmd-ci . ``` -This container is based on Ubuntu 18.04, which is used for `ubuntu-latest` github actions runner -(see ). +This container is based on Ubuntu 18.04, which is used for `ubuntu-latest` github actions runner, +see [Virtual Environment](https://github.com/actions/virtual-environments). You can run a local instance with docker and mount your local pmd checkout into the container: @@ -82,4 +96,5 @@ You'll be dropped into a bash. Start e.g. with ``` cd workspaces/pmd/pmd .ci/check-environment.sh +.ci/build-pr.sh ``` diff --git a/.github/workflows/troubleshooting.yml b/.github/workflows/troubleshooting.yml index dcdeeb9338..a8c1f299a0 100644 --- a/.github/workflows/troubleshooting.yml +++ b/.github/workflows/troubleshooting.yml @@ -26,7 +26,9 @@ jobs: with: ruby-version: 2.7 - name: Install OpenJDK 11 - run: .ci/install-openjdk.sh 11 + run: | + source .ci/inc/install-openjdk.inc + install_openjdk 11 shell: bash - name: Setup tmate session uses: mxschmitt/action-tmate@v3 From e2514c546c8e894ab1f151501d4b9c1066540a03 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 20:34:37 +0100 Subject: [PATCH 37/52] Implement release builds --- .ci/build-pr.sh | 16 +- .ci/build.sh | 251 ++++++++++++++++++++++-------- .ci/docker_ubuntu18.04/Dockerfile | 2 +- .ci/inc/github-releases-api.inc | 214 +++++++++++++++++++++++++ .ci/inc/install-openjdk.inc | 3 - .ci/inc/maven-dependencies.inc | 9 +- .ci/inc/pmd-doc.inc | 2 +- .ci/inc/regression-tester.inc | 2 +- .ci/render_release_notes.rb | 44 ++++++ .github/workflows/pushes.yml | 5 +- .github/workflows/releases.yml | 11 +- docs/sitemap_generator.sh | 56 +++++++ 12 files changed, 528 insertions(+), 87 deletions(-) create mode 100644 .ci/inc/github-releases-api.inc create mode 100755 .ci/render_release_notes.rb create mode 100755 docs/sitemap_generator.sh diff --git a/.ci/build-pr.sh b/.ci/build-pr.sh index 38ea0c6f41..dad6e7ae95 100755 --- a/.ci/build-pr.sh +++ b/.ci/build-pr.sh @@ -6,13 +6,17 @@ source $(dirname $0)/inc/regression-tester.inc source $(dirname $0)/inc/maven-dependencies.inc set -e -#set -x -install_openjdk_setdefault 11 -maven_dependencies_resolve +log_group_start "Installing OpenJDK 11" + install_openjdk_setdefault 11 +log_group_end + +log_group_start "Downloading maven dependencies" + maven_dependencies_resolve +log_group_end log_group_start "Building with maven" -./mvnw -e -V clean verify + ./mvnw -e -V clean verify -Pgenerate-rule-docs log_group_end @@ -20,8 +24,8 @@ log_group_end case "$(uname)" in Linux*) log_group_start "Executing danger" - regression_tester_setup_ci - regression_tester_executeDanger + regression_tester_setup_ci + regression_tester_executeDanger log_group_end ;; esac diff --git a/.ci/build.sh b/.ci/build.sh index 6b68291bc1..715afb02f7 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -6,99 +6,226 @@ source $(dirname $0)/inc/sourceforge-api.inc source $(dirname $0)/inc/pmd-doc.inc source $(dirname $0)/inc/pmd-code-api.inc source $(dirname $0)/inc/regression-tester.inc +source $(dirname $0)/inc/github-releases-api.inc +source $(dirname $0)/inc/maven-dependencies.inc set -e -#set -x function pmd_ci_build_main() { - pmd_ci_setup_private_env - pmd_ci_setup_gpg_key - pmd_ci_setup_ssh + log_group_start "Setting up private secrets" + pmd_ci_setup_private_env + pmd_ci_setup_gpg_key + pmd_ci_setup_ssh + log_group_end - install_openjdk_setdefault 11 - pmd_ci_build_setup_maven - pmd_ci_build_setup_oraclejdk7 - pmd_ci_build_setup_bundler + log_group_start "Prepare Java 11, Maven, Bundler" + install_openjdk_setdefault 11 + pmd_ci_build_setup_maven + pmd_ci_build_setup_oraclejdk7 + pmd_ci_build_setup_bundler + pmd_ci_build_setup_env + log_group_end - VERSION=$(pmd_ci_build_get_pom_version) - log_info "Building PMD ${VERSION}..." + log_group_start "Downloading maven dependencies" + maven_dependencies_resolve + log_group_end - pmd_ci_build_run + log_group_start "Build and Deploy" + pmd_ci_build_run + pmd_ci_deploy_build_artifacts + log_group_end - # Deploy to sourceforge files - sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip" - sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip" + log_group_start "Build and Upload documentation" + pmd_ci_build_and_upload_doc + log_group_end - pmd_ci_build_and_upload_doc + if pmd_ci_build_isRelease; then + log_group_start "Publishing Release" + gh_release_publishRelease "$GH_RELEASE" + sourceforge_selectDefault "${VERSION}" + log_group_end + fi - regression_tester_setup_ci - regression_tester_uploadBaseline + log_group_start "Creating new baseline for regression tester" + regression_tester_setup_ci + regression_tester_uploadBaseline + log_group_end exit 0 } + +# +# Configures maven. +# Needed for deploy to central (both snapshots and releases) +# and for signing the artifacts. +# +function pmd_ci_build_setup_maven() { + mkdir -p ${HOME}/.m2 + cp .ci/files/maven-settings.xml ${HOME}/.m2/settings.xml +} + +# +# Installs jdk7 for integration test +# +function pmd_ci_build_setup_oraclejdk7() { + local local_dir="${HOME}/.cache/jdk7" + local target_dir="${HOME}/oraclejdk7" + local download_url="https://pmd-code.org/oraclejdk/jdk-7u80-linux-x64.tar.gz" + local archive=$(basename $download_url) + + mkdir -p ${local_dir} + mkdir -p ${target_dir} + if [ ! -e ${local_dir}/${archive} ]; then + log_info "Downloading from ${download_url} to ${local_dir}" + curl --location --output ${local_dir}/${archive} ${download_url} + else + log_info "Skipped download, file ${local_dir}/${archive} already exists" + fi + log_info "Extracting to ${target_dir}" + tar --extract --file ${local_dir}/${archive} -C ${target_dir} --strip-components=1 + + log_info "OracleJDK7 can be used via -Djava7.home=${HOME}/oraclejdk7" +} + +# +# Installs bundler, which is needed for doc generation and regression tester +# +function pmd_ci_build_setup_bundler() { + log_info "Installing bundler..." + gem install bundler +} + +# +# Setups common build parameters: +# * Determines the VERSION of PMD, that is being built +# * Determines the PMD_CI_BRANCH or PMD_CI_TAG, that is being built +# +function pmd_ci_build_setup_env() { + VERSION=$(pmd_ci_build_get_pom_version) + + if [[ "${PMD_CI_GIT_REF}" == refs/heads/* ]]; then + PMD_CI_BRANCH=${PMD_CI_GIT_REF##refs/heads/} + unset PMD_CI_TAG + log_info "Building PMD ${VERSION} on branch ${PMD_CI_BRANCH}" + elif [[ "${PMD_CI_GIT_REF}" == refs/tags/* ]]; then + unset PMD_CI_BRANCH + PMD_CI_TAG=${PMD_CI_GIT_REF##refs/tags/} + log_info "Building PMD ${VERSION} on tag ${PMD_CI_TAG}" + else + log_error "Unknown branch/tag: PMD_CI_GIT_REF=${PMD_CI_GIT_REF}" + exit 1 + fi + + if [[ "${VERSION}" == *-SNAPSHOT && -z "$PMD_CI_BRANCH" ]]; then + log_error "Invalid combination: snapshot version ${VERSION} but no branch in PMD_CI_GIT_REF=${PMD_CI_GIT_REF}" + exit 1 + fi + + if [[ "${VERSION}" != *-SNAPSHOT && -z "$PMD_CI_TAG" ]]; then + log_error "Invalid combination: non-snapshot version ${VERSION} but no tag in PMD_CI_GIT_REF=${PMD_CI_GIT_REF}" + exit 1 + fi +} + +# +# Performs the actual build. +# Deploys the artifacts to maven central. +# Also generates rule documentation. +# +function pmd_ci_build_run() { + local mvn_profiles="ossrh,sign,generate-rule-docs" + + if pmd_ci_build_isRelease; then + log_info "This is a release build" + mvn_profiles="${mvn_profiles},pmd-release" + else + log_info "This is a snapshot build" + fi + + ./mvnw clean deploy -P${mvn_profiles} -e -V -Djava7.home=${HOME}/oraclejdk7 +} + +# +# Deploys the binary distribution +# +function pmd_ci_deploy_build_artifacts() { + # Deploy to sourceforge files + sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip" + sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip" + + if pmd_ci_build_isRelease; then + # create a draft github release + gh_releases_createDraftRelease "${PMD_CI_TAG}" "$(git rev-list -n 1 ${PMD_CI_TAG})" + GH_RELEASE="$RESULT" + + # Deploy to github releases + gh_release_uploadAsset "$GH_RELEASE" "pmd-dist/target/pmd-bin-${VERSION}.zip" + gh_release_uploadAsset "$GH_RELEASE" "pmd-dist/target/pmd-src-${VERSION}.zip" + fi +} + +# +# Builds and uploads the documentation site +# function pmd_ci_build_and_upload_doc() { pmd_doc_generate_jekyll_site pmd_doc_create_archive sourceforge_uploadFile "${VERSION}" "docs/pmd-doc-${VERSION}.zip" + if pmd_ci_build_isRelease; then + gh_release_uploadAsset "$GH_RELEASE" "docs/pmd-doc-${VERSION}.zip" + fi # Deploy doc to https://docs.pmd-code.org/pmd-doc-${VERSION}/ pmd_code_uploadDocumentation "${VERSION}" "docs/pmd-doc-${VERSION}.zip" # Deploy javadoc to https://docs.pmd-code.org/apidocs/*/${VERSION}/ pmd_code_uploadJavadoc "${VERSION}" "$(pwd)" - pmd_code_createSymlink "${VERSION}" "snapshot" + if [[ "${VERSION}" == *-SNAPSHOT && "${PMD_CI_BRANCH}" == "master" ]]; then + # only for snapshot builds from branch master + pmd_code_createSymlink "${VERSION}" "snapshot" - # update github pages https://pmd.github.io/pmd/ - pmd_doc_publish_to_github_pages - # rsync site to https://pmd.sourceforge.io/snapshot - sourceforge_rsyncSnapshotDocumentation "${VERSION}" "snapshot" + # update github pages https://pmd.github.io/pmd/ + pmd_doc_publish_to_github_pages + # rsync site to https://pmd.sourceforge.io/snapshot + sourceforge_rsyncSnapshotDocumentation "${VERSION}" "snapshot" + fi + + if pmd_ci_build_isRelease; then + # documentation is already uploaded to https://docs.pmd-code.org/pmd-doc-${VERSION} + # we only need to setup symlinks for the released version + pmd_code_createSymlink "${VERSION}" "latest" + # remove old doc and point to the new version + pmd_code_removeDocumentation "${VERSION}-SNAPSHOT" + pmd_code_createSymlink "${VERSION}" "${VERSION}-SNAPSHOT" + # remove old javadoc + pmd_code_removeJavadoc "${VERSION}-SNAPSHOT" + + # updating github release text + # renders, and skips the first 6 lines - the Jekyll front-matter + local rendered_release_notes=$(bundle exec .ci/render_release_notes.rb docs/pages/release_notes.md | tail -n +6) + local release_name="PMD ${VERSION} ($(date -u +%d-%B-%Y))" + gh_release_updateRelease "$GH_RELEASE" "$release_name" "$rendered_release_notes" + sourceforge_uploadReleaseNotes "${VERSION}" "${rendered_release_notes}" + + publish_release_documentation_github + sourceforge_rsyncSnapshotDocumentation "${VERSION}" "pmd-${VERSION}" + fi +} + + +function pmd_ci_build_isRelease() { + if [[ "${VERSION}" != *-SNAPSHOT && -n "${PMD_CI_TAG}" && -z "${PMD_CI_BRANCH}" ]]; then + return 0 + else + return 1 + fi } function pmd_ci_build_get_pom_version() { echo $(./mvnw -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) } -function pmd_ci_build_setup_maven() { - # configure maven - mkdir -p ${HOME}/.m2 - cp .ci/files/maven-settings.xml ${HOME}/.m2/settings.xml -} - -function pmd_ci_build_setup_oraclejdk7() { - # install jdk7 for integration test - - LOCAL_DIR="${HOME}/.cache/jdk7" - TARGET_DIR="${HOME}/oraclejdk7" - DOWNLOAD_URL="https://pmd-code.org/oraclejdk/jdk-7u80-linux-x64.tar.gz" - ARCHIVE=$(basename $DOWNLOAD_URL) - - mkdir -p ${LOCAL_DIR} - mkdir -p ${TARGET_DIR} - if [ ! -e ${LOCAL_DIR}/${ARCHIVE} ]; then - log_info "Downloading from ${DOWNLOAD_URL} to ${LOCAL_DIR}" - curl --location --output ${LOCAL_DIR}/${ARCHIVE} ${DOWNLOAD_URL} - else - log_info "Skipped download, file ${LOCAL_DIR}/${ARCHIVE} already exists" - fi - log_info "Extracting to ${TARGET_DIR}" - tar --extract --file ${LOCAL_DIR}/${ARCHIVE} -C ${TARGET_DIR} --strip-components=1 - - log_info "OracleJDK7 can be used via -Djava7.home=${HOME}/oraclejdk7" -} - -function pmd_ci_build_run() { - log_info "This is a snapshot build" - #export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" - #export MAVEN_OPTS="-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false" - ./mvnw deploy -Possrh,sign,generate-rule-docs -e -V -Djava7.home=${HOME}/oraclejdk7 -} - -# Needed for doc generation and regression tester -function pmd_ci_build_setup_bundler() { - log_info "Installing bundler..." - gem install bundler -} pmd_ci_build_main diff --git a/.ci/docker_ubuntu18.04/Dockerfile b/.ci/docker_ubuntu18.04/Dockerfile index 6999125846..ff76744c38 100644 --- a/.ci/docker_ubuntu18.04/Dockerfile +++ b/.ci/docker_ubuntu18.04/Dockerfile @@ -23,7 +23,7 @@ ENV GITHUB_EVENT_PATH=/workspaces/event.json ENV GITHUB_REPOSITORY=pmd/pmd ENV GITHUB_ACTION=run1 ENV GITHUB_EVENT_NAME=pull_request -ENV PMD_CI_BRANCH=refs/heads/master +ENV PMD_CI_BRANCH=master ENV PMD_CI_PULL_REQUEST_NUMBER=2913 COPY create-gh-pull-request-event.sh . COPY install-ruby.sh . diff --git a/.ci/inc/github-releases-api.inc b/.ci/inc/github-releases-api.inc new file mode 100644 index 0000000000..e71e67f941 --- /dev/null +++ b/.ci/inc/github-releases-api.inc @@ -0,0 +1,214 @@ +# +# The functions here require the following scripts: +# logger.inc +# +# The functions here require the following environment variables: +# GITHUB_OAUTH_TOKEN +# GITHUB_BASE_URL +# + +# +# Creates a new release on github with the given tag and target_commit. +# The release is draft and not published. +# +# $RESULT = release json string +# +# See: https://developer.github.com/v3/repos/releases/#create-a-release +# +function gh_releases_createDraftRelease() { + local tagName="$1" + local targetCommitish="$2" + + log_debug "$FUNCNAME: Creating new draft release for tag=$tagName and commit=$targetCommitish" + + local request=$(cat <<-EOF + { + "tag_name": "${tagName}", + "target_commitish": "${targetCommitish}", + "name": "${tagName}", + "draft": true + } + EOF + ) + + log_debug "POST $GITHUB_BASE_URL/releases" + log_info "Creating github draft release" + RESULT=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ + -H "Content-Type: application/json" \ + -X POST \ + --data "${request}" \ + "$GITHUB_BASE_URL/releases") + log_debug " -> response: $RESULT" + + log_success "Created draft release with id $(echo $RESULT | jq --raw-output ".url")" +} + +# +# Gets the latest release, if it is a draft and returns with 0. +# Returns with 1, if the latest release is not a draft - meaning, there is no +# draft release (yet?). +# +# RESULT = release json string +# +# See: https://developer.github.com/v3/repos/releases/#list-releases-for-a-repository +# +function gh_releases_getLatestDraftRelease() { + log_debug "$FUNCNAME" + log_debug "GET $GITHUB_BASE_URL/releases?per_page=1" + RESULT=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ + "$GITHUB_BASE_URL/releases?per_page=1" | jq ".[0]") + log_debug " -> response: $RESULT" + local draft=$(echo $RESULT | jq ".draft") + if [ "$draft" != "true" ]; then + RESULT="" + log_error "Could not find draft release!" + return 1 + fi + log_info "Found draft release: $(echo $RESULT | jq --raw-output ".url")" +} + +# +# Deletes a release. +# +# See: https://developer.github.com/v3/repos/releases/#delete-a-release +# +function gh_release_deleteRelease() { + local release="$1" + + gh_release_getIdFromData "$release" + local releaseId="$RESULT" + log_debug "$FUNCNAME id=$releaseId" + log_debug "DELETE $GITHUB_BASE_URL/releases/$releaseId" + log_info "Deleting github release $releaseId" + local response + response=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ + -X DELETE \ + "$GITHUB_BASE_URL/releases/$releaseId") + log_debug " -> response: $response" + log_success "Deleted release with id $releaseId" +} + +# +# Determines the release id from the given JSON release data. +# +# RESULT = "the release id" +# +function gh_release_getIdFromData() { + local release="$1" + + RESULT=$(echo $release | jq --raw-output ".id") +} + +# +# Determines the tag_name from the given JSON release data. +# +# RESULT = "the tag name" +# +function gh_release_getTagNameFromData() { + local release="$1" + + RESULT=$(echo $release | jq --raw-output ".tag_name") +} + +# +# Uploads a asset to an existing release. +# +# See: https://developer.github.com/v3/repos/releases/#upload-a-release-asset +# +function gh_release_uploadAsset() { + local release="$1" + local filename="$2" + local name=$(basename $filename) + + gh_release_getIdFromData "$release" + local releaseId="$RESULT" + log_debug "$FUNCNAME: releaseId=$releaseId file=$filename name=$name" + + local uploadUrl=$(echo "$release" | jq --raw-output ".upload_url") + uploadUrl="${uploadUrl%%\{\?name,label\}}" + uploadUrl="${uploadUrl}?name=${name}" + log_debug "POST $uploadUrl" + log_info "Uploading $filename to github release $releaseId" + local response + response=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ + -H "Content-Type: application/zip" \ + --data-binary "@$filename" \ + -X POST \ + "${uploadUrl}") + log_debug " -> response: $response" + log_success "Uploaded release asset $filename for release $releaseId" +} + +# +# Updates the release info: name and body. +# The body is escaped to fit into JSON, so it is allowed for the body to be +# a multi-line string. +# +# See: https://developer.github.com/v3/repos/releases/#edit-a-release +# +function gh_release_updateRelease() { + local release="$1" + local name="$2" + local body="$3" + + gh_release_getIdFromData "$release" + local releaseId="$RESULT" + gh_release_getTagNameFromData "$release" + local tagName="$RESULT" + log_debug "$FUNCNAME releaseId=$releaseId name=$name tag_name=$tagName" + + body="${body//'\'/\\\\}" + body="${body//$'\r'/}" + body="${body//$'\n'/\\r\\n}" + body="${body//'"'/\\\"}" + + local request=$(cat <<-EOF + { + "tag_name": "${tagName}", + "name": "${name}", + "body": "${body}" + } + EOF + ) + + log_debug "PATCH $GITHUB_BASE_URL/releases/${releaseId}" + log_debug " -> request: $request" + log_info "Updating github release $releaseId" + local response + response=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ + -H "Content-Type: application/json" \ + --data "${request}" \ + -X PATCH \ + "$GITHUB_BASE_URL/releases/${releaseId}") + log_debug " -> response: $response" + log_success "Updated release with id=$releaseId" +} + +# +# Publish a release by setting draft="false". +# Note: This will send out the notification emails if somebody +# watched the releases. +# +# See: https://developer.github.com/v3/repos/releases/#edit-a-release +# +function gh_release_publishRelease() { + local release="$1" + + gh_release_getIdFromData "$release" + local releaseId="$RESULT" + log_debug "$FUNCNAME releaseId=$releaseId" + + local request='{"draft":false}' + log_debug "PATCH $GITHUB_BASE_URL/releases/${releaseId}" + log_debug " -> request: $request" + log_info "Publishing github release $releaseId" + local response + response=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ + -H "Content-Type: application/json" \ + --data "${request}" \ + -X PATCH \ + "$GITHUB_BASE_URL/releases/${releaseId}") + log_debug " -> response: $response" + local htmlUrl=$(echo "$response" | jq --raw-output ".html_url") + log_success "Published release with id=$releaseId at $htmlUrl" +} diff --git a/.ci/inc/install-openjdk.inc b/.ci/inc/install-openjdk.inc index 2a69ba6382..f84ff32540 100644 --- a/.ci/inc/install-openjdk.inc +++ b/.ci/inc/install-openjdk.inc @@ -9,7 +9,6 @@ function install_openjdk() { OPENJDK_VERSION=$1 - log_group_start "Installing OpenJDK ${OPENJDK_VERSION}" case "$(uname)" in Linux*) @@ -65,8 +64,6 @@ function install_openjdk() { exit 1 ;; esac - - log_group_end } function install_openjdk_setdefault() { diff --git a/.ci/inc/maven-dependencies.inc b/.ci/inc/maven-dependencies.inc index 64afe8b95c..711e0144b9 100644 --- a/.ci/inc/maven-dependencies.inc +++ b/.ci/inc/maven-dependencies.inc @@ -17,9 +17,8 @@ # execpt for dokka-maven-plugin, as it does not play well with dependency-plugin. # function maven_dependencies_resolve() { - log_group_start "Downloading maven dependencies" - - dokka_version=$(./mvnw -q -Dexec.executable="echo" -Dexec.args='${dokka.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) + dokka_version=$(./mvnw -q -Dexec.executable="echo" -Dexec.args='${dokka.version}' + --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) ./mvnw dependency:resolve ./mvnw dependency:get -DgroupId=org.jetbrains.dokka \ @@ -27,7 +26,5 @@ function maven_dependencies_resolve() { -Dversion=${dokka_version} \ -Dpackaging=jar \ -DremoteRepositories=jcenter::default::https://jcenter.bintray.com/ - ./mvnw dependency:resolve-plugins -DexcludeGroupIds=org.jetbrains.dokka - - log_group_end + ./mvnw dependency:resolve-plugins -DexcludeGroupIds=org.jetbrains.dokka -Psign } diff --git a/.ci/inc/pmd-doc.inc b/.ci/inc/pmd-doc.inc index c55f18fd92..6054baca46 100644 --- a/.ci/inc/pmd-doc.inc +++ b/.ci/inc/pmd-doc.inc @@ -72,7 +72,7 @@ function publish_release_documentation_github() { git commit -q -m "Copying pmd-${VERSION} to latest" log_info "Generating sitemap.xml" - ../.travis/sitemap_generator.sh > sitemap.xml + ../docs/sitemap_generator.sh > sitemap.xml echo "Executing: git add sitemap.xml" git add sitemap.xml echo "Executing: git commit..." diff --git a/.ci/inc/regression-tester.inc b/.ci/inc/regression-tester.inc index abbd15586f..1083c81e39 100644 --- a/.ci/inc/regression-tester.inc +++ b/.ci/inc/regression-tester.inc @@ -11,7 +11,7 @@ # PMD_CI_CHUNK_TOKEN function regression_tester_setup_ci() { - # install openjdk8 for pmd-regression-tests + log_info "Install openjdk8 for pmd-regression-tests" install_openjdk 8 gpg --batch --yes --decrypt --passphrase="GnxdjywUEPveyCD1RLiTd7t8CImnefYr" \ diff --git a/.ci/render_release_notes.rb b/.ci/render_release_notes.rb new file mode 100755 index 0000000000..c7d0542cd7 --- /dev/null +++ b/.ci/render_release_notes.rb @@ -0,0 +1,44 @@ +#!/usr/bin/env ruby + +# Renders the release notes for Github releases, +# and prints them to standard output + +# Doesn't trim the header, which is done in shell + +# Args: +# ARGV[0] : location of the file to render + +require "liquid" +require "safe_yaml" + +# include some custom liquid extensions +require_relative "../docs/_plugins/all_extensions" + +# explicitly setting safe mode to get rid of the warning +SafeYAML::OPTIONS[:default_mode] = :safe + +# START OF THE SCRIPT + +unless ARGV.length == 1 && File.exists?(ARGV[0]) + print "\e[31m[ERROR] In #{$0}: The first arg must be a valid file name\e[0m\n" + exit 1 +end + +release_notes_file = ARGV[0] + +# Make the script execute wherever we are +travis_dir = File.expand_path File.dirname(__FILE__) + +liquid_env = { + # wrap the config under a "site." namespace because that's how jekyll does it + 'site' => YAML.load_file(travis_dir + "/../docs/_config.yml"), + # This signals the links in {% rule %} tags that they should be rendered as absolute + 'is_release_notes_processor' => true +} + + +to_render = File.read(release_notes_file) +rendered = Liquid::Template.parse(to_render).render(liquid_env) + + +print(rendered) diff --git a/.github/workflows/pushes.yml b/.github/workflows/pushes.yml index 6dc8d81b0d..df2c2d3849 100644 --- a/.github/workflows/pushes.yml +++ b/.github/workflows/pushes.yml @@ -28,14 +28,15 @@ jobs: - name: Check Environment run: .ci/check-environment.sh shell: bash - - name: build + - name: Build run: .ci/build.sh shell: bash env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} PMD_CI_JOB_URL: "https://github.com/pmd/pmd/runs/${{ github.run_id }}" PMD_CI_PUSH_COMMIT_COMPARE: ${{ github.event.compare }} - PMD_CI_BRANCH: ${{ github.ref }} + PMD_CI_GIT_REF: ${{ github.ref }} windows: runs-on: windows-latest diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index cc031398c6..2948cb6653 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -13,11 +13,12 @@ jobs: - name: Check Environment run: .ci/check-environment.sh shell: bash - - name: Install OpenJDK 11 - run: .ci/install-openjdk.sh 11 - shell: bash - - name: Run Release Script - run: .ci/release.sh + - name: Build + run: .ci/build.sh shell: bash env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} + PMD_CI_JOB_URL: "https://github.com/pmd/pmd/runs/${{ github.run_id }}" + PMD_CI_PUSH_COMMIT_COMPARE: ${{ github.event.compare }} + PMD_CI_GIT_REF: ${{ github.ref }} diff --git a/docs/sitemap_generator.sh b/docs/sitemap_generator.sh new file mode 100755 index 0000000000..03e25a0bcf --- /dev/null +++ b/docs/sitemap_generator.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Sitemap generator for pmd.github.io main landing page. +# Assumes we have the latest version of the site under "latest" +# https://www.sitemaps.org/protocol.html + +WEBSITE_PREFIX="https://pmd.github.io/" +DOC_PREFIX="latest/" +DATE=`date +%Y-%m-%d` +# Priority is relative to the website, can be chosen in {0.1, 0.2, ..., 1} +# Default priority is 0.5 +LATEST_PRIORITY=0.8 + + +# Writes to standard output + +cat << HEADER_END + + + + + ${WEBSITE_PREFIX}index.html + 1 + monthly + $DATE + + + + ${WEBSITE_PREFIX}${DOC_PREFIX}index.html + 0.9 + monthly + $DATE + + + + +HEADER_END + + +for page in ${DOC_PREFIX}pmd_*.html +do + + cat << ENTRY_END + + ${WEBSITE_PREFIX}$page + $LATEST_PRIORITY + monthly + $DATE + + +ENTRY_END + +done + +echo "" + From 5cc5e34b7e134a0c9a167f3125c6406fa4996b4a Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 20:45:11 +0100 Subject: [PATCH 38/52] Fix maven-dependencies --- .ci/inc/maven-dependencies.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/inc/maven-dependencies.inc b/.ci/inc/maven-dependencies.inc index 711e0144b9..0aaa020b81 100644 --- a/.ci/inc/maven-dependencies.inc +++ b/.ci/inc/maven-dependencies.inc @@ -17,7 +17,7 @@ # execpt for dokka-maven-plugin, as it does not play well with dependency-plugin. # function maven_dependencies_resolve() { - dokka_version=$(./mvnw -q -Dexec.executable="echo" -Dexec.args='${dokka.version}' + dokka_version=$(./mvnw -q -Dexec.executable="echo" -Dexec.args='${dokka.version}' \ --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) ./mvnw dependency:resolve From 9659a663e9507863a4560b88a3a49f5a9cca23b1 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 21:03:41 +0100 Subject: [PATCH 39/52] Add missing include install-openjdk --- .ci/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/build.sh b/.ci/build.sh index 715afb02f7..4bd9fc72e2 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -8,6 +8,7 @@ source $(dirname $0)/inc/pmd-code-api.inc source $(dirname $0)/inc/regression-tester.inc source $(dirname $0)/inc/github-releases-api.inc source $(dirname $0)/inc/maven-dependencies.inc +source $(dirname $0)/inc/install-openjdk.inc set -e From 574bccb6bff4796e7eafdc6586c79f7b15d86749 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 21:10:52 +0100 Subject: [PATCH 40/52] Adjust integration test for new folder .ci --- pmd-dist/src/main/resources/assemblies/pmd-src.xml | 7 ++++--- .../net/sourceforge/pmd/it/SourceDistributionIT.java | 11 +++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pmd-dist/src/main/resources/assemblies/pmd-src.xml b/pmd-dist/src/main/resources/assemblies/pmd-src.xml index 1513da9f56..a364b8f88f 100644 --- a/pmd-dist/src/main/resources/assemblies/pmd-src.xml +++ b/pmd-dist/src/main/resources/assemblies/pmd-src.xml @@ -29,9 +29,10 @@ **/.idea/** **/*.iml - .travis/secrets.tar - .travis/id_rsa - .travis/*.gpg + .ci/files/id_rsa + .ci/files/private-env + .ci/files/public-env + .ci/files/release-signing-key-D0BF1D737C9A1C22.gpg.gpg .bundle/** vendor/** diff --git a/pmd-dist/src/test/java/net/sourceforge/pmd/it/SourceDistributionIT.java b/pmd-dist/src/test/java/net/sourceforge/pmd/it/SourceDistributionIT.java index d483644813..acfdb2106d 100644 --- a/pmd-dist/src/test/java/net/sourceforge/pmd/it/SourceDistributionIT.java +++ b/pmd-dist/src/test/java/net/sourceforge/pmd/it/SourceDistributionIT.java @@ -10,7 +10,6 @@ import java.io.File; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.regex.Pattern; import org.junit.Assert; import org.junit.Test; @@ -19,7 +18,6 @@ import net.sourceforge.pmd.PMDVersion; public class SourceDistributionIT { private static final String BASE_PATH = "pmd-src-" + PMDVersion.VERSION; - private static final Pattern GPG_PATTERN = Pattern.compile(Pattern.quote(BASE_PATH + "/.travis/") + ".+\\.[gG][pP][gG]"); private File getSourceDistribution() { return new File(".", "target/" + BASE_PATH + ".zip"); @@ -33,13 +31,14 @@ public class SourceDistributionIT { @Test public void verifyExclusions() throws Exception { Set exclusions = new HashSet<>(); - exclusions.add(BASE_PATH + "/.travis/secrets.tar"); - exclusions.add(BASE_PATH + "/.travis/id_rsa"); + exclusions.add(BASE_PATH + "/.ci/files/id_rsa"); + exclusions.add(BASE_PATH + "/.ci/files/private-env"); + exclusions.add(BASE_PATH + "/.ci/files/public-env"); + exclusions.add(BASE_PATH + "/.ci/files/release-signing-key-D0BF1D737C9A1C22.gpg.gpg"); List files = ZipFileExtractor.readZipFile(getSourceDistribution().toPath()); for (String file : files) { - Assert.assertFalse("File " + file + " must not be included", exclusions.contains(file) - || GPG_PATTERN.matcher(file).matches()); + Assert.assertFalse("File " + file + " must not be included", exclusions.contains(file)); } } } From 244ef6157d67bf19316f19635ed49057aee0bfae Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 21:37:45 +0100 Subject: [PATCH 41/52] Add missing tool zip --- .ci/check-environment.sh | 1 + .ci/docker_ubuntu18.04/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/check-environment.sh b/.ci/check-environment.sh index 51dfc28791..d164569a19 100755 --- a/.ci/check-environment.sh +++ b/.ci/check-environment.sh @@ -45,6 +45,7 @@ case "$(uname)" in check "git" "git --version" "git version" check "mvn" "mvn --version" "Apache Maven" check "unzip" "unzip --version" "UnZip" + check "zip" "zip --version" "This is Zip" #check "7z" "7z -version" "7-Zip" ;; Darwin*) diff --git a/.ci/docker_ubuntu18.04/Dockerfile b/.ci/docker_ubuntu18.04/Dockerfile index ff76744c38..ff4f47e005 100644 --- a/.ci/docker_ubuntu18.04/Dockerfile +++ b/.ci/docker_ubuntu18.04/Dockerfile @@ -5,7 +5,7 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get upgrade --yes && \ apt-get install --yes curl jq gpg rsync ssh git p7zip-full openjdk-8-jdk \ libgdbm-dev libncurses5-dev automake libtool bison libffi-dev \ - sudo nano bash tzdata unzip && \ + sudo nano bash tzdata unzip zip && \ apt-get clean RUN cd opt && \ curl https://mirror.checkdomain.de/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz | tar xz && \ From c19d1823a24d9a405620983699fbb1e9926a9a20 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 Nov 2020 22:08:28 +0100 Subject: [PATCH 42/52] Don't check rule docs on pull requests This check fails under Windows --- .ci/build-pr.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/build-pr.sh b/.ci/build-pr.sh index dad6e7ae95..322e3918c1 100755 --- a/.ci/build-pr.sh +++ b/.ci/build-pr.sh @@ -16,7 +16,7 @@ log_group_start "Downloading maven dependencies" log_group_end log_group_start "Building with maven" - ./mvnw -e -V clean verify -Pgenerate-rule-docs + ./mvnw -e -V clean verify log_group_end From e686d0726319f51aba3353ce30eae1f4f54727df Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 14 Nov 2020 10:07:34 +0100 Subject: [PATCH 43/52] Update test instructions --- .ci/README.md | 89 +++++++++++++++++++++++++++++-- .ci/docker_ubuntu18.04/Dockerfile | 7 --- 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/.ci/README.md b/.ci/README.md index a84fbe0a4a..181823afce 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -85,16 +85,97 @@ docker build -t pmd-ci . This container is based on Ubuntu 18.04, which is used for `ubuntu-latest` github actions runner, see [Virtual Environment](https://github.com/actions/virtual-environments). -You can run a local instance with docker and mount your local pmd checkout into the container: +You can run a local instance with docker: ``` -docker run -it --mount type=bind,source=path/to/pmd,target=/workspaces/pmd/pmd pmd-ci +docker run -it pmd-ci ``` -You'll be dropped into a bash. Start e.g. with +You'll be dropped into a bash. + +#### Testing a push build (snapshot) + +Start docker without binding to local directory, so that we can do a fresh checkout: `docker run -it pmd-ci`. +You'll be dropped into a bash. Use the following script, to setup and start the build: ``` -cd workspaces/pmd/pmd +MAIN_BRANCH="master" +export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" +export PMD_CI_JOB_URL="manual job execution in docker" +export PMD_CI_PUSH_COMMIT_COMPARE="" +export PMD_CI_GIT_REF="refs/heads/${MAIN_BRANCH}" + +export PMD_CI_SECRET_PASSPHRASE="xyz" + +cd /workspace/pmd +rmdir pmd && mkdir pmd +cd pmd +git init +git remote add origin https://github.com/pmd/pmd +git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/heads/${MAIN_BRANCH}:refs/remotes/origin/${MAIN_BRANCH} +git checkout --progress --force -B master refs/remotes/origin/${MAIN_BRANCH} + .ci/check-environment.sh + +.ci/build.sh +``` + +#### Performing a release (push) build + +Start docker without binding to local directory, so that we can do a fresh checkout: `docker run -it pmd-ci`. +You'll be dropped into a bash. Use the following script, to setup and start the build: + +``` +TAG_NAME="pmd_releases/0.0.0_release_test" +export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" +export PMD_CI_JOB_URL="manual job execution in docker" +export PMD_CI_PUSH_COMMIT_COMPARE="" +export PMD_CI_GIT_REF="refs/tags/${TAG_NAME}" + +export PMD_CI_SECRET_PASSPHRASE="xyz" + +cd /workspace/pmd +rmdir pmd && mkdir pmd +cd pmd +git init +git remote add origin https://github.com/pmd/pmd +git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +refs/tags/${TAG_NAME}:refs/tags/${TAG_NAME} +git checkout --progress --force refs/tags/${TAG_NAME} + +.ci/check-environment.sh + +.ci/build.sh +``` + +**Warning:** This will build and upload to maven central! + + +#### Testing a pull request + +Start docker without binding to local directory, so that we can do a fresh checkout: `docker run -it pmd-ci`. +You'll be dropped into a bash. Use the following script, to setup and start the build: + +``` +export MAVEN_OPTS="-Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3" +export PMD_CI_BRANCH="master" # base branch +export PMD_CI_PULL_REQUEST_NUMBER=2913 + +# these are used by danger +export GITHUB_EVENT_PATH=/workspaces/event.json +export GITHUB_REPOSITORY=pmd/pmd +export GITHUB_ACTION=run1 +export GITHUB_EVENT_NAME=pull_request +/home/pmd-ci/create-gh-pull-request-event.sh + +cd /workspace/pmd +rmdir pmd && mkdir pmd +cd pmd +git init +git remote add origin https://github.com/pmd/pmd +git fetch --no-tags --prune --progress --no-recurse-submodules --depth=2 origin +refs/pull/${PMD_CI_PULL_REQUEST_NUMBER}/merge:refs/remotes/pull/${PMD_CI_PULL_REQUEST_NUMBER}/merge +git checkout --progress --force refs/remotes/pull/${PMD_CI_PULL_REQUEST_NUMBER}/merge + +.ci/check-environment.sh + .ci/build-pr.sh ``` diff --git a/.ci/docker_ubuntu18.04/Dockerfile b/.ci/docker_ubuntu18.04/Dockerfile index ff4f47e005..8317beb799 100644 --- a/.ci/docker_ubuntu18.04/Dockerfile +++ b/.ci/docker_ubuntu18.04/Dockerfile @@ -19,16 +19,9 @@ RUN mkdir -p /workspaces/pmd/pmd && chown -R pmd-ci:pmd-ci /workspaces && ln -sf USER pmd-ci:pmd-ci WORKDIR /home/pmd-ci -ENV GITHUB_EVENT_PATH=/workspaces/event.json -ENV GITHUB_REPOSITORY=pmd/pmd -ENV GITHUB_ACTION=run1 -ENV GITHUB_EVENT_NAME=pull_request -ENV PMD_CI_BRANCH=master -ENV PMD_CI_PULL_REQUEST_NUMBER=2913 COPY create-gh-pull-request-event.sh . COPY install-ruby.sh . RUN ./install-ruby.sh -RUN ./create-gh-pull-request-event.sh CMD ["/bin/bash", "--login"] From e10198402a0d54f7473896b20c2500e0e88f218e Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 14 Nov 2020 10:54:29 +0100 Subject: [PATCH 44/52] Apparently spring-framework doesn't build anymore without maven central --- .ci/files/project-list.xml | 70 ++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/.ci/files/project-list.xml b/.ci/files/project-list.xml index 600061f674..d150ae3443 100644 --- a/.ci/files/project-list.xml +++ b/.ci/files/project-list.xml @@ -18,6 +18,8 @@ if test -e classpath.txt; then exit fi +set -e + mvn test-compile mvn dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=classpath.txt ]]> @@ -35,30 +37,56 @@ if test -e classpath.txt; then exit fi +set -e + # Note: openjdk8 will be installed by "before_install.sh" -JAVA_HOME=${HOME}/openjdk8 -PATH=$JAVA_HOME/bin:$PATH +export JAVA_HOME=${HOME}/openjdk8 +export PATH=$JAVA_HOME/bin:$PATH + +# keep the tabs!! +(cat <> build.gradle < classpath.txt ]]> cat classpath.txt From 4246cb77ca1214f9833f1a53946264c7c1ac4b6a Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 14 Nov 2020 12:20:11 +0100 Subject: [PATCH 45/52] Add sonar and coveralls jobs --- .ci/README.md | 2 + .ci/build-coveralls.sh | 37 +++++++++++++++ .ci/build-sonar.sh | 33 +++++++++++++ .ci/files/private-env.gpg | Bin 489 -> 611 bytes .github/workflows/pull-requests.yml | 1 - .github/workflows/pushes.yml | 71 ++++++++++++++++++++++++---- pom.xml | 1 + 7 files changed, 136 insertions(+), 9 deletions(-) create mode 100755 .ci/build-coveralls.sh create mode 100755 .ci/build-sonar.sh diff --git a/.ci/README.md b/.ci/README.md index 181823afce..3c5bc286b0 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -18,6 +18,8 @@ for a ssh key, which is used to copy files to sourceforge. * PMD_SF_APIKEY * GITHUB_OAUTH_TOKEN * GITHUB_BASE_URL +* COVERALLS_REPO_TOKEN +* SONAR_TOKEN * DANGER_GITHUB_API_TOKEN * PMD_CI_CHUNK_TOKEN diff --git a/.ci/build-coveralls.sh b/.ci/build-coveralls.sh new file mode 100755 index 0000000000..89a0743fb2 --- /dev/null +++ b/.ci/build-coveralls.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +source $(dirname $0)/inc/logger.inc +source $(dirname $0)/inc/setup-secrets.inc +source $(dirname $0)/inc/install-openjdk.inc +source $(dirname $0)/inc/maven-dependencies.inc + +set -e + +log_group_start "Setup private env and OpenJDK" + pmd_ci_setup_private_env + install_openjdk_setdefault 11 + export CI_NAME="github actions" + export CI_BUILD_URL="${PMD_CI_JOB_URL}" + export CI_BRANCH="${PMD_CI_GIT_REF##refs/heads/}" +log_group_end + +log_group_start "Downloading maven dependencies" + maven_dependencies_resolve +log_group_end + +log_group_start "Executing build with coveralls" + ./mvnw \ + -Dmaven.javadoc.skip=true \ + -Dmaven.source.skip \ + -Dcheckstyle.skip \ + -DrepoToken=${COVERALLS_REPO_TOKEN} \ + -B -V -e \ + clean package jacoco:report \ + coveralls:report -Pcoveralls + + if [ $? -ne 0 ]; then + log_error "Error creating coveralls report" + else + log_success "New coveralls result: https://coveralls.io/github/pmd/pmd" + fi +log_group_end diff --git a/.ci/build-sonar.sh b/.ci/build-sonar.sh new file mode 100755 index 0000000000..461fd317a0 --- /dev/null +++ b/.ci/build-sonar.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +source $(dirname $0)/inc/logger.inc +source $(dirname $0)/inc/setup-secrets.inc +source $(dirname $0)/inc/install-openjdk.inc +source $(dirname $0)/inc/maven-dependencies.inc + +set -e + +log_group_start "Setup private env and OpenJDK" + pmd_ci_setup_private_env + install_openjdk_setdefault 11 +log_group_end + +log_group_start "Downloading maven dependencies" + maven_dependencies_resolve +log_group_end + +log_group_start "Executing build with sonar" + ./mvnw \ + -Dmaven.javadoc.skip=true \ + -Dmaven.source.skip \ + -Dcheckstyle.skip \ + -B -V -e \ + clean package \ + sonar:sonar -Dsonar.login=${SONAR_TOKEN} -Psonar + + if [ $? -ne 0 ]; then + log_error "Error updating sonar..." + else + log_success "New sonar results: https://sonarcloud.io/dashboard?id=net.sourceforge.pmd%3Apmd" + fi +log_group_end diff --git a/.ci/files/private-env.gpg b/.ci/files/private-env.gpg index d379ecefe2134949dedee3142a0c33d14132ec39..af292ee8b509f5ac8706b4c430886d4bf45507f3 100644 GIT binary patch literal 611 zcmV-p0-XJf4Fm}T0%ntHiUuqoGyl@*0Y|e$-<{CT>g4Z@)yY&AH!x4vsIy9TIu$+|)SqI&@_<4&5V*PO{5l?sq# zYawHZwi}Vc)ubf2zv~rN>^Pn8{Rbqo$_DA6Qdz4 zy#IFXfgOOBku`edUjWI|=}j7V7_1aPD=3qDqLFzIXAG2^0!xh0JFjdMHFu%IrM%-P zZy~Z#fuH*#Q&?H8F*5GeAv?@lIB}*uLfql4gVWzGOb29EHzOOg54h5Cn{jb|5u*%km&lrrHM0V=1e}D~hVy zAG7-Frz5RB#yp-qo18`ev(X~ugDL7O7sP;m%1VtoXxdz)pgxvlo58TmWBC2D0avCHP}hki}6v|m%G`;+b^D()-~4XdW9E7MLn2nU_A z3CUHcX{K3#&9{F--Q0+8i74jTb265brVy{r$hla^^P3OS#`@MsM7ZYVHnQkyvYq+w z0qNCjo#cKsSL=XQT85iudD9STL`wbZR^5hzOMp9)$5_siz;DBP;dA?4f%u#A4@b1% z&*Bn3>`Rd^N$kmDP(9`;HFLHlF%G7K^eGmrg0GY((v!hSo<_3- z-FNlJDllWjrQQL&uCQswy@8;ihV+kNPWo22dD`{?j(oUS6Qo^YEA1u6rhzZuZ7i)0 zyn+oj&Fk+YxtIo9#@vz@uBI2Db^DAFs>WULK>j*n>JMhtN*Xdm&W^C6B-%A5N{l7m z2=;fA@al{Zdw&QJLl}!}51$w9BhQgQGJW^~@Ys{1^`!h_Hn$W0ZF0R@CKs-b;c(nE flam{g#_A<1HUX=ast~esrY2$33)nqFF1+0U*~I${ diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index bfa9e6eea3..f9b93f1106 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -7,7 +7,6 @@ jobs: runs-on: ${{ matrix.os }} continue-on-error: false timeout-minutes: 30 - if: "!contains(github.event.head_commit.message, '[skip ci]')" strategy: matrix: os: [ ubuntu-latest, windows-latest, macos-latest ] diff --git a/.github/workflows/pushes.yml b/.github/workflows/pushes.yml index df2c2d3849..5f0c3c8237 100644 --- a/.github/workflows/pushes.yml +++ b/.github/workflows/pushes.yml @@ -11,6 +11,8 @@ on: jobs: linux: runs-on: ubuntu-latest + continue-on-error: false + if: "!contains(github.event.head_commit.message, '[skip ci]')" steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 @@ -40,6 +42,7 @@ jobs: windows: runs-on: windows-latest + needs: linux steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 @@ -50,16 +53,19 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}- - - name: Install OpenJDK - run: .ci/install-openjdk.sh + - name: Check Environment + run: .ci/check-environment.sh shell: bash - name: Build with mvnw run: | - source ${HOME}/java.env - ./mvnw -V clean install + source .ci/inc/install-openjdk.inc + install_openjdk_setdefault 11 + ./mvnw -V clean verify + shell: bash macos: runs-on: macos-latest + needs: linux steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 @@ -70,10 +76,59 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}- - - name: Install OpenJDK - run: .ci/install-openjdk.sh + - name: Check Environment + run: .ci/check-environment.sh shell: bash - name: Build with mvnw run: | - source ${HOME}/java.env - ./mvnw -V clean install + source .ci/inc/install-openjdk.inc + install_openjdk_setdefault 11 + ./mvnw -V clean verify + shell: bash + + coveralls: + runs-on: ubuntu-latest + needs: [linux, windows, macos] + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + key: ${{ runner.os }}-coveralls-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-coveralls- + - name: Check Environment + run: .ci/check-environment.sh + shell: bash + - name: Build + run: .ci/build-coveralls.sh + shell: bash + env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 + PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} + PMD_CI_JOB_URL: "https://github.com/pmd/pmd/runs/${{ github.run_id }}" + + sonar: + runs-on: ubuntu-latest + needs: [linux, windows, macos] + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.m2/repository + ~/.cache + key: ${{ runner.os }}-sonar-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-sonar- + - name: Check Environment + run: .ci/check-environment.sh + shell: bash + - name: Build + run: .ci/build-sonar.sh + shell: bash + env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 + PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }} diff --git a/pom.xml b/pom.xml index 2312e8579f..9516b42d59 100644 --- a/pom.xml +++ b/pom.xml @@ -1028,6 +1028,7 @@ https://sonarcloud.io pmd + 1.8 From 91835315cc0da013fd34fbaab2eadcd44e354281 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 14 Nov 2020 12:44:57 +0100 Subject: [PATCH 46/52] Add vendor/bundle to cache --- .github/workflows/pull-requests.yml | 1 + .github/workflows/pushes.yml | 5 +++++ .github/workflows/troubleshooting.yml | 1 + 3 files changed, 7 insertions(+) diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index f9b93f1106..a9e08dbcbf 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -20,6 +20,7 @@ jobs: path: | ~/.m2/repository ~/.cache + vendor/bundle key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}- diff --git a/.github/workflows/pushes.yml b/.github/workflows/pushes.yml index 5f0c3c8237..4b5ff40f59 100644 --- a/.github/workflows/pushes.yml +++ b/.github/workflows/pushes.yml @@ -20,6 +20,7 @@ jobs: path: | ~/.m2/repository ~/.cache + vendor/bundle key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}- @@ -50,6 +51,7 @@ jobs: path: | ~/.m2/repository ~/.cache + vendor/bundle key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}- @@ -73,6 +75,7 @@ jobs: path: | ~/.m2/repository ~/.cache + vendor/bundle key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}- @@ -96,6 +99,7 @@ jobs: path: | ~/.m2/repository ~/.cache + vendor/bundle key: ${{ runner.os }}-coveralls-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-coveralls- @@ -120,6 +124,7 @@ jobs: path: | ~/.m2/repository ~/.cache + vendor/bundle key: ${{ runner.os }}-sonar-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-sonar- diff --git a/.github/workflows/troubleshooting.yml b/.github/workflows/troubleshooting.yml index a8c1f299a0..bb4cd64fb0 100644 --- a/.github/workflows/troubleshooting.yml +++ b/.github/workflows/troubleshooting.yml @@ -18,6 +18,7 @@ jobs: path: | ~/.m2/repository ~/.cache + vendor/bundle key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}- From 471ea18b0dcb1f24e26e3cedc27ee7c9f1bf91f7 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 14 Nov 2020 12:45:21 +0100 Subject: [PATCH 47/52] Add github actions badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index a1bed340d3..20c3cf3b1e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # PMD [![Join the chat at https://gitter.im/pmd/pmd](https://badges.gitter.im/pmd/pmd.svg)](https://gitter.im/pmd/pmd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Build Status](https://github.com/pmd/pmd/workflows/.github/workflows/pushes.yml/badge.svg?branch=master)](https://github.com/pmd/pmd/actions) [![Build Status](https://travis-ci.com/pmd/pmd.svg?branch=master)](https://travis-ci.com/pmd/pmd) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/net.sourceforge.pmd/pmd/badge.svg)](https://maven-badges.herokuapp.com/maven-central/net.sourceforge.pmd/pmd) [![Reproducible Builds](https://img.shields.io/badge/Reproducible_Builds-ok-green?labelColor=blue)](https://github.com/jvm-repo-rebuild/reproducible-central#net.sourceforge.pmd:pmd) From ac07a0a594e15533171b74b1512f0c0b2d291ac9 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 14 Nov 2020 12:45:52 +0100 Subject: [PATCH 48/52] Remove travis config --- .travis.yml | 94 --------- .travis/all-java.xml | 18 -- .travis/before_install.sh | 51 ----- .travis/build-coveralls.sh | 28 --- .travis/build-deploy.sh | 86 --------- .travis/build-doc.sh | 188 ------------------ .travis/build-publish.sh | 28 --- .travis/build-sonar.sh | 33 ---- .travis/common-functions.sh | 64 ------- .travis/configure-maven.sh | 6 - .travis/github-releases-api.sh | 214 --------------------- .travis/id_rsa.pub | 1 - .travis/install-openjdk.sh | 64 ------- .travis/logger.sh | 23 --- .travis/pmd-code-api.sh | 94 --------- .travis/project-list.xml | 73 ------- .travis/regression-tester.sh | 82 -------- .travis/render_release_notes.rb | 44 ----- .travis/secrets.tar.enc | Bin 20496 -> 0 bytes .travis/setup-secrets.sh | 65 ------- .travis/sitemap_generator.sh | 56 ------ .travis/sourceforge-api.sh | 166 ---------------- .travis/travis-settings.xml | 31 --- .travis/travis_wait | 324 -------------------------------- README.md | 1 - pom.xml | 2 +- 26 files changed, 1 insertion(+), 1835 deletions(-) delete mode 100644 .travis.yml delete mode 100644 .travis/all-java.xml delete mode 100644 .travis/before_install.sh delete mode 100755 .travis/build-coveralls.sh delete mode 100755 .travis/build-deploy.sh delete mode 100755 .travis/build-doc.sh delete mode 100644 .travis/build-publish.sh delete mode 100755 .travis/build-sonar.sh delete mode 100755 .travis/common-functions.sh delete mode 100755 .travis/configure-maven.sh delete mode 100644 .travis/github-releases-api.sh delete mode 100644 .travis/id_rsa.pub delete mode 100644 .travis/install-openjdk.sh delete mode 100644 .travis/logger.sh delete mode 100644 .travis/pmd-code-api.sh delete mode 100644 .travis/project-list.xml delete mode 100644 .travis/regression-tester.sh delete mode 100755 .travis/render_release_notes.rb delete mode 100644 .travis/secrets.tar.enc delete mode 100755 .travis/setup-secrets.sh delete mode 100755 .travis/sitemap_generator.sh delete mode 100644 .travis/sourceforge-api.sh delete mode 100644 .travis/travis-settings.xml delete mode 100755 .travis/travis_wait diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 34758b8d66..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,94 +0,0 @@ -language: java -os: linux -env: - global: - - secure: "KBEuB6U1p5RQXSYe157AwydFr/zpXQPA0IChVCgZV+X1mMyy9ZtrjH1J1AXuviseDDXDbaT25sRnsvpl82rfRw2xOkMGXHy4N95/ylTSr8DjHxTao71BhXsvFycNobFva5y2EGNWqDvpS8I2oSZo7Qk4la3yep3rcJQvcy6RDbbhpDTbL1QMFyadunIBm0WtqbunrMqtjSqaoPsXz8TiQuxHvX4vEXzVbaxV1QQt79Vi+daa6wAV3mRQAugnx+UffsC8JqMxgm06usWeJgCJzxgm8E7clZCLmf53B2TL8dK6bIYbqyvOY3uFxitsTG0d8Z0GOJwXBgZNgbniTRO8ZJSty5eZP8LBybbjVLSL25DNTWtCjADUL/uySnXIEidlMt2N/3QmH7zrGAfAk/tIwKpdRca2GLLydeXf6PSkiahnPEkIY/QupcsOLELhdifpdOjb8QW1OenA+vUbNM9dccLwKnX6Fj9cu4VQG601AcYDr2eyhq8WYkr3wYdw/6KdUa3hmplowTBs+qguppP+eOSgGuEsy38KLtqnvm6WlHy6tcLmcVYKG3DmR1b7TWXsOXC6/VMH8BHBkvsF1QdRg9+Cgx07vX3Hw7roPiYzmaO9Ajs20ATsUfRskMuWCTeTSK5pN8X27veRCZlhFjeKQMDdmfVwzpAfRgKsl3TEn1I=" - - secure: "U1DfAv6acUUWe+dao/ZSDUX64JRadNJY16rITsdrM4ZNAJSuXpEY3p/LWcYjN7D49YmyutbXH9+L3KKQUQGrGXj9QTarfYvd8ZsKt4FK8yv7AFy+RQNIbAjNEHBzHx15p+srMheTaetl7aLwY0qhF+D/RtGapxHKyY4dBHrb0lp8VGyiCiL7Aop8GGskosi1mtirPBp/BStPZ2bEyxG0QzU5SsVWkJWwV9aWLPVAR/n7Xgx/6Gjl6Fed2c/WSrWi4vchm3Ny8pfTweOax3PGYYjBVxIfuX0mqmwuJsY7gNfXCfN3dPiPKGJPFy1pC+LGyGkklO5ReKFLd4O1ME6fU0dlIGfD6n+Q4H6/w9FHXegcKTfWIJm/MFa6vA/tJM5R6zJQuiTQJboHm/UmS/iQj76z0p6sK15Xp5vFId+/dHKqa8xY+Bt6HiXy6z401HOc8QcYBAf7TqhqUt/ZE7HN4be46uR90KmzIrWz6wEoDW7HfwQ9ZMbs55zoOXrvekyE9/gXskypO0p2JT3Y0vlvO27KQvIrSwI480kOVOrlyrYA+LZqlcKaayOuCuZh4lITQUYinUoZZict5joYthH+Cyh2zovoBpxsntDJdMnaZNLtSC7hlhpbMBYaT2y1O6vZH5Yix0mxuDvs/x6ogP5CNBeUYlXhaL+g4GnwKyr0ZA0=" - - secure: "gxI7W4V4fUPQLMCvecXXdet1/mCh6m0RIQMtErVVOnwxEEjeDmko/3rHSl+wk6IeFaaQKcmHJJSJEj+e5TgWRH1uuzCnbFHZhuH28ce/H0EqLJ3GTXeDEgFLzhh//T3ySOZChZELgJ67cKxZJNFMhBVOR2/QIGjZjdzvl36ugsu80Ak3XJb8HCm8D2P8Vuezz/OlCkSOGXaiqZXfflV/cuaLWuueGfQGW9x/UyDTNsuBdB7YnEcxOWt8RgB4JrcPGV7/etxLHXTw9IMEeUhQ/RsxLZHKNYt9cx/QOZkpXemhdT9L2pAIi77eO98x+yAfB1qV6T6IeDd/OiABUzfb7asNBwE+bYRuVkCiWLo3x3hs8FcgsPe4L10guxfRMJPxax9E7uYb9TFxtvc42dDWNyp1BVCtEtmjErO4CJeW97ukhgkUkPhzUZgQ2Gj5Tn4VRjQOQyzh/S/YsHyfxoAQrscc/3bq5PBmtdowyGQd5dsvShdpE6glf+HnBZ8TCw5BaxQuTjapQWjsGuJVCVsrNDtwOFy5UTfxK9OeTUsjwfO5JjkoIeqGyLHPnTMWyZ6EYa//nWPOU1KgVE6Hrzfr3zxEzL1nN/TYahzhKOxXE701W9YhTmhCVljMlSYioTUfTaXHgGLpojvSWlhsTbc3r0gj4NglqakIy7MKgQLgES8=" - - secure: "otZkFNhApMofp2Jl2baZdpNEPcTa79Xc9wyOw1gl1+ubCLtOGr32FDUSr0HQWKrGXcYZR4bc0A80hx0/ykZ/twJaRAabcNYtjwRMDKKrp3plvp8RTZlB1QW85nsdmk3qmHACzc94wKnmcaDohM5FPxnWspZG5Y4ejYX4HEK6r1fwL4FjMPSLd5QW8yCaAc7dpBSm2UyIXuuDjkekXblJFe8ydbf9aF97WKJMCwqIs9Zo5oJa2pnWCw47ixwjVdkaH5x5727I+YmFqn5rjdEideUFEAlggqMaxuOtfYhuG/8k5W3Q/+WX+G7rEPTcemjmaosbbpoOUE0YUHV3Wi3R5D66NuWIpQwX6kw4rTwL0YIh//0aNb8jv4Z8KwkiTMrYuj0we5tXzARp3yYn56P0JBAm6BihNhfEBv06655Tg5LUsWqUTZCl3KJhwtO1N+OuOkE5qoZm59knq6DeZOsYWAVb3MpzklP06kXGYTjb44PVMUc6ynuuZjgPJDv/0906feA3REHBJVxd3PvJp1c8VvQSeW3ndghEz0akJHY/jxV1rJZAZC1ZeshYUpBNmh0sFDty/UE43IvIB4EInypQHyTtQvMc6LuLjTeu2E5ZMsA3udj66EMrYm4Y2eheJycSfttDqzhsKXIVJ4LmA/xtDR01cp1jsqbbZBhVI67rA40=" - - secure: "mmHRnK/8CHfgtwaTN3MgjFD1R8EVQhIb/a0Aag3gPaHWW52Ex6xWG7GsF4kRcdzHdIdrQxHZTiDa2oVMWbnQ16CAu4dZJadwTxhBW4JmE1tsi2C6i+uwMBPvnKBXHwt+QOAqNB0L/ByYwWLkucSLoyRs6qU0i7sJ3ZC/EPCuLI8VfBlFUVegSbp2WmclmFnxOi5MKr15Zwd8BGgyWUsvd1OHxhvO3Xr/ZPO4R+1kIgyojloTMu6oGM/shYgsrUXgmI/WvZRTq+25SRCjUmATJtuDXXgVeJtYabi/+Zju5Z7K06oaKvFYM+BgZWPmGQkVCxKVjSlug0rx6+qgRKIf/B/cPxmQDS0ZZVDmDcbxeibBL9JhRqIDwDb0XS9t1qBXvIJ5E4ZmdX/XuvLCnC4xJptu/vHyKl1wbRM3dcXEBk8J0knJc4JvY2yupF3LS32sfEHftLwzr63U3/0dOQvsCSJRBy5zgI8ieABUZAeZmmaQ6RCGOTEYrGO2G4CCF2uVq4YdrCOsfmrb0DzH8llNMkXKzXlITRvmkuBlTRRKxGnPzS3vf0iicdAkmTiU5u7lArFFc7IKcW4ypO+RKDTUpq74K+UZBI9eBjV7Po368EO8epJ5aIx32a2kyS/hYWkS/dZh7I0fNZa2qBUceZO/KZGRjH4dQFW9t0J9Q7LLXJY=" - - secure: "B2quYqla3mqu3QHqeRGMde66IFAluUHhzY87q/giPvONm/V7xmIcxzC6JNQ7X4pHotC7MUbsMf7vdcL1tyuPP5nqyfaxk0nnGnVoJZIQhMaBQ6Lkyt2Ac5u5/QBeHujrj8gqGjT7vzgPIYMYiSLV09VcPgMaYS6WxCrFmXOEJOeVGOGjG9J2wApAmlf6m/hBogLEbdbzRheqJwX5gMxppKX5XRaiqvKGR7hHFiW5HBaAiQ6caQEw6DZL9GnjMsqd5VMVL7DmbETncNE1utLVDO7mFwDMbkeonmwjsmQsJAo5Be+WFl/a/yRQLrqhhTFwag+V7g7RjcBKAdJ5IZJkN0EC9GjY0ZGOjE2cmcrX03U1g1voO7oreH6+25VqPTTGWHv5K8wDWM5QnhBJGtH2yH76p5XYK8iNOXPTqvjubN9sG3INsnTjxBHuQw+aY2vgwBGCvW+Asqe73BV8SDrmqcCqgmm/BUy0kZRd8mhaQdFL0SAO3suV7uf+fzIsmRtndFx24J5Cm6LE6NnTmtbhHhs4mUEqfngMq0XZBXj5z4LDhzOH4g3rf/Y7jHBjAAa6sKeh1sINdHrMdB843dDfNQPp/7g3hfKi31U9ZMczndsr2QYqm2OC/3lPiRHC+8Grj5PxXmiBplzTppyhl+1pjZWy5SCoQi6ulBOyaZcwfYM=" - - secure: "Kmh31kozw9NBbwtkucpf+QslinJIclB7rcktHz4RokVS5xiMD7njWvD8ptZhXqZG/tNcNiXkWskgM9EV/OvLTTavJToHE1gMhJel5ODxKM0b5QZDy6tkW/wI0Kdl0lck90EAJ7bmsOmF+oGczGsauiLPSlBOZV0tn1zdeJKwwmg46C9UyIhTDcFqUD1ehxExkDHnbOk6HmEPGCsZQCGNXY2t++dO7fRodIgjZAKcyTwgrh6WEZDfQDjvC51yQlVzL2mrcFqHi8qYAx3OHnuidRoMZ4huaoxLimw/1bGii/txTZwJ/pQdF90kGqSqLpRiO2lHLq5l8WAd4Z1XPSs1ZzbKUEhWO8VGgfR0Cp+Bkkd6P/vcuTq7bdhpvdnZn9FRUI/bfgZi827Skj/WoUh0RR7ns6RuDfQ5uGD9NiWDsbfoKLbvvYjyQycQlQTKglYQVcU/pWsuqVlx+27vM6JltR0Pct4uAJxAN+EFA4sTT/u4HiZ0NQyZWisZauu7CyMRN29fkKVVyYHHCr6QcNMUpvp1vvKYaUN7GeykGNVfoHB7CWAKcc6banh33c+QqG5DSTE++ZNa712gxEizQnO5MmLECUxNDpOGZnLsTlgbABwDxxe4PxFxXvBLm9pkJftq5cp9T9Syzz/yyT7z5weSGvRes94eUtSt0agoeX2MKsE=" - - secure: "VezxV+VdmbmtrQYT8AZIyg41WBROxuxpumerkcubADF7V4wV6lwx9Rd2G6yAr0VuHCNUUhS4m+gPFIsuiQbAhyupiEkwhzUYqk1tF+LITlLLPegLypjiLmhJMwGUNuDSSsih1Icmg9FzrP4VyzgGn9pBjoG9QYj1civBZeGwg++e/XDYlHMXrpd/UEfMKVB71JwB0tle4fKJZSvblIqP62yvbBaKHx6A4+ZWzJV5Vps0DoIeNtKCNmNNloKZVHfjbsvqSjnMYUJzkOzyPkM822q41N/D+3IAufO16+jH/W0vAZeN0e4GXiN5W+CVkr2Gbh0FwkVQcI3bekaOIn45XLUMLKdf+JsWDPKz9RraHelR9YxL5GoJ7ntwvmucxw0p8EVyJ/xLk/pBCP8iHq0Jb8//js25XHgxzzAWI37MErPAAGgTKZAVdAN0mGXbe63tWmwaBlEbK8h2A8di6abW5x6YHTkTo2BRlHUSTU8dE3VqTnpSkne5n1SlEa4g1Bci3J45M0/pLmHV6yCxCM5BrVXS5ByaB61py/umSbpmdIBFV6TM1MaKK3lAucQrR+8To/vCbm8XqPyujJdOR+ENIuuDgEU/Yh5Hv5SAODekUYaCp4pjfGzFADHQWVNDxIOXrwBN4OfSiAvRc1x6HXndOmNI4QtOxheuCRFFthq8VZI=" - - GITHUB_BASE_URL=https://api.github.com/repos/pmd/pmd - -jobs: - fast_finish: true - include: - - stage: build - name: "linux - mvn deploy" - os: linux - dist: bionic - env: BUILD=deploy - - name: "linux - build documentation" - os: linux - dist: bionic - env: BUILD=doc - - name: "linux - run sonar" - os: linux - dist: bionic - env: BUILD=sonar - if: "type = push && repo = pmd/pmd && fork = false" - - name: "linux - run coveralls" - os: linux - dist: bionic - env: BUILD=coveralls - if: "type = push && repo = pmd/pmd && fork = false" - - stage: publish - name: "Release Build - Publish" - os: linux - dist: bionic - if: "type = push && repo = pmd/pmd && fork = false && tag is present" - env: BUILD=publish - -before_install: - - bash .travis/before_install.sh "11" - - source ${HOME}/java.env -install: true -before_script: true -script: source .travis/build-$BUILD.sh -after_success: true - - -notifications: - webhooks: - urls: - - https://webhooks.gitter.im/e/0e6dd310db54b96c2653 - - https://webhooks.gitter.im/e/65efc1ea65175ad23ab6 - on_success: always # options: [always|never|change] default: always - on_failure: always # options: [always|never|change] default: always - on_start: never # options: [always|never|change] default: always - email: - recipients: - - andreas.dangel@adangel.org - on_success: change - on_failure: always -cache: - directories: - - "$HOME/.m2" - - "$HOME/.cache" - - vendor/bundle - - "$HOME/.rvm/" - -# Secure Keys, that need to be set for snapshot builds -# -# CI_DEPLOY_USERNAME - the user which can upload net.sourceforge.pmd:* to https://oss.sonatype.org/ -# CI_DEPLOY_PASSWORD -# -# SONAR_TOKEN - the token used to update https://sonarqube.com/dashboard/?id=393826 -# -# PMD_SF_USER - the sourceforge user, which is used to upload created binaries to sf files section. Note: an ssh key is -# required. See "before_install". -# -# Secure Keys, that need to be set for releases: -# -# PMD_SF_APIKEY - used to make the new release the default file in the files section. See https://sourceforge.net/auth/preferences/ -# under "Release API Key" -# -# CI_SIGN_KEYNAME -# CI_SIGN_PASSPHRASE -# add the encrypted GPG keyring file to repo (https://docs.travis-ci.com/user/encrypting-files/#Automated-Encryption), decrypt it and install it at the beginning of .travis-deploy.sh -# -# GITHUB_OAUTH_TOKEN - the token used to upload the binaries to github releases -# GITHUB_BASE_URL - the api url to use for github releases - does not need to be secure (https://api.github.com/repos/pmd/pmd) -# diff --git a/.travis/all-java.xml b/.travis/all-java.xml deleted file mode 100644 index 2a4f1345dc..0000000000 --- a/.travis/all-java.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - Every java rule in PMD which is used for the regression tests with pmdtester - - - - - - - - - - - diff --git a/.travis/before_install.sh b/.travis/before_install.sh deleted file mode 100644 index b1234a3909..0000000000 --- a/.travis/before_install.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -set -e - -source .travis/logger.sh -source .travis/common-functions.sh - -travis_debug - -OPENJDK_VERSION=$1 - -bash .travis/setup-secrets.sh -bash .travis/configure-maven.sh -bash .travis/install-openjdk.sh $OPENJDK_VERSION - - -function install_jdk() { - LOCAL_DIR=$1 - TARGET_DIR=$2 - DOWNLOAD_URL=$3 - ARCHIVE=$(basename $DOWNLOAD_URL) - - mkdir -p ${LOCAL_DIR} - mkdir -p ${TARGET_DIR} - if [ ! -e ${LOCAL_DIR}/${ARCHIVE} ]; then - log_info "Downloading from ${DOWNLOAD_URL} to ${LOCAL_DIR}" - wget --directory-prefix ${LOCAL_DIR} --timestamping --continue ${DOWNLOAD_URL} - else - log_info "Skipped download, file ${LOCAL_DIR}/${ARCHIVE} already exists" - fi - log_info "Extracting to ${TARGET_DIR}" - tar --extract --file ${LOCAL_DIR}/${ARCHIVE} -C ${TARGET_DIR} --strip-components=1 -} - -if travis_isLinux; then - change_ruby_version - gem install bundler - bundle config set --local path vendor/bundle - bundle config set --local with release_notes_preprocessing - bundle install - - # install jdk7 for integration test - install_jdk "${HOME}/.cache/jdk7" "${HOME}/oraclejdk7" "https://pmd-code.org/oraclejdk/jdk-7u80-linux-x64.tar.gz" - log_info "OracleJDK7 can be used via -Djava7.home=${HOME}/oraclejdk7" - - # install openjdk8 for pmd-regression-tests - install_jdk "${HOME}/.cache/openjdk" "${HOME}/openjdk8" "https://pmd-code.org/openjdk/latest/jdk-8-linux64.tar.gz" - log_info "OpenJDK8 can be used from ${HOME}/openjdk8" -else - log_info "Not setting up ruby and additional jvms for ${TRAVIS_OS_NAME}." - exit 0 -fi diff --git a/.travis/build-coveralls.sh b/.travis/build-coveralls.sh deleted file mode 100755 index c3834512a7..0000000000 --- a/.travis/build-coveralls.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -e - -source .travis/logger.sh -source .travis/common-functions.sh - -VERSION=$(get_pom_version) -log_info "Building PMD Coveralls.io report ${VERSION} on branch ${TRAVIS_BRANCH}" - -if ! travis_isPush; then - echo "Not proceeding, since this is not a push!" - exit 0 -fi - -( - # disable fast fail, exit immediately, in this subshell - set +e - - ./mvnw clean install -DskipTests=true -Dmaven.javadoc.skip=true -B -V - ./mvnw package jacoco:report coveralls:report -Pcoveralls -B -V - - if [ $? -ne 0 ]; then - log_error "Error creating coveralls report" - else - log_success "New coveralls result: https://coveralls.io/github/pmd/pmd" - fi - true -) diff --git a/.travis/build-deploy.sh b/.travis/build-deploy.sh deleted file mode 100755 index e9250ad7c4..0000000000 --- a/.travis/build-deploy.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash -set -e - -source .travis/logger.sh -source .travis/common-functions.sh -source .travis/github-releases-api.sh -source .travis/sourceforge-api.sh -source .travis/regression-tester.sh - -VERSION=$(get_pom_version) -log_info "Building PMD ${VERSION} on branch ${TRAVIS_BRANCH}" - -if travis_isLinux; then - MVN_BUILD_FLAGS="-B -V -Djava7.home=${HOME}/oraclejdk7" -else - MVN_BUILD_FLAGS="-B -V" -fi - -if travis_isOSX; then - - log_info "The build is running on OSX" - ./mvnw verify $MVN_BUILD_FLAGS - -elif travis_isWindows; then - - log_info "The build is running on Windows" - ./mvnw verify $MVN_BUILD_FLAGS - -elif travis_isPullRequest; then - - log_info "This is a pull-request build" - ./mvnw verify $MVN_BUILD_FLAGS - - regression-tester_executeDanger - -elif travis_isPush; then - - if [[ "${VERSION}" != *-SNAPSHOT && "${TRAVIS_TAG}" != "" ]]; then - echo -e "\n\n" - log_info "This is a release build for tag ${TRAVIS_TAG}" - echo -e "\n\n" - - # create a draft github release - gh_releases_createDraftRelease "${TRAVIS_TAG}" "$(git rev-list -n 1 ${TRAVIS_TAG})" - GH_RELEASE="$RESULT" - - # Build and deploy to ossrh / maven-central - ./mvnw deploy -Possrh,sign,pmd-release $MVN_BUILD_FLAGS - echo -e "\n\n" - - # Deploy to github releases - gh_release_uploadAsset "$GH_RELEASE" "pmd-dist/target/pmd-bin-${VERSION}.zip" - gh_release_uploadAsset "$GH_RELEASE" "pmd-dist/target/pmd-src-${VERSION}.zip" - - # Deploy to sourceforge files - sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip" - sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip" - - regression-tester_uploadBaseline - - elif [[ "${VERSION}" == *-SNAPSHOT ]]; then - log_info "This is a snapshot build" - ./mvnw deploy -Possrh,sign $MVN_BUILD_FLAGS - - # Deploy to sourceforge files - sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip" - sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip" - - if [ "${TRAVIS_BRANCH}" != "java-grammar" ]; then - regression-tester_uploadBaseline - else - log_info "Skipping regression tester for branch ${TRAVIS_BRANCH}" - fi - - else - # other build. Can happen during release: the commit with a non snapshot version is built, but not from the tag. - log_info "This is some other build, probably during release: commit with a non-snapshot version on branch master..." - ./mvnw verify $MVN_BUILD_FLAGS - # we stop here - no need to execute further steps - exit 0 - fi - -else - log_info "This is neither a pull request nor a push. Not executing any build." - exit 1 -fi diff --git a/.travis/build-doc.sh b/.travis/build-doc.sh deleted file mode 100755 index 4ac8028b1d..0000000000 --- a/.travis/build-doc.sh +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env bash -set -e - -source .travis/logger.sh -source .travis/common-functions.sh -source .travis/github-releases-api.sh -source .travis/sourceforge-api.sh -source .travis/pmd-code-api.sh - -function main() { - VERSION=$(get_pom_version) - log_info "Building PMD Documentation ${VERSION} on branch ${TRAVIS_BRANCH}" - - # - # First step: build pmd with profile "generate-rule-docs" - # The docs should appear under "docs/pages/rules/..." for each language - # With this profile, also the checks are executed (e.g. DeadLinksChecker). - # - ./mvnw clean verify -Dmaven.test.skip=true -P generate-rule-docs - - if ! travis_isPush; then - log_info "Not publishing site, since this is not a push!" - exit 0 - fi - - # - # For pushes, we'll update the online documentation - # - generate_jekyll_doc - create_pmd-doc_archive - - # Deploy to sourceforge files - sourceforge_uploadFile "${VERSION}" "docs/pmd-doc-${VERSION}.zip" - - # Deploy doc to https://docs.pmd-code.org/pmd-doc-${VERSION}/ - pmd_code_uploadDocumentation "${VERSION}" "docs/pmd-doc-${VERSION}.zip" - # Deploy javadoc to https://docs.pmd-code.org/apidocs/*/${VERSION}/ - pmd_code_uploadJavadoc "${VERSION}" "$(pwd)" - - - if [[ "${VERSION}" == *-SNAPSHOT && "${TRAVIS_BRANCH}" == "master" ]]; then - # only for snapshot builds from branch master - - pmd_code_createSymlink "${VERSION}" "snapshot" - - # update github pages https://pmd.github.io/pmd/ - publish_to_github_pages - # rsync site to https://pmd.sourceforge.io/snapshot - sourceforge_rsyncSnapshotDocumentation "${VERSION}" "snapshot" - fi - - - if [[ "${VERSION}" != *-SNAPSHOT && "${TRAVIS_TAG}" != "" ]]; then - log_info "This is a release documentation build for pmd ${VERSION}" - - # documentation is already uploaded to https://docs.pmd-code.org/pmd-doc-${VERSION} - pmd_code_createSymlink "${VERSION}" "latest" - # remove old doc and point to the new version - pmd_code_removeDocumentation "${VERSION}-SNAPSHOT" - pmd_code_createSymlink "${VERSION}" "${VERSION}-SNAPSHOT" - # remove old javadoc - pmd_code_removeJavadoc "${VERSION}-SNAPSHOT" - - # Deploy to github releases - gh_releases_getLatestDraftRelease - GH_RELEASE="$RESULT" - - gh_release_uploadAsset "$GH_RELEASE" "docs/pmd-doc-${VERSION}.zip" - - # updating github release text - # renders, and skips the first 6 lines - the Jekyll front-matter - RENDERED_RELEASE_NOTES=$(bundle exec .travis/render_release_notes.rb docs/pages/release_notes.md | tail -n +6) - RELEASE_NAME="PMD ${VERSION} ($(date -u +%d-%B-%Y))" - gh_release_updateRelease "$GH_RELEASE" "$RELEASE_NAME" "$RENDERED_RELEASE_NOTES" - sourceforge_uploadReleaseNotes "${VERSION}" "${RENDERED_RELEASE_NOTES}" - - publish_release_documentation_github - sourceforge_rsyncSnapshotDocumentation "${VERSION}" "pmd-${VERSION}" - fi -} - -# -# Executes jekyll and generates the documentation -# The documentation will be generated in the directory "docs/_site". -# -function generate_jekyll_doc() { - pushd docs - - echo -e "\n\n" - log_info "Building documentation using jekyll..." - bundle config set --local path vendor/bundle - bundle install - bundle exec jekyll build - - popd -} - -# -# Creates the pmd-doc.zip archive. It will be placed in "docs/". -# -function create_pmd-doc_archive() { - pushd docs - - echo -e "\n\n" - log_info "Creating pmd-doc archive..." - mv _site pmd-doc-${VERSION} - zip -qr pmd-doc-${VERSION}.zip pmd-doc-${VERSION}/ - log_success "Successfully created pmd-doc-${VERSION}.zip" - - popd -} - -# -# Publishes the site to https://pmd.github.io/pmd-${VERSION} and -# https://pmd.github.io/latest/ -# -function publish_release_documentation_github() { - echo -e "\n\n" - log_info "Adding the new doc to pmd.github.io..." - # clone pmd.github.io. Note: This uses the ssh key setup earlier - # In order to speed things up, we use a sparse checkout - no need to checkout all directories here - mkdir pmd.github.io - ( - cd pmd.github.io - git init - git config user.name "Travis CI (pmd-bot)" - git config user.email "andreas.dangel+pmd-bot@adangel.org" - git config core.sparsecheckout true - git remote add origin git@github.com:pmd/pmd.github.io.git - echo "/latest/" > .git/info/sparse-checkout - echo "/sitemap.xml" >> .git/info/sparse-checkout - git pull --depth=1 origin master - log_info "Copying documentation from ../docs/pmd-doc-${VERSION}/ to pmd-${VERSION}/ ..." - rsync -ah --stats ../docs/pmd-doc-${VERSION}/ pmd-${VERSION}/ - git status - echo "Executing: git add pmd-${VERSION}" - git add pmd-${VERSION} - echo "Executing: git commit..." - git commit -q -m "Added pmd-${VERSION}" - - log_info "Copying pmd-${VERSION} to latest ..." - git rm -qr latest - cp -a pmd-${VERSION} latest - echo "Executing: git add latest" - git add latest - echo "Executing: git commit..." - git commit -q -m "Copying pmd-${VERSION} to latest" - - log_info "Generating sitemap.xml" - ../.travis/sitemap_generator.sh > sitemap.xml - echo "Executing: git add sitemap.xml" - git add sitemap.xml - echo "Executing: git commit..." - git commit -q -m "Generated sitemap.xml" - - echo "Executing: git push origin master" - git push origin master - ) -} - -# -# Updates github pages of the main repository, -# so that https://pmd.github.io/pmd/ has the latest (snapshot) content -# -function publish_to_github_pages() { - echo -e "\n\n" - log_info "Pushing the new site to github pages..." - git clone --branch gh-pages --depth 1 git@github.com:pmd/pmd.git pmd-gh-pages - # clear the files first - rm -rf pmd-gh-pages/* - # copy the new site - cp -a docs/pmd-doc-${VERSION}/* pmd-gh-pages/ - ( - cd pmd-gh-pages - git config user.name "Travis CI (pmd-bot)" - git config user.email "andreas.dangel+pmd-bot@adangel.org" - git add -A - MSG="Update documentation - -TRAVIS_JOB_NUMBER=${TRAVIS_JOB_NUMBER} -TRAVIS_COMMIT_RANGE=${TRAVIS_COMMIT_RANGE}" - git commit -q -m "$MSG" - git push git@github.com:pmd/pmd.git HEAD:gh-pages - log_success "Successfully pushed site to https://pmd.github.io/pmd/" - ) -} - -main diff --git a/.travis/build-publish.sh b/.travis/build-publish.sh deleted file mode 100644 index 5599103585..0000000000 --- a/.travis/build-publish.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -set -e - -source .travis/logger.sh -source .travis/common-functions.sh -source .travis/github-releases-api.sh -source .travis/sourceforge-api.sh - -VERSION=$(get_pom_version) -log_info "PMD Release ${VERSION}" - -if ! travis_isPush; then - log_info "Not publishing release, since this is not a push!" - exit 0 -fi - -if [[ "${VERSION}" == *-SNAPSHOT || "${TRAVIS_TAG}" == "" ]]; then - log_info "Not publishing release, since this is not a release!" - exit 0 -fi - - -# Publish github release -gh_releases_getLatestDraftRelease -GH_RELEASE="$RESULT" - -gh_release_publishRelease "$GH_RELEASE" -sourceforge_selectDefault "${VERSION}" diff --git a/.travis/build-sonar.sh b/.travis/build-sonar.sh deleted file mode 100755 index 36cc2cc6b4..0000000000 --- a/.travis/build-sonar.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -set -e - -source .travis/logger.sh -source .travis/common-functions.sh - -VERSION=$(get_pom_version) -log_info "Building PMD Sonar ${VERSION} on branch ${TRAVIS_BRANCH}" - -if ! travis_isPush; then - echo "Not updating sonar, since this is not a push!" - exit 0 -fi - -( - # for sonar, we need to use java10, until sonarjava 5.8.0 is released (Sept. 2018) - JAVA_HOME=$(bash ./install-jdk.sh -F 10 -L GPL -W $HOME/jdk --emit-java-home | tail --lines 1) - export JAVA_HOME - export PATH=${JAVA_HOME}/bin:$PATH - - # disable fast fail, exit immediately, in this subshell - set +e - - # Run the build - ./mvnw clean package sonar:sonar -Dsonar.login=${SONAR_TOKEN} -Psonar -B -V - - if [ $? -ne 0 ]; then - log_error "Error updating sonar..." - else - log_success "New sonar results: https://sonarcloud.io/dashboard?id=net.sourceforge.pmd%3Apmd" - fi - true -) diff --git a/.travis/common-functions.sh b/.travis/common-functions.sh deleted file mode 100755 index c217bb3c6f..0000000000 --- a/.travis/common-functions.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -set -e - - -function travis_debug() { - echo "TRAVIS_REPO_SLUG: ${TRAVIS_REPO_SLUG}" - echo "TRAVIS_PULL_REQUEST_SLUG: ${TRAVIS_PULL_REQUEST_SLUG}" - echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}" - echo "TRAVIS_PULL_REQUEST: ${TRAVIS_PULL_REQUEST}" - echo "TRAVIS_SECURE_ENV_VARS: ${TRAVIS_SECURE_ENV_VARS}" - echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}" - echo "TRAVIS_TAG: ${TRAVIS_TAG}" - echo "TRAVIS_ALLOW_FAILURE: ${TRAVIS_ALLOW_FAILURE}" - echo "TRAVIS_OS_NAME: ${TRAVIS_OS_NAME}" -} - -function travis_isPullRequest() { - if [ "${TRAVIS_REPO_SLUG}" != "pmd/pmd" ] || [ "${TRAVIS_PULL_REQUEST}" != "false" ]; then - return 0 - else - return 1 - fi -} - -function travis_isPush() { - if [ "${TRAVIS_REPO_SLUG}" = "pmd/pmd" ] && [ "${TRAVIS_PULL_REQUEST}" = "false" ] && [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ]; then - return 0 - else - return 1 - fi -} - -function travis_isOSX() { - if [[ $TRAVIS_OS_NAME == 'osx' ]]; then - return 0 - else - return 1 - fi -} - -function travis_isLinux() { - if [[ $TRAVIS_OS_NAME == 'linux' ]]; then - return 0 - else - return 1 - fi -} - -function travis_isWindows() { - if [[ $TRAVIS_OS_NAME == 'windows' ]]; then - return 0 - else - return 1 - fi -} - -function get_pom_version() { - echo $(./mvnw -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec) -} - -function change_ruby_version() { - source "$HOME/.rvm/scripts/rvm" - rvm use ruby-2.7 -} diff --git a/.travis/configure-maven.sh b/.travis/configure-maven.sh deleted file mode 100755 index 0b8f10a17b..0000000000 --- a/.travis/configure-maven.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -e - -echo "MAVEN_OPTS='-Xms1g -Xmx1g'" > ${HOME}/.mavenrc -mkdir -p ${HOME}/.m2 -cp .travis/travis-settings.xml ${HOME}/.m2/settings.xml diff --git a/.travis/github-releases-api.sh b/.travis/github-releases-api.sh deleted file mode 100644 index c30c223f75..0000000000 --- a/.travis/github-releases-api.sh +++ /dev/null @@ -1,214 +0,0 @@ -# -# The functions here require the following scripts: -# logger.sh -# -# The functions here require the following environment variables: -# GITHUB_OAUTH_TOKEN -# GITHUB_BASE_URL -# - -# -# Creates a new release on github with the given tag and target_commit. -# The release is draft and not published. -# -# $RESULT = release json string -# -# See: https://developer.github.com/v3/repos/releases/#create-a-release -# -function gh_releases_createDraftRelease() { - local tagName="$1" - local targetCommitish="$2" - - log_debug "$FUNCNAME: Creating new draft release for tag=$tagName and commit=$targetCommitish" - - local request=$(cat <<-EOF - { - "tag_name": "${tagName}", - "target_commitish": "${targetCommitish}", - "name": "${tagName}", - "draft": true - } - EOF - ) - - log_debug "POST $GITHUB_BASE_URL/releases" - log_info "Creating github draft release" - RESULT=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ - -H "Content-Type: application/json" \ - -X POST \ - --data "${request}" \ - "$GITHUB_BASE_URL/releases") - log_debug " -> response: $RESULT" - - log_success "Created draft release with id $(echo $RESULT | jq --raw-output ".url")" -} - -# -# Gets the latest release, if it is a draft and returns with 0. -# Returns with 1, if the latest release is not a draft - meaning, there is no -# draft release (yet?). -# -# RESULT = release json string -# -# See: https://developer.github.com/v3/repos/releases/#list-releases-for-a-repository -# -function gh_releases_getLatestDraftRelease() { - log_debug "$FUNCNAME" - log_debug "GET $GITHUB_BASE_URL/releases?per_page=1" - RESULT=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ - "$GITHUB_BASE_URL/releases?per_page=1" | jq ".[0]") - log_debug " -> response: $RESULT" - local draft=$(echo $RESULT | jq ".draft") - if [ "$draft" != "true" ]; then - RESULT="" - log_error "Could not find draft release!" - return 1 - fi - log_info "Found draft release: $(echo $RESULT | jq --raw-output ".url")" -} - -# -# Deletes a release. -# -# See: https://developer.github.com/v3/repos/releases/#delete-a-release -# -function gh_release_deleteRelease() { - local release="$1" - - gh_release_getIdFromData "$release" - local releaseId="$RESULT" - log_debug "$FUNCNAME id=$releaseId" - log_debug "DELETE $GITHUB_BASE_URL/releases/$releaseId" - log_info "Deleting github release $releaseId" - local response - response=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ - -X DELETE \ - "$GITHUB_BASE_URL/releases/$releaseId") - log_debug " -> response: $response" - log_success "Deleted release with id $releaseId" -} - -# -# Determines the release id from the given JSON release data. -# -# RESULT = "the release id" -# -function gh_release_getIdFromData() { - local release="$1" - - RESULT=$(echo $release | jq --raw-output ".id") -} - -# -# Determines the tag_name from the given JSON release data. -# -# RESULT = "the tag name" -# -function gh_release_getTagNameFromData() { - local release="$1" - - RESULT=$(echo $release | jq --raw-output ".tag_name") -} - -# -# Uploads a asset to an existing release. -# -# See: https://developer.github.com/v3/repos/releases/#upload-a-release-asset -# -function gh_release_uploadAsset() { - local release="$1" - local filename="$2" - local name=$(basename $filename) - - gh_release_getIdFromData "$release" - local releaseId="$RESULT" - log_debug "$FUNCNAME: releaseId=$releaseId file=$filename name=$name" - - local uploadUrl=$(echo "$release" | jq --raw-output ".upload_url") - uploadUrl="${uploadUrl%%\{\?name,label\}}" - uploadUrl="${uploadUrl}?name=${name}" - log_debug "POST $uploadUrl" - log_info "Uploading $filename to github release $releaseId" - local response - response=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ - -H "Content-Type: application/zip" \ - --data-binary "@$filename" \ - -X POST \ - "${uploadUrl}") - log_debug " -> response: $response" - log_success "Uploaded release asset $filename for release $releaseId" -} - -# -# Updates the release info: name and body. -# The body is escaped to fit into JSON, so it is allowed for the body to be -# a multi-line string. -# -# See: https://developer.github.com/v3/repos/releases/#edit-a-release -# -function gh_release_updateRelease() { - local release="$1" - local name="$2" - local body="$3" - - gh_release_getIdFromData "$release" - local releaseId="$RESULT" - gh_release_getTagNameFromData "$release" - local tagName="$RESULT" - log_debug "$FUNCNAME releaseId=$releaseId name=$name tag_name=$tagName" - - body="${body//'\'/\\\\}" - body="${body//$'\r'/}" - body="${body//$'\n'/\\r\\n}" - body="${body//'"'/\\\"}" - - local request=$(cat <<-EOF - { - "tag_name": "${tagName}", - "name": "${name}", - "body": "${body}" - } - EOF - ) - - log_debug "PATCH $GITHUB_BASE_URL/releases/${releaseId}" - log_debug " -> request: $request" - log_info "Updating github release $releaseId" - local response - response=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "${request}" \ - -X PATCH \ - "$GITHUB_BASE_URL/releases/${releaseId}") - log_debug " -> response: $response" - log_success "Updated release with id=$releaseId" -} - -# -# Publish a release by setting draft="false". -# Note: This will send out the notification emails if somebody -# watched the releases. -# -# See: https://developer.github.com/v3/repos/releases/#edit-a-release -# -function gh_release_publishRelease() { - local release="$1" - - gh_release_getIdFromData "$release" - local releaseId="$RESULT" - log_debug "$FUNCNAME releaseId=$releaseId" - - local request='{"draft":false}' - log_debug "PATCH $GITHUB_BASE_URL/releases/${releaseId}" - log_debug " -> request: $request" - log_info "Publishing github release $releaseId" - local response - response=$(curl --fail -s -H "Authorization: token ${GITHUB_OAUTH_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "${request}" \ - -X PATCH \ - "$GITHUB_BASE_URL/releases/${releaseId}") - log_debug " -> response: $response" - local htmlUrl=$(echo "$response" | jq --raw-output ".html_url") - log_success "Published release with id=$releaseId at $htmlUrl" -} diff --git a/.travis/id_rsa.pub b/.travis/id_rsa.pub deleted file mode 100644 index 7e1b51e198..0000000000 --- a/.travis/id_rsa.pub +++ /dev/null @@ -1 +0,0 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC8yHVLHYDsKui8tYg/sFvkDqcs97pEZz0BzK9HtBF4O+/It1drRYRdUAFfjoImfprpKSxkJCTglHixGRp24eNaZ6woWVJ4/bmiMkEqEZAjr1NZ3qw7zIruMJMSkCV+YTtmL4cYcZlvMRPzzOZOnFbV05oi79oy41MUFHYjolK9QxMFNsVNN5iyzFxM3HqSFozz+ylKbFBtDk6ZHZQNRL/Xl2V9DJ69fVzjG4OZfcWNGmmKHHARmsnJyUOMeeKpLjDOe1M6ZdI8HkXWac8yCr9JTETNZZwemZAcS/RKoKCDqfIUOzkZfIPmyaznfVetTGsMi7yQrJhAyjznuNGF4+3lfgTcmRF8wz5FCeUkdYTmy2wNSFi5HiLPfC5OgRtjKzC6yb8rbRjDx6XQ2ph15PKOaXwzk49TaMc0xJvoiGDMZaTU0iTm3Y1/QUtfLvo3/jGMbtUdV3soWpuBAV2JUI4aB5xdLX9iNmcrVzoUe3y9DWuuTX46eoCvpUNXv/DXKhQw1D7xd7J67db5qUck/Akiqi0JR+e0SoBJvZFtYwVNLGC2bIJ/s8SR8X5Zp+1+ypf3WYjIylxQTkO1r4NfI0Cd9qXg7nmUrHAU7Z6xtJmUK8ZWzSST4wul8WkRJURtODLxt5firtlKhyZ93t9Mjuk6mATIPxr/b3x20T+IH463kw== ssh key for pmd. used for travis accessing sourceforge and github. diff --git a/.travis/install-openjdk.sh b/.travis/install-openjdk.sh deleted file mode 100644 index af59870c50..0000000000 --- a/.travis/install-openjdk.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -set -e - -# -# Downloads AdoptOpenJDK Builds from -# https://pmd-code.org/openjdk/latest/jdk-11-linux64.tar.gz -# https://pmd-code.org/openjdk/latest/jdk-11-windows64.zip -# https://pmd-code.org/openjdk/latest/jdk-11-mac64.tar.gz -# -# The Build are originally from: -# https://github.com/AdoptOpenJDK/openjdk11-binaries/releases -# - -source .travis/logger.sh -source .travis/common-functions.sh - -# OPENJDK_VERSION e.g. "11" -OPENJDK_VERSION=$1 -BASE_URL=https://pmd-code.org/openjdk/latest/jdk-${OPENJDK_VERSION}- - -log_info "Installing OpenJDK${OPENJDK_VERSION} for ${TRAVIS_OS_NAME}" - -if travis_isOSX; then - DOWNLOAD_URL=${BASE_URL}mac64.tar.gz - COMPONENTS_TO_STRIP=3 # e.g. jdk-11.0.3+7/Contents/Home/bin/java -elif travis_isWindows; then - DOWNLOAD_URL=${BASE_URL}windows64.zip -else - DOWNLOAD_URL=${BASE_URL}linux64.tar.gz - COMPONENTS_TO_STRIP=1 # e.g. openjdk-11.0.3+7/bin/java -fi - -OPENJDK_ARCHIVE=$(basename $DOWNLOAD_URL) - -LOCAL_DIR=${HOME}/.cache/openjdk -TARGET_DIR=${HOME}/openjdk${OPENJDK_VERSION} - -mkdir -p ${LOCAL_DIR} -mkdir -p ${TARGET_DIR} -if [ ! -e ${LOCAL_DIR}/${OPENJDK_ARCHIVE} ]; then - log_info "Downloading from ${DOWNLOAD_URL} to ${LOCAL_DIR}" - wget --directory-prefix ${LOCAL_DIR} --timestamping --continue ${DOWNLOAD_URL} -else - log_info "Skipped download, file ${LOCAL_DIR}/${OPENJDK_ARCHIVE} already exists" -fi - -log_info "Extracting to ${TARGET_DIR}" -if travis_isWindows; then - 7z x ${LOCAL_DIR}/${OPENJDK_ARCHIVE} -o${TARGET_DIR} - mv ${TARGET_DIR}/*/* ${TARGET_DIR}/ -else - tar --extract --file ${LOCAL_DIR}/${OPENJDK_ARCHIVE} -C ${TARGET_DIR} --strip-components=${COMPONENTS_TO_STRIP} -fi - -cat > ${HOME}/java.env < https://docs.pmd-code.org/pmd-doc-${pmdVersion}/" -} - -function pmd_code_uploadJavadoc() { - local pmdVersion="$1" - local basePath="$2" - - log_debug "$FUNCNAME pmdVersion=$pmdVersion basePath=$basePath" - - for i in ${basePath}/*/target/*-javadoc.jar */*/target/*-javadoc.jar; do - pmd_code_uploadJavadocModule "$pmdVersion" "$i" - done - - pmd_code_fixPmdLangTestStyle "${basePath}" - - # make sure https://docs.pmd-code.org/apidocs/ shows directory index - ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH}/apidocs && \ - echo 'Options +Indexes' > .htaccess" - log_info "Directory index enabled for https://docs.pmd-code.org/apidocs/" -} - -function pmd_code_uploadJavadocModule() { - local pmdVersion="$1" - local moduleJavadocJar="$2" - local moduleJavadocJarBasename="$(basename $moduleJavadocJar)" - local module=${moduleJavadocJarBasename%%-${pmdVersion}-javadoc.jar} - - log_debug "$FUNCNAME pmdVersion=$pmdVersion moduleJavadocJar=$moduleJavadocJar module=$module" - - scp "$moduleJavadocJar" ${PMD_CODE_SSH_USER}@pmd-code.org:${PMD_CODE_DOCS_PATH} - ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ - mkdir -p apidocs/${module}/${pmdVersion} && \ - unzip -qo -d apidocs/${module}/${pmdVersion} ${moduleJavadocJarBasename} && \ - rm ${moduleJavadocJarBasename}" - log_info "JavaDoc for $module uploaded: https://docs.pmd-code.org/apidocs/${module}/${pmdVersion}/" -} - -function pmd_code_fixPmdLangTestStyle { - local basePath="$1" - - log_debug "$FUNCNAME basePath=$basePath" - scp "${basePath}/pmd-lang-test/target/dokka/style.css" ${PMD_CODE_SSH_USER}@pmd-code.org:${PMD_CODE_DOCS_PATH}/apidocs/pmd-lang-test/ - log_info "Fixed style for https://docs.pmd-code.org/apidocs/pmd-lang-test/*/" -} - -function pmd_code_removeJavadoc() { - local pmdVersion="$1" - - log_debug "$FUNCNAME pmdVersion=$pmdVersion" - ssh ${PMD_CODE_SSH_USER}@pmd-code.org "cd ${PMD_CODE_DOCS_PATH} && \ - rm -rf apidocs/*/${pmdVersion}" - log_info "Removed Javadoc: https://docs.pmd-code.org/apidocs/*/${pmdVersion}/ is gone" -} diff --git a/.travis/project-list.xml b/.travis/project-list.xml deleted file mode 100644 index 600061f674..0000000000 --- a/.travis/project-list.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - Standard Projects - - - checkstyle - git - https://github.com/checkstyle/checkstyle - checkstyle-8.10 - - .*/target/test-classes/com/puppycrawl/tools/checkstyle/.* - .*/target/generated-sources/.* - - - echo -n "$(pwd)/target/classes:$(pwd)/target/test-classes:"; cat classpath.txt - - - - spring-framework - git - https://github.com/spring-projects/spring-framework - v5.0.6.RELEASE - - > build.gradle < classpath.txt -]]> - cat classpath.txt - - - - diff --git a/.travis/regression-tester.sh b/.travis/regression-tester.sh deleted file mode 100644 index 6d48090dbd..0000000000 --- a/.travis/regression-tester.sh +++ /dev/null @@ -1,82 +0,0 @@ -# -# The functions here require the following scripts: -# .travis/logger.sh -# .travis/common-functions.sh -# -# The functions here require the following environment variables: -# PMD_SF_USER - -# -# Generate a new baseline and upload it to sourceforge -# -# Note: this function always succeeds, even if the upload fails. -# In that case, just a error logging is provided. -# -function regression-tester_uploadBaseline() { - change_ruby_version - log_debug "$FUNCNAME branch=${TRAVIS_BRANCH}" - local targetUrl="https://sourceforge.net/projects/pmd/files/pmd-regression-tester/" - - local errexitstate="$(shopt -po errexit)" - set +e # disable errexit - ( - # This handler is called if any command fails - function upload_failed() { - log_error "Error while uploading ${BRANCH_FILENAME}-baseline.zip to sourceforge!" - log_error "Please upload manually: ${targetUrl}" - } - - # exit subshell after trap - set -e - trap upload_failed ERR - - log_info "Generating and uploading baseline for pmdtester..." - cd .. - bundle config --local gemfile pmd/Gemfile - pmd/.travis/travis_wait "bundle exec pmdtester --mode single --local-git-repo ./pmd --patch-branch ${TRAVIS_BRANCH} --patch-config ./pmd/.travis/all-java.xml --list-of-project ./pmd/.travis/project-list.xml --html-flag" - cd target/reports - BRANCH_FILENAME="${TRAVIS_BRANCH/\//_}" - zip -q -r ${BRANCH_FILENAME}-baseline.zip ${BRANCH_FILENAME}/ - ../../pmd/.travis/travis_wait "rsync -avh ${BRANCH_FILENAME}-baseline.zip ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd-regression-tester/" - log_success "Successfully uploaded ${BRANCH_FILENAME}-baseline.zip to ${targetUrl}" - ) - # restore errexit state - eval "$errexitstate" -} - -# -# Execute danger, which executes pmd-regression-tester (via Dangerfile). -# -# Note: this function always succeeds, even if the danger fails. -# In that case, just a error logging is provided. -# -function regression-tester_executeDanger() { - change_ruby_version - log_debug "$FUNCNAME" - - local errexitstate="$(shopt -po errexit)" - set +e # disable errexit - ( - # This handler is called if any command fails - function danger_failed() { - log_error "Error while executing danger/pmd-regression-tester" - } - - # exit subshell after trap - set -e - trap danger_failed ERR - - # Create a corresponding remote branch locally - if ! git show-ref --verify --quiet refs/heads/${TRAVIS_BRANCH}; then - git fetch --no-tags origin +refs/heads/${TRAVIS_BRANCH}:refs/remotes/origin/${TRAVIS_BRANCH} - git branch ${TRAVIS_BRANCH} origin/${TRAVIS_BRANCH} - log_debug "Created local branch ${TRAVIS_BRANCH}" - fi - - log_info "Running danger on branch ${TRAVIS_BRANCH}" - bundle exec danger --verbose - log_success "Executing danger successfully" - ) - # restore errexit state - eval "$errexitstate" -} diff --git a/.travis/render_release_notes.rb b/.travis/render_release_notes.rb deleted file mode 100755 index c7d0542cd7..0000000000 --- a/.travis/render_release_notes.rb +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env ruby - -# Renders the release notes for Github releases, -# and prints them to standard output - -# Doesn't trim the header, which is done in shell - -# Args: -# ARGV[0] : location of the file to render - -require "liquid" -require "safe_yaml" - -# include some custom liquid extensions -require_relative "../docs/_plugins/all_extensions" - -# explicitly setting safe mode to get rid of the warning -SafeYAML::OPTIONS[:default_mode] = :safe - -# START OF THE SCRIPT - -unless ARGV.length == 1 && File.exists?(ARGV[0]) - print "\e[31m[ERROR] In #{$0}: The first arg must be a valid file name\e[0m\n" - exit 1 -end - -release_notes_file = ARGV[0] - -# Make the script execute wherever we are -travis_dir = File.expand_path File.dirname(__FILE__) - -liquid_env = { - # wrap the config under a "site." namespace because that's how jekyll does it - 'site' => YAML.load_file(travis_dir + "/../docs/_config.yml"), - # This signals the links in {% rule %} tags that they should be rendered as absolute - 'is_release_notes_processor' => true -} - - -to_render = File.read(release_notes_file) -rendered = Liquid::Template.parse(to_render).render(liquid_env) - - -print(rendered) diff --git a/.travis/secrets.tar.enc b/.travis/secrets.tar.enc deleted file mode 100644 index dcf3ece91b7ceef632152ef8ca5a4fae6df1cc60..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20496 zcmV(tK@HLg^E)aD<#2{9BOpF{Wwr(h?AoVg zjMRTsg(NC|xYGWNTfGFa7?5aJE&#D;U*xyxO_Km4v8l58#h&z{t|8GX ziQ$70GFST;=3C*rs8SD;=9n|A@KkQmxWEBKz^g= zy-qvWL|XQEwyJ0@?0%%M<{g$kaK+dsiB6*FtC=b-cLvS7WhC!9Lwn*Z4^JAH8qLr+ zU|Hna`gTP|SxO33vv50257|gBgk<8L+>=m&S}C~+j3K#9gamnV!aN<$Pz#DgNA8EP ztl=Urj(W2@!<^AtZ;F(TJPbx*Q94Ay01))NDi*Kd=%ep3`5&pzq|om02pzm6{piu` z<7e2+8VUhA#;S*jS5cdxJy+BaJFZK?o$d!d&YLnS5;F1&8Vz`q{>W+869nHm*5vWz zAD6E^(V-v%R(0V1H^=sBaq|5{afPm~-JL^q6^hezcudCUK?3RaYG_l774S`G6$ z$reg*+s#r(7)T<=;W}hqh#qNs3t{OKHvn;Tl0;X=>l=E*+c`fRz^2)?8fl zrd^2W6d8~}kCSN5%#q?M?t(avvFB-TAqpx<5)R%!e)F3F#pz?bSj`Mcv6!bH_}>I# zB~)tnihQRPTOeF_m-S@!gk=K%_8f5PoNR4{+5lbW3jH zPZ82Y*6kDH=G{Wy`gQdj**CO-YH}mMs^|l5YU}&#Vxc zw+QPOTDZQC#F1l`Cis86pGPg8Tzjgv94yqb<1Oe1S57PzEsefID2>PQqoYn|apm!^ zk-9r-;|OIB6#Giu{S<6sW>m1=MPN~*ilhf2rYKw!{h#d2r8c=3#{u%xSoO3^8O=Qc z^rEuo=5G>4I<$o}G2#e;i56+GdgH0qXUj*1|;?#%c4K3*eh$xZ{>_>@Wk{s7_hF_zcEArruQ5hX9JC{cGE3`&8iN6 zw=99A|FJXuj)B&zWq_m&3TnMd!K4hsK4<}GM|5lb7Dp5V`k;JW5_%335_k`Pxx` zQlj8eF-e+`3zbzDuM{jb53%F>Wk~QvjZ$DIN)F4ROx~m0zT=OdZrcH~SxEqxS{P~- zk}6oFPI%~*yNh8e%Nk~iNb!&~(SvrEq4c25bnf!B$z80OSf@B$YGat?zFC!3Qjl7i z`M!yy!HqaVFLd9*ggFnWq)6~zgF!(3_Y+Wk=jnk74^#}OUe&BP7|<%5t*A};QH}uY zS1z_>9}o;AU9uwPCfo(5Gcifg&)xGWAmdI`v?c&Ti`*J`l3 zwi~{?l-_<)yiY(k`UXOpsZt*Wxl-)tx z@fw}l`+sIH5HQ3^!;2voA2mEe=^4FYh~0p9!p5sWo-rHl4}WVIQX3R9#06 zoy=+szb?>>c(2f4M)G4b0eQBwzq1tlAJ&M#FV*BbUf;we^+J*ciBRfvOrS^><`6`GC{r?sio52&NhYTR5h@xsBkgX#`~lwh6M&$z|ue@xMtF8@qPEJnR-Nso`Rrvo&HuaD+%{+pW9e3;n7^QLwB zbLF{54YE!7mw*iwE3ft4e%zfBhC#|P<9I~lG2RF03>{#}EU*s(ab@r)`-07uULva0LO?YIDqo)q$g%_sim6p>_st2nVC<8_D`HLOaq7EWO6Q#>y{J=ZRBnK0 zOJflHF#6LRSjlZtx~2ow*G)>)df6&=3FMbsY>_zB$p}=YFTN}j=@)xmQP2`AObO-` z@~;^={Mh{o8iIriLJ5UIgddnK%iH}2 zAyI`@&N11jpRWbNsSGF(K`QIZBx|^P2WWHLY(3bhW+0Al->^MNca9%yiW?N$VFi~}-Yty(<1A`Id{Ku2E`%(8!nV>{ zh3&*HZzpyB?;iO1jci_~NIDK+RdcZ*nD6`rf6dRM^)n{B`K!95|18?F3$A_Jz6fSP zR6Yqt9;Y-Z?!Ykz3$m#$rTk zQlGh)HsB6}W0RR;euFLt{V?xcUjEB-V-K6d_-ERT_6-$;cc8>>_8Q@+IW1TPu`}{{ zER-ff9pzWE<;JokdpV8SHNb^?ohl$;99Q!FXOEL-vWIxE6pvxuZQG4TcP7HD?dk%K z)DgD`AQgQxO)Flj87j*oYz+NNgL~puwF$$dpFe!q*k6lih2;iJ>fGAJH%VjbJUa>h zX#f|>ehO2A!vkg)a`NMzP&lft0?;=XTQz2|7Gi&&lv{dKFmBCo)h^dOUie;4upgh- z@Kr8i$6`SYlw7pu%tolRY%7FmA3Y5oGL0PU!cd)Qkh`RLk`Qv-Ix$p?XM6XG@+wzC zzv}8LZ*RXtcnln)0*c*&>8ptRWkU^tWxzik z*)-F+vBXnUkDjHY+ZIIs5mm%(C!b6)B4|T}JOd8m@asoTII#s=3ASVG5`j$%8TjQB zDst4ubf4=S42?3a_?saCS7<5*^l`Ijax@M!G_X(Lz&%S8#Ur7Hh}lO;M_CI2Y#C0- z4k{J~mY3eQXNB{5{aq#sIPg`#{TOtrsEoJBgfC7hQKg#Q+c`nRu|Bl-Fbgji|M)X=s4Vs&!4X}BK zgO3QLZszHUpsAuD2d4u?cnL-y2~r^pKt>T619n{>PhPeyHA7-7bn!3Ujt&>u>(e;o zA(UAkUxmIKNCPcnZfbz)S}>SPxAe8j!H$Z%G0UFn_IQt#c@oa3E= zE?fkIv5Y_<&qs;&9+H!lB}69PJfDgB3F2NMN9c^DTMw~w1B|Q_>eon7L95c25`Tqi zip7!B(#3(%q$j(&zD^XLsWpOXeV+4_S78`$n}zA;X80g?Vra)}7BfkGZYC+N2gdY1 zBg+?t0rAO@>dSDzhbTu-{tKg6YGur7`ZilN_m{Eb~@wBY3eAM^x8X`OB?lo12+^DM28(w5uZ$%WM=k5-2ngpTPmSS3~&@H0=+uh!rnv@&rMvoJq24;9~G=w%t?zgNdh z!($3AZgV*wM_U}W7`6&z(nx02CM<#WVJXwM=0&`b1*Pm4SQ)Q7bT~Y-tLIBDYf!GS=5yT;=|8+RONGY?%FnNzy?DdDKMDA4s zXp`fY(@~b~MKK6F55wlhk<738vn&6+F-o3wEQ(bXM`8Q4vGOUAFyhz*y~AxopRlk8e$Bx(>AM8DbXdHLY*Y(3W{+C#TG$T>yUu((JOiVvChQ1A_~26P-3odPZn0k26mn` zf*xfNdjW|+=K6B_6P%I*h#S!o7$yv!5j6>3u5pi(SE z1MTAPG{rX*U_XPr>m9i?-NEkH#rI3Itt;D!(%Cw?8h-j&DIv~8|Kp_Km&T|IUj43C zpbiX|`P&)=&V>4Zs1^;Hvd;(VF*7A)OW2f#NQ=yOH?`!yH)_4SR&8WCGsIsB%EqOU zN?#KoIXP#BDO6$D#LsVli17o`XZo~Ff@8ZWOCWL{6rCPHoRhHV^R$CpRGsQQM}y7A zE+lBp+ap-Uw)g@oY^|0k70GJvoJeEH+&pXFtS*wXXHWK(_%&-6Zf;#9YEDhuR00rc zGsydl(v!@nhYm7Sb05E`H1&~;Mz-M-+04wdOyRRLpM>790w)2x!bnPcV2ce+;ac!!CE89Ui{MA=? z2^=rv%RitqT@WCcVKR&jRAR2MI0h7J9ZMXfL}B?P`jua2c8!NolI`=KUJShw%K9_a z=8>1#Lcf+6QN@0mz2Lb0j?imEeL~f)H;okeyBE*7ImVf4vK>1_-e4u@MR2K-(ydme z<=AI3h&SoQjW@|?iw?$-0YOpLz<<^@ zVA3Ob-M?N1hwRLl{-p(ssWoWau!JnA>F%+(?S7!>25HLB<1g9Pb+Z)7xDy0ygLhpX zZ}V?HY_d+BFevOH=HcDLxE2kdY9nqYFiFErP&o2|xpf)}zWytqB=f4`4zRQAXrlq| z3DPBy=#Hrz>zK_V!N-^D@DdYcf&2Ef4?*cD3F*Y`IN#0-cIIzkiPiVz_f*R@-MBg0&N@j!Xe&y%P&axE|fEh#`@CF--jND z&7+2cE^2ZT+_Y01HNC+{zgbn`vYjUmbFL?pjIM5?!x(PQOy;9AqYVA}$R}OGpxNC6 zg>f1NELb4TPw`@F&q|GFLyKMU^bjR?qO31nQdMJaTBb+y6fU!8}i^Uh2h zm?C{~T;0$cxUikpi?*8jx4~?1GK7_k*w20B9t;`=h{+n4{`50Sz6SAj*}v7uf5A?1 z=v$?BLj3*pY+-O29_c$t*s5Tc@rI6&dnQrV^bb@b%cA$mV6FHN0t<2yU_jvX{zheW z+n5SA6ewdsj)N_-M~rq!w>v)*kZ%}AVLe`hV#yMTLn%5pd%$(dls@BZ#ctu7vetKXj(=3eS5*56cB>#b8we@&r&dpGrCOzhC-!{nB8 z-d_AbIX6d3pSre9hakdJZNPVj!Ko!!AZluRx4GNPL13Y&u<73C7W+I1IC^o{U7VN6LKz8lSynHvXT6wi!MUIZo;RAXP&CqTm|LJ&H z9!OS61mGk^299!in&XPh6@%p<`MQ^3Tc@W(1rJwIDH+ddqSU{A0ot3Yb4;gtfP~ON z?)%+ry&EhRe+jlWjlVLO-U4Lepkd3Cdu}+#q4}TjlbPAi{|sQT#tjSGy@kK5&ZBc* z7E77gBGg3~K1oIpEPCn-vq=ngkSe9tku{hu+klsT3(<>;M~K{#9_`?{g>~w{^&4lR ze<~&WjoA?5$?)q6>CsE=f6S@7Aq;RT=rd}7;@O;+N#4HJ?^aB?`1DZo^9My6I-xo= z|96?B-7!ci`~8q+lEihev53;%jaouzzdsX*4v;q?yXO3(Cg@HrBc*l|+yZ~&&c9TQ zX}=EBBX+6~bCo;|8!DjVWUNQnAoALx#kXyK=23%o4y_V0q~*DWDXmFb1*euK+aY)o z*4;I2Q|lBes+!zcDK!JiJ4W2r-ryialAoG?-wtm-wT<7niUJ zo|U$|(i@YV{x8~vZeFAA*V-E@O=2g{psIxN0NiIenPzysdEHR94RHEk4E2H_sF>y~ zg*sQs3xf%jWv`Ct89`YiDD1d$A8fF1s+=TH|V5=JvHiM0tY{7_~0?&u?;8A zO5ZVCX?!*lFTq^jZsm{KgTgyDfRm9X^#|O|6s}>4jTS!X#mJ3 zc9y-@x0Yz#{RU7IQ#*FIk2#r*fCX!=QYP3rf@EzR(R&mukFx|KS`F9X9y{U%#V5&WLtrUvaC{ouTDsNYE%-;F1TnKeCd=^)` z+uSJY)k4hLf^-^vu?yKjhTW2k1(aBM!1zfPG{}zzwb1JO$?yCu(*>L}*jV3ePmciu zA6JW|Ff`;rfoz;}|Fs_sH}8X4?z5eA)6%t67O1)}tv$U+)-&fxl!*+($h)x2fe9HhVjC}k~ZOF-nP$m zfX-D7a5sKM(9iW#x@TdCgrYf*!OVp}heR;^fKQmkE)ZsGwZh-6pWSc1 z#b0#m1XFs{dxfb?;E8pYN z9|lU=yeL|aIyxgo+Cu*ZZfWBt(hwo*K@CmBESTs(xHU35m zb!r&SWwRdnH6GxygXFZvJWmBDgRD+xfKbKV&Qhvi8_ zn=w*yxW|LSU0%IRjF*cx-Tr{>)hpu1GjDo17il{L06SFXCQ%Ng8<(3P3U*Gmo?MGQM<)Tn3N;OF^f-Wm6a6)4 z@MF`lgiBA~FnU@<_*oQ%jUw;3ozAcFIs-&@6+hh^*6fk3eU3yiClYWU6Yvp~0|r<0WxNozP+kL5-f-pFl1jx7F~wkiyW@QS;F+xI+KBgn@Dkoe#=&n^A%h7jt%0Cy0S zOETt&ASj`>XIXm#9agOkjP*?-n2xZ&5$nZfN~Jm<{>7f#;MTv#A*iuqyW&lgDZX zu#V;I>2rv&y>aOlJ8MJa^VNi(c1_~GB|!aL7B5B^tTcT`#&g9-oD&-N|N6J?;BTt> zNefVZjP%Y7+Li^zAFNryyfPGJsGW3OYD5ZryLQ0!wj=jrm=qK00Fq_#s}{Z*qu)VY zGhvIzmbT`*CE@lbx8#RcWe{Tp=Hym-e0n=DWb0Rj;9v02<=-cd#Q?2CvEo%h)BwMS zOopqhy4s4y7?PoKw{aO4Ffs`k1~oB+COpQiHKY1CF#*b5oTUB5HGd#j^vmps_tQFHa_Ja zRB@KwRfG=grv}pTk0qT$ zhHXlUh^X=H$+g<-_jF@DW%K*{3R3_r4*l%NFKZMsP_EM{- zD|A?)Zl`Q+-X~~Tm5<#($-^oj$nonbk;&M&@1>b;G!pKne}u6Y{~XEn4#ri86Q@ni z48!rZOqS|jtjl55SZWEeyRV;9u5XG&N4RL{5(9 zp4;Co{IiH&;Qj9!r-r~YNG5ve*5P;$Y8gV7o-SX~pYx3t5@(JK)Ku#M> z!mwL!b(nU&KJ4}X{A6<)Bg{Gc>@Q9u$8-!=fho_4?9pPZLMyd(sUAc<(jX6{=g0(p zZmhNY_cg4xVyyd=EDF2tg_b^1abi7{YmveplQZ2&kJY3*b3A9zUo4GjTlUGBjAk!B zGf{Q)&s1v;AUD3O=i))hqE}IyEKc_jSSUI{#F&4;|N2}?Hh;cT(zy8}7%f}|($BO1 z%(q2vy*{9BhB-NUrQ2Okj4HUdI7a`g`jAfrSD+tQM}RBY<&KB7M~q?aV6>DjWdF&r zEAKKHOaQ$O)jHVn?Ik$?O}dV5W~K@ML_2r&%bZjojk*8RBLn;4Ms!Jl)*1=wZWf&$ z3FG<$d0QpW9u5i(;Z9mQnV7BzYEtk)>8)iV9rY`ZX$U|22y`tcLVu&pCFN=H@l`fu zO@Q)S5GXopUj7CzDSrbd@Dpa|p%BtWNnX@01%9u(Lht(F#{Kdg=_y3b(pqu; zXj$5e3MHmZES7}ej>4xIQ8#%-;jub#jSf&xd-0%;Nk90!_KY ziauWRisYI8r3N$y6*h8?>It#_{0?%i`#BtjgHt6B zL0cMiF2l^vIizit|e#Z7;G>acQP?Qz|c%SL65N#$e2E9o)kJG0)I$PgO@7I z)vNaCsN$(=2D@RBV2-3`nY02Vyo|(dx^kK!?6o5M2 z0Hph|EFc5gFXgY1?r^;$7^}XehEtCsfqSuKo4uy zJ+JV&o0^M2Mu+tA@mIh_eG57!t9*0VsEueCfBrg5z09~NjKJ4d(BH+Wku1p|B{}ji z1lu_?6kt(e5=+OF6;d6$TK8){>qWLL5hYmAdMZ^I!72fic44k@*~JsQDM7YoS1W{A zoLNiqNKofK=p|o0v7+xaiWIPLu!gTw%L8B&=CMpR8ZfTI{#>zpzV3GI12(%_NjK5v zH3s-p*M^l7O{}P(pNPfXOJ1eC{k^!rUiK(MrzwC)dX5JLxsU5uLl*5gB|}HD_;Y_TGA!K3e4(`>{eoLRq3B zKlgIs&-8rc(YN%MLSCwtEpV$VcV!63Q}zx8D%+j9C0L zW06#%PNBT@D*zdG+=W;#h|{S4vYT3zX8ltA{-`24v9aS5OHiuKITfR4J5z!Er?QCf zs~Q%?pdf+VO>5#!g5{T_=dn(V zTL=(5tc}~FeNXSmES=+AloYX6opz4=NvEQno7K-X?w4s?;0NkoJ*?J4)dU+n1xZGv zon9K>C~u*OI+QfG#0lKQ#uk&zij(F8Kpc&g!VIdv;3$8H%Z4jFZwf$%eaWw|4`$!R)a;X#3*U|eA|&Hw#~Q^`D{zth{M2`4jL zM92niDY$m^ibfg@d;42o8|E|a)R%}0)I3T8s7^aO21ra(lW)(RhvH6%U|PjRaT?Ndex(fjij5N3HxFuu z>7;!WtxNX?OmIOGOagzhK%=CcrMSOayvX`2Z0MD?*zu4?<7T$Y@sYR%-g=x53G+3tk{!gU z%*sokR<~31OtLOOt#?}<{L!5#9{s%3pIoF_M-6OtAat@5mf$z{N0^O04@^azMN}Qm zrc#*o@NS$Llo=fHn@W#)-bb}bz24!0pMT{paw_U}LVqQGyHiSlQSlL9H2qz#T!jx|!;%oK$n8NA9$FkO4 zpme!psWS;0$Niy4(N_rFQv|5_j?+)Mx{8?g81suKS&^`gbLzYrf1p+4va-7iy>t_ zbuB%0T9@(JnAZ+8=x0?=fdX(v0}2dc&pXswp#(tZ1p4E9X4cjk#0^fsNWVOqJ=E=- zQjewDsBrN;p%~pMg@aE%bir?u0$mo4{vuPsu>1^NJl=}Ft}NjZ;uD|zGX63f0^{(h zBqtcUT84SL?uw%rLc%b9M@JutOT0oElHua?&Wk|nPml835HM%Ugg{M@XnJfE9Z9?CqpKs_|UBHNt7YQoa#G!Nh!RFKfC?*6+V+B3C4VG~&Ty9{x;6EZ1ar=Ju zD7kaq?s;+rrMnTgDVGV*7Dr!iVaZOn@0p1?1kf1iSW8k+NeAD(YzzT z)9o=Rt_rJm*7v{5At82zyV&eevkV;Zog=_rQ)G^~5MY%*)J620a1UN-AGYvqoAItf zp&EYbw-N2{M7at+<5D)F{)Nu);uIA7eY%_WVAYj5(kZ!-%~yKo=!N>Y#7}W6Xj`2W zl4tC$9N+b)Vz50T8Mq*ha>bVD=`?ATo!FI1E$r(DjrSy86!Mlc6mAKw-{;T~*tIrC zOqH@U%;mJGF-*H3U6IYqJ(e51QkKJ-Pc30byeI}Q^`Cg@6jwD1YGtlsSh<&EwU5gx z-W2_!jL-ZtCE{_Dj!TD#dOnwGyVI?X2oI6(87-KtA3_HfsLRW&dSz*0cS3SW)PiEi zmM(T*ZKhw8;%u~2`xno5&BoPvXpwGU3r)#oIZsJ&oB-$tCTcA^=vpfTZ*iCkZ%L0K z!ZuOAZuzMfE;HOoPfw;tu`-RW))PJpE!IiMu8IW@({x*>2~kpUrV_xoPy&blJTGTV z9`rZXKyl^!8a?wZ>5vmBw7oV`*Q3^fnQW=Yc@PmF;aTVy*|R%o`ty_j-GVEMd+Y6R z(@q4=uHP!UWn*uehF_z!;|a;~%AAiYq*N&UZD!^%MB`Prr}qT0S&PVASzF{WN+Wb7 zBR=QH85u%QDA(3<*42S?E#H=Wkb%dY9t$^1P?!G!{Bnn->H{f5EpP_NN5hPg>-`et zh_(_@Lo9?-t*7wN6E3dUrCW4{wleNnWgU33QR@;mWp4v-g~-IsNq}MnYldSfJjKWU zHfOEpW{5iaKA(qq#<(`}t?JS;>dzMJ#E5WvH&-uWMzR zlA3a$%YgPbEkZnNdP`_|n5R-rUT#aSD2bJ0F@6*TA{`waHiZIzeTx>vw_~X{0Q^1N zG3=JYWXSQI3c|PlWGCr24JA%yRsMCM6c0t%=o?4vjl>iiJ>~cb?YpS;6xY3;Kn8`> zimlZ%lOD6W84F|OvtiGf(4oO{x$n=j29Grj8YZZJv_Ch}D@oH&K=1Xw_?nLMNhBwZ%1D^?mCkD*~fu)O-*ExK65$C>1`Y9Zn7jrojQn4q7=|gBgdksK{Oc z(|lw+7E`nU6)1cRG7Vyu;2@4yV)d=j2&z8Rh0t(+E_3OU(&f9_D(;6hD}Q{Ml-=YQ zDHDjjeeZ1glq*$H0hH3c`OYP;ooz&wZ_b0)cvLc*Dub!JBn2U$NQ_=^7UakWqO)U^ z(P9no_eU~8>fn3=(p0kPNvLkj5p(|H`-l5f7l8ndU<}?X{-_5fm9}9^6(QnuJz-61##fjxG}FV($gK0&6N?d zhQ1zvuj8o({p9vIzd_4W;K22-J^+bmaJZ`ebVe8j1R6v?X)7Tt3;2^Tvl1xMQ6?>k z<;;_Z*i5gZ4>i9p-X)k<@u!ZeG~fl4>dn0}T<&qlX@3r%$BUi(&HM-p_6CV9LZX3! z^f!%*GWWt*J^@1L+f^<~iw3Hge}@ftI_;fBWl-#GRF@iZMs|*zbt41M<}t_4wgo^} z_h`4*6kZ%}J~I~r#DH#;;fZn?q1faPV&>^T{{!O{1NBz7vQbWYOOzzcd#mJLhLf#F z{%d-9n^4>~pt+6am_;PLE>Q{A>b5A-=T-bs#9DLDY50zVQ*5v*v;Ot1DVx520c=-K zb-tKpIQ^gY8jP~^kDm>rGW0}uzPQ8l65}bhI~YYFTQKq2`M@)BX`7|n$cp8~qwIZh zk$w`O?x8RSx3)&J1~$dc|KdWrF?VL5MVSyzBUKA|sT!vERemiD-LKbuzgFFyX10-g zYm>X8vQiqB&++FYaDRRS2l2Zad=R~Ima0+d_ua}5i@p-^_d+!sTU*_gcX*(F%y*1L zI=V>LpZ`9Jf#MLP6qaxcM_5j6^i6OZLU6T6Sbi5BzX48rDjz_o$_?el+&KCm_Tle2 z7DEK;zAD|(cAl<NFtcUn*W@#Hf}ej40$Fp^W#6t%YXKddRaF;2!BkRp(r@P;m)E zBq;^cR9uxQT*4Lrf*b-4Xh&A^ZqT)m+1(PNY_@OIyxvyBtqjo#J1TbzA%vRs3IE^4 z*v&qaDQD!^HGVS3eF}JEYX~4mU6pWvq1{{sZP>x9 zbZ!7=hnNOn?@MXOY~vb+;(|&(ToCpg++U?!!shBV)wEUA(KgV68R_SIb!mCg6TCG? zgIgAUCjW+fH+?Rfu}VeO8;`}lM(JCT;*NWA8(pKH7vCSg?s<|GpB+72jBq|x98+87 zvF%k$k!VJ563~>Hf04SJ8SCIqy!y!%GX~$_Oc0xBpZE=w%ySZ6t|Wj}S^y{Qw5{gW zIR>w@1D|SdO|MBXa&S|dM%3@D237wTvPgC94+(Zyw|q{g)H`dfy&wgh{gH9=nUoDv z;-pSq-W)A?5G+BY0qt^K61`R(X=IH$E?IzcSp>G^8>d7{QJJ4_n*De$(2|>@{(Q-O&j={>L1uaJ04iR7iHU7U~sm z(8u`};$_%ICtW*z8%J})Au~J2>Z#z`ahm!p%Fvz0D=VtGL|IC|Sj146?f7C}153IK zP8%~;#H`WIL%yk%o@!jpoU^M)UUz$)x}j>pG=1K$G#5{|td14t$xiV8$mF?+D%HME zk(`1HR@#L}k`cDSrTz)7y>6YeAPw&Jl~vx3ke4L+N{l+hgJ&sWV76m7?7!gfqT4F* zmlpK%+p%$@DW=f^9Bqg5cm@9Dq}7Wo2juHePfQLWAP}_=jk0zz)=`h{rFQ<~g1SDj zu9O6(wE{yJgN@#bIq*l1E8E{!{mz14V-F^C93W^dQU13AE(ZAAQvy{xj zY9`?d1Jaxp$3D_gr3TeymZ5NE7cJb~zq$m2Rv)MR&x`shgm*|R{lLl5mBvlgV~H)> zKzIS`%Vi~!n5OXfN-7L`LZ6^lwSaRt(TPga$FKwb>@s-jSBa?8OEPej z#%EAQ~KO?lq~I+~8k$_hRjT#4LW$F3QM@fbw_JvXQPJ{XqaulMdpK?*{b~H zNbj)4gxO?-EYcrAUA!9xV=;0t2oxJ<0yeqb(_yLp`y0|YJ$`K$_DN>Sc%6D{5HmPo z@cP`fScF2P{C0hWDDG91L8=V8?u3XvoOqOseAFvithYk#S^}o$PYFG&Rh|*AIi9RO$Hp-5l z)L2^nXRfRtcJ<@vW5$XnJ{e=w=l=XNt^x$ebHpk&v^If@P7;#-cc>vs2N?b`*X)oH z3JQ%~PtPaX|9y?28??flc+34v92$9hQmQ$$gQS_u7uts^&+AfZ5?(5@DwU}TMqU-57MPR4I-PB2EqXZjf$1DI0qlQQjMLr`17>%&`Xu#b>g zM8)|Gp0|YItl!sQ>cDu|qUTW~)!``=rUA=ZuS@&0@0d-sIou7H8lzMd>H*^3aRA`a zaYZyvIk^}WZ$IA^+&~p7LXc2`7P4#br*OYc*Q&iquM*juuN)5iR+XYcAWQ^$E(AM= z$4sP0@dyaeIuv7{D^mC@O(Ey6nI*P)w0^ScIP{dNrdUYt-Ev!7V4p&?L?hyQlTzgi zBg#QAY#rjgeTa_JboK|@QBS|sPl!`h&XX$1*qldDt=LwC!A?AEvBXmd5$r&1Zgh{g zwfI4ZCXC{p>tK5DEFFrf zjkgDpNEOdXzt`!rq}Lz^yaL1Cv`m~Qbe&kZ4|I%_X^`j)LLHDfhHo63@D)O*-BHv^ zYE$)2!YM{SFX|EWo- zJV_rb=4H%Ug^}fzBOS&*UG1`CrQ;BIX{I-iMa%`Nz#ZUH7@g(cB}-D`^=Z0kZ%Wck z)JG;j!P`ZX*BZ2Q^SCUUn3T7s=1s)*j!tss6`3l6Jp0|2ZI7304jWK`=o&L#of<4O z8VjQw(|vm^13F^I(Q4`YUc}!DG)C}I&EGPL3RlNdeW}h3y3L={MIH#HwTZoV(q9=q zBO%s&hU`bP`#gjnNl}oqCfp;Kd*i?uvcZC27+lqN{_|iM{DVZjcs%=Avq2*X7 zN*$Xe>FUN&qjmG#z(_C`U;g7-I}EvGQ3z;dHgIqt1@ry&fbCvcMUi?2!Ll2WV4Grj*kC|D)=!VYziv%0?PKpU49A&|CN=(UTVYIlT=qX$7$F6#?p`sc zZaeYWWRvgc2~sCQ4({e3LP5#z6#mcfI0Bsn2*ZlwGugs!N?#4TywY7GL) zkzA86g8>nFH2rxY=fWI(6F=(rUltMGTK?~{X-J1?Xv^P-DG8pJE;Z>PZDtcnJeaY` z&j8CDi=0lMP&H9^|f~M z-cTTIvt5pBI#5dOeCKo^r|?6ew{ygW4em&wPW{23#Dy6c^VjY>aw?nqMw*~yA%2(Y zI8IvU0`!8I5P*(BEh$ZO<)lZ|RGDlBuNh_Hc2CyI!+TA8{9U9}PormGJ<@8OLPC~O z6^4UDjU^Gz1_x7xyTDo!zYNtu;{1~r?T%O@0aqP7-Xg|7Uj?0`$|^MgE(L)VY0pnVFd&GIy?WN_y3(l#HfK9cl$& z+3BR%Y~bMy87K;>7dn(#RBn>B=`e3BQim_Yb-Fu~^Y@>DJV_FSz){{X4wq$V*@Bxu z&?KOkto;XGvi4Pjkg#ZdBTTP5WvyttDrL1nJtCnA`6J-&Pzk`$^wB`dq+c%N?=!vm zie^nwIWEzQoBzX3Rb(q5U;8vH_-6$AN~m=TiJN&UxM(dJRWUl+qt!5mA!!M1Arv08 zPaS0AWVIlpS}gwamXD%(d|vXhb4WNb6b6DzJdi21H#9QQXWuA^KC?$NTrw4pUNeXS z2DyL|D&V{FSKq1Z0XEh~^I%@KB8NM~5ckmH{rF?@ zTqtJ2Su;|Xt0NVpO-(|*)QdxmYrP^9ie^8wo6#7KTG(jT2L1EKC6K5o#rldqIsVmBj%5%`H;G&kYmh1TaK^mN<2tJX0;z&3v$0R>Kw4y3a!@wcNV$nT z{h%x@`|Ue71&Uc`zyZa=&B|>OF|7*O6F#2D1UIGzWqXqo@BSUn>#ojujspOqudQ=i zp`6!8@Qb=E$191pF+MdoCghfCnQqDAW8{xoeze>aN_6RVfy1-~O*uN4l>E?#<6!*b zC-Ov6oln<~yZs05D*8H+lJT$vYfrNR!xR$y|y>?P^$O?H9K=_8@=FJjJIu(?uy}M+Ovg@&J2xXH!%{g(b4g@;V7_! zK(gF-vPVL+$Uv!ggv{asS^2@mf+kLzn3PiNT1%XSa-DM>@Az82;r%@uFE zRR{_C#u*T{VdJ@y>}Q>^c9@Ir8?u8J-}+d4bnOO3p?eHoQ$A@AxwEI9PsCAYwa)|W z5r=(cj_ezjl`T)C<(~)P=1ngi!5Hxl%nlaZ9(2Vn8WIG|6d>rsE-D_>W1k{r;9Jd2 zZ{iLJMd7!~%we2Oj;T^}v-AiwZJjQp;q#~vpddLGN7)!5X2`Q%qLIE$?O+C9!nA%q za~>eZw^J?gh-h}5AFPU^77{UjTiyh67e|x|kexgLv4z7%>TX~m!d9rW<+Lr^Zy3vP zo|=^A7F8fi9|PljVtt9!ZetNiDm}eoh+l#}jV_hc$rL>NAdro5fNygNHKK+zWuT>^ zzkC0f%S1lX;qJ{Qxu2tJreoLYvb2^dEmEfe>SJ*nZL3*w=+iE32R-q|7{g(;XRe;Jl9Q$C#)61VH z2O2vc$)*|G{ET|T&uTtlk|W^rBB$;MC;8Ft=+yXvkueaoXpLkDrUoDYz}A|GliQV@ zC}InmPnDWBOuYrJDSC<-YW^!<)Aa@uA(mzXv0G)cb*vb#(cAi$)Y1x2`B(qD5x~nU zZ%I{6DH|^&oAM&`%8!fTFl`T+O$ZrPb=obKhJ9YwLVX{f(-Y)fBu! z;5$;!ErApo{B9ld`cma;U~hC!#qjPvEz+vItw>7Mp@tvDj-OCOK0FlyCJ=t#hiN@J|R)B*6$ZnXF-jOF}_7Uk&h zK72tLtYa~1g~2IdscYLI6&O)fx-JMs975TfpY6+BCcBTncHjIF1_8Iw-}kmWjt zxj&=QSMx4of6?1mRNu7?>*jj%;^iqInFc$Iwj&KWppXeJLW{K+sybZnhVO!=v!8cM zeYjkklAyXbqbsb!XD+B=zDI!!#hkG__U{D1GX z39`nP?Vnwqo-6X0|_Z zYs1vOKfsM9MG&%lvQU{CdUPc<76uJ{mlyn{1-9=DjbW@B=z-Xp6ysrmfk+BI(~eMa z4ZF5#;J`O2Y05-3^cAb{T?Ks;-EubA0gETa$IR?qzrQW$8$hyK1k||O^O08M^D~_P z4v3uZ?$n0(%_Gj`1K#D-7;-w!hLi9|pWpfxrF&>^v_$}WE)LWA;fM9I5>-wgN&e*r zhAt@~7N}O)A@||}Ri=yDe*3vS%ffk|3!dY*rouv#V*;#NXyR2<8b4Ti=BDKA;=W*e zveQ3eVG}ZGNd=^9jh`b;^-t7OJSdEZYWi^6Om==-R z%KSz!@{macgY0bQe-g7zf7tg@D`D^Kut^4ef@z#nse4~LAWJ!+qx=yc7?bM^iD9Dj z-L&)m{Njko+2hGx65@TqDK-qa3b%IO3t3jKf?UF=HpwDHxS^f5m)K=fS`aFa4Xl9G zGObS=&qKKXT+P>XRJqhDgbiTH5%z?TEcexH)1}wcXuOk$A8fxcQ+v?a8vUnQU;chn z%u5J3vchYSvhN7Zq7bIV*84Vd{kSj#!bs z8yyw@Omg0?Rm}kC9<9-RLnLT*^duWu-YEXAQv)epADRI020GP;O@#UVE632jtgA+# zd-pHuVWU;AfxY198!K2eKke_AtGiFgfzfk%>?@R> "$HOME/.ssh/known_hosts" -echo 'web.sourceforge.net ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCwsY6sZT4MTTkHfpRzYjxG7mnXrGL74RCT2cO/NFvRrZVNB5XNwKNn7G5fHbYLdJ6UzpURDRae1eMg92JG0+yo=' >> "$HOME/.ssh/known_hosts" -echo 'web.sourceforge.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOQD35Ujalhh+JJkPvMckDlhu4dS7WH6NsOJ15iGCJLC' >> "$HOME/.ssh/known_hosts" - -# add pmd-code.org (ssh-keyscan pmd-code.org) -echo 'pmd-code.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVsIeF6xU0oPb/bMbxG1nU1NDyBpR/cBEPZcm/PuJwdI9B0ydPHA6FysqAnt32fNFznC2SWisnWyY3iNsP3pa8RQJVwmnnv9OboGFlW2/61o3iRyydcpPbgl+ADdt8iU9fmMI7dC04UqgHGBoqOwVNna9VylTjp5709cK2qHnwU450F6YcOEiOKeZfJvV4PmpJCz/JcsUVqft6StviR31jKnqbnkZdP8qNoTbds6WmGKyXkhHdLSZE7X1CFQH28tk8XFqditX93ezeCiThFL7EleDexV/3+2+cs5878sDMUMzHS5KShTjkxzhHaodhtIEdNesinq/hOPbxAGkQ0FbD' >> $HOME/.ssh/known_hosts diff --git a/.travis/sitemap_generator.sh b/.travis/sitemap_generator.sh deleted file mode 100755 index 1d38e8ee96..0000000000 --- a/.travis/sitemap_generator.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -# Sitemap generator -# Assumes we have the latest version of the site under "latest" and "pmd-${RELEASE_VERSION}" -# https://www.sitemaps.org/protocol.html - -WEBSITE_PREFIX="https://pmd.github.io/" -DOC_PREFIX="latest/" # "pmd-${RELEASE_VERSION}/" -DATE=`date +%Y-%m-%d` -# Priority is relative to the website, can be chosen in {0.1, 0.2, ..., 1} -# Default priority is 0.5 -LATEST_PRIORITY=0.8 - - -# Writes to standard output - -cat << HEADER_END - - - - - ${WEBSITE_PREFIX}index.html - 1 - monthly - $DATE - - - - ${WEBSITE_PREFIX}${DOC_PREFIX}index.html - 0.9 - monthly - $DATE - - - - -HEADER_END - - -for page in ${DOC_PREFIX}pmd_*.html -do - - cat << ENTRY_END - - ${WEBSITE_PREFIX}$page - $LATEST_PRIORITY - monthly - $DATE - - -ENTRY_END - -done - -echo "" - diff --git a/.travis/sourceforge-api.sh b/.travis/sourceforge-api.sh deleted file mode 100644 index 41933c9a21..0000000000 --- a/.travis/sourceforge-api.sh +++ /dev/null @@ -1,166 +0,0 @@ -# -# The functions here require the following scripts: -# .travis/logger.sh -# -# The functions here require the following environment variables: -# PMD_SF_USER -# PMD_SF_APIKEY -# - -# -# Uploads the release notes to sourceforge files as "ReadMe.md". -# -# Note: this function always succeeds, even if the upload fails. -# In that case, just a error logging is provided. -# -function sourceforge_uploadReleaseNotes() { - local pmdVersion="$1" - local releaseNotes="$2" - - log_debug "$FUNCNAME pmdVersion=$pmdVersion" - local targetUrl="https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}" - - local errexitstate="$(shopt -po errexit)" - set +e # disable errexit - ( - # This handler is called if any command fails - function release_notes_fail() { - log_error "Error while uploading release notes as ReadMe.md to sourceforge!" - log_error "Please upload manually: ${targetUrl}" - cleanup_temp_dir - } - - function cleanup_temp_dir() { - log_debug "Cleanup tempdir $releaseNotesTempDir" - rm "${releaseNotesTempDir}/${pmdVersion}/ReadMe.md" || true - rmdir "${releaseNotesTempDir}/${pmdVersion}" || true - rmdir "${releaseNotesTempDir}" || true - } - - # exit subshell after trap - set -e - trap release_notes_fail ERR - - local releaseNotesTempDir=$(mktemp -d) - log_debug "Tempdir: $releaseNotesTempDir" - mkdir -p "${releaseNotesTempDir}/${pmdVersion}" - echo "$releaseNotes" > "${releaseNotesTempDir}/${pmdVersion}/ReadMe.md" - - log_info "Uploading release notes to sourceforge for version $pmdVersion" - rsync -avz \ - "${releaseNotesTempDir}/" \ - "${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/" - - log_success "Successfully uploaded release notes as ReadMe.md to sourceforge: ${targetUrl}" - - cleanup_temp_dir - ) - # restore errexit state - eval "$errexitstate" -} - -# -# Uploads the given file to sourceforge. -# -# Note: This function always succeeds, even if the upload fails. -# In that case, just a error logging is provided. -# -function sourceforge_uploadFile() { - local pmdVersion="$1" - local filename="$2" - - log_debug "$FUNCNAME pmdVersion=$pmdVersion filename=$filename" - local targetUrl="https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}" - - local errexitstate="$(shopt -po errexit)" - set +e # disable errexit - ( - # This handler is called if any command fails - function upload_failed() { - log_error "Error while uploading ${filename} to sourceforge!" - log_error "Please upload manually: ${targetUrl}" - } - - # exit subshell after trap - set -e - trap upload_failed ERR - - log_info "Uploading $filename to sourceforge..." - .travis/travis_wait "rsync -avh ${filename} ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/${pmdVersion}/" - log_success "Successfully uploaded ${filename} to sourceforge: ${targetUrl}" - ) - # restore errexit state - eval "$errexitstate" -} - -# -# Select the given version as the new default download. -# -# Note: This function always succeeds, even if the request fails. -# In that case, just a error logging is provided. -# -function sourceforge_selectDefault() { - local pmdVersion="$1" - - log_debug "$FUNCNAME pmdVersion=$pmdVersion" - local targetUrl="https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}" - - local errexitstate="$(shopt -po errexit)" - set +e # disable errexit - ( - # This handler is called if any command fails - function request_failed() { - log_error "Error while selecting ${pmdVersion} as new default download on sourceforge!" - log_error "Please do it manually: ${targetUrl}" - } - - # exit subshell after trap - set -e - trap request_failed ERR - - log_info "Selecting $pmdVersion as new default on sourceforge..." - local response - response=$(curl --fail -s -H "Accept: application/json" \ - -X PUT \ - -d "api_key=${PMD_SF_APIKEY}" \ - -d "default=windows&default=mac&default=linux&default=bsd&default=solaris&default=others" \ - "https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}/pmd-bin-${pmdVersion}.zip") - log_debug " -> response: $response" - log_success "Successfully selected $pmdVersion as new default on sourceforge: ${targetUrl}" - ) - # restore errexit state - eval "$errexitstate" -} - -# -# Rsyncs the complete documentation to sourceforge. -# -# Note: This function always succeeds, even if the upload fails. -# In that case, just a error logging is provided. -# -function sourceforge_rsyncSnapshotDocumentation() { - local pmdVersion="$1" - local targetPath="$2" - - log_debug "$FUNCNAME pmdVersion=$pmdVersion targetPath=$targetPath" - local targetUrl="https://pmd.sourceforge.io/${targetPath}/" - - local errexitstate="$(shopt -po errexit)" - set +e # disable errexit - ( - # This handler is called if any command fails - function upload_failed() { - log_error "Couldn't upload the documentation. It won't be current on ${targetUrl}" - } - - # exit subshell after trap - set -e - trap upload_failed ERR - - log_info "Uploading documentation to ${targetUrl}..." - .travis/travis_wait "rsync -ah --stats --delete docs/pmd-doc-${VERSION}/ ${PMD_SF_USER}@web.sourceforge.net:/home/project-web/pmd/htdocs/snapshot/" - log_success "Successfully uploaded documentation: ${targetUrl}" - ) - # restore errexit state - eval "$errexitstate" -} diff --git a/.travis/travis-settings.xml b/.travis/travis-settings.xml deleted file mode 100644 index d3c3c6446c..0000000000 --- a/.travis/travis-settings.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - org.sonarsource.scanner.maven - - - - ossrh - ${env.CI_DEPLOY_USERNAME} - ${env.CI_DEPLOY_PASSWORD} - - - - - - - ossrh - - ${env.CI_SIGN_KEYNAME} - ${env.CI_SIGN_PASSPHRASE} - - - - - diff --git a/.travis/travis_wait b/.travis/travis_wait deleted file mode 100755 index d81dd41089..0000000000 --- a/.travis/travis_wait +++ /dev/null @@ -1,324 +0,0 @@ -#!/usr/bin/env bash - -# The MIT License (MIT) - -# Copyright (c) 2015 -# m3t (96bd6c8bb869fe632b3650fb7156c797ef8c2a055d31dde634565f3edda485b) - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -# Available from https://github.com/m3t/travis_wait -# Please report bugs at https://github.com/m3t/travis_wait/issues - -# Coding (Style) Guidelines: -# https://www.chromium.org/chromium-os/shell-style-guidelines -# http://mywiki.wooledge.org/BashGuide/Practices -# http://wiki.bash-hackers.org/scripting/style - - -# bash available? -if [ -z "$BASH_VERSINFO" ]; then - echo "Please make sure you're using bash!" - exit 1 -fi - - -# INITIALIZE CONSTANTS AND GLOBALS -# Only lower case, esp. for export! -# That ensures that system vars stay untouched in any case -readonly prog_name=$(basename "$0") - - -is_writeable() { - local var="$1" - - is_writeable_empty "${var}" 0 -} - -is_writeable_empty() { - local var="$1" - local empty="$2" - [[ -z "${empty}" ]] && empty=1 - - # http://mywiki.wooledge.org/BashGuide/TestsAndConditionals - # "touch" creates file, if it doesn't exist, - # so further tests won't fail at the beginning - if { touch -a "${var}" >/dev/null 2>&1; }; then - if [[ ! -s "${var}" ]]; then - if [[ ! -w "${var}" ]]; then - #show_warning "${var} is not writeable" - return 1 - fi - else - #show_warning "${var} is not empty" - [[ "${empty}" -eq 1 ]] && return 1 - fi - else - #show_warning "Destination for ${var} is not accessible at all" - return 1 - fi - - return 0 -} - -is_number() { - local int="$1" - # http://mywiki.wooledge.org/BashFAQ/054 - [[ "$int" != *[!0-9]* ]] -} - -is_empty() { - local var="$1" - - [[ -z "$var" ]] -} - -show_error() { - printf "\n%s\n" "${prog_name}: error: $*" >&2 - exit 1 -} - -show_warning() { - printf "\n%s\n" "${prog_name}: $*" >&2 -} - -show_help() { - # http://wiki.bash-hackers.org/syntax/redirection#here_documents - cat <<- EOF - - Usage: ${prog_name} [options] [] - - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor - incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, - quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo - consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse - cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat - non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. - - Arguments: - Slowpoke command - Where 's output will be saved - Default: \$RANDOM-output.log - - Options: - -i, --interval Refresh interval in sec. - Default: 30 - - -l, --limit Limit execution time in sec. - Default: 0 (Off) - - -x, --exit-code Force the exit code - Default: -1 (Off) - - -a, --append PRN append output to existing logfile - Off: 0 (Default) - On: 1 - - -h This help screen - - - Copyright (C) 2015 m3t - The MIT License (MIT) - -EOF - - exit 0 -} - -cleanup() { - kill -0 ${pid_slowpoke} >/dev/null 2>&1 && kill ${pid_slowpoke} >/dev/null 2>&1 -} - -main() { - - # INITIALIZE LOCAL VARIABLES - # Variables to be evaluated as shell arithmetic should be initialized - # to a default or validated beforehand. - # CAUTION: Arguments' (not options) default values will be overwritten here anyway - # So they are set in VALIDATE INPUT - local i=0 - local msg="" - local time_passed=0 - local pid_slowpoke=0 - local exit_slowpoke=0 - # Options: - local interval=30 - local time_limit=0 - local exit_force=-1 - local append=0 - # Arguments: - local cmd_slowpoke="" - local file_log="" - - - # SIGNAL HANDLING - # http://mywiki.wooledge.org/SignalTrap - # http://mywiki.wooledge.org/SignalTrap#Special_Note_On_SIGINT - trap 'cleanup; trap - INT; kill -INT $$' INT QUIT # CTRL+C OR CTRL+\ - trap 'cleanup; exit 1' TERM # kill's default signal - - - # COMMAND-LINE ARGUMENTS AND OPTIONS - # http://mywiki.wooledge.org/BashFAQ/035 - msg="requires a non-empty option argument." - while :; do - case "$1" in - -h|-\?|--help) - show_help - exit - ;; - -l|--limit) - if [ -n "$2" ]; then - time_limit="$2" - shift 2 - continue - else - show_error "--limit ${msg}" - fi - ;; - --limit=?*) - time_limit="${1#*=}" - ;; - --limit=) - show_error "--limit ${msg}" - ;; - -i|--interval) - if [ -n "$2" ]; then - interval="$2" - shift 2 - continue - else - show_error "--interval ${msg}" - fi - ;; - --interval=?*) - interval="${1#*=}" - ;; - --interval=) - show_error "--interval ${msg}" - ;; - -x|--exit-code) - if [ -n "$2" ]; then - exit_force="$2" - shift 2 - continue - else - show_error "--exit-code ${msg}" - fi - ;; - --exit-code=?*) - exit_force="${1#*=}" - ;; - --exit-code=) - show_error "--exit-code ${msg}" - ;; - -a|--append) - if [ -n "$2" ]; then - append="$2" - shift 2 - continue - else - show_error "--append ${msg}" - fi - ;; - --append=?*) - append="${1#*=}" - ;; - --append=) - show_error "--append ${msg}" - ;; - --) # End of all options. - shift - break - ;; - -?*) - show_warning "Unknown option (ignored): $1" - ;; - *) # Default case: If no more options then break out of the loop. - break - esac - - shift - done - # Arguments following the options - # will remain in the "$@" positional parameters. - cmd_slowpoke="$1" - file_log="$2" - - - # VALIDATE INPUT - is_number "${interval}" || show_error "Interval is not a valid number" - is_number "${time_limit}" || show_error "Limit is not a valid number" - is_empty "${cmd_slowpoke}" && show_error "Command to execute is not given. See --help." - is_empty "${file_log}" && file_log="$RANDOM-output.log" # http://mywiki.wooledge.org/BashFAQ/062 - - # START CMD - # http://mywiki.wooledge.org/ProcessManagement - if [[ "${append}" -ne 1 ]]; then - is_writeable_empty "${file_log}" || show_error "${file_log} is not writeable or not empty." - ${cmd_slowpoke} > "${file_log}" & pid_slowpoke=$! - else - is_writeable "${file_log}" || show_error "${file_log} is not writeable." - ${cmd_slowpoke} >> "${file_log}" & pid_slowpoke=$! - fi - - - # WAIT - # Terminates when $cmd_slowpoke is finished - # OR - # $time_limit has reached - i=0 - while kill -0 ${pid_slowpoke} >/dev/null 2>&1; do - : $(( time_passed = i * interval )) - - printf "%s\n" \ - "Still waiting for about ${time_passed} seconds" \ - "Used disk space: $(du -sh .)" - - # Output last line from $file_log - tail -1 "${file_log}" - - # $time_limit - if [[ "${time_limit}" -ne 0 ]] && [[ "${time_passed}" -ge "${time_limit}" ]]; then - cleanup - break - fi - - sleep ${interval} - - : $(( i += 1 )) - done - - - # FINISHED - # Shall I fake the exit code? - if ! is_number "${exit_force}"; then - # Get exit code from child process that is terminated already, see above - wait ${pid_slowpoke}; exit_slowpoke=$? - else - exit_slowpoke=${exit_force} - fi - # Output last couple of lines from $file_log - tail -5 "${file_log}" - show_warning "Your given command has terminated with exit code $exit_slowpoke. So do I." - exit ${exit_slowpoke} - -} - -main "$@" diff --git a/README.md b/README.md index 20c3cf3b1e..d07be46f1f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ [![Join the chat at https://gitter.im/pmd/pmd](https://badges.gitter.im/pmd/pmd.svg)](https://gitter.im/pmd/pmd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Build Status](https://github.com/pmd/pmd/workflows/.github/workflows/pushes.yml/badge.svg?branch=master)](https://github.com/pmd/pmd/actions) -[![Build Status](https://travis-ci.com/pmd/pmd.svg?branch=master)](https://travis-ci.com/pmd/pmd) [![Maven Central](https://maven-badges.herokuapp.com/maven-central/net.sourceforge.pmd/pmd/badge.svg)](https://maven-badges.herokuapp.com/maven-central/net.sourceforge.pmd/pmd) [![Reproducible Builds](https://img.shields.io/badge/Reproducible_Builds-ok-green?labelColor=blue)](https://github.com/jvm-repo-rebuild/reproducible-central#net.sourceforge.pmd:pmd) [![Coverage Status](https://coveralls.io/repos/github/pmd/pmd/badge.svg)](https://coveralls.io/github/pmd/pmd) diff --git a/pom.xml b/pom.xml index 9516b42d59..4085faae59 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ https://pmd.github.io/ - https://travis-ci.com/pmd/pmd + https://github.com/pmd/pmd/actions 2002 From 4f2e8b6c8fd21b670f6d42049a8c0b0c5554d98f Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 14 Nov 2020 14:18:04 +0100 Subject: [PATCH 49/52] Fix docu --- .ci/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/README.md b/.ci/README.md index 3c5bc286b0..8c1afbeab3 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -109,7 +109,7 @@ export PMD_CI_GIT_REF="refs/heads/${MAIN_BRANCH}" export PMD_CI_SECRET_PASSPHRASE="xyz" -cd /workspace/pmd +cd /workspaces/pmd rmdir pmd && mkdir pmd cd pmd git init From 13774182b62beb5705b6a174f650e792b5b06551 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 14 Nov 2020 17:41:06 +0100 Subject: [PATCH 50/52] Disable debug logging --- .ci/inc/logger.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/inc/logger.inc b/.ci/inc/logger.inc index 0ed1f96ce3..cf002de9a1 100644 --- a/.ci/inc/logger.inc +++ b/.ci/inc/logger.inc @@ -16,8 +16,8 @@ function log_success() { } function log_debug() { - #true - echo -e "[DEBUG ] $*" + true + #echo -e "[DEBUG ] $*" } function log_group_start() { From faa0a6109e4364323f74beaa666154a8704d25c9 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 14 Nov 2020 18:29:44 +0100 Subject: [PATCH 51/52] [ci] Fix builds for Windows / MacOS Add missing include for logger --- .github/workflows/pushes.yml | 2 ++ .github/workflows/troubleshooting.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/pushes.yml b/.github/workflows/pushes.yml index 4b5ff40f59..56f786bb8b 100644 --- a/.github/workflows/pushes.yml +++ b/.github/workflows/pushes.yml @@ -60,6 +60,7 @@ jobs: shell: bash - name: Build with mvnw run: | + source .ci/inc/logger.inc source .ci/inc/install-openjdk.inc install_openjdk_setdefault 11 ./mvnw -V clean verify @@ -84,6 +85,7 @@ jobs: shell: bash - name: Build with mvnw run: | + source .ci/inc/logger.inc source .ci/inc/install-openjdk.inc install_openjdk_setdefault 11 ./mvnw -V clean verify diff --git a/.github/workflows/troubleshooting.yml b/.github/workflows/troubleshooting.yml index bb4cd64fb0..8065541981 100644 --- a/.github/workflows/troubleshooting.yml +++ b/.github/workflows/troubleshooting.yml @@ -28,6 +28,7 @@ jobs: ruby-version: 2.7 - name: Install OpenJDK 11 run: | + source .ci/inc/logger.inc source .ci/inc/install-openjdk.inc install_openjdk 11 shell: bash From 4147d09771b6ccfb79b5ebcc95e1f966aec3c16b Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 14 Nov 2020 19:27:40 +0100 Subject: [PATCH 52/52] [ci] Windows needs maven_dependencies_resolve * Use the same build script for Windows+MacOS as for PRs * Also install java7 for PRs (that was missing from the old travis solution) * Run coveralls and sonar in parallel to win/macos after linux --- .ci/{build-pr.sh => build-pr-win-macos.sh} | 5 ++- .ci/build.sh | 27 +------------ .ci/inc/install-openjdk.inc | 25 +++++++++++- .github/workflows/pull-requests.yml | 2 +- .github/workflows/pushes.yml | 46 ++++++---------------- 5 files changed, 41 insertions(+), 64 deletions(-) rename .ci/{build-pr.sh => build-pr-win-macos.sh} (83%) diff --git a/.ci/build-pr.sh b/.ci/build-pr-win-macos.sh similarity index 83% rename from .ci/build-pr.sh rename to .ci/build-pr-win-macos.sh index 322e3918c1..1fdd821c97 100755 --- a/.ci/build-pr.sh +++ b/.ci/build-pr-win-macos.sh @@ -7,8 +7,9 @@ source $(dirname $0)/inc/maven-dependencies.inc set -e -log_group_start "Installing OpenJDK 11" +log_group_start "Installing Java 7+11" install_openjdk_setdefault 11 + install_oraclejdk7 log_group_end log_group_start "Downloading maven dependencies" @@ -16,7 +17,7 @@ log_group_start "Downloading maven dependencies" log_group_end log_group_start "Building with maven" - ./mvnw -e -V clean verify + ./mvnw -e -V clean verify -Djava7.home=${HOME}/oraclejdk7 log_group_end diff --git a/.ci/build.sh b/.ci/build.sh index 4bd9fc72e2..138f4041de 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -19,10 +19,10 @@ function pmd_ci_build_main() { pmd_ci_setup_ssh log_group_end - log_group_start "Prepare Java 11, Maven, Bundler" + log_group_start "Prepare Java 7+11, Maven, Bundler" install_openjdk_setdefault 11 + install_oraclejdk7 pmd_ci_build_setup_maven - pmd_ci_build_setup_oraclejdk7 pmd_ci_build_setup_bundler pmd_ci_build_setup_env log_group_end @@ -66,29 +66,6 @@ function pmd_ci_build_setup_maven() { cp .ci/files/maven-settings.xml ${HOME}/.m2/settings.xml } -# -# Installs jdk7 for integration test -# -function pmd_ci_build_setup_oraclejdk7() { - local local_dir="${HOME}/.cache/jdk7" - local target_dir="${HOME}/oraclejdk7" - local download_url="https://pmd-code.org/oraclejdk/jdk-7u80-linux-x64.tar.gz" - local archive=$(basename $download_url) - - mkdir -p ${local_dir} - mkdir -p ${target_dir} - if [ ! -e ${local_dir}/${archive} ]; then - log_info "Downloading from ${download_url} to ${local_dir}" - curl --location --output ${local_dir}/${archive} ${download_url} - else - log_info "Skipped download, file ${local_dir}/${archive} already exists" - fi - log_info "Extracting to ${target_dir}" - tar --extract --file ${local_dir}/${archive} -C ${target_dir} --strip-components=1 - - log_info "OracleJDK7 can be used via -Djava7.home=${HOME}/oraclejdk7" -} - # # Installs bundler, which is needed for doc generation and regression tester # diff --git a/.ci/inc/install-openjdk.inc b/.ci/inc/install-openjdk.inc index f84ff32540..c9a963242b 100644 --- a/.ci/inc/install-openjdk.inc +++ b/.ci/inc/install-openjdk.inc @@ -77,4 +77,27 @@ function install_openjdk_setdefault() { export PATH="${TARGET_DIR}/bin:${PATH}" java -version -} \ No newline at end of file +} + +# +# Installs jdk7 for integration test +# +function install_oraclejdk7() { + local local_dir="${HOME}/.cache/jdk7" + local target_dir="${HOME}/oraclejdk7" + local download_url="https://pmd-code.org/oraclejdk/jdk-7u80-linux-x64.tar.gz" + local archive=$(basename $download_url) + + mkdir -p ${local_dir} + mkdir -p ${target_dir} + if [ ! -e ${local_dir}/${archive} ]; then + log_info "Downloading from ${download_url} to ${local_dir}" + curl --location --output ${local_dir}/${archive} ${download_url} + else + log_info "Skipped download, file ${local_dir}/${archive} already exists" + fi + log_info "Extracting to ${target_dir}" + tar --extract --file ${local_dir}/${archive} -C ${target_dir} --strip-components=1 + + log_info "OracleJDK7 can be used via -Djava7.home=${HOME}/oraclejdk7" +} diff --git a/.github/workflows/pull-requests.yml b/.github/workflows/pull-requests.yml index a9e08dbcbf..a07af8d1e5 100644 --- a/.github/workflows/pull-requests.yml +++ b/.github/workflows/pull-requests.yml @@ -32,7 +32,7 @@ jobs: run: .ci/check-environment.sh shell: bash - name: Build - run: .ci/build-pr.sh + run: .ci/build-pr-win-macos.sh shell: bash env: MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 diff --git a/.github/workflows/pushes.yml b/.github/workflows/pushes.yml index 56f786bb8b..695f8070fd 100644 --- a/.github/workflows/pushes.yml +++ b/.github/workflows/pushes.yml @@ -41,8 +41,11 @@ jobs: PMD_CI_PUSH_COMMIT_COMPARE: ${{ github.event.compare }} PMD_CI_GIT_REF: ${{ github.ref }} - windows: - runs-on: windows-latest + windows-macos: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ windows-latest, macos-latest ] needs: linux steps: - uses: actions/checkout@v2 @@ -58,42 +61,15 @@ jobs: - name: Check Environment run: .ci/check-environment.sh shell: bash - - name: Build with mvnw - run: | - source .ci/inc/logger.inc - source .ci/inc/install-openjdk.inc - install_openjdk_setdefault 11 - ./mvnw -V clean verify - shell: bash - - macos: - runs-on: macos-latest - needs: linux - steps: - - uses: actions/checkout@v2 - - uses: actions/cache@v2 - with: - path: | - ~/.m2/repository - ~/.cache - vendor/bundle - key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }} - restore-keys: | - ${{ runner.os }}- - - name: Check Environment - run: .ci/check-environment.sh - shell: bash - - name: Build with mvnw - run: | - source .ci/inc/logger.inc - source .ci/inc/install-openjdk.inc - install_openjdk_setdefault 11 - ./mvnw -V clean verify + - name: Build + run: .ci/build-pr-win-macos.sh shell: bash + env: + MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=180 -Dmaven.wagon.http.retryHandler.count=3 coveralls: runs-on: ubuntu-latest - needs: [linux, windows, macos] + needs: [linux] steps: - uses: actions/checkout@v2 - uses: actions/cache@v2 @@ -118,7 +94,7 @@ jobs: sonar: runs-on: ubuntu-latest - needs: [linux, windows, macos] + needs: [linux] steps: - uses: actions/checkout@v2 - uses: actions/cache@v2