Integrate setup secrets into build.sh, add env.gpg, add sourceforge upload

This commit is contained in:
Andreas Dangel
2020-11-11 22:56:02 +01:00
parent 8970b7fcd8
commit 14548e7b80
12 changed files with 345 additions and 75 deletions

View File

@ -12,5 +12,9 @@ for a ssh key, which is used to copy files to sourceforge.
* `PMD_CI_SECRET_PASSPHRASE`
* `CI_DEPLOY_PASSWORD`
* `CI_SIGN_PASSPHRASE`
* ...
## Encrypting
gpg --batch --symmetric --cipher-algo AES256 --passphrase="$PMD_CI_SECRET_PASSPHRASE" file.txt

View File

@ -1,26 +1,83 @@
#!/usr/bin/env bash
source $(dirname $0)/logger.inc
source $(dirname $0)/setup-secrets.inc
source $(dirname $0)/sourceforge-api.inc
source ${HOME}/java.env
set -e
#set -x
# configure maven
# probably not needed? echo "MAVEN_OPTS='-Xms1g -Xmx1g'" > ${HOME}/.mavenrc
mkdir -p ${HOME}/.m2
cp .ci/maven-settings.xml ${HOME}/.m2/settings.xml
function pmd_ci_build_main() {
pmd_ci_setup_env
pmd_ci_setup_gpg_key
pmd_ci_setup_ssh
pmd_ci_build_setup_maven
pmd_ci_build_setup_oraclejdk7
pmd_ci_build_run
# Deploy to sourceforge files
VERSION=$(pmd_ci_build_get_pom_version)
sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip"
sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip"
#build and upload doc
pmd_ci_build_setup_regression-tester
#regression-tester_uploadBaseline
#MVN_BUILD_FLAGS="-B -V -Djava7.home=${HOME}/oraclejdk7"
MVN_BUILD_FLAGS="-B -V"
exit 0
}
log_info "This is a snapshot build"
./mvnw deploy -Possrh,sign $MVN_BUILD_FLAGS
function pmd_ci_build_get_pom_version() {
echo $(./mvnw -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive org.codehaus.mojo:exec-maven-plugin:3.0.0:exec)
}
# Deploy to sourceforge files
#sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-bin-${VERSION}.zip"
#sourceforge_uploadFile "${VERSION}" "pmd-dist/target/pmd-src-${VERSION}.zip"
function pmd_ci_build_setup_maven() {
# configure maven
echo "MAVEN_OPTS='-Xms1g -Xmx1g'" > ${HOME}/.mavenrc
mkdir -p ${HOME}/.m2
cp .ci/maven-settings.xml ${HOME}/.m2/settings.xml
}
#regression-tester_uploadBaseline
function pmd_ci_build_setup_oraclejdk7() {
# install jdk7 for integration test
#build and upload doc
LOCAL_DIR="${HOME}/.cache/jdk7"
TARGET_DIR="${HOME}/oraclejdk7"
DOWNLOAD_URL="https://pmd-code.org/oraclejdk/jdk-7u80-linux-x64.tar.gz"
ARCHIVE=$(basename $DOWNLOAD_URL)
mkdir -p ${LOCAL_DIR}
mkdir -p ${TARGET_DIR}
if [ ! -e ${LOCAL_DIR}/${ARCHIVE} ]; then
log_info "Downloading from ${DOWNLOAD_URL} to ${LOCAL_DIR}"
curl --location --output ${LOCAL_DIR}/${ARCHIVE} ${DOWNLOAD_URL}
else
log_info "Skipped download, file ${LOCAL_DIR}/${ARCHIVE} already exists"
fi
log_info "Extracting to ${TARGET_DIR}"
tar --extract --file ${LOCAL_DIR}/${ARCHIVE} -C ${TARGET_DIR} --strip-components=1
log_info "OracleJDK7 can be used via -Djava7.home=${HOME}/oraclejdk7"
}
function pmd_ci_build_run() {
MVN_BUILD_FLAGS="-B -V -Djava7.home=${HOME}/oraclejdk7"
log_info "This is a snapshot build"
./mvnw deploy -Possrh,sign $MVN_BUILD_FLAGS
}
function pmd_ci_build_setup_regression-tester() {
# install openjdk8 for pmd-regression-tests
.ci/install-openjdk.sh 8
gem install bundler
bundle config set --local path vendor/bundle
bundle config set --local with release_notes_preprocessing
bundle install
}
pmd_ci_build_main

View File

@ -5,17 +5,37 @@
# and are in the correct version.
#
source logger.inc
source $(dirname $0)/logger.inc
set -e
function check() {
local CMD=$1
local VERSION_CMD=$2
echo -n "Checking ${CMD}..."
if [ hash "$CMD" 2>/dev/null ]; then
echo -e "${COL_GREEN}OK${COL_RESET}"
else
echo -e "${COL_RED}failure${COL_RESET}"
fi
}
# every OS:
# curl
# jq
# linux only
ruby --version | grep "ruby 2.7" || (log_error "Ruby is missing"; exit 1)
check "ruby" "ruby --version"
# gpg
# printenv
# rsync
# ssh
# windows only
# 7zip

BIN
.ci/env.gpg Normal file

Binary file not shown.

View File

@ -28,7 +28,7 @@ case "$(uname)" in
esac
OPENJDK_VERSION=11
OPENJDK_VERSION=$1
DOWNLOAD_URL=$(curl --silent -X GET "https://api.adoptopenjdk.net/v3/assets/feature_releases/${OPENJDK_VERSION}/ga?architecture=x64&heap_size=normal&image_type=jdk&jvm_impl=hotspot&os=${JDK_OS}&page=0&page_size=1&project=jdk&sort_method=DEFAULT&sort_order=DESC&vendor=adoptopenjdk" \
-H "accept: application/json" \
| jq -r ".[0].binaries[0].package.link")
@ -65,12 +65,18 @@ case "$OPENJDK_ARCHIVE" in
;;
esac
cat > ${HOME}/java.env <<EOF
if [ ! -e ${HOME}/java.env ]; then
cat > ${HOME}/java.env <<EOF
export JAVA_HOME="${TARGET_DIR}"
export PATH="${TARGET_DIR}/bin:${PATH}"
java -version
EOF
log_info "OpenJDK can be used via ${HOME}/java.env"
cat ${HOME}/java.env
source ${HOME}/java.env
log_info "OpenJDK can be used via ${HOME}/java.env"
cat ${HOME}/java.env
source ${HOME}/java.env
java -version
else
log_info "${HOME}/java.env already existed and has not been changed"
log_info "OpenJDK${OPENJDK_VERSION} can be used from ${TARGET_DIR}"
fi

View File

@ -12,7 +12,7 @@
<servers>
<server>
<id>ossrh</id>
<username>adangel</username>
<username>${env.CI_DEPLY_USER}</username>
<password>${env.CI_DEPLOY_PASSWORD}</password>
</server>
</servers>
@ -22,7 +22,7 @@
<profile>
<id>ossrh</id>
<properties>
<gpg.keyname>0xD0BF1D737C9A1C22</gpg.keyname>
<gpg.keyname>${env.CI_SIGN_KEY}</gpg.keyname>
<gpg.passphrase>${env.CI_SIGN_PASSPHRASE}</gpg.passphrase>
</properties>
</profile>

56
.ci/setup-secrets.inc Normal file
View File

@ -0,0 +1,56 @@
function pmd_ci_setup_env() {
log_info "Setting up secrets as environment variables..."
printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \
--passphrase-fd 0 \
--output .ci/env .ci/env.gpg
source .ci/env
rm .ci/env
}
function pmd_ci_setup_gpg_key() {
log_info "Setting up GPG release signing key..."
mkdir -p "${HOME}/.gpg"
printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \
--passphrase-fd 0 \
--output .ci/release-signing-key-D0BF1D737C9A1C22.gpg .ci/release-signing-key-D0BF1D737C9A1C22.gpg.gpg
gpg --batch --import .ci/release-signing-key-D0BF1D737C9A1C22.gpg
rm .ci/release-signing-key-D0BF1D737C9A1C22.gpg
}
function pmd_ci_setup_ssh() {
log_info "Setting up .ssh/id_rsa..."
mkdir -p ${HOME}/.ssh
chmod 700 "${HOME}/.ssh"
printenv PMD_CI_SECRET_PASSPHRASE | gpg --batch --yes --decrypt \
--passphrase-fd 0 \
--output ${HOME}/.ssh/id_rsa .ci/id_rsa.gpg
chmod 600 "${HOME}/.ssh/id_rsa"
log_info "Setting up .ssh/known_hosts..."
#
# https://sourceforge.net/p/forge/documentation/SSH%20Key%20Fingerprints/
#
# run locally:
# ssh-keyscan web.sourceforge.net | tee -a known_hosts
#
# verify fingerprints:
# ssh-keygen -F web.sourceforge.net -l -f known_hosts
# # Host web.sourceforge.net found: line 1
# web.sourceforge.net RSA SHA256:xB2rnn0NUjZ/E0IXQp4gyPqc7U7gjcw7G26RhkDyk90
# # Host web.sourceforge.net found: line 2
# web.sourceforge.net ECDSA SHA256:QAAxYkf0iI/tc9oGa0xSsVOAzJBZstcO8HqGKfjpxcY
# # Host web.sourceforge.net found: line 3
# web.sourceforge.net ED25519 SHA256:209BDmH3jsRyO9UeGPPgLWPSegKmYCBIya0nR/AWWCY
#
# then add output of `ssh-keygen -F web.sourceforge.net -f known_hosts`
#
echo 'web.sourceforge.net ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2uifHZbNexw6cXbyg1JnzDitL5VhYs0E65Hk/tLAPmcmm5GuiGeUoI/B0eUSNFsbqzwgwrttjnzKMKiGLN5CWVmlN1IXGGAfLYsQwK6wAu7kYFzkqP4jcwc5Jr9UPRpJdYIK733tSEmzab4qc5Oq8izKQKIaxXNe7FgmL15HjSpatFt9w/ot/CHS78FUAr3j3RwekHCm/jhPeqhlMAgC+jUgNJbFt3DlhDaRMa0NYamVzmX8D47rtmBbEDU3ld6AezWBPUR5Lh7ODOwlfVI58NAf/aYNlmvl2TZiauBCTa7OPYSyXJnIPbQXg6YQlDknNCr0K769EjeIlAfY87Z4tw==' >> "$HOME/.ssh/known_hosts"
echo 'web.sourceforge.net ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCwsY6sZT4MTTkHfpRzYjxG7mnXrGL74RCT2cO/NFvRrZVNB5XNwKNn7G5fHbYLdJ6UzpURDRae1eMg92JG0+yo=' >> "$HOME/.ssh/known_hosts"
echo 'web.sourceforge.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOQD35Ujalhh+JJkPvMckDlhu4dS7WH6NsOJ15iGCJLC' >> "$HOME/.ssh/known_hosts"
# add pmd-code.org (ssh-keyscan pmd-code.org)
echo 'pmd-code.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVsIeF6xU0oPb/bMbxG1nU1NDyBpR/cBEPZcm/PuJwdI9B0ydPHA6FysqAnt32fNFznC2SWisnWyY3iNsP3pa8RQJVwmnnv9OboGFlW2/61o3iRyydcpPbgl+ADdt8iU9fmMI7dC04UqgHGBoqOwVNna9VylTjp5709cK2qHnwU450F6YcOEiOKeZfJvV4PmpJCz/JcsUVqft6StviR31jKnqbnkZdP8qNoTbds6WmGKyXkhHdLSZE7X1CFQH28tk8XFqditX93ezeCiThFL7EleDexV/3+2+cs5878sDMUMzHS5KShTjkxzhHaodhtIEdNesinq/hOPbxAGkQ0FbD' >> $HOME/.ssh/known_hosts
}

View File

@ -1,42 +0,0 @@
#!/usr/bin/env bash
source $(dirname $0)/logger.inc
log_info "Setting up secrets..."
mkdir -p ${HOME}/.ssh
chmod 700 "${HOME}/.ssh"
gpg --symmetric --cipher-algo AES256 --batch --passphrase="$PMD_CI_SECRET_PASSPHRASE" \
--decrypt --output ${HOME}/id_rsa .ci/id_rsa.gpg
chmod 600 "${HOME}/.ssh/id_rsa"
mkdir -p "${HOME}/.gpg"
gpg --symmetric --cipher-algo AES256 --batch --passphrase="$PMD_CI_SECRET_PASSPHRASE" \
--decrypt --output .ci/release-signing-key-D0BF1D737C9A1C22.gpg .ci/release-signing-key-D0BF1D737C9A1C22.gpg.gpg
gpg --batch --import .ci/release-signing-key-D0BF1D737C9A1C22.gpg
rm .ci/release-signing-key-D0BF1D737C9A1C22.gpg
log_info "Setting up .ssh/known_hosts..."
#
# https://sourceforge.net/p/forge/documentation/SSH%20Key%20Fingerprints/
#
# run locally:
# ssh-keyscan web.sourceforge.net | tee -a known_hosts
#
# verify fingerprints:
# ssh-keygen -F web.sourceforge.net -l -f known_hosts
# # Host web.sourceforge.net found: line 1
# web.sourceforge.net RSA SHA256:xB2rnn0NUjZ/E0IXQp4gyPqc7U7gjcw7G26RhkDyk90
# # Host web.sourceforge.net found: line 2
# web.sourceforge.net ECDSA SHA256:QAAxYkf0iI/tc9oGa0xSsVOAzJBZstcO8HqGKfjpxcY
# # Host web.sourceforge.net found: line 3
# web.sourceforge.net ED25519 SHA256:209BDmH3jsRyO9UeGPPgLWPSegKmYCBIya0nR/AWWCY
#
# then add output of `ssh-keygen -F web.sourceforge.net -f known_hosts`
#
echo 'web.sourceforge.net ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2uifHZbNexw6cXbyg1JnzDitL5VhYs0E65Hk/tLAPmcmm5GuiGeUoI/B0eUSNFsbqzwgwrttjnzKMKiGLN5CWVmlN1IXGGAfLYsQwK6wAu7kYFzkqP4jcwc5Jr9UPRpJdYIK733tSEmzab4qc5Oq8izKQKIaxXNe7FgmL15HjSpatFt9w/ot/CHS78FUAr3j3RwekHCm/jhPeqhlMAgC+jUgNJbFt3DlhDaRMa0NYamVzmX8D47rtmBbEDU3ld6AezWBPUR5Lh7ODOwlfVI58NAf/aYNlmvl2TZiauBCTa7OPYSyXJnIPbQXg6YQlDknNCr0K769EjeIlAfY87Z4tw==' >> "$HOME/.ssh/known_hosts"
echo 'web.sourceforge.net ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCwsY6sZT4MTTkHfpRzYjxG7mnXrGL74RCT2cO/NFvRrZVNB5XNwKNn7G5fHbYLdJ6UzpURDRae1eMg92JG0+yo=' >> "$HOME/.ssh/known_hosts"
echo 'web.sourceforge.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOQD35Ujalhh+JJkPvMckDlhu4dS7WH6NsOJ15iGCJLC' >> "$HOME/.ssh/known_hosts"
# add pmd-code.org (ssh-keyscan pmd-code.org)
echo 'pmd-code.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVsIeF6xU0oPb/bMbxG1nU1NDyBpR/cBEPZcm/PuJwdI9B0ydPHA6FysqAnt32fNFznC2SWisnWyY3iNsP3pa8RQJVwmnnv9OboGFlW2/61o3iRyydcpPbgl+ADdt8iU9fmMI7dC04UqgHGBoqOwVNna9VylTjp5709cK2qHnwU450F6YcOEiOKeZfJvV4PmpJCz/JcsUVqft6StviR31jKnqbnkZdP8qNoTbds6WmGKyXkhHdLSZE7X1CFQH28tk8XFqditX93ezeCiThFL7EleDexV/3+2+cs5878sDMUMzHS5KShTjkxzhHaodhtIEdNesinq/hOPbxAGkQ0FbD' >> $HOME/.ssh/known_hosts

166
.ci/sourceforge-api.inc Normal file
View File

@ -0,0 +1,166 @@
#
# The functions here require the following scripts:
# logger.inc
#
# The functions here require the following environment variables:
# PMD_SF_USER
# PMD_SF_APIKEY
#
#
# Uploads the release notes to sourceforge files as "ReadMe.md".
#
# Note: this function always succeeds, even if the upload fails.
# In that case, just a error logging is provided.
#
function sourceforge_uploadReleaseNotes() {
local pmdVersion="$1"
local releaseNotes="$2"
log_debug "$FUNCNAME pmdVersion=$pmdVersion"
local targetUrl="https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}"
local errexitstate="$(shopt -po errexit)"
set +e # disable errexit
(
# This handler is called if any command fails
function release_notes_fail() {
log_error "Error while uploading release notes as ReadMe.md to sourceforge!"
log_error "Please upload manually: ${targetUrl}"
cleanup_temp_dir
}
function cleanup_temp_dir() {
log_debug "Cleanup tempdir $releaseNotesTempDir"
rm "${releaseNotesTempDir}/${pmdVersion}/ReadMe.md" || true
rmdir "${releaseNotesTempDir}/${pmdVersion}" || true
rmdir "${releaseNotesTempDir}" || true
}
# exit subshell after trap
set -e
trap release_notes_fail ERR
local releaseNotesTempDir=$(mktemp -d)
log_debug "Tempdir: $releaseNotesTempDir"
mkdir -p "${releaseNotesTempDir}/${pmdVersion}"
echo "$releaseNotes" > "${releaseNotesTempDir}/${pmdVersion}/ReadMe.md"
log_info "Uploading release notes to sourceforge for version $pmdVersion"
rsync -avz \
"${releaseNotesTempDir}/" \
"${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/"
log_success "Successfully uploaded release notes as ReadMe.md to sourceforge: ${targetUrl}"
cleanup_temp_dir
)
# restore errexit state
eval "$errexitstate"
}
#
# Uploads the given file to sourceforge.
#
# Note: This function always succeeds, even if the upload fails.
# In that case, just a error logging is provided.
#
function sourceforge_uploadFile() {
local pmdVersion="$1"
local filename="$2"
log_debug "$FUNCNAME pmdVersion=$pmdVersion filename=$filename"
local targetUrl="https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}"
local errexitstate="$(shopt -po errexit)"
set +e # disable errexit
(
# This handler is called if any command fails
function upload_failed() {
log_error "Error while uploading ${filename} to sourceforge!"
log_error "Please upload manually: ${targetUrl}"
}
# exit subshell after trap
set -e
trap upload_failed ERR
log_info "Uploading $filename to sourceforge..."
.travis/travis_wait "rsync -avh ${filename} ${PMD_SF_USER}@web.sourceforge.net:/home/frs/project/pmd/pmd/${pmdVersion}/"
log_success "Successfully uploaded ${filename} to sourceforge: ${targetUrl}"
)
# restore errexit state
eval "$errexitstate"
}
#
# Select the given version as the new default download.
#
# Note: This function always succeeds, even if the request fails.
# In that case, just a error logging is provided.
#
function sourceforge_selectDefault() {
local pmdVersion="$1"
log_debug "$FUNCNAME pmdVersion=$pmdVersion"
local targetUrl="https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}"
local errexitstate="$(shopt -po errexit)"
set +e # disable errexit
(
# This handler is called if any command fails
function request_failed() {
log_error "Error while selecting ${pmdVersion} as new default download on sourceforge!"
log_error "Please do it manually: ${targetUrl}"
}
# exit subshell after trap
set -e
trap request_failed ERR
log_info "Selecting $pmdVersion as new default on sourceforge..."
local response
response=$(curl --fail -s -H "Accept: application/json" \
-X PUT \
-d "api_key=${PMD_SF_APIKEY}" \
-d "default=windows&default=mac&default=linux&default=bsd&default=solaris&default=others" \
"https://sourceforge.net/projects/pmd/files/pmd/${pmdVersion}/pmd-bin-${pmdVersion}.zip")
log_debug " -> response: $response"
log_success "Successfully selected $pmdVersion as new default on sourceforge: ${targetUrl}"
)
# restore errexit state
eval "$errexitstate"
}
#
# Rsyncs the complete documentation to sourceforge.
#
# Note: This function always succeeds, even if the upload fails.
# In that case, just a error logging is provided.
#
function sourceforge_rsyncSnapshotDocumentation() {
local pmdVersion="$1"
local targetPath="$2"
log_debug "$FUNCNAME pmdVersion=$pmdVersion targetPath=$targetPath"
local targetUrl="https://pmd.sourceforge.io/${targetPath}/"
local errexitstate="$(shopt -po errexit)"
set +e # disable errexit
(
# This handler is called if any command fails
function upload_failed() {
log_error "Couldn't upload the documentation. It won't be current on ${targetUrl}"
}
# exit subshell after trap
set -e
trap upload_failed ERR
log_info "Uploading documentation to ${targetUrl}..."
.travis/travis_wait "rsync -ah --stats --delete docs/pmd-doc-${VERSION}/ ${PMD_SF_USER}@web.sourceforge.net:/home/project-web/pmd/htdocs/snapshot/"
log_success "Successfully uploaded documentation: ${targetUrl}"
)
# restore errexit state
eval "$errexitstate"
}

View File

@ -21,11 +21,15 @@ jobs:
key: ${{ runner.os }}-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-
- name: Install OpenJDK
run: .ci/install-openjdk.sh
- name: Install OpenJDK 11
run: .ci/install-openjdk.sh 11
shell: bash
- name: Build with mvnw
run: |
source ${HOME}/java.env
./mvnw -V clean install
shell: bash
- name: Execute Danger
if: ${{ runner.os == 'Linux' }}
run: echo "TODO"
shell: bash

View File

@ -28,21 +28,14 @@ jobs:
- name: Check Environment
run: .ci/check-environment.sh
shell: bash
- name: Setup Secrets
run: .ci/setup-secrets.sh
shell: bash
env:
PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }}
- name: Install OpenJDK
run: .ci/install-openjdk.sh
- name: Install OpenJDK 11
run: .ci/install-openjdk.sh 11
shell: bash
- name: build
run: .ci/build.sh
shell: bash
env:
PMD_CI_SECRET_PASSPHRASE: ${{ secrets.PMD_CI_SECRET_PASSPHRASE }}
CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }}
CI_SIGN_PASSPHRASE: ${{ secrets.CI_SIGN_PASSPHRASE }}
windows:
runs-on: windows-latest

View File

@ -10,6 +10,12 @@ jobs:
continue-on-error: false
steps:
- uses: actions/checkout@v2
- name: Check Environment
run: .ci/check-environment.sh
shell: bash
- name: Install OpenJDK 11
run: .ci/install-openjdk.sh 11
shell: bash
- name: Run Release Script
run: .ci/release.sh
shell: bash