diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2b8eb5d86..d8ccfc3ed 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -20,10 +20,11 @@ steps: command: > precommit/src/main/shell/test-patch.sh --brief-report-file=/tmp/yetus-out/brief.txt - --bugcomments=briefreport,htmlout,buildkiteannotate + --bugcomments=briefreport,csvout,htmlout,buildkiteannotate --console-report-file=/tmp/yetus-out/console.txt --docker --docker-cache-from=ghcr.io/apache/yetus-base:main + --csv-report-file=/tmp/yetus-out/report.csv --html-report-file=/tmp/yetus-out/report.html --java-home=/usr/lib/jvm/java-17-openjdk-amd64 --patch-dir=/tmp/yetus-out diff --git a/.circleci/config.yml b/.circleci/config.yml index 7b68ed761..08798682f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,8 +42,9 @@ jobs: --patch-dir=/tmp/yetus-out --html-report-file=/tmp/yetus-out/report.html --console-report-file=/tmp/yetus-out/console.txt + --csv-report-file=/tmp/yetus-out/report.csv --brief-report-file=/tmp/yetus-out/brief.txt - --bugcomments=briefreport,htmlout,junit + --bugcomments=briefreport,csvout,htmlout,junit --tests-filter=checkstyle,test4tests --junit-report-xml=/tmp/yetus-out/junit-report.xml diff --git a/.cirrus.yml b/.cirrus.yml index 93695f5eb..086c27440 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -22,8 +22,9 @@ yetus_task: ${CIRRUS_WORKING_DIR}/precommit/src/main/shell/test-patch.sh --basedir="${CIRRUS_WORKING_DIR}" --brief-report-file=/tmp/yetus-out/brief.txt - --bugcomments=briefreport,htmlout,junit + --bugcomments=briefreport,console,csvout,htmlout,junit --console-report-file=/tmp/yetus-out/console.txt + --csv-report-file=/tmp/yetus-out/report.csv --html-report-file=/tmp/yetus-out/report.html --java-home=/usr/lib/jvm/java-11-openjdk-amd64 --junit-report-xml=/tmp/yetus-out/junit.xml diff --git a/.github/workflows/yetus.yml b/.github/workflows/yetus.yml index b7d42800d..5e4eb07e6 100644 --- a/.github/workflows/yetus.yml +++ b/.github/workflows/yetus.yml @@ -44,8 +44,9 @@ jobs: --archive-list="checkstyle-errors.xml,spotbugsXml.xml" --basedir="${GITHUB_WORKSPACE}/src" --brief-report-file="${GITHUB_WORKSPACE}/out/brief.txt" - --bugcomments=console,briefreport,github,htmlout,junit + --bugcomments=console,briefreport,csvout,github,htmlout,junit --console-report-file="${GITHUB_WORKSPACE}/out/console.txt" + --csv-report-file="${GITHUB_WORKSPACE}/out/report.csv" --docker --docker-cache-from=ghcr.io/apache/yetus-base:main --github-token="${{ secrets.GITHUB_TOKEN }}" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1190222d7..eae1a60ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,9 +25,10 @@ buretoolbox-job: --plugins=all --java-home=/usr/lib/jvm/java-11-openjdk-amd64 --html-report-file=/tmp/yetus-out/report.html + --csv-report-file=/tmp/yetus-out/report.csv --console-report-file=/tmp/yetus-out/console.txt --brief-report-file=/tmp/yetus-out/brief.txt - --bugcomments=briefreport,htmlout,gitlab,junit + --bugcomments=briefreport,csvout,htmlout,gitlab,junit --tests-filter=checkstyle,test4tests --junit-report-xml=/tmp/yetus-out/junit-report.xml diff --git a/.semaphore/semaphore-build.sh b/.semaphore/semaphore-build.sh index 4561894bd..ac7d76776 100755 --- a/.semaphore/semaphore-build.sh +++ b/.semaphore/semaphore-build.sh @@ -19,10 +19,12 @@ PRECOMMITDIR=precommit/src/main/shell "${PRECOMMITDIR}/test-patch.sh" \ --plugins=all \ + --bugcomments=briefreport,csvout,htmlout,junit \ --mvn-custom-repos \ --mvn-custom-repos-dir=/tmp/yetus-m2 \ --patch-dir=/tmp/yetus-out \ --tests-filter=checkstyle,test4tests \ + --csv-report-file=/tmp/yetus-out/report.csv \ --junit-report-xml=/tmp/yetus-out/junit-results.xml \ --junit-report-style=full \ --docker \ diff --git a/Jenkinsfile b/Jenkinsfile index 5d545f148..2bc9da6c1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -86,11 +86,12 @@ pipeline { YETUS_ARGS+=(--sentinel) # lots of different output formats + YETUS_ARGS+=("--bugcomments=briefreport,console,csvout,htmlout,junit") YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/brief.txt") YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/console.txt") YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/report.html") YETUS_ARGS+=("--junit-report-xml=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/junit-report.xml") - + YETUS_ARGS+=("--csv-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/yetus-out/report.csv") # enable writing back to Github YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}") diff --git a/precommit/src/main/shell/core.d/00-yetuslib.sh b/precommit/src/main/shell/core.d/00-yetuslib.sh index 9b2177a5d..0c9bbbbf1 100755 --- a/precommit/src/main/shell/core.d/00-yetuslib.sh +++ b/precommit/src/main/shell/core.d/00-yetuslib.sh @@ -609,3 +609,14 @@ function yetus_is_container YETUS_CONTAINER_STATE=false return 1 } + +## @description Given a string, trim extra surrounding whitespace +## @audience public +## @stability stable +## @replaceable no +function yetus_trim { + declare var="$*" + var="${var#"${var%%[![:space:]]*}"}" + var="${var%"${var##*[![:space:]]}"}" + printf '%s' "$var" +} diff --git a/precommit/src/main/shell/plugins.d/csvout.sh b/precommit/src/main/shell/plugins.d/csvout.sh new file mode 100755 index 000000000..17d848dff --- /dev/null +++ b/precommit/src/main/shell/plugins.d/csvout.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_bugsystem csvout + +## @description Usage info for csvout plugin +## @audience private +## @stability evolving +## @replaceable no +function csvout_usage +{ + yetus_add_option "--csv-report-file=" "Save the final report to an CSV-formated file" +} + +## @description Option parsing for csvout plugin +## @audience private +## @stability evolving +## @replaceable no +function csvout_parse_args +{ + declare i + declare fn + + for i in "$@"; do + case ${i} in + --csv-report-file=*) + delete_parameter "${i}" + fn=${i#*=} + ;; + esac + done + + if [[ -n "${fn}" ]]; then + if : > "${fn}"; then + CSVOUT_REPORTFILE_ORIG="${fn}" + CSVOUT_REPORTFILE=$(yetus_abs "${CSVOUT_REPORTFILE_ORIG}") + else + yetus_error "WARNING: cannot create CSV report file ${fn}. Ignoring." + fi + fi +} + +## @description Give access to the CSV report file in docker mode +## @audience private +## @stability evolving +## @replaceable no +function csvout_docker_support +{ + # if for some reason the report file is in PATCH_DIR, then if + # PATCH_DIR gets cleaned out we lose access to the file on the 'outside' + # so we put it into the workdir which never gets cleaned. + + if [[ -n ${CSVOUT_REPORTFILE} ]]; then + DOCKER_EXTRAARGS+=("-v" "${CSVOUT_REPORTFILE}:${DOCKER_WORK_DIR}/report.csv") + USER_PARAMS+=("--csv-report-file=${DOCKER_WORK_DIR}/report.csv") + fi +} + +## @description Write out an CSV version of the final report to a file +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function csvout_report_writer +{ + declare result=$1 + shift + declare i=0 + declare ourstring + declare vote + declare subs + declare ela + declare comment + + if [[ -z "${CSVOUT_REPORTFILE}" ]]; then + return + fi + + : > "${CSVOUT_REPORTFILE}" + i=0 + until [[ $i -ge ${#TP_VOTE_TABLE[@]} ]]; do + ourstring=$(echo "${TP_VOTE_TABLE[i]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\|) + vote=$(yetus_trim "${vote}") + subs=$(echo "${ourstring}" | cut -f3 -d\|) + subs=$(yetus_trim "${subs}") + ela=$(echo "${ourstring}" | cut -f4 -d\|) + ela=$(yetus_trim "${ela}") + comment=$(echo "${ourstring}" | cut -f6 -d\|) + comment=$(yetus_trim "${comment}") + + echo "${vote},${subs},${ela},${comment}" >> "${CSVOUT_REPORTFILE}" + ((i=i+1)) + done +} + +## @description Write out the CSV version of the final report to a file +## @audience private +## @stability evolving +## @replaceable no +## @param runresult +function csvout_finalreport +{ + declare result=$1 + + if [[ -z "${CSVOUT_REPORTFILE}" ]]; then + return + fi + + big_console_header "Writing CSV to ${CSVOUT_REPORTFILE}" + + csvout_report_writer "${result}" "${CSVOUT_REPORTFILE}" +} diff --git a/precommit/src/main/shell/plugins.d/htmlout.sh b/precommit/src/main/shell/plugins.d/htmlout.sh index b14056ea0..c92854b89 100755 --- a/precommit/src/main/shell/plugins.d/htmlout.sh +++ b/precommit/src/main/shell/plugins.d/htmlout.sh @@ -314,7 +314,7 @@ function htmlout_finalreport return fi - big_console_header "Writing HTML to ${commentfile}" + big_console_header "Writing HTML to ${HTMLOUT_REPORTFILE}" htmlout_report_writer "${result}" "${HTMLOUT_REPORTFILE}" } diff --git a/precommit/src/test/shell/yetus_trim.bats b/precommit/src/test/shell/yetus_trim.bats new file mode 100755 index 000000000..32c3cdff2 --- /dev/null +++ b/precommit/src/test/shell/yetus_trim.bats @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load functions_test_helper + +@test "yetus_trim (no trim)" { + run yetus_trim fake + [ "${output}" = "fake" ] +} + +@test "yetus_trim (left trim)" { + run yetus_trim " fake" + [ "${output}" = "fake" ] +} + +@test "yetus_trim (right trim)" { + run yetus_trim "fake " + [ "${output}" = "fake" ] +} + +@test "yetus_trim (dual trim)" { + run yetus_trim " fake " + [ "${output}" = "fake" ] +}