#!/usr/bin/env bash ############################################################################# ## ## Unit testing automation script ## Last updated on October 11, 2024 ## ## This file is part of Logtalk ## SPDX-FileCopyrightText: 1998-2024 Paulo Moura ## SPDX-License-Identifier: Apache-2.0 ## ## Licensed under the Apache License, Version 2.0 (the "License"); ## you may not use this file except in compliance with the License. ## You may obtain a copy of the License at ## ## http://www.apache.org/licenses/LICENSE-2.0 ## ## Unless required by applicable law or agreed to in writing, software ## distributed under the License is distributed on an "AS IS" BASIS, ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ## See the License for the specific language governing permissions and ## limitations under the License. ## ############################################################################# # loosely based on a unit test automation script contributed by Parker Jones set -o pipefail function cleanup { pkill -9 -P $$ } trap cleanup EXIT print_version() { echo "$(basename "$0") 20.0" exit 0 } operating_system=$(uname -s) if [ "${operating_system:0:10}" == "MINGW32_NT" ] || [ "${operating_system:0:10}" == "MINGW64_NT" ] ; then # assume that we're running on Windows using the Git for Windows bash shell extension='.sh' elif [ "$LOGTALKHOME" != "" ] && [ "$LOGTALKUSER" != "" ] && [ "$LOGTALKHOME" == "$LOGTALKUSER" ] ; then # assume that we're running Logtalk without using the installer scripts extension='.sh' else extension='' fi # first, make sure that we don't peak Windows own timeout command, which is not usable for this purpose if [[ "$(command -v timeout)" == *"System32"* ]] || [[ "$(command -v timeout)" == *"system32"* ]] ; then timeout_command="" # second, look for GNU coreutils package timeout command elif [ -x "$(command -v timeout)" ] && [[ "$(timeout --version)" == *"GNU coreutils"* ]] ; then timeout_command="timeout --foreground -s 9 -k 1.0s" elif [ -x "$(command -v gtimeout)" ] && [[ "$(gtimeout --version)" == *"GNU coreutils"* ]] ; then timeout_command="gtimeout --foreground -s 9 -k 1.0s" else timeout_command="" fi # default argument values declare allargs allargs="$@" driver='tester' dot="" output='verbose' if [ "${operating_system:0:10}" == "MINGW32_NT" ] || [ "${operating_system:0:10}" == "MINGW64_NT" ] ; then # assume that we're running on Windows using the Git for Windows bash shell and get the # current directory path using forward slashes for better Prolog backend compatibility base="$(pwd -W)" else base="$PWD" fi level="" exclude="" results="$base/logtalk_tester_logs" mode='normal' format='default' coverage='none' flag_goal="true" initialization_goal="true" wipe='false' # disable timeouts to maintain backward compatibility timeout=0 prefix="$HOME/" issue_server="" issue_labels="bug" url="" # use GNU sed if available instead of BSD sed if gsed --version >/dev/null 2>&1 ; then sed="gsed" else sed="sed" fi format_default_goal="true" format_tap_goal="logtalk_load(lgtunit(tap_report))" format_xunit_goal="logtalk_load(lgtunit(xunit_report))" format_xunit_net_v2_goal="logtalk_load(lgtunit(xunit_net_v2_report))" format_goal=$format_default_goal coverage_default_goal="true" coverage_xml_goal="logtalk_load(lgtunit(coverage_report))" coverage_goal=$coverage_default_goal run_testset() { start_time="$(date +%s)" unit=$(dirname "$1") unit_short=${unit#$prefix} cd "$unit" || exit 1 if [ "$wipe" == 'true' ] ; then rm -rf ./.lgt_tmp rm -rf ./lgt_tmp fi if [ "$output" == 'verbose' ] ; then echo "%" echo "% $unit_short" fi if [ -f "$driver.sh" ] ; then source "./$driver.sh" $allargs source_exit=$? if [ "$source_exit" -gt 0 ] ; then echo "% source $driver.sh returned code $source_exit" exit 9 fi fi # convert any forward slashes so that the derived file name is usable name=${unit////__} # also convert any colon if running on Windows systems so that the derived file name is usable name=${name/:/__} report_goal="logtalk_load(lgtunit(automation_report)),set_logtalk_flag(test_results_directory,'$results'),set_logtalk_flag(test_unit_name,'$name')" if [ "$prefix" != "" ] ; then flag_goal="set_logtalk_flag(suppress_path_prefix,'$prefix')" else flag_goal="true" fi if [ "$issue_server" != "" ] ; then flag_goal="logtalk_load(issue_creator(loader)),set_logtalk_flag(issue_server,'$issue_server'),set_logtalk_flag(issue_labels,'$issue_labels'),$flag_goal" fi if [ "$url" != "" ] ; then flag_goal="set_logtalk_flag(tests_base_url,'$url'),$flag_goal" fi if [[ "$format" != "default" || "$coverage" != "none" ]] ; then flag_goal="set_logtalk_flag(tests_report_directory,'$unit/'),$flag_goal" fi if [ $mode == 'optimal' ] || [ $mode == 'all' ] ; then run_tests "$name" "$initialization_goal,$report_goal,$format_goal,$coverage_goal,$flag_goal,$seed_goal,$tester_optimal_goal" tests_exit=$? mode_prefix="% (opt) " elif [ $mode == 'normal' ] || [ $mode == 'all' ] ; then run_tests "$name" "$initialization_goal,$report_goal,$format_goal,$coverage_goal,$flag_goal,$seed_goal,$tester_normal_goal" tests_exit=$? mode_prefix="% " elif [ $mode == 'debug' ] || [ $mode == 'all' ] ; then run_tests "$name" "$initialization_goal,$report_goal,$format_goal,$coverage_goal,$flag_goal,$seed_goal,$tester_debug_goal" tests_exit=$? mode_prefix="% (debug) " fi if [ "$tests_exit" -eq 0 ] && [ "$output" == 'verbose' ] && grep -q "tests skipped" "$results/$name.results" ; then echo "% skipped" elif [ "$tests_exit" -eq 0 ] && [ "$output" == 'verbose' ] && grep -q "(not applicable)" "$results/$name.results" ; then echo "% not applicable" elif [ "$tests_exit" -eq 0 ] && [ -f "$results/$name.totals" ] && [ "$output" == 'verbose' ] ; then while read -r line ; do echo -n "$mode_prefix" echo -n "$(cut -f 3 <<< "$line")" echo -n ' tests: ' echo -n "$(cut -f 4 <<< "$line")" echo -n ' skipped, ' echo -n "$(cut -f 5 <<< "$line")" echo -n ' passed, ' echo -n "$(cut -f 6 <<< "$line")" echo -n ' failed (' echo -n "$(cut -f 7 <<< "$line")" echo ' flaky)' duration=$(( $(date +%s) - start_time )) echo -n '% completed tests from object ' echo -n "$(cut -f 2 <<< "$line")" if [ $duration -eq 1 ] ; then echo " in $duration second" else echo " in $duration seconds" fi done < <(grep '^object' "$results/$name.totals") echo -n '% clause coverage ' echo "$(grep "^coverage" "$results/$name.totals" | cut -f 2)" elif [ "$tests_exit" -eq 5 ] ; then if [ "$output" == 'verbose' ] ; then echo "% broken" fi ensure_format_report "$unit" "$(basename "$unit")" "Broken" elif [ "$tests_exit" -eq 137 ] || [ "$tests_exit" -eq 124 ] ; then if [ "$output" == 'verbose' ] ; then echo "% timeout" fi echo "LOGTALK_TIMEOUT" >> "$results/$name.errors" ensure_format_report "$unit" "$(basename "$unit")" "Timeout" elif [ "$tests_exit" -ne 0 ] ; then if [ "$output" == 'verbose' ] ; then echo "% crash" fi echo "LOGTALK_CRASH" >> "$results/$name.errors" ensure_format_report "$unit" "$(basename "$unit")" "Crash" fi if [ $coverage == 'xml' ] ; then if [ -d "$LOGTALKUSER" ] ; then cp "$LOGTALKUSER/tools/lgtunit/coverage_report.dtd" . || true cp "$LOGTALKUSER/tools/lgtunit/coverage_report.xsl" . || true elif [ -d "$LOGTALKHOME" ] ; then cp "$LOGTALKHOME/tools/lgtunit/coverage_report.dtd" . || true cp "$LOGTALKHOME/tools/lgtunit/coverage_report.xsl" . || true fi fi return 0 } run_tests() { name="$1" goal="$2" if [ ${#args[@]} -eq 0 ] ; then if [ "$timeout_command" != "" ] && [ $timeout -ne 0 ] ; then $timeout_command $timeout $logtalk_call "$goal" < /dev/null > "$results/$name.results" 2> "$results/$name.errors" else $logtalk_call "$goal" < /dev/null > "$results/$name.results" 2> "$results/$name.errors" fi else if [ "$timeout_command" != "" ] && [ $timeout -ne 0 ] ; then $timeout_command $timeout $logtalk_call "$goal" -- "${args[@]}" < /dev/null > "$results/$name.results" 2> "$results/$name.errors" else $logtalk_call "$goal" -- "${args[@]}" < /dev/null > "$results/$name.results" 2> "$results/$name.errors" fi fi exit=$? if grep -q "Likely bug in the backend Prolog compiler. Please file a bug report." "$results/$name.errors"; then echo "LOGTALK_BROKEN" >> "$results/$name.errors" return 5 elif [ $exit -eq 0 ] && ! grep -q "(not applicable)" "$results/$name.results" && ! grep -q -s "^object" "$results/$name.totals" && ! grep -q "tests skipped" "$results/$name.results"; then echo "LOGTALK_BROKEN" >> "$results/$name.errors" return 5 fi return $exit } ensure_format_report() { directory="$1" name="$2" error="$3" short=$(echo "$directory" | $sed "s|^$prefix||") if [ "$format" == "xunit" ] ; then timestamp=$(date +"%Y-%m-%dT%H:%M:%S") echo "" > "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "$error" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" elif [ "$format" == "xunit_net_v2" ] ; then run_date=$(date +"%Y-%m-%d") run_time=$(date +"%H:%M:%S") echo "" > "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" # hack for Allure ignoring the "errors" tag echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" echo "" >> "$directory/xunit_report.xml" elif [ "$format" == "tap" ] ; then echo "TAP version 13" > "$directory/tap_report.xml" echo "Bail out! $unit $error" >> "$directory/tap_report.xml" fi } usage_help() { echo echo "This script automates running unit tests found in the current directory and" echo "recursively in its sub-directories by scanning by default for \"tester.lgt\"" echo "and \"tester.logtalk\" source files. In case of failed unit tests or test set" echo "errors, this script returns a non-zero exit code. When a \"tester.sh\" file" echo "exists in the tests directory, the file is sourced before running the tests." echo "The \"tester.sh\" file is sourced with all the parameters passed to the script." echo "The timeout option requires availability of the GNU coreutils timeout command." echo echo "Usage:" echo " $(basename "$0") -p prolog [-o output] [-m mode] [-f format] [-d results] [-t timeout] [-n driver] [-s prefix] [-b tracker] [-u url] [-c report] [-l level] [-e exclude] [-i options] [-g goal] [-r seed] [-w] [-- arguments]" echo " $(basename "$0") -v" echo " $(basename "$0") -h" echo echo "Required arguments:" echo " -p backend Prolog compiler" echo " (valid values are b, ciao, cx, eclipse, gnu, gnunc, ji, xvm, sicstus, swi, swipack, tau, trealla, xsb, and yap)" echo echo "Optional arguments:" echo " -v print version of $(basename "$0")" echo " -o output (valid values are verbose and minimal; default is $output)" echo " -m compilation mode (default is $mode)" echo " (valid values are optimal, normal, debug, and all)" echo " -f format for writing the test results (default is $format)" echo " (valid values are default, tap, xunit, and xunit_net_v2)" echo " -d directory to store the test logs (default is ./logtalk_tester_logs)" echo " -t timeout in seconds for running each test set (default is $timeout; i.e. disabled)" echo " -n name of the test driver and sourced files (minus file name extensions; default is $driver)" echo " -s suppress path prefix (default is $prefix)" echo " -b bug report server (valid values are github and gitlab; no default)" echo " -u base URL to generate links to test files (no default)" echo " -c code coverage report (default is $coverage)" echo " (valid values are none and xml)" echo " -l directory depth level to look for test sets (default is to recurse into all sub-directories)" echo " (level 1 means current directory only)" echo " -e exclude directories matching a POSIX-extended regular expression" echo " -i integration script command-line options (no default)" echo " -g initialization goal (default is $initialization_goal)" echo " -r random generator starting seed (no default)" echo " -w wipe default scratch directories (./.lgt_tmp and ./lgt_tmp) before running a test set" echo " (this option should not be used when running parallel processes that use the same test sets)" echo " -- arguments to be passed to the tests (no default)" echo " -h help" echo } while getopts "vp:o:m:f:d:t:n:s:b:u:c:l:e:g:r:i:wh" option do case $option in v) print_version;; p) p_arg="$OPTARG";; o) o_arg="$OPTARG";; m) m_arg="$OPTARG";; f) f_arg="$OPTARG";; d) d_arg="$OPTARG";; t) t_arg="$OPTARG";; n) n_arg="$OPTARG";; s) s_arg="$OPTARG";; b) b_arg="$OPTARG";; u) u_arg="$OPTARG";; c) c_arg="$OPTARG";; l) l_arg="$OPTARG";; e) e_arg="$OPTARG";; i) i_arg="$OPTARG";; g) g_arg="$OPTARG";; r) r_arg="$OPTARG";; w) wipe='true';; h) usage_help; exit;; *) usage_help; exit 1;; esac done shift $((OPTIND - 1)) args=("$@") if [ "$p_arg" == "" ] ; then echo "Error! Backend Prolog compiler not specified!" >&2 usage_help exit 1 elif [ "$p_arg" == "b" ] ; then prolog='B-Prolog' logtalk=bplgt$extension logtalk_call="$logtalk $i_arg -g" elif [ "$p_arg" == "ciao" ] ; then prolog='Ciao Prolog' logtalk=ciaolgt$extension logtalk_call="$logtalk $i_arg -e" elif [ "$p_arg" == "cx" ] ; then prolog='CxProlog' logtalk=cxlgt$extension logtalk_call="$logtalk $i_arg --goal" elif [ "$p_arg" == "eclipse" ] ; then prolog='ECLiPSe' logtalk=eclipselgt$extension logtalk_call="$logtalk $i_arg -e" elif [ "$p_arg" == "gnu" ] ; then prolog='GNU Prolog' logtalk=gplgt$extension logtalk_call="$logtalk $i_arg --query-goal" elif [ "$p_arg" == "gnunc" ] ; then prolog='GNU Prolog (native code)' logtalk=gplgtnc logtalk_call="$logtalk $i_arg --query-goal" elif [ "$p_arg" == "ji" ] ; then prolog='JIProlog' logtalk=jiplgt$extension logtalk_call="$logtalk $i_arg -n -g" elif [ "$p_arg" == "xvm" ] ; then prolog='XVM' logtalk=xvmlgt$extension logtalk_call="$logtalk $i_arg -g" case "$i_arg" in *"--standard-top-level"*) dot=".";; *"--custom-top-level"*) dot="?";; *) dot=".";; esac elif [ "$p_arg" == "sicstus" ] ; then prolog='SICStus Prolog' logtalk=sicstuslgt$extension logtalk_call="$logtalk $i_arg --goal" dot="." elif [ "$p_arg" == "swi" ] ; then prolog='SWI-Prolog' logtalk=swilgt$extension logtalk_call="$logtalk $i_arg -g" elif [ "$p_arg" == "swipack" ] ; then prolog='SWI-Prolog (pack)' logtalk=swipl logtalk_call="$logtalk $i_arg -g" elif [ "$p_arg" == "tau" ] ; then prolog='Tau Prolog' logtalk=taulgt$extension logtalk_call="$logtalk $i_arg -g" dot="." elif [ "$p_arg" == "trealla" ] ; then prolog='Trealla Prolog' logtalk=tplgt$extension logtalk_call="$logtalk $i_arg -g" elif [ "$p_arg" == "xsb" ] ; then prolog='XSB' logtalk=xsblgt$extension logtalk_call="$logtalk $i_arg -e" dot="." elif [ "$p_arg" == "yap" ] ; then prolog='YAP' logtalk=yaplgt$extension logtalk_call="$logtalk $i_arg -g" elif [ "$p_arg" != "" ] ; then echo "Error! Unsupported backend Prolog compiler: $p_arg" >&2 usage_help exit 1 elif [ ! "$(command -v $logtalk)" ] ; then echo "Error! $logtalk integration script for $prolog not found." >&2 echo " Check that its directory is in your execution path." >&2 exit 1 fi if [ "$o_arg" == "verbose" ] ; then output='verbose' elif [ "$o_arg" == "minimal" ] ; then output='minimal' elif [ "$o_arg" != "" ] ; then echo "Error! Unknown output verbosity: $o_arg" >&2 usage_help exit 1 fi if [ "$m_arg" == "optimal" ] ; then mode='optimal' elif [ "$m_arg" == "normal" ] ; then mode='normal' elif [ "$m_arg" == "debug" ] ; then mode='debug' elif [ "$m_arg" == "all" ] ; then mode='all' elif [ "$m_arg" != "" ] ; then echo "Error! Unknown compilation mode: $m_arg" >&2 usage_help exit 1 fi if [ "$f_arg" == "default" ] ; then format='default' format_goal=$format_default_goal elif [ "$f_arg" == "tap" ] ; then format='tap' format_goal=$format_tap_goal elif [ "$f_arg" == "xunit" ] ; then format='xunit' format_goal=$format_xunit_goal elif [ "$f_arg" == "xunit_net_v2" ] ; then format='xunit_net_v2' format_goal=$format_xunit_net_v2_goal elif [ "$f_arg" != "" ] ; then echo "Error! Unknown format: $f_arg" >&2 usage_help exit 1 fi if [ "$c_arg" == "none" ] ; then coverage='none' coverage_goal=$coverage_default_goal elif [ "$c_arg" == "xml" ] ; then coverage='xml' coverage_goal=$coverage_xml_goal elif [ "$c_arg" != "" ] ; then echo "Error! Unknown coverage report: $c_arg" >&2 usage_help exit 1 fi if [ "$d_arg" != "" ] ; then results="$d_arg" fi if [ "$t_arg" != "" ] ; then timeout="$t_arg" fi if [ "$s_arg" != "" ] ; then prefix="$s_arg" fi if [ "$b_arg" != "" ] ; then issue_server="$(echo "$b_arg" | cut -d':' -f1)" labels="$(echo "$b_arg" | cut -d':' -f2)" if [ "$labels" != "" ] && [ "$labels" != "$issue_server" ] ; then issue_labels="$labels" fi if [ "$issue_server" != "github" ] && [ "$issue_server" != "gitlab" ] ; then echo "Error! Issue tracker server must be either github or gitlab: $b_arg" >&2 usage_help exit 1 fi fi if [ "$u_arg" != "" ] ; then url="$u_arg" fi if [ "$n_arg" != "" ] ; then driver="$n_arg" fi if [ "$l_arg" != "" ] ; then if [[ "$l_arg" =~ ^[1-9][0-9]*$ ]] ; then level="-maxdepth $l_arg" else echo "Error! Level must be an integer equal or greater than 1: $l_arg" >&2 usage_help exit 1 fi fi if [ "$e_arg" != "" ] ; then exclude="$e_arg" fi if [ "$g_arg" != "" ] ; then initialization_goal="$g_arg" fi if [ "$r_arg" != "" ] ; then seed_goal="logtalk_load(arbitrary(loader)),type::set_seed($r_arg)" else seed_goal="true" fi if [ "$p_arg" == "swipack" ] ; then initialization_goal="use_module(library(logtalk)),$initialization_goal" fi if [ "$timeout_command" == "" ] ; then echo "Warning! Timeout support not available. The timeout option will be ignored." >&2 fi versions_goal="logtalk_load(library(tester_versions)),halt$dot" tester_optimal_goal="set_logtalk_flag(optimize,on),logtalk_load($driver),halt$dot" tester_normal_goal="logtalk_load($driver),halt$dot" tester_debug_goal="set_logtalk_flag(debug,on),logtalk_load($driver),halt$dot" mkdir -p "$results" rm -f "$results"/*.results rm -f "$results"/*.errors rm -f "$results"/*.totals rm -f "$results"/errors.all rm -f "$results"/tester_versions.txt if [ "$output" == 'verbose' ] ; then start_date=$(eval date \"+%Y-%m-%d %H:%M:%S\") echo "% Batch testing started @ $start_date" $logtalk_call $versions_goal > "$results"/tester_versions.txt 2> /dev/null grep -a "Logtalk version:" "$results"/tester_versions.txt grep -a "Prolog version:" "$results"/tester_versions.txt | $sed "s/Prolog/$prolog/" fi declare drivers declare testsets if [ "$exclude" == "" ] ; then drivers="$(find -L "$base" $level -type f -name "$driver.lgt" -or -name "$driver.logtalk" | LC_ALL=C sort)" testsets=$(find -L "$base" $level -type f -name "$driver.lgt" -or -name "$driver.logtalk" | wc -l | tr -d ' ') find_exit=$? elif [ "$(uname)" == "Darwin" ] ; then drivers="$(find -L -E "$base" $level -type f -not -regex "$exclude" -name "$driver.lgt" -or -name "$driver.logtalk" | LC_ALL=C sort)" testsets=$(find -L -E "$base" $level -type f -not -regex "$exclude" -name "$driver.lgt" -or -name "$driver.logtalk" | wc -l | tr -d ' ') find_exit=$? else drivers="$(find -L "$base" $level -type f -regextype posix-extended -not -regex "$exclude" -name "$driver.lgt" -or -name "$driver.logtalk" | LC_ALL=C sort)" testsets=$(find -L "$base" $level -type f -regextype posix-extended -not -regex "$exclude" -name "$driver.lgt" -or -name "$driver.logtalk" | wc -l | tr -d ' ') find_exit=$? fi if [ "$find_exit" -gt 0 ] ; then echo "% find command returned code $find_exit" exit 11 fi if [ "$testsets" -eq 0 ] ; then echo "%" echo "% 0 test sets: 0 completed, 0 skipped, 0 broken, 0 timedout, 0 crashed" echo "% 0 tests: 0 skipped, 0 passed, 0 failed" exit 0 fi if [ "$output" == 'verbose' ] ; then while read -r file && [ "$file" != "" ]; do run_testset "$file" done <<< "$drivers" else counter=1 while read -r file && [ "$file" != "" ]; do echo -ne "% running $testsets test sets: " echo -ne "$counter"'\r' run_testset "$file" ((counter++)) done <<< "$drivers" echo fi cd "$results" || exit 1 testsetskipped=$(cat -- *.results | grep -c 'tests skipped') testsetskipped=$((testsetskipped+$(cat -- *.results | grep -c '(not applicable)'))) timeouts=$(cat -- *.errors | grep -c 'LOGTALK_TIMEOUT') crashed=$(cat -- *.errors | grep -c 'LOGTALK_CRASH') broken=$(cat -- *.errors | grep -c 'LOGTALK_BROKEN') testsetruns=$((testsets-testsetskipped-timeouts-crashed-broken)) skipped=0 passed=0 failed=0 flaky=0 while read -r line ; do skipped=$((skipped+$(cut -f 4 <<< "$line"))) passed=$((passed+$(cut -f 5 <<< "$line"))) failed=$((failed+$(cut -f 6 <<< "$line"))) flaky=$((flaky+$(cut -f 7 <<< "$line"))) done < <(grep -s '^object' ./*.totals) total=$((skipped+passed+failed)) if grep -q -s -a -h '^!' -- *.errors || grep -q -s -a -h '^!' -- *.results || grep -q -s -a -h '^\*' -- *.errors || grep -q -s -a -h '^\*' -- *.results ; then echo "%" echo "% Compilation errors/warnings and failed unit tests" echo "% (compilation errors/warnings might be expected depending on the test)" grep -s -a -h '^!' -- *.errors | $sed 's/.errors//' | tee -a errors.all grep -s -a -h '^!' -- *.results | $sed 's/.results//' | tee -a errors.all grep -s -a -h '^\*' -- *.errors | $sed 's/.errors//' | tee -a errors.all grep -s -a -h '^\*' -- *.results | $sed 's/.results//' | tee -a errors.all fi if grep -q -s -a 'tests skipped' -- *.results || grep -q -s -a '(not applicable)' -- *.results ; then echo "%" echo "% Skipped test sets" grep -s -a 'tests skipped' -- *.results | $sed 's/% tests skipped//' | $sed 's/.results://' | $sed 's|__|/|g' | $sed "s|^$prefix||" grep -s -a '(not applicable)' -- *.results | $sed 's/(not applicable)//' | $sed 's/.results://' | $sed 's|__|/|g' | $sed "s|^$prefix||" fi if grep -q -s -a 'LOGTALK_BROKEN' -- *.errors; then echo "%" echo "% Broken" grep -s -a 'LOGTALK_BROKEN' -- *.errors | $sed 's/LOGTALK_BROKEN//' | $sed 's/.errors://' | $sed 's|__|/|g' | $sed "s|^$prefix||" fi if grep -q -s -a 'LOGTALK_TIMEOUT' -- *.errors; then echo "%" echo "% Timedout" grep -s -a 'LOGTALK_TIMEOUT' -- *.errors | $sed 's/LOGTALK_TIMEOUT//' | $sed 's/.errors://' | $sed 's|__|/|g' | $sed "s|^$prefix||" fi if grep -q -s -a 'LOGTALK_CRASH' -- *.errors; then echo "%" echo "% Crashed" grep -s -a 'LOGTALK_CRASH' -- *.errors | $sed 's/LOGTALK_CRASH//' | $sed 's/.errors://' | $sed 's|__|/|g' | $sed "s|^$prefix||" fi if grep -s -q '^skipped' -- *.totals; then echo "%" echo "% Skipped tests" for file in *.totals; do if grep -s -q '^skipped' "$file"; then grep '^skipped' "$file" | cut -f 2,3 | $sed "s|^$prefix||" | $sed "s|\t| - |" fi done fi if grep -s -q '^failed' -- *.totals; then echo "%" echo "% Failed tests" for file in *.totals; do if grep -s -q '^failed' "$file"; then grep '^failed' "$file" | cut -f 2,3 | $sed "s|^$prefix||" | $sed "s|\t| - |" fi done fi echo "%" echo "% $testsets test sets: $testsetruns completed, $testsetskipped skipped, $broken broken, $timeouts timedout, $crashed crashed" echo "% $total tests: $skipped skipped, $passed passed, $failed failed ($flaky flaky)" if [ "$output" == 'verbose' ] ; then end_date=$(eval date \"+%Y-%m-%d %H:%M:%S\") echo "%" echo "% Batch testing ended @ $end_date" fi if [ "$crashed" -gt 0 ] ; then exit 7 elif [ "$broken" -gt 0 ] ; then exit 5 elif [ "$timeouts" -gt 0 ] ; then exit 3 elif [ "$((failed - flaky))" -gt 0 ] ; then exit 1 else exit 0 fi