Bound remaining hangable operations in test scripts

scripts/benchmark.test

@@ -2,6 +2,12 @@
 
 #benchmark.test
 
+# timeout(1) is GNU coreutils and absent on macOS; where it's missing, run the
+# command unbounded (the flaky hang this guards against is Linux-only CI).
+if ! command -v timeout >/dev/null 2>&1; then
+    timeout() { while [ "${1:-}" = "-s" ] || [ "${1:-}" = "-k" ]; do shift 2; done; shift; "$@"; }
+fi
+
 # if we can, isolate the network namespace to eliminate port collisions.
 if [ -n "$NETWORK_UNSHARE_HELPER" ]; then
      if [ -z "$NETWORK_UNSHARE_HELPER_CALLED" ]; then
@@ -111,7 +117,7 @@ if [ $1 -eq 1 ]
 then
     echo "Starting example client to benchmark connection average time"
     # start client to benchmark average time for each connection using port
-    ./examples/client/client -b $2 -p $bench_port $3
+    timeout -s KILL 2m ./examples/client/client -b $2 -p $bench_port $3
     client_result=$?
 fi
 
@@ -120,7 +126,7 @@ if [ $1 -eq 2 ]
 then
     echo "Starting example client to benchmark throughput"
     # start client in non-blocking mode, benchmark throughput using port
-    ./examples/client/client -N -B $2 -p $bench_port $3
+    timeout -s KILL 2m ./examples/client/client -N -B $2 -p $bench_port $3
     client_result=$?
 fi
 

scripts/dtls.test

@@ -1,5 +1,11 @@
 #!/usr/bin/env bash
 
+# timeout(1) is GNU coreutils and absent on macOS; where it's missing, run the
+# command unbounded (the flaky hang this guards against is Linux-only CI).
+if ! command -v timeout >/dev/null 2>&1; then
+    timeout() { while [ "${1:-}" = "-s" ] || [ "${1:-}" = "-k" ]; do shift 2; done; shift; "$@"; }
+fi
+
 # This script can be run with several environment variables set dictating its
 # run. You can set the following to what you like:
 WOLFSSL_ROOT=${WOLFSSL_ROOT:-$(pwd)}
@@ -85,9 +91,9 @@ prepend() { # Usage: cmd 2>&1 | prepend "sometext "
 
 run_test() { # usage: run_test "<testName>" "<udp-proxy args>" "<server args>" "<client args>"
     ((NUM_TESTS_RUN++))
-    echo "" | nc -u 127.0.0.1 $SERVER_PORT # This is a marker for the PCAP file
-    echo "$1" | nc -u 127.0.0.1 $SERVER_PORT # This is a marker for the PCAP file
-    echo "" | nc -u 127.0.0.1 $SERVER_PORT # This is a marker for the PCAP file
+    echo "" | nc -u -w 1 127.0.0.1 $SERVER_PORT # This is a marker for the PCAP file
+    echo "$1" | nc -u -w 1 127.0.0.1 $SERVER_PORT # This is a marker for the PCAP file
+    echo "" | nc -u -w 1 127.0.0.1 $SERVER_PORT # This is a marker for the PCAP file
     echo -e "\n${1}\n"
     stdbuf -oL -eL $WOLFSSL_ROOT/examples/server/server -u -p$SERVER_PORT $DTLS_VERSION $3 2>&1 | prepend "[server] " &
     sleep 0.2

scripts/dtlscid.test

@@ -64,7 +64,7 @@ test_cid () {
     timeout -s KILL 2m $WOLFSSL_ROOT/examples/server/server -v4 -u --cid $SCID 1> $SERVER_FILE &
     SERVER_PID=$!
     sleep 0.2
-    $WOLFSSL_ROOT/examples/client/client -v4 -u --cid $CCID 1> $CLIENT_FILE
+    timeout -s KILL 2m $WOLFSSL_ROOT/examples/client/client -v4 -u --cid $CCID 1> $CLIENT_FILE
     wait $SERVER_PID
     SERVER_PID=
     grep "Sending CID is ${HEXSCID}" $CLIENT_FILE > /dev/null

scripts/external.test

@@ -2,6 +2,12 @@
 
 # external.test
 
+# timeout(1) is GNU coreutils and absent on macOS; where it's missing, run the
+# command unbounded (the flaky hang this guards against is Linux-only CI).
+if ! command -v timeout >/dev/null 2>&1; then
+    timeout() { while [ "${1:-}" = "-s" ] || [ "${1:-}" = "-k" ]; do shift 2; done; shift; "$@"; }
+fi
+
 SCRIPT_DIR="$(dirname "$0")"
 
 server=www.wolfssl.com
@@ -45,7 +51,7 @@ RESULT=$?
 [ $RESULT -ne 0 ] && exit 0
 
 # client test against the server
-./examples/client/client -X -C -h $server -p 443 -g -A $ca
+timeout -s KILL 2m ./examples/client/client -X -C -h $server -p 443 -g -A $ca
 RESULT=$?
 [ $RESULT -ne 0 ] && echo -e "\n\nClient connection failed" && exit 1
 

scripts/google.test

@@ -2,6 +2,12 @@
 
 # google.test
 
+# timeout(1) is GNU coreutils and absent on macOS; where it's missing, run the
+# command unbounded (the flaky hang this guards against is Linux-only CI).
+if ! command -v timeout >/dev/null 2>&1; then
+    timeout() { while [ "${1:-}" = "-s" ] || [ "${1:-}" = "-k" ]; do shift 2; done; shift; "$@"; }
+fi
+
 server=www.google.com
 
 [ ! -x ./examples/client/client ] && echo -e "\n\nClient doesn't exist" && exit 1
@@ -26,7 +32,7 @@ RESULT=$?
 [ $RESULT -ne 0 ] && exit 0
 
 # client test against the server
-./examples/client/client -X -C -h $server -p 443 -g -d
+timeout -s KILL 2m ./examples/client/client -X -C -h $server -p 443 -g -d
 RESULT=$?
 [ $RESULT -ne 0 ] && echo -e "\n\nClient connection failed" && exit 1
 

scripts/ocsp-responder-openssl-interop.test

@@ -9,6 +9,12 @@
 # Uses: examples/ocsp_responder/ocsp_responder as the OCSP responder
 #       openssl ocsp as the client
 
+# timeout(1) is GNU coreutils and absent on macOS; where it's missing, run the
+# command unbounded (the flaky hang this guards against is Linux-only CI).
+if ! command -v timeout >/dev/null 2>&1; then
+    timeout() { while [ "${1:-}" = "-s" ] || [ "${1:-}" = "-k" ]; do shift 2; done; shift; "$@"; }
+fi
+
 if ! test -n "$WOLFSSL_OPENSSL_TEST"; then
     echo "WOLFSSL_OPENSSL_TEST NOT set, won't run"
     exit 77
@@ -89,8 +95,18 @@ ready_files=""
 
 cleanup() {
     exit_status=$?
+    local counter
     for p in $resp_pids; do
         kill $p 2>/dev/null
+        # Escalate to KILL so a wedged responder cannot hang this trap.
+        counter=0
+        while kill -0 $p 2>/dev/null && [ "$counter" -lt 50 ]; do
+            sleep 0.1
+            counter=$((counter + 1))
+        done
+        if kill -0 $p 2>/dev/null; then
+            kill -s KILL $p 2>/dev/null
+        fi
         wait $p 2>/dev/null
     done
     # Clean up log files
@@ -131,14 +147,20 @@ print_responder_logs() {
 
 get_first_free_port() {
     local ret="$1"
+    local scanned=0
     while :; do
         if [[ "$ret" -ge 65536 ]]; then
             ret=1024
         fi
-        if ! nc -z ${LOCALHOST_FOR_NC} "$ret"; then
+        if ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then
             break
         fi
         ret=$((ret+1))
+        scanned=$((scanned+1))
+        if [ "$scanned" -ge 100 ]; then
+            echo "ERROR: no free port found after scanning 100 ports" 1>&2
+            exit 1
+        fi
     done
     echo "$ret"
     return 0
@@ -180,7 +202,7 @@ query_ocsp() {
     printf "  TEST %2d: %-55s " "$tests_run" "$desc"
 
     local output
-    output=$($OPENSSL ocsp \
+    output=$(timeout -s KILL 2m $OPENSSL ocsp \
         -issuer "$issuer" \
         -cert "$cert" \
         -url "http://127.0.0.1:$_port/" \
@@ -395,7 +417,7 @@ echo "=== Negative tests: unsupported features ==="
 tests_run=$((tests_run+1))
 printf "  TEST %2d: %-55s " "$tests_run" "SHA-384 hash -> good"
 
-output=$($OPENSSL ocsp \
+output=$(timeout -s KILL 2m $OPENSSL ocsp \
     -sha384 \
     -issuer "$OCSP_DIR/intermediate1-ca-cert.pem" \
     -cert "$OCSP_DIR/server1-cert.pem" \
@@ -433,7 +455,7 @@ printf "  TEST %2d: %-55s " "$tests_run" "Authorized responder"
 
 # Query using OpenSSL - asks about intermediate1-ca which was issued by root-ca
 # Response will be signed by ocsp-responder-cert (authorized responder)
-output=$($OPENSSL ocsp \
+output=$(timeout -s KILL 2m $OPENSSL ocsp \
     -issuer "$OCSP_DIR/root-ca-cert.pem" \
     -cert "$OCSP_DIR/intermediate1-ca-cert.pem" \
     -url "http://127.0.0.1:$port5/" \
@@ -468,7 +490,7 @@ echo "=== Unknown certificate tests ==="
 tests_run=$((tests_run+1))
 printf "  TEST %2d: %-55s " "$tests_run" "Wrong issuer: server3 to responder 1 (wrong issuer)"
 
-output=$($OPENSSL ocsp \
+output=$(timeout -s KILL 2m $OPENSSL ocsp \
     -issuer "$OCSP_DIR/intermediate1-ca-cert.pem" \
     -cert "$OCSP_DIR/server3-cert.pem" \
     -url "http://127.0.0.1:$port1/" \
@@ -495,7 +517,7 @@ fi
 tests_run=$((tests_run+1))
 printf "  TEST %2d: %-55s " "$tests_run" "Wrong issuer: server1 to responder 2 (wrong issuer)"
 
-output=$($OPENSSL ocsp \
+output=$(timeout -s KILL 2m $OPENSSL ocsp \
     -issuer "$OCSP_DIR/intermediate2-ca-cert.pem" \
     -cert "$OCSP_DIR/server1-cert.pem" \
     -url "http://127.0.0.1:$port2/" \
@@ -528,7 +550,7 @@ tests_run=$((tests_run+1))
 printf "  TEST %2d: %-55s " "$tests_run" "Multiple requests (should return OCSP error)"
 
 # Using multiple -cert options creates one OCSP request with multiple certificate IDs
-output=$($OPENSSL ocsp \
+output=$(timeout -s KILL 2m $OPENSSL ocsp \
     -issuer "$OCSP_DIR/intermediate1-ca-cert.pem" \
     -cert "$OCSP_DIR/server1-cert.pem" \
     -cert "$OCSP_DIR/server2-cert.pem" \

scripts/ocsp-stapling-with-ca-as-responder.test

@@ -2,6 +2,12 @@
 
 # ocsp-stapling-with-ca-as-responder.test
 
+# timeout(1) is GNU coreutils and absent on macOS; where it's missing, run the
+# command unbounded (the flaky hang this guards against is Linux-only CI).
+if ! command -v timeout >/dev/null 2>&1; then
+    timeout() { while [ "${1:-}" = "-s" ] || [ "${1:-}" = "-k" ]; do shift 2; done; shift; "$@"; }
+fi
+
 SCRIPT_DIR="$(dirname "$0")"
 
 # if we can, isolate the network namespace to eliminate port collisions.
@@ -197,14 +203,20 @@ ca=certs/external/DigiCertGlobalRootCA.pem
 
 get_first_free_port() {
     local ret="$1"
+    local scanned=0
     while :; do
         if [[ "$ret" -ge 65536 ]]; then
             ret=1024
         fi
-        if ! nc -z 127.0.0.1 "$ret"; then
+        if ! nc -w 1 -z 127.0.0.1 "$ret"; then
             break
         fi
         ret=$((ret+1))
+        scanned=$((scanned+1))
+        if [ "$scanned" -ge 100 ]; then
+            echo "ERROR: no free port found after scanning 100 ports" 1>&2
+            exit 1
+        fi
     done
     echo "$ret"
     return 0
@@ -225,7 +237,7 @@ if [ ! -f "$ready_file" ]; then
 else
     printf '%s\n' "Random port selected: $port1"
     # Use client connection to shutdown the server cleanly
-    ./examples/client/client -p $port1
+    timeout -s KILL 2m ./examples/client/client -p $port1
     create_new_cnf $port1
 fi
 sleep 0.1
@@ -263,7 +275,7 @@ printf '%s\n\n' "------------- TEST CASE 1 SHOULD PASS ------------------------"
                          -p $port2 &
 wolf_pid2=$!
 wait_for_readyFile "$ready_file2" $wolf_pid2 $port2
-./examples/client/client -C -A certs/ocsp/root-ca-cert.pem -W 1 \
+timeout -s KILL 2m ./examples/client/client -C -A certs/ocsp/root-ca-cert.pem -W 1 \
                          -p $port2
 RESULT=$?
 [ $RESULT -ne 0 ] && printf '\n\n%s\n' "Client connection failed" && exit 1
@@ -277,7 +289,7 @@ remove_single_rF "$ready_file2"
                          -p $port2 &
 wolf_pid2=$!
 wait_for_readyFile "$ready_file2" $wolf_pid2 $port2
-./examples/client/client -C -A certs/ocsp/root-ca-cert.pem -W 1 \
+timeout -s KILL 2m ./examples/client/client -C -A certs/ocsp/root-ca-cert.pem -W 1 \
                          -p $port2
 RESULT=$?
 [ $RESULT -ne 1 ] && printf '\n\n%s\n' "Client connection succeeded $RESULT" && exit 1