switch HA-proxy tests to h1load client

This change simplifies current HA-proxy test set up.
Testing no longer requires apache/nginx server as backend.
Instead of using siege as a client the test uses
h1load [1].

The pull request also install httpterm [2] http/1.1 server.
It's unused currently.

The HA-proxy configuration for testing matches the configuration
used in 'State of SSL stacks' write up.

The h1load client currently runs with options as follows :
    h1load
        -l \	# long results, output expected by h1load shell script
        -P \	# report also percentiles for gathared data
        -d ${TEST_TIME} \	# test duration, TEST_TIME is 10secs
        -c 500 \		# 500 concurrent connections
        -t ${THREAD_COUNT} \	# gather data for 1, 2, 4, 8, 16, 32, 64 threads
        -u \			# use runtime instead of system time
        ${BASE_URL}${PORT} 	# url where to connect to

The options above is just the initial version.

[1] https://github.com/wtarreau/h1load

[2] https://github.com/wtarreau/httpterm

[3] https://www.haproxy.com/blog/state-of-ssl-stacks

Author: Sashan

bench-scripts/bench_config_apache.sh

@@ -63,7 +63,6 @@ HOST=${BENCH_HOST:-'127.0.0.1'}
 APACHE_VERSION='2.4.65'
 
 . ./common_util.sh
-. ./bench_config_haproxy.sh
 
 function install_wolfssl_for_apache {
 	typeset VERSION=$1

bench-scripts/bench_config_haproxy.sh

@@ -10,126 +10,300 @@
 
 set -x
 
+. ./common_util.sh
+
 INSTALL_ROOT=${BENCH_INSTALL_ROOT:-"/tmp/bench.binaries"}
 RESULT_DIR=${BENCH_RESULTS:-"${INSTALL_ROOT}/results"}
 WORKSPACE_ROOT=${BENCH_WORKSPACE_ROOT:-"/tmp/bench.workspace"}
 MAKE_OPTS=${BENCH_MAKE_OPTS}
-HAPROXY_NOSSL_PORT='42128'
-HAPROXY_C2P_PORT='42132'
-HAPROXY_P2S_PORT='42134'
-HAPROXY_C2S_PORT='42136'
+HAPROXY_BUILD_TARG=${BENCH_HAPROXY_BUILD_TARG:-'linux-glibc'}
 CERT_SUBJ=${BENCH_CERT_SUBJ:-'/CN=localhost'}
 CERT_ALT_SUBJ=${BENCH_CERT_ALT_SUBJ:-'subjectAltName=DNS:localhost,IP:127.0.0.1'}
 HOST=${BENCH_HOST:-'127.0.0.1'}
+PORT_RSA_REUSE=${BENCH_PORT_RSA_REUSE:-7000}
+PORT_RSA=${BENCH_PORT_RSA:-7100}
+PORT_EC_REUSE=${BENCH_PORT_EC_REUSE:-7200}
+PORT_EC=${BENCH_PORT_EC:-7300}
 HAPROXY_VERSION='v3.2.0'
+CERT_SUBJ=${BENCH_CERT_SUBJ:-'/CN=localhost'}
+CERT_ALT_SUBJ=${BENCH_CERT_ALT_SUBJ:-'subjectAltName=DNS:localhost,IP:127.0.0.1'}
+PROXY_CHAIN=${BENCH_PROXY_CHAIN:-21}
+HOST=${BENCH_HOST:-'127.0.0.1'}
+
+function install_httpterm {
+    typeset SSL_LIB=$1
+    #
+    # FixMe: with https://github.com/wtarreau/httpterm,
+    # once https://github.com/wtarreau/httpterm/pull/1
+    # will be merged
+    #
+    typeset HTTPTERM_REPO="https://github.com/sashan/httpterm"
+    typeset BASENAME='httpterm'
+    typeset DIRNAME="${BASENAME}"
+    typeset SSL_CFLAGS=''
+    typeset SSL_LFLAGS=''
+
+    if [[ -z "${SSL_LIB}" ]] ; then
+        SSL_LIB="openssl-master"
+    fi
+
+    cd "${WORKSPACE_ROOT}" || exit 1
+    git clone -b fix.null-deref "${HTTPTERM_REPO}" "${DIRNAME}" || exit 1
+    cd ${DIRNAME} || exit 1
+    make || exit 1
+    install httpterm "${INSTALL_ROOT}/${SSL_LIB}/bin/httpterm" || exit 1
+}
+
+function install_h1load {
+    typeset SSL_LIB=$1
+    typeset H1LOAD_REPO="https://github.com/sashan/h1load"
+    typeset BASENAME='h1load'
+    typeset DIRNAME="${BASENAME}"
+    typeset SSL_CFLAGS=''
+    typeset SSL_LFLAGS=''
+
+    if [[ -z "${SSL_LIB}" ]] ; then
+        SSL_LIB="openssl-master"
+    fi
+
+    echo $SSL_LIB | grep 'wolfssl' > /dev/null
+    if [[ $? -eq 0 ]] ; then
+        #
+        # adjust flags for wolfssl
+        #
+        SSL_CFLAGS="-I${INSTALL_ROOT}/${SSL_LIB}/include/wolfssl"
+	SSL_CFLAGS="${SSL_CFLAGS} -I${INSTALL_ROOT}/${SSL_LIB}/include"
+        SSL_CFLAGS="${SSL_CFLAGS} -include ${INSTALL_ROOT}/${SSL_LIB}/include/wolfssl/options.h"
+        SSL_LFLAGS="-L ${INSTALL_ROOT}/${SSL_LIB}/lib -lwolfssl -Wl,-rpath=${INSTALL_ROOT}/lib"
+    else
+        SSL_CFLAGS="-I${INSTALL_ROOT}/${SSL_LIB}/include"
+        SSL_LFLAGS="-L ${INSTALL_ROOT}/${SSL_LIB}/lib -lssl -lcrypto"
+    fi
+    #
+    # this fork adds -u option to keep time as uptime
+    #
+    cd "${WORKSPACE_ROOT}" || exit 1
+    git clone -b float "${H1LOAD_REPO}" "${DIRNAME}" || exit 1
+    cd ${DIRNAME} || exit 1
+    make SSL_CFLAGS="${SSL_CFLAGS}" SSL_LFLAGS="${SSL_LFLAGS}" || exit 1
+    install h1load "${INSTALL_ROOT}/${SSL_LIB}/bin/h1load" || exit 1
+    cd scripts
+    for i in *.sh ; do
+        install $i "${INSTALL_ROOT}/${SSL_LIB}/bin/$i" || exit 1
+    done
+}
 
 function install_haproxy {
     typeset SSL_LIB=$1
     typeset VERSION=${HAPROXY_VERSION:-v3.2.0}
     typeset HAPROXY_REPO="https://github.com/haproxy/haproxy.git"
     typeset BASENAME='haproxy'
     typeset DIRNAME="${BASENAME}-${VERSION}"
-    typeset CERTDIR="${INSTALL_ROOT}/${SSL_LIB}/conf/certs"
+    typeset USE_LIB=''
 
     if [[ -z "${SSL_LIB}" ]] ; then
         SSL_LIB="openssl-master"
     fi
 
+    case ${SSL_LIB} in
+        wolf*)
+	        USE_LIB='USE_OPENSSL_WOLFSSL=1'
+	    ;;
+        *)
+                USE_LIB='USE_OPENSSL=1'
+            ;;
+    esac
+
     if [[ -f "${INSTALL_ROOT}/${SSL_LIB}/sbin/haproxy" ]] ; then
         echo "haproxy already installed; skipping.."
     else
-        cd "${WORKSPACE_ROOT}"
+        cd "${WORKSPACE_ROOT}" || exit 1
         mkdir -p "${DIRNAME}" || exit 1
         cd "${DIRNAME}"
         git clone "${HAPROXY_REPO}" -b ${VERSION} --depth 1 . || exit 1
 
         # haproxy does not have a configure script; only a big makefile
         make clean
         make ${MAKE_OPTS} \
-             TARGET=generic \
-             USE_OPENSSL=1 \
+             TARGET=${HAPROXY_BUILD_TARG} \
+             ${USE_LIB} \
+             USE_OPENSSL=USE_QUIC \
              SSL_INC="${INSTALL_ROOT}/${SSL_LIB}/include" \
              SSL_LIB="${INSTALL_ROOT}/${SSL_LIB}/lib" || exit 1
 
         make install ${MAKE_OPTS} \
              PREFIX="${INSTALL_ROOT}/${SSL_LIB}" || exit 1
     fi
 
-    mkdir -p ${CERTDIR}
-
-    # now generate the certificates
-    echo "generating new certificates for haproxy"
-    OPENSSL_BIN="env LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib ${INSTALL_ROOT}/${SSL_LIB}/bin/openssl"
-
-    # generating the key, cert of ca
-    $OPENSSL_BIN genpkey -algorithm rsa -pkeyopt rsa_keygen_bits:2048 -out "${CERTDIR}/ca_key.pem" || exit 1
-    $OPENSSL_BIN req -new -x509 -days 1 -key "${CERTDIR}/ca_key.pem" -out "${CERTDIR}/ca_cert.pem" -subj "/CN=Root CA" \
-        -addext "basicConstraints=critical,CA:true"  \
-        -addext "keyUsage=critical,keyCertSign,cRLSign" || exit 1
-    
-    # generating the client side
-    $OPENSSL_BIN genpkey -algorithm rsa -pkeyopt rsa_keygen_bits:2048 -out "${CERTDIR}/client_key.pem" || exit 1
-    $OPENSSL_BIN pkey -in "${CERTDIR}/client_key.pem" -pubout -out "${CERTDIR}/client_key_pub.pem" || exit 1
-    $OPENSSL_BIN req -new -out "${CERTDIR}/client_csr.pem" -subj "/CN=${HOST}" -key "${CERTDIR}/client_key.pem" \
-        -addext "${CERT_ALT_SUBJ}" \
-        -addext "keyUsage=critical,digitalSignature" || exit 1
-    $OPENSSL_BIN x509 -req -out "${CERTDIR}/client_cert.pem" -CAkey "${CERTDIR}/ca_key.pem" -CA "${CERTDIR}/ca_cert.pem" \
-        -days 1 -in "${CERTDIR}/client_csr.pem" -copy_extensions copy -ext "subjectAltName,keyUsage" \
-        -extfile <(printf "basicConstraints=critical,CA:false\nsubjectKeyIdentifier=none\n") || exit 1
-
-    # generating the server side
-    $OPENSSL_BIN genpkey -algorithm rsa -pkeyopt rsa_keygen_bits:2048 -out "${CERTDIR}/server_key.pem" || exit 1
-    $OPENSSL_BIN pkey -in "${CERTDIR}/server_key.pem" -pubout -out "${CERTDIR}/server_key_pub.pem" || exit 1
-    $OPENSSL_BIN req -new -out "${CERTDIR}/server_csr.pem" -subj "/CN=${HOST}" -key "${CERTDIR}/server_key.pem" \
-        -addext "${CERT_ALT_SUBJ}" \
-        -addext "keyUsage=critical,digitalSignature" || exit 1
-    $OPENSSL_BIN x509 -req -out "${CERTDIR}/server_cert.pem" -CAkey "${CERTDIR}/ca_key.pem" -CA "${CERTDIR}/ca_cert.pem" \
-        -days 1 -in "${CERTDIR}/server_csr.pem" -copy_extensions copy -ext "subjectAltName,keyUsage" \
-        -extfile <(printf "subjectKeyIdentifier=none\n"
-                   printf "${CERT_ALT_SUBJ}\n"
-                   printf "basicConstraints=critical,CA:false\n"
-                   printf "keyUsage=critical,keyEncipherment\n") || exit 1
-
-    # HAProxy PEM must be: server cert + server key (+ chain)
-    cat "${CERTDIR}/server_cert.pem" "${CERTDIR}/server_key.pem" "${CERTDIR}/ca_cert.pem" > "${CERTDIR}/haproxy_server.pem"
-
-    # setting up SSL Termination mode for now
-    # haproxy modes: encoding from client to haproxy, to server from haproxy, both
-    # the first needs a non TLS connection to the server - use the HTTP_PORT, otherwise use the HTTPS_PORT
-    cat <<EOF > "${INSTALL_ROOT}/${SSL_LIB}/conf/haproxy.cfg"
-defaults
-  timeout server 10s
-  timeout client 10s
-  timeout connect 10s
-
-frontend test_no_ssl
-  mode http
-  bind :${HAPROXY_NOSSL_PORT}
-  default_backend http_test
-
-frontend test_client2proxy
-  mode http
-  bind :${HAPROXY_C2P_PORT} ssl crt ${CERTDIR}/haproxy_server.pem ca-file ${CERTDIR}/ca_cert.pem verify required
-  default_backend http_test
-
-frontend test_proxy2server
-  mode http
-  bind :${HAPROXY_P2S_PORT}
-  default_backend https_test
-
-frontend test_client2server
-  mode http
-  bind :${HAPROXY_C2S_PORT} ssl crt ${CERTDIR}/haproxy_server.pem ca-file ${CERTDIR}/ca_cert.pem verify required
-  default_backend https_test
-
-backend http_test
-  mode http
-  balance random
-  server s1 ${HOST}:${HTTP_PORT}
-
-backend https_test
-  mode http
-  balance random
-  server s2 ${HOST}:${HTTPS_PORT} ssl verify required ca-file ${INSTALL_ROOT}/${SSL_LIB}/conf/server.crt
+    cd ${WORKSPACE_ROOT}
+}
+
+#
+# function creates haproxy.conf which ishould be
+# identical to configuration used here [1].
+#
+# The configuration file defines 4 proxy variants:
+#   ssl-reause with rsa+dh certificate,
+#       https client connects to port 7020
+#
+#   no-ssl-reuse, with rsa+dh certificate,
+#       https client connects to port 7120
+#
+#   ssl-reuse with ecdsa-256 certificate,
+#       https client connects to port 7220
+#
+#   no-ssl-reuse with ecdsa-256 certificate,
+#       https client connects to port 7320
+#
+# [1] https://www.haproxy.com/blog/state-of-ssl-stacks
+#   search for 'daisy-chain'
+#
+function config_haproxy {
+    typeset SSL_LIB=$1
+    typeset RSACERTKEY=''
+    typeset ECCERTKEY=''
+    typeset HAPROXY_CONF='etc/haproxy.conf'
+    typeset BASEPORT=''
+    typeset TOPPORT=''
+    typeset PORT=''
+    typeset SSL_REUSE=''
+    typeset REUSE_LABEL=''
+
+    if [[ -z "${SSL_LIB}" ]] ; then
+        SSL_LIB='openssl-=master'
+    fi
+
+    mkdir -p ${INSTALL_ROOT}/${SSL_LIB}/etc || exit 1
+    HAPROXY_CONF=${INSTALL_ROOT}/${SSL_LIB}/${HAPROXY_CONF}
+    RSACERTKEY=${INSTALL_ROOT}/${SSL_LIB}/etc/dh-rsa-2048.pem
+    ECCERTKEY=${INSTALL_ROOT}/${SSL_LIB}/etc/ec-dsa-256.pem
+
+cat <<EOF > ${HAPROXY_CONF}
+global
+	default-path config
+	tune.listener.default-shards by-thread
+	tune.idle-pool.shared off
+	ssl-default-bind-options ssl-min-ver TLSv1.3 ssl-max-ver TLSv1.3
+	ssl-server-verify none
+
 EOF
+
+    for BASEPORT in ${PORT_RSA_REUSE} ${PORT_RSA} ${PORT_EC_REUSE} ${PORT_EC} ; do
+        if [[ ${BASEPORT} -eq ${PORT_RSA_REUSE} || ${BASEPORT} -eq ${PORT_RSA} ]] ; then
+	    PROXYCERT=${RSACERTKEY}
+	else
+	    PROXYCERT=${ECCERTKEY}
+	fi
+        if [[ ${BASEPORT} -eq ${PORT_RSA_REUSE} || ${BASEPORT} -eq ${PORT_EC_REUSE} ]] ; then
+            SSL_REUSE=''
+	    REUSE_LABEL='ssl-reuse'
+        else
+            SSL_REUSE='no-ssl-reuse'
+	    REUSE_LABEL='no-ssl-reuse'
+        fi
+cat <<EOF >> ${HAPROXY_CONF}
+defaults ${REUSE_LABEL}
+	mode http
+	http-reuse never
+	default-server max-reuse 0 ssl ssl-min-ver TLSv1.3 ssl-max-ver TLSv1.3 ${SSL_REUSE}
+	option httpclose
+	timeout client 10s
+	timeout server 10s
+	timeout connect 10s
+
+frontend port${BASEPORT}
+	bind :${BASEPORT} ssl crt ${PROXYCERT}
+	http-request return status 200 content-type "text/plain" string "it works"
+
+EOF
+        BASEPORT=$(( ${BASEPORT} + 1))
+        TOPPORT=$(( ${BASEPORT} + ${PROXY_CHAIN} - 1 ))
+cat <<EOF >> ${HAPROXY_CONF}
+listen port${BASEPORT}
+	bind :${BASEPORT} ssl crt ${PROXYCERT}
+	stats uri /stats
+	server next ${HOST}:$(( ${BASEPORT} - 1))
+
+EOF
+
+        BASEPORT=$(( ${BASEPORT} + 1))
+        for PORT in $(seq ${BASEPORT} ${TOPPORT}) ; do
+cat <<EOF >> ${HAPROXY_CONF}
+listen port${PORT}
+	bind :${PORT} ssl crt ${PROXYCERT}
+	server next ${HOST}:$(( ${PORT} - 1))
+
+EOF
+        done
+    done
+    gen_certkey ${RSACERTKEY} ${RSACERTKEY}.key
+    gen_certkey_ec ${ECCERTKEY} ${ECCERTKEY}.key
+    cd ${WORKSPACE_ROOT} || exit 1
+}
+
+function setup_tests {
+    typeset i=0
+    cd "${WORKSPACE_ROOT}"
+    install_openssl master
+    install_haproxy openssl-master
+    install_httpterm openssl-master
+    install_h1load openssl-master
+    config_haproxy openssl-master
+    clean_build
+
+    for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do
+        cd "${WORKSPACE_ROOT}"
+        install_openssl openssl-$i ;
+        install_haproxy openssl-$i
+        install_httpterm openssl-$i
+        install_h1load openssl-$i
+        config_haproxy openssl-$i
+        clean_build
+    done
+
+    cd "${WORKSPACE_ROOT}"
+    install_openssl OpenSSL_1_1_1-stable
+    install_haproxy OpenSSL_1_1_1-stable
+    install_httpterm OpenSSL_1_1_1-stable
+    install_h1load OpenSSL_1_1_1-stable
+    config_haproxy OpenSSL_1_1_1-stable
+    clean_build
+
+    cd "${WORKSPACE_ROOT}"
+    install_wolfssl 5.8.2 '--enable-haproxy --enable-quic'
+    install_haproxy wolfssl-5.8.2
+    install_httpterm wolfssl-5.8.2
+    install_h1load wolfssl-5.8.2
+    config_haproxy wolfssl-5.8.2
+    clean_build
+
+    cd "${WORKSPACE_ROOT}"
+    install_libressl 4.1.0
+    install_haproxy libressl-4.1.0
+    install_httpterm libressl-4.1.0
+    install_h1load libressl-4.1.0
+    config_haproxy libressl-4.1.0
+    clean_build
+
+    #
+    # does not build with boring
+    #
+    #install_boringssl
+    #install_haproxy boringssl
+    #install_httpterm boringssl
+    #install_h1load boringssl
+    #config_haproxy boringssl
+    #cd "${WORKSPACE_ROOT}"
+    #clean_build
+
+    cd "${WORKSPACE_ROOT}"
+    install_aws_lc
+    install_haproxy aws-lc
+    install_httpterm aws-lc
+    install_h1load aws-lc
+    config_haproxy aws-lc
+    clean_build aws-lc
 }
+
+check_env
+setup_tests