commit 7fa14d9e053dd25dbd88b750c275e3a792a50ecd parent a492d0dae17e710ce56012b235adaab05114bbd0 Author: kyle <kyle@getaddrinfo.net> Date: Tue, 17 Nov 2015 22:43:00 -0700 tests: simplify testsuite - i want to be able to run a single test without the test harness easily! - this makes debugging tests feel a bit less like work - use make to run the tests, do any diff'ing, and clean up afterwards - put a makefile inside every test directory - each makefile has the same commands to run: test, diff, or clean - default make target left for actual compiling of test programs (not used atm) - don't wait for server to come online before starting tests - i'm going to make server changes where we open the socket asap - this lets us not have to poll the server for connectivity - which also lets us not have to clean the server.log file afterwards - reduce unneeded output, tighten main loop, only show ok/fails if there are any - rename expected_out to server.log.good, to match server.log - don't pass port as first argument in tests, pass it as PORT env var Diffstat:
38 files changed, 123 insertions(+), 100 deletions(-)
diff --git a/run_tests.sh b/run_tests.sh @@ -1,13 +1,14 @@ #!/bin/sh -# *never* connect to a real production instance with this script! -port=4729 -temp_db=$(mktemp) +# try not to connect to a real production instance with this script! +export PORT=4729 +temp_db=`mktemp` # start server with temp db, non standard port, and in the background -perl sl -p $port -d $temp_db > server.log & +perl sl -p $PORT -d $temp_db > server.log & server_pid=$! -echo -n "[$server_pid] server started" + +sleep 0.1 # clean up on ctrl-c trap sigint_handler int @@ -19,106 +20,73 @@ sigint_handler() { kill 0 } -if [ $# -eq 3 ]; then - echo "(implement running single test)" -fi - -# if tput is available we can do colors! if which tput > /dev/null; then - # make sure we run on XTERM={screen,xterm} which have 8 colors - if [ $(tput colors) -ne -1 ] - then - red=$(tput setaf 1 0 0) - green=$(tput setaf 2 0 0) - yellow=$(tput setaf 3 0 0) - fi - reset=$(tput sgr0) -fi - -# don't start the test suite until we can connect to the server -while ! nc -z 127.0.0.1 $port 2> /dev/null; do - if ! ps -p $server_pid > /dev/null - then - echo " server died! log file was:" - cat server.log - exit 1 - fi - echo -n "." - sleep 0.1 -done - -# wait for the output triggered from the above `nc` command to be flushed -if grep "disconnected!" server.log > /dev/null; then - sleep 0.1 - echo -n "." + red=`tput setaf 1 0 0` + green=`tput setaf 2 0 0` + reset=`tput sgr0` fi -cat server.log -> server.log -echo " ready" cleanup() { > server.log # clean up the database between runs - sqlite3 ${1} "delete from devices; - delete from lists; - delete from list_members; - delete from list_data; - delete from friends_map; - delete from mutual_friends;" + sqlite3 ${1} "delete from devices; delete from lists; + delete from list_members; delete from list_data; + delete from friends_map; delete from mutual_friends;" } passed=0 failed=0 count=0 -for t in `ls tests/*/test.*`; do +for t in `ls tests/*/Makefile`; do count=$((count + 1)) - printf "%3s %s: %s" $count $(dirname $t) $yellow + test_dir=`dirname ${t}` + make -s -C $test_dir clean - # run test - if ! ./${t} $port; then - echo "$red test failed$reset" + # run test, complain if failed + if ! make -s -C $test_dir "test"; then + printf "%3s %s: %s%s%s\n" $count $test_dir $red "test failed" $reset failed=$((failed + 1)) cleanup $temp_db continue fi - echo -n "${reset}" - # validate the server is still running + # make sure the server is still running if ! kill -0 $server_pid; then - echo "$red test killed server!$reset" + printf "%3s %s: %s%s%s\n" $count $test_dir $red "test killed server" $reset +cat server.log rm $temp_db exit 1 fi - # wait for server output to be flushed - if grep "disconnected!" server.log > /dev/null; then - sleep 0.001 - fi - - # validate server output against known good - processed_log=`mktemp` - sed 's/.*: //' < server.log | sed 's/[0-9]*:[a-zA-Z0-9/+]*/<ph num>:<dev id>/g' > $processed_log + # process server log and remove header, base64 strings and phone numbers + sed 's/.*: //' < server.log \ + | sed "s/'[0-9]*'/<phone_num>/g" \ + | sed "s/'[a-zA-Z0-9/+]*'/<base64>/g" \ + > $test_dir/server.log # truncate server log, don't delete it as it won't be recreated > server.log - if ! diff -u $(dirname $t)/expected_out $processed_log > /dev/null - then - echo "${red}server log output diff failed$reset" - diff -u `dirname $t`/expected_out $processed_log + if ! make -s -C $test_dir diff; then + printf "%3s %s: %s%s%s\n" $count $test_dir $red "diff failed" $reset + failed=$((failed + 1)) - rm $processed_log cleanup $temp_db continue fi - rm $processed_log - echo "${green}ok${reset}" + printf "%3s %s: %s%s%s\n" $count $test_dir $green "ok" $reset passed=$((passed + 1)) cleanup $temp_db + make -s -C $test_dir clean done -printf "\n%i %sok%s %i %sfail%s " $passed $green $reset $failed $red $reset +printf "\n" +if [ $passed -ne 0 ]; then + printf "%i %sok%s " $passed $green $reset +fi +if [ $failed -ne 0 ]; then + printf "%i %sfailed%s " $failed $red $reset +fi # magic shell variable $SECONDS contains number of seconds since script start printf "(took %i min %i sec)\n" $((SECONDS/60)) $((SECONDS%60)) @@ -126,6 +94,4 @@ kill $server_pid rm $temp_db rm server.log -if [ $failed -ne 0 ]; then - exit 1 -fi +exit $failed; diff --git a/tests/add_friend/Makefile b/tests/add_friend/Makefile @@ -0,0 +1,4 @@ +test: + perl test.pl + +.include "../test.mk" diff --git a/tests/add_friend/expected_out b/tests/add_friend/expected_out diff --git a/tests/add_friend/server.log.good b/tests/add_friend/server.log.good @@ -0,0 +1,3 @@ +new connection +added new device <phone_num> <base64> +<base64> adding <phone_num> diff --git a/tests/add_friend/test.pl b/tests/add_friend/test.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -Itests +#!/usr/bin/perl -I../ use strict; use warnings; diff --git a/tests/bad_msg/Makefile b/tests/bad_msg/Makefile @@ -0,0 +1,4 @@ +test: + sh test.sh + +.include "../test.mk" diff --git a/tests/bad_msg/expected_output b/tests/bad_msg/expected_output diff --git a/tests/bad_msg/expected_out b/tests/bad_msg/server.log.good diff --git a/tests/bad_msg/test.sh b/tests/bad_msg/test.sh @@ -1,3 +1,5 @@ #!/bin/sh -echo -n "baddbedd" | xxd -r -p | nc 127.0.0.1 ${1} +. ../test.sh + +echo -n "baddbedd" | xxd -r -p | nc 127.0.0.1 ${PORT} diff --git a/tests/lib.sh b/tests/lib.sh @@ -1,6 +0,0 @@ -#!/bin/sh - -fail() { - echo -n "$1" - exit 1 -} diff --git a/tests/new_device/Makefile b/tests/new_device/Makefile @@ -0,0 +1,4 @@ +test: + perl test.pl + +.include "../test.mk" diff --git a/tests/new_device/expected_out b/tests/new_device/expected_out @@ -1,3 +0,0 @@ -new connection -added new device <ph num>:<dev id> -disconnected! diff --git a/tests/new_device/server.log.good b/tests/new_device/server.log.good @@ -0,0 +1,3 @@ +new connection +added new device <phone_num> <base64> +disconnected! diff --git a/tests/new_device/test.pl b/tests/new_device/test.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -Itests/ -I.. -I../.. +#!/usr/bin/perl -I../ use strict; use warnings; diff --git a/tests/new_device_bad_phnum/Makefile b/tests/new_device_bad_phnum/Makefile @@ -0,0 +1,4 @@ +test: + perl test.pl + +.include "../test.mk" diff --git a/tests/new_device_bad_phnum/expected_out b/tests/new_device_bad_phnum/server.log.good diff --git a/tests/new_device_bad_phnum/test.pl b/tests/new_device_bad_phnum/test.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -Itests +#!/usr/bin/perl -I../ use strict; use warnings; diff --git a/tests/new_list/Makefile b/tests/new_list/Makefile @@ -0,0 +1,4 @@ +test: + perl test.pl + +.include "../test.mk" diff --git a/tests/new_list/expected_out b/tests/new_list/expected_out diff --git a/tests/new_list/server.log.good b/tests/new_list/server.log.good @@ -0,0 +1,6 @@ +new connection +added new device <phone_num> <base64> +'this is a new list' +adding first list member devid = <base64> +list fingerprint = <base64> +disconnected! diff --git a/tests/new_list/test.pl b/tests/new_list/test.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -Itests -I.. -I../.. +#!/usr/bin/perl -I../ use strict; use warnings; diff --git a/tests/new_list_bad_devid/Makefile b/tests/new_list_bad_devid/Makefile @@ -0,0 +1,4 @@ +test: + perl test.pl + +.include "../test.mk" diff --git a/tests/new_list_bad_devid/expected_out b/tests/new_list_bad_devid/expected_out @@ -1,3 +0,0 @@ -new connection -unknown device 'a' -disconnected! diff --git a/tests/new_list_bad_devid/server.log.good b/tests/new_list_bad_devid/server.log.good @@ -0,0 +1,3 @@ +new connection +unknown device <base64> +disconnected! diff --git a/tests/new_list_bad_devid/test.pl b/tests/new_list_bad_devid/test.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -Itests +#!/usr/bin/perl -I../ use strict; use warnings; diff --git a/tests/new_list_missing_name/Makefile b/tests/new_list_missing_name/Makefile @@ -0,0 +1,4 @@ +test: + perl test.pl + +.include "../test.mk" diff --git a/tests/new_list_missing_name/expected_out b/tests/new_list_missing_name/expected_out @@ -1,4 +0,0 @@ -new connection -added new device <ph num>:<dev id> -list name missing -disconnected! diff --git a/tests/new_list_missing_name/server.log.good b/tests/new_list_missing_name/server.log.good @@ -0,0 +1,4 @@ +new connection +added new device <phone_num> <base64> +list name missing +disconnected! diff --git a/tests/new_list_missing_name/test.pl b/tests/new_list_missing_name/test.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -Itests/ +#!/usr/bin/perl -I../ use strict; use warnings; diff --git a/tests/payload_size_zero/Makefile b/tests/payload_size_zero/Makefile @@ -0,0 +1,4 @@ +test: + perl test.pl + +.include "../test.mk" diff --git a/tests/payload_size_zero/expected_out b/tests/payload_size_zero/server.log.good diff --git a/tests/payload_size_zero/test.pl b/tests/payload_size_zero/test.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -Itests +#!/usr/bin/perl -I../ use strict; use warnings; diff --git a/tests/payload_too_large/Makefile b/tests/payload_too_large/Makefile @@ -0,0 +1,4 @@ +test: + perl test.pl + +.include "../test.mk" diff --git a/tests/payload_too_large/expected_out b/tests/payload_too_large/server.log.good diff --git a/tests/payload_too_large/test.pl b/tests/payload_too_large/test.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -Itests +#!/usr/bin/perl -I../ use strict; use warnings; diff --git a/tests/test.mk b/tests/test.mk @@ -0,0 +1,5 @@ +diff: + diff -u server.log.good server.log + +clean: + rm -f server.log diff --git a/tests/test.sh b/tests/test.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +if [ "$PORT" == "" ]; then + echo "PORT environment variable must be set!" + exit 1 +fi + +fail() { + echo -n "$1" + exit 1 +} diff --git a/tests/testlib.pm b/tests/testlib.pm @@ -14,15 +14,15 @@ sub fail { sub new_socket { - if (@ARGV != 1) { - fail "$0: error, test takes exactly one port argument\n"; + if (! defined $ENV{PORT}) { + fail "$0: error, test needs PORT environment variable set\n"; exit 1; } my $sock = new IO::Socket::INET( LocalHost => '127.0.0.1', PeerHost => '127.0.0.1', - PeerPort => $ARGV[0], + PeerPort => $ENV{PORT}, Proto => 'tcp' );