commit 8eae33321610c7c1eeeb4b91223c7a273297ab55 parent 5f586c2e549f74d663d3d0e6e585452a2e26ca79 Author: kyle <kyle@getaddrinfo.net> Date: Wed, 28 Oct 2015 00:29:51 -0600 tests: add new tests, refine harness - we now check the following conditions on all tests - test runs and exits 0 - server didn't die while test was executing - the server log output is identical to known good output - add some new tests, some incomplete - if you want to run a particular test, move into it's directory and start it with port as its only argument (not working on all tests) Diffstat:
23 files changed, 203 insertions(+), 60 deletions(-)
diff --git a/run_tests.sh b/run_tests.sh @@ -1,55 +1,128 @@ #!/bin/sh -if ! pgrep -f perl\ sl; then - echo "server not running, you need to start it!" - exit 1 +# *never* connect to a real production instance with this script! +port=4729 +temp_db=$(mktemp) + +# start server with temp db, non standard port, and in the background +perl sl -p $port -d $temp_db > server.log & +server_pid=$! +echo -n "[$server_pid] server started" + +# clean up on ctrl-c +trap sigint_handler int + +sigint_handler() { + # remove temp db, and kill this process group + rm $temp_db + kill 0 +} + +if [ $# -eq 3 ]; then + echo "(implement running single test)" +fi + +# if tput is available we can do colors! +if which tput > /dev/null; then + # make sure we run on XTERM={screen,xterm} which have 8 colors + if [ $(tput colors) -ne -1 ] + then + red=$(tput setaf 1 0 0) + green=$(tput setaf 2 0 0) + yellow=$(tput setaf 3 0 0) + fi + reset=$(tput sgr0) fi -# if tput is available, we can do colors! -if which tput; then - RED=$(tput setaf 1 0 0) - GREEN=$(tput setaf 2 0 0) - YELLOW=$(tput setaf 3 0 0) - RESET=$(tput sgr0) +# don't start the test suite until we can connect to the server +while ! nc -z 127.0.0.1 $port 2> /dev/null; do + if ! ps -p $server_pid > /dev/null + then + echo " server died! log file was:" + cat server.log + exit 1 + fi + echo -n "." + sleep 0.1 +done + +# wait for the output triggered from the above `nc` command to be flushed +if grep "disconnected!" server.log > /dev/null; then + sleep 0.1 + echo -n "." fi +cat server.log +> server.log +echo " ready" -export TESTS=asdf +cleanup() { + > server.log -PORT=5437 + # clean up the database between runs + sqlite3 ${1} "delete from devices; + delete from lists; + delete from list_members; + delete from list_data; + delete from friends_map; + delete from mutual_friends;" +} passed=0 failed=0 -count=1 +count=0 for t in `ls tests/*/test.*`; do - CWD=$(pwd) - printf "%3s " $count - echo -n "$(dirname $t): $YELLOW" - - # XXX: put PORT in the environment - cd $(dirname $t) && ./$(basename $t) $PORT - if [ $? -ne 0 ]; then - echo "$RED fail$RESET" + count=$((count + 1)) + printf "%3s %s: %s" $count $(dirname $t) $yellow + + # run test + if ! ./${t} $port; then + echo "$red test failed$reset" failed=$((failed + 1)) - else - echo "$GREEN ok$RESET" - passed=$((passed + 1)) + cleanup $temp_db + continue fi - cd $CWD + echo -n "${reset}" - # `ps -o pid= -p $SERVER_PID` - # if [ $? -eq 0 ]; then - # echo ">>> $RED server died!$RESET" - # exit 1 - # fi + # validate the server is still running + if ! kill -0 $server_pid; then + echo "$red test killed server!$reset" + +cat server.log + rm $temp_db + exit 1 + fi - # clean up the database between runs - sqlite3 db "delete from devices" - sqlite3 db "delete from lists" - sqlite3 db "delete from list_members" + # wait for server output to be flushed + if grep "disconnected!" server.log > /dev/null; then + sleep 0.001 + fi - count=$((count + 1)) + # validate server output against known good + processed_log=`mktemp` + sed 's/.*: //' < server.log | sed 's/[0-9]*:[a-zA-Z0-9/+]*/<ph num>:<dev id>/g' > $processed_log + # truncate server log, don't delete it as it won't be recreated + > server.log + + if ! diff -u $(dirname $t)/expected_out $processed_log > /dev/null + then + echo "${red}server log output diff failed$reset" + diff -u `dirname $t`/expected_out $processed_log + failed=$((failed + 1)) + rm $processed_log + cleanup $temp_db + continue + fi + rm $processed_log + + echo "${green}ok${reset}" + passed=$((passed + 1)) + cleanup $temp_db done -echo "\n$passed$GREEN ok$RESET $failed$RED fail$RESET" +printf "\n%i %sok%s %i %sfail%s " $passed $green $reset $failed $red $reset +# magic shell variable $SECONDS contains number of seconds since script start +printf "(took %i min %i sec)\n" $((SECONDS/60)) $((SECONDS%60)) + +kill $server_pid +rm $temp_db if [ $failed -ne 0 ]; then exit 1 diff --git a/tests/add_friend/expected_out b/tests/add_friend/expected_out diff --git a/tests/add_friend/test.pl b/tests/add_friend/test.pl @@ -0,0 +1,29 @@ +#!/usr/bin/perl -Itests + +use strict; +use warnings; + +use testlib; + +# this test: +# - gets a new device id +# - then adds a new friend + +my $sock = new_socket(); +send_msg($sock, 0, "4038675309"); +my ($type, $device_id, $length) = recv_msg($sock); + +fail "got response type $type, expected 0" if ($type != 0); +fail "expected response length of 43, got $length" if ($length != 43); + +send_msg($sock, 2, "$device_id\0" . "4033217654"); +# my ($type2, $list_data, $length2) = recv_msg($sock); + +# fail "got response type $type, expected 1" if ($type2 != 2); + +# my ($id, $name, @members) = split("\0", $list_data); +# my $id_length = length($id); +# +# fail "bad id length $id_length != 43" if ($id_length != 43); +# fail "recv'd name '$name' not equal to '$list_name'" if ($name ne $list_name); +# fail "list does not have exactly 1 member" if (@members != 1); diff --git a/tests/bad_msg/expected_out b/tests/bad_msg/expected_out @@ -0,0 +1,3 @@ +new connection +unknown message type 0xbadd +disconnected! diff --git a/tests/bad_msg/expected_output b/tests/bad_msg/expected_output diff --git a/tests/bad_msg/test.sh b/tests/bad_msg/test.sh @@ -1,3 +1,3 @@ #!/bin/sh -echo -n "baddbedd" | xxd -r -p | nc 127.0.0.1 5437 +echo -n "baddbedd" | xxd -r -p | nc 127.0.0.1 ${1} diff --git a/tests/net_enums.sh b/tests/net_enums.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# GENERaTED @ Thu Oct 22 22:39:09 MDT 2015 BY gen_enums.sh + +export new_device=0000 +export new_list=0001 +export add_friend=0002 +export list_request=0003 +export join_list=0004 +export leave_list=0005 +export list_items=0006 +export new_list_item=0007 +export ok=0008 diff --git a/tests/new_device/expected_out b/tests/new_device/expected_out @@ -0,0 +1,3 @@ +new connection +added new device <ph num>:<dev id> +disconnected! diff --git a/tests/new_device/test.pl b/tests/new_device/test.pl @@ -1,12 +1,13 @@ -#!/usr/bin/perl -I.. +#!/usr/bin/perl -Itests/ -I.. -I../.. use strict; use warnings; +use MsgTypes; use testlib; my $sock = new_socket(); -send_msg($sock, 0, "4038675309"); +send_msg($sock, $msgs{new_device}, "4038675309"); my ($type, $response, $length) = recv_msg($sock); close $sock; diff --git a/tests/new_device_bad_phnum/expected_out b/tests/new_device_bad_phnum/expected_out @@ -0,0 +1,3 @@ +new connection +device phone number 403867530& invalid +disconnected! diff --git a/tests/new_device_bad_phnum/test.pl b/tests/new_device_bad_phnum/test.pl @@ -1,11 +1,12 @@ -#!/usr/bin/perl -I.. +#!/usr/bin/perl -Itests use strict; use warnings; +use MsgTypes; use testlib; # send a new device message with a bad phone number my $sock = new_socket(); -send_msg($sock, 0, "403867530&"); +send_msg($sock, $msgs{new_device}, "403867530&"); close($sock); diff --git a/tests/new_list/expected_out b/tests/new_list/expected_out diff --git a/tests/new_list/test.pl b/tests/new_list/test.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -I.. +#!/usr/bin/perl -Itests -I.. -I../.. use strict; use warnings; diff --git a/tests/new_list_bad_devid/expected_out b/tests/new_list_bad_devid/expected_out @@ -0,0 +1,3 @@ +new connection +unknown device 'a' +disconnected! diff --git a/tests/new_list_bad_devid/test.pl b/tests/new_list_bad_devid/test.pl @@ -0,0 +1,13 @@ +#!/usr/bin/perl -Itests + +use strict; +use warnings; + +use testlib; + +# this test: +# - gets a new device id +# - tries to create a new list with and invalid device id + +my $sock = new_socket(); +send_msg($sock, 1, "a\0some list over here"); diff --git a/tests/new_list_missing_devid/test.pl b/tests/new_list_missing_devid/test.pl @@ -1,13 +0,0 @@ -#!/usr/bin/perl -I.. - -use strict; -use warnings; - -use testlib; - -# this test: -# - gets a new device id -# - tries to create a new list with and invalid device id - -my $sock = new_socket(); -send_msg($sock, 1, "a\0some list over here"); diff --git a/tests/new_list_missing_name/expected_out b/tests/new_list_missing_name/expected_out @@ -0,0 +1,4 @@ +new connection +added new device <ph num>:<dev id> +list name missing +disconnected! diff --git a/tests/new_list_missing_name/test.pl b/tests/new_list_missing_name/test.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -I.. +#!/usr/bin/perl -Itests/ use strict; use warnings; diff --git a/tests/payload_size_zero/expected_out b/tests/payload_size_zero/expected_out @@ -0,0 +1,3 @@ +new connection +message size not 0 < 0 <= 1024 +disconnected! diff --git a/tests/payload_size_zero/test.pl b/tests/payload_size_zero/test.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -I.. +#!/usr/bin/perl -Itests use strict; use warnings; diff --git a/tests/payload_too_large/expected_out b/tests/payload_too_large/expected_out @@ -0,0 +1,3 @@ +new connection +message size not 0 < 4000 <= 1024 +disconnected! diff --git a/tests/payload_too_large/test.pl b/tests/payload_too_large/test.pl @@ -1,4 +1,4 @@ -#!/usr/bin/perl -I.. +#!/usr/bin/perl -Itests use strict; use warnings; diff --git a/tests/testlib.pm b/tests/testlib.pm @@ -14,14 +14,19 @@ sub fail { sub new_socket { + if (@ARGV != 1) { + fail "$0: error, test takes exactly one port argument\n"; + exit 1; + } + my $sock = new IO::Socket::INET( LocalHost => '127.0.0.1', PeerHost => '127.0.0.1', - PeerPort => 5437, + PeerPort => $ARGV[0], Proto => 'tcp' ); - die "error: socket creation failed: $!\n" unless $sock; + die "error: new socket: $!\n" unless $sock; return $sock; } @@ -52,7 +57,6 @@ sub recv_msg } # XXX: do msg type upper bounds checking here - # fail "server sent $resp_type msg instead of $type" if ($resp_type != $type); fail "bad message size not 0 <= $size < 1024" if ($size < 0 || $size > 1023); my $data; @@ -60,6 +64,7 @@ sub recv_msg fail "read() returned $bread instead of $size!"; } + # caller should validate this is the expected type return ($type, $data, $size); }