commit 27f120378eb2dfd739acdc5591bf40fbcb024e49
parent df118906c4874551497b578657d84b36a3580b89
Author: kyle <kyle@0x30.net>
Date: Tue, 29 Dec 2015 17:50:41 -0700
tests: attempt to deal with process non-determinism
- because of the multi process server design, sometimes processes will do things
at different times
- this lead to instability in the server log output check
- introduce DIFF_MOD make variable
- when set like DIFF_MOD=sort, the server.log file will be sorted before diff
- this makes sure all output lines are accounted for
- but is weaker than checking line order explicitly
- this should be a good compromise until a proper unified server design is
complete
- also switch to using gmake, as conditional statements are non-portable across
make implementations
Diffstat:
8 files changed, 70 insertions(+), 52 deletions(-)
diff --git a/run_tests.sh b/run_tests.sh
@@ -9,6 +9,10 @@ if which tput > /dev/null; then
reset=`tput sgr0`
fi
+if [ `uname` = "OpenBSD" ]; then
+ alias make=gmake
+fi
+
fail() {
printf "%3s %s: $red%s$reset\n" $count "$1" "$2"
failed=`expr $failed + 1`
diff --git a/tests/join_friends_list/Makefile b/tests/join_friends_list/Makefile
@@ -1 +1,3 @@
+DIFF_MOD=sort
+
include ../test.mk
diff --git a/tests/join_friends_list/server.log.good b/tests/join_friends_list/server.log.good
@@ -1,26 +1,26 @@
accepting connections on <ip>:<port> (pid = <digits>)
-new connection (pid = <digits>)
-ssl ok, ver = 'TLSv1_2' cipher = 'ECDHE-RSA-AES128-SHA256'
-new_device: success, <digits>:<base64> os <base64>
+add_friend: <base64> adding <digits>
add_friend: <base64> adding <digits>
add_friend: added friend is a member
-add_friend: friends device id is <base64>
+add_friend: added friend is a member
add_friend: found mutual friendship
+add_friend: friends device id is <base64>
+add_friend: friends device id is <base64>
+disconnected!
+disconnected!
join_list: device <base64>
-join_list: list <base64>
join_list: device <base64> has been added to list <base64>
-list_get: gathering lists for <base64>
+join_list: list <base64>
list_get: found list <string> <base64>
-list_get: list has 2 members
+list_get: gathering lists for <base64>
list_get: list has 0 items
-disconnected!
+list_get: list has 2 members
+new connection (pid = <digits>)
new connection (pid = <digits>)
-ssl ok, ver = 'TLSv1_2' cipher = 'ECDHE-RSA-AES128-SHA256'
new_device: success, <digits>:<base64> os <base64>
-add_friend: <base64> adding <digits>
-add_friend: added friend is a member
-add_friend: friends device id is <base64>
+new_device: success, <digits>:<base64> os <base64>
new_list: <string>
new_list: adding first member devid = <base64>
new_list: fingerprint = <base64>
-disconnected!
+ssl ok, ver = 'TLSv1_2' cipher = 'ECDHE-RSA-AES128-SHA256'
+ssl ok, ver = 'TLSv1_2' cipher = 'ECDHE-RSA-AES128-SHA256'
diff --git a/tests/list_get_other/Makefile b/tests/list_get_other/Makefile
@@ -1 +1,3 @@
+DIFF_MOD=sort
+
include ../test.mk
diff --git a/tests/list_get_other/server.log.good b/tests/list_get_other/server.log.good
@@ -1,22 +1,22 @@
accepting connections on <ip>:<port> (pid = <digits>)
-new connection (pid = <digits>)
-ssl ok, ver = 'TLSv1_2' cipher = 'ECDHE-RSA-AES128-SHA256'
-new_device: success, <digits>:<base64> os <base64>
+add_friend: <base64> adding <digits>
add_friend: <base64> adding <digits>
add_friend: added friend is a member
-add_friend: friends device id is <base64>
+add_friend: added friend is a member
add_friend: found mutual friendship
-list_get_other: gathering lists for <base64>
-list_get_other: found mutual friend <base64>
-list_get_other: found list <string>
+add_friend: friends device id is <base64>
+add_friend: friends device id is <base64>
+disconnected!
disconnected!
+list_get_other: found list <string>
+list_get_other: found mutual friend <base64>
+list_get_other: gathering lists for <base64>
+new connection (pid = <digits>)
new connection (pid = <digits>)
-ssl ok, ver = 'TLSv1_2' cipher = 'ECDHE-RSA-AES128-SHA256'
new_device: success, <digits>:<base64> os <base64>
-add_friend: <base64> adding <digits>
-add_friend: added friend is a member
-add_friend: friends device id is <base64>
+new_device: success, <digits>:<base64> os <base64>
new_list: <string>
new_list: adding first member devid = <base64>
new_list: fingerprint = <base64>
-disconnected!
+ssl ok, ver = 'TLSv1_2' cipher = 'ECDHE-RSA-AES128-SHA256'
+ssl ok, ver = 'TLSv1_2' cipher = 'ECDHE-RSA-AES128-SHA256'
diff --git a/tests/multiple_friends_same_other_list/Makefile b/tests/multiple_friends_same_other_list/Makefile
@@ -1 +1,3 @@
+DIFF_MOD=sort
+
include ../test.mk
diff --git a/tests/multiple_friends_same_other_list/server.log.good b/tests/multiple_friends_same_other_list/server.log.good
@@ -1,38 +1,38 @@
accepting connections on <ip>:<port> (pid = <digits>)
-new connection (pid = <digits>)
-ssl ok, ver = 'TLSv1_2' cipher = 'ECDHE-RSA-AES128-SHA256'
-new_device: success, <digits>:<base64> os <base64>
+add_friend: <base64> adding <digits>
+add_friend: <base64> adding <digits>
+add_friend: <base64> adding <digits>
add_friend: <base64> adding <digits>
add_friend: added friend is a member
-add_friend: friends device id is <base64>
+add_friend: added friend is a member
+add_friend: added friend is a member
+add_friend: added friend is a member
add_friend: found mutual friendship
+add_friend: found mutual friendship
+add_friend: friends device id is <base64>
+add_friend: friends device id is <base64>
+add_friend: friends device id is <base64>
+add_friend: friends device id is <base64>
+disconnected!
+disconnected!
+disconnected!
join_list: device <base64>
-join_list: list <base64>
join_list: device <base64> has been added to list <base64>
-disconnected!
+join_list: list <base64>
+list_get_other: found list <string>
+list_get_other: found list <string>
+list_get_other: found mutual friend <base64>
+list_get_other: found mutual friend <base64>
+list_get_other: gathering lists for <base64>
+new connection (pid = <digits>)
+new connection (pid = <digits>)
new connection (pid = <digits>)
-ssl ok, ver = 'TLSv1_2' cipher = 'ECDHE-RSA-AES128-SHA256'
new_device: success, <digits>:<base64> os <base64>
-add_friend: <base64> adding <digits>
-add_friend: added friend is a member
-add_friend: friends device id is <base64>
-add_friend: found mutual friendship
+new_device: success, <digits>:<base64> os <base64>
+new_device: success, <digits>:<base64> os <base64>
new_list: <string>
new_list: adding first member devid = <base64>
new_list: fingerprint = <base64>
-disconnected!
-new connection (pid = <digits>)
ssl ok, ver = 'TLSv1_2' cipher = 'ECDHE-RSA-AES128-SHA256'
-new_device: success, <digits>:<base64> os <base64>
-add_friend: <base64> adding <digits>
-add_friend: added friend is a member
-add_friend: friends device id is <base64>
-add_friend: <base64> adding <digits>
-add_friend: added friend is a member
-add_friend: friends device id is <base64>
-list_get_other: gathering lists for <base64>
-list_get_other: found mutual friend <base64>
-list_get_other: found list <string>
-list_get_other: found mutual friend <base64>
-list_get_other: found list <string>
-disconnected!
+ssl ok, ver = 'TLSv1_2' cipher = 'ECDHE-RSA-AES128-SHA256'
+ssl ok, ver = 'TLSv1_2' cipher = 'ECDHE-RSA-AES128-SHA256'
diff --git a/tests/test.mk b/tests/test.mk
@@ -1,7 +1,7 @@
test:
perl test.pl
-diff:
+process-server-log:
# remove header, phone numbers, base64 and strings from server.log
sed -i -e "s/.*> //" \
-e "s/'[0-9]*'/<digits>/g" \
@@ -9,7 +9,15 @@ diff:
-e "s/'[a-zA-Z0-9 ]*'/<string>/g" \
-e 's/[0-9]\.[0-9]\.[0-9]\.[0-9]:[0-9]*/<ip>:<port>/' \
server.log
+
+ifndef DIFF_MOD
+diff: process-server-log
+ diff -u server.log.good server.log
+else
+diff: process-server-log
+ sort -o server.log < server.log
diff -u server.log.good server.log
+endif
clean:
rm -f server.log
