Proposed test suite changes - ngIRCd Users

25 Apr 2008

Hi everyone,
Brandon, Ali, and I were using the test suite and noticed that we could
add some useful features.  As we were playing with the code, we would run
the test and some would fail.  We felt it took more work than necessary to
figure out which test failed.  As a result we made some changes to the
testsuite.
changes:
-We added puts before each test to act as a label so we could tell which
tests log messages belonged to.
  =This helped us quickly determine what was causing test to fail, because
we could easily see the erroneous server communication.
  =Test labels must begin with ">--". This allows them to easily and
quickly be seen in the log files and allows the second change to
function
-Added the -s option when running the individual tests
  =This option will show all the test that were run and if a test fails
gives the user the option of viewing all the log messages associated with
the failed test.
While these changes add slightly more work to writing tests (adding a
label) we have found them very useful when the tests don't pass. We added
-s as an option instead of a default because there are some situations
where the added information isn't needed and to make the change
transparent to those who don't want to use it.
-Ali, Brandon, Bryan
---------------------------------------------

diff --git a/src/testsuite/channel-test.e b/src/testsuite/channel-test.e
index 16c67ad..88388c1 100644
--- a/src/testsuite/channel-test.e
+++ b/src/testsuite/channel-test.e
@@ -1,11 +1,13 @@
 # $Id: channel-test.e,v 1.4 2008/02/05 13:31:51 fw Exp $
+puts ">-- Connecting to test server ... "
 spawn telnet localhost 6789
 expect {
    timeout { exit 1 }
    "Connected"
 }
+puts ">-- setting nick ... "
 send "nick nick\r"
 send "user user . . :User\r"
 expect {
@@ -13,22 +15,26 @@ expect {
    "376"
 }
+puts ">-- join #channel test ... "
 send "join #channel\r"
 expect {
    timeout { exit 1 }
    "@* JOIN :#channel"
 }
+puts ">-- names list test ... "
 expect {
    timeout { exit 1 }
    "366"
 }
+puts ">-- set topic test ... "
 send "topic #channel :Test-Topic\r"
 expect {
    timeout { exit 1 }
    "@* TOPIC #channel :Test-Topic"
 }
+puts ">-- who #channel test ... "
 send "who #channel\r"
 expect {
    timeout { exit 1 }
@@ -38,37 +44,44 @@ expect {
    timeout { exit 1 }
    "* nick H@ :0 User"
 }
+puts ">-- end of who test ... "
 expect {
    timeout { exit 1 }
    "315 nick #channel"
 }
+puts ">-- names #channel test ... "
 send "names #channel\r"
 expect {
    timeout { exit 1 }
    "353 nick = #channel :@nick"
 }
+puts ">-- end of names test ... "
 expect {
    timeout { exit 1 }
    "366 nick #channel"
 }
+puts ">-- list test ... "
 send "list\r"
 expect {
    timeout { exit 1 }
    "322 nick #channel 1 :Test-Topic"
 }
+puts ">-- end of list test ... "
 expect {
    timeout { exit 1 }
    "323 nick :End of LIST"
 }
+puts ">-- part test ... "
 send "part #channel\r"
 expect {
    timeout { exit 1 }
    "@* PART #channel :nick"
 }
+puts ">-- re-join test ... "
 send "join #channel\r"
 expect {
    timeout { exit 1 }
@@ -79,6 +92,7 @@ expect {
    "366"
 }
+puts ">-- join #channel2 test ... "
 send "join #channel2\r"
 expect {
    timeout { exit 1 }
@@ -89,6 +103,7 @@ expect {
    "366"
 }
+puts ">-- join 0 test ... "
 send "join 0\r"
 expect {
    timeout { exit 1 }
@@ -99,6 +114,7 @@ expect {
    "@* PART #channel :nick"
 }
+puts ">-- quitting ... "
 send "quit\r"
 expect {
    timeout { exit 1 }
diff --git a/src/testsuite/check-idle.e b/src/testsuite/check-idle.e index
e994c14..04191f6 100644
--- a/src/testsuite/check-idle.e
+++ b/src/testsuite/check-idle.e
@@ -1,11 +1,13 @@
 # $Id: check-idle.e,v 1.2 2004/09/04 14:22:13 alex Exp $
+puts ">-- Connecting to test server ... "
 spawn telnet localhost 6789
 expect {
    timeout { exit 1 }
    "Connected"
 }
+puts ">-- Setting nick nick ... "
 send "nick IdleTest\r"
 send "user idle . . :Idle-Test\r"
 expect {
@@ -14,6 +16,7 @@ expect {
    "376"
 }
+puts ">-- lusers test ... "
 send "lusers\r"
 expect {
    timeout { exit 1 }
@@ -21,6 +24,7 @@ expect {
    "251 IdleTest :There are" { set r 99 }
 }
+puts ">-- quitting ... "
 send "quit\r"
 expect {
    timeout { exit 1 }
diff --git a/src/testsuite/connect-test.e b/src/testsuite/connect-test.e
index 49ce70c..9fc28e1 100644
--- a/src/testsuite/connect-test.e
+++ b/src/testsuite/connect-test.e
@@ -1,17 +1,20 @@
 # $Id: connect-test.e,v 1.1 2002/09/09 10:16:24 alex Exp $
+puts ">-- Connecting to test server ... "
 spawn telnet localhost 6789
 expect {
    timeout { exit 1 }
    "Connected"
 }
+puts ">-- oper test ... "
 send "oper\r"
 expect {
    timeout { exit 1 }
    "451"
 }
+puts ">-- quitting ... "
 send "quit\r"
 expect {
    timeout { exit 1 }
diff --git a/src/testsuite/misc-test.e b/src/testsuite/misc-test.e index
4b88379..5b404c3 100644
--- a/src/testsuite/misc-test.e
+++ b/src/testsuite/misc-test.e
@@ -1,11 +1,13 @@
 # $Id: misc-test.e,v 1.2 2008/02/17 13:51:00 alex Exp $
+puts ">-- Connecting to test server ... "
 spawn telnet localhost 6789
 expect {
    timeout { exit 1 }
    "Connected"
 }
+puts ">-- setting nick ... "
 send "nick nick\r"
 send "user user . . :User\r"
 expect {
@@ -13,28 +15,33 @@ expect {
    "376"
 }
+puts ">-- summon is disabled test ... "
 send "summon\r"
 expect {
    timeout { exit 1 }
    "445"
 }
+puts ">-- users is disabled test ... "
 send "users\r"
 expect {
    timeout { exit 1 }
    "446"
 }
+puts ">-- info test ... "
 send "info\r"
 expect {
    timeout { exit 1 }
    "371"
 }
+puts ">-- end of info test ... "
 expect {
    timeout { exit 1 }
    "374"
 }
+puts ">-- quitting ... "
 send "quit\r"
 expect {
    timeout { exit 1 }
diff --git a/src/testsuite/mode-test.e b/src/testsuite/mode-test.e index
260cd03..946833b 100644
--- a/src/testsuite/mode-test.e
+++ b/src/testsuite/mode-test.e
@@ -1,11 +1,13 @@
 # $Id: mode-test.e,v 1.7 2008/02/16 11:27:49 fw Exp $
+puts ">-- Connecting to test server ... "
 spawn telnet localhost 6789
 expect {
    timeout { exit 1 }
    "Connected"
 }
+puts ">-- setting nick ... "
 send "nick nick\r"
 send "user user . . :User\r"
 expect {
@@ -13,24 +15,28 @@ expect {
    "376"
 }
+puts ">-- set mode test ... "
 send "mode nick +i\r"
 expect {
    timeout { exit 1 }
    "@* MODE nick :+i"
 }
+puts ">-- get mode test ... "
 send "mode nick\r"
 expect {
    timeout { exit 1 }
    "221 nick +i"
 }
+puts ">-- unset mode test ... "
 send "mode nick -i\r"
 expect {
    timeout { exit 1 }
    "@* MODE nick :-i"
 }
+puts ">-- oper test ... "
 send "oper TestOp 123\r"
 expect {
    timeout { exit 1 }
@@ -41,12 +47,14 @@ expect {
    "381 nick"
 }
+puts ">-- is op test ... "
 send "mode nick\r"
 expect {
    timeout { exit 1 }
    "221 nick +o"
 }
+puts ">-- joining #channel ... "
 send "join #channel\r"
 expect {
    timeout { exit 1 }
@@ -57,48 +65,56 @@ expect {
    "366"
 }
+puts ">-- set #channel mode test ... "
 send "mode #channel +tn\r"
 expect {
    timeout { exit 1 }
    "@* MODE #channel +tn"
 }
+puts ">-- get #channel mode test ... "
 send "mode #channel\r"
 expect {
    timeout { exit 1 }
    "324 nick #channel +tn"
 }
+puts ">-- voice nick test ... "
 send "mode #channel +v nick\r"
 expect {
    timeout { exit 1 }
    "@* MODE #channel +v nick\r"
 }
+puts ">-- invite only with nick mask test ... "
 send "mode #channel +I nick1\r"
 expect {
    timeout { exit 1 }
    "@* MODE #channel +I nick1!*@*"
 }
+puts ">-- ban nick2 with domain mask test ... "
 send "mode #channel +b nick2@domain\r"
 expect {
    timeout { exit 1 }
    "@* MODE #channel +b nick2!*@domain"
 }
+puts ">-- invite only with user mask test ... "
 send "mode #channel +I nick3!user\r"
 expect {
    timeout { exit 1 }
    "@* MODE #channel +I nick3!user@*"
 }
+puts ">-- unset #channel mode test ... "
 send "mode #channel -vo nick nick\r"
 expect {
    timeout { exit 1 }
    "@* MODE #channel -vo nick nick\r"
 }
+puts ">-- quitting ... "
 send "quit\r"
 expect {
    timeout { exit 1 }
diff --git a/src/testsuite/stress-A.e b/src/testsuite/stress-A.e
index b22a2f3..18fe86d 100644
--- a/src/testsuite/stress-A.e
+++ b/src/testsuite/stress-A.e
@@ -2,6 +2,7 @@
set timeout 30
+puts ">-- Connecting to test server ... "
 spawn telnet localhost 6789
 expect {
    timeout { exit 1 }
diff --git a/src/testsuite/stress-B.e b/src/testsuite/stress-B.e
index e260c6d..f0c6539 100644
--- a/src/testsuite/stress-B.e
+++ b/src/testsuite/stress-B.e
@@ -1,5 +1,6 @@
 # $Id: stress-B.e,v 1.3 2005/12/30 22:12:28 alex Exp $
+puts ">-- setting user ... "
 send "user user . . :User\r"
 expect {
    timeout { exit 1 }
@@ -8,6 +9,7 @@ expect {
sleep 2
+puts ">-- becoming op ... "
 send "oper TestOp 123\r"
 expect {
    timeout { exit 1 }
@@ -20,6 +22,7 @@ expect {
sleep 2
+puts ">-- joining #channel ... "
 send "join #channel\r"
 expect {
    timeout { exit 1 }
@@ -30,12 +33,14 @@ expect {
    " 366 * #channel :"
 }
+puts ">-- getting #channel mode ... "
 send "mode #channel\r"
 expect {
    timeout { exit 1 }
    " 324 test* #channel"
 }
+puts ">-- joining #channel2 ... "
 send "join #channel2\r"
 expect {
    timeout { exit 1 }
@@ -46,6 +51,7 @@ expect {
    " 366 * #channel2 :"
 }
+puts ">-- listing names ... "
 send "names\r"
 expect {
    timeout { exit 1 }
@@ -54,12 +60,14 @@ expect {
sleep 3
+puts ">-- parting #channel2 ... "
 send "part #channel2\r"
 expect {
    timeout { exit 1 }
    " PART #channel2 "
 }
+puts ">-- parting #channel ... "
 send "part #channel\r"
 expect {
    timeout { exit 1 }
@@ -68,6 +76,7 @@ expect {
sleep 1
+puts ">-- quitting ... "
 send "quit\r"
 expect {
    timeout { exit 1 }
diff --git a/src/testsuite/tests.sh b/src/testsuite/tests.sh
index 2bd9ae4..0eaba50 100755
--- a/src/testsuite/tests.sh
+++ b/src/testsuite/tests.sh
@@ -26,9 +26,35 @@ if [ $? -ne 0 ]; then
   echo "      ${name}: "telnet" not found.";  exit 77
 fi
-echo_n "      running ${test} ..."
+if [ "$1" == "-s" ]; then
+  echo "      running ${test} ..."
+else
+  echo_n "      running ${test} ..."
+fi
 expect ${srcdir}/${test}.e > logs/${test}.log 2>&1; r=$?
-[ $r -eq 0 ] && echo " ok." || echo " failure!"
+if [ "$1" == "-s" ]; then
+  cat logs/${test}.log | grep ">--"
+fi
+
+if [ $r -eq 0 ]; then
+  echo " ok."
+else
+  if [ "$1" == "-s" ]; then
+  echo "--> failed."
+    echo_n "Display log for last test? (y/[n]): "
+    read response
+    if [ "$response" == "y" ]; then
+      begLastTest=`grep -n ">--" logs/${test}.log | tail -1`
+      begLastTest=${begLastTest%:*}
+      length=`wc -l logs/${test}.log`
+      length=${length% *}
+      fromEnd=`expr $length - $begLastTest + 1`
+      tail -n $fromEnd logs/${test}.log
+    fi
+  else
+    echo " failed."
+  fi
+fi
exit $r
diff --git a/src/testsuite/who-test.e b/src/testsuite/who-test.e
index c54a190..1573cb2 100644
--- a/src/testsuite/who-test.e
+++ b/src/testsuite/who-test.e
@@ -1,9 +1,12 @@
+
+puts ">-- Connecting to test server ... "
 spawn telnet localhost 6789
 expect {
    timeout { exit 1 }
    "Connected"
 }
+puts ">-- setting nick ... "
 send "nick nick\r"
 send "user user . . :Real Name\r"
 expect {
@@ -11,72 +14,84 @@ expect {
    "376"
 }
+puts ">-- who test ... "
 send "who\r"
 expect {
    timeout { exit 1 }
    ":ngircd.test.server 352 nick * ~user localhost ngircd.test.server nick
H :0 Real Name"
 }
+puts ">-- joining #channel ... "
 send "join #channel\r"
 expect {
    timeout { exit 1 }
    "@* JOIN :#channel"
 }
+puts ">-- who 0 test ... "
 send "who 0\r"
 expect {
    timeout { exit 1 }
    ":ngircd.test.server 352 nick #channel ~user localhost
ngircd.test.server nick H@ :0 Real Name"
 }
+puts ">-- set away test ... "
 send "away :testing\r"
 expect {
    timeout { exit 1 }
    "306 nick"
 }
+puts ">-- away who test ... "
 send "who *\r"
 expect {
    timeout { exit 1 }
    ":ngircd.test.server 352 nick #channel ~user localhost
ngircd.test.server nick G@ :0 Real Name"
 }
+puts ">-- voicing nick on #channel ... "
 send "mode #channel +v nick\r"
 expect {
    timeout { exit 1 }
    "@* MODE #channel +v nick\r"
 }
+puts ">-- who server by ip test ... "
 send "who localhost\r"
 expect {
    timeout { exit 1 }
    ":ngircd.test.server 352 nick #channel ~user localhost
ngircd.test.server nick G@ :0 Real Name"
 }
+puts ">-- un-op nick on #channel ... "
 send "mode #channel -o nick\r"
 expect {
    timeout { exit 1 }
    "@* MODE #channel -o nick\r"
 }
+puts ">-- who server by name test ... "
 send "who ngircd.test.server\r"
 expect {
    timeout { exit 1 }
    ":ngircd.test.server 352 nick #channel ~user localhost
ngircd.test.server nick G+ :0 Real Name"
 }
+puts ">-- parting #channel ... "
 send "part #channel\r"
 expect {
    timeout { exit 1 }
    "@* PART #channel :nick"
 }
+puts ">-- who by real name test ... "
 send "who Real?Name\r"
 expect {
    timeout { exit 1 }
    ":ngircd.test.server 352 nick * ~user localhost ngircd.test.server nick
G :0 Real Name"
 }
+puts ">-- becoming op ... "
 send "oper TestOp 123\r"
 expect {
    timeout { exit 1 }
@@ -87,72 +102,84 @@ expect {
    "381 nick"
 }
+puts ">-- who 0 with mask test ... "
 send "who 0 o\r"
 expect {
    timeout { exit 1 }
    ":ngircd.test.server 352 nick * ~user localhost ngircd.test.server nick
G* :0 Real Name"
 }
+puts ">-- unaway test ... "
 send "away\r"
 expect {
    timeout { exit 1 }
    "305 nick"
 }
+puts ">-- who with mask test ... "
 send "who *cal*ho??\r"
 expect {
    timeout { exit 1 }
    ":ngircd.test.server 352 nick * ~user localhost ngircd.test.server nick
H* :0 Real Name"
 }
+puts ">-- joining #opers ... "
 send "join #opers\r"
 expect {
    timeout { exit 1 }
    "@* JOIN :#opers"
 }
+puts ">-- who channel test ... "
 send "who #opers\r"
 expect {
    timeout { exit 1 }
    ":ngircd.test.server 352 nick #opers ~user localhost ngircd.test.server
nick H*@ :0 Real Name"
 }
+puts ">-- un-op nick on #opers ... "
 send "mode #opers -o nick\r"
 expect {
    timeout { exit 1 }
    "@* MODE #opers -o nick\r"
 }
+puts ">-- who server mask test ... "
 send "who *.server\r"
 expect {
    timeout { exit 1 }
    ":ngircd.test.server 352 nick #opers ~user localhost ngircd.test.server
nick H* :0 Real Name"
 }
+puts ">-- voicing nick in #opers ... "
 send "mode #opers +v nick\r"
 expect {
    timeout { exit 1 }
    "@* MODE #opers +v nick\r"
 }
+puts ">-- who real name mask test ... "
 send "who Real*me\r"
 expect {
    timeout { exit 1 }
    ":ngircd.test.server 352 nick #opers ~user localhost ngircd.test.server
nick H*+ :0 Real Name"
 }
+puts ">-- making #opers secret ... "
 send "mode #opers +s\r"
 expect {
    timeout { exit 1 }
    "@* MODE #opers +s\r"
 }
+puts ">-- who with mask test ... "
 send "who n?c?\r"
 expect {
    timeout { exit 1 }
    ":ngircd.test.server 352 nick * ~user localhost ngircd.test.server nick
H* :0 Real Name"
 }
+puts ">-- quitting ... "
 send "quit\r"
 expect {
    timeout { exit 1 }

    