Refactor tests, add status checks, add child sub reaper tests and

reduce the number of images created for testing.

test/Dockerfile

@@ -9,5 +9,5 @@ COPY  fixtures/config  /reaper/config
 
 #  Use entrypoint, so we can pass command line parameters for running
 #  the different tests.
-ENTRYPOINT ["/reaper/testpid1"]
+ENTRYPOINT ["/reaper/bin/init.sh"]
 CMD [""]

test/Makefile

@@ -8,7 +8,7 @@ OOP_INIT_TEST_IMAGE = "reaper/oop-init-config-test"
 
 all:	build
 
-build:	lint vet tests
+build:	deps lint vet image
 
 clean:
 	@echo "  - Removing testpid1 binary ..."
@@ -27,25 +27,39 @@ clean:
 	$(RM) /tmp/reaper-tests/*.log
 
 test:	tests
-tests:	image test-image test-debug-on test-notify test-non-pid1 test-oop-init
+tests:	build test-image test-debug-on test-notify test-non-pid1 test-oop-init
 	@echo "  - All tests passed."
 
 
 #
 #  Lint and vet targets.
 #
 lint:
-	@echo  "  - Linting sources ..."
+	@echo "  - Linting shell sources ..."
+	@(shellcheck bin/*.sh build/*.sh *.sh || :)
+	@echo "  - shellcheck passed."
+
+	@echo "  - Linting sources ..."
 	gofmt -d -s testpid1.go fixtures/oop-init/testpid1.go
-	@echo  "  - Linter checks passed."
+	@echo "  - Linter checks passed."
 
 vet:
-	@echo  "  - Vetting go sources ..."
+	@echo "  - Vetting go sources ..."
 	go vet testpid1.go
 	go vet fixtures/oop-init/testpid1.go
-	@echo  "  - go vet checks passed."
+	@echo "  - go vet checks passed."
 
 
+#
+#  Update dependencies (modules).
+#
+deps:
+	@echo "  - Update dependencies ..."
+	go mod tidy
+
+	@echo "  - Download go modules ..."
+	go mod download   #  -x
+
 #
 #  Build a single image to test the various reaper fixtures configs with.
 #
@@ -82,14 +96,17 @@ test-status-close:
 	@echo "  - Running reaper image status close test ..."
 	./runtests.sh $(TEST_IMAGE) /reaper/config/status-close-reaper.json
 
+test-non-pid1:	test-child-sub-reaper
+	@echo "  - Running reaper image non pid1 test ..."
+	./runtests.sh $(TEST_IMAGE) /reaper/config/non-pid1-reaper.json
+
+test-child-sub-reaper:
+	@echo "  - Running reaper image non pid1 child-sub-reaper test ..."
+	./runtests.sh $(TEST_IMAGE) /reaper/config/child-sub-reaper.json
 
 #
 #  Custom image tests.
 #
-test-non-pid1:
-	(build/image.sh $(NON_PID1_CONFIG_TEST_IMAGE) "fixtures/non-pid1";  \
-	 ./runtests.sh $(NON_PID1_CONFIG_TEST_IMAGE) )
-
 test-oop-init:
 	(build/image.sh $(OOP_INIT_TEST_IMAGE) "fixtures/oop-init";  \
 	 ./runtests.sh $(OOP_INIT_TEST_IMAGE) )

test/bin/init.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+
+#
+# Check options to figure out if we need to run the reaper as a child
+# process (!pid-1) or as the first process (as pid 1 replacing this script).
+#
+function _start_reaper() {
+    local cfg=${1:-""}
+    local forked=0
+
+    if [ -f "${cfg}" ]; then
+        echo "  - Checking if we need to run reaper as a child process ..."
+        if grep -Ee '"DisablePid1Check":\s*true' "${cfg}" ; then
+            forked=1
+        fi
+    fi
+
+    if [ "${forked}" -eq 1 ]; then
+        echo "  - Running reaper as child process ..."
+
+        # Run testpid1 with a new parent and process group id/session id.
+        bash <<EOF
+            set -m
+            echo "  - New session pid = \$\$"
+            echo "  - Starting testpid1 ..."
+            /reaper/testpid1 "$@"
+EOF
+
+        exit $?
+    fi
+
+
+    echo "  - Replacing init.sh with reaper as pid 1 ..."
+    exec /reaper/testpid1 "$1" "$@"
+
+}  #  End of function  _start_reaper.
+
+
+#
+#  main():
+#
+_start_reaper "$@"

test/bin/oop-workers.sh

@@ -6,11 +6,11 @@ SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
 # Run batched, so that we create a new process group id/session id.
 set -m
 
-echo "grepping root ..."
-cat /etc/passwd | grep root | sort | grep root
+echo "  - grepping root ..."
+grep -v -Ee "^$" /etc/passwd | grep root | sort | grep root
 
-echo "starting worker ..."
-nohup bash -c "$SCRIPT_DIR/worker.sh $@" < /dev/null &> /dev/null &
+echo "  - Starting worker ..."
+nohup bash -c "$SCRIPT_DIR/worker.sh $*" < /dev/null &> /dev/null &
 pid=$!
-echo "  - Started background worker - pid=$pid"
+echo "  - Started background worker - pid=${pid}"
 set +m

test/bin/script.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+
 #
 #  Usage: $0 <num-threads> <worker-args>
 #         where:  <num-threads> = Number of threads - default 5.
@@ -18,10 +19,10 @@ SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}")
 NTIMES=${1:-"5"}
 shift
 
-echo  "  -  $0 started with $NTIMES parallel threads ..."
+echo  "  - pid $$: $0 started with $NTIMES parallel threads ..."
 
-for i in $(seq $NTIMES); do
-  nohup bash -c "$SCRIPT_DIR/worker.sh $@" < /dev/null &> /dev/null &
+for idx in $(seq "${NTIMES}"); do
+  nohup bash -c "$SCRIPT_DIR/worker.sh $*" < /dev/null &> /dev/null &
   pid=$!
-  echo "  - Started background worker - pid=$pid"
+  echo "  - Started background worker #${idx} - pid=${pid}"
 done

test/bin/worker.sh

@@ -11,10 +11,7 @@
 #  Constants.
 readonly DEFAULT_WORKERS=10
 readonly DEFAULT_DELAY_SECS=5
-
-#  Change log file if you want see the script output.
-#    LOGFILE=/tmp/worker.log
-LOGFILE=/dev/null
+readonly LOGFILE=/tmp/worker.log
 
 
 function start_sleeper() {

test/fixtures/config/child-sub-reaper.json

@@ -0,0 +1,8 @@
+{
+    "Pid": 0,
+    "DisablePid1Check": true,
+    "EnableChildSubreaper": true,
+    "Status": true,
+    "Debug": true,
+    "Options": 0
+}

test/fixtures/oop-init/Dockerfile

@@ -0,0 +1,9 @@
+# FROM scratch
+FROM fedora
+
+MAINTAINER  [email protected]
+
+ADD ./bin     /reaper/bin
+ADD testpid1  /reaper/testpid1
+
+CMD ["/reaper/testpid1"]

test/runtests.sh

@@ -12,22 +12,23 @@ logfile="/tmp/reaper-tests/test.log"
 #
 #  Return list of sleeper processes.
 #
-function get_sleepers() {
+function _get_sleepers() {
     #shellcheck disable=SC2009
-    ps -ef -p "$1" | grep sleep | grep -v grep
+    ps --forest -o pid,ppid,time,cmd -g "${pid1}" |  \
+        grep sleep | grep -v grep
 
-}  #  End of function  get_sleepers.
+}  #  End of function  _get_sleepers.
 
 
 #
 #  Check for orphaned processes.
 #
-function check_orphans() {
+function _check_orphans() {
     local pid1=$1
 
     sleep "${MAX_SLEEP_TIME}"
     local orphans=""
-    orphans=$(get_sleepers "${pid1}")
+    orphans=$(_get_sleepers "${pid1}")
 
     if [ -n "${orphans}" ]; then
         echo ""
@@ -41,30 +42,82 @@ function check_orphans() {
 
     return 0
 
-}  #  End of function  check_orphans.
+}  #  End of function  _check_orphans.
 
 
 #
 #  Terminate docker container.
 #
-function terminate_container() {
+function _terminate_container() {
     local logdir=""
     logdir=$(dirname "${logfile}")
 
     mkdir -p "${logdir}"
     docker logs "$1" > "${logfile}"
 
+    #  append worker logs to the logfile.
+    (echo ""; echo "";                                          \
+        echo "----------------------------------------------";  \
+        echo "  - Worker logs:";                                \
+        docker exec "$1" cat /tmp/worker.log;                   \
+        echo "----------------------------------------------";  \
+    ) >> "${logfile}"
+
     echo "  - Container logs saved to ${logfile}"
 
     echo "  - Terminated container $(docker rm -f "$1")"
 
-}  #  End of function  terminate_container.
+}  #  End of function  _terminate_container.
+
+
+#
+#  If we have status notifications turned on (and are not randomly closing
+#  the status channel), then check whether we got the various different
+#  exit codes from the reaped descendants.
+#
+function _check_status_exit_codes() {
+    local name=${1:-""}
+
+    if [ "z${name}" != "zstatus-reaper" ] &&  \
+       [ "z${name}" != "zchild-sub-reaper" ]; then
+        #  Nothing to do. Test doesn't have any status notifications, so
+        #  the log won't contain any reaped descendant exit codes.
+        return 0
+    fi
+
+    #  Check that we have expected status lines.
+    for code in 1 2 7 13 21 29 30 31 64 65 66 69 70 71 74 76 77 78 127; do
+        local pid=""
+        for pid in $(grep -Ee "exitcode=${code}\$" "${logfile}" |  \
+                         awk -F '=' '{print $2}' |                 \
+                         awk -F ',' '{print $1}' | tr '\r\n' ' '); do
+            #echo "  - Descendant pid ${pid} exited with code ${code}"
+
+            #  On a slower vm (circa 2014), might not always get the status
+            #  reported - it could fill up the channel notifications, so ...
+            if grep "status of pid ${pid}" "${logfile}" > /dev/null; then
+                local reported=""
+                reported=$(grep "status of pid ${pid}" "${logfile}" |   \
+                               sed 's/.*status of pid.*exit code //g' |  \
+                               tr -d '\r')
+                if [ "${reported}" -ne "${code}" ]; then
+                    echo "ERROR: Child pid ${pid} exited with code ${code},"
+                    echo "       but reported code is ${reported}"
+                    exit 65
+                fi
+            fi
+        done
+    done
+
+    return 0
+
+}  #  End of function  _check_status_exit_codes.
 
 
 #
 #  Run reaper tests.
 #
-function run_tests() {
+function _run_tests() {
     local image=${1:-"${IMAGE}"}
     shift
 
@@ -91,15 +144,17 @@ function run_tests() {
     local pid1=""
     pid1=$(docker inspect --format '{{.State.Pid}}' "${elcid}")
 
+    sleep 1.42
+
     echo "  - Docker container pid=${pid1}"
-    echo "  - PID ${pid1} has $(get_sleepers "${pid1}" | wc -l) sleepers."
+    echo "  - PID ${pid1} has $(_get_sleepers "${pid1}" | wc -l) sleepers."
     echo "  - Sleeping for ${MAX_SLEEP_TIME} seconds ..."
     sleep "${MAX_SLEEP_TIME}"
 
     echo "  - Checking for orphans attached to pid1=${pid1} ..."
-    if ! check_orphans "${pid1}"; then
+    if ! _check_orphans "${pid1}"; then
         #  Got an error, cleanup and exit with error code.
-        terminate_container "${elcid}"
+        _terminate_container "${elcid}"
         echo ""
         echo "FAIL: All tests failed - (1/1)"
         exit 65
@@ -112,30 +167,41 @@ function run_tests() {
     docker kill -s USR1 "${elcid}"
 
     sleep 1
-    echo "  - PID ${pid1} has $(get_sleepers "${pid1}" | wc -l) sleepers."
+    echo "  - PID ${pid1} has $(_get_sleepers "${pid1}" | wc -l) sleepers."
 
     echo "  - Sleeping once again for ${MAX_SLEEP_TIME} seconds ..."
     sleep "${MAX_SLEEP_TIME}"
 
+    echo "  - Running processes under ${pid1}:"
+    pstree "${pid1}"
+    ps --forest -o pid,ppid,time,cmd -g "${pid1}" || :
+
     echo "  - Checking for orphans attached to pid1=${pid1} ..."
-    if ! check_orphans "${pid1}"; then
+    if ! _check_orphans "${pid1}"; then
         #  Got an error, cleanup and exit with error code.
-        terminate_container "${elcid}"
+        _terminate_container "${elcid}"
         echo ""
         echo "FAIL: Some tests failed - (1/2)"
         exit 65
     fi
 
+    echo "  - Running processes under ${pid1}:"
+    pstree "${pid1}"
+    ps --forest -o pid,ppid,time,cmd -g "${pid1}" || :
+
     #  Do the cleanup.
-    terminate_container "${elcid}"
+    _terminate_container "${elcid}"
+
+    #  If we have the status, check the different exit codes.
+    _check_status_exit_codes  "${name}"
 
     echo ""
     echo "OK: All tests passed - (2/2)"
 
-} #  End of function  run_tests.
+} #  End of function  _run_tests.
 
 
 #
 #  main():
 #
-run_tests "$@"
+_run_tests "$@"