From 63503a156f609cc9dac5ef97adb515f930452219 Mon Sep 17 00:00:00 2001 From: ramr Date: Sun, 3 Nov 2024 20:03:23 -0800 Subject: [PATCH] Refactor tests, add status checks, add child sub reaper tests and reduce the number of images created for testing. --- test/.gitignore | 1 - test/Dockerfile | 2 +- test/Makefile | 37 +++++-- test/bin/init.sh | 43 ++++++++ test/bin/oop-workers.sh | 10 +- test/bin/script.sh | 9 +- test/bin/worker.sh | 5 +- test/fixtures/config/child-sub-reaper.json | 8 ++ .../non-pid1-reaper.json} | 0 test/fixtures/non-pid1/Dockerfile | 10 -- test/fixtures/oop-init/Dockerfile | 9 ++ test/runtests.sh | 102 ++++++++++++++---- test/testpid1.go | 27 ++--- 13 files changed, 198 insertions(+), 65 deletions(-) delete mode 100644 test/.gitignore create mode 100755 test/bin/init.sh create mode 100644 test/fixtures/config/child-sub-reaper.json rename test/fixtures/{non-pid1/reaper.json => config/non-pid1-reaper.json} (100%) delete mode 100644 test/fixtures/non-pid1/Dockerfile create mode 100644 test/fixtures/oop-init/Dockerfile diff --git a/test/.gitignore b/test/.gitignore deleted file mode 100644 index 9414382..0000000 --- a/test/.gitignore +++ /dev/null @@ -1 +0,0 @@ -Dockerfile diff --git a/test/Dockerfile b/test/Dockerfile index affc5e9..7790785 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -9,5 +9,5 @@ COPY fixtures/config /reaper/config # Use entrypoint, so we can pass command line parameters for running # the different tests. -ENTRYPOINT ["/reaper/testpid1"] +ENTRYPOINT ["/reaper/bin/init.sh"] CMD [""] diff --git a/test/Makefile b/test/Makefile index 6659e56..2439ca1 100644 --- a/test/Makefile +++ b/test/Makefile @@ -8,7 +8,7 @@ OOP_INIT_TEST_IMAGE = "reaper/oop-init-config-test" all: build -build: lint vet tests +build: deps lint vet image clean: @echo " - Removing testpid1 binary ..." @@ -27,7 +27,7 @@ clean: $(RM) /tmp/reaper-tests/*.log test: tests -tests: image test-image test-debug-on test-notify test-non-pid1 test-oop-init +tests: build test-image test-debug-on test-notify test-non-pid1 test-oop-init @echo " - All tests passed." @@ -35,17 +35,31 @@ tests: image test-image test-debug-on test-notify test-non-pid1 test-oop-init # Lint and vet targets. # lint: - @echo " - Linting sources ..." + @echo " - Linting shell sources ..." + @(shellcheck bin/*.sh build/*.sh *.sh || :) + @echo " - shellcheck passed." + + @echo " - Linting sources ..." gofmt -d -s testpid1.go fixtures/oop-init/testpid1.go - @echo " - Linter checks passed." + @echo " - Linter checks passed." vet: - @echo " - Vetting go sources ..." + @echo " - Vetting go sources ..." go vet testpid1.go go vet fixtures/oop-init/testpid1.go - @echo " - go vet checks passed." + @echo " - go vet checks passed." +# +# Update dependencies (modules). +# +deps: + @echo " - Update dependencies ..." + go mod tidy + + @echo " - Download go modules ..." + go mod download # -x + # # Build a single image to test the various reaper fixtures configs with. # @@ -82,14 +96,17 @@ test-status-close: @echo " - Running reaper image status close test ..." ./runtests.sh $(TEST_IMAGE) /reaper/config/status-close-reaper.json +test-non-pid1: test-child-sub-reaper + @echo " - Running reaper image non pid1 test ..." + ./runtests.sh $(TEST_IMAGE) /reaper/config/non-pid1-reaper.json + +test-child-sub-reaper: + @echo " - Running reaper image non pid1 child-sub-reaper test ..." + ./runtests.sh $(TEST_IMAGE) /reaper/config/child-sub-reaper.json # # Custom image tests. # -test-non-pid1: - (build/image.sh $(NON_PID1_CONFIG_TEST_IMAGE) "fixtures/non-pid1"; \ - ./runtests.sh $(NON_PID1_CONFIG_TEST_IMAGE) ) - test-oop-init: (build/image.sh $(OOP_INIT_TEST_IMAGE) "fixtures/oop-init"; \ ./runtests.sh $(OOP_INIT_TEST_IMAGE) ) diff --git a/test/bin/init.sh b/test/bin/init.sh new file mode 100755 index 0000000..8d840cc --- /dev/null +++ b/test/bin/init.sh @@ -0,0 +1,43 @@ +#!/bin/bash + + +# +# Check options to figure out if we need to run the reaper as a child +# process (!pid-1) or as the first process (as pid 1 replacing this script). +# +function _start_reaper() { + local cfg=${1:-""} + local forked=0 + + if [ -f "${cfg}" ]; then + echo " - Checking if we need to run reaper as a child process ..." + if grep -Ee '"DisablePid1Check":\s*true' "${cfg}" ; then + forked=1 + fi + fi + + if [ "${forked}" -eq 1 ]; then + echo " - Running reaper as child process ..." + + # Run testpid1 with a new parent and process group id/session id. + bash < /dev/null & +echo " - Starting worker ..." +nohup bash -c "$SCRIPT_DIR/worker.sh $*" < /dev/null &> /dev/null & pid=$! -echo " - Started background worker - pid=$pid" +echo " - Started background worker - pid=${pid}" set +m diff --git a/test/bin/script.sh b/test/bin/script.sh index a9094df..3ab209e 100755 --- a/test/bin/script.sh +++ b/test/bin/script.sh @@ -1,5 +1,6 @@ #!/bin/bash + # # Usage: $0 # where: = Number of threads - default 5. @@ -18,10 +19,10 @@ SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") NTIMES=${1:-"5"} shift -echo " - $0 started with $NTIMES parallel threads ..." +echo " - pid $$: $0 started with $NTIMES parallel threads ..." -for i in $(seq $NTIMES); do - nohup bash -c "$SCRIPT_DIR/worker.sh $@" < /dev/null &> /dev/null & +for idx in $(seq "${NTIMES}"); do + nohup bash -c "$SCRIPT_DIR/worker.sh $*" < /dev/null &> /dev/null & pid=$! - echo " - Started background worker - pid=$pid" + echo " - Started background worker #${idx} - pid=${pid}" done diff --git a/test/bin/worker.sh b/test/bin/worker.sh index f20b8a5..1c241d2 100755 --- a/test/bin/worker.sh +++ b/test/bin/worker.sh @@ -11,10 +11,7 @@ # Constants. readonly DEFAULT_WORKERS=10 readonly DEFAULT_DELAY_SECS=5 - -# Change log file if you want see the script output. -# LOGFILE=/tmp/worker.log -LOGFILE=/dev/null +readonly LOGFILE=/tmp/worker.log function start_sleeper() { diff --git a/test/fixtures/config/child-sub-reaper.json b/test/fixtures/config/child-sub-reaper.json new file mode 100644 index 0000000..b31d288 --- /dev/null +++ b/test/fixtures/config/child-sub-reaper.json @@ -0,0 +1,8 @@ +{ + "Pid": 0, + "DisablePid1Check": true, + "EnableChildSubreaper": true, + "Status": true, + "Debug": true, + "Options": 0 +} diff --git a/test/fixtures/non-pid1/reaper.json b/test/fixtures/config/non-pid1-reaper.json similarity index 100% rename from test/fixtures/non-pid1/reaper.json rename to test/fixtures/config/non-pid1-reaper.json diff --git a/test/fixtures/non-pid1/Dockerfile b/test/fixtures/non-pid1/Dockerfile deleted file mode 100644 index d30b6b8..0000000 --- a/test/fixtures/non-pid1/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -# FROM scratch -FROM fedora - -MAINTAINER smitram@gmail.com - -COPY ./bin /reaper/bin -COPY testpid1 /reaper/testpid1 -COPY reaper.json /reaper/config/reaper.json - -CMD bash -c "/reaper/testpid1" diff --git a/test/fixtures/oop-init/Dockerfile b/test/fixtures/oop-init/Dockerfile new file mode 100644 index 0000000..d3c18d0 --- /dev/null +++ b/test/fixtures/oop-init/Dockerfile @@ -0,0 +1,9 @@ +# FROM scratch +FROM fedora + +MAINTAINER smitram@gmail.com + +ADD ./bin /reaper/bin +ADD testpid1 /reaper/testpid1 + +CMD ["/reaper/testpid1"] diff --git a/test/runtests.sh b/test/runtests.sh index fb5589c..6fa8cf6 100755 --- a/test/runtests.sh +++ b/test/runtests.sh @@ -12,22 +12,23 @@ logfile="/tmp/reaper-tests/test.log" # # Return list of sleeper processes. # -function get_sleepers() { +function _get_sleepers() { #shellcheck disable=SC2009 - ps -ef -p "$1" | grep sleep | grep -v grep + ps --forest -o pid,ppid,time,cmd -g "${pid1}" | \ + grep sleep | grep -v grep -} # End of function get_sleepers. +} # End of function _get_sleepers. # # Check for orphaned processes. # -function check_orphans() { +function _check_orphans() { local pid1=$1 sleep "${MAX_SLEEP_TIME}" local orphans="" - orphans=$(get_sleepers "${pid1}") + orphans=$(_get_sleepers "${pid1}") if [ -n "${orphans}" ]; then echo "" @@ -41,30 +42,82 @@ function check_orphans() { return 0 -} # End of function check_orphans. +} # End of function _check_orphans. # # Terminate docker container. # -function terminate_container() { +function _terminate_container() { local logdir="" logdir=$(dirname "${logfile}") mkdir -p "${logdir}" docker logs "$1" > "${logfile}" + # append worker logs to the logfile. + (echo ""; echo ""; \ + echo "----------------------------------------------"; \ + echo " - Worker logs:"; \ + docker exec "$1" cat /tmp/worker.log; \ + echo "----------------------------------------------"; \ + ) >> "${logfile}" + echo " - Container logs saved to ${logfile}" echo " - Terminated container $(docker rm -f "$1")" -} # End of function terminate_container. +} # End of function _terminate_container. + + +# +# If we have status notifications turned on (and are not randomly closing +# the status channel), then check whether we got the various different +# exit codes from the reaped descendants. +# +function _check_status_exit_codes() { + local name=${1:-""} + + if [ "z${name}" != "zstatus-reaper" ] && \ + [ "z${name}" != "zchild-sub-reaper" ]; then + # Nothing to do. Test doesn't have any status notifications, so + # the log won't contain any reaped descendant exit codes. + return 0 + fi + + # Check that we have expected status lines. + for code in 1 2 7 13 21 29 30 31 64 65 66 69 70 71 74 76 77 78 127; do + local pid="" + for pid in $(grep -Ee "exitcode=${code}\$" "${logfile}" | \ + awk -F '=' '{print $2}' | \ + awk -F ',' '{print $1}' | tr '\r\n' ' '); do + #echo " - Descendant pid ${pid} exited with code ${code}" + + # On a slower vm (circa 2014), might not always get the status + # reported - it could fill up the channel notifications, so ... + if grep "status of pid ${pid}" "${logfile}" > /dev/null; then + local reported="" + reported=$(grep "status of pid ${pid}" "${logfile}" | \ + sed 's/.*status of pid.*exit code //g' | \ + tr -d '\r') + if [ "${reported}" -ne "${code}" ]; then + echo "ERROR: Child pid ${pid} exited with code ${code}," + echo " but reported code is ${reported}" + exit 65 + fi + fi + done + done + + return 0 + +} # End of function _check_status_exit_codes. # # Run reaper tests. # -function run_tests() { +function _run_tests() { local image=${1:-"${IMAGE}"} shift @@ -91,15 +144,17 @@ function run_tests() { local pid1="" pid1=$(docker inspect --format '{{.State.Pid}}' "${elcid}") + sleep 1.42 + echo " - Docker container pid=${pid1}" - echo " - PID ${pid1} has $(get_sleepers "${pid1}" | wc -l) sleepers." + echo " - PID ${pid1} has $(_get_sleepers "${pid1}" | wc -l) sleepers." echo " - Sleeping for ${MAX_SLEEP_TIME} seconds ..." sleep "${MAX_SLEEP_TIME}" echo " - Checking for orphans attached to pid1=${pid1} ..." - if ! check_orphans "${pid1}"; then + if ! _check_orphans "${pid1}"; then # Got an error, cleanup and exit with error code. - terminate_container "${elcid}" + _terminate_container "${elcid}" echo "" echo "FAIL: All tests failed - (1/1)" exit 65 @@ -112,30 +167,41 @@ function run_tests() { docker kill -s USR1 "${elcid}" sleep 1 - echo " - PID ${pid1} has $(get_sleepers "${pid1}" | wc -l) sleepers." + echo " - PID ${pid1} has $(_get_sleepers "${pid1}" | wc -l) sleepers." echo " - Sleeping once again for ${MAX_SLEEP_TIME} seconds ..." sleep "${MAX_SLEEP_TIME}" + echo " - Running processes under ${pid1}:" + pstree "${pid1}" + ps --forest -o pid,ppid,time,cmd -g "${pid1}" || : + echo " - Checking for orphans attached to pid1=${pid1} ..." - if ! check_orphans "${pid1}"; then + if ! _check_orphans "${pid1}"; then # Got an error, cleanup and exit with error code. - terminate_container "${elcid}" + _terminate_container "${elcid}" echo "" echo "FAIL: Some tests failed - (1/2)" exit 65 fi + echo " - Running processes under ${pid1}:" + pstree "${pid1}" + ps --forest -o pid,ppid,time,cmd -g "${pid1}" || : + # Do the cleanup. - terminate_container "${elcid}" + _terminate_container "${elcid}" + + # If we have the status, check the different exit codes. + _check_status_exit_codes "${name}" echo "" echo "OK: All tests passed - (2/2)" -} # End of function run_tests. +} # End of function _run_tests. # # main(): # -run_tests "$@" +_run_tests "$@" diff --git a/test/testpid1.go b/test/testpid1.go index 9c1320e..dc3477b 100644 --- a/test/testpid1.go +++ b/test/testpid1.go @@ -20,12 +20,13 @@ const NAME = "testpid1" // Reaper test options. type TestOptions struct { - Pid int - Options int - DisablePid1Check bool - Debug bool - Status bool - StatusClose bool + Pid int + Options int + DisablePid1Check bool + EnableChildSubreaper bool + Debug bool + Status bool + StatusClose bool } // Test with a process that sleeps for a short time. @@ -165,19 +166,21 @@ func startReaper(options *TestOptions) { if options.Status { flag := options.StatusClose + fmt.Printf("%s: status channel enabled, random close=%v\n", + NAME, flag) - fmt.Printf("%s: testing status channel with %v\n", NAME, flag) // make a buffered channel with max 42 entries. statusChannel = make(chan reaper.Status, 42) go dumpChildExitStatus(statusChannel, flag) } config := reaper.Config{ - Pid: options.Pid, - Options: options.Options, - DisablePid1Check: options.DisablePid1Check, - Debug: options.Debug, - StatusChannel: statusChannel, + Pid: options.Pid, + Options: options.Options, + DisablePid1Check: options.DisablePid1Check, + EnableChildSubreaper: options.EnableChildSubreaper, + Debug: options.Debug, + StatusChannel: statusChannel, } go reaper.Start(config)