Makefile.in

# Makefile for mercury/test
#
#
# USAGE:
#
#   "make IFNAME=<ifname>" to perform all tests
#   "make comp" to compare test cases
#   "make clean" to remove test files
#
# HOW IT WORKS:
#
#    if there is a file ./tests/foo.pcap, then the files ./foo.json
#    and ./foo.fp will be created by "make comp", and then ./foo.fp
#    will be compared to ./tests/foo.fp.  If they are not identical,
#    then an error will be reported

# definitions for colorized output
COLOR_RED    = "\033[0;31m"
COLOR_GREEN  = "\033[0;32m"
COLOR_YELLOW = "\033[0;33m"
COLOR_OFF    = "\033[0m"

include ../Makefile_helper.mk

MERCURY = ../src/mercury
export LD_LIBRARY_PATH =$(shell pwd)/../src/libmerc

have_tcpreplay = @TCPREPLAY@
have_jq = @JQ@
have_valgrind = @VALGRIND@
have_py3    = @PYTHON3@
python      = @PY@
have_jsonschema = @HAVE_JSONSCHEMA@
have_afl        = @HAVE_AFL@
have_clang      = @CLANGPP@

BATCH_GCD = ../src/batch_gcd

# check dependancies to see if any tests need to be omitted
#
omitted_test = no

ifeq ($(have_py3),yes)
ifeq ($(have_jsonschema),yes)
do_analysis = yes
do_cert_check = yes
else
omitted_test = yes
endif
else
omitted_test = yes
endif

ifeq ($(have_tcpreplay),yes)
do_dummy_capture = yes
else
omitted_test = yes
endif

ifeq ($(SUDO_UID),)
DROP_ROOT = -u root
else
DROP_ROOT =
endif

# all test files go in the ./tests subdirectory
#
vpath %.pcap ./data

# for each test, add a filename with the .comp suffix
#
FP_TEST_FILES = $(notdir $(wildcard ./data/*.fp))
JSON_FILES    = $(FP_TEST_FILES:%.fp=%.json)
COMP_FILES    = $(FP_TEST_FILES:%.fp=%.comp)
MCAP_TEST_FILES = $(notdir $(wildcard ./data/*.mcap))
MCAP_COMP_FILES = $(MCAP_TEST_FILES:%.mcap=%.mcap-comp)
JSON_TEST_FILES = $(notdir $(wildcard ./data/*.json))
JSON_COMP_FILES = $(JSON_TEST_FILES:%.json=%.json-comp)
BGCD_TEST_FILES = $(wildcard ./batch_gcd/*.bgcd-in)        # input files
BGCD_OUT_FILES = $(BGCD_TEST_FILES:%.bgcd-in=%.bgcd-out)   # expected output
BGCD_TOUT_FILES = $(BGCD_TEST_FILES:%.bgcd-in=%.bgcd-tout) # test run output
BGCD_COMP_TARG = $(BGCD_TEST_FILES:%.bgcd-in=%.bgcd-comp)  # comp file never exists

.PHONY: all clean
all: clean comp analysis cert-check memcheck json-validity-test stats libmerc_driver # dummy-capture
ifeq ($(omitted_test),no)
	@echo $(COLOR_GREEN) "passed all tests" $(COLOR_OFF)
else
	@echo $(COLOR_GREEN) "passed all tests that could be performed" $(COLOR_OFF)
	@echo $(COLOR_YELLOW) "warning: some tests could not be performed due to missing dependancies" $(COLOR_OFF)
	@echo $(COLOR_YELLOW) "please install the missing packages mentioned above and re-run ./configure" $(COLOR_OFF)
endif

.PHONY: test_libmerc_so
test_libmerc_so: libmerc_driver_tls_only libmerc_driver_multiprotocol # pdu_test

.PHONY: pdu_test
pdu_test:
	./pdu_test.sh

.PHONY: libmerc_driver
libmerc_driver:
	@echo "running unit tests"
	cd ../unit_tests && $(MAKE) run

.PHONY: json-validity-test
json-validity-test:
	@echo "running json validity test"
	./mercury-json-validity-check.sh --dns-json --certs-json --metadata --nonselected-tcp-data --nonselected-udp-data
	@echo $(COLOR_GREEN) "passed json validity test" $(COLOR_OFF)

.PHONY: comp
comp: $(COMP_FILES) $(MCAP_COMP_FILES) $(JSON_COMP_FILES)
	@echo $(COLOR_GREEN) "passed all test/data target tests" $(COLOR_OFF)

# implicit rule to make a JSON file from a PCAP file
#
%.json: %.pcap
	$(MERCURY) -r $< -f $@ --tcp-reassembly

# implicit rule to make a fingerprint file from a JSON file
#
%.fp: %.json
ifneq ($(have_jq),yes)
	@echo $(COLOR_RED) "jq is missing; cannot create .fp file from .json file" $(COLOR_OFF)
	@/bin/false
else
	cat $< | jq .fingerprints.tls | grep -v null | tr -d "\"" > $@
endif

# implicit rule to compare results
#
%.comp: %.fp
	@echo "checking file" $< "against expected output"
	diff $< ./data/$<
	@echo $(COLOR_GREEN) "passed" $(COLOR_OFF)  # this output only happens if diff returns 0

# implicit rule to make an MCAP file from a PCAP file
#
%.mcap: %.pcap
	$(MERCURY) -r $< --tcp-reassembly -w $@

# implicit rule to compare MCAP results
#
%.mcap-comp: %.mcap
	@echo "checking file" $< "against expected output"
	diff $< ./data/$<
	@echo $(COLOR_GREEN) "passed" $(COLOR_OFF)

# implicit rule to compare JSON results
#
%.json-comp: %.json
	@echo "checking file" $< "against expected output"
	diff $< ./data/$<
	@echo $(COLOR_GREEN) "passed" $(COLOR_OFF)

# prevent deletion of intermediate files
#
#.PRECIOUS: %.fp %.mcap %.json



.PHONY: analysis
analysis:
ifeq ($(do_analysis),yes)
	@echo "running analysis test"
	$(MERCURY) -r data/top-https.pcap -f tmp.json -a --resources=../resources/resources.tgz
	$(python) json-test.py tmp.json
	@echo $(COLOR_GREEN) "passed analysis test" $(COLOR_OFF)
	rm -f tmp.json
else
	@echo $(COLOR_YELLOW) "omitting analysis test; python3 or jsonschema unavailable" $(COLOR_OFF)
endif

.PHONY: cert-check
cert-check:
ifeq ($(do_cert_check),yes)
	@echo "running certificate test"
	$(MERCURY) -r data/top-https.pcap -f tmp.json --tcp-reassembly
	$(python) certificate-test.py tmp.json --complete 96 --partial 0
	@echo $(COLOR_GREEN) "passed top-https certificate test" $(COLOR_OFF)
	$(MERCURY) -r data/top_100_fingerprints.pcap -f tmp.json --tcp-reassembly
	$(python) certificate-test.py tmp.json --complete 164 --partial 0
	@echo $(COLOR_GREEN) "passed top_100_fingerprints certificate test" $(COLOR_OFF)
	$(MERCURY) -r data/test_decrypt.pcap -f tmp.json
	$(python) certificate-test.py tmp.json --complete 6 --partial 1
	@echo $(COLOR_GREEN) "passed test_decrypt certificate test" $(COLOR_OFF)
	rm -f tmp.json
#
#   use these parameters when tcp reassembly is in effect
#
#	@echo "running certificate test"
#	$(MERCURY) -r data/top-https.pcap -f tmp.json
#	$(python) certificate-test.py tmp.json --complete 96 --partial 0
#	@echo $(COLOR_GREEN) "passed top-https certificate test" $(COLOR_OFF)
#	$(MERCURY) -r data/top_100_fingerprints.pcap -f tmp.json
#	$(python) certificate-test.py tmp.json --complete 164 --partial 0
#	@echo $(COLOR_GREEN) "passed top_100_fingerprints certificate test" $(COLOR_OFF)
#	$(MERCURY) -r data/test_decrypt.pcap -f tmp.json
#	$(python) certificate-test.py tmp.json --complete 8 --partial 0
#	@echo $(COLOR_GREEN) "passed test_decrypt certificate test" $(COLOR_OFF)
#	rm -f tmp.json
else
	@echo $(COLOR_YELLOW) "omitting certificate test; python3 or jsonschema unavailable" $(COLOR_OFF)
endif

.PHONY: capture
capture:
	@echo "running capture test"
ifeq ($(IFNAME),)
	@echo $(COLOR_RED) "error: IFNAME unspecified (run as 'make capture IFNAME=wlan0', for instance)" $(COLOR_OFF)
	@/bin/false
else
ifeq ($(shell id -u),0)
	$(MERCURY) -c $(IFNAME) $(DROP_ROOT) -f tmp.json & echo $$! > mercury.PID
	capture/https-test-driver.sh
	while kill `cat mercury.PID`; do echo "waiting for mercury capture process to halt"; sleep 1; done
ifeq ($(have_jsonschema),yes)
	$(python) json-test.py tmp.json
endif
	rm mercury.PID tmp.json
	@echo $(COLOR_GREEN) "passed capture test" $(COLOR_OFF)
else
	@echo $(COLOR_RED) "error: capture test must be run as root" $(COLOR_OFF)
	@/bin/false
endif
endif


.PHONY: dummy-capture
dummy-capture:
ifneq ($(shell id -u),0)
	@echo $(COLOR_RED) "error: dummy interface capture test must be run as root" $(COLOR_OFF)
	@/bin/false
endif
ifeq ($(do_dummy_capture),yes)
	@echo "running dummy interface capture test"
	ip link add dummy0 type dummy || true
	ip link set dev dummy0 up || true
ifneq ($(shell ip link show dummy0 > /dev/null 2>&1; echo $$?),0)
	@echo $(COLOR_RED) "error: network interface dummy0 unavailable (create with 'sudo ip link add dummy0 type dummy', for instance)" $(COLOR_OFF)
	@/bin/false
endif
	rm -f tmp.json
	rm -f mercury.PID
	$(MERCURY) -c dummy0 $(DROP_ROOT) -f tmp.json & echo $$! > mercury.PID
	sleep 2
	tcpreplay -t -i dummy0 data/top-https.pcap
	while kill `cat mercury.PID`; do echo "waiting for mercury dummy capture process to halt"; sleep 1; done
	bash -c "diff  <( jq . tmp.json | grep -v event_start) <( jq . data/top-https.json | grep -v event_start )"
ifeq ($(have_jsonschema),yes)
	$(python) json-test.py tmp.json
else
	@echo $(COLOR_YELLOW) "warning: python package jsonschema not available, omitting json-test" $(COLOR_OFF)
endif
	rm mercury.PID tmp.json
	@echo $(COLOR_GREEN) "passed dummy interface capture test" $(COLOR_OFF)
else
	@echo $(COLOR_YELLOW) "omitting dummy-capture test; tcpreplay is unavailable" $(COLOR_OFF)
endif

.PHONY: stats
stats:
	@echo "running stats test"
	$(MERCURY) -r data/top-https.pcap -f tmp.json --metadata -a --resources=../resources/resources.tgz --stats=stats
	bash -c "$(python) ./compare-stats.py -m tmp.json -s stats.json.gz"
	@echo $(COLOR_GREEN) "passed stats test" $(COLOR_OFF)
	rm -f tmp.json stats.json.gz
	@echo "running stats rotate test"
	$(MERCURY) -r data/quic_tls_http.pcap -f tmp.json --metadata -a --resources=../resources/resources.tgz --stats=statsfile --stats-time=1 -p 10
	bash -c "$(python) ./compare-stats.py -m tmp.json -s statsfile -a"
	@echo $(COLOR_GREEN) "passed stats rotate test" $(COLOR_OFF)
	rm -f tmp.json tempstats.json statsfile*

.PHONY: clean
clean:
	rm -rf *.fp *.json *.mcap Makefile~ README.md~ deleteme/* memcheck.tmp tmp.json mercury.PID afl-mercury
	rm -f fuzz/libmerc.a
	find ./fuzz/ -name "*_exec" -exec rm -v {} +
	find ./fuzz/ -name "*.log" -exec rm -v {} +
	find ./fuzz/ -name "*.c" -exec rm -v {} +
	find ./fuzz/ -name "crash*" -exec rm -v {} +
	@echo "cleaned all targets"

.PHONY: distclean
distclean: clean
	rm -f Makefile

# memory check
#
.PHONY: memcheck
memcheck:
ifeq ($(have_valgrind),yes)
	@echo "running memcheck with valgrind"
	valgrind --trace-children=yes --leak-check=full --show-leak-kinds=all $(MERCURY) -r data/top-https.pcap -f tmp.json -a --resources=../resources/resources.tgz 2> memcheck.tmp
#	cat memcheck.tmp | grep "definitely lost: 0 bytes in 0 blocks"
	cat memcheck.tmp | grep "ERROR SUMMARY: 0"
	rm -f memcheck.tmp
	@echo $(COLOR_GREEN) "passed memcheck" $(COLOR_OFF)
else
	@echo $(COLOR_YELLOW) "valgrind unavailable; cannot perform memcheck" $(COLOR_OFF)
	@/bin/false
endif

# consistency test - compares mercury and pmercury output
#
FILE = afl_data/corner-cases.pcap
PMERCURY = ../python/pmercury/pmercury
.PHONY: consistency
consistency:
ifeq ($(wildcard $(PMERCURY)),)
	@echo $(COLOR_YELLOW) "pmercury unavailable; cannot perform consistency test" $(COLOR_OFF)
else
	@echo "running consistency check with pmercury"
	$(PMERCURY) -r $(FILE) -f p.json
	$(MERCURY)  -r $(FILE) -f m.json
	$(python) p-mercury-diff.py -p p.json -m m.json
endif

# fuzz testing with libfuzzer
#
.PHONY: fuzz-test
fuzz-test:
ifeq ($(have_clang),yes)
	@echo "checking libmerc.a for fuzz test"
	cd ../src && $(MAKE) libmerc.a
ifeq ($(openssl_newer),yes)
	cd fuzz && ./generate_fuzz_test.sh -n none -t 200 -r 1000000000 -s true
else
	cd fuzz && ./generate_fuzz_test.sh -n none -t 200 -r 1000000000
endif
else
	@echo $(COLOR_YELLOW) "clang++ unavailable; cannot perform fuzz test" $(COLOR_OFF)
endif

# fuzz testing with american fuzzy lop
#
FUZZ_CMD = --metadata --dns-json --certs-json --analysis -f /dev/null
.PHONY: afl-fuzz-test
afl-fuzz-test: afl-mercury
	afl-fuzz -i afl_data -o afl_findings ./afl-mercury -r @@ $(FUZZ_CMD)

afl-mercury:
ifeq ($(have_afl),yes)
	@echo "building afl-mercury for fuzz test, using afl-g++"
	cd ../src && $(MAKE) clean && $(MAKE) CXX=afl-g++ CC=afl-g++ mercury
	mv ../src/mercury ./afl-mercury
else
	@echo $(COLOR_YELLOW) "afl unavailable; cannot perform fuzz test" $(COLOR_OFF)
endif

# batch GCD tests
#
.PHONY: batch_gcd_test
batch_gcd_test: $(BGCD_COMP_TARG)

%.pem.bgcd-tout: %.pem.bgcd-in
	$(BATCH_GCD) --cert-file $< > $@

%.bgcd-tout: %.bgcd-in
	$(BATCH_GCD) < $< > $@

%.bgcd-comp: %.bgcd-tout
	@echo "checking file" $< "against expected output"
	diff $< $(<:.bgcd-tout=.bgcd-out)
	@echo $(COLOR_GREEN) "passed" $(COLOR_OFF)

# EOF