-
Notifications
You must be signed in to change notification settings - Fork 0
/
test_report.py
245 lines (200 loc) · 8.94 KB
/
test_report.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
"""
this module generates markdown format test report for throughput test.
The test report contains 2 parts:
1. throughput with different configs
2. throughput with RSSI
"""
import os
class ThroughputForConfigsReport(object):
THROUGHPUT_TYPES = ["tcp_tx", "tcp_rx", "udp_tx", "udp_rx"]
REPORT_FILE_NAME = "ThroughputForConfigs.md"
def __init__(self, output_path, ap_ssid, throughput_results, sdkconfig_files):
"""
:param ap_ssid: the ap we expected to use
:param throughput_results: config with the following type::
{
"config_name": {
"tcp_tx": result,
"tcp_rx": result,
"udp_tx": result,
"udp_rx": result,
},
"config_name2": {},
}
"""
self.output_path = output_path
self.ap_ssid = ap_ssid
self.results = throughput_results
self.sdkconfigs = dict()
for config_name in sdkconfig_files:
self.sdkconfigs[config_name] = self._parse_config_file(sdkconfig_files[config_name])
if not os.path.exists(output_path):
os.makedirs(output_path)
self.sort_order = self.sdkconfigs.keys()
self.sort_order.sort()
@staticmethod
def _parse_config_file(config_file_path):
sdkconfig = {}
with open(config_file_path, "r") as f:
for line in f:
if not line.isspace():
if line[0] == "#":
continue
name, value = line.split("=")
value = value.strip("\r\n")
sdkconfig[name] = value if value else "n"
return sdkconfig
def _generate_the_difference_between_configs(self):
"""
generate markdown list for different configs::
default: esp-idf default
low:
* `config name 1`: old value -> new value
* `config name 2`: old value -> new value
* ...
...
"""
data = "## Config Definition:\r\n\r\n"
def find_difference(base, new):
_difference = {}
all_configs = set(base.keys())
all_configs.update(set(new.keys()))
for _config in all_configs:
try:
_base_value = base[_config]
except KeyError:
_base_value = "null"
try:
_new_value = new[_config]
except KeyError:
_new_value = "null"
if _base_value != _new_value:
_difference[_config] = "{} -> {}".format(_base_value, _new_value)
return _difference
for i, _config_name in enumerate(self.sort_order):
current_config = self.sdkconfigs[_config_name]
if i > 0:
previous_config_name = self.sort_order[i - 1]
previous_config = self.sdkconfigs[previous_config_name]
else:
previous_config = previous_config_name = None
if previous_config:
# log the difference
difference = find_difference(previous_config, current_config)
data += "* {} (compared to {}):\r\n".format(_config_name, previous_config_name)
for diff_name in difference:
data += " * `{}`: {}\r\n".format(diff_name, difference[diff_name])
return data
def _generate_report_for_one_type(self, throughput_type):
"""
generate markdown table with the following format::
| config name | throughput (Mbps) | free heap size (bytes) |
|-------------|-------------------|------------------------|
| default | 32.11 | 147500 |
| low | 32.11 | 147000 |
| medium | 33.22 | 120000 |
| high | 43.11 | 100000 |
| max | 45.22 | 79000 |
"""
empty = True
ret = "\r\n### {} {}\r\n\r\n".format(*throughput_type.split("_"))
ret += "| config name | throughput (Mbps) | free heap size (bytes) |\r\n"
ret += "|-------------|-------------------|------------------------|\r\n"
for config in self.sort_order:
try:
result = self.results[config][throughput_type]
throughput = "{:.02f}".format(max(result.throughput_by_att[self.ap_ssid].values()))
heap_size = str(result.heap_size)
# although markdown table will do alignment
# do align here for better text editor presentation
ret += "| {:<12}| {:<18}| {:<23}|\r\n".format(config, throughput, heap_size)
empty = False
except KeyError:
pass
return ret if not empty else ""
def generate_report(self):
data = "# Throughput for different configs\r\n"
data += "\r\nAP: {}\r\n".format(self.ap_ssid)
for throughput_type in self.THROUGHPUT_TYPES:
data += self._generate_report_for_one_type(throughput_type)
data += "\r\n------\r\n"
data += self._generate_the_difference_between_configs()
with open(os.path.join(self.output_path, self.REPORT_FILE_NAME), "w") as f:
f.write(data)
class ThroughputVsRssiReport(object):
REPORT_FILE_NAME = "ThroughputVsRssi.md"
def __init__(self, output_path, throughput_results):
"""
:param throughput_results: config with the following type::
{
"tcp_tx": result,
"tcp_rx": result,
"udp_tx": result,
"udp_rx": result,
}
"""
self.output_path = output_path
self.raw_data_path = os.path.join(output_path, "raw_data")
self.results = throughput_results
self.throughput_types = self.results.keys()
self.throughput_types.sort()
if not os.path.exists(self.raw_data_path):
os.makedirs(self.raw_data_path)
def _generate_summary(self):
"""
generate summary with the following format::
| item | curve analysis | max throughput (Mbps) |
|---------|----------------|-----------------------|
| tcp tx | Success | 32.11 |
| tcp rx | Success | 32.11 |
| udp tx | Success | 45.22 |
| udp rx | Failed | 55.44 |
"""
ret = "\r\n### Summary\r\n\r\n"
ret += "| item | curve analysis | max throughput (Mbps) |\r\n"
ret += "|---------|----------------|-----------------------|\r\n"
for _type in self.throughput_types:
result = self.results[_type]
max_throughput = 0.0
curve_analysis = "Failed" if result.error_list else "Success"
for ap_ssid in result.throughput_by_att:
_max_for_ap = max(result.throughput_by_rssi[ap_ssid].values())
if _max_for_ap > max_throughput:
max_throughput = _max_for_ap
max_throughput = "{:.02f}".format(max_throughput)
ret += "| {:<8}| {:<15}| {:<22}|\r\n".format("{}_{}".format(result.proto, result.direction),
curve_analysis, max_throughput)
return ret
def _generate_report_for_one_type(self, result):
"""
generate markdown table with the following format::
### tcp rx
Errors:
* detected error 1
* ...
AP: ap_ssid
AP: ap_ssid
"""
result.post_analysis()
ret = "\r\n### {} {}\r\n".format(result.proto, result.direction)
if result.error_list:
ret += "\r\nErrors:\r\n\r\n"
for error in result.error_list:
ret += "* " + error + "\r\n"
for ap_ssid in result.throughput_by_rssi:
ret += "\r\nAP: {}\r\n".format(ap_ssid)
# draw figure
file_name = result.draw_throughput_figure(self.raw_data_path, ap_ssid, "rssi")
result.draw_throughput_figure(self.raw_data_path, ap_ssid, "att")
result.draw_rssi_vs_att_figure(self.raw_data_path, ap_ssid)
ret += "\r\n\r\n".format(os.path.join("raw_data", file_name))
return ret
def generate_report(self):
data = "# Throughput Vs RSSI\r\n"
data += self._generate_summary()
for _type in self.throughput_types:
data += self._generate_report_for_one_type(self.results[_type])
with open(os.path.join(self.output_path, self.REPORT_FILE_NAME), "w") as f:
f.write(data)