From 30f4dd1711934da2876666531874efb9543899e4 Mon Sep 17 00:00:00 2001 From: Moustafa Date: Mon, 19 Jul 2021 20:21:57 -0400 Subject: [PATCH] Use 'env' as a separate key that has the env variables in the input YAML file Add new option to run experiments with perf and get the hotness of each line of code Add Hotness Columns for each build in HTML reports of Source Code Only show Hotness coulmn in the reports if we run with perf --- harness.py | 86 +++++++++++++++++---------- hotness.py | 118 ++++++++++++++++++++++++++++++++++++ opt-viewer/optviewer.py | 129 +++++++++++++++++++++++++++++++--------- 3 files changed, 272 insertions(+), 61 deletions(-) create mode 100644 hotness.py diff --git a/harness.py b/harness.py index c784926..d921db6 100755 --- a/harness.py +++ b/harness.py @@ -17,6 +17,7 @@ from yaml import CLoader import argparse import subprocess +from subprocess import PIPE import os import shutil import time @@ -33,13 +34,16 @@ import optviewer import optdiff -def invoke_optviewer(filelist, output_html_dir, jobs, print_progress): +from hotness import * + + +def invoke_optviewer(filelist, output_html_dir, jobs, print_progress, builds=[]): all_remarks, file_remarks, should_display_hotness = \ optrecord.gather_results( filelist, # filelist 1, # num jobs print_progress) # print progress - + print('generating opt viewers for builds', builds) optviewer.map_remarks(all_remarks) optviewer.generate_report(all_remarks, @@ -50,7 +54,9 @@ def invoke_optviewer(filelist, output_html_dir, jobs, print_progress): should_display_hotness, 100, # max hottest remarks in index 1, # number of jobs - print_progress) # print progress + print_progress, # print progress + builds, ) + def invoke_optdiff(yaml_file_1, yaml_file_2, filter_only, out_yaml): optdiff.generate_diff( @@ -62,11 +68,12 @@ def invoke_optdiff(yaml_file_1, yaml_file_2, filter_only, out_yaml): 100000, # max remarks out_yaml) # output yaml -def run(config, program, reps, dry): +def run(config, program, reps, dry, with_perf): print('Launching program', program, 'with modes', config[program]['build']) - exe = config[program]['run'] + ' ' + config[program]['input'] - os.makedirs( './results', exist_ok=True) - results = { program: {} } + perf_command = 'perf record --freq=100000 -o perf.data' if with_perf else '' + exe = config[program]['env'] + ' ' + perf_command + ' ' + config[program]['run'] + ' ' + config[program]['input'] + os.makedirs('./results', exist_ok=True) + results = {program: {}} try: with open('./results/results-%s.yaml'%(program), 'r') as f: results = yaml.load(f, Loader=CLoader) @@ -97,18 +104,21 @@ def run(config, program, reps, dry): for i in range(start, reps): print('path', bin_dir, 'exe',exe) t1 = time.perf_counter() - p = subprocess.run( exe, capture_output=True, cwd=bin_dir, shell=True ) - out = str(p.stdout.decode('utf-8')) - err = str(p.stderr.decode('utf-8')) + # p = subprocess.run( exe, capture_output=True, cwd=bin_dir, shell=True ) + p = subprocess.run(exe, cwd=bin_dir, shell=True, stdout=PIPE, stderr=PIPE) + out = str(p.stdout.decode('utf-8', errors='ignore')) + err = str(p.stderr.decode('utf-8', errors='ignore')) + # out=str(p.stdout) + # err=str(p.stderr) output = out + err print(output) - #print('Out', p.stdout.decode('utf-8') ) - #print('Err', p.stderr.decode('utf-8') ) - with open('%s/stdout-%d.txt'%(bin_dir, i), 'w') as f: - f.write(p.stdout.decode('utf-8')); - with open('%s/stderr-%d.txt'%(bin_dir, i), 'w') as f: - f.write(p.stderr.decode('utf-8')); - + # print('Out', p.stdout.decode('utf-8') ) + # print('Err', p.stderr.decode('utf-8') ) + with open('%s/stdout-%d.txt' % (bin_dir, i), 'w') as f: + f.write(p.stdout.decode('latin-1', errors='replace')); + with open('%s/stderr-%d.txt' % (bin_dir, i), 'w') as f: + f.write(p.stderr.decode('latin-1', errors='replace')); + output = '' if p.returncode != 0: print('ERROR running', program, 'in', mode) sys.exit(p.returncode) @@ -132,6 +142,15 @@ def run(config, program, reps, dry): with open('./results/results-%s.yaml'%(program), 'w') as f: yaml.dump( results, f ) + # if we run with perf, we generate the report + if with_perf: + hotlines = get_hot_lines_percentage(config[program]['bin'], bin_dir) + reports_dir = './reports/' + program + lines_hotness_path = os.path.join(reports_dir, '{}.lines_hotness.yaml'.format(mode)) + print('WRITING HOTNESS OF SRC CODE LINES TO:', lines_hotness_path) + with open(lines_hotness_path, 'w') as f: + yaml.dump(hotlines, f) + def show_stats(config, program): try: @@ -183,10 +202,6 @@ def merge_stats_reports( program, build_dir, mode ): with open(reports_dir + mode + '.opt.yaml', 'r') as f: data = yaml.load_all(f, Loader=CLoader) - print('==== data') - #for d in data: - # print(d) - input('==== end of data') # merge stats filenames = Path(build_dir).rglob('*.stats') @@ -218,7 +233,6 @@ def compile_and_install(config, program, repo_dir, mode): subprocess.run( config[program]['build'][mode], cwd=build_dir, shell=True ) except Exception as e: print('building %s mode %s failed'%(program, mode), e) - input('key...') sys.exit(1) print('Merge stats and reports...') @@ -232,9 +246,10 @@ def compile_and_install(config, program, repo_dir, mode): shutil.copy( build_dir + '/' + copy, bin_dir) -def generate_diff_reports( report_dir, builds, mode ): +def generate_diff_reports(report_dir, builds, mode, with_perf): out_yaml = report_dir + '%s-%s-%s.opt.yaml'%( builds[0], builds[1], mode ) output_html_dir = report_dir + 'html-%s-%s-%s'%( builds[0], builds[1], mode ) + build_for_hotness = builds if with_perf else [] def generate_diff_yaml(): print('Creating diff remark YAML files...') @@ -260,7 +275,8 @@ def generate_diff_html(): [out_yaml], output_html_dir, 1, - True) + True, + build_for_hotness) print('Done generating compilation report for builds %s|%s mode %s'%( builds[0], builds[1], mode )) except: print('Failed generating compilation report for builds %s|%s mode %s'%( builds[0], builds[1], mode )) @@ -279,17 +295,20 @@ def generate_diff_html(): else: generate_diff_html() -def generate_remark_reports( config, program ): + +def generate_remark_reports(config, program, with_perf): report_dir = './reports/' + program + '/' def generate_html(): print('Creating HTML report output for build %s ...' % ( build ) ) + build_for_hotness = [build] if with_perf else [] try: invoke_optviewer( [in_yaml], output_html_dir, 1, - True) + True, + build_for_hotness) print('Done generating compilation reports!') except: print('Failed generating compilation reports (expects build was '\ @@ -309,10 +328,11 @@ def generate_html(): # Create repors for 2-combinations of build options. combos = itertools.combinations( config[program]['build'], 2 ) for builds in combos: - generate_diff_reports( report_dir, builds, 'all' ) - generate_diff_reports( report_dir, builds, 'analysis' ) - generate_diff_reports( report_dir, builds, 'missed' ) - generate_diff_reports( report_dir, builds, 'passed' ) + generate_diff_reports( report_dir, builds, 'all', with_perf ) + generate_diff_reports( report_dir, builds, 'analysis', with_perf ) + generate_diff_reports( report_dir, builds, 'missed', with_perf ) + generate_diff_reports( report_dir, builds, 'passed', with_perf ) + def fetch(config, program): # directories @@ -345,6 +365,7 @@ def main(): parser.add_argument('-s', '--stats', dest='stats', action='store_true', help='show run statistics') parser.add_argument('-d', '--dry-run', dest='dry', action='store_true', help='enable dry run') parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='verbose printing') + parser.add_argument('-pc', '--perf', dest='perf', action='store_true', help='use perf') args = parser.parse_args() with open(args.input, 'r') as f: @@ -358,6 +379,7 @@ def main(): print('args.build', args.build) print('args.run', args.run) print('args.generate', args.generate) + print('args.perf', args.perf) programs = [] if args.programs: @@ -374,9 +396,9 @@ def main(): if args.build: build( config, p ) if args.run: - run( config, p, args.run, args.dry ) + run( config, p, args.run, args.dry, args.perf ) if args.generate: - generate_remark_reports( config, p ) + generate_remark_reports( config, p, args.perf ) if args.stats: show_stats( config, p) diff --git a/hotness.py b/hotness.py new file mode 100644 index 0000000..a278982 --- /dev/null +++ b/hotness.py @@ -0,0 +1,118 @@ +import subprocess +from subprocess import PIPE +import os + + +# reads a perf report that is a product of the following command: +# perf report -b --sort symbol +# this report contains the symbols sorted by their % of usage +# it outputs a dict that has keys as symbols and vals as % of usage +def get_hot_symbols(report_path): + symbols_usage = {} + with open(report_path) as report_file: + for line in report_file: + if line[0] == '#': # skip the header + continue + + if line.strip() == '': # skip empty lines + continue + # print(line) + words = line.strip().split() + percentage = words[0] + symbol = ' '.join(words[3:]) + + percentage = float(percentage[:-1]) # remove % and convert to float + symbols_usage[symbol] = percentage + + return symbols_usage + + +# read perf annotate -P -l symbol and get hot lines in src file +# hot lines are lines that have more than 0.5%(0.005) of execution time of the function +# the function outputs a dict with key "srcfile:line" and value of percentage of time +def get_hotness_from_anno_file(anno_path, hotlines={}, symbol_percentage=100): + skip_keywords = ['Sorted summary for file', '----------------------------'] + with open(anno_path) as anno_file: + for line in anno_file: + if line[0] == '#': # skip the header + continue + + if line.strip() == '': # skip empty lines + continue + + if 'Percent | Source code & Disassembly of' in line: # we only capture src code and terminate before disassembly code + break + + # skip predefined lines + skip_line = False + for skip in skip_keywords: + if skip in line: + skip_line = True + # we cant use continue above because it will escape the inner loop + if skip_line: + continue + + # print(line) + words = line.strip().split() + percentage = float(words[0]) + srccode = ' '.join(words[1:]) + line_hotness = round(percentage * symbol_percentage / 100, 3) + if srccode in hotlines: + hotlines[srccode] += line_hotness + else: + hotlines[srccode] = line_hotness + + return hotlines + + +# @TODO add cwd as a , ALSO ADD relative and absolute percentages +# return the hotlines in the secfile of a symbol. Return only lines with usage time 0.5% or more +def get_symbol_hotness_in_srcfiles(symbol, symbol_percentage, hotlines={}, cwd=''): + # create annotation file of the symbol + annotation_file_name = "perf-annotate.tmp" + exe = "perf annotate {} -P -l > {}".format(symbol, annotation_file_name) + print("executing command: {}".format(exe)) + p = subprocess.run(exe, cwd=cwd, shell=True, stdout=PIPE, stderr=PIPE) + out = str(p.stdout.decode('utf-8', errors='ignore')) + err = str(p.stderr.decode('utf-8', errors='ignore')) + print(out, '\n\n', err) + annotation_file_name = os.path.join(cwd, annotation_file_name) + hotlines = get_hotness_from_anno_file(annotation_file_name, hotlines=hotlines, symbol_percentage=symbol_percentage) + return hotlines + + +# generate report from perf data and return the hot symbols with their percentages +def get_hot_symbols_from_perf_data(binfile, perf_data_file='perf.data', cwd=''): + report_file_name = "perf-report.tmp" + exe = 'perf report --no-child -d {} -i {} --percentage "relative" > {}'.format(binfile, perf_data_file, + report_file_name) + print("executing command: {}".format(exe)) + p = subprocess.run(exe, cwd=cwd, shell=True, stdout=PIPE, stderr=PIPE) + out = str(p.stdout.decode('utf-8', errors='ignore')) + err = str(p.stderr.decode('utf-8', errors='ignore')) + print(out, '\n\n', err) + report_file_name = os.path.join(cwd, report_file_name) + hot_symbols = get_hot_symbols(report_file_name) + return hot_symbols + + +def get_hot_lines_percentage(binfile, cwd): + symbols = get_hot_symbols_from_perf_data(binfile, cwd=cwd) + print(symbols) + print('\n\n\n\n\n\n\n') + hotlines = {} + for symbol in symbols: + # hotlines=get_hotness_from_anno_file('trial') + # skip symbols that are not in the main app + if '@' in symbol: + continue + symbol_percentage = symbols[symbol] + hotlines = get_symbol_hotness_in_srcfiles(symbol, symbol_percentage, hotlines=hotlines, cwd=cwd) + + return hotlines + + +if __name__ == "__main__": + '''hotlines=get_hot_lines_percentage('lulesh2.0') + for key in hotlines: + print("FILE PATH:{}\tPERCENTAGE:{}%".format(key,round(hotlines[key],3)))''' diff --git a/opt-viewer/optviewer.py b/opt-viewer/optviewer.py index 0e052a6..0953d79 100755 --- a/opt-viewer/optviewer.py +++ b/opt-viewer/optviewer.py @@ -26,6 +26,8 @@ import optpmap import optrecord +import yaml +from yaml import CLoader desc = '''Generate HTML output to visualize optimization records from the YAML files generated with -fsave-optimization-record and -fdiagnostics-show-hotness. @@ -48,8 +50,23 @@ def suppress(remark): return remark.getArgDict()['Callee'][0].startswith(('\"Swift.', '\"specialized Swift.')) return False + +def get_hotness_lines(output_dir, builds): + perf_hotness = {} + for build in builds: + perf_hotness_path = os.path.join(output_dir, '..', "{}.lines_hotness.yaml".format(build)) + f = open(perf_hotness_path) + try: + hotness_dict = yaml.load(f, Loader=CLoader) + except Exception as e: + print(e) + perf_hotness[build] = hotness_dict + f.close() + return perf_hotness + + class SourceFileRenderer: - def __init__(self, source_dir, output_dir, filename, no_highlight): + def __init__(self, source_dir, output_dir, filename, no_highlight, builds=[]): self.filename = filename existing_filename = None #print('filename', filename) #ggout @@ -75,6 +92,9 @@ def __init__(self, source_dir, output_dir, filename, no_highlight): self.html_formatter = HtmlFormatter(encoding='utf-8') self.cpp_lexer = CppLexer(stripnl=False) + self.builds = builds + # We assume that we comparison is between each pair of builds + self.perf_hotness = get_hotness_lines(output_dir, builds) def render_source_lines(self, stream, line_remarks): file_text = stream.read() @@ -103,13 +123,19 @@ def render_source_lines(self, stream, line_remarks): html_highlighted = html_highlighted.replace('', '') for (linenum, html_line) in enumerate(html_highlighted.split('\n'), start=1): - print(u''' + html_src_line = u''' {linenum} - - +'''.format(**locals()) + # add place holder for every hotness + for _ in range(len(self.builds)): + html_src_line += u''' +''' + html_src_line += u''' 
{html_line}
-'''.format(**locals()), file=self.stream) +'''.format(**locals()) + print(html_src_line, file=self.stream) + for remark in line_remarks.get(linenum, []): if not suppress(remark): @@ -128,20 +154,28 @@ def render_inline_remarks(self, r, line): indent = line[:max(r.Column, 1) - 1] indent = re.sub('\S', ' ', indent) - print(u''' + entery = u''' - -{r.RelativeHotness} +''' + for build in self.perf_hotness: + file_name, line_num, column = r.DebugLocString.split(':') + file_and_line = file_name + ':' + line_num + entery_hotness = 0 if file_and_line not in self.perf_hotness[build] else self.perf_hotness[build][ + file_and_line] + entery_hotness = "{:.3f}%".format(entery_hotness) + entery += u''' +{entery_hotness}'''.format(**locals()) + entery += u''' {r.PassWithDiffPrefix} 
{indent}
{r.message}  {inlining_context} -'''.format(**locals()), file=self.stream) +'''.format(**locals()) + print(entery, file=self.stream) def render(self, line_remarks): if not self.source_stream: return - - print(''' + header1 = u''' {} @@ -153,14 +187,18 @@ def render(self, line_remarks): - -'''.format(os.path.basename(self.filename)), file=self.stream) +''' + print(header1, file=self.stream) self.render_source_lines(self.source_stream, line_remarks) print(''' @@ -171,23 +209,49 @@ def render(self, line_remarks): class IndexRenderer: - def __init__(self, output_dir, should_display_hotness, max_hottest_remarks_on_index): + def __init__(self, output_dir, should_display_hotness, max_hottest_remarks_on_index, builds=[]): self.stream = codecs.open(os.path.join(output_dir, 'index.html'), 'w', encoding='utf-8') self.should_display_hotness = should_display_hotness self.max_hottest_remarks_on_index = max_hottest_remarks_on_index + self.builds = builds + # We assume that we comparison is between each pair of builds + self.perf_hotness = get_hotness_lines(output_dir, builds) + def render_entry(self, r, odd): escaped_name = html.escape(r.DemangledFunctionName) - print(u''' + # we assume that omp has +ve sign before + # file_name,line_num,column=r.DebugLocString.split(':') + # print(file_name,line_num,column) + # file_and_line=file_name+':'+line_num + # if perf_render_mode is None: + # perf_hotness='' + # elif perf_render_mode == "omp" or (perf_render_mode == "all" and r.PassWithDiffPrefix[0] == "+"): + # perf_hotness = self.perf_hotness_omp + entery = u''' - - +'''.format(**locals()) + + # add perf hotness for each build + for build in self.perf_hotness: + file_name, line_num, column = r.DebugLocString.split(':') + file_and_line = file_name + ':' + line_num + entery_hotness = 0 if file_and_line not in self.perf_hotness[build] else self.perf_hotness[build][ + file_and_line] + entery_hotness = "{:.3f}%".format(entery_hotness) + entery += u''' +'''.format(**locals()) + + # continue entery + entery += u''' -'''.format(**locals()), file=self.stream) +'''.format(**locals()) + # print('entery in render entery:',entery) + print(entery, file=self.stream) def render(self, all_remarks): - print(''' + header = u''' @@ -197,11 +261,17 @@ def render(self, all_remarks):
Line -Hotness +Line'''.format(os.path.basename(self.filename)) + for build in self.perf_hotness: + header1 += u''' +{} Perf Hotness'''.format(build) + header1 += u''' Optimization Source Inline Context
{r.DebugLocString}{r.RelativeHotness}{r.DebugLocString}{entery_hotness}{escaped_name} {r.PassWithDiffPrefix}
- - - +''' + # print('header is now: ',header) + for build in self.perf_hotness: + header += u''' +'''.format(build) + # print('header is now: ',header) + header += u''' -''', file=self.stream) +''' + # print('header in index rendered:',header) + print(header, file=self.stream) max_entries = None if self.should_display_hotness: @@ -216,11 +286,11 @@ def render(self, all_remarks): ''', file=self.stream) -def _render_file(source_dir, output_dir, ctx, no_highlight, entry): +def _render_file(source_dir, output_dir, ctx, no_highlight, builds, entry): global context context = ctx filename, remarks = entry - SourceFileRenderer(source_dir, output_dir, filename, no_highlight).render(remarks) + SourceFileRenderer(source_dir, output_dir, filename, no_highlight, builds).render(remarks) def map_remarks(all_remarks): @@ -246,7 +316,8 @@ def generate_report(all_remarks, should_display_hotness, max_hottest_remarks_on_index, num_jobs, - should_print_progress): + should_print_progress, + builds=[]): try: os.makedirs(output_dir) except OSError as e: @@ -261,12 +332,12 @@ def generate_report(all_remarks, sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function), reverse=True) else: sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function)) - IndexRenderer(output_dir, should_display_hotness, max_hottest_remarks_on_index).render(sorted_remarks) + IndexRenderer(output_dir, should_display_hotness, max_hottest_remarks_on_index, builds).render(sorted_remarks) shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)), "style.css"), output_dir) - _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context, no_highlight) + _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context, no_highlight, builds) if should_print_progress: print('Rendering HTML files...') optpmap.pmap(_render_file_bound,
Source LocationHotnessFunctionSource Location{} perf HotnessFunction Pass