diff --git a/scripts/process_benchmarks.py b/scripts/process_benchmarks.py index 699bf79f..eaa80579 100644 --- a/scripts/process_benchmarks.py +++ b/scripts/process_benchmarks.py @@ -20,6 +20,7 @@ def show_datafram(df): f for f in os.listdir("benchmark_results") if os.path.isdir(os.path.join("benchmark_results", f))] +benchmarks.sort() benchmark = st.selectbox( "Benchmark", @@ -33,6 +34,7 @@ def show_datafram(df): f for f in os.listdir(benchmark_directory) if os.path.isdir(os.path.join(benchmark_directory, f))] +output_directories.sort() bksv_field = "Best known solution value" @@ -117,7 +119,7 @@ def highlight(s): else ('background-color: pink' if s[fieldname] > s[bksv_field] else 'background-color: yellow')) - if fieldname in result_columns + output_directories + if fieldname in result_columns else '' for fieldname in out_fieldnames] df = df.style.apply(highlight, axis = 1) @@ -138,7 +140,10 @@ def highlight(s): # Get fieldnames of CSV output file. out_fieldnames = reader.fieldnames for output_directory in output_directories: - out_fieldnames.append(output_directory) + out_fieldnames.append(output_directory + " / Solution value") + + result_columns = [fieldname for fieldname in reader.fieldnames + if "Solution value" in fieldname] out_rows = [] @@ -152,12 +157,12 @@ def highlight(s): for instance_class in ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10"] for number_of_items in ["20", "40", "60", "80", "100"] - ] + { + ] + [{ "Path": "Total", - bksv_field: 0} - for output_directory in output_directories: + bksv_field: 0}] + for fieldname in [bksv_field] + result_columns: for row in extra_rows: - row[output_directory] = 0 + row[fieldname] = 0 for row in reader: if not row["Path"]: @@ -165,6 +170,17 @@ def highlight(s): row[bksv_field] = int(row[bksv_field]) + # Fill current row. + for output_directory in output_directories: + json_output_path = os.path.join( + benchmark_directory, + output_directory, + row["Path"] + "_output.json") + json_output_file = open(json_output_path, "r") + json_data = json.load(json_output_file) + row[output_directory + " / Solution value"] = ( + json_data["Output"]["Solution"]["NumberOfBins"]) + # Get extra rows to update. instance_class = int(row["Path"].split('_')[1].split('.')[0]) number_of_items = int(row["Path"].split('_')[2]) @@ -173,50 +189,33 @@ def highlight(s): + int(number_of_items / 20 - 1)), -1] - # Update "Best known solution value" column. + # Update "Best known solution value" column of extra row. for row_id in extra_rows_to_update: extra_rows[row_id][bksv_field] += row[bksv_field] - for output_directory in output_directories: - # Retrieve solution value. - json_output_path = os.path.join( - benchmark_directory, - output_directory, - row["Path"] + "_output.json") - json_output_file = open(json_output_path, "r") - json_data = json.load(json_output_file) - number_of_bins = int(json_data["Output"] - ["Solution"] - ["NumberOfBins"]) - - # Update current row. - row[output_directory] = number_of_bins - - # Update extra rows. - extra_rows[row_id][output_directory] += number_of_bins - extra_rows[-1][output_directory] += number_of_bins - # Update result columns of extra rows. for result_column in result_columns: - waste = int(row[result_column]) - row[result_column] = waste + number_of_bins = int(row[result_column]) + row[result_column] = number_of_bins for row_id in extra_rows_to_update: - extra_rows[row_id][result_column] += waste + extra_rows[row_id][result_column] += number_of_bins # Add current row. out_rows.append(row) # Add extra rows. for row in extra_rows: - out_rows.append(extra_rows[i]) + out_rows.append(row) df = pd.DataFrame.from_records(out_rows, columns=out_fieldnames) def highlight(s): return [('background-color: lightgreen' - if s[fieldname] == int(s[bksv_field]) - else 'background-color: pink') - if fieldname in output_directories + if s[fieldname] == s[bksv_field] + else ('background-color: pink' + if s[fieldname] > s[bksv_field] + else 'background-color: yellow')) + if fieldname in result_columns else '' for fieldname in out_fieldnames] df = df.style.apply(highlight, axis = 1) @@ -234,11 +233,17 @@ def highlight(s): with open(datacsv_path, newline='') as csvfile: reader = csv.DictReader(csvfile) - out_rows = [] + # Get fieldnames of CSV output file. out_fieldnames = reader.fieldnames for output_directory in output_directories: - out_fieldnames.append(output_directory) + out_fieldnames.append(output_directory + " / Solution value") + + result_columns = [fieldname for fieldname in reader.fieldnames + if "Solution value" in fieldname] + out_rows = [] + + # Initialize extra rows. extra_rows = [ { "Path": ("Class_" + instance_class @@ -248,12 +253,12 @@ def highlight(s): for instance_class in ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10"] for number_of_items in ["20", "40", "60", "80", "100"] - ] + { + ] + [{ "Path": "Total", - bksv_field: 0} - for output_directory in output_directories: + bksv_field: 0}] + for fieldname in [bksv_field] + result_columns: for row in extra_rows: - extra_rows[i][output_directory] = 0 + row[fieldname] = 0 for row in reader: if not row["Path"]: @@ -261,49 +266,52 @@ def highlight(s): row[bksv_field] = int(row[bksv_field]) - instance_class = int(row["Path"].split('_')[1].split('.')[0]) - number_of_items = int(row["Path"].split('_')[2]) - row_id = ( - (instance_class - 1) * 5 - + int(number_of_items / 20 - 1)) - - # Update "Best known solution value" column. - extra_rows[row_id][bksv_field] += row[bksv_field] - extra_rows[-1][bksv_field] += row[bksv_field] - + # Fill current row. for output_directory in output_directories: - # Retrieve solution value. json_output_path = os.path.join( benchmark_directory, output_directory, row["Path"] + "_output.json") json_output_file = open(json_output_path, "r") json_data = json.load(json_output_file) - number_of_bins = int(json_data["Output"] - ["Solution"] - ["NumberOfBins"]) + row[output_directory + " / Solution value"] = ( + json_data["Output"]["Solution"]["NumberOfBins"]) - # Update current row. - row[output_directory] = number_of_bins + # Get extra rows to update. + instance_class = int(row["Path"].split('_')[1].split('.')[0]) + number_of_items = int(row["Path"].split('_')[2]) + extra_rows_to_update = [ + ((instance_class - 1) * 5 + + int(number_of_items / 20 - 1)), + -1] - # Update extra rows. - extra_rows[row_id][output_directory] += number_of_bins - extra_rows[-1][output_directory] += number_of_bins + # Update "Best known solution value" column of extra row. + for row_id in extra_rows_to_update: + extra_rows[row_id][bksv_field] += row[bksv_field] + + # Update result columns of extra rows. + for result_column in result_columns: + number_of_bins = int(row[result_column]) + row[result_column] = number_of_bins + for row_id in extra_rows_to_update: + extra_rows[row_id][result_column] += number_of_bins # Add current row. out_rows.append(row) # Add extra rows. for row in extra_rows: - out_rows.append(extra_rows[i]) + out_rows.append(row) df = pd.DataFrame.from_records(out_rows, columns=out_fieldnames) def highlight(s): return [('background-color: lightgreen' - if s[fieldname] == int(s[bksv_field]) - else 'background-color: pink') - if fieldname in output_directories + if s[fieldname] == s[bksv_field] + else ('background-color: pink' + if s[fieldname] > s[bksv_field] + else 'background-color: yellow')) + if fieldname in result_columns else '' for fieldname in out_fieldnames] df = df.style.apply(highlight, axis = 1) @@ -324,11 +332,10 @@ def highlight(s): # Get fieldnames of CSV output file. out_fieldnames = reader.fieldnames for output_directory in output_directories: - out_fieldnames.append(output_directory) + out_fieldnames.append(output_directory + " / Solution value") - result_columns = [ - "long2020 / SILP / Solution value", - "long2020 / HCG2 / Solution value"] + result_columns = [fieldname for fieldname in reader.fieldnames + if "Solution value" in fieldname] out_rows = [] @@ -336,20 +343,16 @@ def highlight(s): row[bksv_field] = int(row[bksv_field]) + # Fill current row. for output_directory in output_directories: - # Retrieve solution value. json_output_path = os.path.join( benchmark_directory, output_directory, row["Path"] + "_output.json") json_output_file = open(json_output_path, "r") json_data = json.load(json_output_file) - number_of_bins = int(json_data["Output"] - ["Solution"] - ["NumberOfBins"]) - - # Update current row. - row[output_directory] = number_of_bins + row[output_directory + " / Solution value"] = ( + json_data["Output"]["Solution"]["NumberOfBins"]) # Update result columns of extra rows. for result_column in result_columns: @@ -467,11 +470,18 @@ def highlight(s): with open(datacsv_path, newline='') as csvfile: reader = csv.DictReader(csvfile) - out_rows = [] + # Get fieldnames of CSV output file. out_fieldnames = reader.fieldnames for output_directory in output_directories: - out_fieldnames.append(output_directory) + out_fieldnames.append(output_directory + " / Solution value") + result_columns = [fieldname for fieldname in reader.fieldnames + if "Solution value" in fieldname] + + # Initialize extra rows. + out_rows = [] + + # Initialize extra rows. extra_rows = [ { "Path": ("Class_" + instance_class @@ -481,16 +491,12 @@ def highlight(s): for instance_class in ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10"] for number_of_items in ["20", "40", "60", "80", "100"] - ] - for output_directory in output_directories: + ] + [{ + "Path": "Total", + bksv_field: 0}] + for fieldname in [bksv_field] + result_columns: for row in extra_rows: - row[output_directory] = 0 - - extra_row = { - "Path": "Total", - bksv_field: 0} - for output_directory in output_directories: - extra_row[output_directory] = 0 + row[fieldname] = 0 for row in reader: if not row["Path"]: @@ -498,14 +504,7 @@ def highlight(s): row[bksv_field] = int(row[bksv_field]) - instance_class = int(row["Path"].split('_')[1].split('.')[0]) - number_of_items = int(row["Path"].split('_')[2]) - row_id = ( - (instance_class - 1) * 5 - + int(number_of_items / 20 - 1)) - extra_row[bksv_field] += int(row[bksv_field]) - extra_rows[row_id][bksv_field] += int(row[bksv_field]) - + # Fill current row. for output_directory in output_directories: json_output_path = os.path.join( benchmark_directory, @@ -513,26 +512,44 @@ def highlight(s): row["Path"] + "_output.json") json_output_file = open(json_output_path, "r") json_data = json.load(json_output_file) - number_of_bins = int(json_data["Output"] - ["Solution"] - ["NumberOfBins"]) - row[output_directory] = number_of_bins - extra_rows[row_id][output_directory] += number_of_bins - extra_row[output_directory] += number_of_bins + row[output_directory + " / Solution value"] = ( + json_data["Output"]["Solution"]["NumberOfBins"]) + # Get extra rows to update. + instance_class = int(row["Path"].split('_')[1].split('.')[0]) + number_of_items = int(row["Path"].split('_')[2]) + extra_rows_to_update = [ + ((instance_class - 1) * 5 + + int(number_of_items / 20 - 1)), + -1] + + # Update "Best known solution value" column of extra row. + for row_id in extra_rows_to_update: + extra_rows[row_id][bksv_field] += row[bksv_field] + + # Update result columns of extra rows. + for result_column in result_columns: + number_of_bins = int(row[result_column]) + row[result_column] = number_of_bins + for row_id in extra_rows_to_update: + extra_rows[row_id][result_column] += number_of_bins + + # Add current row. out_rows.append(row) + # Add extra rows. for row in extra_rows: out_rows.append(row) - out_rows.append(extra_row) df = pd.DataFrame.from_records(out_rows, columns=out_fieldnames) def highlight(s): return [('background-color: lightgreen' - if s[fieldname] == int(s[bksv_field]) - else 'background-color: pink') - if fieldname in output_directories + if s[fieldname] == s[bksv_field] + else ('background-color: pink' + if s[fieldname] > s[bksv_field] + else 'background-color: yellow')) + if fieldname in result_columns else '' for fieldname in out_fieldnames] df = df.style.apply(highlight, axis = 1) @@ -550,11 +567,18 @@ def highlight(s): with open(datacsv_path, newline='') as csvfile: reader = csv.DictReader(csvfile) - out_rows = [] + # Get fieldnames of CSV output file. out_fieldnames = reader.fieldnames for output_directory in output_directories: - out_fieldnames.append(output_directory) + out_fieldnames.append(output_directory + " / Solution value") + + result_columns = [fieldname for fieldname in reader.fieldnames + if "Solution value" in fieldname] + + # Initialize extra rows. + out_rows = [] + # Initialize extra rows. extra_rows = [ { "Path": ("Class_" + instance_class @@ -564,16 +588,12 @@ def highlight(s): for instance_class in ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10"] for number_of_items in ["20", "40", "60", "80", "100"] - ] - for output_directory in output_directories: + ] + [{ + "Path": "Total", + bksv_field: 0}] + for fieldname in [bksv_field] + result_columns: for row in extra_rows: - row[output_directory] = 0 - - extra_row = { - "Path": "Total", - bksv_field: 0} - for output_directory in output_directories: - extra_row[output_directory] = 0 + row[fieldname] = 0 for row in reader: if not row["Path"]: @@ -581,14 +601,7 @@ def highlight(s): row[bksv_field] = int(row[bksv_field]) - instance_class = int(row["Path"].split('_')[1].split('.')[0]) - number_of_items = int(row["Path"].split('_')[2]) - row_id = ( - (instance_class - 1) * 5 - + int(number_of_items / 20 - 1)) - extra_row[bksv_field] += int(row[bksv_field]) - extra_rows[row_id][bksv_field] += int(row[bksv_field]) - + # Fill current row. for output_directory in output_directories: json_output_path = os.path.join( benchmark_directory, @@ -596,26 +609,44 @@ def highlight(s): row["Path"] + "_output.json") json_output_file = open(json_output_path, "r") json_data = json.load(json_output_file) - number_of_bins = int(json_data["Output"] - ["Solution"] - ["NumberOfBins"]) - row[output_directory] = number_of_bins - extra_rows[row_id][output_directory] += number_of_bins - extra_row[output_directory] += number_of_bins + row[output_directory + " / Solution value"] = ( + json_data["Output"]["Solution"]["NumberOfBins"]) + + # Get extra rows to update. + instance_class = int(row["Path"].split('_')[1].split('.')[0]) + number_of_items = int(row["Path"].split('_')[2]) + extra_rows_to_update = [ + ((instance_class - 1) * 5 + + int(number_of_items / 20 - 1)), + -1] + # Update "Best known solution value" column of extra row. + for row_id in extra_rows_to_update: + extra_rows[row_id][bksv_field] += row[bksv_field] + + # Update result columns of extra rows. + for result_column in result_columns: + number_of_bins = int(row[result_column]) + row[result_column] = number_of_bins + for row_id in extra_rows_to_update: + extra_rows[row_id][result_column] += number_of_bins + + # Add current row. out_rows.append(row) + # Add extra rows. for row in extra_rows: out_rows.append(row) - out_rows.append(extra_row) df = pd.DataFrame.from_records(out_rows, columns=out_fieldnames) def highlight(s): return [('background-color: lightgreen' - if s[fieldname] == int(s[bksv_field]) - else 'background-color: pink') - if fieldname in output_directories + if s[fieldname] == s[bksv_field] + else ('background-color: pink' + if s[fieldname] > s[bksv_field] + else 'background-color: yellow')) + if fieldname in result_columns else '' for fieldname in out_fieldnames] df = df.style.apply(highlight, axis = 1)