Skip to content

Commit

Permalink
Write nan to scan summary if result is not available
Browse files Browse the repository at this point in the history
  • Loading branch information
@andlessa
andlessa committed Mar 22, 2024
1 parent 3103d11 commit 03ab812
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 92 deletions.
8 changes: 4 additions & 4 deletions smodels/matching/modelTester.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -409,10 +409,10 @@ def testPoints(fileList, inDir, outputDir, parser, database,
outputDict.update(p.get())

# Collect output to build global summary:
summaryFile = os.path.join(outputDir, 'summary.txt')
logger.info("A summary of the results can be found in %s" %
summaryFile)
printScanSummary(outputDict, summaryFile)
scanSummaryFile = os.path.join(outputDir, 'summary.txt')
logger.info("A summary of the scan results can be found in %s" %
scanSummaryFile)
printScanSummary(outputDict, scanSummaryFile)
# Remove summary log from logger
logger.removeHandler(fileLog)
fileLog.close()
Expand Down
125 changes: 58 additions & 67 deletions smodels/tools/printerTools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,25 +49,25 @@ def printScanSummary(outputDict, outputFile):
# default values (in case of empty results):
summaryDict = OrderedDict({'filename': fname,
'MostConstrainingAnalysis': 'N/A',
'r_max': -1,
'r_exp': -1,
'r_max': np.nan,
'r_exp': np.nan,
'MostSensitive(ATLAS)': 'N/A',
'r(ATLAS)': -1,
'r_exp(ATLAS)': -1,
'r(ATLAS)': np.nan,
'r_exp(ATLAS)': np.nan,
'MostSensitive(CMS)': 'N/A',
'r(CMS)': -1,
'r_exp(CMS)': -1,
'r(combined)' : -1,
'r_exp(combined)' : -1,
'r(CMS)': np.nan,
'r_exp(CMS)': np.nan,
'r(combined)' : np.nan,
'r_exp(combined)' : np.nan,
'CombinedAnalyses' : 'N/A'
})

if 'python' in output:
sDict = getSummaryFrom(output['python'], 'python')
sDict = getScanSummaryFrom(output['python'], 'python')
elif 'slha' in output:
sDict = getSummaryFrom(output['slha'], 'slha')
sDict = getScanSummaryFrom(output['slha'], 'slha')
elif 'summary' in output:
sDict = getSummaryFrom(output['summary'], 'summary')
sDict = getScanSummaryFrom(output['summary'], 'summary')
else:
sDict = {}

Expand All @@ -79,7 +79,7 @@ def printScanSummary(outputDict, outputFile):

# If there are no combined results, remove its (dummy) entries
anaList = set([])
if all(summary['r(combined)'] == -1 for summary in summaryList):
if all(np.isnan(summary['r(combined)']) for summary in summaryList):
for summary in summaryList:
summary.pop('r(combined)')
summary.pop('r_exp(combined)')
Expand All @@ -101,7 +101,7 @@ def printScanSummary(outputDict, outputFile):
header += "#The most senstive (ATLAS/CMS) analysis corresponds to the one with largest expected r from those analyses for which this information is available.\n"
if anaList:
header += "#Analyses used for combination = %s.\n" %(','.join(anaList))
header += "#r(combined) = -1 means no analysis from the above combination set produced results.\n"
header += "#r(combined) = nan means no analysis from the above combination set produced results.\n"


# Get column labels and widths:
Expand Down Expand Up @@ -164,7 +164,7 @@ def formatNestedDict(outputDict,ident=0,maxLength=50):
if isinstance(key,str):
keyStr = "'"+key+"'"
else:
keyStr = str(key)
keyStr = str(key)
if ik == 0:
output += ' '*ident+"%s : %s" %(keyStr,valStr)
else:
Expand All @@ -174,11 +174,11 @@ def formatNestedDict(outputDict,ident=0,maxLength=50):
output += ",\n"
else:
output += "\n"

output += ' '*(ident-4)+'}'
return output

def formatNestedList(outputList,ident=0,maxLength=50):
def formatNestedList(outputList,ident=0,maxLength=50):
"""
Convert a nested list to a string
with identation.
Expand Down Expand Up @@ -216,7 +216,7 @@ def formatNestedList(outputList,ident=0,maxLength=50):
output += ' '*(ident-4)+']'
return output

def getSummaryFrom(output, ptype):
def getScanSummaryFrom(output, ptype):
"""
Retrieves information about the output according to the printer type (slha,python or summary)
Expand All @@ -241,43 +241,34 @@ def getSummaryFrom(output, ptype):
else:
rvals, rexp, anaIDs, r_comb, rexp_comb, anaID_comb = info

# Sort results by r_obs:
rvalswo = copy.deepcopy(rvals)
rvalswo[rvalswo is None] = -1
asort = rvalswo.argsort()[::-1]
rvals = rvals[asort]
anaIDs = anaIDs[asort]
rexp = rexp[asort]
summaryDict['r_max'] = rvals[0]
summaryDict['r_exp'] = rexp[0]
summaryDict['MostConstrainingAnalysis'] = anaIDs[0]

# Sort results by r_obs:
rvalswo = copy.deepcopy(rexp)
rvalswo[rvalswo is None] = -1
# Sort results by r_exp:
asort = rvalswo.argsort()[::-1]
rvals = rvals[asort]
anaIDs = anaIDs[asort]
rexp = rexp[asort]
iATLAS, iCMS = -1, -1
for i, anaID in enumerate(anaIDs):
if rexp[i] < 0:
continue
if 'ATLAS' in anaID and iATLAS < 0:
iATLAS = i
elif 'CMS' in anaID and iCMS < 0:
iCMS = i

if iATLAS >= 0:
summaryDict['r(ATLAS)'] = rvals[iATLAS]
summaryDict['r_exp(ATLAS)'] = rexp[iATLAS]
summaryDict['MostSensitive(ATLAS)'] = anaIDs[iATLAS]

if iCMS >= 0:
summaryDict['r(CMS)'] = rvals[iCMS]
summaryDict['r_exp(CMS)'] = rexp[iCMS]
summaryDict['MostSensitive(CMS)'] = anaIDs[iCMS]
# Sort results by r_obs
# (replace nan by -1 for sorting only)
rvals_c = np.nan_to_num(rvals,nan=-1.0)
imax = rvals_c.argsort()[::-1][0]
summaryDict['r_max'] = rvals[imax]
summaryDict['r_exp'] = rexp[imax]
summaryDict['MostConstrainingAnalysis'] = anaIDs[imax]

# Now keep only results with r_exp != nan:
rexp_good = rexp[~np.isnan(rexp)]
rvals_good = rvals[~np.isnan(rexp)]
anaIDs_good = anaIDs[~np.isnan(rexp)]
asort = rexp_good.argsort()[::-1]
rvals_good = rvals_good[asort]
anaIDs_good = anaIDs_good[asort]
rexp_good = rexp_good[asort]
iATLAS = [i for i,ana in enumerate(anaIDs_good) if 'ATLAS' in ana]
iCMS = [i for i,ana in enumerate(anaIDs_good) if 'CMS' in ana]
if len(iATLAS) > 0:
iATLAS = iATLAS[0]
summaryDict['r(ATLAS)'] = rvals_good[iATLAS]
summaryDict['r_exp(ATLAS)'] = rexp_good[iATLAS]
summaryDict['MostSensitive(ATLAS)'] = anaIDs_good[iATLAS]
if len(iCMS) > 0:
iCMS = iCMS[0]
summaryDict['r(CMS)'] = rvals_good[iCMS]
summaryDict['r_exp(CMS)'] = rexp_good[iCMS]
summaryDict['MostSensitive(CMS)'] = anaIDs_good[iCMS]

summaryDict['r(combined)'] = r_comb
summaryDict['r_exp(combined)'] = rexp_comb
Expand All @@ -301,16 +292,16 @@ def getInfoFromPython(output):
return None
rvals = np.array([res['r'] for res in output['ExptRes']])
rexp = np.array([res['r_expected'] if res['r_expected']
else -1 for res in output['ExptRes']])
else np.nan for res in output['ExptRes']])
anaIDs = np.array([res['AnalysisID'] for res in output['ExptRes']])

r_comb = -1
rexp_comb = -1
r_comb = np.nan
rexp_comb = np.nan
anaID_comb = 'N/A'

if 'CombinedRes' in output:
for res in output['CombinedRes']:
if r_comb is None or r_comb < res['r']:
if np.isnan(r_comb) or r_comb < res['r']:
r_comb = res['r']
rexp_comb = res['r_expected']
anaID_comb = res['AnalysisID']
Expand Down Expand Up @@ -347,25 +338,25 @@ def getInfoFromSLHA(output):
resDict = {i : dict(block) for i,block in groups if i != 0}
# Get r values:
rvals = np.array([resDict[i][(i,1)] for i in resDict])
rexp = np.array([resDict[i][(i,2)] if resDict[i][(i,2)] != 'N/A' else -1
rexp = np.array([resDict[i][(i,2)] if resDict[i][(i,2)] != 'N/A' else np.nan
for i in resDict])
anaIDs = np.array([resDict[i][(i,4)] for i in resDict])

if bcombName is None or len(results.blocks[bcombName]) < 1:
r_comb = -1
rexp_comb = -1
r_comb = np.nan
rexp_comb = np.nan
anaID_comb = 'N/A'
else:
# Group combined results by block index:
groups = itertools.groupby(results.blocks[bcombName].items(),
key = lambda k: k[0][0])
resDict = {i : dict(block) for i,block in groups if i != 0}
rvals_comb = np.array([resDict[i][(i,1)] for i in resDict if i != 0])
rexp_comb = np.array([resDict[i][(i,2)] if resDict[i][(i,2)] != 'N/A' else -1
rexp_comb = np.array([resDict[i][(i,2)] if resDict[i][(i,2)] != 'N/A' else np.nan
for i in resDict if i != 0])
anaID_comb = np.array([resDict[i][(i,6)] for i in resDict if i != 0])

r_comb = max(rvals_comb)
r_comb = max(rvals_comb[~np.isnan(rvals_comb)])
rexp_comb = rexp_comb[np.argmax(rvals_comb)]
anaID_comb = anaID_comb[np.argmax(rvals_comb)]

Expand All @@ -387,8 +378,8 @@ def getInfoFromSummary(output):
rvals = []
rexp = []
anaIDs = []
r_comb = -1
rexp_comb = -1
r_comb = np.nan
rexp_comb = np.nan
anaID_comb = 'N/A'

for line in lines:
Expand All @@ -398,7 +389,7 @@ def getInfoFromSummary(output):
for x in rmax])[0][0] # Get when the value ends
rmax = eval(rmax[:ff])
anaMax = line.split('from')[1].split()[0].replace(',', '')
rexpMax = -1
rexpMax = np.nan
if 'r_expected' in line and "r_expected not available" not in line:
rexpMax = line.split('r_expected')[-1]
rexpMax = rexpMax.split('=')[1]
Expand All @@ -414,7 +405,7 @@ def getInfoFromSummary(output):
ff = np.where([((not x.isdigit()) and (x not in ['.', '+', '-']))
for x in rAna])[0][0] # Get when the value ends
rAna = eval(rAna[:ff])
rexpAna = -1
rexpAna = np.nan
if 'r_expected' in line:
rexpAna = line.split('r_expected')[-1]
rexpAna = rexpAna.split('=')[1]
Expand Down
2 changes: 1 addition & 1 deletion smodels/tools/runSModelS.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def run( inFile, parameterFile, outputDir, db, timeout, development ):
or directory name (path to directory containing input files)
:param parameterFile: File containing the input parameters (default =
smodels/etc/parameters_default.ini)
:param outputDir: Output directory to write a summary of results to
:param outputDir: Output directory to write the results to
:param db: supply a smodels.experiment.databaseObj.Database object, so
the database doesn't have to be loaded anymore. Will
render a few parameters in the parameter file irrelevant.
Expand Down
40 changes: 20 additions & 20 deletions unittests/summary_scan_default.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,26 +2,26 @@
#The most constraining analysis corresponds to the one with largest observed r.
#The most senstive (ATLAS/CMS) analysis corresponds to the one with largest expected r from those analyses for which this information is available.
#Analyses used for combination = ATLAS-CONF-2013-037,CMS-SUS-13-012.
#r(combined) = -1 means no analysis from the above combination set produced results.
#r(combined) = nan means no analysis from the above combination set produced results.
#filename MostConstrainingAnalysis r_max r_exp MostSensitive(ATLAS) r(ATLAS) r_exp(ATLAS) MostSensitive(CMS) r(CMS) r_exp(CMS) r(combined) r_exp(combined)
416126634.slha ATLAS-SUSY-2013-12 1.29 -1 N/A -1 -1 N/A -1 -1 -1 -1
T1tttt.slha CMS-PAS-SUS-15-002 30.5 -1 ATLAS-CONF-2013-037 5.92 5.47 CMS-SUS-13-012 5.82 8.24 8.87 10.4
T1ttttoff.slha CMS-PAS-SUS-15-002 8.16 -1 N/A -1 -1 CMS-SUS-13-012 0.662 0.936 0.662 0.936
T6bbHH_pyhf.slha N/A -1 -1 N/A -1 -1 N/A -1 -1 -1 -1
TRV1_1800_300_300.slha N/A -1 -1 N/A -1 -1 N/A -1 -1 -1 -1
broken.slha N/A -1 -1 N/A -1 -1 N/A -1 -1 -1 -1
complicated.slha ATLAS-SUSY-2013-02 0.388 -1 ATLAS-CONF-2013-037 0.00183 0.00209 CMS-SUS-13-012 0.13 0.208 0.133 0.21
gluinoToTops.slha CMS-PAS-SUS-15-002 5.52 -1 ATLAS-CONF-2013-037 0.479 0.443 CMS-SUS-13-012 0.844 0.703 1.12 0.884
gluino_squarks.slha ATLAS-SUSY-2013-02 0.388 -1 ATLAS-CONF-2013-037 0.00183 0.00209 CMS-SUS-13-012 0.13 0.208 0.133 0.21
higgsinoStop.slha ATLAS-SUSY-2013-05 196 -1 ATLAS-CONF-2013-037 37.8 43.2 N/A -1 -1 37.8 43.2
hscpTest_long.slha CMS-EXO-13-006 0.5 0.5 N/A -1 -1 CMS-EXO-13-006 0.5 0.5 -1 -1
hscpTest_mid.slha CMS-EXO-13-006 0.0884 0.0884 N/A -1 -1 CMS-EXO-13-006 0.0884 0.0884 -1 -1
hscpTest_short.slha CMS-EXO-13-006 0.0406 0.0406 N/A -1 -1 CMS-EXO-13-006 0.0406 0.0406 -1 -1
idm_example.slha ATLAS-SUSY-2013-12 0.0136 -1 N/A -1 -1 N/A -1 -1 -1 -1
lifetime.slha CMS-PAS-SUS-15-002 0.000451 -1 N/A -1 -1 N/A -1 -1 -1 -1
lightEWinos.slha ATLAS-SUSY-2013-12 0.646 -1 ATLAS-CONF-2013-037 0.119 0.11 CMS-SUS-13-012 0.00556 0.00786 0.125 0.115
416126634.slha ATLAS-SUSY-2013-12 1.29 nan N/A nan nan N/A nan nan nan nan
T1tttt.slha CMS-PAS-SUS-15-002 30.5 nan ATLAS-CONF-2013-037 5.92 5.47 CMS-SUS-13-012 5.82 8.24 8.87 10.4
T1ttttoff.slha CMS-PAS-SUS-15-002 8.16 nan N/A nan nan CMS-SUS-13-012 0.662 0.936 0.662 0.936
T6bbHH_pyhf.slha N/A nan nan N/A nan nan N/A nan nan nan nan
TRV1_1800_300_300.slha N/A nan nan N/A nan nan N/A nan nan nan nan
broken.slha N/A nan nan N/A nan nan N/A nan nan nan nan
complicated.slha ATLAS-SUSY-2013-02 0.388 nan ATLAS-CONF-2013-037 0.00183 0.00209 CMS-SUS-13-012 0.13 0.208 0.133 0.21
gluinoToTops.slha CMS-PAS-SUS-15-002 5.52 nan ATLAS-CONF-2013-037 0.479 0.443 CMS-SUS-13-012 0.844 0.703 1.12 0.884
gluino_squarks.slha ATLAS-SUSY-2013-02 0.388 nan ATLAS-CONF-2013-037 0.00183 0.00209 CMS-SUS-13-012 0.13 0.208 0.133 0.21
higgsinoStop.slha ATLAS-SUSY-2013-05 196 nan ATLAS-CONF-2013-037 37.8 43.2 N/A nan nan 37.8 43.2
hscpTest_long.slha CMS-EXO-13-006 0.5 0.5 N/A nan nan CMS-EXO-13-006 0.5 0.5 nan nan
hscpTest_mid.slha CMS-EXO-13-006 0.0884 0.0884 N/A nan nan CMS-EXO-13-006 0.0884 0.0884 nan nan
hscpTest_short.slha CMS-EXO-13-006 0.0406 0.0406 N/A nan nan CMS-EXO-13-006 0.0406 0.0406 nan nan
idm_example.slha ATLAS-SUSY-2013-12 0.0136 nan N/A nan nan N/A nan nan nan nan
lifetime.slha CMS-PAS-SUS-15-002 0.000451 nan N/A nan nan N/A nan nan nan nan
lightEWinos.slha ATLAS-SUSY-2013-12 0.646 nan ATLAS-CONF-2013-037 0.119 0.11 CMS-SUS-13-012 0.00556 0.00786 0.125 0.115
lightEWinos_simple.slha ATLAS-CONF-2013-037 0.119 0.11 ATLAS-CONF-2013-037 0.119 0.11 CMS-SUS-13-012 0.00556 0.00786 0.125 0.115
longLived.slha CMS-EXO-13-006 0.835 0.835 ATLAS-CONF-2013-037 0.00801 0.00916 CMS-EXO-13-006 0.835 0.835 0.00801 0.00916
nobdecay.slha ATLAS-SUSY-2013-02 0.803 -1 N/A -1 -1 N/A -1 -1 -1 -1
simplyGluino.slha CMS-PAS-SUS-15-002 25.7 -1 N/A -1 -1 CMS-SUS-13-012 4.45 6.51 4.45 6.51
squarks_degenerate.slha ATLAS-SUSY-2013-02 0.58 -1 N/A -1 -1 CMS-SUS-13-012 0.256 0.304 0.256 0.304
nobdecay.slha ATLAS-SUSY-2013-02 0.803 nan N/A nan nan N/A nan nan nan nan
simplyGluino.slha CMS-PAS-SUS-15-002 25.7 nan N/A nan nan CMS-SUS-13-012 4.45 6.51 4.45 6.51
squarks_degenerate.slha ATLAS-SUSY-2013-02 0.58 nan N/A nan nan CMS-SUS-13-012 0.256 0.304 0.256 0.304

0 comments on commit 03ab812

Please sign in to comment.