From 0abf8d4690b0f552c8da9ccb9c7b9e55c84580b2 Mon Sep 17 00:00:00 2001 From: tmadlener Date: Thu, 5 Sep 2024 09:59:04 +0200 Subject: [PATCH] Patch missing collections on the fly --- .../production/ILDReconstruction.py | 20 ++- .../production/collections_rec_level.txt | 124 ++++++++++++++++++ StandardConfig/production/py_utils.py | 29 +++- 3 files changed, 168 insertions(+), 5 deletions(-) create mode 100644 StandardConfig/production/collections_rec_level.txt diff --git a/StandardConfig/production/ILDReconstruction.py b/StandardConfig/production/ILDReconstruction.py index e16303de..e00971a7 100644 --- a/StandardConfig/production/ILDReconstruction.py +++ b/StandardConfig/production/ILDReconstruction.py @@ -15,7 +15,7 @@ ) from Gaudi.Configuration import INFO -from py_utils import import_from, SequenceLoader +from py_utils import import_from, SequenceLoader, parse_collection_patch_file from k4FWCore.parseArgs import parser from k4MarlinWrapper.parseConstants import parseConstants @@ -51,6 +51,8 @@ "ILD_l5_v11", ) +REC_COLLECTION_CONTENTS_FILE = "collections_rec_level.txt" + parser.add_argument( "--inputFiles", action="extend", @@ -298,16 +300,26 @@ def create_reader(input_files): "Printing": ["0"], "RootFile": [f"{reco_args.outputFileBase}_PfoAnalysis.root"], } - algList.append(MyPfoAnalysis) if reco_args.lcioOutput != "only": + # Attach the LCIO -> EDM4hep conversion to the last processor that is run + # before the output lcioToEDM4hepOutput = Lcio2EDM4hepTool("OutputConversion") # Take care of the different naming conventions lcioToEDM4hepOutput.collNameMapping = {"MCParticle": "MCParticles"} lcioToEDM4hepOutput.OutputLevel = INFO - # Attach the conversion to the last non-output processor that is always run - MyPfoAnalysis.Lcio2EDM4hepTool = lcioToEDM4hepOutput + + # Make sure that all collections are always available by patching in missing + # ones on-the-fly + collPatcherRec = MarlinProcessorWrapper( + "CollPacherREC", OutputLevel=INFO, ProcessorType="PatchCollections" + ) + collPatcherRec.Parameters = { + "PatchCollections": parse_collection_patch_file(REC_COLLECTION_CONTENTS_FILE) + } + collPatcherRec.Lcio2EDM4hepTool = lcioToEDM4hepOutput + algList.append(collPatcherRec) edm4hepOutput = PodioOutput("EDM4hepOutput") edm4hepOutput.filename = f"{reco_args.outputFileBase}_REC.edm4hep.root" diff --git a/StandardConfig/production/collections_rec_level.txt b/StandardConfig/production/collections_rec_level.txt new file mode 100644 index 00000000..1cdfbb3b --- /dev/null +++ b/StandardConfig/production/collections_rec_level.txt @@ -0,0 +1,124 @@ + YokeEndcapsCollection SimCalorimeterHit + VXDCollection SimTrackerHit + V0RecoParticles ReconstructedParticle + TPCTrackerHitRelations LCRelation[TrackerHit,SimTrackerHit] + TPCCollection SimTrackerHit + SITTrackerHitRelations LCRelation[TrackerHitPlane,SimTrackerHit] + SITCollection SimTrackerHit + SETSpacePoints TrackerHit + SETSpacePointRelations LCRelation[TrackerHit,SimTrackerHit] + RecoMCTruthLink LCRelation[ReconstructedParticle,MCParticle] + ProngVertices Vertex + ProngRecoParticles ReconstructedParticle + PrimaryVertex Vertex + PandoraPFOs ReconstructedParticle + RelationBCalHit LCRelation[CalorimeterHit,SimCalorimeterHit] + PandoraPFANewStartVertices Vertex + MarlinTrkTracksProtonMCP LCRelation[Track,MCParticle] + SubsetTracks Track + MarlinTrkTracksProton Track + TPCTrackerHits TrackerHit + MarlinTrkTracks Track + SETTrackerHitRelations LCRelation[TrackerHitPlane,SimTrackerHit] + MUON CalorimeterHit + VXDTrackerHits TrackerHitPlane + MCTruthMarlinTrkTracksLink LCRelation[MCParticle,Track] + MCParticle MCParticle + LumiCalCollection SimCalorimeterHit + LHCalCollection SimCalorimeterHit + LHCAL CalorimeterHit + YokeBarrelCollection SimCalorimeterHit + LCAL CalorimeterHit + KinkVertices Vertex + KinkRecoParticles ReconstructedParticle + MCTruthClusterLink LCRelation[MCParticle,Cluster] + EcalEndcapsCollectionDigi CalorimeterHit + BCAL CalorimeterHit + FTDSpacePointRelations LCRelation[TrackerHit,SimTrackerHit] + PrimaryVertex_RP ReconstructedParticle + HcalEndcapRingCollectionRec CalorimeterHit + HcalEndcapsCollectionDigi CalorimeterHit + EcalEndcapRingRelationsSimRec LCRelation[CalorimeterHit,SimCalorimeterHit] + EcalEndcapRingCollection SimCalorimeterHit + TPCLowPtCollection SimTrackerHit + HcalEndcapRingCollection SimCalorimeterHit + EcalEndcapsCollectionGapHits CalorimeterHit + GammaGammaCandidateEtaPrimes ReconstructedParticle + EcalBarrelRelationsSimRec LCRelation[CalorimeterHit,SimCalorimeterHit] + EcalBarrelRelationsSimDigi LCRelation[CalorimeterHit,SimCalorimeterHit] + EcalBarrelCollectionDigi CalorimeterHit + SITTrackerHits TrackerHitPlane + EcalBarrelCollection SimCalorimeterHit + BuildUpVertex_V0 Vertex + EcalBarrelCollectionRec CalorimeterHit + SplitRecoParticles ReconstructedParticle + PandoraClusters Cluster + EcalEndcapsRelationsSimRec LCRelation[CalorimeterHit,SimCalorimeterHit] + HcalEndcapRingRelationsSimDigi LCRelation[CalorimeterHit,SimCalorimeterHit] + ECalEndcapSiHitsOdd SimCalorimeterHit + BuildUpVertex_V0_RP ReconstructedParticle + GammaGammaCandidatePi0s ReconstructedParticle + MarlinTrkTracksKaonMCP LCRelation[Track,MCParticle] + EcalEndcapRingCollectionDigi CalorimeterHit + ClusterMCTruthLink LCRelation[Cluster,MCParticle] + V0Vertices Vertex + HcalEndcapsCollection SimCalorimeterHit + ECalBarrelSiHitsOdd SimCalorimeterHit + ClupatraTracks Track + SETCollection SimTrackerHit + EcalEndcapsCollection SimCalorimeterHit + EcalEndcapRingRelationsSimDigi LCRelation[CalorimeterHit,SimCalorimeterHit] + FTDPixelTrackerHitRelations LCRelation[TrackerHitPlane,SimTrackerHit] + RelationMuonHit LCRelation[CalorimeterHit,SimCalorimeterHit] + EcalBarrelCollectionGapHits CalorimeterHit + MarlinTrkTracksMCTruthLink LCRelation[Track,MCParticle] + ClupatraTrackSegments Track + SplitVertices Vertex + HcalEndcapRingRelationsSimRec LCRelation[CalorimeterHit,SimCalorimeterHit] + MCParticlesSkimmed MCParticle + ECalEndcapSiHitsEven SimCalorimeterHit + MarlinTrkTracksKaon Track + EcalEndcapsCollectionRec CalorimeterHit + HcalEndcapRingCollectionDigi CalorimeterHit + GammaGammaCandidateEtas ReconstructedParticle + GammaGammaParticles ReconstructedParticle + SiTracks Track + RelationLHcalHit LCRelation[CalorimeterHit,SimCalorimeterHit] + MCTruthRecoLink LCRelation[MCParticle,ReconstructedParticle] + DistilledPFOs ReconstructedParticle + BuildUpVertex_RP ReconstructedParticle + EcalEndcapsRelationsSimDigi LCRelation[CalorimeterHit,SimCalorimeterHit] + VXDTrackerHitRelations LCRelation[TrackerHitPlane,SimTrackerHit] + FTDCollection SimTrackerHit + BuildUpVertex Vertex + FTDPixelTrackerHits TrackerHitPlane + TPCSpacePointCollection SimTrackerHit + FTDStripTrackerHits TrackerHitPlane + ForwardTracks Track + ECalBarrelSiHitsEven SimCalorimeterHit + FTDSpacePoints TrackerHit + FTDStripTrackerHitRelations LCRelation[TrackerHitPlane,SimTrackerHit] + BeamCalCollection SimCalorimeterHit + HcalEndcapsCollectionRec CalorimeterHit + HcalBarrelRelationsSimDigi LCRelation[CalorimeterHit,SimCalorimeterHit] + FTD_STRIPCollection SimTrackerHit + FTD_PIXELCollection SimTrackerHit + HcalBarrelCollectionDigi CalorimeterHit + SETTrackerHits TrackerHitPlane + EcalEndcapRingCollectionRec CalorimeterHit + HcalBarrelCollectionRec CalorimeterHit + HcalBarrelRelationsSimRec LCRelation[CalorimeterHit,SimCalorimeterHit] + HcalBarrelRegCollection SimCalorimeterHit + RelationLcalHit LCRelation[CalorimeterHit,SimCalorimeterHit] + HcalEndcapsRelationsSimRec LCRelation[CalorimeterHit,SimCalorimeterHit] + HcalEndcapsRelationsSimDigi LCRelation[CalorimeterHit,SimCalorimeterHit] +BasicVariablePID PandoraPFOs|electronLikelihood,muonLikelihood,pionLikelihood,kaonLikelihood,protonLikelihood,hadronLikelihood,MVAOutput_mupiSeparation,electronProbability,muonProbability,pionProbability,kaonProbability,protonProbability,hadronProbability,electron_dEdxdistance,muon_dEdxdistance,pion_dEdxdistance,kaon_dEdxdistance,proton_dEdxdistance +LikelihoodPID PandoraPFOs|electronLikelihood,muonLikelihood,pionLikelihood,kaonLikelihood,protonLikelihood,hadronLikelihood,MVAOutput_mupiSeparation,electronProbability,muonProbability,pionProbability,kaonProbability,protonProbability,hadronProbability,electron_dEdxdistance,muon_dEdxdistance,pion_dEdxdistance,kaon_dEdxdistance,proton_dEdxdistance +LowMomMuID PandoraPFOs|electronLikelihood,muonLikelihood,pionLikelihood,kaonLikelihood,protonLikelihood,hadronLikelihood,MVAOutput_mupiSeparation,electronProbability,muonProbability,pionProbability,kaonProbability,protonProbability,hadronProbability,electron_dEdxdistance,muon_dEdxdistance,pion_dEdxdistance,kaon_dEdxdistance,proton_dEdxdistance +ShowerShapesPID PandoraPFOs|electronLikelihood,muonLikelihood,pionLikelihood,kaonLikelihood,protonLikelihood,hadronLikelihood,MVAOutput_mupiSeparation,electronProbability,muonProbability,pionProbability,kaonProbability,protonProbability,hadronProbability,electron_dEdxdistance,muon_dEdxdistance,pion_dEdxdistance,kaon_dEdxdistance,proton_dEdxdistance +TOFEstimators0ps PandoraPFOs|timeOfFlight +TOFEstimators100ps PandoraPFOs|timeOfFlight +TOFEstimators10ps PandoraPFOs|timeOfFlight +TOFEstimators50ps PandoraPFOs|timeOfFlight +TrackLengthProcessor PandoraPFOs|trackLengthToSET,trackLengthToEcal,momentumHMToSET,momentumHMToEcal +dEdxPID PandoraPFOs|electronLikelihood,muonLikelihood,pionLikelihood,kaonLikelihood,protonLikelihood,hadronLikelihood,MVAOutput_mupiSeparation,electronProbability,muonProbability,pionProbability,kaonProbability,protonProbability,hadronProbability,electron_dEdxdistance,muon_dEdxdistance,pion_dEdxdistance,kaon_dEdxdistance,proton_dEdxdistance diff --git a/StandardConfig/production/py_utils.py b/StandardConfig/production/py_utils.py index a3bd846d..c19fd7f3 100644 --- a/StandardConfig/production/py_utils.py +++ b/StandardConfig/production/py_utils.py @@ -1,6 +1,6 @@ import os from io import TextIOWrapper -from typing import Union, Optional, Dict, Any +from typing import Union, Optional, Dict, Any, List import importlib.util import importlib.abc from importlib.machinery import SourceFileLoader @@ -140,3 +140,30 @@ def load(self, sequence: str) -> None: seq = getattr(seq_module, seq_name) self.alg_list.extend(seq) + + +def parse_collection_patch_file(patch_file: Union[str, os.PathLike]) -> List[str]: + """Parse a collection patch file such that it can be used by the + PatchCollections processor. + + This function reads the file that has been passed in and effectively + flattens its contents into one list of strings. The main assumption is that + the file has been produced via `check_missing_colls --minimal <...>` in + which case it can be directly consumed. Note that no real error checking is + done to detect malformed inputs in which case something else at a later + stage will most likely break. + + Args: + patch_file (Union[str, os.PathLike]): The path to the file that should + be parsed + + Returns: + List[str]: A list of strings (pairs of "names" and "types") that can be + consumed by the PatchCollections processor + + """ + with open(patch_file, "r") as pfile: + patch_colls = [l.split() for l in pfile.readlines()] + + # Flatten the list of lists into one large list + return [s for strings in patch_colls for s in strings]