Skip to content

Commit

Permalink
Record overscan statistics units
Browse files Browse the repository at this point in the history
  • Loading branch information
@Alex-Broughton
Alex-Broughton committed Oct 28, 2024
1 parent 55ca087 commit 6cf0085
Show file tree
Hide file tree
Showing 2 changed files with 25 additions and 88 deletions.
90 changes: 25 additions & 65 deletions python/lsst/ip/isr/ptcDataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,15 +193,16 @@ class PhotonTransferCurveDataset(IsrCalib):
the model in Eq. 20 of Astier+19 (units: electron^2).
covariancesModelNoB : `dict`, [`str`, `np.ndarray`]
Dictionary keyed by amp names containing covariances model
(with 'b'=0 in Eq. 20 of Astier+19)
per mean flux (units: adu^2).
(with 'b'=0 in Eq. 20 of Astier+19) per mean flux (units:
adu^2). Will be deprecated in v28.
aMatrixNoB : `dict`, [`str`, `np.ndarray`]
Dictionary keyed by amp names containing the "a" parameters from the
model in Eq. 20 of Astier+19
(and 'b' = 0) (units: 1/electron).
model in Eq. 20 of Astier+19 (and 'b' = 0) (units: 1/electron).
Will be deprecated in v28.
noiseMatrixNoB : `dict`, [`str`, `np.ndarray`]
Dictionary keyed by amp names containing the "noise" parameters from
the model in Eq. 20 of Astier+19, with 'b' = 0 (units: electron^2).
Will be deprecated in v28.
finalVars : `dict`, [`str`, `np.ndarray`]
Dictionary keyed by amp names containing the masked variance of the
difference image of each flat
Expand Down Expand Up @@ -303,9 +304,9 @@ def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1,
self.aMatrix = {ampName: np.array([]) for ampName in ampNames}
self.bMatrix = {ampName: np.array([]) for ampName in ampNames}
self.noiseMatrix = {ampName: np.array([]) for ampName in ampNames}
self.covariancesModelNoB = {ampName: np.array([]) for ampName in ampNames}
self.aMatrixNoB = {ampName: np.array([]) for ampName in ampNames}
self.noiseMatrixNoB = {ampName: np.array([]) for ampName in ampNames}
self._covariancesModelNoB = {ampName: np.array([]) for ampName in ampNames}
self._aMatrixNoB = {ampName: np.array([]) for ampName in ampNames}
self._noiseMatrixNoB = {ampName: np.array([]) for ampName in ampNames}

self.finalVars = {ampName: np.array([]) for ampName in ampNames}
self.finalModelVars = {ampName: np.array([]) for ampName in ampNames}
Expand All @@ -319,9 +320,8 @@ def __init__(self, ampNames=[], ptcFitType=None, covMatrixSide=1,
'rawMeans', 'rawVars', 'rowMeanVariance', 'gain',
'gainErr', 'noise', 'noiseErr', 'noiseList',
'ptcFitPars', 'ptcFitParsError', 'ptcFitChiSq', 'ptcTurnoff',
'aMatrixNoB', 'covariances', 'covariancesModel',
'covariancesSqrtWeights', 'covariancesModelNoB',
'aMatrix', 'bMatrix', 'noiseMatrix', 'noiseMatrixNoB', 'finalVars',
'covariances', 'covariancesModel', 'covariancesSqrtWeights',
'aMatrix', 'bMatrix', 'noiseMatrix', 'finalVars',
'finalModelVars', 'finalMeans', 'photoCharges', 'histVars',
'histChi2Dofs', 'kspValues', 'auxValues', 'ptcTurnoffSamplingError',
'ampOffsets', 'gainUnadjusted', 'gainList'])
Expand Down Expand Up @@ -417,13 +417,9 @@ def setAmpValuesPartialDataset(

# From FULLCOVARIANCE model
self.covariancesModel[ampName] = np.array([nanMatrixFit])
self._covariancesModelNoB[ampName] = np.array([nanMatrixFit])
self.aMatrix[ampName] = nanMatrixFit
self.bMatrix[ampName] = nanMatrixFit
self._aMatrixNoB[ampName] = nanMatrixFit
self.noiseMatrix[ampName] = nanMatrixFit
self._noiseMatrixNoB[ampName] = nanMatrixFit

# Filler values.
self.finalVars[ampName] = np.array([np.nan])
self.finalModelVars[ampName] = np.array([np.nan])
Expand Down Expand Up @@ -544,29 +540,17 @@ def fromDict(cls, dictionary):
calib.bMatrix[ampName] = np.array(dictionary['bMatrix'][ampName],
dtype=np.float64).reshape(
(covMatrixSideFullCovFit, covMatrixSideFullCovFit))
calib._covariancesModelNoB[ampName] = np.array(
dictionary['covariancesModelNoB'][ampName], dtype=np.float64).reshape(
(nSignalPoints, covMatrixSideFullCovFit, covMatrixSideFullCovFit))
calib._aMatrixNoB[ampName] = np.array(
dictionary['aMatrixNoB'][ampName],
dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit))
calib.noiseMatrix[ampName] = np.array(
dictionary['noiseMatrix'][ampName],
dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit))
calib._noiseMatrixNoB[ampName] = np.array(
dictionary['noiseMatrixNoB'][ampName],
dtype=np.float64).reshape((covMatrixSideFullCovFit, covMatrixSideFullCovFit))
else:
# Empty dataset
calib.covariances[ampName] = np.array([], dtype=np.float64)
calib.covariancesModel[ampName] = np.array([], dtype=np.float64)
calib.covariancesSqrtWeights[ampName] = np.array([], dtype=np.float64)
calib.aMatrix[ampName] = np.array([], dtype=np.float64)
calib.bMatrix[ampName] = np.array([], dtype=np.float64)
calib._covariancesModelNoB[ampName] = np.array([], dtype=np.float64)
calib._aMatrixNoB[ampName] = np.array([], dtype=np.float64)
calib.noiseMatrix[ampName] = np.array([], dtype=np.float64)
calib._noiseMatrixNoB[ampName] = np.array([], dtype=np.float64)

calib.finalVars[ampName] = np.array(dictionary['finalVars'][ampName], dtype=np.float64)
calib.finalModelVars[ampName] = np.array(dictionary['finalModelVars'][ampName], dtype=np.float64)
Expand Down Expand Up @@ -640,9 +624,6 @@ def _dictOfArraysToDictOfLists(dictOfArrays):
outDict['aMatrix'] = _dictOfArraysToDictOfLists(self.aMatrix)
outDict['bMatrix'] = _dictOfArraysToDictOfLists(self.bMatrix)
outDict['noiseMatrix'] = _dictOfArraysToDictOfLists(self.noiseMatrix)
outDict['covariancesModelNoB'] = _dictOfArraysToDictOfLists(self._covariancesModelNoB)
outDict['aMatrixNoB'] = _dictOfArraysToDictOfLists(self._aMatrixNoB)
outDict['noiseMatrixNoB'] = _dictOfArraysToDictOfLists(self._noiseMatrixNoB)
outDict['finalVars'] = _dictOfArraysToDictOfLists(self.finalVars)
outDict['finalModelVars'] = _dictOfArraysToDictOfLists(self.finalModelVars)
outDict['finalMeans'] = _dictOfArraysToDictOfLists(self.finalMeans)
Expand Down Expand Up @@ -705,9 +686,6 @@ def fromTable(cls, tableList):
inDict['aMatrix'] = dict()
inDict['bMatrix'] = dict()
inDict['noiseMatrix'] = dict()
inDict['covariancesModelNoB'] = dict()
inDict['aMatrixNoB'] = dict()
inDict['noiseMatrixNoB'] = dict()
inDict['finalVars'] = dict()
inDict['finalModelVars'] = dict()
inDict['finalMeans'] = dict()
Expand Down Expand Up @@ -742,8 +720,6 @@ def fromTable(cls, tableList):
inDict['covariancesSqrtWeights'][ampName] = record['COVARIANCES_SQRT_WEIGHTS']
inDict['aMatrix'][ampName] = record['A_MATRIX']
inDict['bMatrix'][ampName] = record['B_MATRIX']
inDict['covariancesModelNoB'][ampName] = record['COVARIANCES_MODEL_NO_B']
inDict['aMatrixNoB'][ampName] = record['A_MATRIX_NO_B']
inDict['finalVars'][ampName] = record['FINAL_VARS']
inDict['finalModelVars'][ampName] = record['FINAL_MODEL_VARS']
inDict['finalMeans'][ampName] = record['FINAL_MEANS']
Expand All @@ -769,10 +745,8 @@ def fromTable(cls, tableList):
if calibVersion < 1.3:
nanMatrix = np.full_like(inDict['aMatrix'][ampName], np.nan)
inDict['noiseMatrix'][ampName] = nanMatrix
inDict['noiseMatrixNoB'][ampName] = nanMatrix
else:
inDict['noiseMatrix'][ampName] = record['NOISE_MATRIX']
inDict['noiseMatrixNoB'][ampName] = record['NOISE_MATRIX_NO_B']
if calibVersion < 1.5:
# Matched to `COV_MATRIX_SIDE`. Same for all amps.
inDict['covMatrixSideFullCovFit'] = inDict['covMatrixSide']
Expand Down Expand Up @@ -878,16 +852,13 @@ def toTable(self):
'PTC_TURNOFF_SAMPLING_ERROR': self.ptcTurnoffSamplingError[ampName],
'A_MATRIX': self.aMatrix[ampName].ravel(),
'B_MATRIX': self.bMatrix[ampName].ravel(),
'A_MATRIX_NO_B': self._aMatrixNoB[ampName].ravel(),
'NOISE_MATRIX': self.noiseMatrix[ampName].ravel(),
'NOISE_MATRIX_NO_B': self._noiseMatrixNoB[ampName].ravel(),
'BAD_AMPS': badAmps,
'PHOTO_CHARGE': self.photoCharges[ampName],
'AMP_OFFSETS': self.ampOffsets[ampName],
'COVARIANCES': self.covariances[ampName].ravel(),
'COVARIANCES_MODEL': self.covariancesModel[ampName].ravel(),
'COVARIANCES_SQRT_WEIGHTS': self.covariancesSqrtWeights[ampName].ravel(),
'COVARIANCES_MODEL_NO_B': self._covariancesModelNoB[ampName].ravel(),
'FINAL_VARS': self.finalVars[ampName],
'FINAL_MODEL_VARS': self.finalModelVars[ampName],
'FINAL_MEANS': self.finalMeans[ampName],
Expand Down Expand Up @@ -1017,16 +988,6 @@ def appendPartialPtc(self, partialPtc):
self.covMatrixSide,
)
)
self._covariancesModelNoB[ampName] = np.append(
self._covariancesModelNoB[ampName].ravel(),
partialPtc._covariancesModelNoB[ampName].ravel()
).reshape(
(
len(self.rawExpTimes[ampName]),
self.covMatrixSide,
self.covMatrixSide,
)
)

def sort(self, sortIndex):
"""Sort the components of the PTC by a given sort index.
Expand Down Expand Up @@ -1076,7 +1037,6 @@ def sort(self, sortIndex):
self.covariances[ampName] = self.covariances[ampName][index]
self.covariancesSqrtWeights[ampName] = self.covariancesSqrtWeights[ampName][index]
self.covariancesModel[ampName] = self.covariancesModel[ampName][index]
self._covariancesModelNoB[ampName] = self._covariancesModelNoB[ampName][index]

self.finalVars[ampName] = self.finalVars[ampName][index]
self.finalModelVars[ampName] = self.finalModelVars[ampName][index]
Expand Down Expand Up @@ -1292,32 +1252,32 @@ def _validateCovarianceMatrizSizes(self):
"Setting the former to the latter.")
self.covMatrixSideFullCovFit = self.covMatrixSide

@property
@deprecated(
reason="The covariancesModelNoB attribute is deprecated. Will be "
"removed after v28.",
version="v28.0",
category=FutureWarning
reason="The covariancesModelNoB attribute is deprecated. Will be "
"removed after v28.",
version="v28.0",
category=FutureWarning
)
@property
def covariancesModelNoB(self):
return self._covariancesModelNoB

@property
@deprecated(
reason="The aMatrixNoB attribute is deprecated. Will be "
"removed after v28.",
version="v28.0",
category=FutureWarning
reason="The aMatrixNoB attribute is deprecated. Will be "
"removed after v28.",
version="v28.0",
category=FutureWarning
)
@property
def aMatrixNoB(self):
return self._aMatrixNoB

@property
@deprecated(
reason="The noiseMatrixNoB attribute is deprecated. Will be "
"removed after v28.",
version="v28.0",
category=FutureWarning
reason="The noiseMatrixNoB attribute is deprecated. Will be "
"removed after v28.",
version="v28.0",
category=FutureWarning
)
@property
def noiseMatrixNoB(self):
return self._noiseMatrixNoB
23 changes: 0 additions & 23 deletions tests/test_ptcDataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,12 +119,6 @@ def _checkTypes(self, ptcDataset):
self.assertEqual(ptcDataset.bMatrix[ampName].dtype, np.float64)
self.assertIsInstance(ptcDataset.noiseMatrix[ampName], np.ndarray)
self.assertEqual(ptcDataset.noiseMatrix[ampName].dtype, np.float64)
self.assertIsInstance(ptcDataset.covariancesModelNoB[ampName], np.ndarray)
self.assertEqual(ptcDataset.covariancesModelNoB[ampName].dtype, np.float64)
self.assertIsInstance(ptcDataset.aMatrixNoB[ampName], np.ndarray)
self.assertEqual(ptcDataset.aMatrixNoB[ampName].dtype, np.float64)
self.assertIsInstance(ptcDataset.noiseMatrixNoB[ampName], np.ndarray)
self.assertEqual(ptcDataset.noiseMatrixNoB[ampName].dtype, np.float64)
self.assertIsInstance(ptcDataset.finalVars[ampName], np.ndarray)
self.assertEqual(ptcDataset.finalVars[ampName].dtype, np.float64)
self.assertIsInstance(ptcDataset.finalModelVars[ampName], np.ndarray)
Expand Down Expand Up @@ -273,13 +267,6 @@ def test_ptcDataset(self):
nSideCovMatrixFullCovFit), np.nan)
localDataset.noiseMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,
nSideCovMatrixFullCovFit), np.nan)
localDataset.covariancesModelNoB[ampName] = np.full((nSignalPoints,
nSideCovMatrixFullCovFit,
nSideCovMatrixFullCovFit), np.nan)
localDataset.aMatrixNoB[ampName] = np.full(
(nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), np.nan)
localDataset.noiseMatrixNoB[ampName] = np.full(
(nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), np.nan)

if localDataset.ptcFitType in ['FULLCOVARIANCE', ]:
localDataset.ptcFitPars[ampName] = np.array([np.nan, np.nan])
Expand All @@ -300,13 +287,6 @@ def test_ptcDataset(self):
nSideCovMatrixFullCovFit), 1e-7)
localDataset.noiseMatrix[ampName] = np.full((nSideCovMatrixFullCovFit,
nSideCovMatrixFullCovFit), 3.0)
localDataset.covariancesModelNoB[ampName] = np.full((nSignalPoints,
nSideCovMatrixFullCovFit,
nSideCovMatrixFullCovFit), 15.0)
localDataset.aMatrixNoB[ampName] = np.full(
(nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), 2e-6)
localDataset.noiseMatrixNoB[ampName] = np.full(
(nSideCovMatrixFullCovFit, nSideCovMatrixFullCovFit), 3.0)

for useAuxValues in [False, True]:
if useAuxValues:
Expand Down Expand Up @@ -396,7 +376,6 @@ def test_ptcDatasetSort(self):
localDataset.covariances[ampName] = testCov.copy()
localDataset.covariancesSqrtWeights[ampName] = testCov.copy()
localDataset.covariancesModel[ampName] = testCov.copy()
localDataset.covariancesModelNoB[ampName] = testCov.copy()

localDataset.finalMeans[ampName] = testArr.copy()
localDataset.finalMeans[ampName][~localDataset.expIdMask[ampName]] = np.nan
Expand Down Expand Up @@ -442,7 +421,6 @@ def test_ptcDatasetSort(self):
np.testing.assert_array_equal(localDataset.covariances[ampName], testCovSorted)
np.testing.assert_array_equal(localDataset.covariancesSqrtWeights[ampName], testCovSorted)
np.testing.assert_array_equal(localDataset.covariancesModel[ampName], testCovSorted)
np.testing.assert_array_equal(localDataset.covariancesModelNoB[ampName], testCovSorted)
np.testing.assert_array_equal(localDataset.finalVars[ampName], testArrSortedMasked)
np.testing.assert_array_equal(localDataset.finalModelVars[ampName], testArrSortedMasked)
np.testing.assert_array_equal(localDataset.finalMeans[ampName], testArrSortedMasked)
Expand Down Expand Up @@ -518,7 +496,6 @@ def test_ptcDatasetAppend(self):
np.testing.assert_array_equal(ptc.covariancesSqrtWeights[ampName], testCov)
# These two should have the same shape, but no useful values.
self.assertEqual(ptc.covariancesModel[ampName].shape, testCov.shape)
self.assertEqual(ptc.covariancesModelNoB[ampName].shape, testCov.shape)
self.assertEqual(ptc.finalVars[ampName].shape, testArr.shape)
self.assertEqual(ptc.finalModelVars[ampName].shape, testArr.shape)
self.assertEqual(ptc.finalMeans[ampName].shape, testArr.shape)
Expand Down

0 comments on commit 6cf0085

Please sign in to comment.