diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml index 530acb47ad..f57c373739 100644 --- a/.github/workflows/ci_cron.yml +++ b/.github/workflows/ci_cron.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python: ['3.10', '3.11', '3.12'] + python: ['3.11', '3.12'] toxenv: [test-alldeps, test-numpydev, test-linetoolsdev, test-gingadev, test-astropydev] steps: - name: Check out repository diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 9c2f37ec7d..a5f541cc70 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -17,8 +17,8 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python: ['3.10', '3.11', '3.12'] - toxenv: [test, test-alldeps-cov, test-linetoolsdev, test-gingadev, test-astropydev] + python: ['3.11', '3.12'] + toxenv: [test, test-alldeps-cov, test-numpydev, test-linetoolsdev, test-gingadev, test-astropydev] steps: - name: Check out repository uses: actions/checkout@v3 @@ -32,13 +32,13 @@ jobs: - name: Test with tox run: | tox -e ${{ matrix.python }}-${{ matrix.toxenv }} - - name: Upload coverage to codecov - if: "contains(matrix.toxenv, '-cov')" - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV }} - file: ./coverage.xml - fail_ci_if_error: true +# - name: Upload coverage to codecov +# if: "contains(matrix.toxenv, '-cov')" +# uses: codecov/codecov-action@v3 +# with: +# token: ${{ secrets.CODECOV }} +# file: ./coverage.xml +# fail_ci_if_error: true os-tests: name: Python ${{ matrix.python }} on ${{ matrix.os }} @@ -48,7 +48,7 @@ jobs: fail-fast: false matrix: os: [windows-latest, macos-latest] - python: ['3.10', '3.11', '3.12'] + python: ['3.11', '3.12'] toxenv: [test-alldeps] steps: - name: Check out repository @@ -71,7 +71,7 @@ jobs: - name: Conda environment check uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: '3.12' - name: Install base dependencies run: | python -m pip install --upgrade pip tox @@ -86,7 +86,7 @@ jobs: - name: Python codestyle check uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: '3.12' - name: Install base dependencies run: | python -m pip install --upgrade pip diff --git a/MANIFEST.in b/MANIFEST.in index 8f6c6502c1..c884b33e25 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -17,6 +17,7 @@ global-exclude *.pyc *.o *.so *.DS_Store *.ipynb # update the defined_paths dictionary in pypeit.pypeitdata.PypeItDataPaths! recursive-exclude pypeit/data/arc_lines/reid_arxiv *.fits *.json *.pdf *.tar.gz recursive-exclude pypeit/data/arc_lines/NIST *.ascii +recursive-exclude pypeit/data/pixelflats *.fits.gz recursive-exclude pypeit/data/sensfuncs *.fits recursive-exclude pypeit/data/skisim *.dat recursive-exclude pypeit/data/standards *.gz *.fits *.dat diff --git a/README.rst b/README.rst index 2dc294d37c..66cf1c4249 100644 --- a/README.rst +++ b/README.rst @@ -11,9 +11,6 @@ .. |CITests| image:: https://github.com/pypeit/PypeIt/workflows/CI%20Tests/badge.svg :target: https://github.com/pypeit/PypeIt/actions?query=workflow%3A"CI+Tests" -.. |Coverage| image:: https://codecov.io/gh/PypeIt/pypeit/branch/release/graph/badge.svg - :target: https://codecov.io/gh/PypeIt/pypeit - .. |docs| image:: https://readthedocs.org/projects/pypeit/badge/?version=latest :target: https://pypeit.readthedocs.io/en/latest/ @@ -49,7 +46,7 @@ PypeIt |forks| |stars| |github| |pypi| |pypi_downloads| |License| -|docs| |CITests| |Coverage| +|docs| |CITests| |DOI_latest| |JOSS| |arxiv| diff --git a/bin/pypeit_c_enabled b/bin/pypeit_c_enabled index 8a21ba246b..746e82cdc1 100755 --- a/bin/pypeit_c_enabled +++ b/bin/pypeit_c_enabled @@ -13,11 +13,25 @@ else: print('Successfully imported bspline C utilities.') try: - from pypeit.bspline.setup_package import extra_compile_args + + # Check for whether OpenMP support is enabled, by seeing if the bspline + # extension was compiled with it. + # + # The extension_helpers code that is run to figure out OMP support runs + # multiple tests to determine compiler version, some of which output to stderr. + # To make the output pretty we redirect those to /dev/null (or equivalent) + import os + import sys + devnull_fd = os.open(os.devnull,os.O_WRONLY) + os.dup2(devnull_fd,sys.stderr.fileno()) + + from pypeit.bspline.setup_package import get_extensions + bspline_extension = get_extensions()[0] except: print("Can't check status of OpenMP support") else: - if '-fopenmp' in extra_compile_args: + # Windows uses -openmp, other environments use -fopenmp + if any(['openmp' in arg for arg in bspline_extension.extra_compile_args]): print('OpenMP compiler support detected.') else: print('OpenMP compiler support not detected. Bspline utilities single-threaded.') diff --git a/deprecated/arc_old.py b/deprecated/arc_old.py index edecb94dd2..79b58ded49 100644 --- a/deprecated/arc_old.py +++ b/deprecated/arc_old.py @@ -855,3 +855,53 @@ def saturation_mask(a, satlevel): return mask.astype(int) +def mask_around_peaks(spec, inbpm): + """ + Find peaks in the input spectrum and mask pixels around them. + + All pixels to the left and right of a peak is masked until + a pixel has a lower value than the adjacent pixel. At this + point, we assume that spec has reached the noise level. + + Parameters + ---------- + spec: `numpy.ndarray`_ + Spectrum (1D array) in counts + inbpm: `numpy.ndarray`_ + Input bad pixel mask + + Returns + ------- + outbpm: `numpy.ndarray`_ + Bad pixel mask with pixels around peaks masked + """ + # Find the peak locations + pks = detect_peaks(spec) + + # Initialise some useful variables and the output bpm + xarray = np.arange(spec.size) + specdiff = np.append(np.diff(spec), 0.0) + outbpm = inbpm.copy() + + # Loop over the peaks and mask pixels around them + for i in range(len(pks)): + # Find all pixels to the left of the peak that are above the noise level + wl = np.where((xarray <= pks[i]) & (specdiff > 0.0))[0] + ww = (pks[i]-wl)[::-1] + # Find the first pixel to the left of the peak that is below the noise level + nmask = np.where(np.diff(ww) > 1)[0] + if nmask.size != 0 and nmask[0] > 5: + # Mask all pixels to the left of the peak + mini = max(0,wl.size-nmask[0]-1) + outbpm[wl[mini]:pks[i]] = True + # Find all pixels to the right of the peak that are above the noise level + ww = np.where((xarray >= pks[i]) & (specdiff < 0.0))[0] + # Find the first pixel to the right of the peak that is below the noise level + nmask = np.where(np.diff(ww) > 1)[0] + if nmask.size != 0 and nmask[0] > 5: + # Mask all pixels to the right of the peak + maxi = min(nmask[0], ww.size) + outbpm[pks[i]:ww[maxi]+2] = True + # Return the output bpm + return outbpm + diff --git a/deprecated/datacube.py b/deprecated/datacube.py index abeb9a1e4c..83e339e87b 100644 --- a/deprecated/datacube.py +++ b/deprecated/datacube.py @@ -443,3 +443,71 @@ def make_whitelight_frompixels(all_ra, all_dec, all_wave, all_sci, all_wghts, al whitelight_ivar[:, :, ff] = ivar_img.copy() return whitelight_Imgs, whitelight_ivar, whitelightWCS + +def make_sensfunc(ss_file, senspar, blaze_wave=None, blaze_spline=None, grating_corr=False): + """ + Generate the sensitivity function from a standard star DataCube. + + Args: + ss_file (:obj:`str`): + The relative path and filename of the standard star datacube. It + should be fits format, and for full functionality, should ideally of + the form :class:`~pypeit.coadd3d.DataCube`. + senspar (:class:`~pypeit.par.pypeitpar.SensFuncPar`): + The parameters required for the sensitivity function computation. + blaze_wave (`numpy.ndarray`_, optional): + Wavelength array used to construct blaze_spline + blaze_spline (`scipy.interpolate.interp1d`_, optional): + Spline representation of the reference blaze function (based on the illumflat). + grating_corr (:obj:`bool`, optional): + If a grating correction should be performed, set this variable to True. + + Returns: + `numpy.ndarray`_: A mask of the good sky pixels (True = good) + """ + # TODO :: This routine has not been updated to the new spec1d plan of passing in a sensfunc object + # :: Probably, this routine should be removed and the functionality moved to the sensfunc object + msgs.error("coding error - make_sensfunc is not currently supported. Please contact the developers") + # Check if the standard star datacube exists + if not os.path.exists(ss_file): + msgs.error("Standard cube does not exist:" + msgs.newline() + ss_file) + msgs.info(f"Loading standard star cube: {ss_file:s}") + # Load the standard star cube and retrieve its RA + DEC + stdcube = fits.open(ss_file) + star_ra, star_dec = stdcube[1].header['CRVAL1'], stdcube[1].header['CRVAL2'] + + # Extract a spectrum of the standard star + wave, Nlam_star, Nlam_ivar_star, gpm_star = extract_standard_spec(stdcube) + + # Extract the information about the blaze + if grating_corr: + blaze_wave_curr, blaze_spec_curr = stdcube['BLAZE_WAVE'].data, stdcube['BLAZE_SPEC'].data + blaze_spline_curr = interp1d(blaze_wave_curr, blaze_spec_curr, + kind='linear', bounds_error=False, fill_value="extrapolate") + # Perform a grating correction + grat_corr = correct_grating_shift(wave, blaze_wave_curr, blaze_spline_curr, blaze_wave, blaze_spline) + # Apply the grating correction to the standard star spectrum + Nlam_star /= grat_corr + Nlam_ivar_star *= grat_corr ** 2 + + # Read in some information above the standard star + std_dict = flux_calib.get_standard_spectrum(star_type=senspar['star_type'], + star_mag=senspar['star_mag'], + ra=star_ra, dec=star_dec) + # Calculate the sensitivity curve + # TODO :: This needs to be addressed... unify flux calibration into the main PypeIt routines. + msgs.warn("Datacubes are currently flux-calibrated using the UVIS algorithm... this will be deprecated soon") + zeropoint_data, zeropoint_data_gpm, zeropoint_fit, zeropoint_fit_gpm = \ + flux_calib.fit_zeropoint(wave, Nlam_star, Nlam_ivar_star, gpm_star, std_dict, + mask_hydrogen_lines=senspar['mask_hydrogen_lines'], + mask_helium_lines=senspar['mask_helium_lines'], + hydrogen_mask_wid=senspar['hydrogen_mask_wid'], + nresln=senspar['UVIS']['nresln'], + resolution=senspar['UVIS']['resolution'], + trans_thresh=senspar['UVIS']['trans_thresh'], + polyorder=senspar['polyorder'], + polycorrect=senspar['UVIS']['polycorrect'], + polyfunc=senspar['UVIS']['polyfunc']) + wgd = np.where(zeropoint_fit_gpm) + sens = np.power(10.0, -0.4 * (zeropoint_fit[wgd] - flux_calib.ZP_UNIT_CONST)) / np.square(wave[wgd]) + return interp1d(wave[wgd], sens, kind='linear', bounds_error=False, fill_value="extrapolate") diff --git a/deprecated/old_ech_objfind.py b/deprecated/old_ech_objfind.py new file mode 100644 index 0000000000..e0e7be9309 --- /dev/null +++ b/deprecated/old_ech_objfind.py @@ -0,0 +1,681 @@ + +def orig_ech_objfind(image, ivar, slitmask, slit_left, slit_righ, order_vec, maskslits, det='DET01', + inmask=None, spec_min_max=None, fof_link=1.5, plate_scale=0.2, + std_trace=None, ncoeff=5, npca=None, coeff_npoly=None, max_snr=2.0, min_snr=1.0, + nabove_min_snr=2, pca_explained_var=99.0, box_radius=2.0, fwhm=3.0, + use_user_fwhm=False, maxdev=2.0, hand_extract_dict=None, nperorder=2, + extract_maskwidth=3.0, snr_thresh=10.0, + specobj_dict=None, trim_edg=(5, 5), + show_peaks=False, show_fits=False, show_single_fits=False, + show_trace=False, show_single_trace=False, show_pca=False, + debug_all=False, objfindQA_filename=None): + """ + Object finding routine for Echelle spectrographs. + + This routine: + + #. Runs object finding on each order individually + + #. Links the objects found together using a friends-of-friends algorithm + on fractional order position. + + #. For objects which were only found on some orders, the standard (or + the slit boundaries) are placed at the appropriate fractional + position along the order. + + #. A PCA fit to the traces is performed using the routine above pca_fit + + Args: + image (`numpy.ndarray`_): + (Floating-point) Image to use for object search with shape (nspec, + nspat). The first dimension (nspec) is spectral, and second + dimension (nspat) is spatial. Note this image can either have the + sky background in it, or have already been sky subtracted. Object + finding works best on sky-subtracted images. Ideally, object finding + is run in another routine, global sky-subtraction performed, and + then this code should be run. However, it is also possible to run + this code on non-sky-subtracted images. + ivar (`numpy.ndarray`_): + Floating-point inverse variance image for the input image. Shape + must match ``image``, (nspec, nspat). + slitmask (`numpy.ndarray`_): + Integer image indicating the pixels that belong to each order. + Pixels that are not on an order have value -1, and those that are on + an order have a value equal to the slit number (i.e. 0 to nslits-1 + from left to right on the image). Shape must match ``image``, + (nspec, nspat). + slit_left (`numpy.ndarray`_): + Left boundary of orders to be extracted (given as floating point + pixels). Shape is (nspec, norders), where norders is the total + number of traced echelle orders. + slit_righ (`numpy.ndarray`_): + Right boundary of orders to be extracted (given as floating point + pixels). Shape is (nspec, norders), where norders is the total + number of traced echelle orders. + order_vec (`numpy.ndarray`_): + Vector identifying the Echelle orders for each pair of order edges + found. This is saved to the output :class:`~pypeit.specobj.SpecObj` + objects. If the orders are not known, this can be + ``np.arange(norders)`` (but this is *not* recommended). + maskslits (`numpy.ndarray`_): + Boolean array selecting orders that should be ignored (i.e., good + orders are False, bad orders are True). Shape must be (norders,). + det (:obj:`str`, optional): + The name of the detector containing the object. Only used if + ``specobj_dict`` is None. + inmask (`numpy.ndarray`_, optional): + Good-pixel mask for input image. Must have the same shape as + ``image``. If None, all pixels in ``slitmask`` with non-negative + values are considered good. + spec_min_max (`numpy.ndarray`_, optional): + 2D array defining the minimum and maximum pixel in the spectral + direction with useable data for each order. Shape must be (2, + norders). This should only be used for echelle spectrographs for + which the orders do not entirely cover the detector. PCA tracing + will re-map the traces such that they all have the same length, + compute the PCA, and then re-map the orders back. This improves + performance for echelle spectrographs by removing the nonlinear + shrinking of the orders so that the linear pca operation can better + predict the traces. If None, the minimum and maximum values will be + determined automatically from ``slitmask``. + fof_link (:obj:`float`, optional): + Friends-of-friends linking length in arcseconds used to link + together traces across orders. The routine links together at + the same fractional slit position and links them together + with a friends-of-friends algorithm using this linking + length. + plate_scale (:obj:`float`, `numpy.ndarray`_, optional): + Plate scale in arcsec/pix. This can either be a single float for + every order, or an array with shape (norders,) providing the plate + scale of each order. + std_trace (`numpy.ndarray`_, optional): + Vector with the standard star trace, which is used as a crutch for + tracing. Shape must be (nspec,). If None, the slit boundaries are + used as the crutch. + ncoeff (:obj:`int`, optional): + Order of polynomial fit to traces. + npca (:obj:`int`, optional): + Number of PCA components to keep during PCA decomposition of the + object traces. If None, the number of components set by requiring + the PCA accounts for approximately 99% of the variance. + coeff_npoly (:obj:`int`, optional): + Order of polynomial used for PCA coefficients fitting. If None, + value set automatically, see + :func:`~pypeit.tracepca.pca_trace_object`. + max_snr (:obj:`float`, optional): + For an object to be included in the output object, it must have a + max S/N ratio above this value. + min_snr (:obj:`float`, optional): + For an object to be included in the output object, it must have a + a median S/N ratio above this value for at least + ``nabove_min_snr`` orders (see below). + nabove_min_snr (:obj:`int`, optional): + The required number of orders that an object must have with median + SNR greater than ``min_snr`` in order to be included in the output + object. + pca_explained_var (:obj:`float`, optional): + The percentage (i.e., not the fraction) of the variance in the data + accounted for by the PCA used to truncate the number of PCA + coefficients to keep (see ``npca``). Ignored if ``npca`` is provided + directly; see :func:`~pypeit.tracepca.pca_trace_object`. + box_radius (:obj:`float`, optional): + Box_car extraction radius in arcseconds to assign to each detected + object and to be used later for boxcar extraction. In this method + ``box_radius`` is converted into pixels using ``plate_scale``. + ``box_radius`` is also used for SNR calculation and trimming. + fwhm (:obj:`float`, optional): + Estimated fwhm of the objects in pixels + use_user_fwhm (:obj:`bool`, optional): + If True, ``PypeIt`` will use the spatial profile FWHM input by the + user (see ``fwhm``) rather than determine the spatial FWHM from the + smashed spatial profile via the automated algorithm. + maxdev (:obj:`float`, optional): + Maximum deviation of pixels from polynomial fit to trace + used to reject bad pixels in trace fitting. + hand_extract_dict (:obj:`dict`, optional): + Dictionary with info on manual extraction; see + :class:`~pypeit.manual_extract.ManualExtractionObj`. + nperorder (:obj:`int`, optional): + Maximum number of objects allowed per order. If there are more + detections than this number, the code will select the ``nperorder`` + most significant detections. However, hand apertures will always be + returned and do not count against this budget. + extract_maskwidth (:obj:`float`, optional): + Determines the initial size of the region in units of FWHM that will + be used for local sky subtraction; See :func:`objs_in_slit` and + :func:`~pypeit.core.skysub.local_skysub_extract`. + snr_thresh (:obj:`float`, optional): + SNR threshold for finding objects + specobj_dict (:obj:`dict`, optional): + Dictionary containing meta-data for the objects that will be + propagated into the :class:`~pypeit.specobj.SpecObj` objects. The + expected components are: + + - SLITID: The slit ID number + - DET: The detector identifier + - OBJTYPE: The object type + - PYPELINE: The class of pipeline algorithms applied + + If None, the dictionary is filled with the following placeholders:: + + specobj_dict = {'SLITID': 999, 'DET': 'DET01', + 'OBJTYPE': 'unknown', 'PYPELINE': 'unknown'} + + trim_edg (:obj:`tuple`, optional): + A two-tuple of integers or floats used to ignore objects within this + many pixels of the left and right slit boundaries, respectively. + show_peaks (:obj:`bool`, optional): + Plot the QA of the object peak finding in each order. + show_fits (:obj:`bool`, optional): + Plot trace fitting for final fits using PCA as crutch. + show_single_fits (:obj:`bool`, optional): + Plot trace fitting for single order fits. + show_trace (:obj:`bool`, optional): + Display the object traces on top of the image. + show_single_trace (:obj:`bool`, optional): + Display the object traces on top of the single order. + show_pca (:obj:`bool`, optional): + Display debugging plots for the PCA decomposition. + debug_all (:obj:`bool`, optional): + Show all the debugging plots. If True, this also overrides any + provided values for ``show_peaks``, ``show_trace``, and + ``show_pca``, setting them to True. + objfindQA_filename (:obj:`str`, optional): + Full path (directory and filename) for the object profile QA plot. + If None, not plot is produced and saved. + + Returns: + :class:`~pypeit.specobjs.SpecObjs`: Object containing the objects + detected. + """ + raise DeprecationWarning + msgs.error("This ginormous method as been Deprecated") + + # debug_all=True + if debug_all: + show_peaks = True + # show_fits = True + # show_single_fits = True + show_trace = True + show_pca = True + # show_single_trace = True + # TODO: This isn't used, right? + debug = True + + if specobj_dict is None: + specobj_dict = {'SLITID': 999, 'ECH_ORDERINDX': 999, + 'DET': det, 'OBJTYPE': 'unknown', 'PYPELINE': 'Echelle'} + + # TODO Update FOF algorithm here with the one from scikit-learn. + + allmask = slitmask > -1 + if inmask is None: + inmask = allmask + + nspec, nspat = image.shape + norders = len(order_vec) + + # Find the spat IDs + gdslit_spat = np.unique(slitmask[slitmask >= 0]).astype(int) # Unique sorts + if gdslit_spat.size != np.sum(np.invert(maskslits)): + msgs.error('Masking of slitmask not in sync with that of maskslits. This is a bug') + # msgs.error('There is a mismatch between the number of valid orders found by PypeIt and ' + # 'the number expected for this spectrograph. Unable to continue. Please ' + # 'submit an issue on Github: https://github.com/pypeit/PypeIt/issues .') + + if spec_min_max is None: + spec_min_max = np.zeros((2, norders), dtype=int) + for iord in range(norders): + ispec, ispat = np.where(slitmask == gdslit_spat[iord]) + spec_min_max[:, iord] = ispec.min(), ispec.max() + + # Setup the plate scale + if isinstance(plate_scale, (float, int)): + plate_scale_ord = np.full(norders, plate_scale) + elif isinstance(plate_scale, (np.ndarray, list, tuple)): + if len(plate_scale) == norders: + plate_scale_ord = plate_scale + elif len(plate_scale) == 1: + plate_scale_ord = np.full(norders, plate_scale[0]) + else: + msgs.error('Invalid size for plate_scale. It must either have one element or norders elements') + else: + msgs.error('Invalid type for plate scale') + + specmid = nspec // 2 + spec_vec = np.arange(nspec) + slit_width = slit_righ - slit_left + slit_spec_pos = nspec / 2.0 + + # TODO JFH This hand apertures in echelle needs to be completely refactored. + # Hand prep + # Determine the location of the source on *all* of the orders + if hand_extract_dict is not None: + f_spats = [] + for ss, spat, spec in zip(range(len(hand_extract_dict['spec'])), + hand_extract_dict['spat'], + hand_extract_dict['spec']): + # Find the input slit + ispec = int(np.clip(np.round(spec), 0, nspec - 1)) + ispat = int(np.clip(np.round(spat), 0, nspat - 1)) + slit = slitmask[ispec, ispat] + if slit == -1: + msgs.error('You are requesting a manual extraction at a position ' + + f'(spat, spec)={spat, spec} that is not on one of the echelle orders. Check your pypeit file.') + # Fractions + iord_hand = gdslit_spat.tolist().index(slit) + f_spat = (spat - slit_left[ispec, iord_hand]) / ( + slit_righ[ispec, iord_hand] - slit_left[ispec, iord_hand]) + f_spats.append(f_spat) + + # Loop over orders and find objects + sobjs = specobjs.SpecObjs() + # TODO: replace orderindx with the true order number here? Maybe not. Clean + # up SLITID and orderindx! + gdorders = np.arange(norders)[np.invert(maskslits)] + for iord in gdorders: # range(norders): + qa_title = 'Finding objects on order # {:d}'.format(order_vec[iord]) + msgs.info(qa_title) + thisslit_gpm = slitmask == gdslit_spat[iord] + inmask_iord = inmask & thisslit_gpm + specobj_dict['SLITID'] = gdslit_spat[iord] + specobj_dict['ECH_ORDERINDX'] = iord + specobj_dict['ECH_ORDER'] = order_vec[iord] + std_in = None if std_trace is None else std_trace[:, iord] + + # TODO JFH: Fix this. The way this code works, you should only need to create a single hand object, + # not one at every location on the order + if hand_extract_dict is not None: + new_hand_extract_dict = copy.deepcopy(hand_extract_dict) + for ss, spat, spec, f_spat in zip(range(len(hand_extract_dict['spec'])), + hand_extract_dict['spat'], + hand_extract_dict['spec'], f_spats): + ispec = int(spec) + new_hand_extract_dict['spec'][ss] = ispec + new_hand_extract_dict['spat'][ss] = slit_left[ispec, iord] + f_spat * ( + slit_righ[ispec, iord] - slit_left[ispec, iord]) + else: + new_hand_extract_dict = None + + # Get SLTIORD_ID for the objfind QA + ech_objfindQA_filename = objfindQA_filename.replace('S0999', 'S{:04d}'.format(order_vec[iord])) \ + if objfindQA_filename is not None else None + # Run + sobjs_slit = \ + objs_in_slit(image, ivar, thisslit_gpm, slit_left[:, iord], slit_righ[:, iord], + spec_min_max=spec_min_max[:, iord], + inmask=inmask_iord, std_trace=std_in, ncoeff=ncoeff, fwhm=fwhm, use_user_fwhm=use_user_fwhm, + maxdev=maxdev, + hand_extract_dict=new_hand_extract_dict, nperslit=nperorder, + extract_maskwidth=extract_maskwidth, + snr_thresh=snr_thresh, trim_edg=trim_edg, boxcar_rad=box_radius / plate_scale_ord[iord], + show_peaks=show_peaks, show_fits=show_single_fits, + show_trace=show_single_trace, qa_title=qa_title, specobj_dict=specobj_dict, + objfindQA_filename=ech_objfindQA_filename) + sobjs.add_sobj(sobjs_slit) + + nfound = len(sobjs) + + if nfound == 0: + msgs.warn('No objects found') + return sobjs + + FOF_frac = fof_link / (np.median(np.median(slit_width, axis=0) * plate_scale_ord)) + # Run the FOF. We use fake coordinates + fracpos = sobjs.SPAT_FRACPOS + ra_fake = fracpos / 1000.0 # Divide all angles by 1000 to make geometry euclidian + dec_fake = np.zeros_like(fracpos) + if nfound > 1: + inobj_id, multobj_id, firstobj_id, nextobj_id \ + = pydl.spheregroup(ra_fake, dec_fake, FOF_frac / 1000.0) + # TODO spheregroup returns zero based indices but we use one based. We should probably add 1 to inobj_id here, + # i.e. obj_id_init = inobj_id + 1 + obj_id_init = inobj_id.copy() + elif nfound == 1: + obj_id_init = np.zeros(1, dtype='int') + + uni_obj_id_init, uni_ind_init = np.unique(obj_id_init, return_index=True) + + # Now loop over the unique objects and check that there is only one object per order. If FOF + # grouped > 1 objects on the same order, then this will be popped out as its own unique object + obj_id = obj_id_init.copy() + nobj_init = len(uni_obj_id_init) + for iobj in range(nobj_init): + for iord in range(norders): + on_order = (obj_id_init == uni_obj_id_init[iobj]) & (sobjs.ECH_ORDERINDX == iord) + if (np.sum(on_order) > 1): + msgs.warn('Found multiple objects in a FOF group on order iord={:d}'.format(iord) + msgs.newline() + + 'Spawning new objects to maintain a single object per order.') + off_order = (obj_id_init == uni_obj_id_init[iobj]) & (sobjs.ECH_ORDERINDX != iord) + ind = np.where(on_order)[0] + if np.any(off_order): + # Keep the closest object to the location of the rest of the group (on other orders) + # as corresponding to this obj_id, and spawn new obj_ids for the others. + frac_mean = np.mean(fracpos[off_order]) + min_dist_ind = np.argmin(np.abs(fracpos[ind] - frac_mean)) + else: + # If there are no other objects with this obj_id to compare to, then we simply have multiple + # objects grouped together on the same order, so just spawn new object IDs for them to maintain + # one obj_id per order + min_dist_ind = 0 + ind_rest = np.setdiff1d(ind, ind[min_dist_ind]) + # JFH OLD LINE with bug + # obj_id[ind_rest] = (np.arange(len(ind_rest)) + 1) + obj_id_init.max() + obj_id[ind_rest] = (np.arange(len(ind_rest)) + 1) + obj_id.max() + + uni_obj_id, uni_ind = np.unique(obj_id, return_index=True) + nobj = len(uni_obj_id) + msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects') + + gfrac = np.zeros(nfound) + for jj in range(nobj): + this_obj_id = obj_id == uni_obj_id[jj] + gfrac[this_obj_id] = np.median(fracpos[this_obj_id]) + + uni_frac = gfrac[uni_ind] + + # Sort with respect to fractional slit location to guarantee that we have a similarly sorted list of objects later + isort_frac = uni_frac.argsort(kind='stable') + uni_obj_id = uni_obj_id[isort_frac] + uni_frac = uni_frac[isort_frac] + + sobjs_align = sobjs.copy() + # Loop over the orders and assign each specobj a fractional position and a obj_id number + for iobj in range(nobj): + for iord in range(norders): + on_order = (obj_id == uni_obj_id[iobj]) & (sobjs_align.ECH_ORDERINDX == iord) + sobjs_align[on_order].ECH_FRACPOS = uni_frac[iobj] + sobjs_align[on_order].ECH_OBJID = uni_obj_id[iobj] + sobjs_align[on_order].OBJID = uni_obj_id[iobj] + sobjs_align[on_order].ech_frac_was_fit = False + + # Reset names (just in case) + sobjs_align.set_names() + # Now loop over objects and fill in the missing objects and their traces. We will fit the fraction slit position of + # the good orders where an object was found and use that fit to predict the fractional slit position on the bad orders + # where no object was found + for iobj in range(nobj): + # Grab all the members of this obj_id from the object list + indx_obj_id = sobjs_align.ECH_OBJID == uni_obj_id[iobj] + nthisobj_id = np.sum(indx_obj_id) + # Perform the fit if this objects shows up on more than three orders + if (nthisobj_id > 3) and (nthisobj_id < norders): + thisorderindx = sobjs_align[indx_obj_id].ECH_ORDERINDX + goodorder = np.zeros(norders, dtype=bool) + goodorder[thisorderindx] = True + badorder = np.invert(goodorder) + xcen_good = (sobjs_align[indx_obj_id].TRACE_SPAT).T + slit_frac_good = (xcen_good - slit_left[:, goodorder]) / slit_width[:, goodorder] + # Fractional slit position averaged across the spectral direction for each order + frac_mean_good = np.mean(slit_frac_good, 0) + # Perform a linear fit to fractional slit position + # TODO Do this as a S/N weighted fit similar to what is now in the pca_trace algorithm? + # msk_frac, poly_coeff_frac = fitting.robust_fit(order_vec[goodorder], frac_mean_good, 1, + pypeitFit = fitting.robust_fit(order_vec[goodorder], frac_mean_good, 1, + function='polynomial', maxiter=20, lower=2, upper=2, + use_mad=True, sticky=False, + minx=order_vec.min(), maxx=order_vec.max()) + frac_mean_new = np.zeros(norders) + frac_mean_new[badorder] = pypeitFit.eval( + order_vec[badorder]) # , minx = order_vec.min(),maxx=order_vec.max()) + frac_mean_new[goodorder] = frac_mean_good + # TODO This QA needs some work + if show_pca: + frac_mean_fit = pypeitFit.eval(order_vec) + plt.plot(order_vec[goodorder][pypeitFit.bool_gpm], frac_mean_new[goodorder][pypeitFit.bool_gpm], 'ko', + mfc='k', markersize=8.0, label='Good Orders Kept') + plt.plot(order_vec[goodorder][np.invert(pypeitFit.bool_gpm)], + frac_mean_new[goodorder][np.invert(pypeitFit.bool_gpm)], 'ro', mfc='k', markersize=8.0, + label='Good Orders Rejected') + plt.plot(order_vec[badorder], frac_mean_new[badorder], 'ko', mfc='None', markersize=8.0, + label='Predicted Bad Orders') + plt.plot(order_vec, frac_mean_new, '+', color='cyan', markersize=12.0, label='Final Order Fraction') + plt.plot(order_vec, frac_mean_fit, 'r-', label='Fractional Order Position Fit') + plt.xlabel('Order Index', fontsize=14) + plt.ylabel('Fractional Slit Position', fontsize=14) + plt.title('Fractional Slit Position Fit') + plt.legend() + plt.show() + else: + frac_mean_new = np.full(norders, uni_frac[iobj]) + + # Now loop over the orders and add objects on the ordrers for which the current object was not found + for iord in range(norders): + # Is the current object detected on this order? + on_order = (sobjs_align.ECH_OBJID == uni_obj_id[iobj]) & (sobjs_align.ECH_ORDERINDX == iord) + num_on_order = np.sum(on_order) + if num_on_order == 0: + # If it is not, create a new sobjs and add to sobjs_align and assign required tags + thisobj = specobj.SpecObj('Echelle', sobjs_align[0].DET, + OBJTYPE=sobjs_align[0].OBJTYPE, + ECH_ORDERINDX=iord, + ECH_ORDER=order_vec[iord]) + # thisobj.ECH_ORDERINDX = iord + # thisobj.ech_order = order_vec[iord] + thisobj.SPAT_FRACPOS = uni_frac[iobj] + # Assign traces using the fractional position fit above + if std_trace is not None: + x_trace = np.interp(slit_spec_pos, spec_vec, std_trace[:, iord]) + shift = np.interp(slit_spec_pos, spec_vec, + slit_left[:, iord] + slit_width[:, iord] * frac_mean_new[iord]) - x_trace + thisobj.TRACE_SPAT = std_trace[:, iord] + shift + else: + thisobj.TRACE_SPAT = slit_left[:, iord] + slit_width[:, iord] * frac_mean_new[iord] # new trace + thisobj.trace_spec = spec_vec + thisobj.SPAT_PIXPOS = thisobj.TRACE_SPAT[specmid] + # Use the real detections of this objects for the FWHM + this_obj_id = obj_id == uni_obj_id[iobj] + # Assign to the fwhm of the nearest detected order + imin = np.argmin(np.abs(sobjs_align[this_obj_id].ECH_ORDERINDX - iord)) + thisobj.FWHM = sobjs_align[imin].FWHM + thisobj.maskwidth = sobjs_align[imin].maskwidth + thisobj.smash_peakflux = sobjs_align[imin].smash_peakflux + thisobj.smash_snr = sobjs_align[imin].smash_snr + thisobj.BOX_RADIUS = sobjs_align[imin].BOX_RADIUS + thisobj.ECH_FRACPOS = uni_frac[iobj] + thisobj.ECH_OBJID = uni_obj_id[iobj] + thisobj.OBJID = uni_obj_id[iobj] + thisobj.SLITID = gdslit_spat[iord] + thisobj.ech_frac_was_fit = True + thisobj.set_name() + sobjs_align.add_sobj(thisobj) + obj_id = np.append(obj_id, uni_obj_id[iobj]) + gfrac = np.append(gfrac, uni_frac[iobj]) + elif num_on_order == 1: + # Object is already on this order so no need to do anything + pass + elif num_on_order > 1: + msgs.error( + 'Problem in echelle object finding. The same objid={:d} appears {:d} times on echelle orderindx ={:d}' + ' even after duplicate obj_ids the orders were removed. ' + 'Report this bug to PypeIt developers'.format(uni_obj_id[iobj], num_on_order, iord)) + + # Loop over the objects and perform a quick and dirty extraction to assess S/N. + varimg = utils.calc_ivar(ivar) + flux_box = np.zeros((nspec, norders, nobj)) + ivar_box = np.zeros((nspec, norders, nobj)) + mask_box = np.zeros((nspec, norders, nobj)) + SNR_arr = np.zeros((norders, nobj)) + slitfracpos_arr = np.zeros((norders, nobj)) + for iobj in range(nobj): + for iord in range(norders): + iorder_vec = order_vec[iord] + indx = sobjs_align.slitorder_objid_indices(iorder_vec, uni_obj_id[iobj]) + # indx = (sobjs_align.ECH_OBJID == uni_obj_id[iobj]) & (sobjs_align.ECH_ORDERINDX == iord) + # spec = sobjs_align[indx][0] + inmask_iord = inmask & (slitmask == gdslit_spat[iord]) + # TODO make the snippet below its own function quick_extraction() + box_rad_pix = box_radius / plate_scale_ord[iord] + + # TODO -- We probably shouldn't be operating on a SpecObjs but instead a SpecObj + flux_tmp = moment1d(image * inmask_iord, sobjs_align[indx][0].TRACE_SPAT, 2 * box_rad_pix, + row=sobjs_align[indx][0].trace_spec)[0] + var_tmp = moment1d(varimg * inmask_iord, sobjs_align[indx][0].TRACE_SPAT, 2 * box_rad_pix, + row=sobjs_align[indx][0].trace_spec)[0] + ivar_tmp = utils.calc_ivar(var_tmp) + pixtot = moment1d(ivar * 0 + 1.0, sobjs_align[indx][0].TRACE_SPAT, 2 * box_rad_pix, + row=sobjs_align[indx][0].trace_spec)[0] + mask_tmp = moment1d(ivar * inmask_iord == 0.0, sobjs_align[indx][0].TRACE_SPAT, 2 * box_rad_pix, + row=sobjs_align[indx][0].trace_spec)[0] != pixtot + + flux_box[:, iord, iobj] = flux_tmp * mask_tmp + ivar_box[:, iord, iobj] = np.fmax(ivar_tmp * mask_tmp, 0.0) + mask_box[:, iord, iobj] = mask_tmp + mean, med_sn, stddev = astropy.stats.sigma_clipped_stats( + flux_box[mask_tmp, iord, iobj] * np.sqrt(ivar_box[mask_tmp, iord, iobj]), + sigma_lower=5.0, sigma_upper=5.0 + ) + # ToDO assign this to sobjs_align for use in the extraction + SNR_arr[iord, iobj] = med_sn + sobjs_align[indx][0].ech_snr = med_sn + # For hand extractions + slitfracpos_arr[iord, iobj] = sobjs_align[indx][0].SPAT_FRACPOS + + # Purge objects with low SNR that don't show up in enough orders, sort the list of objects with respect to obj_id + # and orderindx + keep_obj = np.zeros(nobj, dtype=bool) + sobjs_trim = specobjs.SpecObjs() + # objids are 1 based so that we can easily asign the negative to negative objects + iobj_keep = 1 + iobj_keep_not_hand = 1 + + # TODO JFH: Fix this ugly and dangerous hack that was added to accomodate hand apertures + hand_frac = [-1000] if hand_extract_dict is None else [int(np.round(ispat * 1000)) for ispat in f_spats] + + ## Loop over objects from highest SNR to lowest SNR. Apply the S/N constraints. Once we hit the maximum number + # objects requested exit, except keep the hand apertures that were requested. + isort_SNR_max = np.argsort(np.median(SNR_arr, axis=0), kind='stable')[::-1] + for iobj in isort_SNR_max: + hand_ap_flag = int(np.round(slitfracpos_arr[0, iobj] * 1000)) in hand_frac + SNR_constraint = (SNR_arr[:, iobj].max() > max_snr) or (np.sum(SNR_arr[:, iobj] > min_snr) >= nabove_min_snr) + nperorder_constraint = (iobj_keep - 1) < nperorder + if (SNR_constraint and nperorder_constraint) or hand_ap_flag: + keep_obj[iobj] = True + ikeep = sobjs_align.ECH_OBJID == uni_obj_id[iobj] + sobjs_keep = sobjs_align[ikeep].copy() + sobjs_keep.ECH_OBJID = iobj_keep + sobjs_keep.OBJID = iobj_keep + # for spec in sobjs_keep: + # spec.ECH_OBJID = iobj_keep + # #spec.OBJID = iobj_keep + sobjs_trim.add_sobj(sobjs_keep[np.argsort(sobjs_keep.ECH_ORDERINDX, kind='stable')]) + iobj_keep += 1 + if not hand_ap_flag: + iobj_keep_not_hand += 1 + else: + if not nperorder_constraint: + msgs.info('Purging object #{:d}'.format(iobj) + + ' since there are already {:d} objects automatically identified ' + 'and you set nperorder={:d}'.format(iobj_keep_not_hand - 1, nperorder)) + else: + msgs.info('Purging object #{:d}'.format( + iobj) + ' which does not satisfy max_snr > {:5.2f} OR min_snr > {:5.2f}'.format(max_snr, min_snr) + + ' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) + ' orders') + + nobj_trim = np.sum(keep_obj) + + if nobj_trim == 0: + msgs.warn('No objects found') + sobjs_final = specobjs.SpecObjs() + return sobjs_final + + # TODO JFH: We need to think about how to implement returning a maximum number of objects, where the objects + # returned are the highest S/N ones. It is a bit complicated with regards to the individual object finding and then + # the linking that is performed above, and also making sure the hand apertures don't get removed. + SNR_arr_trim = SNR_arr[:, keep_obj] + + sobjs_final = sobjs_trim.copy() + # Loop over the objects one by one and adjust/predict the traces + pca_fits = np.zeros((nspec, norders, nobj_trim)) + + # Create the trc_inmask for iterative fitting below + trc_inmask = np.zeros((nspec, norders), dtype=bool) + for iord in range(norders): + trc_inmask[:, iord] = (spec_vec >= spec_min_max[0, iord]) & (spec_vec <= spec_min_max[1, iord]) + + for iobj in range(nobj_trim): + indx_obj_id = sobjs_final.ECH_OBJID == (iobj + 1) + # PCA predict all the orders now (where we have used the standard or slit boundary for the bad orders above) + msgs.info('Fitting echelle object finding PCA for object {:d}/{:d} with median SNR = {:5.3f}'.format( + iobj + 1, nobj_trim, np.median(sobjs_final[indx_obj_id].ech_snr))) + pca_fits[:, :, iobj] \ + = tracepca.pca_trace_object(sobjs_final[indx_obj_id].TRACE_SPAT.T, + order=coeff_npoly, npca=npca, + pca_explained_var=pca_explained_var, + trace_wgt=np.fmax(sobjs_final[indx_obj_id].ech_snr, 1.0) ** 2, + debug=show_pca) + + # Trial and error shows weighting by S/N instead of S/N^2 performs better + # JXP -- Updated to now be S/N**2, i.e. inverse variance, with fitting fit + + # Perform iterative flux weighted centroiding using new PCA predictions + xinit_fweight = pca_fits[:, :, iobj].copy() + inmask_now = inmask & allmask + xfit_fweight = fit_trace(image, xinit_fweight, ncoeff, bpm=np.invert(inmask_now), + trace_bpm=np.invert(trc_inmask), fwhm=fwhm, maxdev=maxdev, + debug=show_fits)[0] + + # Perform iterative Gaussian weighted centroiding + xinit_gweight = xfit_fweight.copy() + xfit_gweight = fit_trace(image, xinit_gweight, ncoeff, bpm=np.invert(inmask_now), + trace_bpm=np.invert(trc_inmask), weighting='gaussian', fwhm=fwhm, + maxdev=maxdev, debug=show_fits)[0] + + # TODO Assign the new traces. Only assign the orders that were not orginally detected and traced. If this works + # well, we will avoid doing all of the iter_tracefits above to make the code faster. + for iord, spec in enumerate(sobjs_final[indx_obj_id]): + # JFH added the condition on ech_frac_was_fit with S/N cut on 7-7-19. + # TODO is this robust against half the order being masked? + if spec.ech_frac_was_fit & (spec.ech_snr > 1.0): + spec.TRACE_SPAT = xfit_gweight[:, iord] + spec.SPAT_PIXPOS = spec.TRACE_SPAT[specmid] + + # TODO Put in some criterion here that does not let the fractional position change too much during the iterative + # tracefitting. The problem is spurious apertures identified on one slit can be pulled over to the center of flux + # resulting in a bunch of objects landing on top of each other. + + # Set the IDs + sobjs_final[:].ECH_ORDER = order_vec[sobjs_final[:].ECH_ORDERINDX] + # for spec in sobjs_final: + # spec.ech_order = order_vec[spec.ECH_ORDERINDX] + sobjs_final.set_names() + + if show_trace: + viewer, ch = display.show_image(image * allmask) + + for spec in sobjs_trim: + color = 'red' if spec.ech_frac_was_fit else 'magenta' + ## Showing the final flux weighted centroiding from PCA predictions + display.show_trace(viewer, ch, spec.TRACE_SPAT, spec.NAME, color=color) + + for iobj in range(nobj_trim): + for iord in range(norders): + ## Showing PCA predicted locations before recomputing flux/gaussian weighted centroiding + display.show_trace(viewer, ch, pca_fits[:, iord, iobj], str(uni_frac[iobj]), color='yellow') + ## Showing the final traces from this routine + display.show_trace(viewer, ch, sobjs_final.TRACE_SPAT[iord].T, sobjs_final.NAME, color='cyan') + + # Labels for the points + text_final = [dict(type='text', args=(nspat / 2 - 40, nspec / 2, 'final trace'), + kwargs=dict(color='cyan', fontsize=20))] + + text_pca = [dict(type='text', args=(nspat / 2 - 40, nspec / 2 - 30, 'PCA fit'), + kwargs=dict(color='yellow', fontsize=20))] + + text_fit = [dict(type='text', args=(nspat / 2 - 40, nspec / 2 - 60, 'predicted'), + kwargs=dict(color='red', fontsize=20))] + + text_notfit = [dict(type='text', args=(nspat / 2 - 40, nspec / 2 - 90, 'originally found'), + kwargs=dict(color='magenta', fontsize=20))] + + canvas = viewer.canvas(ch._chname) + canvas_list = text_final + text_pca + text_fit + text_notfit + canvas.add('constructedcanvas', canvas_list) + # TODO two things need to be debugged. 1) For objects which were found and traced, i don't think we should be updating the tracing with + # the PCA. This just adds a failutre mode. 2) The PCA fit is going wild for X-shooter. Debug that. + # Vette + for sobj in sobjs_final: + if not sobj.ready_for_extraction(): + msgs.error("Bad SpecObj. Can't proceed") + + return sobjs_final \ No newline at end of file diff --git a/deprecated/sensfunc.py b/deprecated/sensfunc.py new file mode 100644 index 0000000000..08f162231c --- /dev/null +++ b/deprecated/sensfunc.py @@ -0,0 +1,56 @@ +def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, + min_blaze_value=1e-3, debug=False): + """ + Compute the blaze function from a flat field image. + + Args: + wave (`numpy.ndarray`_): + Wavelength array. Shape = (nspec, norddet) + trace_spec (`numpy.ndarray`_): + Spectral pixels for the trace of the spectrum. Shape = (nspec, norddet) + trace_spat (`numpy.ndarray`_): + Spatial pixels for the trace of the spectrum. Shape = (nspec, norddet) + flatfile (:obj:`str`): + Filename for the flat field calibration image + box_radius (:obj:`float`, optional): + Radius of the boxcar extraction region used to extract the blaze function in pixels + min_blaze_value (:obj:`float`, optional): + Minimum value of the blaze function. Values below this are clipped and set to this value. Default=1e-3 + debug (:obj:`bool`, optional): + Show plots useful for debugging. Default=False + + Returns: + `numpy.ndarray`_: The log10 blaze function. Shape = (nspec, norddet) + if norddet > 1, else shape = (nspec,) + """ + flatImages = flatfield.FlatImages.from_file(flatfile, chk_version=self.chk_version) + + pixelflat_raw = flatImages.pixelflat_raw + pixelflat_norm = flatImages.pixelflat_norm + pixelflat_proc, flat_bpm = flat.flatfield(pixelflat_raw, pixelflat_norm) + + flux_box = moment1d(pixelflat_proc * np.logical_not(flat_bpm), trace_spat, 2 * box_radius, row=trace_spec)[0] + + pixtot = moment1d(pixelflat_proc * 0 + 1.0, trace_spat, 2 * box_radius, row=trace_spec)[0] + pixmsk = moment1d(flat_bpm, trace_spat, 2 * box_radius, row=trace_spec)[0] + + mask_box = (pixmsk != pixtot) & np.isfinite(wave) & (wave > 0.0) + + # TODO This is ugly and redundant with spec_atleast_2d, but the order of operations compels me to do it this way + blaze_function = (np.clip(flux_box * mask_box, 1e-3, 1e9)).reshape(-1, 1) \ + if flux_box.ndim == 1 else flux_box * mask_box + wave_debug = wave.reshape(-1, 1) if wave.ndim == 1 else wave + log10_blaze_function = np.zeros_like(blaze_function) + norddet = log10_blaze_function.shape[1] + for iorddet in range(norddet): + blaze_function_smooth = utils.fast_running_median(blaze_function[:, iorddet], 5) + blaze_function_norm = blaze_function_smooth / blaze_function_smooth.max() + log10_blaze_function[:, iorddet] = np.log10(np.clip(blaze_function_norm, min_blaze_value, None)) + if debug: + plt.plot(wave_debug[:, iorddet], log10_blaze_function[:, iorddet]) + if debug: + plt.show() + + # TODO It would probably better to just return an array of shape (nspec, norddet) even if norddet = 1, i.e. + # to get rid of this .squeeze() + return log10_blaze_function.squeeze() diff --git a/doc/api/pypeit.scripts.chk_flexure.rst b/doc/api/pypeit.scripts.chk_flexure.rst new file mode 100644 index 0000000000..90d71dfdb8 --- /dev/null +++ b/doc/api/pypeit.scripts.chk_flexure.rst @@ -0,0 +1,8 @@ +pypeit.scripts.chk\_flexure module +================================== + +.. automodule:: pypeit.scripts.chk_flexure + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.scripts.extract_datacube.rst b/doc/api/pypeit.scripts.extract_datacube.rst new file mode 100644 index 0000000000..49f3a9d13e --- /dev/null +++ b/doc/api/pypeit.scripts.extract_datacube.rst @@ -0,0 +1,8 @@ +pypeit.scripts.extract\_datacube module +======================================= + +.. automodule:: pypeit.scripts.extract_datacube + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.scripts.rst b/doc/api/pypeit.scripts.rst index 38fbcf5584..e9d732cc80 100644 --- a/doc/api/pypeit.scripts.rst +++ b/doc/api/pypeit.scripts.rst @@ -12,6 +12,7 @@ Submodules pypeit.scripts.chk_alignments pypeit.scripts.chk_edges pypeit.scripts.chk_flats + pypeit.scripts.chk_flexure pypeit.scripts.chk_for_calibs pypeit.scripts.chk_noise_1dspec pypeit.scripts.chk_noise_2dspec @@ -26,6 +27,7 @@ Submodules pypeit.scripts.compare_sky pypeit.scripts.compile_wvarxiv pypeit.scripts.edge_inspector + pypeit.scripts.extract_datacube pypeit.scripts.flux_calib pypeit.scripts.flux_setup pypeit.scripts.identify @@ -50,6 +52,7 @@ Submodules pypeit.scripts.show_1dspec pypeit.scripts.show_2dspec pypeit.scripts.show_arxiv + pypeit.scripts.show_pixflat pypeit.scripts.show_wvcalib pypeit.scripts.skysub_regions pypeit.scripts.tellfit diff --git a/doc/api/pypeit.scripts.show_pixflat.rst b/doc/api/pypeit.scripts.show_pixflat.rst new file mode 100644 index 0000000000..b66038acae --- /dev/null +++ b/doc/api/pypeit.scripts.show_pixflat.rst @@ -0,0 +1,8 @@ +pypeit.scripts.show\_pixflat module +=================================== + +.. automodule:: pypeit.scripts.show_pixflat + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.spectrographs.aat_uhrf.rst b/doc/api/pypeit.spectrographs.aat_uhrf.rst new file mode 100644 index 0000000000..2f4e657404 --- /dev/null +++ b/doc/api/pypeit.spectrographs.aat_uhrf.rst @@ -0,0 +1,8 @@ +pypeit.spectrographs.aat\_uhrf module +===================================== + +.. automodule:: pypeit.spectrographs.aat_uhrf + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.spectrographs.rst b/doc/api/pypeit.spectrographs.rst index c53968bcbb..239d591126 100644 --- a/doc/api/pypeit.spectrographs.rst +++ b/doc/api/pypeit.spectrographs.rst @@ -7,6 +7,7 @@ Submodules .. toctree:: :maxdepth: 4 + pypeit.spectrographs.aat_uhrf pypeit.spectrographs.bok_bc pypeit.spectrographs.gemini_flamingos pypeit.spectrographs.gemini_gmos diff --git a/doc/calibrations/calibrations.rst b/doc/calibrations/calibrations.rst index 94c5285fd8..33b40837ad 100644 --- a/doc/calibrations/calibrations.rst +++ b/doc/calibrations/calibrations.rst @@ -127,6 +127,7 @@ The primary calibration procedures are, in the order they're performed: flexure wave_calib Slit Alignment (IFU only) + non-linear correction flat_fielding scattlight diff --git a/doc/calibrations/flat.rst b/doc/calibrations/flat.rst index d224400051..7bc244b8a8 100644 --- a/doc/calibrations/flat.rst +++ b/doc/calibrations/flat.rst @@ -51,7 +51,7 @@ spectrograph that we have not included example screen-shots. Raw Flat --------- +++++++++ This is the processed and combined ``pixelflat`` image. Despite the name, it is not completely raw. @@ -60,7 +60,7 @@ This image should look like any one of your input ``pixelflat`` images. Pixel Flat ----------- +++++++++++ This is the normalized to unity image which is used to correct for pixel-to-pixel variations across the detector. @@ -77,17 +77,32 @@ true if there is limited flux at these ends (e.g. the data goes below the atmospheric cutoff). Illumination Flat ------------------ ++++++++++++++++++ This image should also have most values near unity, but there will be vertical coherence. And the edges (left/right) may fall off well below unity. Flat Model ----------- +++++++++++ This image should largely resemble the `Raw Flat`_. +pypeit_show_pixflat +------------------- + +In addition to ``pypeit_chk_flats``, if a custom pixel flat is provided by the user, +another script ``pypeit_show_pixflat`` is available to inspect it. The script is called as follows: + +.. code-block:: console + + pypeit_show_pixflat PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz + +The script usage can be displayed by calling the script with the ``-h`` option: + +.. include:: ../help/pypeit_show_pixflat.rst + + Troubleshooting =============== diff --git a/doc/calibrations/flat_fielding.rst b/doc/calibrations/flat_fielding.rst index 904239b05b..134714c3db 100644 --- a/doc/calibrations/flat_fielding.rst +++ b/doc/calibrations/flat_fielding.rst @@ -204,7 +204,8 @@ need to provide the matching flat field images in your In short, if ``use_pixelflat=True`` for *any* of your frame types, at least one of the data files in the :ref:`pypeit_file` :ref:`data_block` must -be labelled as ``pixelflat`` (unless you `Feed a PixelFlat`_). +be labelled as ``pixelflat``, or ``slitless_pixflat`` +(unless you `Feed a PixelFlat`_). And, if ``use_illumflat=True`` for *any* of your frame types, at least one of the data files in the @@ -216,26 +217,47 @@ frames for the pixel and illumination corrections. This is supported, but we recommend that you set the ``trace`` frames to be the same as the ``illumflat`` frames. +.. _generate-pixflat: + +Generate a Slitless PixelFlat +----------------------------- + +If a set of ``slitless_pixflat`` frames are available in the +:ref:`data_block` of the :ref:`pypeit_file`, PypeIt will generate +a slitless pixel flat (unless you `Feed a PixelFlat`_ instead) +during the main :ref:`run-pypeit`, and will apply it to frame +types that have ``use_pixelflat=True``. +The slitless pixel flat is generated separately for each detector +(even in the case of a mosaic reduction) and it is stored in a FITS +file in the reduction directory, with one extension per detector. +In addition to saving the file in your reduction directory, +the constructed pixelflat is saved to the PypeIt cache (see ref:`data_installation`). +This allows you to use the file for both current and future reductions. +To use this file in future reductions, the user should add the slitless +pixel flat file name to the :ref:`pypeit_file` as shown in `Feed a PixelFlat`_. + +If you generate your own slitless pixel flat, and you think it is generally +applicable for your instrument, please consider sharing it with the PypeIt Developers. + + Feed a PixelFlat ---------------- If you have generated your own pixel flat (or were provided one) and it is trimmed and oriented following the expected :ref:`pypeit-orientation`, -then you may feed this into PypeIt. This is the recommended approach -at present for :ref:`lrisb`. +then you may feed this into PypeIt. -And you perform this by modifying the :ref:`parameter_block`: - -.. TODO: IS THIS STILL THE CORRECT APPROACH? WHAT DO PEOPLE DO IF THEY DON'T -.. HAVE THE DEV SUITE? +To use the available PixelFlat, you need to modify the :ref:`parameter_block` like, e.g.: .. code-block:: ini - [calibrations] - [[flatfield]] - pixelflat_file = /Users/joe/python/PypeIt-development-suite/CALIBS/PYPEIT_LRISb_pixflat_B600_2x2_17sep2009.fits.gz + [calibrations] + [[flatfield]] + pixelflat_file = PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz + +If any of the frames in the :ref:`data_block` are labelled as ``pixelflat``, or ``slitless_pixflat``, +the provided pixel flat file will still be used instead of generating a new one. -None of the frames in the :ref:`data_block` should be labelled as ``pixelflat``. Algorithms ---------- diff --git a/doc/calibrations/image_proc.rst b/doc/calibrations/image_proc.rst index f64d467908..b80caecf80 100644 --- a/doc/calibrations/image_proc.rst +++ b/doc/calibrations/image_proc.rst @@ -42,8 +42,9 @@ where: - the quantity :math:`C=N_{\rm frames}\ c/s\prime=c/s` is the number of electron counts excited by photons hitting the detector, - :math:`1/s=N_{\rm frames}/s\prime` is a factor that accounts for the number - of frames contributing to the electron counts, and the relative - throughput factors (see below) that can be measured from flat-field frames, + of frames contributing to the electron counts (`N_{\rm frames}`), and (`s\prime`) the relative + throughput factors (see below) that can be measured from flat-field frames plus a scaling factor + applied if the counts of each frame are scaled to the mean counts of all frames, - :math:`D` is the dark-current, i.e., the rate at which the detector generates thermal electrons, in e-/pixel/s, - :math:`N_{\rm bin}` is the number of pixels in a binned pixel, diff --git a/doc/calibrations/nonlinear.rst b/doc/calibrations/nonlinear.rst new file mode 100644 index 0000000000..31e9a56776 --- /dev/null +++ b/doc/calibrations/nonlinear.rst @@ -0,0 +1,93 @@ + +===================== +Non-Linear correction +===================== + +The non-linear correction is a simple way to correct the non-linear behavior of the +sensor. Imagine a perfect light source that emits photons with a constant count rate. +An ideal detector would be able to measure this count rate independently of the exposure +time. However, in practice, detectors have a non-linear response to the incoming photons. +This means that the count rate measured by the detector is not proportional to the count +rate of the light source. The non-linear correction is a simple way to correct this +non-linear behavior. Most modern CCD cameras have a linear response to about 1% over +most of the dynamic range, so this correction is not necessary for most applications. +The correction parameters can be measured by the user, but care should be taken with +how you collect the calibration data, since the internal continuum light source used +for most flat-field calibrations is not a perfect light source. + +At present, the non-linear correction is only implemented for KCWI, which is already very +close to linear over most of the dynamic range of the CCD. If you have collected non-linear +calibration data with another instrument, please contact the developers to see if we can +implement this correction for your instrument. + +Calibrations required +--------------------- + +The non-linear correction requires a set of dedicated calibration data that can be used to measure +the non-linear response of the detector. This calibration data should be collected with +a continuum light source (usually internal to the instrument). We currently recommend that +you turn on the internal lamp, and leave it for 30 minutes to warm up and settle. The lamp +count rate will likely decay over time, so we recommend that you collect a series of exposures +that are bracketed by `reference` exposures of constant exposure time (also known as the +`bracketed repeat exposure` technique). The reference exposures are used to monitor the +temporal decay of the lamp count rate. Here is a screenshot to show the lamp decay as a +function of time. The points are the `reference exposures` and are colour-coded by the +wavelength of the light, and the solid lines are the best-fit exponential + 2D polynomial +decay curves. The bottom panel shows a residual of the fit. Thus, both the relative shape +and the intensity of the lamp decay can be monitored as a function of time. + +.. image:: ../figures/nonlinear_lamp_decay.png + +Between a set of reference exposures, you should +acquire a series of exposures with increasing exposure time. The exposure time should be +increased by a factor of 2 between each exposure. The exposure time should be chosen so that +the count rate is not too high, but also not too low. The counts should extend to at least 10% +of the maximum counts of the detector. The `reference` exposures should have an exposure time +that is close to the middle of the range of exposure times used for the other exposures. It is +often good practice to collect an even number of exposures at each exposure time; some shutters +move in opposite directions for adjacent exposures, and the true exposure time may depend on +the direction that the shutter is moving. + +To determine the non-linearity, we assume that the correction is quadratic, such that the +quadratic term is a small perturbation from linear response. The functional form of the +true count rate is then: + +.. math:: + + C_T \cdot t = C_M \cdot t (1 + b \cdot C_M \cdot t) + +where :math:`C_T` is the true count rate, :math:`C_M` is the measured count rate, :math:`t` +is the exposure time, and :math:`b` is the non-linearity coefficient. +The non-linearity coefficient can be determined by fitting +the measured count rate as a function of the exposure time. That that as the true count rate +tends to zero, the measured count rate tends to zero. The non-linearity coefficient can be +measured for each pixel on the detector, and it is a good idea to consider the non-linearity +coefficient for different amplifiers separately. The above equation can be written as a matrix +equation: + +.. math:: + + M \cdot x = e + M = \begin{bmatrix} + C_{M,1}t_1 & (C_{M,1}t_1)^2 \\ + C_{M,2}t_2 & (C_{M,2}t_2)^2 \\ + \vdots & \vdots \\ + C_{M,n}t_n & (C_{M,n}t_n)^2 \\ + \end{bmatrix} + x = \begin{bmatrix} + 1/C_T \\ + b/C_T \\ + \end{bmatrix} + e = \begin{bmatrix} + t_1 \\ + t_2 \\ + \vdots \\ + t_n \\ + \end{bmatrix} + +where :math:`M` is a matrix of measured count rates, :math:`x` is a vector of the non-linearity +coefficient and the true count rate, and :math:`e` is a vector of exposure times. The non-linearity +coefficient can be determined by inverting the matrix :math:`M`, and solving for :math:`x`. The +non-linearity coefficient can be determined for each pixel on the detector. The central value of +the distribution of non-linearity coefficients can be used as a fixed non-linearity coefficient +for each amplifier of the detector. diff --git a/doc/calibrations/slit_tracing.rst b/doc/calibrations/slit_tracing.rst index 84bbdbd574..af2b1ba969 100644 --- a/doc/calibrations/slit_tracing.rst +++ b/doc/calibrations/slit_tracing.rst @@ -369,7 +369,7 @@ case for low-dispersion data, e.g. LRISb 300 grism spectra .. code-block:: ini [calibrations] - [[slits]] + [[slitedges]] smash_range = 0.5,1. Algorithm diff --git a/doc/coadd1d.rst b/doc/coadd1d.rst index 76d27d7781..6520efe6aa 100644 --- a/doc/coadd1d.rst +++ b/doc/coadd1d.rst @@ -303,3 +303,40 @@ and launches a GUI from the `linetools`_ package. e.g.: lt_xspec J1217p3905_coadd.fits +UVES_popler coaddition +====================== + +If you prefer to use a GUI for the coaddition (to manually remove +bad pixels, ghosts, cosmic rays etc.), then you can use the UVES_popler tool. +This tool is developed by Michael Murphy and is available at +`this link `__. +Here is an example of a coadded spectrum using UVES_popler: + +.. image:: figures/uves_popler.png + :scale: 60% + +UVES_popler was originally written to coadd ESO/UVES echelle spectra +that were reduced by the ESO pipeline, and it has been recently modified +to support the reduction of PypeIt longslit and echelle data. +For details on how to use the tool, please refer to the +`UVES_popler documentation `__. +To get you started with reading in PypeIt :doc:`out_spec1D` files, +you need to generate a text file that lists the absolute paths to the +:doc:`out_spec1D` files. Here is an example of how to generate this file: + +.. code-block:: console + + ls -1 /path/to/your/pypeit_output/Science/spec1d/*.fits > /path/to/your/pypeit_output/pypeit_spec1d_files.txt + +Then you can use this file as input to UVES_popler, by using the following command: + +.. code-block:: console + + cd /path/to/your/pypeit_output/ + UVES_popler -disp 50 -filetype 11 pypeit_spec1d_files.txt + +This will launch the GUI, where you can interactively coadd your spectra. The +``-disp 50`` option is used to set the pixel sampling of the spectra to 50 km/s, +and the ``-filetype 11`` option is used to specify that the input files are PypeIt +:doc:`out_spec1D` files. For more information on the options available, you can +specify the ``-h`` option. diff --git a/doc/coadd3d.rst b/doc/coadd3d.rst index 03640ca5e5..4e16390484 100644 --- a/doc/coadd3d.rst +++ b/doc/coadd3d.rst @@ -57,6 +57,7 @@ saved as ``BB1245p4238.coadd3d``: [reduce] [[cube]] combine = True + align = True output_filename = BB1245p4238_datacube.fits save_whitelight = True @@ -74,18 +75,39 @@ If you want to combine all exposures into a single datacube, you need to set ``c as in the above example, and provide an ``output_filename``. This is very useful if you want to combine several standard star exposures into a single datacube for flux calibration, for example. -The spec2d block provides a list of :doc:`out_spec2D` files. You can also specify an optional scale correction -as part of the spec2d block. This relative scale correction ensures that the relative spectral sensitivity of the -datacube is constant across the field of view. The spec2d file used for the ``scale_corr`` column should either be a -twilight or dome flat reduced as a ``science`` frame (see :doc:`spectrographs/keck_kcwi` for a description of what you need to do). -In order to use this functionality, you should not reduce your science data with a spectral illumination correction. -In other words, in your :doc:`pypeit_file` file, set the following when you execute :ref:`run-pypeit`: - -.. code-block:: ini - - [scienceframe] - [[process]] - use_specillum = False +The spec2d block provides a list of :doc:`out_spec2D` files. You can also specify several optional +corrections as part of the spec2d block, including: + +* ``scale_corr``: A relative scale correction file that is used to correct the relative + spectral sensitivity of the datacube. This relative scale correction ensures that the + relative spectral sensitivity of the datacube is constant across the field of view. + The spec2d file used for the ``scale_corr`` column should either be a twilight or dome flat + reduced as a ``science`` frame (see :doc:`spectrographs/keck_kcwi` for a description of what + you need to do). In order to use this functionality, you should not reduce your science data + with a spectral illumination correction. In other words, in your :doc:`pypeit_file` file, set + the following when you execute :ref:`run-pypeit`: + + .. code-block:: ini + + [scienceframe] + [[process]] + use_specillum = False + +* ``grating_corr``: A grating correction file that is used to correct the grating relative sensitivity + of individual spec2d files. It is unlikely that you will require this correction. This is only required + if you are combining spec2d files that have very slightly different wavelength solutions *and* you only + have a sensitivity function for one of these setups. Otherwise, if you have a sensitivity function for + each setup, you should use the ``sensfile`` option to specify the sensitivity function for each wavelength + setup. For further details, see :ref:`coadd3d_gratcorr`. +* ``skysub_frame``: A sky subtraction frame that is used to remove the sky background of the datacube. + For further details, see :ref:`coadd3d_skysub`. +* ``ra_offset``: A right ascension offset that is used to correct the pointing of the datacube. + For further details, see :ref:`coadd3d_offsets`. +* ``dec_offset``: A declination offset that is used to correct the pointing of the datacube. + For further details, see :ref:`coadd3d_offsets`. +* ``sensfile``: A sensitivity function file that is used to correct the absolute sensitivity of the datacube. + The required input file is the sensitivity function, which is generated with the ``pypeit_sensfunc`` script. + For further details, see :ref:`coadd3d_fluxing`. run --- @@ -96,10 +118,58 @@ Then run the script: pypeit_coadd_datacube BB1245p4238.coadd3d -o +There are several recommended steps of the coadd3d process that can be run separately. These are: + +#. Step 1 - Create a datacube of your standard star exposures. It is worthwhile noting that the + standard star exposures should be reduced with the same setup as the science exposures. The + datacube is then used to flux calibrate the science exposures. + The datacube is created by running the following command: + + .. code-block:: console + + pypeit_coadd_datacube StandardStarName.coadd3d -o + +#. Step 2 - Extract the 1D spectra from the datacube. This is done by running the following command, + assuming that the output datacube from the previous step was called ``StandardStarName.fits``. + The ``pypeit_extract_datacube`` script will produce an output file called + ``spec1d_StandardStarName.fits``: + + .. code-block:: console + + pypeit_extract_datacube StandardStarName.fits -o + + This script is only designed for point sources. Both a boxcar and an optimal extraction are calculated. + The boxcar extraction uses a circular aperture with a radius set by the ``boxcar_radius`` parameter. + The optimal extraction uses the white light image as the object profile. Also note that the extraction + if performed on the sky-subtracted datacube. Therefore, the ``spec1d`` file contains the 1D spectra + including a ``BOX_COUNTS_SKY`` and ``OPT_COUNTS_SKY`` columns. These columns do not contain the sky + counts, but rather the residual level of the sky aperture. This is useful for checking the quality of + the sky subtraction. + +#. Step 3 - Generate a sensitivity function from the 1D spectra. This is done by running the following + command, assuming that the output 1D spectra from the previous step was called + ``spec1d_StandardStarName.fits``. The ``pypeit_sensfunc`` script will produce an output file called + ``sens_StandardStarName.fits``: + + .. code-block:: console + + pypeit_sensfunc spec1d_StandardStarName.fits -o sens_StandardStarName.fits + + For further details, see :ref:`sensitivity_function`. + +#. Step 4 - Generate a datacube of the science exposures. This is done by running the following command: + + .. code-block:: console + + pypeit_coadd_datacube ScienceName.coadd3d -o + + Note that you will need to specify the sensitivity function file using the ``sensfile`` option in the + :ref:`coadd3d_file` file. For further details, see :ref:`coadd3d_fluxing`. + Combination options =================== -PypeIt currently supports two different methods to convert an spec2d frame into a datacube; +PypeIt currently supports two different methods to convert a spec2d frame into a datacube; these options are called ``subpixel`` (default) and ``NGP`` (which is short for, nearest grid point), and can be set using the following keyword arguments: @@ -114,20 +184,45 @@ into many subpixels, and assigns each subpixel to a voxel of the datacube. Flux but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. The subpixellation scale can be separately set in the spatial and spectral direction on the 2D detector. If you would like to change the subpixellation factors from -the default values (5), you can set the ``spec_subpixel`` and ``spat_subpixel`` keywords -as follows: +the default values (5), you can optionally set (one or all of) the ``spec_subpixel``, +``spat_subpixel``, and ``slice_subpixel`` parameters as follows: .. code-block:: ini [reduce] [[cube]] method = subpixel - spec_subpixel = 8 - spat_subpixel = 10 + spec_subpixel = 3 + spat_subpixel = 7 + slice_subpixel = 10 The total number of subpixels generated for each detector pixel on the spec2d frame is -spec_subpixel x spat_subpixel. The default values (5) divide each spec2d pixel into 25 subpixels -during datacube creation. As an alternative, you can convert the spec2d frames into a datacube +spec_subpixel x spat_subpixel x slice_subpixel. The default values (5) divide each spec2d pixel +into 5x5x5=125 subpixels during datacube creation. +``spec_subpixel`` is the number of subpixels in the spectral +direction (i.e. predominantly detector columns), +``spat_subpixel`` is the number of subpixels in the +spatial direction (i.e. the long axis of each slice; predominantly along detector rows), and +``slice_subpixel`` is the number of times to divide each of the +slices (i.e. the short axis of each slice). Note that all three of these ``subpixel`` +definitions are perpendicular to each other in the datacube. + +While the ``spec_subpixel`` and ``spat_subpixel`` options are somewhat intuitive (i.e. the code is +dividing detector columns and rows into smaller subpixels), the ``slice_subpixel`` option may not be +immediately obvious, so consider the following example. Imagine the long edge of the slice aligned +East-West. The short edge of a single slice will span a Dec difference of 0.35 arcseconds in the case +of the Keck/KCWI Small slicer. ``slice_subpixel`` is effectively dividing this slice width further. +If ``slice_subpixel=7`` then this is creating seven subslices, each of width 0.05 arcseconds. The +importance of this becomes really noticeable when combined with the differential atmospheric +refraction (DAR) correction. Consider two wavelengths that have a relative DAR of 0.15 arcseconds. +In this case, choosing a value of ``slice_subpixel=1`` would shift the relative spatial positions by +0.35 arcseconds (i.e. the difference between adjacent slices) compared to the true difference of 0.15 +arcseconds. ``slice_subpixel`` divides the flux of the slice into evenly spaced rectangular bins, and +places each of these into a voxel of the final datacube. In the case of a 0.15 arcsecond shift, this +would mean that :math:`3/7` of the flux ends up in one output slice and the remaining :math:`4/7` of +the flux ends up in the adjacent slice. + +As an alternative to the ``subpixel`` method, you can convert the spec2d frames into a datacube with the ``NGP`` method. This algorithm is effectively a 3D histogram. This approach is faster than ``subpixel``, flux is conserved, and voxels are not correlated. However, this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. @@ -135,20 +230,42 @@ This algorithm takes each pixel on the spec2d frame and puts the flux of this pi in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighbouring voxel might contain the flux from two spec2d pixels. +.. _coadd3d_fluxing: + Flux calibration ================ If you would like to flux calibrate your datacube, you need to -produce your standard star datacube first, and when generating -the datacube of the science frame you must pass in the name of -the standard star cube in your ``coadd3d`` file as follows: +produce your standard star datacube first. Then extract the spectrum +of the standard star using the ``pypeit_extract_datacube`` script. This +will produce a ``spec1d`` file that you will need to use to generate a +sensitivity function in the usual way (see :ref:`sensitivity_function`). +Then, when generating the datacube of the science frame you must include +the name of the sensitivity function in your ``coadd3d`` file as follows: .. code-block:: ini [reduce] [[cube]] - standard_cube = standard_star_cube.fits + sensfunc = my_sensfunc.fits + + +Also, an important note for users that wish to combine multiple standard star exposures +into a single datacube: PypeIt currently performs an extinction correction when +generating the sensitivity function; this is perfectly fine for single exposures. +However, if you are combining multiple standard star exposures into a single datacube, +you should note that the extinction correction will be applied at the airmass of the +first standard star exposure listed in the ``coadd3d`` file. This is because the +extinction correction is currently not applied to each individual frame in the datacube. +This control flow will be changed in a future release of PypeIt, but for now, to stay +consistent with the current pipeline, the extinction correction is done in the sensitivity +function algorithms, with the caveat that the standard star exposures are assumed to have +similar airmasses. If you have standard star exposures with significantly different +airmasses, then you should use just one of these exposures to generate the sensitivity +function. + +.. _coadd3d_skysub: Sky Subtraction =============== @@ -172,19 +289,20 @@ then you can specify the ``skysub_frame`` in the ``spec2d`` block of the above. If you have dedicated sky frames, then it is generally recommended to reduce these frames as if they are regular science frames, but add the following keyword arguments at the top of your -:doc:`pypeit_file`: +:ref:`coadd3d_file`: .. code-block:: ini [reduce] [[skysub]] - joint_fit = True - user_regions = : + user_regions = 5:95 [flexure] spec_method = slitcen -This ensures that all pixels in the slit are used to generate a -complete model of the sky. +This ensures that the innermost 90 percent of pixels in each slit are +used to generate a model of the sky. + +.. _coadd3d_gratcorr: Grating correction ================== @@ -193,19 +311,33 @@ The grating correction is needed if any of the data are recorded with even a very slightly different setup (e.g. data taken on two different nights with the same *intended* wavelength coverage, but the grating angle of the two nights were slightly different). -This is also needed if your standard star observations were taken -with a slightly different setup. This correction requires that you +You can avoid this correction if you generate a sensitivity function +for each of the setups. However, if you have not done this, then +the grating correction is needed. This correction requires that you have taken calibrations (i.e. flatfields) with the two different -setups. By default, the grating correction will be applied, but it -can be disabled by setting the following keyword argument in your -``coadd3d`` file: +setups, and uses the relative sensitivity of the two flatfields to +estimate the sensitivity correction. By default, the grating correction +will not be applied. If you want to apply the grating correction, you +will need to specify the relative path+file of the Flat calibration +file for each spec2d file. You will need to specify a ``grating_corr`` +file for each science frame, in the ``spec2d`` block of the +``.coadd3d`` file: .. code-block:: ini - [reduce] - [[cube]] - grating_corr = False + # Read in the data + spec2d read + filename | grating_corr + Science/spec2d_scienceframe_01.fits | Calibrations/Flat_A_0_DET01.fits + Science/spec2d_scienceframe_02.fits | Calibrations/Flat_B_1_DET01.fits + spec2d end +If all spec2d files were reduced with the same Flat calibration file, +then you do not need to specify the grating correction file. Also, if you +generate a sensitivity function for each spec2d file, then you do not need +to specify the grating correction file. The grating correction file is only +needed if you have one sensitivity function for all spec2d files, even though +the spec2d files were acquired with different grating angles. Astrometric correction ====================== @@ -224,12 +356,16 @@ file: [[cube]] astrometric = False +If a :doc:`calibrations/align` frame is not available, then the astrometric +correction will be based on the slit edges. White light image ================= A white light image can be generated for the combined frame, or -for each individual frame if ``combine=False``, by setting the following +for each individual frame if ``combine=False``, by setting the +``save_whitelight`` keyword argument. You can set the wavelength +range of the white light image by setting the ``whitelight_range`` keyword argument: .. code-block:: ini @@ -237,10 +373,13 @@ keyword argument: [reduce] [[cube]] save_whitelight = True + whitelight_range = 5000,6000 White light images are not produced by default. The output filename for the white light images are given the suffix ``_whitelight.fits``. +.. _coadd3d_offsets: + Spatial alignment with different setups ======================================= diff --git a/doc/cookbook.rst b/doc/cookbook.rst index 915aed4865..19b397281e 100644 --- a/doc/cookbook.rst +++ b/doc/cookbook.rst @@ -70,7 +70,8 @@ what we recommend: - We will refer to that folder as ``RAWDIR`` The raw images can be gzip-compressed, although this means opening files will be -slower. +slower. See :ref:`setup-file-searching` for specific comments about the files +in your raw directory. A word on calibration data -------------------------- diff --git a/doc/dev/hiresconfig.rst b/doc/dev/hiresconfig.rst new file mode 100644 index 0000000000..38d07f705c --- /dev/null +++ b/doc/dev/hiresconfig.rst @@ -0,0 +1,118 @@ +.. include:: ../include/links.rst + +.. _hires_config: + +Automated sorting of HIRES frames by instrument configuration +============================================================= + +Version History +--------------- + + +========= ================ =========== =========== +*Version* *Author* *Date* ``PypeIt`` +========= ================ =========== =========== +1.0 Debora Pelliccia 10 Aug 2024 1.16.1.dev +========= ================ =========== =========== + +---- + +Basics +------ + +To prepare for the data reduction, PypeIt, first, automatically associates fits +files to specific :ref:`frame_types` (see :ref:`hires_frames`) and, then, +collects groups of frames in unique instrument configurations (see below). This is performed +by the :ref:`pypeit_setup` script, which sorts the frames and writes a +:ref:`pypeit_file` for each unique configuration. See :ref:`setup_doc`. + + +HIRES configuration identification +---------------------------------- + +The HIRES instrument configurations are determined by the function +:func:`pypeit.metadata.PypeItMetaData.unique_configurations`, +which finds unique combinations of the following keywords: + +=============== ============ +``fitstbl`` key Header Key +=============== ============ +``dispname`` ``XDISPERS`` +``decker`` ``DECKNAME`` +``binning`` ``BINNING`` +``filter1`` ``FIL1NAME`` +``echangle`` ``ECHANGL`` +``xdangle`` ``XDANGL`` +=============== ============ + +The unique configurations are determined by collating the relevant metadata from the headers +of all frames found by a run of :ref:`pypeit_setup`, *except* those that are designated as +bias and slitless_pixflat frames. Bias and slitless_pixflat frames can have header data (e.g., ``filter1``) +that do not match the instrument configuration that an observer intended for their use. +Therefore, PypeIt uses the ``dispname`` and ``binning`` keys to match the bias and +slitless_pixflat frames to the configurations with frames taken with the same cross-disperser +and same binning. +Note that when using the ``echangle`` and ``xdangle`` keys to identify configurations, PypeIt +uses a relative tolerance of 1e-3 and absolute tolerance of 1e-2 for ``echangle``, and a relative +tolerance of 1e-2 for ``xdangle``, to account for small differences in the values of these angles. + +After that, :func:`pypeit.metadata.PypeItMetaData.set_configurations` associates each frame +to the relevant unique configuration ("setup"), by assigning a setup identifier +(e.g., A,B,C,D...) to every frames for which the values of the above keywords match the +values of the specific unique configuration. + +HIRES calibration groups +------------------------ + +PypeIt uses the concept of a "calibration group" to define a complete set of +calibration frames (e.g., arcs, flats) and the science frames to which these calibration +frames should be applied. + +By default, :ref:`pypeit_setup` uses the setup identifier to assign frames to a single +calibration group. Frames that are in the same calibration group will have the same PypeIt +keyword ``calib``. No automated procedure exists to do anything except this. +However, the user can edit the :ref:`pypeit_file` to, within a given configuration, assign +specific calibration frames to specific science frames using the data in the ``calib`` column +of the :ref:`data_block`. + +Testing +------- + +To test that PypeIt can successfully identify multiple +configurations among a set of files, we have added the +``test_setup_keck_hires_multiconfig()`` test to +``${PYPEIT_DEV}/unit_tests/test_setups.py``. + +Here is an example of how to run the test: + +.. code-block:: bash + + cd ${PYPEIT_DEV}/unit_tests + pytest test_setup.py::test_setup_keck_hires_multiconfig -W ignore + +The tests require that you have downloaded the PypeIt +:ref:`dev-suite` and defined the ``PYPEIT_DEV`` environmental +variable that points to the relevant directory. + +The algorithm for this test is as follows: + + 1. Collect the names of all files in selected HIRES directories. + + 2. Use :class:`~pypeit.pypeitsetup.PypeItSetup` to automatically + identify the configurations for these files. + + 3. Check that the code found two configurations and wrote the + pypeit files for each. + + 4. For each configuration: + + a. Read the pypeit file + + b. Check that the name for the setup is correct ('A' or 'B') + + c. Check that the calibration group is the same for all frames ('0' or '1') + + +Because these tests are now included in the PypeIt +:ref:`unit-tests`, these configuration checks are performed by the +developers for every new version of the code. diff --git a/doc/dev/hiresframes.rst b/doc/dev/hiresframes.rst new file mode 100644 index 0000000000..d303081a9a --- /dev/null +++ b/doc/dev/hiresframes.rst @@ -0,0 +1,128 @@ +.. include:: ../include/links.rst + +.. _hires_frames: + +Automated typing of HIRES frames +================================ + +Version History +--------------- + + +========= ================ =========== =========== +*Version* *Author* *Date* ``PypeIt`` +========= ================ =========== =========== +1.0 Debora Pelliccia 10 Aug 2024 1.16.1.dev +========= ================ =========== =========== + +---- + +Basics +------ + +The general procedure used to assign frames a given type is described +here: :ref:`frame_types`. + +HIRES frame typing +------------------ + +The primary typing of HIRES frames is performed by +:func:`pypeit.spectrographs.keck_hires.KECKHIRESSpectrograph.check_frame_type`. +This function checks the values of various header keywords against a +set of criteria used to classify the frame type. +The header cards required for the frame-typing and their associated keyword in the +:class:`~pypeit.metadata.PypeItMetaData` object are: + +=============== ============ +``fitstbl`` key Header Key +=============== ============ +``exptime`` ``ELAPTIME`` +``hatch`` ``HATOPEN`` +``lampstat01`` See below +No key ``XCOVOPEN`` +No key ``AUTOSHUT`` +=============== ============ + +``lampstat01`` is defined using a combination of header keywords, which include +``LAMPCAT1``, ``LAMPCAT2``, ``LAMPQTZ2``, ``LAMPNAME``. If ``LAMPCAT1 = True`` or +``LAMPCAT2 = True``, ``lampstat01`` will be equal to ``'ThAr1'`` or ``'ThAr2'``, respectively. +If ``LAMPQTZ2 = True`` or ``LAMPNAME = 'quartz1'``, ``lampstat01`` will be equal to ``'on'``. + + +The criteria used to select each frame type are as follows: + +==================== ============ ============ ============ ====================================== ====================================================== +Frame ``hatch`` ``AUTOSHUT`` ``XCOVOPEN`` ``lampstat01`` ``exptime`` +==================== ============ ============ ============ ====================================== ====================================================== +``science`` ``True`` ``True`` ``True`` ``'off'`` ``>601s`` +``standard`` ``'open'`` ``True`` ``True`` ``'off'`` ``>1s`` & ``<600s`` +``bias`` ``False`` ``False`` ``True`` ``'off'`` ``<0.001s`` +``dark`` ``False`` ``True`` ``True`` ``'off'`` Not used +``slitless_pixflat`` ``False`` ``True`` ``False`` ``'off'`` ``<60s`` +``pixelflat`` ``False`` ``True`` ``True`` ``'on'`` ``<60s`` +``trace`` ``False`` ``True`` ``True`` ``'on'`` ``<60s`` +``illumflat`` ``False`` ``True`` ``True`` ``'on'`` ``<60s`` +``arc`` ``False`` ``True`` ``True`` ``'ThAr1'`` or ``'ThAr2'`` Not used +``tilt`` ``False`` ``True`` ``True`` ``'ThAr1'`` or ``'ThAr2'`` Not used +==================== ============ ============ ============ ====================================== ====================================================== + +Note that PypeIt employs commonly used value of ``exptime`` to distinguish frame type; +however, if needed, the user can specify a different value by +using the ``exprng`` parameter in the :ref:`pypeit_file`; see also :ref:`frame_types`. + +The ``science`` and ``standard`` frames have identical selection criteria, except for the +``exptime`` value. In order to better distinguish between the two types, the ``RA`` and ``DEC`` header +keywords are also used to assign the ``standard`` type to frames with ``RA`` and ``DEC`` values that are +within 10 arcmin of one of the standard stars available in PypeIt (see :ref:`standards`). + +The criteria used to select ``arc`` and ``tilt`` frames are identical; the same is true for +``pixelflat``, ``trace``, and ``illumflat`` frames. Note that if both ``pixelflat`` and +``slitless_pixflat`` frames are identified, the ``pixelflat`` assignment will be removed +so that the ``slitless_pixflat`` frames will be used for the flat fielding. + +Finally, note that a HIRES frame is never given a ``pinhole`` type. + + +Testing +------- + +To test that PypeIt can successfully identify HIRES framt types +among a set of files, we have added the +``test_hires()`` test to ``${PYPEIT_DEV}/unit_tests/test_frametype.py``. + +Here is an example of how to run the test: + +.. code-block:: bash + + cd ${PYPEIT_DEV}/unit_tests + pytest test_frametype.py::test_hires -W ignore + +The tests requires that you have downloaded the PypeIt +:ref:`dev-suite` and defined the ``PYPEIT_DEV`` environmental +variable that points to the relevant directory. The algorithm for +all these tests is the same and is as follows: + + 1. Find the directories in the :ref:`dev-suite` with Keck + HIRES data. + + 2. For each directory (i.e., instrument setup): + + a. Make sure there is a "by-hand" version of the pypeit file + for this setup where a human (one of the pypeit + developers) has ensured the frame types are correct. + + b. Effectively run :ref:`pypeit_setup` on each of the + instrument setups to construct a new pypeit file with the + automatically generated frame types. + + c. Read both the by-hand and automatically generated frame + types from these two pypeit files and check that they are + identical. This check is *only* performed for the + calibration frames, not any ``science`` or ``standard`` + frames. + +Because this test is now included in the ``PypeIt`` +:ref:`unit-tests`, this frame-typing check is performed by the +developers for every new version of the code. + + diff --git a/doc/dev/lrisconfig.rst b/doc/dev/lrisconfig.rst index 3e73cedcb5..aa0638ffe9 100644 --- a/doc/dev/lrisconfig.rst +++ b/doc/dev/lrisconfig.rst @@ -13,6 +13,7 @@ Version History *Version* *Author* *Date* ``PypeIt`` ========= ================ =========== =========== 1.0 Debora Pelliccia 6 Sep 2023 1.13.1.dev +1.1 Debora Pelliccia 10 Aug 2024 1.16.1.dev ========= ================ =========== =========== ---- @@ -50,12 +51,15 @@ which finds unique combinations of the following keywords: The unique configurations are determined by collating the relevant metadata from the headers of all frames found by a run of :ref:`pypeit_setup`, *except* those that are designated as -bias frames. The reason is that bias frames can have header data (e.g., ``dispangle``) +bias and slitless_pixflat frames. Bias frames can have header data (e.g., ``dispangle``) that do not match the instrument configuration that an observer intended for their use; e.g., the frames were taken before the instrument was fully configured for the night's observations. Therefore, PypeIt uses the ``dateobs``, ``binning``, ``amp`` keys to match the bias frames to the configurations with frames taken on the same date, with -the same binning and on the same amplifier. +the same binning and on the same amplifier. Similarly, slitless_pixflat frames are taken +without a slitmask (i.e., ``decker`` is different from the other frames), therefore +PypeIt uses the ``dateobs``, ``binning``, ``amp``, ``dispname``, ``dichroic`` keys to match +the slitless_pixflat frames to a specific configuration. After that, :func:`pypeit.metadata.PypeItMetaData.set_configurations` associates each frame to the relevant unique configuration ("setup"), by assigning a setup identifier diff --git a/doc/dev/lrisframes.rst b/doc/dev/lrisframes.rst index ef7c845273..ae52ee3c1c 100644 --- a/doc/dev/lrisframes.rst +++ b/doc/dev/lrisframes.rst @@ -13,6 +13,7 @@ Version History *Version* *Author* *Date* ``PypeIt`` ========= ================ =========== =========== 1.0 Debora Pelliccia 6 Sep 2023 1.13.1.dev +1.1 Debora Pelliccia 10 Aug 2024 1.16.1.dev ========= ================ =========== =========== ---- @@ -40,6 +41,7 @@ The header cards required for the frame-typing and their associated keyword in t =============== ====================================================== ``exptime`` ``ELAPTIME`` (``TELAPSE`` for ``keck_lris_red_mark4``) ``hatch`` ``TRAPDOOR`` +``decker`` ``SLITNAME`` ``lampstat01`` See below =============== ====================================================== @@ -51,32 +53,43 @@ of keywords used to define ``lampstat01`` varies depending on the available head The criteria used to select each frame type are as follows: -============= ============ ====================================== =========================================== -Frame ``hatch`` ``lampstat01`` ``exptime`` -============= ============ ====================================== =========================================== -``science`` ``'open'`` ``'off'`` ``>61s`` -``standard`` ``'open'`` ``'off'`` ``>1s`` & ``<61s`` -``bias`` ``'closed'`` ``'off'`` ``<0.001s`` -``pixelflat`` ``'closed'`` ``'Halogen' or '2H'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) -``pixelflat`` ``'open'`` ``'on'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) -``trace`` ``'closed'`` ``'Halogen'`` or ``'2H'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) -``trace`` ``'open'`` ``'on'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) -``illumflat`` ``'closed'`` ``'Halogen'`` or ``'2H'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) -``illumflat`` ``'open'`` ``'on'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) -``arc`` ``'closed'`` ``!= 'Halogen', '2H', 'on', 'off'`` Not used -``tilt`` ``'closed'`` ``!= 'Halogen', '2H', 'on', 'off'`` Not used -============= ============ ====================================== =========================================== +==================== ============ ====================================== ================= ====================================================== +Frame ``hatch`` ``lampstat01`` ``decker`` ``exptime`` +==================== ============ ====================================== ================= ====================================================== +``science`` ``'open'`` ``'off'`` ``!= 'GOH_LRIS'`` ``>61s`` +``standard`` ``'open'`` ``'off'`` ``!= 'GOH_LRIS'`` ``>1s`` & ``<61s`` (LRIS RED) or ``<900s`` (LRIS BLUE) +``bias`` ``'closed'`` ``'off'`` ``!= 'GOH_LRIS'`` ``<1s`` +``slitless_pixflat`` ``'open'`` ``'off'`` ``== 'direct'`` ``<60s`` +``pixelflat`` ``'closed'`` ``'Halogen' or '2H'`` ``!= 'GOH_LRIS'`` ``<60s`` (LRIS RED) or ``<300s`` (LRIS BLUE) +``pixelflat`` ``'open'`` ``'on'`` ``!= 'GOH_LRIS'`` ``<60s`` (LRIS RED) or ``<300s`` (LRIS BLUE) +``trace`` ``'closed'`` ``'Halogen'`` or ``'2H'`` ``!= 'GOH_LRIS'`` ``<60s`` (LRIS RED) or ``<300s`` (LRIS BLUE) +``trace`` ``'open'`` ``'on'`` ``!= 'GOH_LRIS'`` ``<60s`` (LRIS RED) or ``<300s`` (LRIS BLUE) +``illumflat`` ``'closed'`` ``'Halogen'`` or ``'2H'`` ``!= 'GOH_LRIS'`` ``<60s`` (LRIS RED) or ``<300s`` (LRIS BLUE) +``illumflat`` ``'open'`` ``'on'`` ``!= 'GOH_LRIS'`` ``<60s`` (LRIS RED) or ``<300s`` (LRIS BLUE) +``arc`` ``'closed'`` ``!= 'Halogen', '2H', 'on', 'off'`` ``!= 'GOH_LRIS'`` Not used +``tilt`` ``'closed'`` ``!= 'Halogen', '2H', 'on', 'off'`` ``!= 'GOH_LRIS'`` Not used +==================== ============ ====================================== ================= ====================================================== Note that PypeIt employs commonly used value of ``exptime`` to distinguish frame type; however, if needed, the user can specify a different value by using the ``exprng`` parameter in the :ref:`pypeit_file`; see also :ref:`frame_types`. +The ``science`` and ``standard`` frames have identical selection criteria, except for the +``exptime`` value. In order to better distinguish between the two types, the ``RA`` and ``DEC`` header +keywords are also used to assign the ``standard`` type to frames with ``RA`` and ``DEC`` values that are +within 10 arcmin of one of the standard stars available in PypeIt (see :ref:`standards`). + The criteria used to select ``arc`` and ``tilt`` frames are identical; the same is true for -``pixelflat``, ``trace``, and ``illumflat`` frames. However, it's important to note that +``pixelflat``, ``trace``, and ``illumflat`` frames. It's important to note that PypeIt is able to correctly assign the ``pixelflat``, ``trace``, and ``illumflat`` types -to the internal and dome flat frames, but the twilight flats will generally have the -``science`` or ``standard`` type. Therefore, the user should manually change their frame type -in the :ref:`pypeit_file`. +to the internal and dome flat frames, and it tries to do the same for the twilight flats, by selecting +frames that looks like ``science`` frames and include the following words in the ``OBJECT`` +or ``TARGNAME`` header keywords: 'sky', 'blank', 'twilight', 'twiflat', 'twi flat'. This way of +identifying twilight flats is not robust, therefore the user should always check the frame types assigned +and manually change them if needed in the :ref:`pypeit_file`. + +Note, also, that if both ``pixelflat`` and ``slitless_pixflat`` frames are identified, the ``pixelflat`` +assignment will be removed so that the ``slitless_pixflat`` frames will be used for the flat fielding. Finally, note that a LRIS frame is never given a ``pinhole`` or ``dark`` type. diff --git a/doc/figures/Edges_A_0_MSC01_orders_qa.png b/doc/figures/Edges_A_0_MSC01_orders_qa.png new file mode 100644 index 0000000000..72c6c30127 Binary files /dev/null and b/doc/figures/Edges_A_0_MSC01_orders_qa.png differ diff --git a/doc/figures/nonlinear_lamp_decay.png b/doc/figures/nonlinear_lamp_decay.png new file mode 100644 index 0000000000..430b4327f4 Binary files /dev/null and b/doc/figures/nonlinear_lamp_decay.png differ diff --git a/doc/figures/uves_popler.png b/doc/figures/uves_popler.png new file mode 100644 index 0000000000..eb4129d343 Binary files /dev/null and b/doc/figures/uves_popler.png differ diff --git a/doc/frametype.rst b/doc/frametype.rst index 83387add34..148f9cf078 100644 --- a/doc/frametype.rst +++ b/doc/frametype.rst @@ -42,24 +42,25 @@ description can be listed as follows: More detailed descriptions are given in the table below. -================ ============================================================= -Frame Type Description -================ ============================================================= -``align`` Used to align spatial positions in multiple slits. This frame is particularly useful for slit-based IFU, such as Keck KCWI. -``arc`` Spectrum of one or more calibration arc lamps -``bias`` Bias frame; typically a 0s exposure with the shutter closed -``dark`` Dark frame; typically a >0s exposure to assess dark current (shutter closed) -``illumflat`` Spectrum taken to correct illumination profile of the slit(s). This is often the same as the trace flat (below). -``lampoffflats`` Spectrum taken to remove persistence from lamp on flat exposures and/or thermal emission from the telescope and dome. Usually this is an exposure using a flat with lamps OFF -``pinhole`` Spectrum taken through a pinhole slit (i.e. a very short slit length), and is used to define the centre if a slit (currently, this frame is only used for echelle data reduction). Often this is an exposure using a flat lamp, but one can in principle use a standard star frame too (or a science frame if the spectrum is uniform). -``pixelflat`` Spectrum taken to correct for pixel-to-pixel detector variations Often an exposure using a dome (recommended) or internal flat lamp, but for observations in the very blue, this may be on-sky -``science`` Spectrum of one or more science targets -``standard`` Spectrum of spectrophotometric standard star PypeIt includes a list of pre-defined standards -``trace`` Spectrum taken to define the slit edges. Often this is an exposure using a flat lamp, but for observations in the very blue, this may be on-sky. The slit length of a trace frame should be the same as the science slit. -``tilt`` Exposure used to trace the tilt in the wavelength solution. Often the same file(s) as the arc. -``sky`` On-sky observation of the sky used for background subtraction -``None`` File could not be automatically identified by PypeIt -================ ============================================================= +==================== ============================================================= +Frame Type Description +==================== ============================================================= +``align`` Used to align spatial positions in multiple slits. This frame is particularly useful for slit-based IFU, such as Keck KCWI. +``arc`` Spectrum of one or more calibration arc lamps +``bias`` Bias frame; typically a 0s exposure with the shutter closed +``dark`` Dark frame; typically a >0s exposure to assess dark current (shutter closed) +``illumflat`` Spectrum taken to correct illumination profile of the slit(s). This is often the same as the trace flat (below). +``lampoffflats`` Spectrum taken to remove persistence from lamp on flat exposures and/or thermal emission from the telescope and dome. Usually this is an exposure using a flat with lamps OFF +``pinhole`` Spectrum taken through a pinhole slit (i.e. a very short slit length), and is used to define the centre if a slit (currently, this frame is only used for echelle data reduction). Often this is an exposure using a flat lamp, but one can in principle use a standard star frame too (or a science frame if the spectrum is uniform). +``pixelflat`` Spectrum taken to correct for pixel-to-pixel detector variations. Often an exposure using a dome (recommended) or internal flat lamp, but for observations in the very blue, this may be on-sky +``slitless_pixflat`` Spectrum taken without a slitmask or longslit to correct for pixel-to-pixel detector variations. This is often an exposure taken on-sky +``science`` Spectrum of one or more science targets +``standard`` Spectrum of spectrophotometric standard star PypeIt includes a list of pre-defined standards +``trace`` Spectrum taken to define the slit edges. Often this is an exposure using a flat lamp, but for observations in the very blue, this may be on-sky. The slit length of a trace frame should be the same as the science slit. +``tilt`` Exposure used to trace the tilt in the wavelength solution. Often the same file(s) as the arc. +``sky`` On-sky observation of the sky used for background subtraction +``None`` File could not be automatically identified by PypeIt +==================== ============================================================= .. TODO: Need to check that "sky" frametype is correct and/or used! diff --git a/doc/help/pypeit_cache_github_data.rst b/doc/help/pypeit_cache_github_data.rst index 804e0656f2..3ab386e7fd 100644 --- a/doc/help/pypeit_cache_github_data.rst +++ b/doc/help/pypeit_cache_github_data.rst @@ -9,7 +9,7 @@ Script to download/cache PypeIt github data positional arguments: - spectrograph A valid spectrograph identifier: bok_bc, + spectrograph A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, @@ -36,14 +36,15 @@ --exclude EXCLUDE [EXCLUDE ...] A subset of the directories to *exclude* from the list of files to download. Options are: tests, reid_arxiv, - nist, standards, skisim, sensfunc. This option is - mutually exclusive with --include. (default: ['tests']) + nist, standards, skisim, sensfunc, pixelflat. This + option is mutually exclusive with --include. (default: + ['tests']) --include INCLUDE [INCLUDE ...] The directories to *include* in the list of files to download. Use "--include all" to include all directories. Options are: all, tests, reid_arxiv, nist, - standards, skisim, sensfunc. This option is mutually - exclusive with --exclude. (default: None) + standards, skisim, sensfunc, pixelflat. This option is + mutually exclusive with --exclude. (default: None) --spec_dependent_only Only include files that are specific to the provided list of spectrographs. By default, the script also diff --git a/doc/help/pypeit_chk_flexure.rst b/doc/help/pypeit_chk_flexure.rst new file mode 100644 index 0000000000..d7110d32d4 --- /dev/null +++ b/doc/help/pypeit_chk_flexure.rst @@ -0,0 +1,18 @@ +.. code-block:: console + + $ pypeit_chk_flexure -h + usage: pypeit_chk_flexure [-h] (--spec | --spat) [--try_old] + input_file [input_file ...] + + Print QA on flexure to the screen + + positional arguments: + input_file One or more PypeIt spec2d or spec1d file + + options: + -h, --help show this help message and exit + --spec Check the spectral flexure (default: False) + --spat Check the spatial flexure (default: False) + --try_old Attempt to load old datamodel versions. A crash may ensue.. + (default: False) + \ No newline at end of file diff --git a/doc/help/pypeit_chk_for_calibs.rst b/doc/help/pypeit_chk_for_calibs.rst index 06cc2d6357..653b5d1e81 100644 --- a/doc/help/pypeit_chk_for_calibs.rst +++ b/doc/help/pypeit_chk_for_calibs.rst @@ -13,7 +13,7 @@ options: -h, --help show this help message and exit -s SPECTROGRAPH, --spectrograph SPECTROGRAPH - A valid spectrograph identifier: bok_bc, + A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, @@ -35,8 +35,11 @@ vlt_xshooter_nir, vlt_xshooter_uvb, vlt_xshooter_vis, wht_isis_blue, wht_isis_red (default: None) -e EXTENSION, --extension EXTENSION - File extension; compression indicators (e.g. .gz) not - required. (default: .fits) + File extension to use. Must include the period (e.g., + ".fits") and it must be one of the allowed extensions + for this spectrograph. If None, root directory will be + searched for all files with any of the allowed + extensions. (default: None) --save_setups If not toggled, remove setup_files/ folder and its files. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_coadd_2dspec.rst b/doc/help/pypeit_coadd_2dspec.rst index 53cb0d2e57..d5d0aae36c 100644 --- a/doc/help/pypeit_coadd_2dspec.rst +++ b/doc/help/pypeit_coadd_2dspec.rst @@ -3,8 +3,6 @@ $ pypeit_coadd_2dspec -h usage: pypeit_coadd_2dspec [-h] [--show] [--debug_offsets] [--peaks] [--basename BASENAME] [--debug] [-v VERBOSITY] - [--spec_samp_fact SPEC_SAMP_FACT] - [--spat_samp_fact SPAT_SAMP_FACT] coadd2d_file Coadd 2D spectra produced by PypeIt @@ -27,12 +25,4 @@ Verbosity level between 0 [none] and 2 [all]. Default: 1. Level 2 writes a log with filename coadd_2dspec_YYYYMMDD-HHMM.log (default: 1) - --spec_samp_fact SPEC_SAMP_FACT - Make the wavelength grid finer (spec_samp_fact < 1.0) or - coarser (spec_samp_fact > 1.0) by this sampling factor, - i.e. units of spec_samp_fact are pixels. (default: 1.0) - --spat_samp_fact SPAT_SAMP_FACT - Make the spatial grid finer (spat_samp_fact < 1.0) or - coarser (spat_samp_fact > 1.0) by this sampling factor, - i.e. units of spat_samp_fact are pixels. (default: 1.0) \ No newline at end of file diff --git a/doc/help/pypeit_extract_datacube.rst b/doc/help/pypeit_extract_datacube.rst new file mode 100644 index 0000000000..f4b16eab69 --- /dev/null +++ b/doc/help/pypeit_extract_datacube.rst @@ -0,0 +1,29 @@ +.. code-block:: console + + $ pypeit_extract_datacube -h + usage: pypeit_extract_datacube [-h] [-e EXT_FILE] [-s SAVE] [-o] + [-b BOXCAR_RADIUS] [-v VERBOSITY] + file + + Read in a datacube, extract a spectrum of a point source,and save it as a spec1d + file. + + positional arguments: + file spec3d.fits DataCube file + + options: + -h, --help show this help message and exit + -e EXT_FILE, --ext_file EXT_FILE + Configuration file with extraction parameters (default: + None) + -s SAVE, --save SAVE Output spec1d filename (default: None) + -o, --overwrite Overwrite any existing files/directories (default: + False) + -b BOXCAR_RADIUS, --boxcar_radius BOXCAR_RADIUS + Radius of the circular boxcar (in arcseconds) to use for + the extraction. (default: None) + -v VERBOSITY, --verbosity VERBOSITY + Verbosity level between 0 [none] and 2 [all]. Default: + 1. Level 2 writes a log with filename + extract_datacube_YYYYMMDD-HHMM.log (default: 1) + \ No newline at end of file diff --git a/doc/help/pypeit_obslog.rst b/doc/help/pypeit_obslog.rst index 3b4c807e8c..5b399fcee0 100644 --- a/doc/help/pypeit_obslog.rst +++ b/doc/help/pypeit_obslog.rst @@ -10,7 +10,7 @@ using PypeItMetaData. positional arguments: - spec A valid spectrograph identifier: bok_bc, + spec A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, @@ -75,8 +75,11 @@ -s SORT, --sort SORT Metadata keyword (pypeit-specific) to use to sort the output table. (default: mjd) -e EXTENSION, --extension EXTENSION - File extension; compression indicators (e.g. .gz) not - required. (default: .fits) + File extension to use. Must include the period (e.g., + ".fits") and it must be one of the allowed extensions + for this spectrograph. If None, root directory will be + searched for all files with any of the allowed + extensions. (default: None) -d OUTPUT_PATH, --output_path OUTPUT_PATH Path to top-level output directory. (default: current working directory) diff --git a/doc/help/pypeit_ql.rst b/doc/help/pypeit_ql.rst index b8991c356d..1653437b5d 100644 --- a/doc/help/pypeit_ql.rst +++ b/doc/help/pypeit_ql.rst @@ -8,16 +8,17 @@ [--calibs_only] [--overwrite_calibs] [--det DET [DET ...]] [--slitspatnum SLITSPATNUM] [--maskID MASKID] [--boxcar_radius BOXCAR_RADIUS] [--snr_thresh SNR_THRESH] - [--ignore_std] [--skip_display] [--coadd2d] - [--only_slits ONLY_SLITS [ONLY_SLITS ...]] [--offsets OFFSETS] - [--weights WEIGHTS] [--spec_samp_fact SPEC_SAMP_FACT] - [--spat_samp_fact SPAT_SAMP_FACT] [--try_old] + [--ignore_std] [--skip_display] [--removetrace] [--coadd2d] + [--spec_samp_fact SPEC_SAMP_FACT] + [--spat_samp_fact SPAT_SAMP_FACT] [--offsets OFFSETS] + [--weights WEIGHTS] [--only_slits ONLY_SLITS [ONLY_SLITS ...]] + [--try_old] spectrograph Script to produce quick-look PypeIt reductions positional arguments: - spectrograph A valid spectrograph identifier: bok_bc, + spectrograph A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, @@ -110,13 +111,20 @@ detected, ignore those frames. Otherwise, they are included with the reduction of the science frames. (default: False) - --skip_display Run the quicklook without displaying any results. - (default: True) + --skip_display Run the quicklook without displaying any results. The + default skip_display=False will show the results. + (default: False) + --removetrace When the image is shown, do not overplot traces in the + skysub, sky_resid, and resid channels (default: False) --coadd2d Perform default 2D coadding. (default: False) - --only_slits ONLY_SLITS [ONLY_SLITS ...] - If coadding, only coadd this space-separated set of - slits. If not provided, all slits are coadded. (default: - None) + --spec_samp_fact SPEC_SAMP_FACT + If coadding, adjust the wavelength grid sampling by this + factor. For a finer grid, set value to <1.0; for coarser + sampling, set value to >1.0). (default: 1.0) + --spat_samp_fact SPAT_SAMP_FACT + If coadding, adjust the spatial grid sampling by this + factor. For a finer grid, set value to <1.0; for coarser + sampling, set value to >1.0). (default: 1.0) --offsets OFFSETS If coadding, spatial offsets to apply to each image; see the [coadd2d][offsets] parameter. Options are restricted here to either maskdef_offsets or auto. If not @@ -126,14 +134,10 @@ [coadd2d][weights] parameter. Options are restricted here to either uniform or auto. If not specified, the (spectrograph-specific) default is used. (default: None) - --spec_samp_fact SPEC_SAMP_FACT - If coadding, adjust the wavelength grid sampling by this - factor. For a finer grid, set value to <1.0; for coarser - sampling, set value to >1.0). (default: 1.0) - --spat_samp_fact SPAT_SAMP_FACT - If coadding, adjust the spatial grid sampling by this - factor. For a finer grid, set value to <1.0; for coarser - sampling, set value to >1.0). (default: 1.0) + --only_slits ONLY_SLITS [ONLY_SLITS ...] + If coadding, only coadd this space-separated set of + slits. If not provided, all slits are coadded. (default: + None) --try_old Attempt to load old datamodel versions. A crash may ensue.. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_sensfunc.rst b/doc/help/pypeit_sensfunc.rst index 3a5b008e28..26ac6639bb 100644 --- a/doc/help/pypeit_sensfunc.rst +++ b/doc/help/pypeit_sensfunc.rst @@ -1,9 +1,9 @@ .. code-block:: console $ pypeit_sensfunc -h - usage: pypeit_sensfunc [-h] [--algorithm {UVIS,IR}] [--multi MULTI] [-o OUTFILE] - [-s SENS_FILE] [-f FLATFILE] [--debug] - [--par_outfile PAR_OUTFILE] [-v VERBOSITY] + usage: pypeit_sensfunc [-h] [--extr {OPT,BOX}] [--algorithm {UVIS,IR}] + [--multi MULTI] [-o OUTFILE] [-s SENS_FILE] [-f] + [--debug] [--par_outfile PAR_OUTFILE] [-v VERBOSITY] spec1dfile Compute a sensitivity function @@ -14,6 +14,16 @@ options: -h, --help show this help message and exit + --extr {OPT,BOX} Override the default extraction method used for + computing the sensitivity function. Note that it is not + possible to set --extr and simultaneously use a .sens + file with the --sens_file option. If you are using a + .sens file, set the algorithm there via: + + [sensfunc] + extr = BOX + + The extraction options are: OPT or BOX --algorithm {UVIS,IR} Override the default algorithm for computing the sensitivity function. Note that it is not possible to @@ -61,18 +71,20 @@ in the filename. -s SENS_FILE, --sens_file SENS_FILE Configuration file with sensitivity function parameters - -f FLATFILE, --flatfile FLATFILE - Use the flat file for computing the sensitivity - function. Note that it is not possible to set - --flatfile and simultaneously use a .sens file with the - --sens_file option. If you are using a .sens file, set - the flatfile there via e.g.: + -f, --use_flat Use the extracted spectrum of the flatfield calibration + to estimate the blaze function when generating the + sensitivity function. This is helpful to account for + small scale undulations in the sensitivity function. The + spec1dfile must contain the extracted flatfield response + in order to use this option. This spectrum is extracted + by default, unless you did not compute a pixelflat + frame. Note that it is not possible to set --use_flat + and simultaneously use a .sens file with the --sens_file + option. If you are using a .sens file, set the use_flat + flag with the argument: [sensfunc] - flatfile = Calibrations/Flat_A_0_DET01.fits - - Where Flat_A_0_DET01.fits is the flat file in your - Calibrations directory + use_flat = True --debug show debug plots? --par_outfile PAR_OUTFILE Name of output file to save the parameters used by the diff --git a/doc/help/pypeit_setup.rst b/doc/help/pypeit_setup.rst index 32f4da9915..01e0c078dd 100644 --- a/doc/help/pypeit_setup.rst +++ b/doc/help/pypeit_setup.rst @@ -11,7 +11,7 @@ options: -h, --help show this help message and exit -s SPECTROGRAPH, --spectrograph SPECTROGRAPH - A valid spectrograph identifier: bok_bc, + A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, @@ -39,8 +39,11 @@ --extension option to set the types of files to search for. (default: current working directory) -e EXTENSION, --extension EXTENSION - File extension; compression indicators (e.g. .gz) not - required. (default: .fits) + File extension to use. Must include the period (e.g., + ".fits") and it must be one of the allowed extensions + for this spectrograph. If None, root directory will be + searched for all files with any of the allowed + extensions. (default: None) -d OUTPUT_PATH, --output_path OUTPUT_PATH Path to top-level output directory. (default: current working directory) diff --git a/doc/help/pypeit_setup_coadd2d.rst b/doc/help/pypeit_setup_coadd2d.rst index b9d76bd679..54d9f23f6a 100644 --- a/doc/help/pypeit_setup_coadd2d.rst +++ b/doc/help/pypeit_setup_coadd2d.rst @@ -9,6 +9,8 @@ [--exclude_slits EXCLUDE_SLITS [EXCLUDE_SLITS ...]] [--spat_toler SPAT_TOLER] [--offsets OFFSETS] [--weights WEIGHTS] + [--spec_samp_fact SPEC_SAMP_FACT] + [--spat_samp_fact SPAT_SAMP_FACT] Prepare a configuration file for performing 2D coadds @@ -68,4 +70,12 @@ or auto. If not specified, the (spectrograph-specific) default is used. Other options exist but must be entered by directly editing the coadd2d file. (default: None) + --spec_samp_fact SPEC_SAMP_FACT + Make the wavelength grid finer (spec_samp_fact < 1.0) or + coarser (spec_samp_fact > 1.0) by this sampling factor, + i.e. units of spec_samp_fact are pixels. (default: 1.0) + --spat_samp_fact SPAT_SAMP_FACT + Make the spatial grid finer (spat_samp_fact < 1.0) or + coarser (spat_samp_fact > 1.0) by this sampling factor, + i.e. units of spat_samp_fact are pixels. (default: 1.0) \ No newline at end of file diff --git a/doc/help/pypeit_show_2dspec.rst b/doc/help/pypeit_show_2dspec.rst index 1ab710aebf..9b46fbc745 100644 --- a/doc/help/pypeit_show_2dspec.rst +++ b/doc/help/pypeit_show_2dspec.rst @@ -20,7 +20,8 @@ constructed assuming the reduction is for a single detector. If a string, it must match the name of the detector object (e.g., DET01 for a detector, MSC01 for a - mosaic). (default: 1) + mosaic). If not set, the first available detectorin the + spec2d file will be shown (default: None) --spat_id SPAT_ID Restrict plotting to this slit (PypeIt ID notation) (default: None) --maskID MASKID Restrict plotting to this maskID (default: None) diff --git a/doc/help/pypeit_show_pixflat.rst b/doc/help/pypeit_show_pixflat.rst new file mode 100644 index 0000000000..d7394e32e3 --- /dev/null +++ b/doc/help/pypeit_show_pixflat.rst @@ -0,0 +1,17 @@ +.. code-block:: console + + $ pypeit_show_pixflat -h + usage: pypeit_show_pixflat [-h] [--det DET [DET ...]] file + + Show an archived Pixel Flat image in a ginga window. + + positional arguments: + file Pixel Flat filename, e.g. + pixelflat_keck_lris_blue.fits.gz + + options: + -h, --help show this help message and exit + --det DET [DET ...] Detector(s) to show. If more than one, list the detectors + as, e.g. --det 1 2 to show detectors 1 and 2. If not + provided, all detectors will be shown. (default: None) + \ No newline at end of file diff --git a/doc/help/pypeit_trace_edges.rst b/doc/help/pypeit_trace_edges.rst index dc265db21b..a9b5e21814 100644 --- a/doc/help/pypeit_trace_edges.rst +++ b/doc/help/pypeit_trace_edges.rst @@ -26,16 +26,16 @@ default mosaic. (default: None) -s SPECTROGRAPH, --spectrograph SPECTROGRAPH A valid spectrograph identifier, which is only used if - providing files directly: bok_bc, gemini_flamingos1, - gemini_flamingos2, gemini_gmos_north_e2v, - gemini_gmos_north_ham, gemini_gmos_north_ham_ns, - gemini_gmos_south_ham, gemini_gnirs_echelle, - gemini_gnirs_ifu, gtc_maat, gtc_osiris, gtc_osiris_plus, - jwst_nircam, jwst_nirspec, keck_deimos, keck_esi, - keck_hires, keck_kcrm, keck_kcwi, keck_lris_blue, - keck_lris_blue_orig, keck_lris_red, keck_lris_red_mark4, - keck_lris_red_orig, keck_mosfire, keck_nires, - keck_nirspec_high, keck_nirspec_high_old, + providing files directly: aat_uhrf, bok_bc, + gemini_flamingos1, gemini_flamingos2, + gemini_gmos_north_e2v, gemini_gmos_north_ham, + gemini_gmos_north_ham_ns, gemini_gmos_south_ham, + gemini_gnirs_echelle, gemini_gnirs_ifu, gtc_maat, + gtc_osiris, gtc_osiris_plus, jwst_nircam, jwst_nirspec, + keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, + keck_lris_blue, keck_lris_blue_orig, keck_lris_red, + keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, + keck_nires, keck_nirspec_high, keck_nirspec_high_old, keck_nirspec_low, lbt_luci1, lbt_luci2, lbt_mods1b, lbt_mods1r, lbt_mods2b, lbt_mods2r, ldt_deveny, magellan_fire, magellan_fire_long, magellan_mage, diff --git a/doc/help/pypeit_view_fits.rst b/doc/help/pypeit_view_fits.rst index fb6bbd14e2..4ee223de61 100644 --- a/doc/help/pypeit_view_fits.rst +++ b/doc/help/pypeit_view_fits.rst @@ -9,7 +9,7 @@ View FITS files with ginga positional arguments: - spectrograph A valid spectrograph identifier: bok_bc, + spectrograph A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst index 33669bf891..b0262023ad 100644 --- a/doc/help/run_pypeit.rst +++ b/doc/help/run_pypeit.rst @@ -4,14 +4,14 @@ usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c] pypeit_file - ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.16.1.dev84+g643dd5acc + ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.16.1.dev635+g9540496b9 ## ## Available spectrographs include: - ## bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, - ## gemini_gmos_north_ham, gemini_gmos_north_ham_ns, - ## gemini_gmos_south_ham, gemini_gnirs_echelle, gemini_gnirs_ifu, - ## gtc_maat, gtc_osiris, gtc_osiris_plus, jwst_nircam, jwst_nirspec, - ## keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, + ## aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, + ## gemini_gmos_north_e2v, gemini_gmos_north_ham, + ## gemini_gmos_north_ham_ns, gemini_gmos_south_ham, gemini_gnirs_echelle, + ## gemini_gnirs_ifu, gtc_maat, gtc_osiris, gtc_osiris_plus, jwst_nircam, + ## jwst_nirspec, keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, ## keck_lris_blue, keck_lris_blue_orig, keck_lris_red, ## keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, keck_nires, ## keck_nirspec_high, keck_nirspec_high_old, keck_nirspec_low, lbt_luci1, diff --git a/doc/include/class_datamodel_mosaic.rst b/doc/include/class_datamodel_mosaic.rst index ddc075089c..80c523ddef 100644 --- a/doc/include/class_datamodel_mosaic.rst +++ b/doc/include/class_datamodel_mosaic.rst @@ -1,5 +1,5 @@ -**Version**: 1.0.0 +**Version**: 1.0.1 ============== ================ ============================================================ =================================================================================== Attribute Type Array Type Description @@ -7,7 +7,7 @@ Attribute Type Array Type ``binning`` str On-chip binning ``detectors`` `numpy.ndarray`_ :class:`~pypeit.images.detector_container.DetectorContainer` List of objects with detector parameters. ``id`` int Mosaic ID number -``msc_order`` int Order of the interpolation used to construct the mosaic. +``msc_ord`` int Order of the interpolation used to construct the mosaic. ``platescale`` float Detector platescale in arcsec/pixel ``rot`` `numpy.ndarray`_ float Raw, hard-coded rotations (counter-clockwise in degrees) for each unbinned detector ``shape`` tuple Shape of each processed detector image diff --git a/doc/include/class_datamodel_pypeitimage.rst b/doc/include/class_datamodel_pypeitimage.rst index 1f21b5588b..9008ab7bc3 100644 --- a/doc/include/class_datamodel_pypeitimage.rst +++ b/doc/include/class_datamodel_pypeitimage.rst @@ -10,6 +10,7 @@ Attribute Type ``det_img`` `numpy.ndarray`_ `numpy.integer`_ If a detector mosaic, this image provides the detector that contributed to each pixel. ``detector`` :class:`~pypeit.images.detector_container.DetectorContainer`, :class:`~pypeit.images.mosaic.Mosaic` The detector (see :class:`~pypeit.images.detector_container.DetectorContainer`) or mosaic (see :class:`~pypeit.images.mosaic.Mosaic`) parameters ``exptime`` int, float Effective exposure time (s) +``filename`` str Filename for the image ``fullmask`` :class:`~pypeit.images.imagebitmask.ImageBitMaskArray` Image mask ``image`` `numpy.ndarray`_ `numpy.floating`_ Primary image data ``img_scale`` `numpy.ndarray`_ `numpy.floating`_ Image count scaling applied (e.g., 1/flat-field) diff --git a/doc/include/class_datamodel_sensfunc.rst b/doc/include/class_datamodel_sensfunc.rst index fbf98d1922..e57591681c 100644 --- a/doc/include/class_datamodel_sensfunc.rst +++ b/doc/include/class_datamodel_sensfunc.rst @@ -8,6 +8,7 @@ Attribute Type Array Type Description ``airmass`` float Airmass of the observation ``algorithm`` str Algorithm used for the sensitivity calculation. ``exptime`` float Exposure time +``extr`` str Extraction method used for the standard star (OPT or BOX) ``pypeline`` str PypeIt pipeline reduction path ``sens`` `astropy.table.table.Table`_ Table with the sensitivity function ``spec1df`` str PypeIt spec1D file used to for sensitivity function diff --git a/doc/include/class_datamodel_specobj.rst b/doc/include/class_datamodel_specobj.rst index 11492ddf32..9c6afca3d2 100644 --- a/doc/include/class_datamodel_specobj.rst +++ b/doc/include/class_datamodel_specobj.rst @@ -1,5 +1,5 @@ -**Version**: 1.1.10 +**Version**: 1.1.11 ======================= =================================================================================================== ===================== ==================================================================================================================================================================================== Attribute Type Array Type Description @@ -14,6 +14,7 @@ Attribute Type ``BOX_FLAM`` `numpy.ndarray`_ float Boxcar flux (erg/s/cm^2/Ang) ``BOX_FLAM_IVAR`` `numpy.ndarray`_ float Boxcar flux inverse variance (1e-17 erg/s/cm^2/Ang)^-2 ``BOX_FLAM_SIG`` `numpy.ndarray`_ float Boxcar flux uncertainty (1e-17 erg/s/cm^2/Ang) +``BOX_FLAT`` `numpy.ndarray`_ float Boxcar extracted flatfield spectrum, normalized to the peak value. ``BOX_FRAC_USE`` `numpy.ndarray`_ float Fraction of pixels in the object profile subimage used for this extraction ``BOX_FWHM`` `numpy.ndarray`_ float Spectral FWHM (in Angstroms) at every pixel of the boxcar extracted flux. ``BOX_MASK`` `numpy.ndarray`_ `numpy.bool`_ Mask for boxcar extracted flux. True=good @@ -51,6 +52,7 @@ Attribute Type ``OPT_FLAM`` `numpy.ndarray`_ float Optimal flux (1e-17 erg/s/cm^2/Ang) ``OPT_FLAM_IVAR`` `numpy.ndarray`_ float Optimal flux inverse variance (1e-17 erg/s/cm^2/Ang)^-2 ``OPT_FLAM_SIG`` `numpy.ndarray`_ float Optimal flux uncertainty (1e-17 erg/s/cm^2/Ang) +``OPT_FLAT`` `numpy.ndarray`_ float Optimally extracted flatfield spectrum, normalised to the peak value. ``OPT_FRAC_USE`` `numpy.ndarray`_ float Fraction of pixels in the object profile subimage used for this extraction ``OPT_FWHM`` `numpy.ndarray`_ float Spectral FWHM (in Angstroms) at every pixel of the optimally extracted flux. ``OPT_MASK`` `numpy.ndarray`_ `numpy.bool`_ Mask for optimally extracted flux. True=good diff --git a/doc/include/data_dir.rst b/doc/include/data_dir.rst index adda09f16a..1c8e42c4d8 100644 --- a/doc/include/data_dir.rst +++ b/doc/include/data_dir.rst @@ -7,6 +7,7 @@ extinction extinction ... filters filters ... linelist arc_lines/lists ... nist arc_lines/NIST github +pixelflat pixelflats github reid_arxiv arc_lines/reid_arxiv github sensfunc sensfuncs github skisim skisim github diff --git a/doc/include/datamodel_specobj.rst b/doc/include/datamodel_specobj.rst index deb7cb2c19..7de4f07a1a 100644 --- a/doc/include/datamodel_specobj.rst +++ b/doc/include/datamodel_specobj.rst @@ -1,6 +1,6 @@ -Version: 1.1.10 +Version: 1.1.11 ======================= ========================= ================= ==================================================================================================================================================================================== Obj Key Obj Type Array Type Description @@ -15,6 +15,7 @@ Obj Key Obj Type Array Type Descripti ``BOX_FLAM`` ndarray float Boxcar flux (erg/s/cm^2/Ang) ``BOX_FLAM_IVAR`` ndarray float Boxcar flux inverse variance (1e-17 erg/s/cm^2/Ang)^-2 ``BOX_FLAM_SIG`` ndarray float Boxcar flux uncertainty (1e-17 erg/s/cm^2/Ang) +``BOX_FLAT`` ndarray float Boxcar extracted flatfield spectrum, normalized to the peak value. ``BOX_FRAC_USE`` ndarray float Fraction of pixels in the object profile subimage used for this extraction ``BOX_FWHM`` ndarray float Spectral FWHM (in Angstroms) at every pixel of the boxcar extracted flux. ``BOX_MASK`` ndarray bool Mask for boxcar extracted flux. True=good @@ -52,6 +53,7 @@ Obj Key Obj Type Array Type Descripti ``OPT_FLAM`` ndarray float Optimal flux (1e-17 erg/s/cm^2/Ang) ``OPT_FLAM_IVAR`` ndarray float Optimal flux inverse variance (1e-17 erg/s/cm^2/Ang)^-2 ``OPT_FLAM_SIG`` ndarray float Optimal flux uncertainty (1e-17 erg/s/cm^2/Ang) +``OPT_FLAT`` ndarray float Optimally extracted flatfield spectrum, normalised to the peak value. ``OPT_FRAC_USE`` ndarray float Fraction of pixels in the object profile subimage used for this extraction ``OPT_FWHM`` ndarray float Spectral FWHM (in Angstroms) at every pixel of the optimally extracted flux. ``OPT_MASK`` ndarray bool Mask for optimally extracted flux. True=good diff --git a/doc/include/dependencies_table.rst b/doc/include/dependencies_table.rst index eedeadd17a..5156b259ec 100644 --- a/doc/include/dependencies_table.rst +++ b/doc/include/dependencies_table.rst @@ -1,5 +1,5 @@ -======================= ========================================================================================================================================================================================================================================================================================================================================================= -Python Version ``>=3.10,<3.13`` -Required for users ``IPython>=7.10.0``, ``PyERFA>=2.0.0``, ``PyYAML>=5.1``, ``astropy>=6.0``, ``bottleneck``, ``configobj>=5.0.6``, ``extension-helpers>=0.1``, ``fast-histogram>=0.11``, ``ginga>=5.1.0``, ``linetools>=0.3.1``, ``matplotlib>=3.7``, ``numpy>=1.23``, ``packaging>=0.19``, ``pygithub``, ``pyqt6``, ``qtpy>=2.0.1``, ``scikit-learn>=1.0``, ``scipy>=1.7`` -Required for developers ``coverage``, ``docutils<0.21``, ``psutil``, ``pygit2``, ``pytest-astropy``, ``pytest-cov``, ``pytest-qt``, ``pytest>=6.0.0``, ``scikit-image``, ``specutils>=1.13``, ``sphinx-automodapi``, ``sphinx>=1.6,<8``, ``sphinx_rtd_theme==2.0.0``, ``tox`` -======================= ========================================================================================================================================================================================================================================================================================================================================================= +======================= ======================================================================================================================================================================================================================================================================================================================================================== +Python Version ``>=3.11,<3.13`` +Required for users ``IPython>=8.0.0``, ``PyERFA>=2.0.0``, ``PyYAML>=6.0``, ``astropy>=6.0``, ``bottleneck``, ``configobj>=5.0.6``, ``extension-helpers>=1.0``, ``fast-histogram>=0.11``, ``ginga>=5.1.0``, ``linetools>=0.3.2``, ``matplotlib>=3.7``, ``numpy>=1.24``, ``packaging>=22.0``, ``pygithub``, ``pyqt6``, ``qtpy>=2.2.0``, ``scikit-learn>=1.2``, ``scipy>=1.9`` +Required for developers ``coverage``, ``docutils<0.21``, ``psutil``, ``pygit2``, ``pytest-astropy``, ``pytest-cov``, ``pytest-qt``, ``pytest>=7.0.0``, ``scikit-image>=0.23``, ``specutils>=1.13``, ``sphinx-automodapi``, ``sphinx>=1.6,<8``, ``sphinx_rtd_theme==2.0.0``, ``tox`` +======================= ======================================================================================================================================================================================================================================================================================================================================================== diff --git a/doc/include/inst_detector_table.rst b/doc/include/inst_detector_table.rst index cbb0df365a..41c4c8b132 100644 --- a/doc/include/inst_detector_table.rst +++ b/doc/include/inst_detector_table.rst @@ -1,6 +1,7 @@ ============================ === ======== ======== ======== ======== ========================== ====================== ======== ======== ============ ========= ========== Instrument Det specaxis specflip spatflip namp gain RN darkcurr min sat nonlinear platescale ============================ === ======== ======== ======== ======== ========================== ====================== ======== ======== ============ ========= ========== +``aat_uhrf`` 1 0 False False 1 1.0 0.0 0.0 -1.0e+10 65535.0 0.7600 0.0500 ``bok_bc`` 1 1 False False 1 1.5 3.0 5.4 -1.0e+10 65535.0 1.0000 0.2000 ``gemini_flamingos1`` 1 0 False False 1 3.8 6.0 1080.0 -1.0e+10 320000.0 0.8750 0.1500 ``gemini_flamingos2`` 1 0 True False 1 4.44 5.0 1800.0 -1.0e+10 700000.0 1.0000 0.1787 @@ -85,9 +86,9 @@ Instrument Det specaxis specflip spatflip namp gain ``vlt_fors2`` 1 1 False False 1 0.7 2.9 2.1 -1.0e+10 200000.0 0.8000 0.1260 ... 2 1 False False 1 0.7 3.15 1.4 -1.0e+10 200000.0 0.8000 0.1260 ``vlt_sinfoni`` 1 0 True False 1 2.42 7.0 540.0 -1.0e+10 1000000000.0 1.0000 0.0125 -``vlt_xshooter_nir`` 1 1 False False 1 2.12 8.0 0.0 -1.0e+10 200000.0 0.8600 0.1970 -``vlt_xshooter_uvb`` 1 0 True True 1 1.61 2.6 0.0 -1.0e+10 65000.0 0.8600 0.1610 -``vlt_xshooter_vis`` 1 0 False False 1 0.595 3.1 0.0 -1.0e+10 65535.0 0.8600 0.1600 +``vlt_xshooter_nir`` 1 1 False False 1 2.29 8.0 72.0 -1.0e+10 200000.0 0.8600 0.2450 +``vlt_xshooter_uvb`` 1 0 True True 1 ``None`` ``None`` 0.0 -1.0e+10 65000.0 0.8600 0.1640 +``vlt_xshooter_vis`` 1 0 False False 1 ``None`` ``None`` 0.0 -1.0e+10 65535.0 0.8600 0.1540 ``wht_isis_blue`` 1 0 False False 1 1.2 5.0 0.0 -1.0e+10 65535.0 0.7600 0.2000 ``wht_isis_red`` 1 0 False False 1 0.98 4.0 0.0 -1.0e+10 65535.0 0.7600 0.2200 ============================ === ======== ======== ======== ======== ========================== ====================== ======== ======== ============ ========= ========== diff --git a/doc/include/links.rst b/doc/include/links.rst index e4e1f1b84f..f1c47e7444 100644 --- a/doc/include/links.rst +++ b/doc/include/links.rst @@ -128,7 +128,7 @@ .. _pip: https://pip.pypa.io/en/stable/ .. _anaconda: https://www.anaconda.com/products/individual .. _conda: https://docs.conda.io/projects/conda/en/latest/index.html -.. _virtualenv: https://virtualenv.pypa.io/en/latest/ +.. _venv: https://docs.python.org/3/library/venv.html .. _pdb: https://docs.python.org/3/library/pdb.html .. _IPython.embed: https://ipython.readthedocs.io/en/stable/api/generated/IPython.terminal.embed.html#function .. _pytest: https://docs.pytest.org/en/latest/ diff --git a/doc/include/spectrographs_table.rst b/doc/include/spectrographs_table.rst index 1244f6e8e8..2b877fac53 100644 --- a/doc/include/spectrographs_table.rst +++ b/doc/include/spectrographs_table.rst @@ -1,6 +1,7 @@ ======================== ============================================================================ ========= ============ =============================================================================================================================== ========= ========= ========= =============================================================================================== ``PypeIt`` Name ``PypeIt`` Class Telescope Camera URL Pipeline Supported QL Tested Comments ======================== ============================================================================ ========= ============ =============================================================================================================================== ========= ========= ========= =============================================================================================== +aat_uhrf :class:`~pypeit.spectrographs.aat_uhrf.AATUHRFSpectrograph` AAT UHRF `Link `__ MultiSlit True False bok_bc :class:`~pypeit.spectrographs.bok_bc.BokBCSpectrograph` BOK BC `Link `__ MultiSlit True False Bok B&C spectrometer gemini_flamingos1 :class:`~pypeit.spectrographs.gemini_flamingos.GeminiFLAMINGOS1Spectrograph` GEMINI-S FLAMINGOS `Link `__ MultiSlit False False gemini_flamingos2 :class:`~pypeit.spectrographs.gemini_flamingos.GeminiFLAMINGOS2Spectrograph` GEMINI-S FLAMINGOS `Link `__ MultiSlit True False Flamingos-2 NIR spectrograph @@ -17,9 +18,9 @@ jwst_nircam :class:`~pypeit.spectrographs.jwst_nircam.JWSTNIRCamSp jwst_nirspec :class:`~pypeit.spectrographs.jwst_nirspec.JWSTNIRSpecSpectrograph` JWST NIRSPEC `Link `__ MultiSlit True False keck_deimos :class:`~pypeit.spectrographs.keck_deimos.KeckDEIMOSSpectrograph` KECK DEIMOS `Link `__ MultiSlit True True Supported gratings: 600ZD, 830G, 900ZD, 1200B, 1200G; see :doc:`deimos` keck_esi :class:`~pypeit.spectrographs.keck_esi.KeckESISpectrograph` KECK ESI Echelle True False -keck_hires :class:`~pypeit.spectrographs.keck_hires.KECKHIRESSpectrograph` KECK HIRES `Link `__ Echelle False False +keck_hires :class:`~pypeit.spectrographs.keck_hires.KECKHIRESSpectrograph` KECK HIRES `Link `__ Echelle False False Post detector upgrade (~ August 2004). See :doc:`keck_hires` keck_kcrm :class:`~pypeit.spectrographs.keck_kcwi.KeckKCRMSpectrograph` KECK KCRM `Link `__ SlicerIFU True False Supported setups: RL, RM1, RM2, RH3; see :doc:`keck_kcwi` -keck_kcwi :class:`~pypeit.spectrographs.keck_kcwi.KeckKCWISpectrograph` KECK KCWI `Link `__ SlicerIFU True False Supported setups: BL, BM, BH2; see :doc:`keck_kcwi` +keck_kcwi :class:`~pypeit.spectrographs.keck_kcwi.KeckKCWISpectrograph` KECK KCWI `Link `__ SlicerIFU True False Supported setups: BL, BM, BH2, BH3; see :doc:`keck_kcwi` keck_lris_blue :class:`~pypeit.spectrographs.keck_lris.KeckLRISBSpectrograph` KECK LRISb `Link `__ MultiSlit True False Blue camera; Current FITS file format; used from May 2009, see :doc:`lris` keck_lris_blue_orig :class:`~pypeit.spectrographs.keck_lris.KeckLRISBOrigSpectrograph` KECK LRISb `Link `__ MultiSlit True False Blue camera; Original FITS file format; used until April 2009; see :doc:`lris` keck_lris_red :class:`~pypeit.spectrographs.keck_lris.KeckLRISRSpectrograph` KECK LRISr `Link `__ MultiSlit True True Red camera; Current FITS file format; LBNL detector, 2kx4k; used from May 2009, see :doc:`lris` diff --git a/doc/installing.rst b/doc/installing.rst index 79e92d9de6..bf190ca83e 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -36,7 +36,7 @@ Setup a clean python environment PypeIt is available from the `Python Package Index `_ (PyPI) and is installed via ``pip``. This process also installs and/or upgrades -PypeIt's :ref:`dependencies`, and for this reason, we highly (!!) recommend you +PypeIt's :ref:`dependencies`, and for this reason you should always first set up a clean python environment in which to install PypeIt. This mitigates any possible dependency conflicts with other packages you use. @@ -47,20 +47,18 @@ You can set up a new python environment using either `conda`_: conda create -n pypeit python=3.11 conda activate pypeit -or `virtualenv`_: +or `venv`_: .. code-block:: console - virtualenv pypeit + python -m venv pypeit source pypeit/bin/activate See the `Managing Environments with Conda -`_ -and/or `Virtualenv documentation `_ -for more details. See also `virtualenvwrapper -`_ as an option for more -easily managing `virtualenv`_ environments. The `conda`_ installation method described below -creates an environment for you. +`__ +and/or the `venv documentation `__ for +more details. The `conda`_ installation method described below creates an +environment for you. .. _installing-pip: @@ -222,6 +220,41 @@ for x86-64: Solutions/Recommendations/Feedback for these installation options are welcome; please `Submit an issue`_. +.. _install_windows: + +User Installation on Windows +--------------------------------------------- + +#. Download `Python for Windows `_. + +#. Run the installer. + + * Make sure "Add python.exe to Path" or "Add Python to environment + variables" is selected before installing. + + * If you have Admin privileges click "Disable path length limit" after the + installation succeeds. + +#. Download and run the `Visual Studio build tools + `_ installer. + + * Only "Desktop Development with C++" needs to be checked. + + * Click install + +#. Create a virtual environment as in `Setup a clean python environment + `__ and install PypeIt as described above. + +If running ``python`` on Windows brings up a window for the Microsoft Store you +may want to change the application alias. This is under ``Settings -> Apps -> +App execution aliases`` on Windows 10 and ``Settings -> Apps -> Advanced app +settings -> App execution aliases`` on Windows 11. Disable the ``App Installer`` +options for the ``python.exe`` and ``python3.exe`` executables. + +An alternative for running under Windows is to install the `Windows Subsystem +for Linux (WSL) `_. This +in effect allows you to run PypeIt under Linux under Windows. + ---- .. _data_installation: @@ -518,11 +551,12 @@ will work single-threaded if OpenMP is not available. GCC supports OpenMP out of the box, however the ``clang`` compiler that Apple's XCode provides does not. So for optimal performance on Apple hardware, you will want to install GCC via ``homebrew`` or ``macports`` and specify its use when installing ``pypeit``. For example, if you installed -GCC 12.x via ``homebrew``, you would get ``pypeit`` to use it by doing, for example: +GCC via ``homebrew``, you would get ``pypeit`` to use it by doing, for example: .. code-block:: console - CC=gcc-12 pip install pypeit + $ export CC=/opt/homebrew/bin/gcc + $ pip install pypeit Basically, ``pypeit`` checks the ``CC`` environment variable for what compiler to use so configure that as needed to use your desired compiler. The ``pypeit_c_enabled`` script can be used to check if @@ -726,4 +760,85 @@ In either case, over 100 tests should pass, nearly 100 will be skipped and none should fail. The skipped tests only run if the PypeIt development is installed and configured; see :ref:`dev-suite`. +---- + +.. _install_troubleshoot: + +Troubleshooting +=============== + +If you have trouble installing pypeit, you're encouraged to `join +`__ +our `PypeIt Users Slack `__ and post your issue +to the #installing channel. Here is an incomplete list of issues that users +have reported in the past. In addition to posting to the Users Slack if your +issue isn't among those listed below, *please let us know if these suggestions +do not work for you.* + +**I am trying to install pypeit for the first time and it fails!**: The root +problem of this can be system dependent: + + - First, *always* make sure you install the code into a fresh environment. + - If you're on Windows, make sure you follow the :ref:`install_windows` + instructions. If you're still having trouble, it may be because PypeIt + includes some C code to accelerate some computations. If the issue is + because the C compiler is not properly linking, you can try typing ``set + CC=help`` at the command prompt before running the ``pip install`` command. + + - Occasionally, the installation may fail because of incompatible dependencies. + This may be because of recent releases of one of PypeIt's dependencies; i.e., + updates to packages since the most recent PypeIt release. Please let us know + if this happens, and we will try to issue a new release asap that corrects + the incompatibility. In the short-term, we may ask you to install old + versions of packages that we know work. + +**I am trying to upgrade pypeit and it fails!**: First try uninstalling your +current pypeit version: + +.. code-block:: bash + + pip uninstall pypeit + +Then reinstall it. If that also fails, try creating a fresh environment and +reinstalling pypeit in that new environment. + +**The installation process succeeded, but the code is faulting!**: This could +be for a few reasons: + + - Recall that pypeit isn't necessarily backwards compatible. If you've + upgraded pypeit and tried to use it with data that was reduced by a previous + version, the fault may because of changes between versions. You will either + need to revert to your previous version or reprocess the data. + + - This may be because of dependency changes. A tell-tale signature of this is + if you get errors associate with missing or unknown keywords or arguments. + This is may be because of recent releases of one of PypeIt's dependencies; i.e., + updates to packages since the most recent PypeIt release. Please let us know + if this happens, and we will try to issue a new release asap that corrects + the incompatibility. In the short-term, we may ask you to install old + versions of packages that we know work. + +**The installation process succeeded and the code completes without faulting, +but the output looks wrong!**: This could happen for any number of reasons. +*We always welcome reports of failures!* Either `submit an issue +`__ or report it on the PypeIt Users +Slack. However, here are a few things to note and/or try: + + - Make sure you have checked your calibrations; see :ref:`calibrations`. The + issue may be related to a parameter that you can change. + + - If you don't see any ``spec1d`` files in your ``Science`` folder, this is + likely because the code didn't find any objects; see :ref:`object_finding`. + + - If you've recently upgraded the code, this may be related to changes in + dependencies that the developers didn't catch. PypeIt performs *a lot* of + testing before issuing a new release, but does not have complete test + coverage and performance validation. This means silent failures are the most + difficult to catch. + + - And, of course, the code will have bugs. If you find one, the more + information you provide the developers, the easier it will be for us to track + down the issue. Valuable information includes your OS, OS version, python + version, and pypeit version, as well as QA plots and ``ginga`` screen grabs + that illustrate the issue. diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index db0e4ae1c8..668bc3d32c 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -185,6 +185,9 @@ Current PypeItPar Parameter Hierarchy | ``[[lampoffflatsframe]]``: :ref:`framegrouppar` | ``[[[process]]]``: :ref:`processimagespar` | ``[[[[scattlight]]]]``: :ref:`scatteredlightpar` +| ``[[slitless_pixflatframe]]``: :ref:`framegrouppar` +| ``[[[process]]]``: :ref:`processimagespar` +| ``[[[[scattlight]]]]``: :ref:`scatteredlightpar` | ``[[scattlightframe]]``: :ref:`framegrouppar` | ``[[[process]]]``: :ref:`processimagespar` | ``[[[[scattlight]]]]``: :ref:`scatteredlightpar` @@ -252,32 +255,33 @@ CalibrationsPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.CalibrationsPar` -===================== ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= -Key Type Options Default Description -===================== ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= -``alignframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the align frames -``alignment`` :class:`~pypeit.par.pypeitpar.AlignPar` .. `AlignPar Keywords`_ Define the procedure for the alignment of traces -``arcframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the wavelength calibration -``biasframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the bias correction -``bpm_usebias`` bool .. False Make a bad pixel mask from bias frames? Bias frames must be provided. -``calib_dir`` str .. ``Calibrations`` The name of the directory for the processed calibration frames. The host path for the directory is set by the redux_path (see :class:`~pypeit.par.pypeitpar.ReduxPar`). Beware that success when changing the default value is not well tested! -``darkframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the dark-current correction -``flatfield`` :class:`~pypeit.par.pypeitpar.FlatFieldPar` .. `FlatFieldPar Keywords`_ Parameters used to set the flat-field procedure -``illumflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the illumination flat -``lampoffflatsframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the lamp off flats -``pinholeframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the pinholes -``pixelflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the pixel flat -``raise_chk_error`` bool .. True Raise an error if the calibration check fails -``scattlight_pad`` int .. 5 Number of unbinned pixels to extend the slit edges by when masking the slits. -``scattlightframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the scattered light frames -``skyframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the sky background observations -``slitedges`` :class:`~pypeit.par.pypeitpar.EdgeTracePar` .. `EdgeTracePar Keywords`_ Slit-edge tracing parameters -``standardframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the spectrophotometric standard observations -``tiltframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the wavelength tilts -``tilts`` :class:`~pypeit.par.pypeitpar.WaveTiltsPar` .. `WaveTiltsPar Keywords`_ Define how to trace the slit tilts using the trace frames -``traceframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for images used for slit tracing -``wavelengths`` :class:`~pypeit.par.pypeitpar.WavelengthSolutionPar` .. `WavelengthSolutionPar Keywords`_ Parameters used to derive the wavelength solution -===================== ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= +========================= ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= +Key Type Options Default Description +========================= ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= +``alignframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the align frames +``alignment`` :class:`~pypeit.par.pypeitpar.AlignPar` .. `AlignPar Keywords`_ Define the procedure for the alignment of traces +``arcframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the wavelength calibration +``biasframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the bias correction +``bpm_usebias`` bool .. False Make a bad pixel mask from bias frames? Bias frames must be provided. +``calib_dir`` str .. ``Calibrations`` The name of the directory for the processed calibration frames. The host path for the directory is set by the redux_path (see :class:`~pypeit.par.pypeitpar.ReduxPar`). Beware that success when changing the default value is not well tested! +``darkframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the dark-current correction +``flatfield`` :class:`~pypeit.par.pypeitpar.FlatFieldPar` .. `FlatFieldPar Keywords`_ Parameters used to set the flat-field procedure +``illumflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the illumination flat +``lampoffflatsframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the lamp off flats +``pinholeframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the pinholes +``pixelflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the pixel flat +``raise_chk_error`` bool .. True Raise an error if the calibration check fails +``scattlight_pad`` int .. 5 Number of unbinned pixels to extend the slit edges by when masking the slits. +``scattlightframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the scattered light frames +``skyframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the sky background observations +``slitedges`` :class:`~pypeit.par.pypeitpar.EdgeTracePar` .. `EdgeTracePar Keywords`_ Slit-edge tracing parameters +``slitless_pixflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the slitless pixel flat +``standardframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the spectrophotometric standard observations +``tiltframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the wavelength tilts +``tilts`` :class:`~pypeit.par.pypeitpar.WaveTiltsPar` .. `WaveTiltsPar Keywords`_ Define how to trace the slit tilts using the trace frames +``traceframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for images used for slit tracing +``wavelengths`` :class:`~pypeit.par.pypeitpar.WavelengthSolutionPar` .. `WavelengthSolutionPar Keywords`_ Parameters used to derive the wavelength solution +========================= ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= ---- @@ -308,32 +312,33 @@ FlatFieldPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.FlatFieldPar` -========================== ================= ================================= =========== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ -Key Type Options Default Description -========================== ================= ================================= =========== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ -``fit_2d_det_response`` bool .. False Set this variable to True if you want to compute and account for the detector response in the flatfield image. Note that ``detector response`` refers to pixel sensitivity variations that primarily depend on (x,y) detector coordinates. In most cases, the default 2D bspline is sufficient to account for detector response (i.e. set this parameter to False). Note that this correction will _only_ be performed for the spectrographs that have a dedicated response correction implemented. Currently,this correction is only implemented for Keck+KCWI. -``illum_iter`` int .. 0 The number of rejection iterations to perform when constructing the slit-illumination profile. No rejection iterations are performed if 0. WARNING: Functionality still being tested. -``illum_rej`` int, float .. 5.0 The sigma threshold used in the rejection iterations used to refine the slit-illumination profile. Rejection iterations are only performed if ``illum_iter > 0``. -``method`` str ``bspline``, ``skip`` ``bspline`` Method used to flat field the data; use skip to skip flat-fielding. Options are: None, bspline, skip -``pixelflat_file`` str .. .. Filename of the image to use for pixel-level field flattening -``pixelflat_max_wave`` int, float .. .. All values of the normalized pixel flat are set to 1 for wavelengths above this value. -``pixelflat_min_wave`` int, float .. .. All values of the normalized pixel flat are set to 1 for wavelengths below this value. -``rej_sticky`` bool .. False Propagate the rejected pixels through the stages of the flat-field fitting (i.e, from the spectral fit, to the spatial fit, and finally to the 2D residual fit). If False, pixels rejected in each stage are included in each subsequent stage. -``saturated_slits`` str ``crash``, ``mask``, ``continue`` ``crash`` Behavior when a slit is encountered with a large fraction of saturated pixels in the flat-field. The options are: 'crash' - Raise an error and halt the data reduction; 'mask' - Mask the slit, meaning no science data will be extracted from the slit; 'continue' - ignore the flat-field correction, but continue with the reduction. -``slit_illum_finecorr`` bool .. True If True, a fine correction to the spatial illumination profile will be performed. The fine correction is a low order 2D polynomial fit to account for a gradual change to the spatial illumination profile as a function of wavelength. -``slit_illum_pad`` int, float .. 5.0 The number of pixels to pad the slit edges when constructing the slit-illumination profile. Single value applied to both edges. -``slit_illum_ref_idx`` int .. 0 The index of a reference slit (0-indexed) used for estimating the relative spectral sensitivity (or the relative blaze). This parameter is only used if ``slit_illum_relative = True``. -``slit_illum_relative`` bool .. False Generate an image of the relative spectral illumination for a multi-slit setup. If you set ``use_specillum = True`` for any of the frames that use the flatfield model, this *must* be set to True. Currently, this is only used for SlicerIFU reductions. -``slit_illum_smooth_npix`` int .. 10 The number of pixels used to determine smoothly varying relative weights is given by ``nspec/slit_illum_smooth_npix``, where nspec is the number of spectral pixels. -``slit_trim`` int, float, tuple .. 3.0 The number of pixels to trim each side of the slit when selecting pixels to use for fitting the spectral response function. Single values are used for both slit edges; a two-tuple can be used to trim the left and right sides differently. -``spat_samp`` int, float .. 5.0 Spatial sampling for slit illumination function. This is the width of the median filter in pixels used to determine the slit illumination function, and thus sets the minimum scale on which the illumination function will have features. -``spec_samp_coarse`` int, float .. 50.0 bspline break point spacing in units of pixels for 2-d bspline-polynomial fit to flat field image residuals. This should be a large number unless you are trying to fit a sky flat with lots of narrow spectral features. -``spec_samp_fine`` int, float .. 1.2 bspline break point spacing in units of pixels for spectral fit to flat field blaze function. -``tweak_slits`` bool .. True Use the illumination flat field to tweak the slit edges. This will work even if illumflatten is set to False -``tweak_slits_maxfrac`` float .. 0.1 If tweak_slit is True, this sets the maximum fractional amount (of a slits width) allowed for trimming each (i.e. left and right) slit boundary, i.e. the default is 10% which means slits would shrink or grow by at most 20% (10% on each side) -``tweak_slits_thresh`` float .. 0.93 If tweak_slits is True, this sets the illumination function threshold used to tweak the slit boundaries based on the illumination flat. It should be a number less than 1.0 -``twod_fit_npoly`` int .. .. Order of polynomial used in the 2D bspline-polynomial fit to flat-field image residuals. The code determines the order of these polynomials to each slit automatically depending on the slit width, which is why the default is None. Alter this paramter at your own risk! -========================== ================= ================================= =========== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ +========================== ================= ================================= ============= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ +Key Type Options Default Description +========================== ================= ================================= ============= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ +``fit_2d_det_response`` bool .. False Set this variable to True if you want to compute and account for the detector response in the flatfield image. Note that ``detector response`` refers to pixel sensitivity variations that primarily depend on (x,y) detector coordinates. In most cases, the default 2D bspline is sufficient to account for detector response (i.e. set this parameter to False). Note that this correction will _only_ be performed for the spectrographs that have a dedicated response correction implemented. Currently,this correction is only implemented for Keck+KCWI. +``illum_iter`` int .. 0 The number of rejection iterations to perform when constructing the slit-illumination profile. No rejection iterations are performed if 0. WARNING: Functionality still being tested. +``illum_rej`` int, float .. 5.0 The sigma threshold used in the rejection iterations used to refine the slit-illumination profile. Rejection iterations are only performed if ``illum_iter > 0``. +``method`` str ``bspline``, ``skip`` ``bspline`` Method used to flat field the data; use skip to skip flat-fielding. Options are: None, bspline, skip +``pixelflat_file`` str .. .. Filename of the image to use for pixel-level field flattening +``pixelflat_max_wave`` int, float .. .. All values of the normalized pixel flat are set to 1 for wavelengths above this value. +``pixelflat_min_wave`` int, float .. .. All values of the normalized pixel flat are set to 1 for wavelengths below this value. +``rej_sticky`` bool .. False Propagate the rejected pixels through the stages of the flat-field fitting (i.e, from the spectral fit, to the spatial fit, and finally to the 2D residual fit). If False, pixels rejected in each stage are included in each subsequent stage. +``saturated_slits`` str ``crash``, ``mask``, ``continue`` ``crash`` Behavior when a slit is encountered with a large fraction of saturated pixels in the flat-field. The options are: 'crash' - Raise an error and halt the data reduction; 'mask' - Mask the slit, meaning no science data will be extracted from the slit; 'continue' - ignore the flat-field correction, but continue with the reduction. +``slit_illum_finecorr`` bool .. True If True, a fine correction to the spatial illumination profile will be performed. The fine correction is a low order 2D polynomial fit to account for a gradual change to the spatial illumination profile as a function of wavelength. +``slit_illum_pad`` int, float .. 5.0 The number of pixels to pad the slit edges when constructing the slit-illumination profile. Single value applied to both edges. +``slit_illum_ref_idx`` int .. 0 The index of a reference slit (0-indexed) used for estimating the relative spectral sensitivity (or the relative blaze). This parameter is only used if ``slit_illum_relative = True``. +``slit_illum_relative`` bool .. False Generate an image of the relative spectral illumination for a multi-slit setup. If you set ``use_specillum = True`` for any of the frames that use the flatfield model, this *must* be set to True. Currently, this is only used for SlicerIFU reductions. +``slit_illum_smooth_npix`` int .. 10 The number of pixels used to determine smoothly varying relative weights is given by ``nspec/slit_illum_smooth_npix``, where nspec is the number of spectral pixels. +``slit_trim`` int, float, tuple .. 3.0 The number of pixels to trim each side of the slit when selecting pixels to use for fitting the spectral response function. Single values are used for both slit edges; a two-tuple can be used to trim the left and right sides differently. +``spat_samp`` int, float .. 5.0 Spatial sampling for slit illumination function. This is the width of the median filter in pixels used to determine the slit illumination function, and thus sets the minimum scale on which the illumination function will have features. +``spec_samp_coarse`` int, float .. 50.0 bspline break point spacing in units of pixels for 2-d bspline-polynomial fit to flat field image residuals. This should be a large number unless you are trying to fit a sky flat with lots of narrow spectral features. +``spec_samp_fine`` int, float .. 1.2 bspline break point spacing in units of pixels for spectral fit to flat field blaze function. +``tweak_method`` str ``threshold``, ``gradient`` ``threshold`` Method used to tweak the slit edges (when "tweak_slits" is set to True). Options include: threshold, gradient. The "threshold" method determines when the left and right slit edges fall below a threshold relative to the peak illumination. The "gradient" method determines where the gradient is the highest at the left and right slit edges. This method performs better when there is systematic vignetting in the spatial direction. +``tweak_slits`` bool .. True Use the illumination flat field to tweak the slit edges. This will work even if illumflatten is set to False +``tweak_slits_maxfrac`` float .. 0.1 If tweak_slit is True, this sets the maximum fractional amount (of a slits width) allowed for trimming each (i.e. left and right) slit boundary, i.e. the default is 10% which means slits would shrink or grow by at most 20% (10% on each side) +``tweak_slits_thresh`` float .. 0.93 If tweak_slits is True, this sets the illumination function threshold used to tweak the slit boundaries based on the illumination flat. It should be a number less than 1.0 +``twod_fit_npoly`` int .. .. Order of polynomial used in the 2D bspline-polynomial fit to flat-field image residuals. The code determines the order of these polynomials to each slit automatically depending on the slit width, which is why the default is None. Alter this paramter at your own risk! +========================== ================= ================================= ============= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ ---- @@ -345,77 +350,80 @@ EdgeTracePar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.EdgeTracePar` -=========================== ================ =========================================== ============== ====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -=========================== ================ =========================================== ============== ====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== -``add_missed_orders`` bool .. False For any Echelle spectrograph (fixed-format or otherwise), attempt to add orders that have been missed by the automated edge tracing algorithm. For *fixed-format* Echelles, this is based on the expected positions on on the detector. Otherwise, the detected orders are modeled and roughly used to predict the locations of missed orders; see additional parameters ``order_width_poly``, ``order_gap_poly``, and ``order_spat_range``. -``add_predict`` str .. ``nearest`` Sets the method used to predict the shape of the left and right traces for a user-defined slit inserted. Options are (1) ``straight`` inserts traces with a constant spatial pixels position, (2) ``nearest`` inserts traces with a form identical to the automatically identified trace at the nearest spatial position to the inserted slit, or (3) ``pca`` uses the PCA decomposition to predict the shape of the traces. -``add_slits`` str, list .. .. Add one or more user-defined slits. The syntax to define a slit to add is: 'det:spec:spat_left:spat_right' where det=detector, spec=spectral pixel, spat_left=spatial pixel of left slit boundary, and spat_righ=spatial pixel of right slit boundary. For example, '2:2000:2121:2322,3:2000:1201:1500' will add a slit to detector 2 passing through spec=2000 extending spatially from 2121 to 2322 and another on detector 3 at spec=2000 extending from 1201 to 1500. -``auto_pca`` bool .. True During automated tracing, attempt to construct a PCA decomposition of the traces. When True, the edge traces resulting from the initial detection, centroid refinement, and polynomial fitting must meet a set of criteria for performing the pca; see :func:`pypeit.edgetrace.EdgeTraceSet.can_pca`. If False, the ``sync_predict`` parameter *cannot* be set to ``pca``; if it is not, the value is set to ``nearest`` and a warning is issued when validating the parameter set. -``bound_detector`` bool .. False When the code is ready to synchronize the left/right trace edges, the traces should have been constructed, vetted, and cleaned. This can sometimes lead to *no* valid traces. This parameter dictates what to do next. If ``bound_detector`` is True, the code will artificially add left and right edges that bound the detector; if False, the code identifies the slit-edge tracing as being unsuccessful, warns the user, and ends gracefully. Note that setting ``bound_detector`` to True is needed for some long-slit data where the slit edges are, in fact, beyond the edges of the detector. -``clip`` bool .. True Remove traces flagged as bad, instead of only masking them. This is currently only used by :func:`~pypeit.edgetrace.EdgeTraceSet.centroid_refine`. -``det_buffer`` int .. 5 The minimum separation between the detector edges and a slit edge for any added edge traces. Must be positive. -``det_min_spec_length`` int, float .. 0.33 The minimum spectral length (as a fraction of the detector size) of a trace determined by direct measurements of the detector data (as opposed to what should be included in any modeling approach; see fit_min_spec_length). -``dlength_range`` int, float .. .. Similar to ``minimum_slit_dlength``, but constrains the *fractional* change in the slit length as a function of wavelength. For example, a value of 0.2 means that slit length should not vary more than 20%as a function of wavelength. -``edge_detect_clip`` int, float .. .. Sigma clipping level for peaks detected in the collapsed, Sobel-filtered significance image. -``edge_thresh`` int, float .. 20.0 Threshold for finding edges in the Sobel-filtered significance image. -``exclude_regions`` list, str .. .. User-defined regions to exclude from the slit tracing. To set this parameter, the text should be a comma separated list of pixel ranges (in the x direction) to be excluded and the detector number. For example, the following string 1:0:20,1:300:400 would select two regions in det=1 between pixels 0 and 20 and between 300 and 400. -``filt_iter`` int .. 0 Number of median-filtering iterations to perform on sqrt(trace) image before applying to Sobel filter to detect slit/order edges. -``fit_function`` str ``polynomial``, ``legendre``, ``chebyshev`` ``legendre`` Function fit to edge measurements. Options are: polynomial, legendre, chebyshev -``fit_maxdev`` int, float .. 5.0 Maximum deviation between the fitted and measured edge position for rejection in spatial pixels. -``fit_maxiter`` int .. 25 Maximum number of rejection iterations during edge fitting. -``fit_min_spec_length`` float .. 0.6 Minimum unmasked spectral length of a traced slit edge to use in any modeling procedure (polynomial fitting or PCA decomposition). -``fit_niter`` int .. 1 Number of iterations of re-measuring and re-fitting the edge data; see :func:`~pypeit.core.trace.fit_trace`. -``fit_order`` int .. 5 Order of the function fit to edge measurements. -``follow_span`` int .. 20 In the initial connection of spectrally adjacent edge detections, this sets the number of previous spectral rows to consider when following slits forward. -``fwhm_gaussian`` int, float .. 3.0 The `fwhm` parameter to use when using Gaussian weighting in :func:`~pypeit.core.trace.fit_trace` when refining the PCA predictions of edges. See description :func:`~pypeit.core.trace.peak_trace`. -``fwhm_uniform`` int, float .. 3.0 The `fwhm` parameter to use when using uniform weighting in :func:`~pypeit.core.trace.fit_trace` when refining the PCA predictions of edges. See description of :func:`~pypeit.core.trace.peak_trace`. -``gap_offset`` int, float .. 5.0 Offset (pixels) used for the slit edge gap width when inserting slit edges (see `sync_center`) or when nudging predicted slit edges to avoid slit overlaps. This should be larger than `minimum_slit_gap` when converted to arcseconds. -``left_right_pca`` bool .. False Construct a PCA decomposition for the left and right traces separately. This can be important for cross-dispersed echelle spectrographs (e.g., Keck-NIRES) -``length_range`` int, float .. .. Allowed range in slit length compared to the median slit length. For example, a value of 0.3 means that slit lengths should not vary more than 30%. Relatively shorter or longer slits are masked or clipped. Most useful for echelle or multi-slit data where the slits should have similar or identical lengths. -``maskdesign_filename`` str, list .. .. Mask design info contained in this file or files (comma separated) -``maskdesign_maxsep`` int, float .. 50 Maximum allowed offset in pixels between the slit edges defined by the slit-mask design and the traced edges. -``maskdesign_sigrej`` int, float .. 3 Number of sigma for sigma-clipping rejection during slit-mask design matching. -``maskdesign_step`` int, float .. 1 Step in pixels used to generate a list of possible offsets (within +/- `maskdesign_maxsep`) between the slit edges defined by the mask design and the traced edges. -``match_tol`` int, float .. 3.0 Same-side slit edges below this separation in pixels are considered part of the same edge. -``max_nudge`` int, float .. .. If parts of any (predicted) trace fall off the detector edge, allow them to be nudged away from the detector edge up to and including this maximum number of pixels. If None, no limit is set; otherwise should be 0 or larger. -``max_shift_abs`` int, float .. 0.5 Maximum spatial shift in pixels between an input edge location and the recentroided value. -``max_shift_adj`` int, float .. 0.15 Maximum spatial shift in pixels between the edges in adjacent spectral positions. -``max_spat_error`` int, float .. .. Maximum error in the spatial position of edges in pixels. -``minimum_slit_dlength`` int, float .. .. Minimum *change* in the slit length (arcsec) as a function of wavelength in arcsec. This is mostly meant to catch cases when the polynomial fit to the detected edges becomes ill-conditioned (e.g., when the slits run off the edge of the detector) and leads to wild traces. If reducing the order of the polynomial (``fit_order``) does not help, try using this to remove poorly constrained slits. -``minimum_slit_gap`` int, float .. .. Minimum slit gap in arcsec. Gaps between slits are determined by the median difference between the right and left edge locations of adjacent slits. Slits with small gaps are merged by removing the intervening traces.If None, no minimum slit gap is applied. This should be smaller than `gap_offset` when converted to pixels. -``minimum_slit_length`` int, float .. .. Minimum slit length in arcsec. Slit lengths are determined by the median difference between the left and right edge locations for the unmasked trace locations. This is used to identify traces that are *erroneously* matched together to form slits. Short slits are expected to be ignored or removed (see ``clip``). If None, no minimum slit length applied. -``minimum_slit_length_sci`` int, float .. .. Minimum slit length in arcsec for a science slit. Slit lengths are determined by the median difference between the left and right edge locations for the unmasked trace locations. Used in combination with ``minimum_slit_length``, this parameter is used to identify box or alignment slits; i.e., those slits that are shorter than ``minimum_slit_length_sci`` but larger than ``minimum_slit_length`` are box/alignment slits. Box slits are *never* removed (see ``clip``), but no spectra are extracted from them. If None, no minimum science slit length is applied. -``niter_gaussian`` int .. 6 The number of iterations of :func:`~pypeit.core.trace.fit_trace` to use when using Gaussian weighting. -``niter_uniform`` int .. 9 The number of iterations of :func:`~pypeit.core.trace.fit_trace` to use when using uniform weighting. -``order_gap_poly`` int .. 3 Order of the Legendre polynomial used to model the spatial gap between orders as a function of the order spatial position. See ``add_missed_orders``. -``order_match`` int, float .. .. Orders for *fixed-format* echelle spectrographs are always matched to a predefined expectation for the number of orders found and their relative placement in the detector. This sets the tolerance allowed for matching identified "slits" to echelle orders. Must be relative to the fraction of the detector spatial scale (i.e., a value of 0.05 means that the order locations must be within 5% of the expected value). If None, no limit is used. -``order_offset`` int, float .. .. Orders for *fixed-format* echelle spectrographs are always matched to a predefined expectation for the number of orders found and their relative placement in the detector. This sets the offset to introduce to the expected order positions to improve the match for this specific data. This is an additive offset to the measured slit positions; i.e., this should minimize the difference between the expected order positions and ``self.slit_spatial_center() + offset``. Must be in the fraction of the detector spatial scale. If None, no offset is applied. -``order_spat_range`` list .. .. The spatial range of the detector/mosaic over which to predict order locations. If None, the full detector/mosaic range is used. See ``add_missed_orders``. -``order_width_poly`` int .. 2 Order of the Legendre polynomial used to model the spatial width of each order as a function of spatial pixel position. See ``add_missed_orders``. -``overlap`` bool .. False Assume slits identified as abnormally short are actually due to overlaps between adjacent slits/orders. If set to True, you *must* have also used ``length_range`` to identify left-right edge pairs that have an abnormally short separation. For those short slits, the code attempts to convert the short slits into slit gaps. This is particularly useful for blue orders in Keck-HIRES data. -``pad`` int .. 0 Integer number of pixels to consider beyond the slit edges when selecting pixels that are 'on' the slit. -``pca_function`` str ``polynomial``, ``legendre``, ``chebyshev`` ``polynomial`` Type of function fit to the PCA coefficients for each component. Options are: polynomial, legendre, chebyshev -``pca_maxiter`` int .. 25 Maximum number of rejection iterations when fitting the PCA coefficients. -``pca_maxrej`` int .. 1 Maximum number of PCA coefficients rejected during a given fit iteration. -``pca_min_edges`` int .. 4 Minimum number of edge traces required to perform a PCA decomposition of the trace form. If left_right_pca is True, this minimum applies to the number of left and right traces separately. -``pca_n`` int .. .. The number of PCA components to keep, which must be less than the number of detected traces. If not provided, determined by calculating the minimum number of components required to explain a given percentage of variance in the edge data; see `pca_var_percent`. -``pca_order`` int .. 2 Order of the function fit to the PCA coefficients. -``pca_sigrej`` int, float, list .. 2.0, 2.0 Sigma rejection threshold for fitting PCA components. Individual numbers are used for both lower and upper rejection. A list of two numbers sets these explicitly (e.g., [2., 3.]). -``pca_var_percent`` int, float .. 99.8 The percentage (i.e., not the fraction) of the variance in the edge data accounted for by the PCA used to truncate the number of PCA coefficients to keep (see `pca_n`). Ignored if `pca_n` is provided directly. -``rm_slits`` str, list .. .. Remove one or more user-specified slits. The syntax used to define a slit to remove is: 'det:spec:spat' where det=detector, spec=spectral pixel, spat=spatial pixel. For example, '2:2000:2121,3:2000:1500' will remove the slit on detector 2 that contains pixel (spat,spec)=(2000,2121) and on detector 3 that contains pixel (2000,2121). -``smash_range`` list .. 0.0, 1.0 Range of the slit in the spectral direction (in fractional units) to smash when searching for slit edges. If the spectrum covers only a portion of the image, use that range. -``sobel_enhance`` int .. 0 Enhance the sobel filtering? A value of 0 will not enhance the sobel filtering. Any other value > 0 will sum the sobel values. For example, a value of 3 will combine the sobel values for the 3 nearest pixels. This is useful when a slit edge is poorly defined (e.g. vignetted). -``sobel_mode`` str ``nearest``, ``constant`` ``nearest`` Mode for Sobel filtering. Default is 'nearest'; note we find'constant' works best for DEIMOS. -``sync_center`` str ``median``, ``nearest``, ``gap`` ``median`` Mode to use for determining the location of traces to insert. Use `median` to use the median of the matched left and right edge pairs, `nearest` to use the length of the nearest slit, or `gap` to offset by a fixed gap width from the next slit edge. -``sync_predict`` str ``pca``, ``nearest``, ``auto`` ``pca`` Mode to use when predicting the form of the trace to insert. Use `pca` to use the PCA decomposition, `nearest` to reproduce the shape of the nearest trace, or `auto` to let PypeIt decide which mode to use between `pca` and `nearest`. In general, it will first try `pca`, and if that is not possible, it will use `nearest`. -``sync_to_edge`` bool .. True If adding a first left edge or a last right edge, ignore `center_mode` for these edges and place them at the edge of the detector (with the relevant shape). -``trace_median_frac`` int, float .. .. After detection of peaks in the rectified Sobel-filtered image and before refitting the edge traces, the rectified image is median filtered with a kernel width of `trace_median_frac*nspec` along the spectral dimension. -``trace_rms_tol`` int, float .. .. After retracing edges using peaks detected in the rectified and collapsed image, the RMS difference (in pixels) between the original and refit traces are calculated. This sets the upper limit of the RMS for traces that will be removed. If None, no limit is set and all new traces are kept. -``trace_thresh`` int, float .. .. After rectification and median filtering of the Sobel-filtered image (see `trace_median_frac`), values in the median-filtered image *below* this threshold are masked in the refitting of the edge trace data. If None, no masking applied. -``trim_spec`` list .. .. User-defined truncation of all slits in the spectral direction.Should be two integers, e.g. 100,150 trims 100 pixels from the short wavelength end and 150 pixels from the long wavelength end of the spectral axis of the detector. -``use_maskdesign`` bool .. False Use slit-mask designs to identify slits. -=========================== ================ =========================================== ============== ====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== +=========================== ================ =========================================== ============== =================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +=========================== ================ =========================================== ============== =================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== +``add_missed_orders`` bool .. False For any Echelle spectrograph (fixed-format or otherwise), attempt to add orders that have been missed by the automated edge tracing algorithm. For *fixed-format* echelles, this is based on the expected positions on on the detector. Otherwise, the detected orders are modeled and used to predict the locations of missed orders; see additional parameters ``order_width_poly``, ``order_gap_poly``, ``order_fitrej``, ``order_outlier``, and ``order_spat_range``. +``add_predict`` str .. ``nearest`` Sets the method used to predict the shape of the left and right traces for a user-defined slit inserted. Options are (1) ``straight`` inserts traces with a constant spatial pixels position, (2) ``nearest`` inserts traces with a form identical to the automatically identified trace at the nearest spatial position to the inserted slit, or (3) ``pca`` uses the PCA decomposition to predict the shape of the traces. +``add_slits`` str, list .. .. Add one or more user-defined slits. The syntax to define a slit to add is: 'det:spec:spat_left:spat_right' where det=detector, spec=spectral pixel, spat_left=spatial pixel of left slit boundary, and spat_righ=spatial pixel of right slit boundary. For example, '2:2000:2121:2322,3:2000:1201:1500' will add a slit to detector 2 passing through spec=2000 extending spatially from 2121 to 2322 and another on detector 3 at spec=2000 extending from 1201 to 1500. +``auto_pca`` bool .. True During automated tracing, attempt to construct a PCA decomposition of the traces. When True, the edge traces resulting from the initial detection, centroid refinement, and polynomial fitting must meet a set of criteria for performing the pca; see :func:`pypeit.edgetrace.EdgeTraceSet.can_pca`. If False, the ``sync_predict`` parameter *cannot* be set to ``pca``; if it is not, the value is set to ``nearest`` and a warning is issued when validating the parameter set. +``bound_detector`` bool .. False When the code is ready to synchronize the left/right trace edges, the traces should have been constructed, vetted, and cleaned. This can sometimes lead to *no* valid traces. This parameter dictates what to do next. If ``bound_detector`` is True, the code will artificially add left and right edges that bound the detector; if False, the code identifies the slit-edge tracing as being unsuccessful, warns the user, and ends gracefully. Note that setting ``bound_detector`` to True is needed for some long-slit data where the slit edges are, in fact, beyond the edges of the detector. +``clip`` bool .. True Remove traces flagged as bad, instead of only masking them. This is currently only used by :func:`~pypeit.edgetrace.EdgeTraceSet.centroid_refine`. +``det_buffer`` int .. 5 The minimum separation between the detector edges and a slit edge for any added edge traces. Must be positive. +``det_min_spec_length`` int, float .. 0.33 The minimum spectral length (as a fraction of the detector size) of a trace determined by direct measurements of the detector data (as opposed to what should be included in any modeling approach; see fit_min_spec_length). +``dlength_range`` int, float .. .. Similar to ``minimum_slit_dlength``, but constrains the *fractional* change in the slit length as a function of wavelength. For example, a value of 0.2 means that slit length should not vary more than 20%as a function of wavelength. +``edge_detect_clip`` int, float .. .. Sigma clipping level for peaks detected in the collapsed, Sobel-filtered significance image. +``edge_thresh`` int, float .. 20.0 Threshold for finding edges in the Sobel-filtered significance image. +``exclude_regions`` list, str .. .. User-defined regions to exclude from the slit tracing. To set this parameter, the text should be a comma separated list of pixel ranges (in the x direction) to be excluded and the detector number. For example, the following string 1:0:20,1:300:400 would select two regions in det=1 between pixels 0 and 20 and between 300 and 400. +``filt_iter`` int .. 0 Number of median-filtering iterations to perform on sqrt(trace) image before applying to Sobel filter to detect slit/order edges. +``fit_function`` str ``polynomial``, ``legendre``, ``chebyshev`` ``legendre`` Function fit to edge measurements. Options are: polynomial, legendre, chebyshev +``fit_maxdev`` int, float .. 5.0 Maximum deviation between the fitted and measured edge position for rejection in spatial pixels. +``fit_maxiter`` int .. 25 Maximum number of rejection iterations during edge fitting. +``fit_min_spec_length`` float .. 0.6 Minimum unmasked spectral length of a traced slit edge to use in any modeling procedure (polynomial fitting or PCA decomposition). +``fit_niter`` int .. 1 Number of iterations of re-measuring and re-fitting the edge data; see :func:`~pypeit.core.trace.fit_trace`. +``fit_order`` int .. 5 Order of the function fit to edge measurements. +``follow_span`` int .. 20 In the initial connection of spectrally adjacent edge detections, this sets the number of previous spectral rows to consider when following slits forward. +``fwhm_gaussian`` int, float .. 3.0 The `fwhm` parameter to use when using Gaussian weighting in :func:`~pypeit.core.trace.fit_trace` when refining the PCA predictions of edges. See description :func:`~pypeit.core.trace.peak_trace`. +``fwhm_uniform`` int, float .. 3.0 The `fwhm` parameter to use when using uniform weighting in :func:`~pypeit.core.trace.fit_trace` when refining the PCA predictions of edges. See description of :func:`~pypeit.core.trace.peak_trace`. +``gap_offset`` int, float .. 5.0 Offset (pixels) used for the slit edge gap width when inserting slit edges (see `sync_center`) or when nudging predicted slit edges to avoid slit overlaps. This should be larger than `minimum_slit_gap` when converted to arcseconds. +``left_right_pca`` bool .. False Construct a PCA decomposition for the left and right traces separately. This can be important for cross-dispersed echelle spectrographs (e.g., Keck-NIRES) +``length_range`` int, float .. .. Allowed range in slit length compared to the median slit length. For example, a value of 0.3 means that slit lengths should not vary more than 30%. Relatively shorter or longer slits are masked or clipped. Most useful for echelle or multi-slit data where the slits should have similar or identical lengths. +``maskdesign_filename`` str, list .. .. Mask design info contained in this file or files (comma separated) +``maskdesign_maxsep`` int, float .. 50 Maximum allowed offset in pixels between the slit edges defined by the slit-mask design and the traced edges. +``maskdesign_sigrej`` int, float .. 3 Number of sigma for sigma-clipping rejection during slit-mask design matching. +``maskdesign_step`` int, float .. 1 Step in pixels used to generate a list of possible offsets (within +/- `maskdesign_maxsep`) between the slit edges defined by the mask design and the traced edges. +``match_tol`` int, float .. 3.0 Same-side slit edges below this separation in pixels are considered part of the same edge. +``max_nudge`` int, float .. .. If parts of any (predicted) trace fall off the detector edge, allow them to be nudged away from the detector edge up to and including this maximum number of pixels. If None, no limit is set; otherwise should be 0 or larger. +``max_overlap`` float .. .. When adding missing echelle orders based on where existing orders are found, the prediction can yield overlapping orders. The edges of these orders are adjusted to eliminate the overlap, and orders can be added up over the spatial range of the detector set by ``order_spate_range``. If this value is None, orders are added regardless of how much they overlap. If not None, this defines the maximum fraction of an order spatial width that can overlap with other orders. For example, if ``max_overlap=0.5``, any order that overlaps its neighboring orders by more than 50% will not be added as a missing order. +``max_shift_abs`` int, float .. 0.5 Maximum spatial shift in pixels between an input edge location and the recentroided value. +``max_shift_adj`` int, float .. 0.15 Maximum spatial shift in pixels between the edges in adjacent spectral positions. +``max_spat_error`` int, float .. .. Maximum error in the spatial position of edges in pixels. +``minimum_slit_dlength`` int, float .. .. Minimum *change* in the slit length (arcsec) as a function of wavelength in arcsec. This is mostly meant to catch cases when the polynomial fit to the detected edges becomes ill-conditioned (e.g., when the slits run off the edge of the detector) and leads to wild traces. If reducing the order of the polynomial (``fit_order``) does not help, try using this to remove poorly constrained slits. +``minimum_slit_gap`` int, float .. .. Minimum slit gap in arcsec. Gaps between slits are determined by the median difference between the right and left edge locations of adjacent slits. Slits with small gaps are merged by removing the intervening traces.If None, no minimum slit gap is applied. This should be smaller than `gap_offset` when converted to pixels. +``minimum_slit_length`` int, float .. .. Minimum slit length in arcsec. Slit lengths are determined by the median difference between the left and right edge locations for the unmasked trace locations. This is used to identify traces that are *erroneously* matched together to form slits. Short slits are expected to be ignored or removed (see ``clip``). If None, no minimum slit length applied. +``minimum_slit_length_sci`` int, float .. .. Minimum slit length in arcsec for a science slit. Slit lengths are determined by the median difference between the left and right edge locations for the unmasked trace locations. Used in combination with ``minimum_slit_length``, this parameter is used to identify box or alignment slits; i.e., those slits that are shorter than ``minimum_slit_length_sci`` but larger than ``minimum_slit_length`` are box/alignment slits. Box slits are *never* removed (see ``clip``), but no spectra are extracted from them. If None, no minimum science slit length is applied. +``niter_gaussian`` int .. 6 The number of iterations of :func:`~pypeit.core.trace.fit_trace` to use when using Gaussian weighting. +``niter_uniform`` int .. 9 The number of iterations of :func:`~pypeit.core.trace.fit_trace` to use when using uniform weighting. +``order_fitrej`` int, float .. 3.0 When fitting the width of and gap beteween echelle orders with Legendre polynomials, this is the sigma-clipping threshold when excluding data from the fit. See ``add_missed_orders``. +``order_gap_poly`` int .. 3 Order of the Legendre polynomial used to model the spatial gap between orders as a function of the order spatial position. See ``add_missed_orders``. +``order_match`` int, float .. .. Orders for *fixed-format* echelle spectrographs are always matched to a predefined expectation for the number of orders found and their relative placement in the detector. This sets the tolerance allowed for matching identified "slits" to echelle orders. Must be relative to the fraction of the detector spatial scale (i.e., a value of 0.05 means that the order locations must be within 5% of the expected value). If None, no limit is used. +``order_offset`` int, float .. .. Orders for *fixed-format* echelle spectrographs are always matched to a predefined expectation for the number of orders found and their relative placement in the detector. This sets the offset to introduce to the expected order positions to improve the match for this specific data. This is an additive offset to the measured slit positions; i.e., this should minimize the difference between the expected order positions and ``self.slit_spatial_center() + offset``. Must be in the fraction of the detector spatial scale. If None, no offset is applied. +``order_outlier`` int, float .. .. When fitting the width of echelle orders with Legendre polynomials, this is the sigma-clipping threshold used to identify outliers. Orders clipped by this threshold are *removed* from further consideration, whereas orders clipped by ``order_fitrej`` are excluded from the polynomial fit but are not removed. Note this is *only applied to the order widths*, not the order gaps. If None, no "outliers" are identified/removed. Should be larger or equal to ``order_fitrej``. +``order_spat_range`` list .. .. The spatial range of the detector/mosaic over which to predict order locations. If None, the full detector/mosaic range is used. See ``add_missed_orders``. +``order_width_poly`` int .. 2 Order of the Legendre polynomial used to model the spatial width of each order as a function of spatial pixel position. See ``add_missed_orders``. +``overlap`` bool .. False Assume slits identified as abnormally short are actually due to overlaps between adjacent slits/orders. If set to True, you *must* have also used ``length_range`` to identify left-right edge pairs that have an abnormally short separation. For those short slits, the code attempts to convert the short slits into slit gaps. This is particularly useful for blue orders in Keck-HIRES data. +``pad`` int .. 0 Integer number of pixels to consider beyond the slit edges when selecting pixels that are 'on' the slit. +``pca_function`` str ``polynomial``, ``legendre``, ``chebyshev`` ``polynomial`` Type of function fit to the PCA coefficients for each component. Options are: polynomial, legendre, chebyshev +``pca_maxiter`` int .. 25 Maximum number of rejection iterations when fitting the PCA coefficients. +``pca_maxrej`` int .. 1 Maximum number of PCA coefficients rejected during a given fit iteration. +``pca_min_edges`` int .. 4 Minimum number of edge traces required to perform a PCA decomposition of the trace form. If left_right_pca is True, this minimum applies to the number of left and right traces separately. +``pca_n`` int .. .. The number of PCA components to keep, which must be less than the number of detected traces. If not provided, determined by calculating the minimum number of components required to explain a given percentage of variance in the edge data; see `pca_var_percent`. +``pca_order`` int .. 2 Order of the function fit to the PCA coefficients. +``pca_sigrej`` int, float, list .. 2.0, 2.0 Sigma rejection threshold for fitting PCA components. Individual numbers are used for both lower and upper rejection. A list of two numbers sets these explicitly (e.g., [2., 3.]). +``pca_var_percent`` int, float .. 99.8 The percentage (i.e., not the fraction) of the variance in the edge data accounted for by the PCA used to truncate the number of PCA coefficients to keep (see `pca_n`). Ignored if `pca_n` is provided directly. +``rm_slits`` str, list .. .. Remove one or more user-specified slits. The syntax used to define a slit to remove is: 'det:spec:spat' where det=detector, spec=spectral pixel, spat=spatial pixel. For example, '2:2000:2121,3:2000:1500' will remove the slit on detector 2 that contains pixel (spat,spec)=(2000,2121) and on detector 3 that contains pixel (2000,2121). +``smash_range`` list .. 0.0, 1.0 Range of the slit in the spectral direction (in fractional units) to smash when searching for slit edges. If the spectrum covers only a portion of the image, use that range. +``sobel_enhance`` int .. 0 Enhance the sobel filtering? A value of 0 will not enhance the sobel filtering. Any other value > 0 will sum the sobel values. For example, a value of 3 will combine the sobel values for the 3 nearest pixels. This is useful when a slit edge is poorly defined (e.g. vignetted). +``sobel_mode`` str ``nearest``, ``constant`` ``nearest`` Mode for Sobel filtering. Default is 'nearest'; note we find'constant' works best for DEIMOS. +``sync_center`` str ``median``, ``nearest``, ``gap`` ``median`` Mode to use for determining the location of traces to insert. Use `median` to use the median of the matched left and right edge pairs, `nearest` to use the length of the nearest slit, or `gap` to offset by a fixed gap width from the next slit edge. +``sync_predict`` str ``pca``, ``nearest``, ``auto`` ``pca`` Mode to use when predicting the form of the trace to insert. Use `pca` to use the PCA decomposition, `nearest` to reproduce the shape of the nearest trace, or `auto` to let PypeIt decide which mode to use between `pca` and `nearest`. In general, it will first try `pca`, and if that is not possible, it will use `nearest`. +``sync_to_edge`` bool .. True If adding a first left edge or a last right edge, ignore `center_mode` for these edges and place them at the edge of the detector (with the relevant shape). +``trace_median_frac`` int, float .. .. After detection of peaks in the rectified Sobel-filtered image and before refitting the edge traces, the rectified image is median filtered with a kernel width of `trace_median_frac*nspec` along the spectral dimension. +``trace_rms_tol`` int, float .. .. After retracing edges using peaks detected in the rectified and collapsed image, the RMS difference (in pixels) between the original and refit traces are calculated. This sets the upper limit of the RMS for traces that will be removed. If None, no limit is set and all new traces are kept. +``trace_thresh`` int, float .. .. After rectification and median filtering of the Sobel-filtered image (see `trace_median_frac`), values in the median-filtered image *below* this threshold are masked in the refitting of the edge trace data. If None, no masking applied. +``trim_spec`` list .. .. User-defined truncation of all slits in the spectral direction.Should be two integers, e.g. 100,150 trims 100 pixels from the short wavelength end and 150 pixels from the long wavelength end of the spectral axis of the detector. +``use_maskdesign`` bool .. False Use slit-mask designs to identify slits. +=========================== ================ =========================================== ============== =================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== ---- @@ -474,8 +482,8 @@ Key Type Options ``echelle_pad`` int .. 3 Number of orders by which to pad the echellogram reference in the echelle method. Values > 0 allow for some error in the reddest order guess, but require sufficient reference orders. ``frac_rms_thresh`` float .. 1.5 For echelle spectrographs (i.e., ``echelle=True``), this is the fractional change in the RMS threshold used when a 1D fit is re-attempted for failed orders. ``func`` str .. ``legendre`` Function used for wavelength solution fits -``fwhm`` int, float .. 4.0 Spectral sampling of the arc lines. This is the FWHM of an arcline in binned pixels of the input arc image -``fwhm_fromlines`` bool .. True Estimate spectral resolution in each slit using the arc lines. If True, the estimated FWHM will override ``fwhm`` only in the determination of the wavelength solution (including the calculation of the threshold for the solution RMS, see ``rms_thresh_frac_fwhm``), but not for the wave tilts calibration. +``fwhm`` int, float .. 4.0 Spectral sampling of the arc lines. This is the FWHM of an arcline in binned pixels of the input arc image. Note that this is used also in the wave tilts calibration. +``fwhm_fromlines`` bool .. True Estimate spectral resolution in each slit using the arc lines. If True, the estimated FWHM will override ``fwhm`` in the determination of the wavelength solution (including the calculation of the threshold for the solution RMS, see ``rms_thresh_frac_fwhm``), and ALSO for the wave tilts calibration. ``fwhm_spat_order`` int .. 0 This parameter determines the spatial polynomial order to use in the 2D polynomial fit to the FWHM of the arc lines. See also, fwhm_spec_order. ``fwhm_spec_order`` int .. 1 This parameter determines the spectral polynomial order to use in the 2D polynomial fit to the FWHM of the arc lines. See also, fwhm_spat_order. ``lamps`` list .. .. Name of one or more ions used for the wavelength calibration. Use ``None`` for no calibration. Choose ``use_header`` to use the list of lamps recorded in the header of the arc frames (this is currently available only for Keck DEIMOS, Keck LRIS, MMT Blue Channel, and LDT DeVeny). @@ -565,7 +573,9 @@ Key Type Options Default Description ``manual`` str .. .. Manual extraction parameters. det:spat:spec:fwhm:boxcar_radius. Multiple manual extractions are semi-colon separated, and spat,spec are in the pseudo-image generated by COADD2D.boxcar_radius is optional and in pixels (not arcsec!). ``offsets`` str, list .. ``auto`` Offsets for the images being combined (spat pixels). Options are: ``maskdef_offsets``, ``header``, ``auto``, and a list of offsets. Use ``maskdef_offsets`` to use the offsets computed during the slitmask design matching (currently available for these :ref:`slitmask_info_instruments` only). If equal to ``header``, the dither offsets recorded in the header, when available, will be used. If ``auto`` is chosen, PypeIt will try to compute the offsets using a reference object with the highest S/N, or an object selected by the user (see ``user_obj``). If a list of offsets is provided, PypeIt will use it. ``only_slits`` str, list .. .. Restrict coaddition to one or more of slits. Example syntax -- DET01:175,DET02:205 or MSC02:2234. This and ``exclude_slits`` are mutually exclusive. If both are provided, ``only_slits`` takes precedence. +``spat_samp_fact`` float .. 1.0 Make the spatial sampling finer (``spat_samp_fact`` lessthan 1.0) or coarser (``spat_samp_fact`` greather than 1.0) bythis sampling factor. This basically multiples the 'native'spatial pixel size by ``spat_samp_fact``, i.e. the units of``spat_samp_fact`` are pixels. ``spat_toler`` int .. 5 This parameter provides the desired tolerance in spatial pixel used to identify slits in different exposures +``spec_samp_fact`` float .. 1.0 Make the wavelength grid sampling finer (``spec_samp_fact`` less than 1.0)or coarser (``spec_samp_fact`` greater than 1.0) by this sampling factor.This multiples the 'native' spectral pixel size by ``spec_samp_fact``,i.e. the units of ``spec_samp_fact`` are pixels. ``use_slits4wvgrid`` bool .. False If True, use the slits to set the trace down the center ``user_obj`` int, list .. .. Object that the user wants to use to compute the weights and/or the offsets for coadding images. For longslit/multislit spectroscopy, provide the ``SLITID`` and the ``OBJID``, separated by comma, of the selected object. For echelle spectroscopy, provide the ``ECH_OBJID`` of the selected object. See :doc:`out_spec1D` for more info about ``SLITID``, ``OBJID`` and ``ECH_OBJID``. If this parameter is not ``None``, it will be used to compute the offsets only if ``offsets = auto``, and it will used to compute the weights only if ``weights = auto``. ``wave_method`` str .. .. Argument to :func:`~pypeit.core.wavecal.wvutils.get_wave_grid` method, which determines how the 2d coadd wavelength grid is constructed. The default is None, which will use a linear gridfor longslit/multislit coadds and a log10 grid for echelle coadds. Currently supported options with 2d coadding are:* 'iref' -- Use one of the exposures (the first) as the reference for the wavelength grid * 'velocity' -- Grid is uniform in velocity* 'log10' -- Grid is uniform in log10(wave). This is the same as velocity.* 'linear' -- Grid is uniform in wavelength @@ -706,7 +716,6 @@ Key Type Options ``correct_dar`` bool .. True If True, the data will be corrected for differential atmospheric refraction (DAR). ``dec_max`` float .. .. Maximum DEC to use when generating the WCS. If None, the default is maximum DEC based on the WCS of all spaxels. Units should be degrees. ``dec_min`` float .. .. Minimum DEC to use when generating the WCS. If None, the default is minimum DEC based on the WCS of all spaxels. Units should be degrees. -``grating_corr`` bool .. True This option performs a small correction for the relative blaze function of all input frames that have (even slightly) different grating angles, or if you are flux calibrating your science data with a standard star that was observed with a slightly different setup. ``method`` str ``subpixel``, ``ngp`` ``subpixel`` What method should be used to generate the datacube. There are currently two options: (1) "subpixel" (default) - this algorithm divides each pixel in the spec2d frames into subpixels, and assigns each subpixel to a voxel of the datacube. Flux is conserved, but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. See also, spec_subpixel and spat_subpixel. (2) "ngp" (nearest grid point) - this algorithm is effectively a 3D histogram. Flux is conserved, voxels are not correlated, however this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. This algorithm takes each pixel on the spec2d frame and puts the flux of this pixel into one voxel in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighboring voxel might contain the flux from two spec2d pixels. Note that all spec2d pixels that contribute to the same voxel are inverse variance weighted (e.g. if two pixels have the same variance, the voxel would be assigned the average flux of the two pixels). ``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with ``spec3d_*`` ``ra_max`` float .. .. Maximum RA to use when generating the WCS. If None, the default is maximum RA based on the WCS of all spaxels. Units should be degrees. @@ -714,13 +723,13 @@ Key Type Options ``reference_image`` str .. .. White light image of a previously combined datacube. The white light image will be used as a reference when calculating the offsets of the input spec2d files. Ideally, the reference image should have the same shape as the data to be combined (i.e. set the ra_min, ra_max etc. params so they are identical to the reference image). ``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. To set the wavelength range, use the "whitelight_range" parameter. If combine=False, the individual spec3d files will have a suffix "_whitelight". ``scale_corr`` str .. .. This option performs a small correction for the relative spectral illumination scale of different spec2D files. Specify the relative path+file to the spec2D file that you would like to use for the relative scaling. If you want to perform this correction, it is best to use the spec2d file with the highest S/N sky spectrum. You should choose the same frame for both the standards and science frames. +``sensfile`` str .. .. Filename of a sensitivity function to use to flux calibrate your datacube. The sensitivity function file will also be used to correct the relative scales of the slits. ``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time; the sky model will be scaled to the science frame based on the relative exposure time. ``slice_subpixel`` int .. 5 When method=subpixel, slice_subpixel sets the subpixellation scale of each IFU slice. The default option is to divide each slice into 5 sub-slices during datacube creation. See also, spec_subpixel and spat_subpixel. ``slit_spec`` bool .. True If the data use slits in one spatial direction, set this to True. If the data uses fibres for all spaxels, set this to False. ``spat_subpixel`` int .. 5 When method=subpixel, spat_subpixel sets the subpixellation scale of each detector pixel in the spatial direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spec_subpixel and slice_subpixel. ``spatial_delta`` float .. .. The spatial size of each spaxel to use when generating the WCS (in arcsec). If None, the default is set by the spectrograph file. ``spec_subpixel`` int .. 5 When method=subpixel, spec_subpixel sets the subpixellation scale of each detector pixel in the spectral direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spat_subpixel and slice_subpixel. -``standard_cube`` str .. .. Filename of a standard star datacube. This cube will be used to correct the relative scales of the slits, and to flux calibrate the science datacube. ``wave_delta`` float .. .. The wavelength step to use when generating the WCS (in Angstroms). If None, the default is set by the wavelength solution. ``wave_max`` float .. .. Maximum wavelength to use when generating the WCS. If None, the default is maximum wavelength based on the WCS of all spaxels. Units should be Angstroms. ``wave_min`` float .. .. Minimum wavelength to use when generating the WCS. If None, the default is minimum wavelength based on the WCS of all spaxels. Units should be Angstroms. @@ -781,8 +790,9 @@ Key Type Options Default Description ``skip_second_find`` bool .. False Only perform one round of object finding (mainly for quick_look) ``skip_skysub`` bool .. False If True, do not sky subtract when performing object finding. This should be set to True for example when running on data that is already sky-subtracted. Note that for near-IR difference imaging one still wants to remove sky-residuals via sky-subtraction, and so this is typically set to False ``snr_thresh`` int, float .. 10.0 S/N threshold for object finding in wavelength direction smashed image. -``std_spec1d`` str .. .. A PypeIt spec1d file of a previously reduced standard star. The trace of the standard star spectrum is used as a crutch for tracing the object spectra, when a direct trace is not possible (i.e., faint sources). If provided, this overrides use of any standards included in your pypeit file; the standard exposures will still be reduced. +``std_spec1d`` str .. .. A PypeIt spec1d file of a previously reduced standard star. This can be used to trace the object spectra, but the ``use_std_trace`` parameter must be set to True. If provided, this overrides use of any standards included in your pypeit file; the standard exposures will still be reduced. ``trace_npoly`` int .. 5 Order of legendre polynomial fits to object traces. +``use_std_trace`` bool .. True If True, the trace of the standard star spectrum is used as a crutch for tracing the object spectra. This is useful when a direct trace is not possible (i.e., faint sources). Note that a standard star exposure must be included in your pypeit file, or the ``std_spec1d`` parameter must be set for this to work. =========================== ========== ======= ======= ============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= @@ -845,14 +855,14 @@ FrameGroupPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.FrameGroupPar` -============= =============================================== ============================================================================================================================================================================ ============================ =============================================================================================================================================================================================================================================================== -Key Type Options Default Description -============= =============================================== ============================================================================================================================================================================ ============================ =============================================================================================================================================================================================================================================================== -``exprng`` list .. None, None Used in identifying frames of this type. This sets the minimum and maximum allowed exposure times. There must be two items in the list. Use None to indicate no limit; i.e., to select exposures with any time greater than 30 sec, use exprng = [30, None]. -``frametype`` str ``align``, ``arc``, ``bias``, ``dark``, ``pinhole``, ``pixelflat``, ``illumflat``, ``lampoffflats``, ``scattlight``, ``science``, ``standard``, ``trace``, ``tilt``, ``sky`` ``science`` Frame type. Options are: align, arc, bias, dark, pinhole, pixelflat, illumflat, lampoffflats, scattlight, science, standard, trace, tilt, sky -``process`` :class:`~pypeit.par.pypeitpar.ProcessImagesPar` .. `ProcessImagesPar Keywords`_ Low level parameters used for basic image processing -``useframe`` str .. .. A calibrations file to use if it exists. -============= =============================================== ============================================================================================================================================================================ ============================ =============================================================================================================================================================================================================================================================== +============= =============================================== ================================================================================================================================================================================================== ============================ =============================================================================================================================================================================================================================================================== +Key Type Options Default Description +============= =============================================== ================================================================================================================================================================================================== ============================ =============================================================================================================================================================================================================================================================== +``exprng`` list .. None, None Used in identifying frames of this type. This sets the minimum and maximum allowed exposure times. There must be two items in the list. Use None to indicate no limit; i.e., to select exposures with any time greater than 30 sec, use exprng = [30, None]. +``frametype`` str ``align``, ``arc``, ``bias``, ``dark``, ``pinhole``, ``pixelflat``, ``illumflat``, ``lampoffflats``, ``slitless_pixflat``, ``scattlight``, ``science``, ``standard``, ``trace``, ``tilt``, ``sky`` ``science`` Frame type. Options are: align, arc, bias, dark, pinhole, pixelflat, illumflat, lampoffflats, slitless_pixflat, scattlight, science, standard, trace, tilt, sky +``process`` :class:`~pypeit.par.pypeitpar.ProcessImagesPar` .. `ProcessImagesPar Keywords`_ Low level parameters used for basic image processing +``useframe`` str .. .. A calibrations file to use if it exists. +============= =============================================== ================================================================================================================================================================================================== ============================ =============================================================================================================================================================================================================================================================== ---- @@ -864,43 +874,45 @@ ProcessImagesPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.ProcessImagesPar` -======================== ================================================ =================================================================== ============================= ============================================================================================================================================================================================================================================================================================================================================================ -Key Type Options Default Description -======================== ================================================ =================================================================== ============================= ============================================================================================================================================================================================================================================================================================================================================================ -``apply_gain`` bool .. True Convert the ADUs to electrons using the detector gain -``clip`` bool .. True Perform sigma clipping when combining. Only used with combine=mean -``comb_sigrej`` float .. .. Sigma-clipping level for when clip=True; Use None for automatic limit (recommended). -``combine`` str ``median``, ``mean`` ``mean`` Method used to combine multiple frames. Options are: median, mean -``dark_expscale`` bool .. False If designated dark frames are used and have a different exposure time than the science frames, scale the counts by the by the ratio in the exposure times to adjust the dark counts for the difference in exposure time. WARNING: You should always take dark frames that have the same exposure time as your science frames, so use this option with care! -``empirical_rn`` bool .. False If True, use the standard deviation in the overscan region to measure an empirical readnoise to use in the noise model. -``grow`` int, float .. 1.5 Factor by which to expand regions with cosmic rays detected by the LA cosmics routine. -``lamaxiter`` int .. 1 Maximum number of iterations for LA cosmics routine. -``mask_cr`` bool .. False Identify CRs and mask them -``n_lohi`` list .. 0, 0 Number of pixels to reject at the lowest and highest ends of the distribution; i.e., n_lohi = low, high. Use None for no limit. -``noise_floor`` float .. 0.0 Impose a noise floor by adding the provided fraction of the bias- and dark-subtracted electron counts to the error budget. E.g., a value of 0.01 means that the S/N of the counts in the image will never be greater than 100. -``objlim`` int, float .. 3.0 Object detection limit in LA cosmics routine -``orient`` bool .. True Orient the raw image into the PypeIt frame -``overscan_method`` str ``chebyshev``, ``polynomial``, ``savgol``, ``median``, ``odd_even`` ``savgol`` Method used to fit the overscan. Options are: chebyshev, polynomial, savgol, median, odd_even Note: Method "polynomial" is identical to "chebyshev"; the former is deprecated and will be removed. -``overscan_par`` int, list .. 5, 65 Parameters for the overscan subtraction. For 'chebyshev' or 'polynomial', set overcan_par = order; for 'savgol', set overscan_par = order, window size ; for 'median', set overscan_par = None or omit the keyword. -``rmcompact`` bool .. True Remove compact detections in LA cosmics routine -``satpix`` str ``reject``, ``force``, ``nothing`` ``reject`` Handling of saturated pixels. Options are: reject, force, nothing -``scattlight`` :class:`~pypeit.par.pypeitpar.ScatteredLightPar` .. `ScatteredLightPar Keywords`_ Scattered light subtraction parameters. -``shot_noise`` bool .. True Use the bias- and dark-subtracted image to calculate and include electron count shot noise in the image processing error budget -``sigclip`` int, float .. 4.5 Sigma level for rejection in LA cosmics routine -``sigfrac`` int, float .. 0.3 Fraction for the lower clipping threshold in LA cosmics routine. -``spat_flexure_correct`` bool .. False Correct slits, illumination flat, etc. for flexure -``spat_flexure_maxlag`` int .. 20 Maximum of possible spatial flexure correction, in pixels -``subtract_continuum`` bool .. False Subtract off the continuum level from an image. This parameter should only be set to True to combine arcs with multiple different lamps. For all other cases, this parameter should probably be False. -``subtract_scattlight`` bool .. False Subtract off the scattered light from an image. This parameter should only be set to True for spectrographs that have dedicated methods to subtract scattered light. For all other cases, this parameter should be False. -``trim`` bool .. True Trim the image to the detector supplied region -``use_biasimage`` bool .. True Use a bias image. If True, one or more must be supplied in the PypeIt file. -``use_darkimage`` bool .. False Subtract off a dark image. If True, one or more darks must be provided. -``use_illumflat`` bool .. True Use the illumination flat to correct for the illumination profile of each slit. -``use_overscan`` bool .. True Subtract off the overscan. Detector *must* have one or code will crash. -``use_pattern`` bool .. False Subtract off a detector pattern. This pattern is assumed to be sinusoidal along one direction, with a frequency that is constant across the detector. -``use_pixelflat`` bool .. True Use the pixel flat to make pixel-level corrections. A pixelflat image must be provied. -``use_specillum`` bool .. False Use the relative spectral illumination profiles to correct the spectral illumination profile of each slit. This is primarily used for slicer IFUs. To use this, you must set ``slit_illum_relative=True`` in the ``flatfield`` parameter set! -======================== ================================================ =================================================================== ============================= ============================================================================================================================================================================================================================================================================================================================================================ +======================== ================================================ =================================================================== ============================= ======================================================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +======================== ================================================ =================================================================== ============================= ======================================================================================================================================================================================================================================================================================================================================================================================== +``apply_gain`` bool .. True Convert the ADUs to electrons using the detector gain +``clip`` bool .. True Perform sigma clipping when combining. Only used with combine=mean +``comb_sigrej`` float .. .. Sigma-clipping level for when clip=True; Use None for automatic limit (recommended). +``combine`` str ``median``, ``mean`` ``mean`` Method used to combine multiple frames. Options are: median, mean +``correct_nonlinear`` list .. .. Correct for non-linear response of the detector. If None, no correction is performed. If a list, then the list should be the non-linear correction parameter (alpha), where the functional form is given by Ct = Cm (1 + alpha x Cm), with Ct and Cm the true and measured counts. This parameter is usually hard-coded for a given spectrograph, and should otherwise be left as None. +``dark_expscale`` bool .. False If designated dark frames are used and have a different exposure time than the science frames, scale the counts by the by the ratio in the exposure times to adjust the dark counts for the difference in exposure time. WARNING: You should always take dark frames that have the same exposure time as your science frames, so use this option with care! +``empirical_rn`` bool .. False If True, use the standard deviation in the overscan region to measure an empirical readnoise to use in the noise model. +``grow`` int, float .. 1.5 Factor by which to expand regions with cosmic rays detected by the LA cosmics routine. +``lamaxiter`` int .. 1 Maximum number of iterations for LA cosmics routine. +``mask_cr`` bool .. False Identify CRs and mask them +``n_lohi`` list .. 0, 0 Number of pixels to reject at the lowest and highest ends of the distribution; i.e., n_lohi = low, high. Use None for no limit. +``noise_floor`` float .. 0.0 Impose a noise floor by adding the provided fraction of the bias- and dark-subtracted electron counts to the error budget. E.g., a value of 0.01 means that the S/N of the counts in the image will never be greater than 100. +``objlim`` int, float .. 3.0 Object detection limit in LA cosmics routine +``orient`` bool .. True Orient the raw image into the PypeIt frame +``overscan_method`` str ``chebyshev``, ``polynomial``, ``savgol``, ``median``, ``odd_even`` ``savgol`` Method used to fit the overscan. Options are: chebyshev, polynomial, savgol, median, odd_even Note: Method "polynomial" is identical to "chebyshev"; the former is deprecated and will be removed. +``overscan_par`` int, list .. 5, 65 Parameters for the overscan subtraction. For 'chebyshev' or 'polynomial', set overcan_par = order; for 'savgol', set overscan_par = order, window size ; for 'median', set overscan_par = None or omit the keyword. +``rmcompact`` bool .. True Remove compact detections in LA cosmics routine +``satpix`` str ``reject``, ``force``, ``nothing`` ``reject`` Handling of saturated pixels. Options are: reject, force, nothing +``scale_to_mean`` bool .. False If True, scale the input images to have the same mean before combining. +``scattlight`` :class:`~pypeit.par.pypeitpar.ScatteredLightPar` .. `ScatteredLightPar Keywords`_ Scattered light subtraction parameters. +``shot_noise`` bool .. True Use the bias- and dark-subtracted image to calculate and include electron count shot noise in the image processing error budget +``sigclip`` int, float .. 4.5 Sigma level for rejection in LA cosmics routine +``sigfrac`` int, float .. 0.3 Fraction for the lower clipping threshold in LA cosmics routine. +``spat_flexure_correct`` bool .. False Correct slits, illumination flat, etc. for flexure +``spat_flexure_maxlag`` int .. 20 Maximum of possible spatial flexure correction, in pixels +``subtract_continuum`` bool .. False Subtract off the continuum level from an image. This parameter should only be set to True to combine arcs with multiple different lamps. For all other cases, this parameter should probably be False. +``subtract_scattlight`` bool .. False Subtract off the scattered light from an image. This parameter should only be set to True for spectrographs that have dedicated methods to subtract scattered light. For all other cases, this parameter should be False. +``trim`` bool .. True Trim the image to the detector supplied region +``use_biasimage`` bool .. True Use a bias image. If True, one or more must be supplied in the PypeIt file. +``use_darkimage`` bool .. False Subtract off a dark image. If True, one or more darks must be provided. +``use_illumflat`` bool .. True Use the illumination flat to correct for the illumination profile of each slit. +``use_overscan`` bool .. True Subtract off the overscan. Detector *must* have one or code will crash. +``use_pattern`` bool .. False Subtract off a detector pattern. This pattern is assumed to be sinusoidal along one direction, with a frequency that is constant across the detector. +``use_pixelflat`` bool .. True Use the pixel flat to make pixel-level corrections. A pixelflat image must be provied. +``use_specillum`` bool .. False Use the relative spectral illumination profiles to correct the spectral illumination profile of each slit. This is primarily used for slicer IFUs. To use this, you must set ``slit_illum_relative=True`` in the ``flatfield`` parameter set! +======================== ================================================ =================================================================== ============================= ======================================================================================================================================================================================================================================================================================================================================================================================== ---- @@ -938,9 +950,9 @@ Key Type Options ``IR`` :class:`~pypeit.par.pypeitpar.TelluricPar` .. `TelluricPar Keywords`_ Parameters for the IR sensfunc algorithm ``UVIS`` :class:`~pypeit.par.pypeitpar.SensfuncUVISPar` .. `SensfuncUVISPar Keywords`_ Parameters for the UVIS sensfunc algorithm ``algorithm`` str ``UVIS``, ``IR`` ``UVIS`` Specify the algorithm for computing the sensitivity function. The options are: (1) UVIS = Should be used for data with :math:`\lambda < 7000` A. No detailed model of telluric absorption but corrects for atmospheric extinction. (2) IR = Should be used for data with :math:`\lambda > 7000` A. Peforms joint fit for sensitivity function and telluric absorption using HITRAN models. +``extr`` str .. ``OPT`` Extraction method to use for the sensitivity function. Options are: 'OPT' (optimal extraction), 'BOX' (boxcar extraction). Default is 'OPT'. ``extrap_blu`` float .. 0.1 Fraction of minimum wavelength coverage to grow the wavelength coverage of the sensitivitity function in the blue direction (`i.e.`, if the standard star spectrum cuts off at ``wave_min``) the sensfunc will be extrapolated to cover down to (1.0 - ``extrap_blu``) * ``wave_min`` ``extrap_red`` float .. 0.1 Fraction of maximum wavelength coverage to grow the wavelength coverage of the sensitivitity function in the red direction (`i.e.`, if the standard star spectrumcuts off at ``wave_max``) the sensfunc will be extrapolated to cover up to (1.0 + ``extrap_red``) * ``wave_max`` -``flatfile`` str .. .. Flat field file to be used if the sensitivity function model will utilize the blaze function computed from a flat field file in the Calibrations directory, e.g.Calibrations/Flat_A_0_DET01.fits ``hydrogen_mask_wid`` float .. 10.0 Mask width from line center for hydrogen recombination lines in Angstroms (total mask width is 2x this value). ``mask_helium_lines`` bool .. False Mask certain ``HeII`` recombination lines prominent in O-type stars in the sensitivity function fit A region equal to 0.5 * ``hydrogen_mask_wid`` on either side of the line center is masked. ``mask_hydrogen_lines`` bool .. True Mask hydrogen Balmer, Paschen, Brackett, and Pfund recombination lines in the sensitivity function fit. A region equal to ``hydrogen_mask_wid`` on either side of the line center is masked. @@ -951,6 +963,7 @@ Key Type Options ``star_mag`` float .. .. Magnitude of the standard star (for near-IR mainly) ``star_ra`` float .. .. RA of the standard star. This will override values in the header (`i.e.`, if they are wrong or absent) ``star_type`` str .. .. Spectral type of the standard star (for near-IR mainly) +``use_flat`` bool .. False If True, the flatfield spectrum will be used when computing the sensitivity function. ======================= ============================================== ================ =========================== ============================================================================================================================================================================================================================================================================================================================================================================================ @@ -1047,6 +1060,104 @@ these in the PypeIt file, you would be reproducing the effect of the `default_pypeit_par` method specific to each derived :class:`~pypeit.spectrographs.spectrograph.Spectrograph` class. +.. _instr_par-aat_uhrf: + +AAT UHRF (``aat_uhrf``) +----------------------- +Alterations to the default parameters are: + +.. code-block:: ini + + [rdx] + spectrograph = aat_uhrf + [calibrations] + [[biasframe]] + [[[process]]] + combine = median + use_biasimage = False + shot_noise = False + use_pixelflat = False + use_illumflat = False + [[darkframe]] + [[[process]]] + mask_cr = True + use_pixelflat = False + use_illumflat = False + [[arcframe]] + exprng = None, 60.0, + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[tiltframe]] + exprng = None, 60.0, + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[pixelflatframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[alignframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[traceframe]] + exprng = None, 60.0, + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[illumflatframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[lampoffflatsframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[scattlightframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[skyframe]] + [[[process]]] + mask_cr = True + noise_floor = 0.01 + [[standardframe]] + [[[process]]] + mask_cr = True + noise_floor = 0.01 + [[wavelengths]] + lamps = ThAr, + n_final = 3 + [[slitedges]] + sync_predict = nearest + bound_detector = True + [[tilts]] + spat_order = 4 + spec_order = 1 + [scienceframe] + exprng = 61, None, + [[process]] + mask_cr = True + sigclip = 10.0 + noise_floor = 0.01 + [reduce] + [[skysub]] + bspline_spacing = 3.0 + no_poly = True + user_regions = :10,75: + .. _instr_par-bok_bc: BOK BC (``bok_bc``) @@ -1129,6 +1240,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1257,6 +1376,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1385,6 +1512,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1497,6 +1632,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1586,6 +1727,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1675,6 +1822,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1764,6 +1917,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1880,6 +2039,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2009,6 +2176,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2031,12 +2206,11 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[flatfield]] + tweak_method = gradient tweak_slits_thresh = 0.0 tweak_slits_maxfrac = 0.0 slit_trim = 2 slit_illum_finecorr = False - [[slitedges]] - pad = 2 [[tilts]] spat_order = 1 spec_order = 1 @@ -2061,8 +2235,6 @@ Alterations to the default parameters are: [[extraction]] model_full_slit = True skip_extraction = True - [[cube]] - grating_corr = False [flexure] spec_maxshift = 0 [sensfunc] @@ -2137,6 +2309,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2152,6 +2330,9 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[flatfield]] + tweak_method = gradient + tweak_slits_thresh = 0.0 + tweak_slits_maxfrac = 0.0 slit_illum_finecorr = False [[wavelengths]] method = full_template @@ -2251,6 +2432,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2350,6 +2537,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2440,6 +2633,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2531,6 +2730,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2639,6 +2844,13 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2739,6 +2951,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2872,6 +3090,15 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = median + combine = median + satpix = nothing + scale_to_mean = True + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = median @@ -2934,6 +3161,8 @@ Alterations to the default parameters are: [reduce] [[findobj]] find_trim_edge = 3, 3, + maxnumber_sci = 2 + maxnumber_std = 1 [[skysub]] global_sky_std = False [[extraction]] @@ -3008,6 +3237,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3023,18 +3258,16 @@ Alterations to the default parameters are: noise_floor = 0.01 [[flatfield]] spec_samp_coarse = 20.0 + tweak_method = gradient tweak_slits_thresh = 0.0 tweak_slits_maxfrac = 0.0 slit_illum_relative = True slit_illum_ref_idx = 14 - slit_illum_smooth_npix = 5 - fit_2d_det_response = True [[wavelengths]] fwhm_spat_order = 2 [[slitedges]] edge_thresh = 5 fit_order = 4 - pad = 2 [scienceframe] [[process]] mask_cr = True @@ -3085,16 +3318,18 @@ Alterations to the default parameters are: use_pattern = True [[arcframe]] [[[process]]] + correct_nonlinear = -1.4e-07, -1.4e-07, -1.2e-07, -1.8e-07, use_pixelflat = False use_illumflat = False [[tiltframe]] [[[process]]] + correct_nonlinear = -1.4e-07, -1.4e-07, -1.2e-07, -1.8e-07, use_pixelflat = False use_illumflat = False [[pixelflatframe]] [[[process]]] - combine = median satpix = nothing + correct_nonlinear = -1.4e-07, -1.4e-07, -1.2e-07, -1.8e-07, use_pixelflat = False use_illumflat = False subtract_scattlight = True @@ -3112,6 +3347,7 @@ Alterations to the default parameters are: [[illumflatframe]] [[[process]]] satpix = nothing + correct_nonlinear = -1.4e-07, -1.4e-07, -1.2e-07, -1.8e-07, use_illumflat = False use_pattern = True subtract_scattlight = True @@ -3120,6 +3356,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3132,32 +3374,32 @@ Alterations to the default parameters are: [[standardframe]] [[[process]]] mask_cr = True + correct_nonlinear = -1.4e-07, -1.4e-07, -1.2e-07, -1.8e-07, noise_floor = 0.01 use_pattern = True [[flatfield]] spec_samp_coarse = 20.0 spat_samp = 1.0 + tweak_method = gradient tweak_slits_thresh = 0.0 tweak_slits_maxfrac = 0.0 slit_illum_relative = True slit_illum_ref_idx = 14 - slit_illum_smooth_npix = 5 fit_2d_det_response = True [[wavelengths]] fwhm_spat_order = 2 [[slitedges]] edge_thresh = 5 fit_order = 4 - pad = 2 [scienceframe] [[process]] mask_cr = True sigclip = 4.0 objlim = 1.5 + correct_nonlinear = -1.4e-07, -1.4e-07, -1.2e-07, -1.8e-07, noise_floor = 0.01 use_specillum = True use_pattern = True - subtract_scattlight = True [[[scattlight]]] finecorr_method = median [reduce] @@ -3183,7 +3425,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_blue [calibrations] [[biasframe]] - exprng = None, 0.001, + exprng = None, 1, [[[process]]] combine = median use_biasimage = False @@ -3233,6 +3475,14 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + exprng = 0, 60, + [[[process]]] + combine = median + satpix = nothing + scale_to_mean = True + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3243,11 +3493,13 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = 1, 61, + exprng = 1, 901, [[[process]]] mask_cr = True noise_floor = 0.01 spat_flexure_correct = True + [[flatfield]] + slit_illum_finecorr = False [[wavelengths]] sigdetect = 10.0 rms_thresh_frac_fwhm = 0.06 @@ -3285,7 +3537,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_blue_orig [calibrations] [[biasframe]] - exprng = None, 0.001, + exprng = None, 1, [[[process]]] combine = median use_biasimage = False @@ -3335,6 +3587,14 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + exprng = 0, 60, + [[[process]]] + combine = median + satpix = nothing + scale_to_mean = True + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3345,11 +3605,13 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = 1, 61, + exprng = 1, 901, [[[process]]] mask_cr = True noise_floor = 0.01 spat_flexure_correct = True + [[flatfield]] + slit_illum_finecorr = False [[wavelengths]] sigdetect = 10.0 rms_thresh_frac_fwhm = 0.06 @@ -3387,7 +3649,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_red [calibrations] [[biasframe]] - exprng = None, 0.001, + exprng = None, 1, [[[process]]] combine = median use_biasimage = False @@ -3409,7 +3671,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -3422,12 +3684,12 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] use_pixelflat = False use_illumflat = False [[illumflatframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -3437,6 +3699,14 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + exprng = 0, 60, + [[[process]]] + combine = median + satpix = nothing + scale_to_mean = True + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3452,6 +3722,8 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 spat_flexure_correct = True + [[flatfield]] + slit_illum_finecorr = False [[wavelengths]] sigdetect = 10.0 rms_thresh_frac_fwhm = 0.05 @@ -3500,7 +3772,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_red_mark4 [calibrations] [[biasframe]] - exprng = None, 0.001, + exprng = None, 1, [[[process]]] combine = median use_biasimage = False @@ -3522,7 +3794,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -3535,12 +3807,12 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] use_pixelflat = False use_illumflat = False [[illumflatframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -3550,6 +3822,14 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + exprng = 0, 60, + [[[process]]] + combine = median + satpix = nothing + scale_to_mean = True + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3565,6 +3845,8 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 spat_flexure_correct = True + [[flatfield]] + slit_illum_finecorr = False [[wavelengths]] sigdetect = 10.0 rms_thresh_frac_fwhm = 0.05 @@ -3613,7 +3895,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_red_orig [calibrations] [[biasframe]] - exprng = None, 0.001, + exprng = None, 1, [[[process]]] combine = median use_biasimage = False @@ -3635,7 +3917,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -3648,12 +3930,12 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] use_pixelflat = False use_illumflat = False [[illumflatframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -3663,6 +3945,14 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + exprng = 0, 60, + [[[process]]] + combine = median + satpix = nothing + scale_to_mean = True + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3678,6 +3968,8 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 spat_flexure_correct = True + [[flatfield]] + slit_illum_finecorr = False [[wavelengths]] sigdetect = 10.0 rms_thresh_frac_fwhm = 0.05 @@ -3792,6 +4084,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3920,6 +4220,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3970,6 +4278,8 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [reduce] + [[findobj]] + maxnumber_std = 1 [[skysub]] bspline_spacing = 0.8 [[extraction]] @@ -4063,6 +4373,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -4208,6 +4526,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -4352,6 +4678,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -4482,6 +4816,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -4602,6 +4944,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -4731,6 +5081,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = odd_even + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = odd_even @@ -4856,6 +5214,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = odd_even + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = odd_even @@ -4987,6 +5353,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = odd_even + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = odd_even @@ -5112,6 +5486,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = odd_even + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = odd_even @@ -5241,6 +5623,14 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = chebyshev + overscan_par = 1 + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = chebyshev @@ -5389,6 +5779,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -5533,6 +5931,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -5644,6 +6050,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -5758,6 +6170,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -5847,6 +6265,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -5954,6 +6378,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = odd_even + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = odd_even @@ -6058,6 +6490,13 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6176,6 +6615,13 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6296,6 +6742,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6428,6 +6882,13 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6536,6 +6997,13 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6624,6 +7092,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6723,6 +7197,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6816,6 +7296,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6930,6 +7416,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7058,6 +7552,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7159,6 +7659,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7251,6 +7757,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7350,6 +7862,13 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7453,6 +7972,13 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7554,6 +8080,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7642,6 +8174,13 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = median + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = median @@ -7763,6 +8302,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7899,6 +8446,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7929,6 +8484,7 @@ Alterations to the default parameters are: ech_sigrej = 3.0 lamps = OH_XSHOOTER, sigdetect = 10.0 + fwhm_fromlines = False reid_arxiv = vlt_xshooter_nir.fits cc_thresh = 0.5 cc_local_thresh = 0.5 @@ -7957,7 +8513,7 @@ Alterations to the default parameters are: use_illumflat = False [reduce] [[findobj]] - trace_npoly = 8 + trace_npoly = 10 maxnumber_sci = 2 maxnumber_std = 1 [[skysub]] @@ -8055,6 +8611,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = median + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = median @@ -8200,6 +8764,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = median + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = median @@ -8343,6 +8915,13 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -8448,6 +9027,13 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing diff --git a/doc/qa.rst b/doc/qa.rst index e6e61396e8..f895fc6003 100644 --- a/doc/qa.rst +++ b/doc/qa.rst @@ -9,9 +9,11 @@ PypeIt QA ========= As part of the standard reduction, PypeIt generates a series -of Quality Assurance (QA) files. This document describes +of fixed-format Quality Assurance (QA) figures. This document describes the typical outputs, in the typical order that they appear. +*This page is still a work in progress.* + The basic arrangement is that individual PNG files are created and then a set of HTML files are generated to organize viewing of the PNGs. @@ -20,16 +22,10 @@ viewing of the PNGs. HTML ==== -When the code completes (or crashes out), a set of -HTML files are generated in the ``QA/`` folder. There -is one HTML file per calibration frame set and one -HTML file per science exposure. Example names are -``MF_A.html``. - -Open in your browser and have at 'em. -Quick links are provided to allow one to jump between -the various files. - +When the code completes (or crashes out), an HTML file is generated in the +``QA/`` folder, one per setup that has been reduced (typically one). An example +filename is ``MF_A.html``. These HTML files are out of date, so you're better +off opening the PNG files in the ``PNGs`` directory directly. Calibration QA ============== @@ -39,9 +35,38 @@ to calibration processing. There is a unique one generated for each setup and detector and (possibly) calibration set. -Generally, the title describes the type of QA and the -sub-title indicates the user who ran PypeIt and the -date of the processing. +Generally, the title describes the type of QA plotted. + +.. _qa-order-predict: + +Echelle Order Prediction +------------------------ + +When reducing echelle observations and inserting missing orders, a QA plot is +produced to assess the success of the predicted locations. The example below is +for Keck/HIRES. + +.. figure:: figures/Edges_A_0_MSC01_orders_qa.png + :width: 60% + + Example QA plot showing the measured order spatial widths (blue) and gaps + (green) in pixels. The widths should be nearly constant as a function of + position, whereas the gaps should change monotonically with spatial pixel. + +In the figure above, measured values that are included in the polynomial fit are +shown as filled points. The colored lines show the best fit polynomial model +used for the predicted order locations. The fit allows for an iterative +rejection of points; measured widths and gaps that are rejected during the fit +are shown as orange and purple crosses, respectively. The measurements that are +rejected during the fit are not necessarily *removed* as invalid traces, but the +code allows you to identify outlier traces that *will be* removed. None of the +traces in the example image above are identified as outliers; if they exist, +they will be plotted as orange and purple triangles for widths and gaps, +respectively. Missing orders that will be added are included as open squares; +gaps are green, widths are blue. To deal with overlap, "bracketing" orders are +added for the overlap calculation but are removed in the final set of traces; +the title of the plot indicates if bracketing orders are included and the +vertical dashed lines shows the edges of the detector/mosaic. .. _qa-wave-fit: @@ -52,7 +77,7 @@ PypeIt produces plots like the one below showing the result of the wavelength calibration. .. figure:: figures/deimos_arc1d.png - :width: 60 % + :width: 60% An example QA plot for Keck/DEIMOS wavelength calibration. The extracted arc spectrum is shown to the left with arc lines used for the wavelength solution diff --git a/doc/releases/1.16.1dev.rst b/doc/releases/1.16.1dev.rst deleted file mode 100644 index 33b59d4259..0000000000 --- a/doc/releases/1.16.1dev.rst +++ /dev/null @@ -1,44 +0,0 @@ - -Version 1.16.1dev -================= - -Installation Changes --------------------- - -- Significant expansion of PypeIt's use of the cache system; see - :ref:`data_installation`. Important changes include that most cached files - are now version dependent. When upgrading to new versions of PypeIt, users - should delete their cache and start fresh. - -Dependency Changes ------------------- - -Functionality/Performance Improvements and Additions ----------------------------------------------------- - -Instrument-specific Updates ---------------------------- - -Script Changes --------------- - -- Modifications to the cache-related :ref:`install_scripts` to accommodate - expansion of and changes to the cache system. -- Added ``pypeit_clean_cache`` script to facilitate both viewing and removing - files in the cache. - -Datamodel Changes ------------------ - -Under-the-hood Improvements ---------------------------- - -- Introduced :class:`~pypeit.pypeitdata.PypeItDataPaths` to handle all - interactions with the ``pypeit/data`` directory, which provides a unified - interface for accessing on-disk and cached files. - -Bug Fixes ---------- - -- None - diff --git a/doc/releases/1.17.0.rst b/doc/releases/1.17.0.rst new file mode 100644 index 0000000000..774daf53a1 --- /dev/null +++ b/doc/releases/1.17.0.rst @@ -0,0 +1,175 @@ + +Version 1.17.0 +============== + +Installation Changes +-------------------- + +- Significant expansion of PypeIt's use of the cache system; see + :ref:`data_installation`. Important changes include that most cached + files are now version dependent. When upgrading to new versions of + PypeIt, users should delete their cache and start fresh. + +Dependency Changes +------------------ + +- Support added for numpy>=2.0.0; numpy<=2.0.0 should still be supported +- Deprecated support for python 3.10 +- General update to dependencies to be roughly consistent with release + of python 3.11 + +Functionality/Performance Improvements and Additions +---------------------------------------------------- + +- Added the ``max_overlap`` parameter, which limits the set of new order + traces added, to compensate for orders missed during automated + edge-tracing, to those that have less than a given fractional overlap + with adjacent orders. +- Added the ``order_fitrej`` and ``order_outlier`` parameters used to + set the sigma-clipping threshold used when fitting Legendre functions + to the order widths and gaps. +- Added the possibility to decide if the extracted standard star + spectrum should be used as a crutch for tracing the object in the + science frame (before it was done as default). This is done by + setting the parameter ``use_std_trace`` in FindObjPar. +- Now PypeIt can handle the case where "Standard star trace does not + match the number of orders in the echelle data" both in ``run_pypeit`` + and in ``pypeit_coadd_1dspec``. +- Added the functionality to use slitless flats to create pixelflats. + Note: new frametype ``slitless_pixflat`` is added to the PypeIt + frametype list. +- The created pixelflats are stored in the reduction directory and in + the PypeIt cache directory ``data/pixelflats``. +- Added a functionality that allows, when multiple frames are combined, + to scale each frame to have the same mean value before combining. To + use this functionality, the new parameter ``scale_mean`` should be set + to ``True``. +- Added the possibility to use the parameter ``fwhm_fromlines`` also for + the tilts calibration. + +Instrument-specific Updates +--------------------------- + +- Improved LRIS frame typing, including the typing of slitless flats and + sky flats. +- Improved HIRES frame typing and configuration setup. +- Added support for Keck/KCWI BH3 grating configuration. +- Updated the requirements of a spectral flip for KCWI (blue) data. If + all amplifiers are used, the data will not be flipped in the spectral + direction. Otherwise, the data will be flipped. +- Added support for the (decommissioned) AAT/UHRF instrument +- Updated X-Shooter detector gain and read noise to come from header, + and updated plate scales to the most recent values from the manual. + Detailed changes are: + + - NIR arm: + + - Platescale updated from 0.197 to 0.245 arcsec/pixel + - Dark current updated from 0. to 72. e-/pixel/hr + - Gain updated from 2.12 to 2.29 e-/DN + + - VIS arm: + + - Platescale updated from an order-dependent value, to being + 0.154 arcsec/pixel for all orders + + - UVB arm: + + - Platescale updated from an order-dependent value, to being + 0.164 arcsec/pixel for all orders + +- Add new P200/DBSP reid_arxiv template for 1200/7100 with D55 dichroic +- Add B480 as a supported option for Gemini-S/GMOS + +Script Changes +-------------- + +- Modifications to the cache-related :ref:`install_scripts` to + accommodate expansion of and changes to the cache system. +- Added ``pypeit_clean_cache`` script to facilitate both viewing and + removing files in the cache. +- Changed the name of the multi-dimensional specdata to + ``specdata_multi`` in ``pypeit_identify`` and improved the robustness + of the saving dialog when calibrating single trace spectra. +- Fixed a read-in error for the high resolution A0V PHOENIX model. +- A new script, called ``pypeit_extract_datacube``, allows 1D spectra of + point sources to be extracted from datacubes. +- The sensitivity function is now generated outside of datacube + generation. +- The ``grating_corr`` column is now used to select the correct grating + correction file for each spec2d file when generating the datacube. +- Added the ``--extr`` parameter in the ``pypeit_sensfunc`` script (also + as a ``SensFuncPar``) to allow the user to specify the extraction + method to use when computing the sensitivity function (before only + optimal extraction was used). +- Added ``pypeit_show_pixflat`` script to inspect the (slitless) pixel + flat generated during the reduction and stored in ``data/pixelflats``. +- Added ``pypeit_chk_flexure`` script to check both spatial and spectral + flexure applied to the reduced data. +- Treatment of file names is now more formal. Compression signatures + are now considered, and filename matching is now more strict. +- Removed ``--spec_samp_fact`` and ``--spat_samp_fact`` command line + options from ``pypeit_coadd_2d``. These options are now parameters in + ``Coadd2dPar``. +- ``pypeit_show_2dspec`` now shows the first available detector in the + 2D spectrum by default. The user can specify the detector to show with + the ``--det`` option. +- Added ``--removetrace`` command line option to ``pypeit_ql`` to not + show the object trace when displaying the 2D spectrum. +- Change the default value for ``--skip_display`` in ``pypeit_ql`` to + ``True``. + +Datamodel Changes +----------------- + +- Adjusted spec1d datamodel to enable use with UVES_popler GUI tool + +Under-the-hood Improvements +--------------------------- + +- Introduced :class:`~pypeit.pypeitdata.PypeItDataPaths` to handle all + interactions with the ``pypeit/data`` directory, which provides a + unified interface for accessing on-disk and cached files. +- When adding missing orders, the full syncing procedure is no longer + performed. The code now only checks that the edges are still synced + after the missed orders are added. +- When detecting overlapping orders/slits, the code now forces each edge + used to have been directly detected; i.e., if an edge is inserted, the + fact that the resulting slit is abnormally short should not trigger + the overlap detection. +- Improved the QA plot resulting from fitting order widths and gaps as a + function of spatial position. +- Updated general raw image reader so that it correctly accounts for + spectrographs that read the data and overscan sections directly from + the file headers. + +Bug Fixes +--------- + +- Fix "The system cannot find the file specified" errors when installing + on Windows. +- Fixed a fault caused when all frames in a pypeit file are identified + as being part of ``all`` calibration groups. +- Allow for empty 2D wavecal solution in HDU extension of WaveCalib file +- Fixed a bug in the ginga display function, when the user doesn't + provide the ``trc_name`` argument. +- Fix a **MAJOR BUT SUBTLE** bug in the use of ``numpy.argsort``. When + using ``numpy.argsort`` the parameter ``kind='stable'`` should be used + to ensure that a sorting algorithm more robust than "quicksort" is + used. +- Fix error "ValueError: setting an array element with a sequence. The + requested array has an inhomogeneous shape after 1 dimensions..." + occurring when unpacking the ``SpecObj`` spectrum but having an + attribute of the ``SpecObj`` object that is ``None``. +- Fixed a hidden bug that was causing the spatial flexure to fail. The + bug was in the ``SlitTraceBitMask`` class, where the function + ``exclude_for_flexure()`` was not returning the ``'BOXSLIT'`` flag. +- Fix a bug in ``pypeit_coadd_2d`` related to how the binning was taken + into account in the mask definition, and in the calculation of the + offset between frames. +- Fix bug when trying to open mosaic data from previous versions; + version checking flag was not being propagated. + + + + diff --git a/doc/scripts.rst b/doc/scripts.rst index 0b37cc7b0c..e5c4aee3d6 100644 --- a/doc/scripts.rst +++ b/doc/scripts.rst @@ -566,6 +566,26 @@ The script usage can be displayed by calling the script with the .. include:: help/pypeit_chk_flats.rst +.. _pypeit_show_pixflat: + +pypeit_show_pixflat +------------------- + +Inspect in a Ginga window the (slitless) pixel flat produced by PypeIt and stored +in the PypeIt cache (see ref:`data_installation`). It displays each detector separately +in different channels. The script is useful for assessing the quality of the pixel-to-pixel +response of the detector. Typical call is: + +.. code-block:: console + + pypeit_show_pixflat PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz + + +The script usage can be displayed by calling the script with the +``-h`` option: + +.. include:: help/pypeit_show_pixflat.rst + pypeit_show_2dspec ------------------ diff --git a/doc/setup.rst b/doc/setup.rst index 2a046b17e5..523ed2290d 100644 --- a/doc/setup.rst +++ b/doc/setup.rst @@ -22,6 +22,27 @@ preparatory script, :ref:`pypeit_obslog`, which provides a simple listing of the available data files; however, use of this script is optional in terms of setup for reducing your data. +.. _setup-file-searching: + +Raw File Searches +================= + +PypeIt scripts that search for raw files in a given directory base the search on +a list of file extensions. These are generally ``.fits`` and ``.fits.gz``, but +some spectrographs specify a different set. + +Some scripts allow you to specify the extension to use for the search, which +*must* be one of the allowed extensions for that spectrograph. E.g., +for a spectrograph that allows ``.fits`` and ``.fits.gz`` extension, you can +specify to only look for the ``.fits`` files, but you *cannot* have it look for +``.fits.bz2`` files. If your raw files have extensions that are currently not +allowed by the code, please `Submit an issue`_. + +If you have both compressed and uncompressed files in your directory, the search +function will generally find both. You are strongly encouraged to only include +one version (compressed or uncompressed) of each file in the directory with your +raw data. + .. _setup-metadata: Use of Metadata to Identify Instrument Configurations @@ -168,7 +189,9 @@ to be the same directory that holds the raw data. 1. First Execution ------------------ -We recommend you first execute ``pypeit_setup`` like this:: +We recommend you first execute ``pypeit_setup`` like this: + +.. code-block:: bash pypeit_setup -r path_to_your_raw_data/LB -s keck_lris_blue @@ -187,7 +210,9 @@ This execution of ``pypeit_setup`` searches for all `*.fits` and `*.fits.gz` files with the provided root directory. Generally, the provided path should **not** contain a wild-card and it is best if you provide the *full* path; however, you can search through multiple -directories as follows:: +directories as follows: + +.. code-block:: bash pypeit_setup -r "/Users/xavier/Keck/LRIS/data/2016apr06/Raw/*/LB" -s keck_lris_blue diff --git a/doc/spectrographs/aat_uhrf.rst b/doc/spectrographs/aat_uhrf.rst new file mode 100644 index 0000000000..ebef209ecf --- /dev/null +++ b/doc/spectrographs/aat_uhrf.rst @@ -0,0 +1,33 @@ +.. highlight:: rest + +******** +AAT UHRF +******** + + +Overview +======== + +This file summarizes several instrument specific +settings that are related to AAT/UHRF. + + +Wavelength Calibration +---------------------- + +UHRF has many wavelength setups, and the wavelength calibration +must be performed manually for each setup using :ref:`wvcalib-byhand` +approach and the :ref:`pypeit_identify` script. Since this spectrograph +is decommissioned, we do not expect to have a general solution +for this spectrograph. + +Object profile +-------------- + +UHRF is a slicer spectrograph, and the data are usually binned aggressively. +The object profile tends to be poorly characterised with the automated approach, +and you may need to generate your own wavelength dependent profile. Previously, +a Gaussian KDE profile was used, and this performed well, but is not available +by default. For users that are interested in this functionality, please contact +the PypeIt developers on the PypeIt User's Slack, or see the `aat_uhrf_rjc` +branch. diff --git a/doc/spectrographs/deimos.rst b/doc/spectrographs/deimos.rst index a6e4d8bc81..d9131773c8 100644 --- a/doc/spectrographs/deimos.rst +++ b/doc/spectrographs/deimos.rst @@ -15,9 +15,9 @@ settings that are related to the Keck/DEIMOS spectrograph. .. warning:: PypeIt currently *cannot* reduce images produced by reading - the DEIMOS CCDs with the A amplifier or those taken in imaging + the DEIMOS CCDs with the A+B amplifier or those taken in imaging mode. All image-handling assumes DEIMOS images have been read - with the B amplifier in the "Spectral" observing mode. PypeIt + with the A or B amplifier in the "Spectral" observing mode. PypeIt handles files that do not meet these criteria in two ways: - When running :ref:`pypeit_setup`, any frames not in diff --git a/doc/spectrographs/keck_hires.rst b/doc/spectrographs/keck_hires.rst index f8d3277554..f1628da181 100644 --- a/doc/spectrographs/keck_hires.rst +++ b/doc/spectrographs/keck_hires.rst @@ -6,13 +6,85 @@ Overview ======== This file summarizes several instrument specific settings that are related to the Keck/HIRES spectrograph. +Currently, PypeIt only supports the reduction of the post detector upgrade HIRES data (around August 2004). +Information on the frame typing and instrument configuration can be found in ref:`hiresframes` and :ref:`hires_config`. + +Default Settings +---------------- + +See :ref:`instr_par-keck_hires` for a list of modifications to the default settings. +*You do not have to add these changes to your PypeIt reduction file!* This is just a list of +how the parameters used for HIRES differ from the defaults listed in the preceding tables on that page. +Moreover, additional modifications may have been made for specific setups, e.g, for different binning, +etc. You can see a list of all the used parameters in the ``keck_hires_XXX.par`` +file generated by PypeIt at the beginning of the reduction. + +MOSAIC +====== + +PypeIt, by default, uses a mosaic approach for the reduction. It basically constructs a mosaic +of the blue, green, and red detector data and reduces it, instead of processing the detector data individually. +The mosaic reduction is switched on by setting the parameter ``detnum`` in :ref:`reduxpar` to be a +tuple of the detector indices that are mosaiced together. For HIRES, it looks like: + +.. code-block:: ini + + [rdx] + spectrograph = keck_hires + detnum = (1,2,3) + +This is already the default for HIRES, but the user can modify it in the :ref:`pypeit_file` to +turn off the mosaic reduction. + + +Calibrations +============ + +Flat Fielding +------------- +Special type of "slitless" flat images are sometime used for correcting +pixel-to-pixel variations in the HIRES data. These flats images are taken +with the cross disperser cover on, so that the light is somewhat diffused +and the orders are not visible. If this type of frames are present in the +:ref:`pypeit_file`, during the main reduction PypeIt will be able to identify them, +assign the correct frame type (`slitless_pixflat`), and use them for creating a "slitless" +pixel flat. Since, these frames are taken with different exposure time and +cross-disperser angle for each detector, PypeIt is able to select the correct +observations for each detector. The pixelflat is generated by first scaling +each flat frame to have the same mean counts, then median combining them. +A pixelflat file will be generated and stored in the reduction folder. +In addition, the constructed pixelflat is saved to the PypeIt cache (see ref:`data_installation`). +This allows you to use the file for both current and future reductions. +To use this file in future reductions, the user can add the +slitless pixelflat file name to the :ref:`pypeit_file` as shown below: + +.. code-block:: ini + + [calibrations] + [[flatfield]] + pixelflat_file = pixelflat_keck_hires_RED_1x2_20160330.fits.gz + +See more in :ref:`generate-pixflat`. + Wavelengths -=========== +----------- See :ref:`wvcalib-echelle` for details on the wavelength calibration. -We also note that several Orders from 40-45 are -frequently flagged as bad in the wavelength solution. -This is due, in part, to very bright ThAr line contamination. \ No newline at end of file + +Additional Reading +================== + +Here are additional docs related to Keck/HIRES. Note all of them are related +to the development of PypeIt for use with HIRES data: + +.. TODO: Generally useful information in these dev docs should be moved into +.. user-level doc pages, even if that means repeating information. + +.. toctree:: + :maxdepth: 1 + + ../dev/hiresframes + ../dev/hiresconfig \ No newline at end of file diff --git a/doc/spectrographs/lris.rst b/doc/spectrographs/lris.rst index f6b4b786c9..e28b7cf3d5 100644 --- a/doc/spectrographs/lris.rst +++ b/doc/spectrographs/lris.rst @@ -212,21 +212,45 @@ Pixel Flat It is recommend to correct for pixel-to-pixel variations using a slitless flat. If you did not take such calibration frames or cannot process them, -you may wish to use an archival. -`This link `__ -has the existing ones staged by the PypeIt team. +you may wish to use an archival one available in the PypeIt cache (see ref:`data_installation`). And then set the following in your :ref:`pypeit_file`: .. code-block:: ini - [calibrations] - [[flatfield]] - frame = path_to_the_file/PYPEIT_LRISb_pixflat_B600_2x2_17sep2009.fits.gz - -.. warning:: - - Internal flats may be too bright and need to be tested. + [calibrations] + [[flatfield]] + pixelflat_file = PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz + + +If, instead, you have taken slitless pixelflat frames, PypeIt will be able to identify them +(see :ref:`lris_frames_report`) and process them during the reduction. The slitless pixelflat +is generated by first scaling each slitless flat frame to have the same mean counts, then +median combining them. A slitless pixelflat file will be generated +and stored in the reduction folder. In addition, the constructed pixelflat is saved to the +PypeIt cache. This allows you to use the file for both current and future reductions. +To use this file in future reductions, the user can add the +slitless pixelflat file name to the :ref:`pypeit_file` as shown above. Consider sharing +your slitless pixelflat file with the PypeIt Developers. See more in :ref:`generate-pixflat`. + +To generate slitless pixelflat, we recommend using twilight flats. As these are taken +on the twilight sky, it is important to dither between exposures to avoid bright stars +from landing in the same part of the detector. Related, it is best to take these as close +to sunset (or sunrise) as possible (about 10 minutes after/before sunset/sunrise) so that +one can use the shortest exposure time possible (3s for LRIS given its shutter). + +Internal flats may be too bright and need to be tested. +Our recommended recipe for taking twilight flats is: + + 1. Point away from the Galaxy (i.e. avoid stars) + 2. Select the desired setup for your spectroscopy + 3. Remove any selected slitmask/longslit + 4. Expose for 3s + 5. Check counts and make sure you have more than 40,000 + 6. Offset by ~1' and 1' in NE + 7. Repeat 4-6, increasing the exposure each time you dip below 30,000 counts or so + +About 5 exposures is enough but the more the better. Trace Flat ---------- diff --git a/doc/spectrographs/spectrographs.rst b/doc/spectrographs/spectrographs.rst index 275915a2c9..1fffa34730 100644 --- a/doc/spectrographs/spectrographs.rst +++ b/doc/spectrographs/spectrographs.rst @@ -33,6 +33,7 @@ instrument-specific details for running PypeIt. :caption: Spectrographs :maxdepth: 1 + aat_uhrf gemini_flamingos2 gemini_gmos gemini_gnirs diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 66935e6aec..85de8cae70 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -11,7 +11,7 @@ What's New in PypeIt ---- -.. include:: releases/1.16.1dev.rst +.. include:: releases/1.17.0.rst ---- diff --git a/environment.yml b/environment.yml index b52aa30b6e..c020c2fd15 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: pypeit channels: - defaults dependencies: - - python>=3.10,<3.13 + - python>=3.11,<3.13 - pip - pip: - pypeit diff --git a/presentations/py/users.py b/presentations/py/users.py index 9f3affcddd..73902997e4 100644 --- a/presentations/py/users.py +++ b/presentations/py/users.py @@ -21,9 +21,9 @@ def set_fontsize(ax, fsz): ax.get_xticklabels() + ax.get_yticklabels()): item.set_fontsize(fsz) -user_dates = ["2021-03-11", "2022-04-29", "2022-11-07", "2022-12-06", "2023-06-08", "2023-06-29", "2023-07-11", "2023-09-03", "2023-10-13", "2023-12-01", "2023-12-15", "2024-02-22", "2024-03-21", "2024-04-09", "2024-05-02", "2024-05-19", "2024-06-06", "2024-06-10"] +user_dates = ["2021-03-11", "2022-04-29", "2022-11-07", "2022-12-06", "2023-06-08", "2023-06-29", "2023-07-11", "2023-09-03", "2023-10-13", "2023-12-01", "2023-12-15", "2024-02-22", "2024-03-21", "2024-04-09", "2024-05-02", "2024-05-19", "2024-06-06", "2024-06-10", "2024-08-20"] user_dates = numpy.array([numpy.datetime64(date) for date in user_dates]) -user_number = numpy.array([125, 293, 390, 394, 477, 487, 506, 518, 531, 544, 551, 568, 579, 588, 596, 603, 616, 620]) +user_number = numpy.array([125, 293, 390, 394, 477, 487, 506, 518, 531, 544, 551, 568, 579, 588, 596, 603, 616, 620, 643]) user_pred_dates = numpy.array([numpy.datetime64(date) for date in ["2024-06-10", "2024-12-31", "2025-12-31", "2026-12-31", diff --git a/pypeit/alignframe.py b/pypeit/alignframe.py index 9a28491d21..fb899525a9 100644 --- a/pypeit/alignframe.py +++ b/pypeit/alignframe.py @@ -181,8 +181,8 @@ def build_traces(self, show_peaks=False, debug=False): dict: self.align_dict """ # Generate slits - slitid_img_init = self.slits.slit_img(initial=True) - left, right, _ = self.slits.select_edges(initial=True) + slitid_img_init = self.slits.slit_img() + left, right, _ = self.slits.select_edges() align_prof = dict({}) # Go through the slits @@ -316,8 +316,7 @@ def show_alignment(alignframe, align_traces=None, slits=None, clear=False): if slt%2 == 0: color = 'magenta' # Display the trace - display.show_trace(viewer, channel, align_traces[:, bar, slt], trc_name="", - color=color) + display.show_trace(viewer, channel, align_traces[:, bar, slt], color=color) class AlignmentSplines: diff --git a/pypeit/bitmask.py b/pypeit/bitmask.py index fd35e61c0f..cfa4eae9ca 100644 --- a/pypeit/bitmask.py +++ b/pypeit/bitmask.py @@ -575,7 +575,7 @@ def from_header(cls, hdr, prefix=None): # Fill in any missing bits keys, values, descr = cls._fill_sequence(keys, values, descr=descr) # Make sure the bits are sorted - srt = numpy.argsort(values) + srt = numpy.argsort(values, kind='stable') # Instantiate the BitMask return cls(keys[srt], descr=descr[srt]) diff --git a/pypeit/bspline/bspline.py b/pypeit/bspline/bspline.py index d4efc6c10b..2867460710 100644 --- a/pypeit/bspline/bspline.py +++ b/pypeit/bspline/bspline.py @@ -532,7 +532,7 @@ def value(self, x, x2=None, action=None, lower=None, upper=None): is good). """ # TODO: Is the sorting necessary? - xsort = x.argsort() + xsort = x.argsort(kind='stable') if action is None: action, lower, upper = self.action(x[xsort], x2=None if x2 is None else x2[xsort]) else: @@ -550,12 +550,12 @@ def value(self, x, x2=None, action=None, lower=None, upper=None): mask[(x < gb[self.nord-1]) | (x > gb[n])] = False hmm = (np.diff(goodbk) > 2).nonzero()[0] if hmm.size == 0: - return yfit[np.argsort(xsort)], mask + return yfit[np.argsort(xsort, kind='stable')], mask for jj in range(hmm.size): mask[(x >= self.breakpoints[goodbk[hmm[jj]]]) & (x <= self.breakpoints[goodbk[hmm[jj]+1]-1])] = False - return yfit[np.argsort(xsort)], mask + return yfit[np.argsort(xsort, kind='stable')], mask def maskpoints(self, err): """Perform simple logic of which breakpoints to mask. diff --git a/pypeit/bspline/setup_package.py b/pypeit/bspline/setup_package.py index 415759c0c1..122cff8dc0 100644 --- a/pypeit/bspline/setup_package.py +++ b/pypeit/bspline/setup_package.py @@ -1,50 +1,9 @@ import os import sys -import tempfile, subprocess, shutil from setuptools import Extension +from extension_helpers import add_openmp_flags_if_available -# the most reliable way to check for openmp support in the C compiler is to try to build -# some test code with the -fopenmp flag. openmp provides a big performance boost, but some -# systems, notably apple's version of clang that xcode provides, don't support it out of the box. - -# see http://openmp.org/wp/openmp-compilers/ -omp_test = \ -r""" -#include -#include -int main() { -#pragma omp parallel -printf("Hello from thread %d, nthreads %d\n", omp_get_thread_num(), omp_get_num_threads()); -} -""" - -def check_for_openmp(): - tmpdir = tempfile.mkdtemp() - curdir = os.getcwd() - os.chdir(tmpdir) - - if 'CC' in os.environ: - c_compiler = os.environ['CC'] - else: - c_compiler = 'gcc' - - filename = r'test.c' - with open(filename, 'w') as file: - file.write(omp_test) - with open(os.devnull, 'w') as fnull: - result = subprocess.call([c_compiler, '-fopenmp', filename], - stdout=fnull, stderr=fnull) - - os.chdir(curdir) - # clean up test code - shutil.rmtree(tmpdir) - - # return code from compiler process is 0 if it completed successfully - if result == 0: - return True - else: - return False C_BSPLINE_PKGDIR = os.path.relpath(os.path.dirname(__file__)) @@ -55,16 +14,17 @@ def check_for_openmp(): if not sys.platform.startswith('win'): extra_compile_args.append('-fPIC') -if check_for_openmp(): - extra_compile_args.append('-fopenmp') - extra_link_args = ['-fopenmp'] -else: - extra_link_args = [] def get_extensions(): - return [Extension(name='pypeit.bspline._bspline', sources=SRC_FILES, - extra_compile_args=extra_compile_args, language='c', - extra_link_args=extra_link_args, - export_symbols=['bspline_model', 'solution_arrays', - 'cholesky_band', 'cholesky_solve', - 'intrv'])] + extension = Extension(name='pypeit.bspline._bspline', sources=SRC_FILES, + extra_compile_args=extra_compile_args, language='c', + export_symbols=['bspline_model', 'solution_arrays', + 'cholesky_band', 'cholesky_solve', + 'intrv']) + + # extension_helpers will check for opnmp support by trying to build + # some test code with the appropriate flag. openmp provides a big performance boost, but some + # systems, notably apple's version of clang that xcode provides, don't support it out of the box. + + add_openmp_flags_if_available(extension) + return [extension] diff --git a/pypeit/cache.py b/pypeit/cache.py index b761b19f09..b1d6555d1f 100644 --- a/pypeit/cache.py +++ b/pypeit/cache.py @@ -63,13 +63,22 @@ def git_branch(): """ Return the name/hash of the currently checked out branch - + Returns: - :obj:`str`: Branch name or hash + :obj:`str`: Branch name or hash. Defaults to "develop" if PypeIt is not currently in a repository + or pygit2 is inot installed. + """ - if Repository is None: + if Repository is not None: + try: + repo = Repository(resources.files('pypeit')) + except Exception as e: + # PypeIt not in a git repo + repo = None + + if Repository is None or repo is None: return 'develop' if '.dev' in __version__ else __version__ - repo = Repository(resources.files('pypeit')) + return str(repo.head.target) if repo.head_is_detached else str(repo.head.shorthand) @@ -153,6 +162,7 @@ def fetch_remote_file( install_script: bool=False, force_update: bool=False, full_url: str=None, + return_none: bool=False, ) -> pathlib.Path: """ Use `astropy.utils.data`_ to fetch file from remote or cache @@ -182,6 +192,8 @@ def fetch_remote_file( full_url (:obj:`str`, optional): The full url. If None, use :func:`_build_remote_url`). Defaults to None. + return_none (:obj:`bool`, optional): + Return None if the file is not found. Defaults to False. Returns: `Path`_: The local path to the desired file in the cache @@ -239,6 +251,9 @@ def fetch_remote_file( "https://pypeit.readthedocs.io/en/latest/fluxing.html#extinction-correction" ) + elif return_none: + return None + else: err_msg = ( f"Error downloading {filename}: {error}{msgs.newline()}" diff --git a/pypeit/calibrations.py b/pypeit/calibrations.py index e8875595df..b2f2e8fd71 100644 --- a/pypeit/calibrations.py +++ b/pypeit/calibrations.py @@ -4,6 +4,7 @@ .. include common links, assuming primary doc root is up one directory .. include:: ../include/links.rst """ +import os from pathlib import Path from datetime import datetime from copy import deepcopy @@ -38,10 +39,13 @@ from pypeit.core import framematch from pypeit.core import parse from pypeit.core import scattlight as core_scattlight +from pypeit.core.mosaic import build_image_mosaic from pypeit.par import pypeitpar from pypeit.spectrographs.spectrograph import Spectrograph from pypeit import io from pypeit import utils +from pypeit import cache +from pypeit import dataPaths class Calibrations: @@ -209,6 +213,46 @@ def __init__(self, fitstbl, par, spectrograph, caldir, qadir=None, self.success = False self.failed_step = None + def check_calibrations(self, file_list, check_lamps=True): + """ + Check if the input calibration files are consistent with each other. + This step is usually needed when combining calibration frames of a given type. + This routine currently only prints out warning messages if the calibration files are not consistent. + + Note: The exposure times are currently checked in the combine step, so they are not checked here. + + Parameters + ---------- + file_list : list + List of calibration files to check + check_lamps : bool, optional + Check if the lamp status is the same for all the files. Default is True. + """ + + lampstat = [None] * len(file_list) + # Loop on the files + for ii, ifile in enumerate(file_list): + # Save the lamp status + headarr = deepcopy(self.spectrograph.get_headarr(ifile)) + lampstat[ii] = self.spectrograph.get_lamps_status(headarr) + + # Check that the lamps being combined are all the same + if check_lamps: + if not lampstat[1:] == lampstat[:-1]: + msgs.warn("The following files contain different lamp status") + # Get the longest strings + maxlen = max([len("Filename")] + [len(os.path.split(x)[1]) for x in file_list]) + maxlmp = max([len("Lamp status")] + [len(x) for x in lampstat]) + strout = "{0:" + str(maxlen) + "} {1:s}" + # Print the messages + print(msgs.indent() + '-' * maxlen + " " + '-' * maxlmp) + print(msgs.indent() + strout.format("Filename", "Lamp status")) + print(msgs.indent() + '-' * maxlen + " " + '-' * maxlmp) + for ff, file in enumerate(file_list): + print(msgs.indent() + + strout.format(os.path.split(file)[1], " ".join(lampstat[ff].split("_")))) + print(msgs.indent() + '-' * maxlen + " " + '-' * maxlmp) + def find_calibrations(self, frametype, frameclass): """ Find calibration files and identifiers. @@ -334,6 +378,9 @@ def get_arc(self): # Reset the BPM self.get_bpm(frame=raw_files[0]) + # Perform a check on the files + self.check_calibrations(raw_files) + # Otherwise, create the processed file. msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.') self.msarc = buildimage.buildimage_fromlist(self.spectrograph, self.det, @@ -377,6 +424,9 @@ def get_tiltimg(self): # Reset the BPM self.get_bpm(frame=raw_files[0]) + # Perform a check on the files + self.check_calibrations(raw_files) + # Otherwise, create the processed file. msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.') self.mstilt = buildimage.buildimage_fromlist(self.spectrograph, self.det, @@ -426,6 +476,9 @@ def get_align(self): # Reset the BPM self.get_bpm(frame=raw_files[0]) + # Perform a check on the files + self.check_calibrations(raw_files) + # Otherwise, create the processed file. msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.') msalign = buildimage.buildimage_fromlist(self.spectrograph, self.det, @@ -473,6 +526,9 @@ def get_bias(self): self.msbias = frame['class'].from_file(cal_file, chk_version=self.chk_version) return self.msbias + # Perform a check on the files + self.check_calibrations(raw_files) + # Otherwise, create the processed file. msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.') self.msbias = buildimage.buildimage_fromlist(self.spectrograph, self.det, @@ -523,6 +579,9 @@ def get_dark(self): # there any reason why creation of the bpm should come after the dark, # or can we change the order? + # Perform a check on the files + self.check_calibrations(raw_files) + # Otherwise, create the processed file. self.msdark = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['darkframe'], raw_files, @@ -597,6 +656,9 @@ def get_scattlight(self): # Reset the BPM self.get_bpm(frame=raw_scattlight_files[0]) + # Perform a check on the files + self.check_calibrations(raw_scattlight_files) + binning = self.fitstbl[scatt_idx[0]]['binning'] dispname = self.fitstbl[scatt_idx[0]]['dispname'] scattlightImage = buildimage.buildimage_fromlist(self.spectrograph, self.det, @@ -607,7 +669,7 @@ def get_scattlight(self): spatbin = parse.parse_binning(binning)[1] pad = self.par['scattlight_pad'] // spatbin - offslitmask = self.slits.slit_img(pad=pad, initial=True, flexure=None) == -1 + offslitmask = self.slits.slit_img(pad=pad, flexure=None) == -1 # Get starting parameters for the scattered light model x0, bounds = self.spectrograph.scattered_light_archive(binning, dispname) @@ -673,21 +735,50 @@ def get_flats(self): # Check internals self._chk_set(['det', 'calib_ID', 'par']) - pixel_frame = {'type': 'pixelflat', 'class': flatfield.FlatImages} - raw_pixel_files, pixel_cal_file, pixel_calib_key, pixel_setup, pixel_calib_id, detname \ - = self.find_calibrations(pixel_frame['type'], pixel_frame['class']) - + # generate the slitless pixel flat (if frames available). + slitless_rows = self.fitstbl.find_frames('slitless_pixflat', calib_ID=self.calib_ID, index=True) + if len(slitless_rows) > 0: + sflat = flatfield.SlitlessFlat(self.fitstbl, slitless_rows, self.spectrograph, + self.par, qa_path=self.qa_path) + # A pixel flat will be saved to disc and self.par['flatfield']['pixelflat_file'] will be updated + self.par['flatfield']['pixelflat_file'] = \ + sflat.make_slitless_pixflat(msbias=self.msbias, msdark=self.msdark, calib_dir=self.calib_dir, + write_qa=self.write_qa, show=self.show) + + # get illumination flat frames illum_frame = {'type': 'illumflat', 'class': flatfield.FlatImages} raw_illum_files, illum_cal_file, illum_calib_key, illum_setup, illum_calib_id, detname \ = self.find_calibrations(illum_frame['type'], illum_frame['class']) + # get pixel flat frames + pixel_frame = {'type': 'pixelflat', 'class': flatfield.FlatImages} + raw_pixel_files, pixel_cal_file, pixel_calib_key, pixel_setup, pixel_calib_id, detname \ + = [], None, None, illum_setup, None, detname + # read in the raw pixelflat frames only if the user has not provided a pixelflat_file + if self.par['flatfield']['pixelflat_file'] is None: + raw_pixel_files, pixel_cal_file, pixel_calib_key, pixel_setup, pixel_calib_id, detname \ + = self.find_calibrations(pixel_frame['type'], pixel_frame['class']) + + # get lamp off flat frames raw_lampoff_files = self.fitstbl.find_frame_files('lampoffflats', calib_ID=self.calib_ID) + # Check if we have any calibration frames to work with if len(raw_pixel_files) == 0 and pixel_cal_file is None \ and len(raw_illum_files) == 0 and illum_cal_file is None: - msgs.warn(f'No raw {pixel_frame["type"]} or {illum_frame["type"]} frames found and ' - 'unable to identify a relevant processed calibration frame. Continuing...') - self.flatimages = None + # if no calibration frames are found, check if the user has provided a pixel flat file + if self.par['flatfield']['pixelflat_file'] is not None: + msgs.warn(f'No raw {pixel_frame["type"]} or {illum_frame["type"]} frames found but a ' + 'user-defined pixel flat file was provided. Using that file.') + self.flatimages = flatfield.FlatImages(PYP_SPEC=self.spectrograph.name, spat_id=self.slits.spat_id) + self.flatimages.calib_key = flatfield.FlatImages.construct_calib_key(self.fitstbl['setup'][self.frame], + self.calib_ID, detname) + self.flatimages = flatfield.load_pixflat(self.par['flatfield']['pixelflat_file'], self.spectrograph, + self.det, self.flatimages, calib_dir=self.calib_dir, + chk_version=self.chk_version) + else: + msgs.warn(f'No raw {pixel_frame["type"]} or {illum_frame["type"]} frames found and ' + 'unable to identify a relevant processed calibration frame. Continuing...') + self.flatimages = None return self.flatimages # If a processed calibration frame exists and we want to reuse it, do @@ -709,10 +800,9 @@ def get_flats(self): # Load user defined files if self.par['flatfield']['pixelflat_file'] is not None: # Load - msgs.info(f'Using user-defined file: {self.par["flatfield"]["pixelflat_file"]}') - with io.fits_open(self.par['flatfield']['pixelflat_file']) as hdu: - nrm_image = flatfield.FlatImages(pixelflat_norm=hdu[self.det].data) - self.flatimages = flatfield.merge(self.flatimages, nrm_image) + self.flatimages = flatfield.load_pixflat(self.par['flatfield']['pixelflat_file'], self.spectrograph, + self.det, self.flatimages, calib_dir=self.calib_dir, + chk_version=self.chk_version) # update slits self.slits.mask_flats(self.flatimages) return self.flatimages @@ -725,6 +815,10 @@ def get_flats(self): if len(raw_pixel_files) > 0: # Reset the BPM self.get_bpm(frame=raw_pixel_files[0]) + + # Perform a check on the files + self.check_calibrations(raw_pixel_files) + msgs.info('Creating pixel-flat calibration frame using files: ') for f in raw_pixel_files: msgs.prindent(f'{Path(f).name}') @@ -737,6 +831,10 @@ def get_flats(self): if len(raw_lampoff_files) > 0: # Reset the BPM self.get_bpm(frame=raw_lampoff_files[0]) + + # Perform a check on the files + self.check_calibrations(raw_lampoff_files) + msgs.info('Subtracting lamp off flats using files: ') for f in raw_lampoff_files: msgs.prindent(f'{Path(f).name}') @@ -750,8 +848,8 @@ def get_flats(self): # Initialise the pixel flat pixelFlatField = flatfield.FlatField(pixel_flat, self.spectrograph, - self.par['flatfield'], self.slits, self.wavetilts, - self.wv_calib, qa_path=self.qa_path, + self.par['flatfield'], self.slits, wavetilts=self.wavetilts, + wv_calib=self.wv_calib, qa_path=self.qa_path, calib_key=calib_key) # Generate pixelflatImages = pixelFlatField.run(doqa=self.write_qa, show=self.show) @@ -763,9 +861,14 @@ def get_flats(self): if not pix_is_illum and len(raw_illum_files) > 0: # Reset the BPM self.get_bpm(frame=raw_illum_files[0]) + + # Perform a check on the files + self.check_calibrations(raw_illum_files) + msgs.info('Creating slit-illumination flat calibration frame using files: ') for f in raw_illum_files: msgs.prindent(f'{Path(f).name}') + illum_flat = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['illumflatframe'], raw_illum_files, dark=self.msdark, bias=self.msbias, scattlight=self.msscattlight, @@ -775,6 +878,10 @@ def get_flats(self): for f in raw_lampoff_files: msgs.prindent(f'{Path(f).name}') if lampoff_flat is None: + # Perform a check on the files + self.check_calibrations(raw_lampoff_files) + + # Build the image lampoff_flat = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['lampoffflatsframe'], raw_lampoff_files, @@ -787,8 +894,8 @@ def get_flats(self): # Initialise the illum flat illumFlatField = flatfield.FlatField(illum_flat, self.spectrograph, - self.par['flatfield'], self.slits, self.wavetilts, - self.wv_calib, spat_illum_only=True, + self.par['flatfield'], self.slits, wavetilts=self.wavetilts, + wv_calib=self.wv_calib, spat_illum_only=True, qa_path=self.qa_path, calib_key=calib_key) # Generate illumflatImages = illumFlatField.run(doqa=self.write_qa, show=self.show) @@ -823,10 +930,9 @@ def get_flats(self): # Should we allow that? if self.par['flatfield']['pixelflat_file'] is not None: # Load - msgs.info(f'Using user-defined file: {self.par["flatfield"]["pixelflat_file"]}') - with io.fits_open(self.par['flatfield']['pixelflat_file']) as hdu: - self.flatimages = flatfield.merge(self.flatimages, - flatfield.FlatImages(pixelflat_norm=hdu[self.det].data)) + self.flatimages = flatfield.load_pixflat(self.par['flatfield']['pixelflat_file'], self.spectrograph, + self.det, self.flatimages, calib_dir=self.calib_dir, + chk_version=self.chk_version) return self.flatimages @@ -889,6 +995,9 @@ def get_slits(self): # Reset the BPM self.get_bpm(frame=raw_trace_files[0]) + # Perform a check on the files + self.check_calibrations(raw_trace_files) + traceImage = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['traceframe'], raw_trace_files, bias=self.msbias, bpm=self.msbpm, @@ -903,6 +1012,9 @@ def get_slits(self): # Reset the BPM self.get_bpm(frame=raw_trace_files[0]) + # Perform a check on the files + self.check_calibrations(raw_lampoff_files) + lampoff_flat = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['lampoffflatsframe'], raw_lampoff_files, dark=self.msdark, @@ -1064,11 +1176,14 @@ def get_tilts(self): _spat_flexure = self.mstilt.spat_flexure \ if self.par['tiltframe']['process']['spat_flexure_correct'] else None + # get measured fwhm from wv_calib + measured_fwhms = [wvfit.fwhm for wvfit in self.wv_calib.wv_fits] + # Build buildwaveTilts = wavetilts.BuildWaveTilts( self.mstilt, self.slits, self.spectrograph, self.par['tilts'], self.par['wavelengths'], det=self.det, qa_path=self.qa_path, - spat_flexure=_spat_flexure) + spat_flexure=_spat_flexure, measured_fwhms=measured_fwhms) # TODO still need to deal with syntax for LRIS ghosts. Maybe we don't need it self.wavetilts = buildwaveTilts.run(doqa=self.write_qa, show=self.show) @@ -1196,6 +1311,7 @@ def get_association(fitstbl, spectrograph, caldir, setup, calib_ID, det, must_ex 'pixelflat': [flatfield.FlatImages], 'illumflat': [flatfield.FlatImages], 'lampoffflats': [flatfield.FlatImages], + 'slitless_pixflat': [flatfield.FlatImages], 'trace': [edgetrace.EdgeTraceSet, slittrace.SlitTraceSet], 'tilt': [buildimage.TiltImage, wavetilts.WaveTilts] } @@ -1231,7 +1347,8 @@ def get_association(fitstbl, spectrograph, caldir, setup, calib_ID, det, must_ex indx = fitstbl.find_frames(frametype) & in_grp if not any(indx): continue - if not all(fitstbl['calib'][indx] == fitstbl['calib'][indx][0]): + if not (all(fitstbl['calib'][indx] == fitstbl['calib'][indx][0]) or + all([fitstbl['calib'][indx][0] in cc.split(',') for cc in fitstbl['calib'][indx]])): msgs.error(f'CODING ERROR: All {frametype} frames in group {calib_ID} ' 'are not all associated with the same subset of calibration ' 'groups; calib for the first file is ' @@ -1457,8 +1574,13 @@ def check_for_calibs(par, fitstbl, raise_error=True, cut_cfg=None): if ftype == 'pixelflat' \ and par['calibrations']['flatfield']['pixelflat_file'] is not None: continue + # Allow for no pixelflat but slitless_pixflat needs to exist + elif ftype == 'pixelflat' \ + and len(fitstbl.find_frame_files('slitless_pixflat', calib_ID=calib_ID)) > 0: + continue # Otherwise fail - msg = f'No frames of type={ftype} provide for the *{key}* processing ' \ + add_msg = ' or slitless_pixflat' if ftype == 'pixelflat' else '' + msg = f'No frames of type={ftype}{add_msg} provided for the *{key}* processing ' \ 'step. Add them to your PypeIt file!' pass_calib = False if raise_error: diff --git a/pypeit/coadd1d.py b/pypeit/coadd1d.py index fc6eafcbbf..7084aacbc2 100644 --- a/pypeit/coadd1d.py +++ b/pypeit/coadd1d.py @@ -202,7 +202,6 @@ def __init__(self, spec1dfiles, objids, spectrograph=None, par=None, sensfuncfil super().__init__(spec1dfiles, objids, spectrograph=spectrograph, par=par, sensfuncfile=sensfuncfile, setup_id=setup_id, debug=debug, show=show, chk_version=chk_version) - def load(self): """ Load the arrays we need for performing coadds. @@ -236,7 +235,7 @@ def load(self): msgs.error("Error in spec1d file for exposure {:d}: " "More than one object was identified with the OBJID={:s} in file={:s}".format( iexp, self.objids[iexp], self.spec1dfiles[iexp])) - wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, _, _, _, header = \ + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, blaze_iexp, _, header = \ sobjs[indx].unpack_object(ret_flam=self.par['flux_value'], extract_type=self.par['ex_value']) waves.append(wave_iexp) fluxes.append(flux_iexp) @@ -461,8 +460,7 @@ def coadd(self): lower=self.par['lower'], upper=self.par['upper'], maxrej=self.par['maxrej'], sn_clip=self.par['sn_clip'], debug=self.debug, show=self.show, show_exp=self.show) - - + return wave_grid_mid, wave_coadd, flux_coadd, ivar_coadd, gpm_coadd, order_stacks def load_ech_arrays(self, spec1dfiles, objids, sensfuncfiles): @@ -474,7 +472,7 @@ def load_ech_arrays(self, spec1dfiles, objids, sensfuncfiles): List of spec1d files for this setup. objids (list): List of objids. This is aligned with spec1dfiles - sensfuncfile (list): + sensfuncfiles (list): List of sensfuncfiles. This is aligned with spec1dfiles and objids Returns: @@ -488,7 +486,7 @@ def load_ech_arrays(self, spec1dfiles, objids, sensfuncfiles): indx = sobjs.name_indices(objids[iexp]) if not np.any(indx): msgs.error("No matching objects for {:s}. Odds are you input the wrong OBJID".format(objids[iexp])) - wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, _, _, _, header = \ + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, blaze_iexp, meta_spec, header = \ sobjs[indx].unpack_object(ret_flam=self.par['flux_value'], extract_type=self.par['ex_value']) # This np.atleast2d hack deals with the situation where we are wave_iexp is actually Multislit data, i.e. we are treating # it like an echelle spectrograph with a single order. This usage case arises when we want to use the @@ -496,6 +494,7 @@ def load_ech_arrays(self, spec1dfiles, objids, sensfuncfiles): if wave_iexp.ndim == 1: wave_iexp, flux_iexp, ivar_iexp, gpm_iexp = np.atleast_2d(wave_iexp).T, np.atleast_2d(flux_iexp).T, np.atleast_2d(ivar_iexp).T, np.atleast_2d(gpm_iexp).T weights_sens_iexp = sensfunc.SensFunc.sensfunc_weights(sensfuncfiles[iexp], wave_iexp, + ech_order_vec=meta_spec['ECH_ORDERS'], debug=self.debug, chk_version=self.chk_version) # Allocate arrays on first iteration @@ -511,14 +510,18 @@ def load_ech_arrays(self, spec1dfiles, objids, sensfuncfiles): header_out['RA_OBJ'] = sobjs[indx][0]['RA'] header_out['DEC_OBJ'] = sobjs[indx][0]['DEC'] - # Store the information - waves[...,iexp], fluxes[...,iexp], ivars[..., iexp], gpms[...,iexp], weights_sens[...,iexp] \ - = wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, weights_sens_iexp - + # TODO :: The error below can be removed if we refactor to use a list of numpy arrays. But, if we do that, + # we need to make several changes to the ech_combspec function. + try: + # Store the information + waves[...,iexp], fluxes[...,iexp], ivars[..., iexp], gpms[...,iexp], weights_sens[...,iexp] \ + = wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, weights_sens_iexp + except ValueError: + msgs.error('The shape (Nspec,Norder) of spectra is not consistent between exposures. ' + 'These spec1ds cannot be coadded at this time.') return waves, fluxes, ivars, gpms, weights_sens, header_out - def load(self): """ Load the arrays we need for performing echelle coadds. @@ -563,8 +566,6 @@ def load(self): for c, l in zip(combined, loaded): c.append(l) - - return waves, fluxes, ivars, gpms, weights_sens, headers diff --git a/pypeit/coadd2d.py b/pypeit/coadd2d.py index 65d92b3542..7864a2e63f 100644 --- a/pypeit/coadd2d.py +++ b/pypeit/coadd2d.py @@ -27,13 +27,13 @@ from pypeit.core import findobj_skymask from pypeit.core.wavecal import wvutils from pypeit.core import coadd -#from pypeit.core import parse -from pypeit import calibrations +from pypeit.core import parse from pypeit import spec2dobj from pypeit.core.moment import moment1d from pypeit.manual_extract import ManualExtractionObj - +#TODO We should decide which parameters go in through the parset +# and which parameters are passed in to the method as arguments class CoAdd2D: """ @@ -208,6 +208,7 @@ def __init__(self, spec2d, spectrograph, par, det=1, offsets=None, weights='auto self.pseudo_dict = None self.objid_bri = None + self.spat_pixpos_bri = None self.slitidx_bri = None self.snr_bar_bri = None self.use_weights = None # This is a list of length self.nexp that is assigned by the compute_weights method @@ -834,20 +835,20 @@ def reduce(self, pseudo_dict, show=False, clear_ginga=True, show_peaks=False, sh # maskdef stuff if parcopy['reduce']['slitmask']['assign_obj'] and slits.maskdef_designtab is not None: - # Get plate scale - platescale = sciImage.detector.platescale * self.spat_samp_fact + # Get pixel scale, binned and resampled (if requested), i.e., pixel scale of the pseudo image + resampled_pixscale = parse.parse_binning(sciImage.detector.binning)[1]*sciImage.detector.platescale*self.spat_samp_fact # Assign slitmask design information to detected objects - slits.assign_maskinfo(sobjs_obj, platescale, None, TOLER=parcopy['reduce']['slitmask']['obj_toler']) + slits.assign_maskinfo(sobjs_obj, resampled_pixscale, None, TOLER=parcopy['reduce']['slitmask']['obj_toler']) if parcopy['reduce']['slitmask']['extract_missing_objs'] is True: # Set the FWHM for the extraction of missing objects - fwhm = slits.get_maskdef_extract_fwhm(sobjs_obj, platescale, + fwhm = slits.get_maskdef_extract_fwhm(sobjs_obj, resampled_pixscale, parcopy['reduce']['slitmask']['missing_objs_fwhm'], parcopy['reduce']['findobj']['find_fwhm']) # Assign undetected objects sobjs_obj = slits.mask_add_missing_obj(sobjs_obj, None, fwhm, - parcopy['reduce']['slitmask']['missing_objs_boxcar_rad']/platescale) + parcopy['reduce']['slitmask']['missing_objs_boxcar_rad']/resampled_pixscale) # Initiate Extract object exTract = extraction.Extract.get_instance(sciImage, pseudo_dict['slits'], sobjs_obj, self.spectrograph, parcopy, @@ -870,51 +871,29 @@ def reduce(self, pseudo_dict, show=False, clear_ginga=True, show_peaks=False, sh objmodel_pseudo, ivarmodel_pseudo, outmask_pseudo, sobjs, sciImage.detector, slits, \ pseudo_dict['tilts'], pseudo_dict['waveimg'] - def snr_report(self, snr_bar, slitid=None): + @staticmethod + def offsets_report(offsets, pixscale, offsets_method): """ - ..todo.. I need a doc string + Print out a report on the offsets of the frames to be coadded Args: - snr_bar: - slitid: - - Returns: - - """ - - # Print out a report on the SNR - msg_string = msgs.newline() + '-------------------------------------' - msg_string += msgs.newline() + ' Summary for highest S/N object' - if slitid is not None: - msg_string += msgs.newline() + ' found on slitid = {:d} '.format(slitid) - msg_string += msgs.newline() + '-------------------------------------' - msg_string += msgs.newline() + ' exp# S/N' - for iexp, snr in enumerate(snr_bar): - msg_string += msgs.newline() + ' {:d} {:5.2f}'.format(iexp, snr) - - msg_string += msgs.newline() + '-------------------------------------' - msgs.info(msg_string) - - def offsets_report(self, offsets, offsets_method): - """ - Print out a report on the offsets - - Args: - offsets: - offsets_method: - - Returns: + offsets (`numpy.ndarray`_) + Array of offsets + pixscale (float): + The (binned) pixelscale in arcsec/pixel. + offsets_method (str): + A string describing the method used to determine the offsets """ if offsets_method is not None and offsets is not None: - msg_string = msgs.newline() + '---------------------------------------------' + msg_string = msgs.newline() + '---------------------------------------------------------------------------------' msg_string += msgs.newline() + ' Summary of offsets from {} '.format(offsets_method) - msg_string += msgs.newline() + '---------------------------------------------' - msg_string += msgs.newline() + ' exp# offset ' + msg_string += msgs.newline() + '---------------------------------------------------------------------------------' + msg_string += msgs.newline() + ' exp# offset (pixels) offset (arcsec)' for iexp, off in enumerate(offsets): - msg_string += msgs.newline() + ' {:d} {:5.2f}'.format(iexp, off) - msg_string += msgs.newline() + '-----------------------------------------------' + msg_string += msgs.newline() + ' {:2d} {:6.2f} {:6.3f}'.format(iexp, off, off*pixscale) + msg_string += msgs.newline() + '---------------------------------------------------------------------------------' msgs.info(msg_string) def offset_slit_cen(self, slitid, offsets): @@ -1126,17 +1105,18 @@ def compute_offsets(self, offsets): """ msgs.info('Get Offsets') + # binned pixel scale of the frames to be coadded + pixscale = parse.parse_binning(self.stack_dict['detectors'][0].binning)[1]*self.stack_dict['detectors'][0].platescale # 1) offsets are provided in the header of the spec2d files if offsets == 'header': msgs.info('Using offsets from header') - pscale = self.stack_dict['detectors'][0].platescale dithoffs = [self.spectrograph.get_meta_value(f, 'dithoff') for f in self.spec2d] if None in dithoffs: msgs.error('Dither offsets keyword not found for one or more spec2d files. ' 'Choose another option for `offsets`') - dithoffs_pix = - np.array(dithoffs) / pscale + dithoffs_pix = - np.array(dithoffs) / pixscale self.offsets = dithoffs_pix[0] - dithoffs_pix - self.offsets_report(self.offsets, 'header keyword') + self.offsets_report(self.offsets, pixscale, 'header keyword') elif self.objid_bri is None and offsets == 'auto': msgs.error('Offsets cannot be computed because no unique reference object ' @@ -1147,7 +1127,7 @@ def compute_offsets(self, offsets): msgs.info('Using user input offsets') # use them self.offsets = self.check_input(offsets, 'offsets') - self.offsets_report(self.offsets, 'user input') + self.offsets_report(self.offsets, pixscale, 'user input') # 3) parset `offsets` is = 'maskdef_offsets' (no matter if we have a bright object or not) elif offsets == 'maskdef_offsets': @@ -1155,7 +1135,7 @@ def compute_offsets(self, offsets): # the offsets computed during the main reduction (`run_pypeit`) are used msgs.info('Determining offsets using maskdef_offset recoded in SlitTraceSet') self.offsets = self.maskdef_offset[0] - self.maskdef_offset - self.offsets_report(self.offsets, 'maskdef_offset') + self.offsets_report(self.offsets, pixscale, 'maskdef_offset') else: # if maskdef_offsets were not computed during the main reduction, we cannot continue msgs.error('No maskdef_offset recoded in SlitTraceSet') @@ -1210,8 +1190,46 @@ def compute_weights(self, weights): else: msgs.error('Invalid value for `weights`') + @staticmethod + def unpack_specobj(spec, spatord_id=None): + """ + Utility routine to unpack flux, ivar, and gpm from a single SpecObj + object. + + Args: + spec (:class:`~pypeit.specobj.SpecObj`): + SpecObj object to unpack. + spatord_id (:obj:`int`, optional): + Slit/order ID to unpack. If None, the Slit/order ID + of the SpecObj object is used. + + Returns: + :obj:`tuple`: Returns the following: flux (`numpy.ndarray`_), ivar + (`numpy.ndarray`_), gpm (`numpy.ndarray`_). + + """ + # Get the slit/order ID if not provided + if spatord_id is None: + spatord_id = spec.ECH_ORDER if spec.ECH_ORDER is not None else spec.SLITID + + # get OBJID, which is different for Echelle and MultiSlit + objid = spec.ECH_OBJID if spec.ECH_OBJID is not None else spec.OBJID + + # check if OPT_COUNTS is available + if spec.has_opt_ext() and np.any(spec.OPT_MASK): + _, flux, ivar, gpm = spec.get_opt_ext() + # check if BOX_COUNTS is available + elif spec.has_box_ext() and np.any(spec.BOX_MASK): + _, flux, ivar, gpm = spec.get_box_ext() + msgs.warn(f'Optimal extraction not available for obj {objid} ' + f'in slit/order {spatord_id}. Using box extraction.') + else: + msgs.warn(f'Optimal and Boxcar extraction not available for obj {objid} in slit/order {spatord_id}.') + _, flux, ivar, gpm = None, None, None, None + + return flux, ivar, gpm - def get_brightest_object(self, specobjs_list, spat_ids): + def get_brightest_obj(self, specobjs_list, spat_ids): """ Dummy method to identify the brightest object. Overloaded by child methods. @@ -1292,30 +1310,43 @@ def __init__(self, spec2d_files, spectrograph, par, det=1, offsets=None, weights wave_method = 'linear' if self.par['coadd2d']['wave_method'] is None else self.par['coadd2d']['wave_method'] self.wave_grid, self.wave_grid_mid, self.dsamp = self.get_wave_grid(wave_method) - # Check if the user-input object to compute offsets and weights exists - if self.par['coadd2d']['user_obj'] is not None: + # If a user-input object to compute offsets and weights is provided, check if it exists and get the needed info + if len(self.stack_dict['specobjs_list']) > 0 and self.par['coadd2d']['user_obj'] is not None: if len(self.par['coadd2d']['user_obj']) != 2: msgs.error('Parameter `user_obj` must include both SLITID and OBJID.') else: user_slit, user_objid = self.par['coadd2d']['user_obj'] # does it exists? - user_obj_exist = [] - for sobjs in self.stack_dict['specobjs_list']: - user_obj_exist.append(np.any(sobjs.slitorder_objid_indices(user_slit, user_objid, - toler=self.par['coadd2d']['spat_toler']))) + user_obj_exist = np.zeros(self.nexp, dtype=bool) + # get the flux, ivar, gpm, and spatial pixel position of the user object + fluxes, ivars, gpms, spat_pixpos = [], [], [], [] + for i, sobjs in enumerate(self.stack_dict['specobjs_list']): + user_idx = sobjs.slitorder_objid_indices(user_slit, user_objid, + toler=self.par['coadd2d']['spat_toler']) + if np.any(user_idx): + this_sobj = sobjs[user_idx][0] + flux_iobj, ivar_iobj, gpm_iobj = self.unpack_specobj(this_sobj) + spat_pixpos_iobj = this_sobj.SPAT_PIXPOS + if flux_iobj is not None and ivar_iobj is not None and gpm_iobj is not None: + fluxes.append(flux_iobj) + ivars.append(ivar_iobj) + gpms.append(gpm_iobj) + spat_pixpos.append(spat_pixpos_iobj) + user_obj_exist[i] = True + # check if the user object exists in all the exposures if not np.all(user_obj_exist): msgs.error('Object provided through `user_obj` does not exist in all the exposures.') - - # find if there is a bright object we could use - if len(self.stack_dict['specobjs_list']) > 0 and self.par['coadd2d']['user_obj'] is not None: - _slitidx_bri = np.where(np.abs(self.spat_ids - user_slit) <= self.par['coadd2d']['spat_toler'])[0][0] - self.objid_bri, self.slitidx_bri, self.spatid_bri, self.snr_bar_bri = \ - np.repeat(user_objid, self.nexp), _slitidx_bri, user_slit, None + # get the needed info about the user object + self.objid_bri = np.repeat(user_objid, self.nexp) + self.spat_pixpos_bri = spat_pixpos + self.slitidx_bri = np.where(np.abs(self.spat_ids - user_slit) <= self.par['coadd2d']['spat_toler'])[0][0] + self.spatid_bri = user_slit + self.snr_bar_bri, _ = coadd.calc_snr(fluxes, ivars, gpms) + + # otherwise, find if there is a bright object we could use elif len(self.stack_dict['specobjs_list']) > 0 and (offsets == 'auto' or weights == 'auto'): - self.objid_bri, self.slitidx_bri, self.spatid_bri, self.snr_bar_bri = \ + self.objid_bri, self.spat_pixpos_bri, self.slitidx_bri, self.spatid_bri, self.snr_bar_bri = \ self.get_brightest_obj(self.stack_dict['specobjs_list'], self.spat_ids) - else: - self.objid_bri, self.slitidx_bri, self.spatid_bri, self.snr_bar_bri = (None,)*4 # get self.use_weights self.compute_weights(weights) @@ -1402,7 +1433,9 @@ def compute_offsets(self, offsets): plt.show() self.offsets = offsets - self.offsets_report(self.offsets, offsets_method) + # binned pixel scale of the frames to be coadded + pixscale = parse.parse_binning(self.stack_dict['detectors'][0].binning)[1]*self.stack_dict['detectors'][0].platescale + self.offsets_report(self.offsets, pixscale, offsets_method) def compute_weights(self, weights): """ @@ -1429,72 +1462,67 @@ def compute_weights(self, weights): msgs.info(f'Weights computed using a unique reference object in slit={self.spatid_bri} provided by the user') else: msgs.info(f'Weights computed using a unique reference object in slit={self.spatid_bri} with the highest S/N') - self.snr_report(self.snr_bar_bri, slitid=self.spatid_bri) + self.snr_report(self.spatid_bri, self.spat_pixpos_bri, self.snr_bar_bri) - def get_brightest_obj(self, specobjs_list, spat_ids): + def get_brightest_obj(self, specobjs_list, slit_spat_ids): """ - Utility routine to find the brightest object in each exposure given a specobjs_list for MultiSlit reductions. + Utility routine to find the brightest reference object in each exposure given a specobjs_list + for MultiSlit reductions. Args: specobjs_list: list List of SpecObjs objects. - spat_ids (`numpy.ndarray`_): + slit_spat_ids (`numpy.ndarray`_): Returns: tuple: Returns the following: - - objid: ndarray, int, shape (len(specobjs_list),): - Array of object ids representing the brightest object + - objid: ndarray, int, shape=(len(specobjs_list),): + Array of object ids representing the brightest reference object + in each exposure + - spatid_pixpos: ndarray, float, shape=(len(specobjs_list),): + Array of spatial pixel positions of the brightest reference object in each exposure - - slit_idx (int): 0-based index - - spat_id (int): SPAT_ID for slit that highest S/N ratio object is on - (only for pypeline=MultiSlit) + - slit_idx (int): + A zero-based index for the slit that the brightest object is on + - spat_id (int): + The SPAT_ID for the slit that the highest S/N ratio object is on - snr_bar: ndarray, float, shape (len(list),): Average - S/N over all the orders for this object + S/N computed over all the exposures for this brightest reference object """ msgs.info('Finding brightest object') nexp = len(specobjs_list) - nslits = spat_ids.size + nslits = slit_spat_ids.size slit_snr_max = np.zeros((nslits, nexp), dtype=float) bpm = np.ones(slit_snr_max.shape, dtype=bool) objid_max = np.zeros((nslits, nexp), dtype=int) + spat_pixpos_max = np.zeros((nslits, nexp), dtype=float) # Loop over each exposure, slit, find the brightest object on that slit for every exposure for iexp, sobjs in enumerate(specobjs_list): msgs.info("Working on exposure {}".format(iexp)) - for islit, spat_id in enumerate(spat_ids): + for islit, spat_id in enumerate(slit_spat_ids): if len(sobjs) == 0: continue ithis = np.abs(sobjs.SLITID - spat_id) <= self.par['coadd2d']['spat_toler'] if np.any(ithis): objid_this = sobjs[ithis].OBJID + spat_pixpos_this = sobjs[ithis].SPAT_PIXPOS fluxes, ivars, gpms = [], [], [] - for iobj, spec in enumerate(sobjs[ithis]): - # check if OPT_COUNTS is available - if spec.has_opt_ext() and np.any(spec.OPT_MASK): - _, flux_iobj, ivar_iobj, gpm_iobj = spec.get_opt_ext() - fluxes.append(flux_iobj) - ivars.append(ivar_iobj) - gpms.append(gpm_iobj) - # check if BOX_COUNTS is available - elif spec.has_box_ext() and np.any(spec.BOX_MASK): - _, flux_iobj, ivar_iobj, gpm_iobj = spec.get_box_ext() + for spec in sobjs[ithis]: + flux_iobj, ivar_iobj, gpm_iobj = self.unpack_specobj(spec, spatord_id=spat_id) + if flux_iobj is not None and ivar_iobj is not None and gpm_iobj is not None: fluxes.append(flux_iobj) ivars.append(ivar_iobj) gpms.append(gpm_iobj) - msgs.warn(f'Optimal extraction not available for obj {spec.OBJID} ' - f'in slit {spat_id}. Using box extraction.') - # if both are not available, we remove the object in this slit, - # because otherwise coadd.sn_weights will crash - else: - msgs.warn(f'Optimal and Boxcar extraction not available for obj {spec.OBJID} in slit {spat_id}.') - #remove_indx.append(iobj) + # if there are objects on this slit left, we can proceed with computing rms_sn if len(fluxes) > 0: rms_sn, _ = coadd.calc_snr(fluxes, ivars, gpms) imax = np.argmax(rms_sn) slit_snr_max[islit, iexp] = rms_sn[imax] objid_max[islit, iexp] = objid_this[imax] + spat_pixpos_max[islit, iexp] = spat_pixpos_this[imax] bpm[islit, iexp] = False # If a slit has bpm = True for some exposures and not for others, set bpm = True for all exposures @@ -1517,8 +1545,40 @@ def get_brightest_obj(self, specobjs_list, spat_ids): snr_bar_mean = slit_snr[slitid] snr_bar = slit_snr_max[slitid, :] objid = objid_max[slitid, :] + spat_pixpos = spat_pixpos_max[slitid, :] + + return objid, spat_pixpos, slitid, slit_spat_ids[slitid], snr_bar + + def snr_report(self, slitid, spat_pixpos, snr_bar): + """ + + Print out a SNR report for the reference object used to compute the weights for multislit 2D coadds. + + Args: + slitid (:obj:`int`): + The SPAT_ID of the slit that the reference object is on + spat_pixpos (:obj:`numpy.ndarray`): + Array of spatial pixel position of the reference object in the slit for each exposure shape = (nexp,) + snr_bar (:obj:`numpy.ndarray`): + Array of average S/N ratios for the reference object in each exposure, shape = (nexp,) + + + Returns: + + """ + + # Print out a report on the SNR + msg_string = msgs.newline() + '-------------------------------------' + msg_string += msgs.newline() + ' Summary for highest S/N object' + msg_string += msgs.newline() + ' found on slitid = {:d} '.format(slitid) + msg_string += msgs.newline() + '-------------------------------------' + msg_string += msgs.newline() + ' exp# spat_pixpos S/N' + msg_string += msgs.newline() + '-------------------------------------' + for iexp, (spat,snr) in enumerate(zip(spat_pixpos, snr_bar)): + msg_string += msgs.newline() + ' {:2d} {:7.1f} {:5.2f}'.format(iexp, spat, snr) + msg_string += msgs.newline() + '-------------------------------------' + msgs.info(msg_string) - return objid, slitid, spat_ids[slitid], snr_bar # TODO add an option here to actually use the reference trace for cases where they are on the same slit and it is # single slit??? @@ -1648,28 +1708,30 @@ def __init__(self, spec2d_files, spectrograph, par, det=1, offsets=None, weights wave_method = 'log10' if self.par['coadd2d']['wave_method'] is None else self.par['coadd2d']['wave_method'] self.wave_grid, self.wave_grid_mid, self.dsamp = self.get_wave_grid(wave_method) - # Check if the user-input object to compute offsets and weights exists - if self.par['coadd2d']['user_obj'] is not None: + # If a user-input object to compute offsets and weights is provided, check if it exists and get the needed info + if len(self.stack_dict['specobjs_list']) > 0 and self.par['coadd2d']['user_obj'] is not None: if not isinstance(self.par['coadd2d']['user_obj'], int): msgs.error('Parameter `user_obj` must include only the object OBJID.') else: user_objid = self.par['coadd2d']['user_obj'] # does it exists? - user_obj_exist = [] - for sobjs in self.stack_dict['specobjs_list']: + user_obj_exist = np.zeros((self.nexp,self.nslits_single), dtype=bool) + for i, sobjs in enumerate(self.stack_dict['specobjs_list']): for iord in range(self.nslits_single): - user_obj_exist.append(np.any(sobjs.slitorder_objid_indices(sobjs.ECH_ORDER[iord], user_objid))) + # check if the object exists in this exposure + ind = (sobjs.ECH_ORDERINDX == iord) & (sobjs.ECH_OBJID == user_objid) + flux, ivar, mask = self.unpack_specobj(sobjs[ind][0]) + if flux is not None and ivar is not None and mask is not None: + user_obj_exist[i, iord] = True if not np.all(user_obj_exist): msgs.error('Object provided through `user_obj` does not exist in all the exposures.') - # find if there is a bright object we could use - if len(self.stack_dict['specobjs_list']) > 0 and self.par['coadd2d']['user_obj'] is not None: - self.objid_bri, self.slitidx_bri, self.snr_bar_bri = np.repeat(user_objid, self.nexp), None, None + # get the needed info about the user object + self.objid_bri, self.slitidx_bri, self.snr_bar_bri = np.repeat(user_objid, self.nexp), None, None + elif len(self.stack_dict['specobjs_list']) > 0 and (offsets == 'auto' or weights == 'auto'): self.objid_bri, self.slitidx_bri, self.snr_bar_bri = \ self.get_brightest_obj(self.stack_dict['specobjs_list'], self.nslits_single) - else: - self.objid_bri, self.slitidx_bri, self.snr_bar_bri = (None,)*3 # get self.use_weights self.compute_weights(weights) @@ -1726,6 +1788,7 @@ def compute_weights(self, weights): self.use_weights.append(iweights) if self.par['coadd2d']['user_obj'] is not None: msgs.info('Weights computed using a unique reference object provided by the user') + # TODO: implement something here to print out the snr_report else: msgs.info('Weights computed using a unique reference object with the highest S/N') self.snr_report(self.snr_bar_bri) @@ -1760,21 +1823,10 @@ def get_brightest_obj(self, specobjs_list, nslits): bpm = np.ones((nslits, nobjs), dtype=bool) for iord in range(nslits): for iobj in range(nobjs): - flux = None ind = (sobjs.ECH_ORDERINDX == iord) & (sobjs.ECH_OBJID == uni_objid[iobj]) - # check if OPT_COUNTS is available - if sobjs[ind][0].has_opt_ext() and np.any(sobjs[ind][0].OPT_MASK): - _, flux, ivar, mask = sobjs[ind][0].get_opt_ext() - # check if BOX_COUNTS is available - elif sobjs[ind][0].has_box_ext() and np.any(sobjs[ind][0].BOX_MASK): - _, flux, ivar, mask = sobjs[ind][0].get_box_ext() - msgs.warn(f'Optimal extraction not available for object {sobjs[ind][0].ECH_OBJID} ' - f'in order {sobjs[ind][0].ECH_ORDER}. Using box extraction.') - else: - msgs.warn(f'Optimal and Boxcar extraction not available for ' - f'object {sobjs[ind][0].ECH_OBJID} in order {sobjs[ind][0].ECH_ORDER}.') - continue - if flux is not None: + flux, ivar, mask = self.unpack_specobj(sobjs[ind][0], spatord_id=sobjs[ind][0].ECH_ORDER) + + if flux is not None and ivar is not None and mask is not None: rms_sn, _ = coadd.calc_snr([flux], [ivar], [mask]) order_snr[iord, iobj] = rms_sn[0] bpm[iord, iobj] = False @@ -1799,6 +1851,30 @@ def get_brightest_obj(self, specobjs_list, nslits): return None, None, None return objid, None, snr_bar + def snr_report(self, snr_bar): + """ + Printo out a SNR report for echelle 2D coadds. + + Args: + snr_bar (:obj:`numpy.ndarray`): + Array of average S/N ratios for the brightest object in each exposure. Shape = (nexp,) + + Returns: + + """ + + # Print out a report on the SNR + msg_string = msgs.newline() + '-------------------------------------' + msg_string += msgs.newline() + ' Summary for highest S/N object' + msg_string += msgs.newline() + '-------------------------------------' + msg_string += msgs.newline() + ' exp# S/N' + for iexp, snr in enumerate(snr_bar): + msg_string += msgs.newline() + ' {:d} {:5.2f}'.format(iexp, snr) + + msg_string += msgs.newline() + '-------------------------------------' + msgs.info(msg_string) + + def reference_trace_stack(self, slitid, offsets=None, objid=None): """ Utility function for determining the reference trace about diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index e6cc7d356c..5638cb6f77 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -15,7 +15,7 @@ import numpy as np from pypeit import msgs -from pypeit import alignframe, datamodel, flatfield, io, spec2dobj, utils +from pypeit import alignframe, datamodel, flatfield, io, sensfunc, spec2dobj, utils from pypeit.core.flexure import calculate_image_phase from pypeit.core import datacube, extract, flux_calib, parse from pypeit.spectrographs.util import load_spectrograph @@ -93,7 +93,7 @@ class DataCube(datamodel.DataContainer): 'spectrograph', 'spect_meta', '_ivar', # This is set internally, and should be accessed with self.ivar - '_wcs' # This is set internally, and should be accessed with self.wcs + '_wcs' ] def __init__(self, flux, sig, bpm, wave, PYP_SPEC, blaze_wave, blaze_spec, sensfunc=None, @@ -106,6 +106,7 @@ def __init__(self, flux, sig, bpm, wave, PYP_SPEC, blaze_wave, blaze_spec, sensf # Initialise the internals self._ivar = None self._wcs = None + self.head0 = None # This contains the primary header of the spec2d used to make the datacube def _bundle(self): """ @@ -162,9 +163,12 @@ def to_file(self, ofile, primary_hdr=None, hdr=None, **kwargs): subheader = spectrograph.subheader_for_spec(self.head0, self.head0) else: subheader = {} - # Add em in + # Add them in for key in subheader: primary_hdr[key] = subheader[key] + # Set the exposure time to 1, since datacubes are counts/second. + # This is needed for the sensitivity function calculation + primary_hdr['EXPTIME'] = 1.0 # Do it super(DataCube, self).to_file(ofile, primary_hdr=primary_hdr, hdr=hdr, **kwargs) @@ -175,7 +179,7 @@ def from_file(cls, ifile, verbose=True, chk_version=True, **kwargs): Over-load :func:`~pypeit.datamodel.DataContainer.from_file` to deal with the header - + Args: ifile (:obj:`str`, `Path`_): Fits file with the data to read @@ -191,12 +195,12 @@ def from_file(cls, ifile, verbose=True, chk_version=True, **kwargs): self = cls.from_hdu(hdu, chk_version=chk_version, **kwargs) # Internals self.filename = ifile - self.head0 = hdu[1].header # Actually use the first extension here, since it contains the WCS + self.head0 = hdu[0].header # Meta self.spectrograph = load_spectrograph(self.PYP_SPEC) self.spect_meta = self.spectrograph.parse_spec_header(hdu[0].header) self._ivar = None - self._wcs = None + self._wcs = wcs.WCS(hdu[1].header) return self @property @@ -214,20 +218,35 @@ def ivar(self): self._ivar = utils.inverse(self.sig**2) return self._ivar - @property - def wcs(self): + def extract_spec(self, parset, outname=None, boxcar_radius=None, overwrite=False): """ - Utility function to provide the world coordinate system of the datacube + Extract a spectrum from the datacube - Returns - ------- - self._wcs : `astropy.wcs.WCS`_ - The WCS based on the stored header information. Note that self._wcs should - not be accessed directly, and you should only call self.wcs + Parameters + ---------- + parset : dict + A dictionary containing the :class:`~pypeit.par.pypeitpar.ReducePar` parameters. + outname : str, optional + Name of the output file + boxcar_radius : float, optional + Radius of the circular boxcar (in arcseconds) to use for the extraction + overwrite : bool, optional + Overwrite any existing files """ - if self._wcs is None: - self._wcs = wcs.WCS(self.head0) - return self._wcs + # Extract the spectrum + fwhm = parset['findobj']['find_fwhm'] if parset['extraction']['use_user_fwhm'] else None + + # Datacube's are counts/second, so set the exposure time to 1 + exptime = 1.0 + # TODO :: Avoid transposing these large cubes + sobjs = datacube.extract_point_source(self.wave, self.flux.T, self.ivar.T, self.bpm.T, self._wcs, + exptime=exptime, pypeline=self.spectrograph.pypeline, + fluxed=self.fluxed, boxcar_radius=boxcar_radius, + optfwhm=fwhm, whitelight_range=parset['cube']['whitelight_range']) + + # Save the extracted spectrum + spec1d_filename = 'spec1d_' + self.filename if outname is None else outname + sobjs.write_to_fits(self.head0, spec1d_filename, overwrite=overwrite) class DARcorrection: @@ -340,9 +359,9 @@ class CoAdd3D: """ # Superclass factory method generates the subclass instance @classmethod - def get_instance(cls, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, - dec_offsets=None, spectrograph=None, det=1, overwrite=False, show=False, - debug=False): + def get_instance(cls, spec2dfiles, par, skysub_frame=None, sensfile=None, scale_corr=None, grating_corr=None, + ra_offsets=None, dec_offsets=None, spectrograph=None, det=1, + overwrite=False, show=False, debug=False): """ Instantiate the subclass appropriate for the provided spectrograph. @@ -357,13 +376,13 @@ def get_instance(cls, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_o """ return next(c for c in cls.__subclasses__() if c.__name__ == (spectrograph.pypeline + 'CoAdd3D'))( - spec2dfiles, par, skysub_frame=skysub_frame, scale_corr=scale_corr, - ra_offsets=ra_offsets, dec_offsets=dec_offsets, spectrograph=spectrograph, - det=det, overwrite=overwrite, show=show, debug=debug) + spec2dfiles, par, skysub_frame=skysub_frame, sensfile=sensfile, scale_corr=scale_corr, + grating_corr=grating_corr, ra_offsets=ra_offsets, dec_offsets=dec_offsets, + spectrograph=spectrograph, det=det, overwrite=overwrite, show=show, debug=debug) - def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, - dec_offsets=None, spectrograph=None, det=None, overwrite=False, show=False, - debug=False): + def __init__(self, spec2dfiles, par, skysub_frame=None, sensfile=None, scale_corr=None, grating_corr=None, + ra_offsets=None, dec_offsets=None, spectrograph=None, det=None, + overwrite=False, show=False, debug=False): """ Args: @@ -378,9 +397,16 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs skysub_frame (:obj:`list`, optional): If not None, this should be a list of frames to use for the sky subtraction of each individual entry of spec2dfiles. It should be the same length as spec2dfiles. + sensfile (:obj:`list`, optional): + If not None, this should be a list of frames to use for the sensitivity function of each individual + entry of spec2dfiles. It should be the same length as spec2dfiles. scale_corr (:obj:`list`, optional): If not None, this should be a list of relative scale correction options. It should be the same length as spec2dfiles. + grating_corr (:obj:`list`, optional): + If not None, this should be a list of `str`, where each element is the relative path to the + Flat calibration file that was used to reduce each spec2d file. It should be the + same length as spec2dfiles. ra_offsets (:obj:`list`, optional): If not None, this should be a list of relative RA offsets of each frame. It should be the same length as spec2dfiles. The units should be degrees. @@ -418,8 +444,12 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs # Do some quick checks on the input options if skysub_frame is not None and len(skysub_frame) != self.numfiles: msgs.error("The skysub_frame list should be identical length to the spec2dfiles list") + if sensfile is not None and len(sensfile) != self.numfiles: + msgs.error("The sensfile list should be identical length to the spec2dfiles list") if scale_corr is not None and len(scale_corr) != self.numfiles: msgs.error("The scale_corr list should be identical length to the spec2dfiles list") + if grating_corr is not None and len(grating_corr) != self.numfiles: + msgs.error("The grating_corr list should be identical length to the spec2dfiles list") if ra_offsets is not None and len(ra_offsets) != self.numfiles: msgs.error("The ra_offsets list should be identical length to the spec2dfiles list") if dec_offsets is not None and len(dec_offsets) != self.numfiles: @@ -430,8 +460,18 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs if ra_offsets is not None and dec_offsets is None: msgs.error("If you provide ra_offsets, you must also provide dec_offsets") # Set the frame specific options + self.sensfile = None + if sensfile is None: + # User didn't provide a sensfile for each frame. Check if they provided a single one. + if self.cubepar['sensfile'] is not None: + # User provided a single sensfile. Use this for all frames. + self.sensfile = self.numfiles*[self.cubepar['sensfile']] + else: + # User provided a sensfile for each frame. Use these. + self.sensfile = sensfile self.skysub_frame = skysub_frame self.scale_corr = scale_corr + self.grating_corr = grating_corr self.ra_offsets = list(ra_offsets) if isinstance(ra_offsets, np.ndarray) else ra_offsets self.dec_offsets = list(dec_offsets) if isinstance(dec_offsets, np.ndarray) else dec_offsets # If there is only one frame being "combined" AND there's no reference image, then don't compute the translation. @@ -456,6 +496,8 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs # Initialise the lists of ra_offsets and dec_offsets self.ra_offsets = [0.0]*self.numfiles self.dec_offsets = [0.0]*self.numfiles + if self.grating_corr is None: + self.grating_corr = [None] * self.numfiles # Check on Spectrograph input if spectrograph is None: @@ -469,7 +511,7 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs self.ifu_ra, self.ifu_dec = np.array([]), np.array([]) # The RA and Dec at the centre of the IFU, as stored in the header self.all_sci, self.all_ivar, self.all_wave, self.all_slitid, self.all_wghts = [], [], [], [], [] - self.all_tilts, self.all_slits, self.all_align = [], [], [] + self.all_tilts, self.all_slits, self.all_align, self.all_header = [], [], [], [] self.all_wcs, self.all_ra, self.all_dec, self.all_dar = [], [], [], [] self.weights = np.ones(self.numfiles) # Weights to use when combining cubes @@ -506,12 +548,10 @@ def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offs self.check_outputs() # Check the reference cube and image exist, if requested - self.fluxcal = False + self.fluxcal = False if self.sensfile is None else True self.blaze_wave, self.blaze_spec = None, None self.blaze_spline, self.flux_spline = None, None self.flat_splines = dict() # A dictionary containing the splines of the flatfield - if self.cubepar['standard_cube'] is not None: - self.make_sensfunc() # If a reference image has been set, check that it exists if self.cubepar['reference_image'] is not None: @@ -578,22 +618,6 @@ def set_blaze_spline(self, wave_spl, spec_spl): self.blaze_spline = interp1d(wave_spl, spec_spl, kind='linear', bounds_error=False, fill_value="extrapolate") - def make_sensfunc(self): - """ - Generate the sensitivity function to be used for the flux calibration. - """ - self.fluxcal = True - # The first standard star cube is used as the reference blaze spline - if self.cubepar['grating_corr']: - # Load the blaze information - stdcube = fits.open(self.cubepar['standard_cube']) - # If a reference blaze spline has not been set, do that now. - self.set_blaze_spline(stdcube['BLAZE_WAVE'].data, stdcube['BLAZE_SPEC'].data) - # Generate a spline representation of the sensitivity function - self.flux_spline = datacube.make_sensfunc(self.cubepar['standard_cube'], self.senspar, - blaze_wave=self.blaze_wave, blaze_spline=self.blaze_spline, - grating_corr=self.cubepar['grating_corr']) - def set_default_scalecorr(self): """ Set the default mode to use for relative spectral scale correction. @@ -814,8 +838,8 @@ def add_grating_corr(self, flatfile, waveimg, slits, spat_flexure=None): msgs.info("Calculating relative sensitivity for grating correction") # Load the Flat file flatimages = flatfield.FlatImages.from_file(flatfile, chk_version=self.chk_version) - total_illum = flatimages.fit2illumflat(slits, finecorr=False, frametype='illum', initial=True, spat_flexure=spat_flexure) * \ - flatimages.fit2illumflat(slits, finecorr=True, frametype='illum', initial=True, spat_flexure=spat_flexure) + total_illum = flatimages.fit2illumflat(slits, finecorr=False, frametype='illum', spat_flexure=spat_flexure) * \ + flatimages.fit2illumflat(slits, finecorr=True, frametype='illum', spat_flexure=spat_flexure) flatframe = flatimages.pixelflat_raw / total_illum if flatimages.pixelflat_spec_illum is None: # Calculate the relative scale @@ -877,12 +901,13 @@ class SlicerIFUCoAdd3D(CoAdd3D): - White light images are also produced, if requested. """ - def __init__(self, spec2dfiles, par, skysub_frame=None, scale_corr=None, ra_offsets=None, - dec_offsets=None, spectrograph=None, det=1, overwrite=False, show=False, - debug=False): - super().__init__(spec2dfiles, par, skysub_frame=skysub_frame, scale_corr=scale_corr, - ra_offsets=ra_offsets, dec_offsets=dec_offsets, spectrograph=spectrograph, - det=det, overwrite=overwrite, show=show, debug=debug) + def __init__(self, spec2dfiles, par, skysub_frame=None, sensfile=None, scale_corr=None, grating_corr=None, + ra_offsets=None, dec_offsets=None, spectrograph=None, det=1, + overwrite=False, show=False, debug=False): + super().__init__(spec2dfiles, par, skysub_frame=skysub_frame, sensfile=sensfile, + scale_corr=scale_corr, grating_corr=grating_corr, + ra_offsets=ra_offsets, dec_offsets=dec_offsets, spectrograph=spectrograph, det=det, + overwrite=overwrite, show=show, debug=debug) self.mnmx_wv = None # Will be used to store the minimum and maximum wavelengths of every slit and frame. self._spatscale = np.zeros((self.numfiles, 2)) # index 0, 1 = pixel scale, slicer scale self._specscale = np.zeros(self.numfiles) @@ -925,7 +950,7 @@ def get_alignments(self, spec2DObj, slits, spat_flexure=None): msgs.info("Using slit edges for astrometric transform") # If nothing better was provided, use the slit edges if alignments is None: - left, right, _ = slits.select_edges(initial=True, flexure=spat_flexure) + left, right, _ = slits.select_edges(flexure=spat_flexure) locations = [0.0, 1.0] traces = np.append(left[:, None, :], right[:, None, :], axis=1) else: @@ -972,7 +997,7 @@ def load(self): # Load all spec2d files and prepare the data for making a datacube for ff, fil in enumerate(self.spec2d): # Load it up - msgs.info("Loading PypeIt spec2d frame:" + msgs.newline() + fil) + msgs.info(f"Loading PypeIt spec2d frame ({ff+1}/{len(self.spec2d)}):" + msgs.newline() + fil) spec2DObj = spec2dobj.Spec2DObj.from_file(fil, self.detname, chk_version=self.chk_version) detector = spec2DObj.detector @@ -980,6 +1005,7 @@ def load(self): # Load the header hdr0 = spec2DObj.head0 + self.all_header.append(hdr0) self.ifu_ra = np.append(self.ifu_ra, self.spec.compound_meta([hdr0], 'ra')) self.ifu_dec = np.append(self.ifu_dec, self.spec.compound_meta([hdr0], 'dec')) @@ -989,8 +1015,8 @@ def load(self): # Initialise the slit edges msgs.info("Constructing slit image") slits = spec2DObj.slits - slitid_img_init = slits.slit_img(pad=0, initial=True, flexure=spat_flexure) - slits_left, slits_right, _ = slits.select_edges(initial=True, flexure=spat_flexure) + slitid_img = slits.slit_img(pad=0, flexure=spat_flexure) + slits_left, slits_right, _ = slits.select_edges(flexure=spat_flexure) # The order of operations below proceeds as follows: # (1) Get science image @@ -1015,7 +1041,7 @@ def load(self): bpmmask = spec2DObj.bpmmask # Mask the edges of the spectrum where the sky model is bad - sky_is_good = datacube.make_good_skymask(slitid_img_init, spec2DObj.tilts) + sky_is_good = datacube.make_good_skymask(slitid_img, spec2DObj.tilts) # TODO :: Really need to write some detailed information in the docs about all of the various corrections that can optionally be applied @@ -1048,7 +1074,7 @@ def load(self): if self.mnmx_wv is None: self.mnmx_wv = np.zeros((len(self.spec2d), slits.nslits, 2)) for slit_idx, slit_spat in enumerate(slits.spat_id): - onslit_init = (slitid_img_init == slit_spat) + onslit_init = (slitid_img == slit_spat) self.mnmx_wv[ff, slit_idx, 0] = np.min(waveimg[onslit_init]) self.mnmx_wv[ff, slit_idx, 1] = np.max(waveimg[onslit_init]) @@ -1071,7 +1097,7 @@ def load(self): # Construct a good pixel mask # TODO: This should use the mask function to figure out which elements are masked. - onslit_gpm = (slitid_img_init > 0) & (bpmmask.mask == 0) & sky_is_good + onslit_gpm = (slitid_img > 0) & (bpmmask.mask == 0) & sky_is_good # Generate the alignment splines, and then retrieve images of the RA and Dec of every pixel, # and the number of spatial pixels in each slit @@ -1083,18 +1109,18 @@ def load(self): crval_wv = self.cubepar['wave_min'] if self.cubepar['wave_min'] is not None else wave0 cd_wv = self.cubepar['wave_delta'] if self.cubepar['wave_delta'] is not None else dwv self.all_wcs.append(self.spec.get_wcs(spec2DObj.head0, slits, detector.platescale, crval_wv, cd_wv)) - ra_img, dec_img, minmax = slits.get_radec_image(self.all_wcs[ff], alignSplines, spec2DObj.tilts, initial=True, flexure=spat_flexure) + ra_img, dec_img, minmax = slits.get_radec_image(self.all_wcs[ff], alignSplines, spec2DObj.tilts, flexure=spat_flexure) # Extract wavelength and delta wavelength arrays from the images wave_ext = waveimg[onslit_gpm] dwav_ext = dwaveimg[onslit_gpm] # For now, work in sorted wavelengths - wvsrt = np.argsort(wave_ext) + wvsrt = np.argsort(wave_ext, kind='stable') wave_sort = wave_ext[wvsrt] dwav_sort = dwav_ext[wvsrt] # Here's an array to get back to the original ordering - resrt = np.argsort(wvsrt) + resrt = np.argsort(wvsrt, kind='stable') # Compute the DAR correction cosdec = np.cos(self.ifu_dec[ff] * np.pi / 180.0) @@ -1105,54 +1131,74 @@ def load(self): humidity = self.spec.get_meta_value([spec2DObj.head0], 'humidity') # Expressed as a percentage (not a fraction!) darcorr = DARcorrection(airmass, parangle, pressure, temperature, humidity, cosdec) - # Compute the extinction correction - msgs.info("Applying extinction correction") - extinct = flux_calib.load_extinction_data(self.spec.telescope['longitude'], - self.spec.telescope['latitude'], - self.senspar['UVIS']['extinct_file']) - # extinction_correction requires the wavelength is sorted - extcorr_sort = flux_calib.extinction_correction(wave_sort * units.AA, airmass, extinct) + # TODO :: Need to make a note somewhere that the extinction correction cannot currently be done + # in the datacube because the sensitivity function algorithms correct the standard star + # for extinction when generating the sensitivity function. Including the extinction correction + # in the datacube would result in a double correction of the standard star for extinction. + # This could be wrong when combining multiple standard star exposures if the airmass of the + # standard star exposures is significantly different. For now, to stay consistent with the + # current pipeline, the extinction correction is done in the sensitivity function algorithms, + # with the caveat that the standard star exposures are assumed to have similar airmasses. + # For now, comment out the extinction correction, and reapply this later when the sensitivity + # function algorithms are unified. + extcorr_sort = 1.0 + if False: + # Compute the extinction correction + msgs.info("Applying extinction correction") + # TODO :: Change the ['UVIS']['extinct_file'] here when the sensitivity function calculation is unified. + extinct = flux_calib.load_extinction_data(self.spec.telescope['longitude'], + self.spec.telescope['latitude'], + self.senspar['UVIS']['extinct_file']) + # extinction_correction requires the wavelength is sorted + extcorr_sort = flux_calib.extinction_correction(wave_sort * units.AA, airmass, extinct) # Correct for sensitivity as a function of grating angle # (this assumes the spectrum of the flatfield lamp has the same shape for all setups) gratcorr_sort = 1.0 - if self.cubepar['grating_corr']: - # Load the flatfield file - key = flatfield.FlatImages.calib_type.upper() - if key not in spec2DObj.calibs: - msgs.error('Processed flat calibration file not recorded by spec2d file!') - flatfile = os.path.join(spec2DObj.calibs['DIR'], spec2DObj.calibs[key]) + if self.grating_corr[ff] is not None: # Setup the grating correction + flatfile = self.grating_corr[ff] self.add_grating_corr(flatfile, waveimg, slits, spat_flexure=spat_flexure) # Calculate the grating correction gratcorr_sort = datacube.correct_grating_shift(wave_sort, self.flat_splines[flatfile + "_wave"], self.flat_splines[flatfile], self.blaze_wave, self.blaze_spline) - # Sensitivity function - sensfunc_sort = 1.0 + # Sensitivity function - note that the sensitivity function factors in the exposure time and the + # wavelength sampling, so if the flux calibration will not be applied, the sens_factor needs to be + # scaled by the exposure time and the wavelength sampling + sens_sort = 1.0/(exptime * dwav_sort) # If no sensitivity function is provided if self.fluxcal: msgs.info("Calculating the sensitivity function") - sensfunc_sort = self.flux_spline(wave_sort) - # Convert the flux_sav to counts/s, correct for the relative sensitivity of different setups - extcorr_sort *= sensfunc_sort / (exptime * gratcorr_sort) + # Load the sensitivity function + sens = sensfunc.SensFunc.from_file(self.sensfile[ff], chk_version=self.par['rdx']['chk_version']) + # Interpolate the sensitivity function onto the wavelength grid of the data + # TODO :: Change the ['UVIS']['extinct_file'] here when the sensitivity function calculation is unified. + sens_sort = flux_calib.get_sensfunc_factor( + wave_sort, sens.wave[:, 0], sens.zeropoint[:, 0], exptime, delta_wave=dwav_sort, + extinct_correct=True, longitude=self.spec.telescope['longitude'], + latitude=self.spec.telescope['latitude'], extinctfilepar=self.senspar['UVIS']['extinct_file'], + airmass=airmass, extrap_sens=self.par['fluxcalib']['extrap_sens']) + # Convert the flux units to counts/s, and correct for the relative sensitivity of different setups + sens_sort *= extcorr_sort/gratcorr_sort # Correct for extinction - sciImg[onslit_gpm] *= extcorr_sort[resrt] - ivar[onslit_gpm] /= extcorr_sort[resrt] ** 2 + sciImg[onslit_gpm] *= sens_sort[resrt] + ivar[onslit_gpm] /= sens_sort[resrt] ** 2 # Convert units to Counts/s/Ang/arcsec2 # Slicer sampling * spatial pixel sampling + unitscale = self.all_wcs[ff].wcs.cunit[0].to(units.arcsec) * self.all_wcs[ff].wcs.cunit[1].to(units.arcsec) sl_deg = np.sqrt(self.all_wcs[ff].wcs.cd[0, 0] ** 2 + self.all_wcs[ff].wcs.cd[1, 0] ** 2) px_deg = np.sqrt(self.all_wcs[ff].wcs.cd[1, 1] ** 2 + self.all_wcs[ff].wcs.cd[0, 1] ** 2) - scl_units = dwav_sort * (3600.0 * sl_deg) * (3600.0 * px_deg) - sciImg[onslit_gpm] /= scl_units[resrt] - ivar[onslit_gpm] *= scl_units[resrt] ** 2 + scl_units = unitscale * sl_deg * px_deg + sciImg[onslit_gpm] /= scl_units + ivar[onslit_gpm] *= scl_units ** 2 # Calculate the weights relative to the zeroth cube self.weights[ff] = 1.0 # exptime #np.median(flux_sav[resrt]*np.sqrt(ivar_sav[resrt]))**2 wghts = self.weights[ff] * np.ones(sciImg.shape) # Get the slit image and then unset pixels in the slit image that are bad - slitid_img_gpm = slitid_img_init * onslit_gpm.astype(int) + slitid_img_gpm = slitid_img * onslit_gpm.astype(int) # If individual frames are to be output without aligning them, # there's no need to store information, just make the cubes now @@ -1163,7 +1209,7 @@ def load(self): else: outfile = datacube.get_output_filename(fil, self.cubepar['output_filename'], self.combine, ff + 1) # Get the coordinate bounds - slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) + slitlength = int(np.round(np.median(slits.get_slitlengths(median=True)))) numwav = int((np.max(waveimg) - wave0) / dwv) bins = self.spec.get_datacube_bins(slitlength, minmax, numwav) # Set the wavelength range of the white light image. @@ -1195,7 +1241,7 @@ def load(self): msgs.info("Saving datacube as: {0:s}".format(outfile)) final_cube = DataCube(flxcube, sigcube, bpmcube, wave, self.specname, self.blaze_wave, self.blaze_spec, sensfunc=None, fluxed=self.fluxcal) - final_cube.to_file(outfile, hdr=hdr, overwrite=self.overwrite) + final_cube.to_file(outfile, primary_hdr=self.all_header[ff], hdr=hdr, overwrite=self.overwrite) # No need to proceed and store arrays - we are writing individual datacubes continue @@ -1404,7 +1450,8 @@ def run(self): msgs.info("Saving datacube as: {0:s}".format(outfile)) final_cube = DataCube(flxcube, sigcube, bpmcube, wave, self.specname, self.blaze_wave, self.blaze_spec, sensfunc=sensfunc, fluxed=self.fluxcal) - final_cube.to_file(outfile, hdr=hdr, overwrite=self.overwrite) + # Note, we only store in the primary header the first spec2d file + final_cube.to_file(outfile, primary_hdr=self.all_header[0], hdr=hdr, overwrite=self.overwrite) else: for ff in range(self.numfiles): outfile = datacube.get_output_filename("", self.cubepar['output_filename'], False, ff) @@ -1431,4 +1478,4 @@ def run(self): msgs.info("Saving datacube as: {0:s}".format(outfile)) final_cube = DataCube(flxcube, sigcube, bpmcube, wave, self.specname, self.blaze_wave, self.blaze_spec, sensfunc=sensfunc, fluxed=self.fluxcal) - final_cube.to_file(outfile, hdr=hdr, overwrite=self.overwrite) + final_cube.to_file(outfile, primary_hdr=self.all_header[ff], hdr=hdr, overwrite=self.overwrite) diff --git a/pypeit/core/arc.py b/pypeit/core/arc.py index e69230cf28..e41fc7fc8b 100644 --- a/pypeit/core/arc.py +++ b/pypeit/core/arc.py @@ -507,6 +507,7 @@ def get_censpec(slit_cen, slitmask, arcimg, gpm=None, box_rad=3.0, arc_spec[arc_spec_bpm] = 0.0 return arc_spec, arc_spec_bpm, np.all(arc_spec_bpm, axis=0) + def detect_peaks(x, mph=None, mpd=1, threshold=0, edge='rising', kpsh=False, valley=False, show=False, ax=None): """Detect peaks in data based on their amplitude and other features. @@ -644,7 +645,7 @@ def detect_peaks(x, mph=None, mpd=1, threshold=0, edge='rising', ind = np.delete(ind, np.where(dx < threshold)[0]) # detect small peaks closer than minimum peak distance if ind.size and mpd > 1: - ind = ind[np.argsort(x[ind])][::-1] # sort ind by peak height + ind = ind[np.argsort(x[ind], kind='stable')][::-1] # sort ind by peak height idel = np.zeros(ind.size, dtype=bool) for i in range(ind.size): if not idel[i]: @@ -783,7 +784,7 @@ def iter_continuum(spec, gpm=None, fwhm=4.0, sigthresh = 2.0, sigrej=3.0, niter_ max_nmask = int(np.ceil((max_mask_frac)*nspec_available)) for iter in range(niter_cont): spec_sub = spec - cont_now - mask_sigclip = np.invert(cont_mask & gpm) + mask_sigclip = np.logical_not(cont_mask & gpm) (mean, med, stddev) = stats.sigma_clipped_stats(spec_sub, mask=mask_sigclip, sigma_lower=sigrej, sigma_upper=sigrej, cenfunc='median', stdfunc=utils.nan_mad_std) # be very liberal in determining threshold for continuum determination @@ -808,7 +809,7 @@ def iter_continuum(spec, gpm=None, fwhm=4.0, sigthresh = 2.0, sigrej=3.0, niter_ #cont_mask = np.ones_like(cont_mask) & gpm # Short circuit the masking and just mask the 0.70 most offending pixels peak_mask_ind = np.where(np.logical_not(peak_mask) & gpm)[0] - isort = np.argsort(np.abs(spec[peak_mask_ind]))[::-1] + isort = np.argsort(np.abs(spec[peak_mask_ind]), kind='stable')[::-1] peak_mask_new = np.ones_like(peak_mask) peak_mask_new[peak_mask_ind[isort[0:max_nmask]]] = False cont_mask = peak_mask_new & gpm diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index 364198cd67..db5157fee2 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -688,12 +688,12 @@ def interp_spec(wave_new, waves, fluxes, ivars, gpms, log10_blaze_function=None, for ii in range(nexp): fluxes_inter[:,ii], ivars_inter[:,ii], gpms_inter[:,ii], log10_blazes_inter[:,ii] \ = interp_oned(wave_new, waves[:,ii], fluxes[:,ii], ivars[:,ii], gpms[:,ii], - log10_blaze_function = log10_blaze_function[:, ii], sensfunc=sensfunc, kind=kind) + log10_blaze_function=log10_blaze_function[:, ii], sensfunc=sensfunc, kind=kind) else: for ii in range(nexp): fluxes_inter[:,ii], ivars_inter[:,ii], gpms_inter[:,ii], _ \ = interp_oned(wave_new, waves[:,ii], fluxes[:,ii], ivars[:,ii], gpms[:,ii], sensfunc=sensfunc, kind=kind) - log10_blazes_inter=None + log10_blazes_inter = None return fluxes_inter, ivars_inter, gpms_inter, log10_blazes_inter @@ -882,7 +882,7 @@ def sn_weights(fluxes, ivars, gpms, sn_smooth_npix=None, weight_method='auto', v # Check if relative weights input if verbose: - msgs.info('Computing weights with weight_method=%s'.format(weight_method)) + msgs.info('Computing weights with weight_method={:s}'.format(weight_method)) weights = [] @@ -1859,21 +1859,16 @@ def spec_reject_comb(wave_grid, wave_grid_mid, waves_list, fluxes_list, ivars_li # Compute the stack wave_stack, flux_stack, ivar_stack, gpm_stack, nused = compute_stack( wave_grid, waves_list, fluxes_list, ivars_list, utils.array_to_explist(this_gpms, nspec_list=nspec_list), weights_list) - # Interpolate the individual spectra onto the wavelength grid of the stack. Use wave_grid_mid for this - # since it has no masked values + # Interpolate the stack onto the wavelength grids of the individual spectra. This will be used to perform + # the rejection of pixels in the individual spectra. flux_stack_nat, ivar_stack_nat, gpm_stack_nat, _ = interp_spec( waves, wave_grid_mid, flux_stack, ivar_stack, gpm_stack) - ## TESTING - #nused_stack_nat, _, _ = interp_spec( - # waves, wave_grid_mid, nused, ivar_stack, mask_stack) - #embed() - rejivars, sigma_corrs, outchi, chigpm = update_errors(fluxes, ivars, this_gpms, - flux_stack_nat, ivar_stack_nat, gpm_stack_nat, sn_clip=sn_clip) - this_gpms, qdone = pydl.djs_reject(fluxes, flux_stack_nat, outmask=this_gpms,inmask=gpms, invvar=rejivars, + rejivars, sigma_corrs, outchi, chigpm = update_errors( + fluxes, ivars, this_gpms, flux_stack_nat, ivar_stack_nat, gpm_stack_nat, sn_clip=sn_clip) + this_gpms, qdone = pydl.djs_reject(fluxes, flux_stack_nat, outmask=this_gpms, inmask=gpms, invvar=rejivars, lower=lower,upper=upper, maxrej=maxrej, sticky=False) iter += 1 - if (iter == maxiter_reject) & (maxiter_reject != 0): msgs.warn('Maximum number of iterations maxiter={:}'.format(maxiter_reject) + ' reached in spec_reject_comb') out_gpms = np.copy(this_gpms) @@ -2406,7 +2401,7 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se nbests=None, which will just use one fourth of the orders for a given setup. wave_method : str, optional - method for generating new wavelength grid with get_wave_grid. Deafult is + method for generating new wavelength grid with get_wave_grid. Default is 'log10' which creates a uniformly space grid in log10(lambda), which is typically the best for echelle spectrographs dwave : float, optional @@ -2562,6 +2557,8 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se # data shape nsetups=len(waves_arr_setup) + msgs.info(f'Number of setups to cycle through is: {nsetups}') + if setup_ids is None: setup_ids = list(string.ascii_uppercase[:nsetups]) @@ -2610,7 +2607,7 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se wave_grid_min=wave_grid_min, wave_grid_max=wave_grid_max, dwave=dwave, dv=dv, dloglam=dloglam, spec_samp_fact=spec_samp_fact) - + msgs.info(f'The shape of the giant wave grid here is: {np.shape(wave_grid)}') # Evaluate the sn_weights. This is done once at the beginning weights = [] rms_sn_setup_list = [] @@ -2738,14 +2735,28 @@ def ech_combspec(waves_arr_setup, fluxes_arr_setup, ivars_arr_setup, gpms_arr_se for iord in range(norders[isetup]): ind_start = iord*nexps[isetup] ind_end = (iord+1)*nexps[isetup] + wave_grid_ord = waves_setup_list[isetup][ind_start] + # if the wavelength grid is non-monotonic, resample onto a loglam grid + wave_grid_diff_ord = np.diff(wave_grid_ord) + if np.any(wave_grid_diff_ord < 0): + msgs.warn(f'This order ({iord}) has a non-monotonic wavelength solution. Resampling now: ') + wave_grid_ord = np.linspace(np.min(wave_grid_ord), np.max(wave_grid_ord), len(wave_grid_ord)) + wave_grid_diff_ord = np.diff(wave_grid_ord) + + wave_grid_diff_ord = np.append(wave_grid_diff_ord, wave_grid_diff_ord[-1]) + wave_grid_mid_ord = wave_grid_ord + wave_grid_diff_ord / 2.0 + # removing the last bin since the midpoint now falls outside of wave_grid rightmost bin. This matches + # the convention in wavegrid above + wave_grid_mid_ord = wave_grid_mid_ord[:-1] + #trim off first and last pixel in case of edge effects in wavelength calibration wave_order_stack_iord, flux_order_stack_iord, ivar_order_stack_iord, gpm_order_stack_iord, \ nused_order_stack_iord, outgpms_order_stack_iord = spec_reject_comb( - wave_grid, wave_grid_mid, waves_setup_list[isetup][ind_start:ind_end], + wave_grid_ord, wave_grid_mid_ord, waves_setup_list[isetup][ind_start:ind_end], fluxes_scale_setup_list[isetup][ind_start:ind_end], ivars_scale_setup_list[isetup][ind_start:ind_end], gpms_setup_list[isetup][ind_start:ind_end], weights_setup_list[isetup][ind_start:ind_end], sn_clip=sn_clip, lower=lower, upper=upper, maxrej=maxrej, maxiter_reject=maxiter_reject, debug=debug_order_stack, title='order_stacks') - waves_order_stack.append(wave_order_stack_iord) + waves_order_stack.append(wave_grid_mid_ord) fluxes_order_stack.append(flux_order_stack_iord) ivars_order_stack.append(ivar_order_stack_iord) gpms_order_stack.append(gpm_order_stack_iord) diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 57596b6bed..3b5ff61923 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -14,9 +14,8 @@ from scipy.interpolate import interp1d import numpy as np -from pypeit import msgs -from pypeit import utils -from pypeit.core import coadd, flux_calib +from pypeit import msgs, utils, specobj, specobjs +from pypeit.core import coadd, extract, flux_calib # Use a fast histogram for speed! from fast_histogram import histogramdd @@ -35,13 +34,13 @@ def gaussian2D(tup, intflux, xo, yo, sigma_x, sigma_y, theta, offset): intflux (float): The Integrated flux of the 2D Gaussian xo (float): - The centre of the Gaussian along the x-coordinate when z=0 + The centre of the Gaussian along the x-coordinate when z=0 (units of pixels) yo (float): - The centre of the Gaussian along the y-coordinate when z=0 + The centre of the Gaussian along the y-coordinate when z=0 (units of pixels) sigma_x (float): - The standard deviation in the x-direction + The standard deviation in the x-direction (units of pixels) sigma_y (float): - The standard deviation in the y-direction + The standard deviation in the y-direction (units of pixels) theta (float): The orientation angle of the 2D Gaussian offset (float): @@ -97,16 +96,19 @@ def fitGaussian2D(image, norm=False): x = np.linspace(0, image.shape[0] - 1, image.shape[0]) y = np.linspace(0, image.shape[1] - 1, image.shape[1]) xx, yy = np.meshgrid(x, y, indexing='ij') - # Setup the fitting params - idx_max = [image.shape[0]/2, image.shape[1]/2] # Just use the centre of the image as the best guess - #idx_max = np.unravel_index(np.argmax(image), image.shape) + # Setup the fitting params - Estimate a starting point for the fit using a median filter + med_filt_image = signal.medfilt2d(image, kernel_size=3) + idx_max = np.unravel_index(np.argmax(med_filt_image), image.shape) initial_guess = (1, idx_max[0], idx_max[1], 2, 2, 0, 0) bounds = ([0, 0, 0, 0.5, 0.5, -np.pi, -np.inf], [np.inf, image.shape[0], image.shape[1], image.shape[0], image.shape[1], np.pi, np.inf]) # Perform the fit + # TODO :: May want to generate the image on a finer pixel scale first popt, pcov = opt.curve_fit(gaussian2D, (xx, yy), image.ravel() / wlscl, bounds=bounds, p0=initial_guess) + # Generate a best fit model + model = gaussian2D((xx, yy), *popt).reshape(image.shape) * wlscl # Return the fitting results - return popt, pcov + return popt, pcov, model def dar_fitfunc(radec, coord_ra, coord_dec, datfit, wave, obstime, location, pressure, @@ -187,76 +189,119 @@ def correct_grating_shift(wave_eval, wave_curr, spl_curr, wave_ref, spl_ref, ord return grat_corr -def extract_standard_spec(stdcube, subpixel=20): +def extract_point_source(wave, flxcube, ivarcube, bpmcube, wcscube, exptime, + subpixel=20, boxcar_radius=None, optfwhm=None, whitelight_range=None, + pypeline="SlicerIFU", fluxed=False): """ Extract a spectrum of a standard star from a datacube Parameters ---------- - std_cube : `astropy.io.fits.HDUList`_ - An HDU list of fits files - subpixel : int + wave : `numpy.ndarray`_ + Wavelength array for the datacube + flxcube : `numpy.ndarray`_ + Datacube of the flux + ivarcube : `numpy.ndarray`_ + Datacube of the inverse variance + bpmcube : `numpy.ndarray`_ + Datacube of the bad pixel mask + wcscube : `astropy.wcs.WCS`_ + WCS of the datacube + exptime : float + Exposure time listed in the header of the datacube + subpixel : int, optional Number of pixels to subpixelate spectrum when creating mask + boxcar_radius : float, optional + Radius of the circular boxcar (in arcseconds) to use for the extraction + optfwhm : float, optional + FWHM of the PSF in pixels that is used to generate a Gaussian profile + for the optimal extraction. + pypeline : str, optional + PypeIt pipeline used to reduce the datacube + fluxed : bool, optional + Is the datacube fluxed? Returns ------- - wave : `numpy.ndarray`_ - Wavelength of the star. - Nlam_star : `numpy.ndarray`_ - counts/second/Angstrom - Nlam_ivar_star : `numpy.ndarray`_ - inverse variance of Nlam_star - gpm_star : `numpy.ndarray`_ - good pixel mask for Nlam_star + sobjs : :class:`~pypeit.specobjs.SpecObjs` + SpecObjs object containing the extracted spectrum """ - # Extract some information from the HDU list - flxcube = stdcube['FLUX'].data.T.copy() - varcube = stdcube['SIG'].data.T.copy()**2 - bpmcube = stdcube['BPM'].data.T.copy() - numwave = flxcube.shape[2] + if whitelight_range is None: + whitelight_range = [np.min(wave), np.max(wave)] + + # Generate a spec1d object to hold the extracted spectrum + msgs.info("Initialising a PypeIt SpecObj spec1d file") + sobj = specobj.SpecObj(pypeline, "DET01", SLITID=0) + sobj.RA = wcscube.wcs.crval[0] + sobj.DEC = wcscube.wcs.crval[1] + sobj.SLITID = 0 + + # Convert from counts/s/Ang/arcsec**2 to counts. The sensitivity function expects counts as input + numxx, numyy, numwave = flxcube.shape + arcsecSQ = (wcscube.wcs.cdelt[0] * wcscube.wcs.cunit[0].to(units.arcsec)) * \ + (wcscube.wcs.cdelt[1] * wcscube.wcs.cunit[1].to(units.arcsec)) + if fluxed: + # The datacube is flux calibrated, in units of 10^-17 erg/s/cm**2/Ang/arcsec**2 + # Scale the flux and ivar cubes to be in units of erg/s/cm**2/Ang + unitscale = arcsecSQ + else: + # Scale the flux and ivar cubes to be in units of counts. pypeit_sensfunc expects counts as input + deltawave = wcscube.wcs.cdelt[2]*wcscube.wcs.cunit[2].to(units.Angstrom) + unitscale = exptime * deltawave * arcsecSQ - # Setup the WCS - stdwcs = wcs.WCS(stdcube['FLUX'].header) + # Apply the relevant scaling + _flxcube = flxcube * unitscale + _ivarcube = ivarcube / unitscale**2 - wcs_scale = (1.0 * stdwcs.spectral.wcs.cunit[0]).to(units.Angstrom).value # Ensures the WCS is in Angstroms - wave = wcs_scale * stdwcs.spectral.wcs_pix2world(np.arange(numwave), 0)[0] + # Calculate the variance cube + _varcube = utils.inverse(_ivarcube) # Generate a whitelight image, and fit a 2D Gaussian to estimate centroid and width - wl_img = make_whitelight_fromcube(flxcube) - popt, pcov = fitGaussian2D(wl_img, norm=True) - wid = max(popt[3], popt[4]) + msgs.info("Making white light image") + wl_img = make_whitelight_fromcube(_flxcube, bpmcube, wave=wave, wavemin=whitelight_range[0], wavemax=whitelight_range[1]) + popt, pcov, model = fitGaussian2D(wl_img, norm=True) + if boxcar_radius is None: + nsig = 4 # 4 sigma should be far enough... Note: percentage enclosed for 2D Gaussian = 1-np.exp(-0.5 * nsig**2) + wid = nsig * max(popt[3], popt[4]) + else: + # Set the user-defined radius + wid = boxcar_radius / np.sqrt(arcsecSQ) + # Set the radius of the extraction boxcar for the sky determination + msgs.info("Using a boxcar radius of {:0.2f} arcsec".format(wid*np.sqrt(arcsecSQ))) + widsky = 2 * wid # Setup the coordinates of the mask - x = np.linspace(0, flxcube.shape[0] - 1, flxcube.shape[0] * subpixel) - y = np.linspace(0, flxcube.shape[1] - 1, flxcube.shape[1] * subpixel) + x = np.linspace(0, numxx - 1, numxx * subpixel) + y = np.linspace(0, numyy - 1, numyy * subpixel) xx, yy = np.meshgrid(x, y, indexing='ij') # Generate a mask - newshape = (flxcube.shape[0] * subpixel, flxcube.shape[1] * subpixel) + msgs.info("Generating an object mask") + newshape = (numxx * subpixel, numyy * subpixel) mask = np.zeros(newshape) - nsig = 4 # 4 sigma should be far enough... Note: percentage enclosed for 2D Gaussian = 1-np.exp(-0.5 * nsig**2) - ww = np.where((np.sqrt((xx - popt[1]) ** 2 + (yy - popt[2]) ** 2) < nsig * wid)) + ww = np.where((np.sqrt((xx - popt[1]) ** 2 + (yy - popt[2]) ** 2) < wid)) mask[ww] = 1 - mask = utils.rebinND(mask, (flxcube.shape[0], flxcube.shape[1])).reshape(flxcube.shape[0], flxcube.shape[1], 1) + mask = utils.rebinND(mask, (numxx, numyy)).reshape(numxx, numyy, 1) # Generate a sky mask - newshape = (flxcube.shape[0] * subpixel, flxcube.shape[1] * subpixel) + msgs.info("Generating a sky mask") + newshape = (numxx * subpixel, numyy * subpixel) smask = np.zeros(newshape) - nsig = 8 # 8 sigma should be far enough - ww = np.where((np.sqrt((xx - popt[1]) ** 2 + (yy - popt[2]) ** 2) < nsig * wid)) + ww = np.where((np.sqrt((xx - popt[1]) ** 2 + (yy - popt[2]) ** 2) < widsky)) smask[ww] = 1 - smask = utils.rebinND(smask, (flxcube.shape[0], flxcube.shape[1])).reshape(flxcube.shape[0], flxcube.shape[1], 1) + smask = utils.rebinND(smask, (numxx, numyy)).reshape(numxx, numyy, 1) + # Subtract off the object mask region, so that we just have an annulus around the object smask -= mask - # Subtract the residual sky + msgs.info("Subtracting the residual sky") + # Subtract the residual sky from the datacube skymask = np.logical_not(bpmcube) * smask - skycube = flxcube * skymask - skyspec = skycube.sum(0).sum(0) - nrmsky = skymask.sum(0).sum(0) + skycube = _flxcube * skymask + skyspec = skycube.sum(axis=(0,1)) + nrmsky = skymask.sum(axis=(0,1)) skyspec *= utils.inverse(nrmsky) - flxcube -= skyspec.reshape((1, 1, numwave)) - - # Subtract the residual sky from the whitelight image + _flxcube -= skyspec.reshape((1, 1, numwave)) + # Now subtract the residual sky from the white light image sky_val = np.sum(wl_img[:, :, np.newaxis] * smask) / np.sum(smask) wl_img -= sky_val @@ -267,88 +312,102 @@ def extract_standard_spec(stdcube, subpixel=20): norm_flux /= np.sum(norm_flux) # Extract boxcar cntmask = np.logical_not(bpmcube) * mask # Good pixels within the masked region around the standard star - flxscl = (norm_flux * cntmask).sum(0).sum(0) # This accounts for the flux that is missing due to masked pixels - scimask = flxcube * cntmask - varmask = varcube * cntmask**2 + flxscl = (norm_flux * cntmask).sum(axis=(0,1)) # This accounts for the flux that is missing due to masked pixels + scimask = _flxcube * cntmask + varmask = _varcube * cntmask**2 nrmcnt = utils.inverse(flxscl) - box_flux = scimask.sum(0).sum(0) * nrmcnt - box_var = varmask.sum(0).sum(0) * nrmcnt**2 + box_flux = scimask.sum(axis=(0,1)) * nrmcnt + box_var = varmask.sum(axis=(0,1)) * nrmcnt**2 box_gpm = flxscl > 1/3 # Good pixels are those where at least one-third of the standard star flux is measured - # Setup the return values - ret_flux, ret_var, ret_gpm = box_flux, box_var, box_gpm - - # Convert from counts/s/Ang/arcsec**2 to counts/s/Ang - arcsecSQ = 3600.0*3600.0*(stdwcs.wcs.cdelt[0]*stdwcs.wcs.cdelt[1]) - ret_flux *= arcsecSQ - ret_var *= arcsecSQ**2 - # Return the box extraction results - return wave, ret_flux, utils.inverse(ret_var), ret_gpm - -def make_sensfunc(ss_file, senspar, blaze_wave=None, blaze_spline=None, grating_corr=False): - """ - Generate the sensitivity function from a standard star DataCube. - - Args: - ss_file (:obj:`str`): - The relative path and filename of the standard star datacube. It - should be fits format, and for full functionality, should ideally of - the form :class:`~pypeit.coadd3d.DataCube`. - senspar (:class:`~pypeit.par.pypeitpar.SensFuncPar`): - The parameters required for the sensitivity function computation. - blaze_wave (`numpy.ndarray`_, optional): - Wavelength array used to construct blaze_spline - blaze_spline (`scipy.interpolate.interp1d`_, optional): - Spline representation of the reference blaze function (based on the illumflat). - grating_corr (:obj:`bool`, optional): - If a grating correction should be performed, set this variable to True. - - Returns: - `numpy.ndarray`_: A mask of the good sky pixels (True = good) - """ - # Check if the standard star datacube exists - if not os.path.exists(ss_file): - msgs.error("Standard cube does not exist:" + msgs.newline() + ss_file) - msgs.info(f"Loading standard star cube: {ss_file:s}") - # Load the standard star cube and retrieve its RA + DEC - stdcube = fits.open(ss_file) - star_ra, star_dec = stdcube[1].header['CRVAL1'], stdcube[1].header['CRVAL2'] - - # Extract a spectrum of the standard star - wave, Nlam_star, Nlam_ivar_star, gpm_star = extract_standard_spec(stdcube) - - # Extract the information about the blaze - if grating_corr: - blaze_wave_curr, blaze_spec_curr = stdcube['BLAZE_WAVE'].data, stdcube['BLAZE_SPEC'].data - blaze_spline_curr = interp1d(blaze_wave_curr, blaze_spec_curr, - kind='linear', bounds_error=False, fill_value="extrapolate") - # Perform a grating correction - grat_corr = correct_grating_shift(wave, blaze_wave_curr, blaze_spline_curr, blaze_wave, blaze_spline) - # Apply the grating correction to the standard star spectrum - Nlam_star /= grat_corr - Nlam_ivar_star *= grat_corr ** 2 - - # Read in some information above the standard star - std_dict = flux_calib.get_standard_spectrum(star_type=senspar['star_type'], - star_mag=senspar['star_mag'], - ra=star_ra, dec=star_dec) - # Calculate the sensitivity curve - # TODO :: This needs to be addressed... unify flux calibration into the main PypeIt routines. - msgs.warn("Datacubes are currently flux-calibrated using the UVIS algorithm... this will be deprecated soon") - zeropoint_data, zeropoint_data_gpm, zeropoint_fit, zeropoint_fit_gpm = \ - flux_calib.fit_zeropoint(wave, Nlam_star, Nlam_ivar_star, gpm_star, std_dict, - mask_hydrogen_lines=senspar['mask_hydrogen_lines'], - mask_helium_lines=senspar['mask_helium_lines'], - hydrogen_mask_wid=senspar['hydrogen_mask_wid'], - nresln=senspar['UVIS']['nresln'], - resolution=senspar['UVIS']['resolution'], - trans_thresh=senspar['UVIS']['trans_thresh'], - polyorder=senspar['polyorder'], - polycorrect=senspar['UVIS']['polycorrect'], - polyfunc=senspar['UVIS']['polyfunc']) - wgd = np.where(zeropoint_fit_gpm) - sens = np.power(10.0, -0.4 * (zeropoint_fit[wgd] - flux_calib.ZP_UNIT_CONST)) / np.square(wave[wgd]) - return interp1d(wave[wgd], sens, kind='linear', bounds_error=False, fill_value="extrapolate") + # Store the BOXCAR extraction information + sobj.BOX_RADIUS = wid # Size of boxcar radius in pixels + sobj.BOX_WAVE = wave.astype(float) + if fluxed: + sobj.BOX_FLAM = box_flux + sobj.BOX_FLAM_SIG = np.sqrt(box_var) + sobj.BOX_FLAM_IVAR = utils.inverse(box_var) + else: + sobj.BOX_COUNTS = box_flux + sobj.BOX_COUNTS_SIG = np.sqrt(box_var) + sobj.BOX_COUNTS_IVAR = utils.inverse(box_var) + sobj.BOX_COUNTS_SKY = skyspec # This is not the real sky, it is the residual sky. The datacube is presumed to be sky subtracted + sobj.BOX_MASK = box_gpm + sobj.S2N = np.median(box_flux * np.sqrt(utils.inverse(box_var))) + + # Now do the OPTIMAL extraction + msgs.info("Extracting an optimal spectrum of datacube") + # First, we need to rearrange the datacube and inverse variance cube into a 2D array. + # The 3D -> 2D conversion is done so that there is a spectral and spatial dimension, + # and the brightest white light pixel is transformed to be at the centre column of the 2D + # array. Then, the second brightest white light pixel is transformed to be next to the centre + # column of the 2D array, and so on. This is done so that the optimal extraction algorithm + # can be applied. + optkern = wl_img + if optfwhm is not None: + msgs.info("Generating a 2D Gaussian kernel for the optimal extraction, with FWHM = {:.2f} pixels".format(optfwhm)) + x = np.linspace(0, wl_img.shape[0] - 1, wl_img.shape[0]) + y = np.linspace(0, wl_img.shape[1] - 1, wl_img.shape[1]) + xx, yy = np.meshgrid(x, y, indexing='ij') + # Generate a Gaussian kernel + intflux = 1 + xo, yo = popt[1], popt[2] + fwhm2sigma = 1.0 / (2 * np.sqrt(2 * np.log(2))) + sigma_x, sigma_y = optfwhm*fwhm2sigma, optfwhm*fwhm2sigma + theta, offset, = 0.0, 0.0 + optkern = gaussian2D((xx, yy), intflux, xo, yo, sigma_x, sigma_y, theta, offset).reshape(wl_img.shape) + # Normalise the kernel + optkern /= np.sum(optkern) + + optkern_masked = optkern * mask[:,:,0] + # Normalise the white light image + optkern_masked /= np.sum(optkern_masked) + asrt = np.argsort(optkern_masked, axis=None) + # Need to ensure that the number of pixels in the object profile is even + if asrt.size % 2 != 0: + # Remove the pixel with the lowest kernel weight. + # It should be a zero value (given the mask), so it doesn't matter if we remove it + asrt = asrt[1:] + # Now sort the indices of the pixels in the object profile + tmp = asrt.reshape((asrt.size//2, 2)) + objprof_idx = np.append(tmp[:,0], tmp[::-1,1]) + objprof = optkern_masked[np.unravel_index(objprof_idx, optkern.shape)] + + # Now slice the datacube and inverse variance cube into a 2D array + spat, spec = np.meshgrid(objprof_idx, np.arange(numwave), indexing='ij') + spatspl = np.apply_along_axis(np.unravel_index, 1, spat, optkern.shape) + # Now slice the datacube and corresponding cubes/vectors into a series of 2D arrays + numspat = objprof_idx.size + flxslice = (spatspl[:,0,:], spatspl[:,1,:], spec) + flxcube2d = _flxcube[flxslice].T + ivarcube2d = _ivarcube[flxslice].T + gpmcube2d = np.logical_not(bpmcube[flxslice].T) + waveimg = wave.reshape((numwave,1)).repeat(numspat, axis=1) + skyimg = np.zeros((numwave, numspat)) # Note, the residual sky has already been subtracted off _flxcube + oprof = objprof.reshape((1,numspat)).repeat(numwave, axis=0) + thismask = np.ones_like(flxcube2d, dtype=bool) + + # Now do the optimal extraction + extract.extract_optimal(flxcube2d, ivarcube2d, gpmcube2d, waveimg, skyimg, thismask, oprof, + sobj, min_frac_use=0.05, fwhmimg=None, base_var=None, count_scale=None, noise_floor=None) + + # TODO :: The optimal extraction may suffer from residual DAR correction issues. This is because the + # :: object profile assumes that the white light image represents the true spatial profile of the + # :: object. One possibility is to fit a (linear?) model to the ratio of box/optimal extraction + # :: and then apply this model to the optimal extraction. This is a bit of a fudge. + # Note that extract.extract_optimal() stores the optimal extraction in the + # sobj.OPT_COUNTS, sobj.OPT_COUNTS_SIG, and sobj.OPT_COUNTS_IVAR attributes. + # We need to store the fluxed extraction into the FLAM attributes (a slight fudge). + if fluxed: + sobj.OPT_FLAM = sobj.OPT_COUNTS + sobj.OPT_FLAM_SIG = sobj.OPT_COUNTS_SIG + sobj.OPT_FLAM_IVAR = sobj.OPT_COUNTS_IVAR + + # Make a specobjs object + sobjs = specobjs.SpecObjs() + sobjs.add_sobj(sobj) + # Return the specobj object + return sobjs def make_good_skymask(slitimg, tilts): @@ -537,13 +596,16 @@ def get_whitelight_range(wavemin, wavemax, wl_range): return wlrng -def make_whitelight_fromcube(cube, wave=None, wavemin=None, wavemax=None): +def make_whitelight_fromcube(cube, bpmcube, wave=None, wavemin=None, wavemax=None): """ Generate a white light image using an input cube. Args: cube (`numpy.ndarray`_): 3D datacube (the final element contains the wavelength dimension) + bpmcube (`numpy.ndarray`_): + 3D bad pixel mask cube (the final element contains the wavelength dimension). + A value of 1 indicates a bad pixel. wave (`numpy.ndarray`_, optional): 1D wavelength array. Only required if wavemin or wavemax are not None. @@ -560,7 +622,6 @@ def make_whitelight_fromcube(cube, wave=None, wavemin=None, wavemax=None): A whitelight image of the input cube (of type `numpy.ndarray`_). """ # Make a wavelength cut, if requested - cutcube = cube.copy() if wavemin is not None or wavemax is not None: # Make some checks on the input if wave is None: @@ -576,10 +637,15 @@ def make_whitelight_fromcube(cube, wave=None, wavemin=None, wavemax=None): ww = np.where((wave >= wavemin) & (wave <= wavemax))[0] wmin, wmax = ww[0], ww[-1]+1 cutcube = cube[:, :, wmin:wmax] + # Cut the bad pixel mask and convert it to a good pixel mask + cutgpmcube = np.logical_not(bpmcube[:, :, wmin:wmax]) + else: + cutcube = cube.copy() + cutgpmcube = np.logical_not(bpmcube) # Now sum along the wavelength axis - nrmval = np.sum(cutcube != 0.0, axis=2) - nrmval[nrmval == 0.0] = 1.0 - wl_img = np.sum(cutcube, axis=2) / nrmval + nrmval = np.sum(cutgpmcube, axis=2) + nrmval[nrmval == 0] = 1.0 + wl_img = np.sum(cutcube*cutgpmcube, axis=2) / nrmval return wl_img @@ -1117,7 +1183,7 @@ def compute_weights_frompix(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, # Compute the weights return compute_weights(raImg, decImg, waveImg, sciImg, ivarImg, slitidImg, all_wcs, all_tilts, all_slits, all_align, all_dar, ra_offsets, dec_offsets, - wl_full[:, :, 0], dspat, dwv, + wl_full, dspat, dwv, ra_min=ra_min, ra_max=ra_max, dec_min=dec_min, dec_max=dec_max, wave_min=wave_min, sn_smooth_npix=sn_smooth_npix, weight_method=weight_method, correct_dar=correct_dar) @@ -1367,36 +1433,39 @@ def generate_image_subpixel(image_wcs, bins, sciImg, ivarImg, waveImg, slitid_im correction will not be applied. Returns: - `numpy.ndarray`_: The white light images for all frames + `numpy.ndarray`_: The white light images for all frames. If combine=True, + this will be a single 2D image. Otherwise, it will be a 3D array with + dimensions (numra, numdec, numframes). """ # Perform some checks on the input -- note, more complete checks are performed in subpixellate() _sciImg, _ivarImg, _waveImg, _slitid_img_gpm, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ check_inputs([sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, all_wcs, tilts, slits, astrom_trans, all_dar, ra_offset, dec_offset]) - numframes = len(_sciImg) - - # Prepare the array of white light images to be stored - numra = bins[0].size-1 - numdec = bins[1].size-1 - all_wl_imgs = np.zeros((numra, numdec, numframes)) - # Loop through all frames and generate white light images - for fr in range(numframes): - msgs.info(f"Creating image {fr+1}/{numframes}") - if combine: - # Subpixellate - img, _, _ = subpixellate(image_wcs, bins, _sciImg, _ivarImg, _waveImg, _slitid_img_gpm, _wghtImg, - _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset, - spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel, - skip_subpix_weights=True, correct_dar=correct_dar) - else: + # Generate the white light images + if combine: + # Subpixellate + img, _, _ = subpixellate(image_wcs, bins, _sciImg, _ivarImg, _waveImg, _slitid_img_gpm, _wghtImg, + _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset, + spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel, + skip_subpix_weights=True, correct_dar=correct_dar) + return img[:, :, 0] + else: + # Prepare the array of white light images to be stored + numframes = len(_sciImg) + numra = bins[0].size - 1 + numdec = bins[1].size - 1 + all_wl_imgs = np.zeros((numra, numdec, numframes)) + # Loop through all frames and generate white light images + for fr in range(numframes): + msgs.info(f"Creating image {fr + 1}/{numframes}") # Subpixellate img, _, _ = subpixellate(image_wcs, bins, _sciImg[fr], _ivarImg[fr], _waveImg[fr], _slitid_img_gpm[fr], _wghtImg[fr], _all_wcs[fr], _tilts[fr], _slits[fr], _astrom_trans[fr], _all_dar[fr], _ra_offset[fr], _dec_offset[fr], spec_subpixel=spec_subpixel, spat_subpixel=spat_subpixel, slice_subpixel=slice_subpixel, skip_subpix_weights=True, correct_dar=correct_dar) - all_wl_imgs[:, :, fr] = img[:, :, 0] - # Return the constructed white light images - return all_wl_imgs + all_wl_imgs[:, :, fr] = img[:, :, 0] + # Return the constructed white light images + return all_wl_imgs def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wghtImg, @@ -1532,7 +1601,7 @@ def generate_cube_subpixel(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_im whitelight_range[0], whitelight_range[1])) # Get the output filename for the white light image out_whitelight = get_output_whitelight_filename(outfile) - whitelight_img = make_whitelight_fromcube(flxcube, wave=wave, wavemin=whitelight_range[0], wavemax=whitelight_range[1]) + whitelight_img = make_whitelight_fromcube(flxcube, bpmcube, wave=wave, wavemin=whitelight_range[0], wavemax=whitelight_range[1]) msgs.info("Saving white light image as: {0:s}".format(out_whitelight)) img_hdu = fits.PrimaryHDU(whitelight_img.T, header=whitelight_wcs.to_header()) img_hdu.writeto(out_whitelight, overwrite=overwrite) @@ -1636,8 +1705,7 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh Returns: :obj:`tuple`: Three or four `numpy.ndarray`_ objects containing (1) the datacube generated from the subpixellated inputs, (2) the corresponding - variance cube, (3) the corresponding bad pixel mask cube, and (4) the - residual cube. The latter is only returned if debug is True. + variance cube, and (3) the corresponding bad pixel mask cube. """ # Check the inputs for combinations of lists or not _sciImg, _ivarImg, _waveImg, _gpmImg, _wghtImg, _all_wcs, _tilts, _slits, _astrom_trans, _all_dar, _ra_offset, _dec_offset = \ @@ -1686,7 +1754,7 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh yspl = this_tilts[wpix] * (this_slits.nspec - 1) tiltpos = np.add.outer(yspl, spec_y).flatten() wspl = this_wav[this_sl] - asrt = np.argsort(yspl) + asrt = np.argsort(yspl, kind='stable') wave_spl = interp1d(yspl[asrt], wspl[asrt], kind='linear', bounds_error=False, fill_value='extrapolate') # Calculate the wavelength at each subpixel this_wave_subpix = wave_spl(tiltpos) @@ -1701,7 +1769,7 @@ def subpixellate(output_wcs, bins, sciImg, ivarImg, waveImg, slitid_img_gpm, wgh # Transform this to spatial location spatpos_subpix = _astrom_trans[fr].transform(sl, spat_xx, spec_yy) spatpos = _astrom_trans[fr].transform(sl, wpix[1], wpix[0]) - ssrt = np.argsort(spatpos) + ssrt = np.argsort(spatpos, kind='stable') # Initialize the voxel coordinates for each spec2D pixel vox_coord = np.full((numpix, num_all_subpixels, 3), -1, dtype=float) # Loop over the subslices diff --git a/pypeit/core/extract.py b/pypeit/core/extract.py index 246a6f0c9c..83b2e1ad48 100644 --- a/pypeit/core/extract.py +++ b/pypeit/core/extract.py @@ -24,7 +24,8 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, - spec, min_frac_use=0.05, fwhmimg=None, base_var=None, count_scale=None, noise_floor=None): + spec, min_frac_use=0.05, fwhmimg=None, flatimg=None, + base_var=None, count_scale=None, noise_floor=None): r""" Perform optimal extraction `(Horne 1986) @@ -42,6 +43,7 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, - spec.OPT_COUNTS_NIVAR --> Optimally extracted noise variance (sky + read noise) only - spec.OPT_MASK --> Mask for optimally extracted flux - spec.OPT_FWHM --> Spectral FWHM (in A) for optimally extracted flux + - spec.OPT_FLAT --> Flat field spectrum, normalised at the peak value, for the optimally extracted flux - spec.OPT_COUNTS_SKY --> Optimally extracted sky - spec.OPT_COUNTS_SIG_DET --> Square root of optimally extracted read noise squared - spec.OPT_FRAC_USE --> Fraction of pixels in the object profile subimage used for this extraction @@ -88,6 +90,10 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, fwhmimg : `numpy.ndarray`_, None, optional: Floating-point image containing the modeled spectral FWHM (in pixels) at every pixel location. Must have the same shape as ``sciimg``, :math:`(N_{\rm spec}, N_{\rm spat})`. + flatimg : `numpy.ndarray`_, None, optional: + Floating-point image containing the unnormalized flat-field image. This image + is used to extract the blaze function. Must have the same shape as ``sciimg``, + :math:`(N_{\rm spec}, N_{\rm spat})`. base_var : `numpy.ndarray`_, optional Floating-point "base-level" variance image set by the detector properties and the image processing steps. See :func:`~pypeit.core.procimg.base_variance`. @@ -145,6 +151,8 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, oprof_sub = oprof[:,mincol:maxcol] if fwhmimg is not None: fwhmimg_sub = fwhmimg[:,mincol:maxcol] + if flatimg is not None: + flatimg_sub = flatimg[:,mincol:maxcol] # enforce normalization and positivity of object profiles norm = np.nansum(oprof_sub,axis = 1) norm_oprof = np.outer(norm, np.ones(nsub)) @@ -190,6 +198,9 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, fwhm_opt = None if fwhmimg is not None: fwhm_opt = np.nansum(mask_sub*ivar_sub*fwhmimg_sub*oprof_sub, axis=1) * utils.inverse(tot_weight) + blaze_opt = None + if flatimg is not None: + blaze_opt = np.nansum(mask_sub*ivar_sub*flatimg_sub*oprof_sub, axis=1) * utils.inverse(tot_weight) # Interpolate wavelengths over masked pixels badwvs = (mivar_num <= 0) | np.invert(np.isfinite(wave_opt)) | (wave_opt <= 0.0) if badwvs.any(): @@ -221,6 +232,9 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, # Calculate the Angstroms/pixel and Spectral FWHM if fwhm_opt is not None: fwhm_opt *= np.gradient(wave_opt) # Convert pixel FWHM to Angstroms + # Normalize the blaze function + if blaze_opt is not None: + blaze_opt /= np.nanmax(blaze_opt) # Fill in the optimally extraction tags spec.OPT_WAVE = wave_opt # Optimally extracted wavelengths spec.OPT_COUNTS = flux_opt # Optimally extracted flux @@ -229,6 +243,8 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, spec.OPT_COUNTS_NIVAR = None if nivar_opt is None else nivar_opt*np.logical_not(badwvs) # Optimally extracted noise variance (sky + read noise) only spec.OPT_MASK = mask_opt*np.logical_not(badwvs) # Mask for optimally extracted flux spec.OPT_FWHM = fwhm_opt # Spectral FWHM (in Angstroms) for the optimally extracted spectrum + if blaze_opt is not None: + spec.OPT_FLAT = blaze_opt # Flat field spectrum, normalised to the peak value spec.OPT_COUNTS_SKY = sky_opt # Optimally extracted sky spec.OPT_COUNTS_SIG_DET = base_opt # Square root of optimally extracted read noise squared spec.OPT_FRAC_USE = frac_use # Fraction of pixels in the object profile subimage used for this extraction @@ -307,7 +323,7 @@ def extract_asym_boxcar(sciimg, left_trace, righ_trace, gpm=None, ivar=None): return flux_out, gpm_box, box_npix, ivar_out -def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, base_var=None, +def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, flatimg=None, base_var=None, count_scale=None, noise_floor=None): r""" Perform boxcar extraction for a single :class:`~pypeit.specobj.SpecObj`. @@ -325,6 +341,7 @@ def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, b - spec.BOX_COUNTS_NIVAR --> Box car extracted noise variance - spec.BOX_MASK --> Box car extracted mask - spec.BOX_FWHM --> Box car extracted spectral FWHM + - spec.BOX_FLAT --> Box car extracted flatfield spectrum function (normalized to peak value) - spec.BOX_COUNTS_SKY --> Box car extracted sky - spec.BOX_COUNTS_SIG_DET --> Box car extracted read noise - spec.BOX_NPIX --> Number of pixels used in boxcar sum @@ -354,9 +371,12 @@ def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, b Container that holds object, trace, and extraction information for the object in question. **This object is altered in place!** Note that this routine operates one object at a time. - fwhmimg : `numpy.ndarray`_, None, optional: + fwhmimg : `numpy.ndarray`_, None, optional Floating-point image containing the modeled spectral FWHM (in pixels) at every pixel location. Must have the same shape as ``sciimg``, :math:`(N_{\rm spec}, N_{\rm spat})`. + flatimg : `numpy.ndarray`_, None, optional + Floating-point image containing the normalized flat-field. Must have the same shape as + ``sciimg``, :math:`(N_{\rm spec}, N_{\rm spat})`. base_var : `numpy.ndarray`_, optional Floating-point "base-level" variance image set by the detector properties and the image processing steps. See :func:`~pypeit.core.procimg.base_variance`. @@ -365,7 +385,8 @@ def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, b A scale factor, :math:`s`, that *has already been applied* to the provided science image. It accounts for the number of frames contributing to the provided counts, and the relative throughput factors that can be measured - from flat-field frames. For example, if the image has been flat-field + from flat-field frames plus a scaling factor applied if the counts of each frame are + scaled to the mean counts of all frames. For example, if the image has been flat-field corrected, this is the inverse of the flat-field counts. If None, set to 1. If a single float, assumed to be constant across the full image. If an array, the shape must match ``base_var``. The variance will be 0 @@ -406,6 +427,9 @@ def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, b fwhm_box = None if fwhmimg is not None: fwhm_box = moment1d(fwhmimg*mask, spec.TRACE_SPAT, 2*box_radius, row=spec.trace_spec)[0] + blaze_box = None + if flatimg is not None: + blaze_box = moment1d(flatimg*mask, spec.TRACE_SPAT, 2*box_radius, row=spec.trace_spec)[0] varimg = 1.0/(ivar + (ivar == 0.0)) var_box = moment1d(varimg*mask, spec.TRACE_SPAT, 2*box_radius, row=spec.trace_spec)[0] nvar_box = None if var_no is None \ @@ -437,15 +461,21 @@ def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, b # Calculate the Angstroms/pixel and the final spectral FWHM value if fwhm_box is not None: ang_per_pix = np.gradient(wave_box) - fwhm_box *= ang_per_pix / (pixtot - pixmsk) # Need to divide by total number of unmasked pixels + fwhm_box *= ang_per_pix * utils.inverse(pixtot - pixmsk) # Need to divide by total number of unmasked pixels + # Normalize the blaze function + if blaze_box is not None: + blaze_box *= utils.inverse(pixtot - pixmsk) # Need to divide by total number of unmasked pixels + blaze_box *= utils.inverse(np.nanmax(blaze_box[mask_box])) # Now normalize to the peak value # Fill em up! spec.BOX_WAVE = wave_box spec.BOX_COUNTS = flux_box*mask_box spec.BOX_COUNTS_IVAR = ivar_box*mask_box*np.logical_not(bad_box) - spec.BOX_COUNTS_SIG = np.sqrt(utils.inverse( spec.BOX_COUNTS_IVAR)) + spec.BOX_COUNTS_SIG = np.sqrt(utils.inverse(spec.BOX_COUNTS_IVAR)) spec.BOX_COUNTS_NIVAR = None if nivar_box is None else nivar_box*mask_box*np.logical_not(bad_box) spec.BOX_MASK = mask_box*np.logical_not(bad_box) spec.BOX_FWHM = fwhm_box # Spectral FWHM (in Angstroms) for the boxcar extracted spectrum + if blaze_box is not None: + spec.BOX_FLAT = blaze_box # Flat field spectrum, normalised to the peak value spec.BOX_COUNTS_SKY = sky_box spec.BOX_COUNTS_SIG_DET = base_box # TODO - Confirm this should be float, not int @@ -644,7 +674,7 @@ def qa_fit_profile(x_tot, y_tot, model_tot, l_limit = None, closest = (dist[close]).argmin() model_samp[i] = (model[close])[closest] if nclose > 3: - s = yclose.argsort() + s = yclose.argsort(kind='stable') y50[i] = yclose[s[int(np.rint((nclose - 1)*0.5))]] y80[i] = yclose[s[int(np.rint((nclose - 1)*0.8))]] y20[i] = yclose[s[int(np.rint((nclose - 1)*0.2))]] @@ -656,7 +686,7 @@ def qa_fit_profile(x_tot, y_tot, model_tot, l_limit = None, else: ax.plot(plot_mid, y50, marker='o', color='lime', markersize=2, fillstyle = 'full', linestyle='None') - isort = x.argsort() + isort = x.argsort(kind='stable') ax.plot(x[isort], model[isort], color='red', linewidth=1.0) @@ -905,7 +935,7 @@ def fit_profile(image, ivar, waveimg, thismask, spat_img, trace_in, wave, spline_flux1[ispline] = spline_tmp cont_tmp, _ = c_answer.value(wave[ispline]) cont_flux1[ispline] = cont_tmp - isrt = np.argsort(wave[indsp]) + isrt = np.argsort(wave[indsp], kind='stable') s2_1_interp = scipy.interpolate.interp1d(wave[indsp][isrt], sn2[isrt],assume_sorted=False, bounds_error=False,fill_value = 0.0) sn2_1[ispline] = s2_1_interp(wave[ispline]) bmask = np.zeros(nspec,dtype='bool') @@ -952,7 +982,7 @@ def fit_profile(image, ivar, waveimg, thismask, spat_img, trace_in, wave, # Create the normalized object image if np.any(totmask): igd = (wave >= wave_min) & (wave <= wave_max) - isrt1 = np.argsort(wave[igd]) + isrt1 = np.argsort(wave[igd], kind='stable') #plt.plot(wave[igd][isrt1], spline_flux1[igd][isrt1]) #plt.show() spline_img_interp = scipy.interpolate.interp1d(wave[igd][isrt1],spline_flux1[igd][isrt1],assume_sorted=False, @@ -1018,7 +1048,7 @@ def fit_profile(image, ivar, waveimg, thismask, spat_img, trace_in, wave, inside, = np.where(IN_PIX.flatten()) - si = inside[np.argsort(sigma_x.flat[inside])] + si = inside[np.argsort(sigma_x.flat[inside], kind='stable')] sr = si[::-1] bset, bmask = fitting.iterfit(sigma_x.flat[si],norm_obj.flat[si], invvar = norm_ivar.flat[si], @@ -1075,7 +1105,7 @@ def fit_profile(image, ivar, waveimg, thismask, spat_img, trace_in, wave, return (profile_model, trace_in, fwhmfit, med_sn2) sigma_iter = 3 - isort = (xtemp.flat[si[inside]]).argsort() + isort = (xtemp.flat[si[inside]]).argsort(kind='stable') inside = si[inside[isort]] pb = np.ones(inside.size) @@ -1152,7 +1182,7 @@ def fit_profile(image, ivar, waveimg, thismask, spat_img, trace_in, wave, # Update the profile B-spline fit for the next iteration if iiter < sigma_iter-1: - ss = sigma_x.flat[inside].argsort() + ss = sigma_x.flat[inside].argsort(kind='stable') pb = (np.outer(area, np.ones(nspat,dtype=float))).flat[inside] keep = (bkpt >= sigma_x.flat[inside].min()) & (bkpt <= sigma_x.flat[inside].max()) if keep.sum() == 0: @@ -1176,7 +1206,7 @@ def fit_profile(image, ivar, waveimg, thismask, spat_img, trace_in, wave, xnew = trace_in fwhmfit = sigma*2.3548 - ss=sigma_x.flatten().argsort() + ss=sigma_x.flatten().argsort(kind='stable') inside, = np.where((sigma_x.flat[ss] >= min_sigma) & (sigma_x.flat[ss] <= max_sigma) & mask[ss] & @@ -1195,7 +1225,7 @@ def fit_profile(image, ivar, waveimg, thismask, spat_img, trace_in, wave, sigma_x_igood = sigma_x.flat[igood] yfit_out, _ = bset.value(sigma_x_igood) full_bsp[igood] = yfit_out - isrt2 = sigma_x_igood.argsort() + isrt2 = sigma_x_igood.argsort(kind='stable') (peak, peak_x, lwhm, rwhm) = findfwhm(yfit_out[isrt2] - median_fit, sigma_x_igood[isrt2]) diff --git a/pypeit/core/findobj_skymask.py b/pypeit/core/findobj_skymask.py index 8d5ba51880..6e4698c844 100644 --- a/pypeit/core/findobj_skymask.py +++ b/pypeit/core/findobj_skymask.py @@ -11,6 +11,7 @@ import matplotlib.pyplot as plt import astropy.stats +from astropy import table from pypeit import msgs from pypeit import utils @@ -66,57 +67,68 @@ def create_skymask(sobjs, thismask, slit_left, slit_righ, box_rad_pix=None, trim global sky subtraction (True means the pixel is usable for sky subtraction, False means it should be masked when subtracting sky). """ + # Number of objects nobj = len(sobjs) - ximg, _ = pixels.ximg_and_edgemask(slit_left, slit_righ, thismask, trim_edg=trim_edg) - # How many pixels wide is the slit at each Y? - xsize = slit_righ - slit_left - #nsamp = np.ceil(np.median(xsize)) # JFH Changed 07-07-19 - nsamp = np.ceil(xsize.max()) + if nobj == 0: + msgs.info('No objects were detected. The entire slit will be used for sky subtraction.') + return thismask[thismask] - # Objmask + # Compute the object mask skymask_objsnr = np.copy(thismask) - if nobj == 0: - msgs.info('No objects were detected. The entire slit will be used to determine the sky subtraction.') - else: - # Compute some inputs for the object mask - xtmp = (np.arange(nsamp) + 0.5)/nsamp - # threshold for object finding - for iobj in range(nobj): - # this will skip also sobjs with THRESHOLD=0, because are the same that have smash_snr=0. - if (sobjs[iobj].smash_snr != 0.) and (sobjs[iobj].smash_snr != None): - qobj = np.zeros_like(xtmp) - sep = np.abs(xtmp-sobjs[iobj].SPAT_FRACPOS) - sep_inc = sobjs[iobj].maskwidth/nsamp - close = sep <= sep_inc - # This is an analytical SNR profile with a Gaussian shape. - # JFH modified to use SNR here instead of smash peakflux. I believe that the 2.77 is supposed to be - # 2.355**2/2, i.e. the argument of a gaussian with sigma = FWHM/2.35 - qobj[close] = sobjs[iobj].smash_snr * \ - np.exp(np.fmax(-2.77*(sep[close]*nsamp)**2/sobjs[iobj].FWHM**2, -9.0)) - skymask_objsnr[thismask] &= np.interp(ximg[thismask], xtmp, qobj) < skymask_snr_thresh - # FWHM + # Create an image with pixel values equal to the fraction of the spatial + # position along the slit, ranging from 0 -> 1 + ximg, _ = pixels.ximg_and_edgemask(slit_left, slit_righ, thismask, trim_edg=trim_edg) + # Maximum spatial width rounded up + nsamp = np.ceil(np.amax(slit_righ - slit_left)) + # Fractional position within the maximum spatial width + xtmp = (np.arange(nsamp) + 0.5)/nsamp + # threshold for object finding + for iobj in range(nobj): + # this will skip also sobjs with THRESHOLD=0, because are the same that have smash_snr=0. + if sobjs[iobj].smash_snr is None or sobjs[iobj].smash_snr <= 0.: + continue + # Select pixels within the defined width of the object + sep = np.absolute(xtmp-sobjs[iobj].SPAT_FRACPOS) + sep_inc = sobjs[iobj].maskwidth/nsamp + close = sep <= sep_inc + # This is an analytical SNR profile with a Gaussian shape. + # JFH modified to use SNR here instead of smash peakflux. I believe that + # the 2.77 is supposed to be 2.355**2/2, i.e. the argument of a gaussian + # with sigma = FWHM/2.35 + qobj = np.zeros_like(xtmp) + qobj[close] = sobjs[iobj].smash_snr * \ + np.exp(np.fmax(-2.77*(sep[close]*nsamp)**2/sobjs[iobj].FWHM**2, -9.0)) + skymask_objsnr[thismask] &= np.interp(ximg[thismask], xtmp, qobj) < skymask_snr_thresh + + # Compute the FWHM mask skymask_fwhm = np.copy(thismask) - if nobj > 0: - nspec, nspat = thismask.shape - # spatial position everywhere along image - spat_img = np.outer(np.ones(nspec, dtype=int),np.arange(nspat, dtype=int)) - # Boxcar radius? - if box_rad_pix is not None: - msgs.info("Using boxcar radius for masking") - # Loop me - for iobj in range(nobj): - # Create a mask for the pixels that will contribute to the object - skymask_radius = box_rad_pix if box_rad_pix is not None else sobjs[iobj].FWHM - msgs.info(f"Masking around object {iobj+1} within a radius = {skymask_radius} pixels") - slit_img = np.outer(sobjs[iobj].TRACE_SPAT, np.ones(nspat)) # central trace replicated spatially - objmask_now = thismask & (spat_img > (slit_img - skymask_radius)) & (spat_img < (slit_img + skymask_radius)) - skymask_fwhm &= np.invert(objmask_now) - - # Check that we have not performed too much masking - if (np.sum(skymask_fwhm)/np.sum(thismask) < 0.10): - msgs.warn('More than 90% of usable area on this slit would be masked and not used by global sky subtraction. ' - 'Something is probably wrong with object finding for this slit. Not masking object for global sky subtraction.') - skymask_fwhm = np.copy(thismask) + nspec, nspat = thismask.shape + # spatial position everywhere along image +# spat_img = np.outer(np.ones(nspec, dtype=int),np.arange(nspat, dtype=int)) + spat_img = np.tile(np.arange(nspat, dtype=int), (nspec,1)) + # Boxcar radius? + if box_rad_pix is not None: + msgs.info("Using boxcar radius for masking") + # Loop me + for iobj in range(nobj): + # Create a mask for the pixels that will contribute to the object + skymask_radius = box_rad_pix if box_rad_pix is not None else sobjs[iobj].FWHM + msgs.info(f"Masking around object {iobj+1} within a radius = {skymask_radius} pixels") +# slit_img = np.outer(sobjs[iobj].TRACE_SPAT, np.ones(nspat)) # central trace replicated spatially + slit_img = np.tile(sobjs[iobj].TRACE_SPAT, (nspat,1)).T + objmask_now = thismask \ + & (spat_img > slit_img - skymask_radius) \ + & (spat_img < slit_img + skymask_radius) + skymask_fwhm &= np.logical_not(objmask_now) + + # Check that we have not performed too much masking + # TODO: There is this hard-coded check here, and then there is a similar + # check in skysub.global_skysub. Do we need both? + if np.sum(skymask_fwhm)/np.sum(thismask) < 0.10: + msgs.warn('More than 90% of usable area on this slit would be masked and not used by ' + 'global sky subtraction. Something is probably wrong with object finding for ' + 'this slit. Not masking object for global sky subtraction.') + skymask_fwhm = np.copy(thismask) # Still have to make the skymask # # TODO -- Make sure this is right @@ -135,8 +147,10 @@ def create_skymask(sobjs, thismask, slit_left, slit_righ, box_rad_pix=None, trim # computation from objs_in_slit is not necessarily that reliable and when large amounts of masking are performed # on narrow slits/orders, we have problems. We should revisit this after object finding is refactored since # maybe then the fwhm estimates will be more robust. - if box_rad_pix is None and np.all([sobj.smash_snr is not None for sobj in sobjs]) \ - and np.all([sobj.smash_snr != 0. for sobj in sobjs]) and not np.all(skymask_objsnr == thismask): + if box_rad_pix is None \ + and np.all([sobj.smash_snr is not None for sobj in sobjs]) \ + and np.all([sobj.smash_snr != 0. for sobj in sobjs]) \ + and not np.all(skymask_objsnr == thismask): # TODO This is a kludge until we refactor this routine. Basically mask design objects that are not auto-ID # always have smash_snr undefined. If there is a hybrid situation of auto-ID and maskdesign, the logic # here does not really make sense. Soution would be to compute thershold and smash_snr for all objects. @@ -225,10 +239,11 @@ def ech_findobj_ineach_order( Good-pixel mask for input image. Must have the same shape as ``image``. If None, all pixels in ``slitmask`` with non-negative values are considered good. - std_trace (`numpy.ndarray`_, optional): - Vector with the standard star trace, which is used as a crutch for - tracing. Shape must be (nspec,). If None, the slit boundaries are - used as the crutch. + std_trace (`astropy.table.Table`_, optional): + Table with the trace of the standard star on the input detector, + which is used as a crutch for tracing. The table has two columns: + `ECH_ORDER` and `TRACE_SPAT`. The shape of each row must be (nspec,). + If None, or for missing orders, the slit boundaries are used as the crutch. ncoeff (:obj:`int`, optional): Order of polynomial fit to traces. box_radius (:obj:`float`, optional): @@ -300,7 +315,10 @@ def ech_findobj_ineach_order( specobj_dict['SLITID'] = slit_spats[iord] specobj_dict['ECH_ORDERINDX'] = iord specobj_dict['ECH_ORDER'] = iorder - std_in = None if std_trace is None else std_trace[:, iord] + std_in = None + if std_trace is not None and 'ECH_ORDER' in std_trace.keys() and np.any(std_trace['ECH_ORDER'] == iorder): + idx = np.where(std_trace['ECH_ORDER'] == iorder)[0][0] + std_in = std_trace[idx]['TRACE_SPAT'] # Get SLTIORD_ID for the objfind QA ech_objfindQA_filename = objfindQA_filename.replace('S0999', 'S{:04d}'.format(order_vec[iord])) \ @@ -431,7 +449,7 @@ def ech_fill_in_orders(sobjs:specobjs.SpecObjs, slit_spat_id: np.ndarray, order_vec:np.ndarray, obj_id:np.ndarray, - std_trace:specobjs.SpecObjs=None, + std_trace:table.Table=None, show:bool=False): """ For objects which were only found on some orders, the standard (or @@ -471,9 +489,11 @@ def ech_fill_in_orders(sobjs:specobjs.SpecObjs, ``np.arange(norders)`` (but this is *not* recommended). obj_id (`numpy.ndarray`_): Object IDs of the objects linked together. - std_trace (:class:`~pypeit.specobjs.SpecObjs`, optional): - Standard star objects (including the traces) - Defaults to None. + std_trace (`astropy.table.Table`_, optional): + Table with the trace of the standard star on the input detector, + which is used as a crutch for tracing. The table has two columns: + `ECH_ORDER` and `TRACE_SPAT`. The shape of each row must be (nspec,). + If None, or for missing orders, the slit boundaries are used as the crutch. show (bool, optional): Plot diagnostics related to filling the missing orders @@ -493,8 +513,9 @@ def ech_fill_in_orders(sobjs:specobjs.SpecObjs, slit_width = slit_righ - slit_left # Check standard star - if std_trace is not None and std_trace.shape[1] != norders: - msgs.error('Standard star trace does not match the number of orders in the echelle data.') + if std_trace is not None and len(std_trace) != norders: + msgs.warn('Standard star trace does not match the number of orders in the echelle data.' + ' Will use the slit edges to trace the object in the missing orders.') # For traces nspec = slit_left.shape[0] @@ -510,7 +531,7 @@ def ech_fill_in_orders(sobjs:specobjs.SpecObjs, uni_frac = gfrac[uni_ind] # Sort with respect to fractional slit location to guarantee that we have a similarly sorted list of objects later - isort_frac = uni_frac.argsort() + isort_frac = uni_frac.argsort(kind='stable') uni_obj_id = uni_obj_id[isort_frac] uni_frac = uni_frac[isort_frac] @@ -594,12 +615,16 @@ def ech_fill_in_orders(sobjs:specobjs.SpecObjs, #thisobj.ech_order = order_vec[iord] thisobj.SPAT_FRACPOS = uni_frac[iobj] # Assign traces using the fractional position fit above - if std_trace is not None: - x_trace = np.interp(slit_spec_pos, spec_vec, std_trace[:,iord]) + if std_trace is not None and 'ECH_ORDER' in std_trace.keys() and \ + np.any(std_trace['ECH_ORDER'] == this_order): + idx = np.where(std_trace['ECH_ORDER'] == this_order)[0][0] + # standard star trace in this order + std_in = std_trace[idx]['TRACE_SPAT'] + x_trace = np.interp(slit_spec_pos, spec_vec, std_in) shift = np.interp( slit_spec_pos, spec_vec, slit_left[:,iord] + slit_width[:,iord]*frac_mean_new[iord]) - x_trace - thisobj.TRACE_SPAT = std_trace[:,iord] + shift + thisobj.TRACE_SPAT = std_in + shift else: thisobj.TRACE_SPAT = slit_left[:, iord] + slit_width[:, iord] * frac_mean_new[iord] # new trace thisobj.trace_spec = spec_vec @@ -775,7 +800,7 @@ def ech_cutobj_on_snr( ## Loop over objects from highest SNR to lowest SNR. Apply the S/N constraints. Once we hit the maximum number # objects requested exit, except keep the hand apertures that were requested. - isort_SNR_max = np.argsort(np.median(SNR_arr,axis=0))[::-1] + isort_SNR_max = np.argsort(np.median(SNR_arr,axis=0), kind='stable')[::-1] for iobj in isort_SNR_max: hand_ap_flag = hand_flag[iobj] SNR_constraint = (SNR_arr[:,iobj].max() > max_snr) or ( @@ -787,7 +812,7 @@ def ech_cutobj_on_snr( sobjs_keep = sobjs_align[ikeep].copy() sobjs_keep.ECH_OBJID = iobj_keep sobjs_keep.OBJID = iobj_keep - sobjs_trim.add_sobj(sobjs_keep[np.argsort(sobjs_keep.SLITID)]) + sobjs_trim.add_sobj(sobjs_keep[np.argsort(sobjs_keep.SLITID, kind='stable')]) iobj_keep += 1 if not hand_ap_flag: iobj_keep_not_hand += 1 @@ -1111,10 +1136,11 @@ def ech_objfind(image, ivar, slitmask, slit_left, slit_righ, slit_spat_id, order Plate scale in arcsec/pix. This can either be a single float for every order, or an array with shape (norders,) providing the plate scale of each order. - std_trace (`numpy.ndarray`_, optional): - Vector with the standard star trace, which is used as a crutch for - tracing. Shape must be (nspec,). If None, the slit boundaries are - used as the crutch. + std_trace (`astropy.table.Table`_, optional): + Table with the trace of the standard star on the input detector, + which is used as a crutch for tracing. The table has two columns: + `ECH_ORDER` and `TRACE_SPAT`. The shape of each row must be (nspec,). + If None, or for missing orders, the slit boundaries are used as the crutch. ncoeff (:obj:`int`, optional): Order of polynomial fit to traces. npca (:obj:`int`, optional): @@ -1252,6 +1278,7 @@ def ech_objfind(image, ivar, slitmask, slit_left, slit_righ, slit_spat_id, order det=det, inmask=inmask, std_trace=std_trace, + ncoeff=ncoeff, specobj_dict=specobj_dict, snr_thresh=snr_thresh, show_peaks=show_peaks, @@ -1313,31 +1340,210 @@ def ech_objfind(image, ivar, slitmask, slit_left, slit_righ, slit_spat_id, order return sobjs_ech -def orig_ech_objfind(image, ivar, slitmask, slit_left, slit_righ, order_vec, maskslits, det='DET01', - inmask=None, spec_min_max=None, fof_link=1.5, plate_scale=0.2, - std_trace=None, ncoeff=5, npca=None, coeff_npoly=None, max_snr=2.0, min_snr=1.0, - nabove_min_snr=2, pca_explained_var=99.0, box_radius=2.0, fwhm=3.0, - use_user_fwhm=False, maxdev=2.0, hand_extract_dict=None, nperorder=2, - extract_maskwidth=3.0, snr_thresh=10.0, - specobj_dict=None, trim_edg=(5,5), - show_peaks=False, show_fits=False, show_single_fits=False, - show_trace=False, show_single_trace=False, show_pca=False, - debug_all=False, objfindQA_filename=None): +def objfind_QA(spat_peaks, snr_peaks, spat_vector, snr_vector, snr_thresh, qa_title, peak_gpm, + near_edge_bpm, nperslit_bpm, objfindQA_filename=None, show=False): """ - Object finding routine for Echelle spectrographs. - - This routine: + Utility routine for making object finding QA plots. - #. Runs object finding on each order individually + Args: + spat_peaks (`numpy.ndarray`_): + Array of locations in the spatial direction at which objects were + identified. Shape is ``(npeaks,)``, where ``npeaks`` is the number + of peaks identified. + snr_peaks (`numpy.ndarray`_): + S/N ratio in the spectral direction after collapsing along the + spectral direction, evaluated at the location of each spatial peak. + Shape must match ``spat_peaks``. + spat_vector (`numpy.ndarray`_): + A 1D array of spatial locations along the slit. Shape is + ``(nsamp,)``, where ``nsamp`` is the number of spatial pixels + defined by the slit edges. + snr_vector (`numpy.ndarray`_): + A 1D array with the S/N ratio sampled along the slit at each spatial + location (i.e., spectral direction has been smashed out) defined by + ``spat_vector``. Shape must match ``spat_vector``. + snr_thresh (:obj:`float`): + The threshold S/N ratio adopted by the object finding. + qa_title (:obj:`str`): + Title for the QA file plot. + peak_gpm (`numpy.ndarray`_): + Boolean array containing a good pixel mask for each peak indicating + whether it will be used as an object (True) or not (False). Shape + must match ``spat_peaks``. + near_edge_bpm (`numpy.ndarray`_): + A bad pixel mask (True is masked, False is unmasked) indicating + which objects are masked because they are near the slit edges. + Shape must match ``spat_peaks``. + nperslit_bpm (`numpy.ndarray`_): + A bad pixel mask (True is masked, False is unmasked) indicating + which objects are masked because they exceed the maximum number of + objects (see :func:`objs_in_slit` parameter ``nperslit``) that were + specified as being on this slit. + objfindQA_filename (:obj:`str`, optional): + Output filename for the QA plot. If None, plot is not saved. + show (:obj:`bool`, optional): + If True, show the plot as a matplotlib interactive plot. - #. Links the objects found together using a friends-of-friends algorithm - on fractional order position. + """ - #. For objects which were only found on some orders, the standard (or - the slit boundaries) are placed at the appropriate fractional - position along the order. + plt.plot(spat_vector, snr_vector, drawstyle='steps-mid', color='black', label = 'Collapsed SNR (FWHM convol)') + plt.hlines(snr_thresh,spat_vector.min(),spat_vector.max(), color='red',linestyle='--', + label='SNR_THRESH={:5.3f}'.format(snr_thresh)) + if np.any(peak_gpm): + plt.plot(spat_peaks[peak_gpm], snr_peaks[peak_gpm], color='red', marker='o', markersize=10.0, + mfc='lawngreen', fillstyle='full',linestyle='None', zorder = 10,label='{:d} Good Objects'.format(np.sum(peak_gpm))) + if np.any(near_edge_bpm): + plt.plot(spat_peaks[near_edge_bpm], snr_peaks[near_edge_bpm], color='red', marker='o', markersize=10.0, + mfc='cyan', fillstyle='full', linestyle='None', zorder = 10,label='{:d} Rejected: Near Edge'.format(np.sum(near_edge_bpm))) + if np.any(nperslit_bpm): + plt.plot(spat_peaks[nperslit_bpm], snr_peaks[nperslit_bpm], color='red', marker='o', markersize=10.0, + mfc='yellow', fillstyle='full', linestyle='None', zorder = 10,label='{:d} Rejected: Nperslit'.format(np.sum(nperslit_bpm))) + plt.legend() + plt.xlabel('Approximate Spatial Position (pixels)') + plt.ylabel('SNR') + plt.title(qa_title) + #plt.ylim(np.fmax(snr_vector.min(), -20.0), 1.3*snr_vector.max()) + fig = plt.gcf() + if show: + plt.show() + if objfindQA_filename is not None: + fig.savefig(objfindQA_filename, dpi=400) + plt.close('all') - #. A PCA fit to the traces is performed using the routine above pca_fit +def get_fwhm(fwhm_in, nsamp, smash_peakflux, spat_fracpos, flux_smash_smth): + """ + Utility routine to measure the FWHM of an object trace from the spectrally + collapsed flux profile by determining the locations along the spatial + direction where this profile reaches have its peak value. + + Args: + fwhm_in (:obj:`float`): + Best guess for the FWHM of this object. + nsamp (:obj:`int`): + Number of pixels along the spatial direction. + smash_peakflux (:obj:`float`): + The peak flux in the 1d (spectrally collapsed) flux profile at the + object location. + spat_fracpos (:obj:`float`): + Fractional spatial position along the slit where the object is + located and at which the ``flux_smash_smth`` array has values + provided by ``smash_peakflux`` (see above and below). + flux_smash_smth (`numpy.ndarray`_): + A 1D array with the flux averaged along the spectral direction at + each location along the slit in the spatial direction location. + Shape is ``(nsamp,)``. + + Returns: + :obj:`float`: The FWHM determined from the object flux profile, unless + the FWHM could not be found from the profile, in which case the input + guess (``fwhm_in``) is simply returned. + """ + + # Determine the fwhm max + yhalf = 0.5*smash_peakflux + xpk = spat_fracpos*nsamp + x0 = int(np.rint(xpk)) + # TODO It seems we have two codes that do similar things, i.e. findfwhm in arextract.py. Could imagine having one + # Find right location where smash profile croses yhalf + if x0 < (int(nsamp) - 1): + ind_righ, = np.where(flux_smash_smth[x0:] < yhalf) + if len(ind_righ) > 0: + i2 = ind_righ[0] + if i2 == 0: + xrigh = None + else: + xarr_righ = x0 + np.array([i2 - 1, i2], dtype=float) + xrigh_int = scipy.interpolate.interp1d(flux_smash_smth[x0 + i2 - 1:x0 + i2 + 1], xarr_righ, + assume_sorted=False, bounds_error=False, + fill_value=(xarr_righ[0], xarr_righ[1])) + xrigh = xrigh_int([yhalf])[0] + else: + xrigh = None + else: + xrigh = None + # Find left location where smash profile crosses yhalf + if x0 > 0: + ind_left, = np.where(flux_smash_smth[0:np.fmin(x0 + 1, int(nsamp) - 1)] < yhalf) + if len(ind_left) > 0: + i1 = (ind_left[::-1])[0] + if i1 == (int(nsamp) - 1): + xleft = None + else: + xarr_left = np.array([i1, i1 + 1], dtype=float) + xleft_int = scipy.interpolate.interp1d(flux_smash_smth[i1:i1 + 2], xarr_left, + assume_sorted=False, bounds_error=False, + fill_value=(xarr_left[0], xarr_left[1])) + xleft = xleft_int([yhalf])[0] + else: + xleft = None + else: + xleft = None + + # Set FWHM for the object + if (xleft is None) & (xrigh is None): + fwhm_measure = None + elif xrigh is None: + fwhm_measure = 2.0 * (xpk - xleft) + elif xleft is None: + fwhm_measure = 2.0 * (xrigh - xpk) + else: + fwhm_measure = (xrigh - xleft) + + if fwhm_measure is not None: + fwhm_out = np.sqrt(np.fmax(fwhm_measure ** 2 - fwhm_in ** 2, (fwhm_in / 2.0) ** 2)) # Set a floor of fwhm/2 on fwhm + else: + fwhm_out = fwhm_in + + return fwhm_out + + +def objs_in_slit(image, ivar, thismask, slit_left, slit_righ, + inmask=None, fwhm=3.0, + sigclip_smash=5.0, use_user_fwhm=False, boxcar_rad=7., + maxdev=2.0, spec_min_max=None, hand_extract_dict=None, std_trace=None, + ncoeff=5, nperslit=None, snr_thresh=10.0, trim_edg=(5,5), + extract_maskwidth=4.0, specobj_dict=None, find_min_max=None, + show_peaks=False, show_fits=False, show_trace=False, + debug_all=False, qa_title='objfind', objfindQA_filename=None): + """ + Find the location of objects in a slitmask slit or a echelle order. + + The algorithm for this function is: + + - Rectify the image by extracting along the edge traces. + + - Compute the sigma-clipped mean spatial profile and its variance by + collapsing the image along the spectral direction to constuct a S/N + spatial profile of the slit/order. + + - Smooth the S/N profile by a Gaussian with the provided FWHM (see + ``fwhm``) and detect peaks in the smoothed profile using + :func:`~pypeit.core.arc.detect_lines`. Ignore peaks found near the + slit edges, and limit the number of peaks to the number requested (see + ``nperslit``). + + - Instantiate a :class:`~pypeit.specobj.SpecObj` object for each valid + detection, construct preliminary spectral traces for them, and estimate + the object spatial FWHM if one is not provided (see + ``use_user_fwhm``). + + - For automatically identified objects (i.e., not manual extractions), + improve the object trace by fitting the spatial position of the peak + as a function of wavelength. For manual apertures, use either the + form of the brightest object on the slit, the trace of a standard star + (see ``std_trace``), or the left edge trace to set the trace for the + object. Finally, remove automatically identified objects that overlap + with manually defined extraction apertures. + + At the end of this function, the list of objects is ready for extraction. + + **Revision History:** + + - 10-Mar-2005 -- First version written by D. Schlegel, LBL + - 2005-2018 -- Improved by J. F. Hennawi and J. X. Prochaska + - 23-June-2018 -- Ported to python by J. F. Hennawi and significantly + improved + - 01-Feb-2022 -- Skymask stripped out by JXP Args: image (`numpy.ndarray`_): @@ -1352,867 +1558,13 @@ def orig_ech_objfind(image, ivar, slitmask, slit_left, slit_righ, order_vec, mas ivar (`numpy.ndarray`_): Floating-point inverse variance image for the input image. Shape must match ``image``, (nspec, nspat). - slitmask (`numpy.ndarray`_): - Integer image indicating the pixels that belong to each order. - Pixels that are not on an order have value -1, and those that are on - an order have a value equal to the slit number (i.e. 0 to nslits-1 - from left to right on the image). Shape must match ``image``, - (nspec, nspat). + thismask (`numpy.ndarray`_): + Boolean mask image selecting pixels associated with the slit/order + to search for objects on (True means on the slit/order). Shape must + match ``image``. slit_left (`numpy.ndarray`_): - Left boundary of orders to be extracted (given as floating point - pixels). Shape is (nspec, norders), where norders is the total - number of traced echelle orders. - slit_righ (`numpy.ndarray`_): - Right boundary of orders to be extracted (given as floating point - pixels). Shape is (nspec, norders), where norders is the total - number of traced echelle orders. - order_vec (`numpy.ndarray`_): - Vector identifying the Echelle orders for each pair of order edges - found. This is saved to the output :class:`~pypeit.specobj.SpecObj` - objects. If the orders are not known, this can be - ``np.arange(norders)`` (but this is *not* recommended). - maskslits (`numpy.ndarray`_): - Boolean array selecting orders that should be ignored (i.e., good - orders are False, bad orders are True). Shape must be (norders,). - det (:obj:`str`, optional): - The name of the detector containing the object. Only used if - ``specobj_dict`` is None. - inmask (`numpy.ndarray`_, optional): - Good-pixel mask for input image. Must have the same shape as - ``image``. If None, all pixels in ``slitmask`` with non-negative - values are considered good. - spec_min_max (`numpy.ndarray`_, optional): - 2D array defining the minimum and maximum pixel in the spectral - direction with useable data for each order. Shape must be (2, - norders). This should only be used for echelle spectrographs for - which the orders do not entirely cover the detector. PCA tracing - will re-map the traces such that they all have the same length, - compute the PCA, and then re-map the orders back. This improves - performance for echelle spectrographs by removing the nonlinear - shrinking of the orders so that the linear pca operation can better - predict the traces. If None, the minimum and maximum values will be - determined automatically from ``slitmask``. - fof_link (:obj:`float`, optional): - Friends-of-friends linking length in arcseconds used to link - together traces across orders. The routine links together at - the same fractional slit position and links them together - with a friends-of-friends algorithm using this linking - length. - plate_scale (:obj:`float`, `numpy.ndarray`_, optional): - Plate scale in arcsec/pix. This can either be a single float for - every order, or an array with shape (norders,) providing the plate - scale of each order. - std_trace (`numpy.ndarray`_, optional): - Vector with the standard star trace, which is used as a crutch for - tracing. Shape must be (nspec,). If None, the slit boundaries are - used as the crutch. - ncoeff (:obj:`int`, optional): - Order of polynomial fit to traces. - npca (:obj:`int`, optional): - Number of PCA components to keep during PCA decomposition of the - object traces. If None, the number of components set by requiring - the PCA accounts for approximately 99% of the variance. - coeff_npoly (:obj:`int`, optional): - Order of polynomial used for PCA coefficients fitting. If None, - value set automatically, see - :func:`~pypeit.tracepca.pca_trace_object`. - max_snr (:obj:`float`, optional): - For an object to be included in the output object, it must have a - max S/N ratio above this value. - min_snr (:obj:`float`, optional): - For an object to be included in the output object, it must have a - a median S/N ratio above this value for at least - ``nabove_min_snr`` orders (see below). - nabove_min_snr (:obj:`int`, optional): - The required number of orders that an object must have with median - SNR greater than ``min_snr`` in order to be included in the output - object. - pca_explained_var (:obj:`float`, optional): - The percentage (i.e., not the fraction) of the variance in the data - accounted for by the PCA used to truncate the number of PCA - coefficients to keep (see ``npca``). Ignored if ``npca`` is provided - directly; see :func:`~pypeit.tracepca.pca_trace_object`. - box_radius (:obj:`float`, optional): - Box_car extraction radius in arcseconds to assign to each detected - object and to be used later for boxcar extraction. In this method - ``box_radius`` is converted into pixels using ``plate_scale``. - ``box_radius`` is also used for SNR calculation and trimming. - fwhm (:obj:`float`, optional): - Estimated fwhm of the objects in pixels - use_user_fwhm (:obj:`bool`, optional): - If True, ``PypeIt`` will use the spatial profile FWHM input by the - user (see ``fwhm``) rather than determine the spatial FWHM from the - smashed spatial profile via the automated algorithm. - maxdev (:obj:`float`, optional): - Maximum deviation of pixels from polynomial fit to trace - used to reject bad pixels in trace fitting. - hand_extract_dict (:obj:`dict`, optional): - Dictionary with info on manual extraction; see - :class:`~pypeit.manual_extract.ManualExtractionObj`. - nperorder (:obj:`int`, optional): - Maximum number of objects allowed per order. If there are more - detections than this number, the code will select the ``nperorder`` - most significant detections. However, hand apertures will always be - returned and do not count against this budget. - extract_maskwidth (:obj:`float`, optional): - Determines the initial size of the region in units of FWHM that will - be used for local sky subtraction; See :func:`objs_in_slit` and - :func:`~pypeit.core.skysub.local_skysub_extract`. - snr_thresh (:obj:`float`, optional): - SNR threshold for finding objects - specobj_dict (:obj:`dict`, optional): - Dictionary containing meta-data for the objects that will be - propagated into the :class:`~pypeit.specobj.SpecObj` objects. The - expected components are: - - - SLITID: The slit ID number - - DET: The detector identifier - - OBJTYPE: The object type - - PYPELINE: The class of pipeline algorithms applied - - If None, the dictionary is filled with the following placeholders:: - - specobj_dict = {'SLITID': 999, 'DET': 'DET01', - 'OBJTYPE': 'unknown', 'PYPELINE': 'unknown'} - - trim_edg (:obj:`tuple`, optional): - A two-tuple of integers or floats used to ignore objects within this - many pixels of the left and right slit boundaries, respectively. - show_peaks (:obj:`bool`, optional): - Plot the QA of the object peak finding in each order. - show_fits (:obj:`bool`, optional): - Plot trace fitting for final fits using PCA as crutch. - show_single_fits (:obj:`bool`, optional): - Plot trace fitting for single order fits. - show_trace (:obj:`bool`, optional): - Display the object traces on top of the image. - show_single_trace (:obj:`bool`, optional): - Display the object traces on top of the single order. - show_pca (:obj:`bool`, optional): - Display debugging plots for the PCA decomposition. - debug_all (:obj:`bool`, optional): - Show all the debugging plots. If True, this also overrides any - provided values for ``show_peaks``, ``show_trace``, and - ``show_pca``, setting them to True. - objfindQA_filename (:obj:`str`, optional): - Full path (directory and filename) for the object profile QA plot. - If None, not plot is produced and saved. - - Returns: - :class:`~pypeit.specobjs.SpecObjs`: Object containing the objects - detected. - """ - raise DeprecationWarning - msgs.error("This ginormous method as been Deprecated") - - #debug_all=True - if debug_all: - show_peaks = True - #show_fits = True - #show_single_fits = True - show_trace = True - show_pca = True - #show_single_trace = True - # TODO: This isn't used, right? - debug = True - - - if specobj_dict is None: - specobj_dict = {'SLITID': 999, 'ECH_ORDERINDX': 999, - 'DET': det, 'OBJTYPE': 'unknown', 'PYPELINE': 'Echelle'} - - # TODO Update FOF algorithm here with the one from scikit-learn. - - allmask = slitmask > -1 - if inmask is None: - inmask = allmask - - nspec, nspat = image.shape - norders = len(order_vec) - - # Find the spat IDs - gdslit_spat = np.unique(slitmask[slitmask >= 0]).astype(int) # Unique sorts - if gdslit_spat.size != np.sum(np.invert(maskslits)): - msgs.error('Masking of slitmask not in sync with that of maskslits. This is a bug') - #msgs.error('There is a mismatch between the number of valid orders found by PypeIt and ' - # 'the number expected for this spectrograph. Unable to continue. Please ' - # 'submit an issue on Github: https://github.com/pypeit/PypeIt/issues .') - - if spec_min_max is None: - spec_min_max = np.zeros((2,norders), dtype=int) - for iord in range(norders): - ispec, ispat = np.where(slitmask == gdslit_spat[iord]) - spec_min_max[:,iord] = ispec.min(), ispec.max() - - # Setup the plate scale - if isinstance(plate_scale,(float, int)): - plate_scale_ord = np.full(norders, plate_scale) - elif isinstance(plate_scale,(np.ndarray, list, tuple)): - if len(plate_scale) == norders: - plate_scale_ord = plate_scale - elif len(plate_scale) == 1: - plate_scale_ord = np.full(norders, plate_scale[0]) - else: - msgs.error('Invalid size for plate_scale. It must either have one element or norders elements') - else: - msgs.error('Invalid type for plate scale') - - specmid = nspec // 2 - spec_vec = np.arange(nspec) - slit_width = slit_righ - slit_left - slit_spec_pos = nspec/2.0 - - # TODO JFH This hand apertures in echelle needs to be completely refactored. - # Hand prep - # Determine the location of the source on *all* of the orders - if hand_extract_dict is not None: - f_spats = [] - for ss, spat, spec in zip(range(len(hand_extract_dict['spec'])), - hand_extract_dict['spat'], - hand_extract_dict['spec']): - # Find the input slit - ispec = int(np.clip(np.round(spec),0,nspec-1)) - ispat = int(np.clip(np.round(spat),0,nspat-1)) - slit = slitmask[ispec, ispat] - if slit == -1: - msgs.error('You are requesting a manual extraction at a position ' + - f'(spat, spec)={spat, spec} that is not on one of the echelle orders. Check your pypeit file.') - # Fractions - iord_hand = gdslit_spat.tolist().index(slit) - f_spat = (spat - slit_left[ispec, iord_hand]) / ( - slit_righ[ispec, iord_hand] - slit_left[ispec, iord_hand]) - f_spats.append(f_spat) - - # Loop over orders and find objects - sobjs = specobjs.SpecObjs() - # TODO: replace orderindx with the true order number here? Maybe not. Clean - # up SLITID and orderindx! - gdorders = np.arange(norders)[np.invert(maskslits)] - for iord in gdorders: #range(norders): - qa_title = 'Finding objects on order # {:d}'.format(order_vec[iord]) - msgs.info(qa_title) - thisslit_gpm = slitmask == gdslit_spat[iord] - inmask_iord = inmask & thisslit_gpm - specobj_dict['SLITID'] = gdslit_spat[iord] - specobj_dict['ECH_ORDERINDX'] = iord - specobj_dict['ECH_ORDER'] = order_vec[iord] - std_in = None if std_trace is None else std_trace[:, iord] - - # TODO JFH: Fix this. The way this code works, you should only need to create a single hand object, - # not one at every location on the order - if hand_extract_dict is not None: - new_hand_extract_dict = copy.deepcopy(hand_extract_dict) - for ss, spat, spec, f_spat in zip(range(len(hand_extract_dict['spec'])), - hand_extract_dict['spat'], - hand_extract_dict['spec'], f_spats): - ispec = int(spec) - new_hand_extract_dict['spec'][ss] = ispec - new_hand_extract_dict['spat'][ss] = slit_left[ispec,iord] + f_spat*( - slit_righ[ispec,iord]-slit_left[ispec,iord]) - else: - new_hand_extract_dict = None - - # Get SLTIORD_ID for the objfind QA - ech_objfindQA_filename = objfindQA_filename.replace('S0999', 'S{:04d}'.format(order_vec[iord])) \ - if objfindQA_filename is not None else None - # Run - sobjs_slit = \ - objs_in_slit(image, ivar, thisslit_gpm, slit_left[:,iord], slit_righ[:,iord], spec_min_max=spec_min_max[:,iord], - inmask=inmask_iord,std_trace=std_in, ncoeff=ncoeff, fwhm=fwhm, use_user_fwhm=use_user_fwhm, maxdev=maxdev, - hand_extract_dict=new_hand_extract_dict, nperslit=nperorder, extract_maskwidth=extract_maskwidth, - snr_thresh=snr_thresh, trim_edg=trim_edg, boxcar_rad=box_radius/plate_scale_ord[iord], - show_peaks=show_peaks, show_fits=show_single_fits, - show_trace=show_single_trace, qa_title=qa_title, specobj_dict=specobj_dict, - objfindQA_filename=ech_objfindQA_filename) - sobjs.add_sobj(sobjs_slit) - - nfound = len(sobjs) - - if nfound == 0: - msgs.warn('No objects found') - return sobjs - - FOF_frac = fof_link/(np.median(np.median(slit_width,axis=0)*plate_scale_ord)) - # Run the FOF. We use fake coordinates - fracpos = sobjs.SPAT_FRACPOS - ra_fake = fracpos/1000.0 # Divide all angles by 1000 to make geometry euclidian - dec_fake = np.zeros_like(fracpos) - if nfound>1: - inobj_id, multobj_id, firstobj_id, nextobj_id \ - = pydl.spheregroup(ra_fake, dec_fake, FOF_frac/1000.0) - # TODO spheregroup returns zero based indices but we use one based. We should probably add 1 to inobj_id here, - # i.e. obj_id_init = inobj_id + 1 - obj_id_init = inobj_id.copy() - elif nfound==1: - obj_id_init = np.zeros(1,dtype='int') - - uni_obj_id_init, uni_ind_init = np.unique(obj_id_init, return_index=True) - - # Now loop over the unique objects and check that there is only one object per order. If FOF - # grouped > 1 objects on the same order, then this will be popped out as its own unique object - obj_id = obj_id_init.copy() - nobj_init = len(uni_obj_id_init) - for iobj in range(nobj_init): - for iord in range(norders): - on_order = (obj_id_init == uni_obj_id_init[iobj]) & (sobjs.ECH_ORDERINDX == iord) - if (np.sum(on_order) > 1): - msgs.warn('Found multiple objects in a FOF group on order iord={:d}'.format(iord) + msgs.newline() + - 'Spawning new objects to maintain a single object per order.') - off_order = (obj_id_init == uni_obj_id_init[iobj]) & (sobjs.ECH_ORDERINDX != iord) - ind = np.where(on_order)[0] - if np.any(off_order): - # Keep the closest object to the location of the rest of the group (on other orders) - # as corresponding to this obj_id, and spawn new obj_ids for the others. - frac_mean = np.mean(fracpos[off_order]) - min_dist_ind = np.argmin(np.abs(fracpos[ind] - frac_mean)) - else: - # If there are no other objects with this obj_id to compare to, then we simply have multiple - # objects grouped together on the same order, so just spawn new object IDs for them to maintain - # one obj_id per order - min_dist_ind = 0 - ind_rest = np.setdiff1d(ind,ind[min_dist_ind]) - # JFH OLD LINE with bug - #obj_id[ind_rest] = (np.arange(len(ind_rest)) + 1) + obj_id_init.max() - obj_id[ind_rest] = (np.arange(len(ind_rest)) + 1) + obj_id.max() - - uni_obj_id, uni_ind = np.unique(obj_id, return_index=True) - nobj = len(uni_obj_id) - msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects') - - gfrac = np.zeros(nfound) - for jj in range(nobj): - this_obj_id = obj_id == uni_obj_id[jj] - gfrac[this_obj_id] = np.median(fracpos[this_obj_id]) - - uni_frac = gfrac[uni_ind] - - # Sort with respect to fractional slit location to guarantee that we have a similarly sorted list of objects later - isort_frac = uni_frac.argsort() - uni_obj_id = uni_obj_id[isort_frac] - uni_frac = uni_frac[isort_frac] - - sobjs_align = sobjs.copy() - # Loop over the orders and assign each specobj a fractional position and a obj_id number - for iobj in range(nobj): - for iord in range(norders): - on_order = (obj_id == uni_obj_id[iobj]) & (sobjs_align.ECH_ORDERINDX == iord) - sobjs_align[on_order].ECH_FRACPOS = uni_frac[iobj] - sobjs_align[on_order].ECH_OBJID = uni_obj_id[iobj] - sobjs_align[on_order].OBJID = uni_obj_id[iobj] - sobjs_align[on_order].ech_frac_was_fit = False - - # Reset names (just in case) - sobjs_align.set_names() - # Now loop over objects and fill in the missing objects and their traces. We will fit the fraction slit position of - # the good orders where an object was found and use that fit to predict the fractional slit position on the bad orders - # where no object was found - for iobj in range(nobj): - # Grab all the members of this obj_id from the object list - indx_obj_id = sobjs_align.ECH_OBJID == uni_obj_id[iobj] - nthisobj_id = np.sum(indx_obj_id) - # Perform the fit if this objects shows up on more than three orders - if (nthisobj_id > 3) and (nthisobj_id 1: - msgs.error('Problem in echelle object finding. The same objid={:d} appears {:d} times on echelle orderindx ={:d}' - ' even after duplicate obj_ids the orders were removed. ' - 'Report this bug to PypeIt developers'.format(uni_obj_id[iobj],num_on_order, iord)) - - - - # Loop over the objects and perform a quick and dirty extraction to assess S/N. - varimg = utils.calc_ivar(ivar) - flux_box = np.zeros((nspec, norders, nobj)) - ivar_box = np.zeros((nspec, norders, nobj)) - mask_box = np.zeros((nspec, norders, nobj)) - SNR_arr = np.zeros((norders, nobj)) - slitfracpos_arr = np.zeros((norders, nobj)) - for iobj in range(nobj): - for iord in range(norders): - iorder_vec = order_vec[iord] - indx = sobjs_align.slitorder_objid_indices(iorder_vec, uni_obj_id[iobj]) - #indx = (sobjs_align.ECH_OBJID == uni_obj_id[iobj]) & (sobjs_align.ECH_ORDERINDX == iord) - #spec = sobjs_align[indx][0] - inmask_iord = inmask & (slitmask == gdslit_spat[iord]) - # TODO make the snippet below its own function quick_extraction() - box_rad_pix = box_radius/plate_scale_ord[iord] - - # TODO -- We probably shouldn't be operating on a SpecObjs but instead a SpecObj - flux_tmp = moment1d(image*inmask_iord, sobjs_align[indx][0].TRACE_SPAT, 2*box_rad_pix, - row=sobjs_align[indx][0].trace_spec)[0] - var_tmp = moment1d(varimg*inmask_iord, sobjs_align[indx][0].TRACE_SPAT, 2*box_rad_pix, - row=sobjs_align[indx][0].trace_spec)[0] - ivar_tmp = utils.calc_ivar(var_tmp) - pixtot = moment1d(ivar*0 + 1.0, sobjs_align[indx][0].TRACE_SPAT, 2*box_rad_pix, - row=sobjs_align[indx][0].trace_spec)[0] - mask_tmp = moment1d(ivar*inmask_iord == 0.0, sobjs_align[indx][0].TRACE_SPAT, 2*box_rad_pix, - row=sobjs_align[indx][0].trace_spec)[0] != pixtot - - flux_box[:,iord,iobj] = flux_tmp*mask_tmp - ivar_box[:,iord,iobj] = np.fmax(ivar_tmp*mask_tmp,0.0) - mask_box[:,iord,iobj] = mask_tmp - mean, med_sn, stddev = astropy.stats.sigma_clipped_stats( - flux_box[mask_tmp,iord,iobj]*np.sqrt(ivar_box[mask_tmp,iord,iobj]), - sigma_lower=5.0,sigma_upper=5.0 - ) - # ToDO assign this to sobjs_align for use in the extraction - SNR_arr[iord,iobj] = med_sn - sobjs_align[indx][0].ech_snr = med_sn - # For hand extractions - slitfracpos_arr[iord,iobj] = sobjs_align[indx][0].SPAT_FRACPOS - - # Purge objects with low SNR that don't show up in enough orders, sort the list of objects with respect to obj_id - # and orderindx - keep_obj = np.zeros(nobj,dtype=bool) - sobjs_trim = specobjs.SpecObjs() - # objids are 1 based so that we can easily asign the negative to negative objects - iobj_keep = 1 - iobj_keep_not_hand = 1 - - # TODO JFH: Fix this ugly and dangerous hack that was added to accomodate hand apertures - hand_frac = [-1000] if hand_extract_dict is None else [int(np.round(ispat*1000)) for ispat in f_spats] - - ## Loop over objects from highest SNR to lowest SNR. Apply the S/N constraints. Once we hit the maximum number - # objects requested exit, except keep the hand apertures that were requested. - isort_SNR_max = np.argsort(np.median(SNR_arr,axis=0))[::-1] - for iobj in isort_SNR_max: - hand_ap_flag = int(np.round(slitfracpos_arr[0, iobj]*1000)) in hand_frac - SNR_constraint = (SNR_arr[:,iobj].max() > max_snr) or (np.sum(SNR_arr[:,iobj] > min_snr) >= nabove_min_snr) - nperorder_constraint = (iobj_keep-1) < nperorder - if (SNR_constraint and nperorder_constraint) or hand_ap_flag: - keep_obj[iobj] = True - ikeep = sobjs_align.ECH_OBJID == uni_obj_id[iobj] - sobjs_keep = sobjs_align[ikeep].copy() - sobjs_keep.ECH_OBJID = iobj_keep - sobjs_keep.OBJID = iobj_keep -# for spec in sobjs_keep: -# spec.ECH_OBJID = iobj_keep -# #spec.OBJID = iobj_keep - sobjs_trim.add_sobj(sobjs_keep[np.argsort(sobjs_keep.ECH_ORDERINDX)]) - iobj_keep += 1 - if not hand_ap_flag: - iobj_keep_not_hand += 1 - else: - if not nperorder_constraint: - msgs.info('Purging object #{:d}'.format(iobj) + - ' since there are already {:d} objects automatically identified ' - 'and you set nperorder={:d}'.format(iobj_keep_not_hand-1, nperorder)) - else: - msgs.info('Purging object #{:d}'.format(iobj) + ' which does not satisfy max_snr > {:5.2f} OR min_snr > {:5.2f}'.format(max_snr, min_snr) + - ' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) + ' orders') - - - nobj_trim = np.sum(keep_obj) - - if nobj_trim == 0: - msgs.warn('No objects found') - sobjs_final = specobjs.SpecObjs() - return sobjs_final - - # TODO JFH: We need to think about how to implement returning a maximum number of objects, where the objects - # returned are the highest S/N ones. It is a bit complicated with regards to the individual object finding and then - # the linking that is performed above, and also making sure the hand apertures don't get removed. - SNR_arr_trim = SNR_arr[:,keep_obj] - - - sobjs_final = sobjs_trim.copy() - # Loop over the objects one by one and adjust/predict the traces - pca_fits = np.zeros((nspec, norders, nobj_trim)) - - # Create the trc_inmask for iterative fitting below - trc_inmask = np.zeros((nspec, norders), dtype=bool) - for iord in range(norders): - trc_inmask[:,iord] = (spec_vec >= spec_min_max[0,iord]) & (spec_vec <= spec_min_max[1,iord]) - - for iobj in range(nobj_trim): - indx_obj_id = sobjs_final.ECH_OBJID == (iobj + 1) - # PCA predict all the orders now (where we have used the standard or slit boundary for the bad orders above) - msgs.info('Fitting echelle object finding PCA for object {:d}/{:d} with median SNR = {:5.3f}'.format( - iobj + 1,nobj_trim,np.median(sobjs_final[indx_obj_id].ech_snr))) - pca_fits[:,:,iobj] \ - = tracepca.pca_trace_object(sobjs_final[indx_obj_id].TRACE_SPAT.T, - order=coeff_npoly, npca=npca, - pca_explained_var=pca_explained_var, - trace_wgt=np.fmax(sobjs_final[indx_obj_id].ech_snr, 1.0)**2, - debug=show_pca) - - # Trial and error shows weighting by S/N instead of S/N^2 performs better - # JXP -- Updated to now be S/N**2, i.e. inverse variance, with fitting fit - - # Perform iterative flux weighted centroiding using new PCA predictions - xinit_fweight = pca_fits[:,:,iobj].copy() - inmask_now = inmask & allmask - xfit_fweight = fit_trace(image, xinit_fweight, ncoeff, bpm=np.invert(inmask_now), - trace_bpm=np.invert(trc_inmask), fwhm=fwhm, maxdev=maxdev, - debug=show_fits)[0] - - # Perform iterative Gaussian weighted centroiding - xinit_gweight = xfit_fweight.copy() - xfit_gweight = fit_trace(image, xinit_gweight, ncoeff, bpm=np.invert(inmask_now), - trace_bpm=np.invert(trc_inmask), weighting='gaussian', fwhm=fwhm, - maxdev=maxdev, debug=show_fits)[0] - - #TODO Assign the new traces. Only assign the orders that were not orginally detected and traced. If this works - # well, we will avoid doing all of the iter_tracefits above to make the code faster. - for iord, spec in enumerate(sobjs_final[indx_obj_id]): - # JFH added the condition on ech_frac_was_fit with S/N cut on 7-7-19. - # TODO is this robust against half the order being masked? - if spec.ech_frac_was_fit & (spec.ech_snr > 1.0): - spec.TRACE_SPAT = xfit_gweight[:,iord] - spec.SPAT_PIXPOS = spec.TRACE_SPAT[specmid] - - #TODO Put in some criterion here that does not let the fractional position change too much during the iterative - # tracefitting. The problem is spurious apertures identified on one slit can be pulled over to the center of flux - # resulting in a bunch of objects landing on top of each other. - - # Set the IDs - sobjs_final[:].ECH_ORDER = order_vec[sobjs_final[:].ECH_ORDERINDX] - #for spec in sobjs_final: - # spec.ech_order = order_vec[spec.ECH_ORDERINDX] - sobjs_final.set_names() - - if show_trace: - viewer, ch = display.show_image(image*allmask) - - for spec in sobjs_trim: - color = 'red' if spec.ech_frac_was_fit else 'magenta' - ## Showing the final flux weighted centroiding from PCA predictions - display.show_trace(viewer, ch, spec.TRACE_SPAT, spec.NAME, color=color) - - for iobj in range(nobj_trim): - for iord in range(norders): - ## Showing PCA predicted locations before recomputing flux/gaussian weighted centroiding - display.show_trace(viewer, ch, pca_fits[:,iord, iobj], str(uni_frac[iobj]), color='yellow') - ## Showing the final traces from this routine - display.show_trace(viewer, ch, sobjs_final.TRACE_SPAT[iord].T, sobjs_final.NAME, color='cyan') - - # Labels for the points - text_final = [dict(type='text', args=(nspat / 2 -40, nspec / 2, 'final trace'), - kwargs=dict(color='cyan', fontsize=20))] - - text_pca = [dict(type='text', args=(nspat / 2 -40, nspec / 2 - 30, 'PCA fit'),kwargs=dict(color='yellow', fontsize=20))] - - text_fit = [dict(type='text', args=(nspat / 2 -40, nspec / 2 - 60, 'predicted'),kwargs=dict(color='red', fontsize=20))] - - text_notfit = [dict(type='text', args=(nspat / 2 -40, nspec / 2 - 90, 'originally found'),kwargs=dict(color='magenta', fontsize=20))] - - canvas = viewer.canvas(ch._chname) - canvas_list = text_final + text_pca + text_fit + text_notfit - canvas.add('constructedcanvas', canvas_list) - # TODO two things need to be debugged. 1) For objects which were found and traced, i don't think we should be updating the tracing with - # the PCA. This just adds a failutre mode. 2) The PCA fit is going wild for X-shooter. Debug that. - # Vette - for sobj in sobjs_final: - if not sobj.ready_for_extraction(): - msgs.error("Bad SpecObj. Can't proceed") - - return sobjs_final - - - -def objfind_QA(spat_peaks, snr_peaks, spat_vector, snr_vector, snr_thresh, qa_title, peak_gpm, - near_edge_bpm, nperslit_bpm, objfindQA_filename=None, show=False): - """ - Utility routine for making object finding QA plots. - - Args: - spat_peaks (`numpy.ndarray`_): - Array of locations in the spatial direction at which objects were - identified. Shape is ``(npeaks,)``, where ``npeaks`` is the number - of peaks identified. - snr_peaks (`numpy.ndarray`_): - S/N ratio in the spectral direction after collapsing along the - spectral direction, evaluated at the location of each spatial peak. - Shape must match ``spat_peaks``. - spat_vector (`numpy.ndarray`_): - A 1D array of spatial locations along the slit. Shape is - ``(nsamp,)``, where ``nsamp`` is the number of spatial pixels - defined by the slit edges. - snr_vector (`numpy.ndarray`_): - A 1D array with the S/N ratio sampled along the slit at each spatial - location (i.e., spectral direction has been smashed out) defined by - ``spat_vector``. Shape must match ``spat_vector``. - snr_thresh (:obj:`float`): - The threshold S/N ratio adopted by the object finding. - qa_title (:obj:`str`): - Title for the QA file plot. - peak_gpm (`numpy.ndarray`_): - Boolean array containing a good pixel mask for each peak indicating - whether it will be used as an object (True) or not (False). Shape - must match ``spat_peaks``. - near_edge_bpm (`numpy.ndarray`_): - A bad pixel mask (True is masked, False is unmasked) indicating - which objects are masked because they are near the slit edges. - Shape must match ``spat_peaks``. - nperslit_bpm (`numpy.ndarray`_): - A bad pixel mask (True is masked, False is unmasked) indicating - which objects are masked because they exceed the maximum number of - objects (see :func:`objs_in_slit` parameter ``nperslit``) that were - specified as being on this slit. - objfindQA_filename (:obj:`str`, optional): - Output filename for the QA plot. If None, plot is not saved. - show (:obj:`bool`, optional): - If True, show the plot as a matplotlib interactive plot. - - """ - - plt.plot(spat_vector, snr_vector, drawstyle='steps-mid', color='black', label = 'Collapsed SNR (FWHM convol)') - plt.hlines(snr_thresh,spat_vector.min(),spat_vector.max(), color='red',linestyle='--', - label='SNR_THRESH={:5.3f}'.format(snr_thresh)) - if np.any(peak_gpm): - plt.plot(spat_peaks[peak_gpm], snr_peaks[peak_gpm], color='red', marker='o', markersize=10.0, - mfc='lawngreen', fillstyle='full',linestyle='None', zorder = 10,label='{:d} Good Objects'.format(np.sum(peak_gpm))) - if np.any(near_edge_bpm): - plt.plot(spat_peaks[near_edge_bpm], snr_peaks[near_edge_bpm], color='red', marker='o', markersize=10.0, - mfc='cyan', fillstyle='full', linestyle='None', zorder = 10,label='{:d} Rejected: Near Edge'.format(np.sum(near_edge_bpm))) - if np.any(nperslit_bpm): - plt.plot(spat_peaks[nperslit_bpm], snr_peaks[nperslit_bpm], color='red', marker='o', markersize=10.0, - mfc='yellow', fillstyle='full', linestyle='None', zorder = 10,label='{:d} Rejected: Nperslit'.format(np.sum(nperslit_bpm))) - plt.legend() - plt.xlabel('Approximate Spatial Position (pixels)') - plt.ylabel('SNR') - plt.title(qa_title) - #plt.ylim(np.fmax(snr_vector.min(), -20.0), 1.3*snr_vector.max()) - fig = plt.gcf() - if show: - plt.show() - if objfindQA_filename is not None: - fig.savefig(objfindQA_filename, dpi=400) - plt.close('all') - -def get_fwhm(fwhm_in, nsamp, smash_peakflux, spat_fracpos, flux_smash_smth): - """ - Utility routine to measure the FWHM of an object trace from the spectrally - collapsed flux profile by determining the locations along the spatial - direction where this profile reaches have its peak value. - - Args: - fwhm_in (:obj:`float`): - Best guess for the FWHM of this object. - nsamp (:obj:`int`): - Number of pixels along the spatial direction. - smash_peakflux (:obj:`float`): - The peak flux in the 1d (spectrally collapsed) flux profile at the - object location. - spat_fracpos (:obj:`float`): - Fractional spatial position along the slit where the object is - located and at which the ``flux_smash_smth`` array has values - provided by ``smash_peakflux`` (see above and below). - flux_smash_smth (`numpy.ndarray`_): - A 1D array with the flux averaged along the spectral direction at - each location along the slit in the spatial direction location. - Shape is ``(nsamp,)``. - - Returns: - :obj:`float`: The FWHM determined from the object flux profile, unless - the FWHM could not be found from the profile, in which case the input - guess (``fwhm_in``) is simply returned. - """ - - # Determine the fwhm max - yhalf = 0.5*smash_peakflux - xpk = spat_fracpos*nsamp - x0 = int(np.rint(xpk)) - # TODO It seems we have two codes that do similar things, i.e. findfwhm in arextract.py. Could imagine having one - # Find right location where smash profile croses yhalf - if x0 < (int(nsamp) - 1): - ind_righ, = np.where(flux_smash_smth[x0:] < yhalf) - if len(ind_righ) > 0: - i2 = ind_righ[0] - if i2 == 0: - xrigh = None - else: - xarr_righ = x0 + np.array([i2 - 1, i2], dtype=float) - xrigh_int = scipy.interpolate.interp1d(flux_smash_smth[x0 + i2 - 1:x0 + i2 + 1], xarr_righ, - assume_sorted=False, bounds_error=False, - fill_value=(xarr_righ[0], xarr_righ[1])) - xrigh = xrigh_int([yhalf])[0] - else: - xrigh = None - else: - xrigh = None - # Find left location where smash profile crosses yhalf - if x0 > 0: - ind_left, = np.where(flux_smash_smth[0:np.fmin(x0 + 1, int(nsamp) - 1)] < yhalf) - if len(ind_left) > 0: - i1 = (ind_left[::-1])[0] - if i1 == (int(nsamp) - 1): - xleft = None - else: - xarr_left = np.array([i1, i1 + 1], dtype=float) - xleft_int = scipy.interpolate.interp1d(flux_smash_smth[i1:i1 + 2], xarr_left, - assume_sorted=False, bounds_error=False, - fill_value=(xarr_left[0], xarr_left[1])) - xleft = xleft_int([yhalf])[0] - else: - xleft = None - else: - xleft = None - - # Set FWHM for the object - if (xleft is None) & (xrigh is None): - fwhm_measure = None - elif xrigh is None: - fwhm_measure = 2.0 * (xpk - xleft) - elif xleft is None: - fwhm_measure = 2.0 * (xrigh - xpk) - else: - fwhm_measure = (xrigh - xleft) - - if fwhm_measure is not None: - fwhm_out = np.sqrt(np.fmax(fwhm_measure ** 2 - fwhm_in ** 2, (fwhm_in / 2.0) ** 2)) # Set a floor of fwhm/2 on fwhm - else: - fwhm_out = fwhm_in - - return fwhm_out - - -def objs_in_slit(image, ivar, thismask, slit_left, slit_righ, - inmask=None, fwhm=3.0, - sigclip_smash=5.0, use_user_fwhm=False, boxcar_rad=7., - maxdev=2.0, spec_min_max=None, hand_extract_dict=None, std_trace=None, - ncoeff=5, nperslit=None, snr_thresh=10.0, trim_edg=(5,5), - extract_maskwidth=4.0, specobj_dict=None, find_min_max=None, - show_peaks=False, show_fits=False, show_trace=False, - debug_all=False, qa_title='objfind', objfindQA_filename=None): - """ - Find the location of objects in a slitmask slit or a echelle order. - - The algorithm for this function is: - - - Rectify the image by extracting along the edge traces. - - - Compute the sigma-clipped mean spatial profile and its variance by - collapsing the image along the spectral direction to constuct a S/N - spatial profile of the slit/order. - - - Smooth the S/N profile by a Gaussian with the provided FWHM (see - ``fwhm``) and detect peaks in the smoothed profile using - :func:`~pypeit.core.arc.detect_lines`. Ignore peaks found near the - slit edges, and limit the number of peaks to the number requested (see - ``nperslit``). - - - Instantiate a :class:`~pypeit.specobj.SpecObj` object for each valid - detection, construct preliminary spectral traces for them, and estimate - the object spatial FWHM if one is not provided (see - ``use_user_fwhm``). - - - For automatically identified objects (i.e., not manual extractions), - improve the object trace by fitting the spatial position of the peak - as a function of wavelength. For manual apertures, use either the - form of the brightest object on the slit, the trace of a standard star - (see ``std_trace``), or the left edge trace to set the trace for the - object. Finally, remove automatically identified objects that overlap - with manually defined extraction apertures. - - At the end of this function, the list of objects is ready for extraction. - - **Revision History:** - - - 10-Mar-2005 -- First version written by D. Schlegel, LBL - - 2005-2018 -- Improved by J. F. Hennawi and J. X. Prochaska - - 23-June-2018 -- Ported to python by J. F. Hennawi and significantly - improved - - 01-Feb-2022 -- Skymask stripped out by JXP - - Args: - image (`numpy.ndarray`_): - (Floating-point) Image to use for object search with shape (nspec, - nspat). The first dimension (nspec) is spectral, and second - dimension (nspat) is spatial. Note this image can either have the - sky background in it, or have already been sky subtracted. Object - finding works best on sky-subtracted images. Ideally, object finding - is run in another routine, global sky-subtraction performed, and - then this code should be run. However, it is also possible to run - this code on non-sky-subtracted images. - ivar (`numpy.ndarray`_): - Floating-point inverse variance image for the input image. Shape - must match ``image``, (nspec, nspat). - thismask (`numpy.ndarray`_): - Boolean mask image selecting pixels associated with the slit/order - to search for objects on (True means on the slit/order). Shape must - match ``image``. - slit_left (`numpy.ndarray`_): - Left boundary of a single slit/orders to be extracted (given as - floating point pixels). Shape is (nspec,). + Left boundary of a single slit/orders to be extracted (given as + floating point pixels). Shape is (nspec,). slit_righ (`numpy.ndarray`_): Right boundary of a single slit/orders to be extracted (given as floating point pixels). Shape is (nspec,). @@ -2412,8 +1764,7 @@ def objs_in_slit(image, ivar, thismask, slit_left, slit_righ, flux_sum_smash = np.sum((image_rect*gpm_sigclip)[find_min_max_out[0]:find_min_max_out[1]], axis=0) flux_smash = flux_sum_smash*gpm_smash/(npix_smash + (npix_smash == 0.0)) flux_smash_mean, flux_smash_med, flux_smash_std = astropy.stats.sigma_clipped_stats( - flux_smash, mask=np.logical_not(gpm_smash), sigma_lower=3.0, sigma_upper=3.0 - ) + flux_smash, mask=np.logical_not(gpm_smash), sigma_lower=3.0, sigma_upper=3.0) flux_smash_recen = flux_smash - flux_smash_med # Return if none found and no hand extraction @@ -2524,6 +1875,9 @@ def objs_in_slit(image, ivar, thismask, slit_left, slit_righ, # If no standard trace is provided shift left slit boundary over to be initial trace else: # ToDO make this the average left and right boundary instead. That would be more robust. + # Print a status message for the first object + if iobj == 0: + msgs.info('Using slit edges as crutch for object tracing') sobjs[iobj].TRACE_SPAT = slit_left + xsize*sobjs[iobj].SPAT_FRACPOS sobjs[iobj].trace_spec = spec_vec @@ -2553,7 +1907,7 @@ def objs_in_slit(image, ivar, thismask, slit_left, slit_righ, if len(sobjs) > 0: msgs.info('Fitting the object traces') # Note the transpose is here to pass in the TRACE_SPAT correctly. - xinit_fweight = np.copy(sobjs.TRACE_SPAT.T) + xinit_fweight = np.copy(sobjs.TRACE_SPAT.T).astype(float) spec_mask = (spec_vec >= spec_min_max_out[0]) & (spec_vec <= spec_min_max_out[1]) trc_inmask = np.outer(spec_mask, np.ones(len(sobjs), dtype=bool)) xfit_fweight = fit_trace(image, xinit_fweight, ncoeff, bpm=np.invert(inmask), maxshift=1., @@ -2671,7 +2025,7 @@ def objs_in_slit(image, ivar, thismask, slit_left, slit_righ, # Sort objects according to their spatial location nobj = len(sobjs) spat_pixpos = sobjs.SPAT_PIXPOS - sobjs = sobjs[spat_pixpos.argsort()] + sobjs = sobjs[spat_pixpos.argsort(kind='stable')] # Assign integer objids sobjs.OBJID = np.arange(nobj) + 1 diff --git a/pypeit/core/fitting.py b/pypeit/core/fitting.py index 7f4346ebe4..eddc49ec07 100644 --- a/pypeit/core/fitting.py +++ b/pypeit/core/fitting.py @@ -943,7 +943,7 @@ def iterfit(xdata, ydata, invvar=None, inmask=None, upper=5, lower=5, x2=None, outmask = True else: outmask = np.ones(invvar.shape, dtype='bool') - xsort = xdata.argsort() + xsort = xdata.argsort(kind='stable') maskwork = (outmask & inmask & (invvar > 0.0))[xsort] # `maskwork` is in xsort order if 'oldset' in kwargs_bspline: sset = kwargs_bspline['oldset'] diff --git a/pypeit/core/flat.py b/pypeit/core/flat.py index 1c104d1d96..bac66b4c9a 100644 --- a/pypeit/core/flat.py +++ b/pypeit/core/flat.py @@ -16,44 +16,8 @@ from IPython import embed from pypeit import msgs -from pypeit.core import parse -from pypeit.core import pixels -from pypeit.core import tracewave from pypeit.core import coadd from pypeit import utils -from pypeit.core import pydl - -# TODO: Put this in utils -def linear_interpolate(x1, y1, x2, y2, x): - r""" - Interplate or extrapolate between two points. - - Given a line defined two points, :math:`(x_1,y_1)` and - :math:`(x_2,y_2)`, return the :math:`y` value of a new point on - the line at coordinate :math:`x`. - - This function is meant for speed. No type checking is performed and - the only check is that the two provided ordinate coordinates are not - numerically identical. By definition, the function will extrapolate - without any warning. - - Args: - x1 (:obj:`float`): - First abscissa position - y1 (:obj:`float`): - First ordinate position - x2 (:obj:`float`): - Second abscissa position - y3 (:obj:`float`): - Second ordinate position - x (:obj:`float`): - Abcissa for new value - - Returns: - :obj:`float`: Interpolated/extrapolated value of ordinate at - :math:`x`. - """ - return y1 if np.isclose(x1,x2) else y1 + (x-x1)*(y2-y1)/(x2-x1) # TODO: Make this function more general and put it in utils. @@ -103,7 +67,7 @@ def sorted_flat_data(data, coo, gpm=None): # np.argsort sorts the data over the last axis. To avoid coo[gpm] # returning an array (which will happen if the gpm is not provided # as an argument), all the arrays are explicitly flattened. - srt = np.argsort(coo[gpm].ravel()) + srt = np.argsort(coo[gpm].ravel(), kind='stable') coo_data = coo[gpm].ravel()[srt] flat_data = data[gpm].ravel()[srt] return gpm, srt, coo_data, flat_data @@ -277,7 +241,7 @@ def construct_illum_profile(norm_spec, spat_coo, slitwidth, spat_gpm=None, spat_ plt.show() # Include the rejected data in the full image good-pixel mask - _spat_gpm[_spat_gpm] = spat_gpm_data_raw[np.argsort(spat_srt)] + _spat_gpm[_spat_gpm] = spat_gpm_data_raw[np.argsort(spat_srt, kind='stable')] # Recreate the illumination profile data _spat_gpm, spat_srt, spat_coo_data, spat_flat_data_raw \ = sorted_flat_data(norm_spec, spat_coo, gpm=_spat_gpm) @@ -470,7 +434,7 @@ def poly_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, deg=3, slitmask_spatid = np.sort(slitmask_spatid[slitmask_spatid > 0]) # Create a spline between the raw data and the error - flxsrt = np.argsort(np.ravel(rawimg)) + flxsrt = np.argsort(np.ravel(rawimg), kind='stable') spl = scipy.interpolate.interp1d(np.ravel(rawimg)[flxsrt], np.ravel(rawivar)[flxsrt], kind='linear', bounds_error=False, fill_value=0.0, assume_sorted=True) modelmap = np.ones_like(rawimg) @@ -505,10 +469,120 @@ def poly_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg, deg=3, return modelmap, relscale +def tweak_slit_edges_gradient(left, right, spat_coo, norm_flat, maxfrac=0.1, debug=False): + r""" Adjust slit edges based on the gradient of the normalized + flat-field illumination profile. + + Args: + left (`numpy.ndarray`_): + Array with the left slit edge for a single slit. Shape is + :math:`(N_{\rm spec},)`. + right (`numpy.ndarray`_): + Array with the right slit edge for a single slit. Shape + is :math:`(N_{\rm spec},)`. + spat_coo (`numpy.ndarray`_): + Spatial pixel coordinates in fractions of the slit width + at each spectral row for the provided normalized flat + data. Coordinates are relative to the left edge (with the + left edge at 0.). Shape is :math:`(N_{\rm flat},)`. + Function assumes the coordinate array is sorted. + norm_flat (`numpy.ndarray`_) + Normalized flat data that provide the slit illumination + profile. Shape is :math:`(N_{\rm flat},)`. + maxfrac (:obj:`float`, optional): + The maximum fraction of the slit width that the slit edge + can be adjusted by this algorithm. If ``maxfrac = 0.1``, + this means the maximum change in the slit width (either + narrowing or broadening) is 20% (i.e., 10% for either + edge). + debug (:obj:`bool`, optional): + If True, the function will output plots to test if the + fitting is working correctly. + + Returns: + tuple: Returns six objects: + + - The threshold used to set the left edge + - The fraction of the slit that the left edge is shifted to + the right + - The adjusted left edge + - The threshold used to set the right edge + - The fraction of the slit that the right edge is shifted to + the left + - The adjusted right edge + """ + # Check input + nspec = len(left) + if len(right) != nspec: + msgs.error('Input left and right traces must have the same length!') + + # Median slit width + slitwidth = np.median(right - left) + + # Calculate the gradient of the normalized flat profile + grad_norm_flat = np.gradient(norm_flat) + # Smooth with a Gaussian kernel + # The standard deviation of the kernel is set to be one detector pixel. Since the norm_flat array is oversampled, + # we need to set the kernel width (sig_res) to be the oversampling factor. + sig_res = norm_flat.size / slitwidth + # The scipy.ndimage module is faster than the astropy convolution module + grad_norm_flat_smooth = scipy.ndimage.gaussian_filter1d(grad_norm_flat, sig_res, mode='nearest') + + # Find the location of the minimum/maximum gradient - this is the amount of shift required + left_shift = spat_coo[np.argmax(grad_norm_flat_smooth)] + right_shift = spat_coo[np.argmin(grad_norm_flat_smooth)]-1.0 + + # Check if the shift is within the allowed range + if np.abs(left_shift) > maxfrac: + msgs.warn('Left slit edge shift of {0:.1f}% exceeds the maximum allowed of {1:.1f}%'.format( + 100*left_shift, 100*maxfrac) + msgs.newline() + + 'The left edge will not be tweaked.') + left_shift = 0.0 + else: + msgs.info('Tweaking left slit boundary by {0:.1f}%'.format(100 * left_shift) + + ' ({0:.2f} pixels)'.format(left_shift * slitwidth)) + if np.abs(right_shift) > maxfrac: + msgs.warn('Right slit edge shift of {0:.1f}% exceeds the maximum allowed of {1:.1f}%'.format( + 100*right_shift, 100*maxfrac) + msgs.newline() + + 'The right edge will not be tweaked.') + right_shift = 0.0 + else: + msgs.info('Tweaking right slit boundary by {0:.1f}%'.format(100 * right_shift) + + ' ({0:.2f} pixels)'.format(right_shift * slitwidth)) + + # Calculate the tweak for the left edge + new_left = left + left_shift * slitwidth + new_right = right + right_shift * slitwidth + + # Calculate the value of the threshold at the new slit edges + left_thresh = np.interp(left_shift, spat_coo, norm_flat) + right_thresh = np.interp(1+right_shift, spat_coo, norm_flat) + + if debug: + plt.subplot(211) + plt.plot(spat_coo, norm_flat, 'k-') + plt.axvline(0.0, color='b', linestyle='-', label='initial') + plt.axvline(1.0, color='b', linestyle='-') + plt.axvline(left_shift, color='g', linestyle='-', label='tweak (gradient)') + plt.axvline(1+right_shift, color='g', linestyle='-') + plt.axhline(left_thresh, xmax=0.5, color='lightgreen', linewidth=3.0, zorder=10) + plt.axhline(right_thresh, xmin=0.5, color='lightgreen', linewidth=3.0, zorder=10) + plt.legend() + plt.subplot(212) + plt.plot(spat_coo, grad_norm_flat, 'k-') + plt.plot(spat_coo, grad_norm_flat_smooth, 'm-') + plt.axvline(0.0, color='b', linestyle='-') + plt.axvline(1.0, color='b', linestyle='-') + plt.axvline(left_shift, color='g', linestyle='-') + plt.axvline(1+right_shift, color='g', linestyle='-') + plt.show() + return left_thresh, left_shift, new_left, right_thresh, right_shift, new_right + + # TODO: See pypeit/deprecated/flat.py for the previous version. We need # to continue to vet this algorithm to make sure there are no # unforeseen corner cases that cause errors. -def tweak_slit_edges(left, right, spat_coo, norm_flat, thresh=0.93, maxfrac=0.1, debug=False): +def tweak_slit_edges_threshold(left, right, spat_coo, norm_flat, thresh=0.93, maxfrac=0.1, debug=False): r""" Adjust slit edges based on the normalized slit illumination profile. @@ -614,10 +688,10 @@ def tweak_slit_edges(left, right, spat_coo, norm_flat, thresh=0.93, maxfrac=0.1, 100*maxfrac)) left_shift = maxfrac else: - left_shift = linear_interpolate(norm_flat[i], spat_coo[i], norm_flat[i+1], - spat_coo[i+1], left_thresh) + left_shift = utils.linear_interpolate(norm_flat[i], spat_coo[i], norm_flat[i+1], + spat_coo[i+1], left_thresh) msgs.info('Tweaking left slit boundary by {0:.1f}%'.format(100*left_shift) + - ' % ({0:.2f} pixels)'.format(left_shift*slitwidth)) + ' ({0:.2f} pixels)'.format(left_shift*slitwidth)) new_left += left_shift * slitwidth # ------------------------------------------------------------------ @@ -668,15 +742,15 @@ def tweak_slit_edges(left, right, spat_coo, norm_flat, thresh=0.93, maxfrac=0.1, 100*maxfrac)) right_shift = maxfrac else: - right_shift = 1-linear_interpolate(norm_flat[i-1], spat_coo[i-1], norm_flat[i], - spat_coo[i], right_thresh) + right_shift = 1-utils.linear_interpolate(norm_flat[i-1], spat_coo[i-1], norm_flat[i], + spat_coo[i], right_thresh) msgs.info('Tweaking right slit boundary by {0:.1f}%'.format(100*right_shift) + - ' % ({0:.2f} pixels)'.format(right_shift*slitwidth)) + ' ({0:.2f} pixels)'.format(right_shift*slitwidth)) new_right -= right_shift * slitwidth return left_thresh, left_shift, new_left, right_thresh, right_shift, new_right -#def flatfield(sciframe, flatframe, bpm=None, illum_flat=None, snframe=None, varframe=None): + def flatfield(sciframe, flatframe, varframe=None): r""" Field flatten the input image. diff --git a/pypeit/core/flexure.py b/pypeit/core/flexure.py index b1d94a02a2..487dd504bf 100644 --- a/pypeit/core/flexure.py +++ b/pypeit/core/flexure.py @@ -17,6 +17,7 @@ from astropy import stats from astropy import units from astropy.io import ascii +from astropy.table import Table import scipy.signal import scipy.optimize as opt from scipy import interpolate @@ -1218,7 +1219,8 @@ def calculate_image_phase(imref, imshift, gpm_ref=None, gpm_shift=None, maskval= if gpm_shift is None: gpm_shift = np.ones(imshift.shape, dtype=bool) if maskval is None else imshift != maskval # Get a crude estimate of the shift - shift = phase_cross_correlation(imref, imshift, reference_mask=gpm_ref, moving_mask=gpm_shift).astype(int) + shift, _, _ = phase_cross_correlation(imref, imshift, reference_mask=gpm_ref, moving_mask=gpm_shift) + shift = shift.astype(int) # Extract the overlapping portion of the images exref = imref.copy() exshf = imshift.copy() diff --git a/pypeit/core/flux_calib.py b/pypeit/core/flux_calib.py index b94e1bdf4d..66481a6e8a 100644 --- a/pypeit/core/flux_calib.py +++ b/pypeit/core/flux_calib.py @@ -31,6 +31,7 @@ from pypeit import dataPaths + # TODO: Put these in the relevant functions TINY = 1e-15 SN2_MAX = (20.0) ** 2 @@ -411,7 +412,7 @@ def get_standard_spectrum(star_type=None, star_mag=None, ra=None, dec=None): # Pull star spectral model from archive msgs.info("Getting archival standard spectrum") # Grab closest standard within a tolerance - std_dict = find_standard_file(ra, dec) + std_dict = find_standard_file(ra, dec,to_pkg='symlink') elif (star_mag is not None) and (star_type is not None): ## using vega spectrum @@ -431,7 +432,7 @@ def get_standard_spectrum(star_type=None, star_mag=None, ra=None, dec=None): elif 'PHOENIX' in star_type: msgs.info('Getting PHOENIX 10000 K, logg = 4.0 spectrum') ## Vega model from TSPECTOOL - vega_file = data.Paths.standards / 'PHOENIX_10000K_4p0.fits' + vega_file = dataPaths.standards.get_file_path('PHOENIX_10000K_4p0.fits') vega_data = table.Table.read(vega_file, format='fits') std_dict = dict(cal_file='PHOENIX_10000K_4p0', name=star_type, Vmag=star_mag, std_ra=ra, std_dec=dec) @@ -696,15 +697,9 @@ def sensfunc(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, If you have significant telluric absorption you should be using telluric.sensnfunc_telluric. default = 0.9 Returns: - Tuple: Returns: - - Returns - ------- - meta_table: `astropy.table.Table`_ - Table containing meta data for the sensitivity function - out_table: `astropy.table.Table`_ - Table containing the sensitivity function - + tuple: Returns the following: + - meta_table: `astropy.table.Table`_ Table containing meta data for the sensitivity function + - out_table: `astropy.table.Table`_ Table containing the sensitivity function """ wave_arr, counts_arr, ivar_arr, mask_arr, log10_blaze_func, nspec, norders = utils.spec_atleast_2d(wave, counts, counts_ivar, counts_mask) @@ -755,9 +750,9 @@ def sensfunc(wave, counts, counts_ivar, counts_mask, exptime, airmass, std_dict, return meta_table, out_table -def get_sensfunc_factor(wave, wave_zp, zeropoint, exptime, tellmodel=None, extinct_correct=False, - airmass=None, longitude=None, latitude=None, extinctfilepar=None, - extrap_sens=False): + +def get_sensfunc_factor(wave, wave_zp, zeropoint, exptime, tellmodel=None, delta_wave=None, extinct_correct=False, + airmass=None, longitude=None, latitude=None, extinctfilepar=None, extrap_sens=False): """ Get the final sensitivity function factor that will be multiplied into a spectrum in units of counts to flux calibrate it. This code interpolates the sensitivity function and can also multiply in extinction and telluric corrections. @@ -766,27 +761,30 @@ def get_sensfunc_factor(wave, wave_zp, zeropoint, exptime, tellmodel=None, extin Args: wave (float `numpy.ndarray`_): shape = (nspec,) - Senstivity + Wavelength vector for the spectrum to be flux calibrated wave_zp (float `numpy.ndarray`_): - Zerooint wavelength vector shape = (nsens,) + Zeropoint wavelength vector shape = (nsens,) zeropoint (float `numpy.ndarray`_): shape = (nsens,) - Zeropoint, i.e. sensitivity function + Zeropoint, i.e. sensitivity function exptime (float): - tellmodel (float `numpy.ndarray`_, optional): shape = (nspec,) - Apply telluric correction if it is passed it. Note this is deprecated. + Exposure time in seconds + tellmodel (float, `numpy.ndarray`_, optional): + Apply telluric correction if it is passed it (shape = (nspec,)). Note this is deprecated. + delta_wave (float, `numpy.ndarray`_, optional): + The wavelength sampling of the spectrum to be flux calibrated. extinct_correct (bool, optional) - If True perform an extinction correction. Deafult = False + If True perform an extinction correction. Default = False airmass (float, optional): - Airmass used if extinct_correct=True. This is required if extinct_correct=True + Airmass used if extinct_correct=True. This is required if extinct_correct=True longitude (float, optional): longitude in degree for observatory Required for extinction correction latitude: latitude in degree for observatory - Required for extinction correction + Required for extinction correction extinctfilepar (str): - [sensfunc][UVIS][extinct_file] parameter - Used for extinction correction + [sensfunc][UVIS][extinct_file] parameter + Used for extinction correction extrap_sens (bool, optional): Extrapolate the sensitivity function (instead of crashing out) @@ -796,10 +794,23 @@ def get_sensfunc_factor(wave, wave_zp, zeropoint, exptime, tellmodel=None, extin This quantity is defined to be sensfunc_interp/exptime/delta_wave. shape = (nspec,) """ - + # Initialise some variables zeropoint_obs = np.zeros_like(wave) wave_mask = wave > 1.0 # filter out masked regions or bad wavelengths - delta_wave = wvutils.get_delta_wave(wave, wave_mask) + if delta_wave is not None: + # Check that the delta_wave is the same size as the wave vector + if isinstance(delta_wave, float): + _delta_wave = delta_wave + elif isinstance(delta_wave, np.ndarray): + if wave.size != delta_wave.size: + msgs.error('The wavelength vector and delta_wave vector must be the same size') + _delta_wave = delta_wave + else: + msgs.warn('Invalid type for delta_wave - using a default value') + _delta_wave = wvutils.get_delta_wave(wave, wave_mask) + else: + # If delta_wave is not passed in, then we will use the native wavelength sampling of the spectrum + _delta_wave = wvutils.get_delta_wave(wave, wave_mask) # print(f'get_sensfunc_factor: {np.amin(wave_zp):.1f}, {np.amax(wave_zp):.1f}, ' # f'{np.amin(wave[wave_mask]):.1f}, {np.amax(wave[wave_mask]):.1f}') @@ -829,10 +840,10 @@ def get_sensfunc_factor(wave, wave_zp, zeropoint, exptime, tellmodel=None, extin # Did the user request a telluric correction? if tellmodel is not None: # This assumes there is a separate telluric key in this dict. + msgs.warn("Telluric corrections via this method are deprecated") msgs.info('Applying telluric correction') sensfunc_obs = sensfunc_obs * (tellmodel > 1e-10) / (tellmodel + (tellmodel < 1e-10)) - if extinct_correct: if longitude is None or latitude is None: msgs.error('You must specify longitude and latitude if we are extinction correcting') @@ -848,7 +859,7 @@ def get_sensfunc_factor(wave, wave_zp, zeropoint, exptime, tellmodel=None, extin # senstot is the conversion from N_lam to F_lam, and the division by exptime and delta_wave are to convert # the spectrum in counts/pixel into units of N_lam = counts/sec/angstrom - return senstot/exptime/delta_wave + return senstot/exptime/_delta_wave def counts2Nlam(wave, counts, counts_ivar, counts_mask, exptime, airmass, longitude, latitude, extinctfilepar): @@ -1302,6 +1313,7 @@ def Nlam_to_Flam(wave, zeropoint, zp_min=5.0, zp_max=30.0): factor[gpm] = np.power(10.0, -0.4*(zeropoint[gpm] - ZP_UNIT_CONST))/np.square(wave[gpm]) return factor + def Flam_to_Nlam(wave, zeropoint, zp_min=5.0, zp_max=30.0): r""" The factor that when multiplied into F_lam converts to N_lam, @@ -1390,7 +1402,7 @@ def zeropoint_to_throughput(wave, zeropoint, eff_aperture): """ eff_aperture_m2 = eff_aperture*units.m**2 - S_lam_units = 1e-17*units.erg/units.cm**2 + S_lam_units = PYPEIT_FLUX_SCALE*units.erg/units.cm**2 # Set the throughput to be -1 in places where it is not defined. throughput = np.full_like(zeropoint, -1.0) zeropoint_gpm = (zeropoint > 5.0) & (zeropoint < 30.0) & (wave > 1.0) diff --git a/pypeit/core/framematch.py b/pypeit/core/framematch.py index 4125f7011e..94200c405f 100644 --- a/pypeit/core/framematch.py +++ b/pypeit/core/framematch.py @@ -32,6 +32,7 @@ def __init__(self): ('lampoffflats', 'Flat-field exposure with lamps off used to remove ' 'persistence from lamp on flat exposures and/or thermal emission ' 'from the telescope and dome'), + ('slitless_pixflat', 'Flat-field exposure without slitmask used for pixel-to-pixel response'), ('scattlight', 'Frame (ideally with lots of counts) used to determine the scattered light model'), ('science', 'On-sky observation of a primary target'), ('standard', 'On-sky observation of a flux calibrator'), diff --git a/pypeit/core/gui/identify.py b/pypeit/core/gui/identify.py index 271abefdda..6a232579af 100644 --- a/pypeit/core/gui/identify.py +++ b/pypeit/core/gui/identify.py @@ -716,7 +716,7 @@ def get_results(self): def store_solution(self, final_fit, binspec, rmstol=0.15, force_save=False, wvcalib=None, multi=False, - fits_dicts=None, specdata=None, slits=None, + fits_dicts=None, specdata_multi=None, slits=None, lines_pix_arr=None, lines_wav_arr=None, lines_fit_ord=None, custom_wav=None, custom_wav_ind=None): """Check if the user wants to store this solution in the reid arxiv, when doing the wavelength solution @@ -737,8 +737,9 @@ def store_solution(self, final_fit, binspec, rmstol=0.15, Flag if the template has multiple slits/traces. fits_dict : list, optional List of dictionaries containing the _fitdict of previous calls, if multi-trace data - specdata : array, optional - Numpy array containing the flux information from all the traces + specdata_multi : array, optional + Numpy array containing the flux information from all the traces, if multiple traces are + being fit. wvcalib : :class:`pypeit.wavecalib.WaveCalib`, optional Wavelength solution lines_pix_arr : array, optional @@ -782,7 +783,7 @@ def store_solution(self, final_fit, binspec, rmstol=0.15, if ans == 'y': # Arxiv solution # prompt the user to give the orders that were used here - if '"echelle": true' in wvcalib.strpar: + if wvcalib is not None and '"echelle": true' in wvcalib.strpar: while True: try: print('') @@ -818,15 +819,15 @@ def store_solution(self, final_fit, binspec, rmstol=0.15, if make_arxiv != 'n': if multi: # check that specdata is defined - if specdata is None: + if specdata_multi is None: msgs.warn('Skipping arxiv save because arc line spectra are not defined by pypeit/scripts/identify.py') # check that the number of spectra in specdata is the same as the number of wvcalib solutions - elif specdata is not None and np.shape(specdata)[0] != len(wvcalib.wv_fits): + elif specdata_multi is not None and np.shape(specdata_multi)[0] != len(wvcalib.wv_fits): msgs.warn('Skipping arxiv save because there are not enough orders for full template') msgs.warn('To generate a valid arxiv to save, please rerun with the "--slits all" option.') else: - norder = np.shape(specdata)[0] - wavelengths = np.copy(specdata) + norder = np.shape(specdata_multi)[0] + wavelengths = np.copy(specdata_multi) for iord in range(norder): if fits_dicts is not None: fitdict = fits_dicts[iord] @@ -834,8 +835,8 @@ def store_solution(self, final_fit, binspec, rmstol=0.15, msgs.warn('skipping saving fits because fits_dicts is not defined by pypeit/scripts/identify.py') fitdict = None if fitdict is not None and fitdict['full_fit'] is not None: - wavelengths[iord,:] = fitdict['full_fit'].eval(np.arange(specdata[iord,:].size) / - (specdata[iord,:].size - 1)) + wavelengths[iord,:] = fitdict['full_fit'].eval(np.arange(specdata_multi[iord,:].size) / + (specdata_multi[iord,:].size - 1)) elif wvcalib is not None and wvcalib.wv_fits[iord] is None and iord in custom_wav_ind: wavelengths[iord,:] = custom_wav[np.where(iord == custom_wav_ind)[0]] else: @@ -858,7 +859,7 @@ def store_solution(self, final_fit, binspec, rmstol=0.15, wvarxiv_name = wvarxiv_name_new # Write the wvarxiv file - _specdata = specdata if specdata is not None else self.specdata + _specdata = specdata_multi if specdata_multi is not None else self.specdata order_vec = np.flip(order_vec, axis=0) if order_vec is not None else None wvutils.write_template(wavelengths, _specdata, binspec, './', wvarxiv_name, to_cache=True, order = order_vec, @@ -870,43 +871,44 @@ def store_solution(self, final_fit, binspec, rmstol=0.15, wvcalib.to_file(outfname, overwrite=True) msgs.info("A WaveCalib container was written to wvcalib.fits") - # Ask if overwrite the existing WVCalib file only if force_save=False, otherwise don't overwrite - ow_wvcalib = '' - if not force_save: - while ow_wvcalib != 'y' and ow_wvcalib != 'n': - print('') - msgs.warn('Do you want to overwrite existing Calibrations/WaveCalib*.fits file? ' + msgs.newline() + - 'NOTE: To use this WaveCalib file the user will need to delete the other files in Calibrations/ ' + msgs.newline() + - ' and re-run run_pypeit. ') - print('') - ow_wvcalib = input('Proceed with overwrite? (y/[n]): ') - - if ow_wvcalib == 'y': - wvcalib.to_file() - if multi: - slit_list_str = ''; slit_list = np.arange(np.shape(specdata)[0]) - for islit in slit_list: - if islit < len(slit_list) - 1: - slit_list_str += str(islit) + ',' - else: slit_list_str += str(islit) - - if slits: - print(' ') - msgs.info('Unflagging Slits from WaveCalib: ') - slits.mask = np.zeros(slits.nslits, dtype=slits.bitmask.minimum_dtype()) - slits.ech_order = order_vec - slits.to_file() - print(' ') - print(' ') - # ask to clean up the Calibrations directory only if force_save=False, otherwise don't clean up + # Ask if overwrite the existing WVCalib file only if force_save=False, otherwise don't overwrite + ow_wvcalib = '' if not force_save: - clean_calib = input('Clean up the Calibrations/ directory? This will delete all of the existing' - ' calibrations except the Arcs and WaveCalib files. y/[n]: ') - if clean_calib == 'y': - cal_root = Path('Calibrations').resolve() - for cal in ['Tilt', 'Flat', 'Edge', 'Slit']: - for f in cal_root.glob(f'{cal}*'): - f.unlink() + while ow_wvcalib != 'y' and ow_wvcalib != 'n': + print('') + msgs.warn('Do you want to overwrite existing Calibrations/WaveCalib*.fits file? ' + msgs.newline() + + 'NOTE: To use this WaveCalib file the user will need to delete the other files in Calibrations/ ' + msgs.newline() + + ' and re-run run_pypeit. ') + print('') + ow_wvcalib = input('Proceed with overwrite? (y/[n]): ') + + if ow_wvcalib == 'y': + wvcalib.to_file() + if multi: + slit_list_str = '' + slit_list = np.arange(np.shape(specdata_multi)[0]) + for islit in slit_list: + if islit < len(slit_list) - 1: + slit_list_str += str(islit) + ',' + else: slit_list_str += str(islit) + + if slits: + print(' '*10) + msgs.info('Unflagging Slits from WaveCalib: ') + slits.mask = np.zeros(slits.nslits, dtype=slits.bitmask.minimum_dtype()) + slits.ech_order = order_vec + slits.to_file() + print(' '*10) + print(' '*10) + # ask to clean up the Calibrations directory only if force_save=False, otherwise don't clean up + if not force_save: + clean_calib = input('Clean up the Calibrations/ directory? This will delete all of the existing' + ' calibrations except the Arcs and WaveCalib files. y/[n]: ') + if clean_calib == 'y': + cal_root = Path('Calibrations').resolve() + for cal in ['Tilt', 'Flat', 'Edge', 'Slit']: + for f in cal_root.glob(f'{cal}*'): + f.unlink() # Print some helpful information print("\n\nPlease visit the following site if you want to include your solution in PypeIt:") diff --git a/pypeit/core/procimg.py b/pypeit/core/procimg.py index 25a4826536..795a198446 100644 --- a/pypeit/core/procimg.py +++ b/pypeit/core/procimg.py @@ -813,6 +813,7 @@ def subtract_pattern(rawframe, datasec_img, oscansec_img, frequency=None, axis=1 frequency = np.mean(frq) # Perform the overscan subtraction for each amplifier + full_model = np.zeros_like(frame_orig) # Store the model pattern for all amplifiers in this array for aa, amp in enumerate(amps): # Get the frequency to use for this amplifier if isinstance(frequency, list): @@ -823,9 +824,9 @@ def subtract_pattern(rawframe, datasec_img, oscansec_img, frequency=None, axis=1 use_fr = frequency # Extract overscan - overscan, os_slice = rect_slice_with_mask(frame_orig, tmp_oscan, amp) + overscan, os_slice = rect_slice_with_mask(frame_orig.copy(), tmp_oscan, amp) # Extract overscan+data - oscandata, osd_slice = rect_slice_with_mask(frame_orig, tmp_oscan+tmp_data, amp) + oscandata, osd_slice = rect_slice_with_mask(frame_orig.copy(), tmp_oscan+tmp_data, amp) # Subtract the DC offset overscan -= np.median(overscan, axis=1)[:, np.newaxis] @@ -854,7 +855,7 @@ def subtract_pattern(rawframe, datasec_img, oscansec_img, frequency=None, axis=1 tmpamp = np.fft.rfft(overscan, axis=1) idx = (np.arange(overscan.shape[0]), np.argmax(np.abs(tmpamp), axis=1)) # Convert result to amplitude and phase - amps = (np.abs(tmpamp))[idx] * (2.0 / overscan.shape[1]) + ampls = (np.abs(tmpamp))[idx] * (2.0 / overscan.shape[1]) # STEP 2 - Using the model frequency, calculate how amplitude depends on pixel row (usually constant) # Use the above to as initial guess parameters for a chi-squared minimisation of the amplitudes @@ -877,7 +878,7 @@ def subtract_pattern(rawframe, datasec_img, oscansec_img, frequency=None, axis=1 try: # Now fit it popt, pcov = scipy.optimize.curve_fit( - cosfunc, cent[wgd], hist[wgd], p0=[amps[ii], 0.0], + cosfunc, cent[wgd], hist[wgd], p0=[ampls[ii], 0.0], bounds=([0, -np.inf],[np.inf, np.inf]) ) except ValueError: @@ -920,21 +921,15 @@ def subtract_pattern(rawframe, datasec_img, oscansec_img, frequency=None, axis=1 model_pattern[ii, :] = cosfunc_full(xdata_all, amp_mod[ii], frq_mod[ii], popt[0]) # Estimate the improvement of the effective read noise - tmp = outframe.copy() - tmp[osd_slice] -= model_pattern - mod_oscan, _ = rect_slice_with_mask(tmp, tmp_oscan, amp) - old_ron = astropy.stats.sigma_clipped_stats(overscan, sigma=5)[-1] - new_ron = astropy.stats.sigma_clipped_stats(overscan-mod_oscan, sigma=5)[-1] + full_model[osd_slice] = model_pattern + old_ron = astropy.stats.sigma_clipped_stats(overscan, sigma=5, stdfunc='mad_std')[-1] + new_ron = astropy.stats.sigma_clipped_stats(overscan-full_model[os_slice], sigma=5, stdfunc='mad_std')[-1] msgs.info(f'Effective read noise of amplifier {amp} reduced by a factor of {old_ron/new_ron:.2f}x') - # Subtract the model pattern from the full datasec - outframe[osd_slice] -= model_pattern - # Transpose if the input frame if applied along a different axis if axis == 0: - outframe = outframe.T - # Return the result - return outframe + return (outframe - full_model).T + return outframe - full_model def pattern_frequency(frame, axis=1): @@ -1162,7 +1157,9 @@ def base_variance(rn_var, darkcurr=None, exptime=None, proc_var=None, count_scal - :math:`C` is the observed number of sky + object counts, - :math:`s=s\prime / N_{\rm frames}` is a scale factor derived from the (inverse of the) flat-field frames plus the number - of frames contributing to the object counts (see ``count_scale``), + of frames contributing to the object counts plus a scaling + factor applied if the counts of each frame are scaled to the + mean counts of all frames (see ``count_scale``), - :math:`D` is the dark current in electrons per **hour** (see ``darkcurr``), - :math:`t_{\rm exp}` is the effective exposure time in seconds (see @@ -1234,8 +1231,9 @@ def base_variance(rn_var, darkcurr=None, exptime=None, proc_var=None, count_scal A scale factor that *has already been applied* to the provided counts. It accounts for the number of frames contributing to the provided counts, and the relative throughput factors that - can be measured from flat-field frames. For example, if the image - has been flat-field corrected, this is the inverse of the flat-field counts. + can be measured from flat-field frames plus a scaling factor applied + if the counts of each frame are scaled to the mean counts of all frames. + For example, if the image has been flat-field corrected, this is the inverse of the flat-field counts. If None, set to 1. If a single float, assumed to be constant across the full image. If an array, the shape must match ``rn_var``. The variance will be 0 wherever :math:`s \leq 0`, modulo the provided ``noise_floor``. @@ -1292,7 +1290,9 @@ def variance_model(base, counts=None, count_scale=None, noise_floor=None): - :math:`C` is the observed number of sky + object counts, - :math:`s=s\prime / N_{\rm frames}` is a scale factor derived from the (inverse of the) flat-field frames plus the number - of frames contributing to the object counts (see ``count_scale``), + of frames contributing to the object counts plus a scaling factor + applied if the counts of each frame are scaled to the mean counts + of all frames (see ``count_scale``), - :math:`D` is the dark current in electrons per **hour**, - :math:`t_{\rm exp}` is the effective exposure time in seconds, - :math:`V_{\rm rn}` is the detector readnoise variance (i.e., @@ -1352,7 +1352,9 @@ def variance_model(base, counts=None, count_scale=None, noise_floor=None): A scale factor that *has already been applied* to the provided counts; see :math:`s` in the equations above. It accounts for the number of frames contributing to the provided counts, and - the relative throughput factors that can be measured from flat-field frames. + the relative throughput factors that can be measured from flat-field frames + plus a scaling factor applied if the counts of each frame are + scaled to the mean counts of all frames. For example, if the image has been flat-field corrected, this is the inverse of the flat-field counts. If None, no scaling is expected, meaning ``counts`` are exactly the observed detector counts. If a single @@ -1395,3 +1397,51 @@ def variance_model(base, counts=None, count_scale=None, noise_floor=None): return var +def nonlinear_counts(counts, ampimage, nonlinearity_coeffs): + r""" + Apply a nonlinearity correction to the provided counts. + + The nonlinearity correction is applied to the provided ``counts`` using the + hard-coded parameters in the provided ``nonlinearity_coeffs``. The + correction is applied to the provided ``counts`` using the following + equation: + + .. math:: + + C_{\rm corr} = C \left[ 1 + a_i C \right] + + where :math:`C` is the provided counts, :math:`C_{\rm corr}` is the corrected counts + :math:`a_i` are the provided coefficients (one for each amplifier). + + Parameters + ---------- + counts : `numpy.ndarray`_ + Array with the counts to correct. + ampimage : `numpy.ndarray`_ + Array with the amplifier image. This is used to determine the + amplifier-dependent nonlinearity correction coefficients. + nonlinearity_coeffs : `numpy.ndarray`_ + Array with the nonlinearity correction coefficients. The shape of the + array must be :math:`(N_{\rm amp})`, where :math:`N_{\rm amp}` is the + number of amplifiers. The coefficients are applied to the counts using + the equation above. + + Returns + ------- + corr_counts : + Array with the corrected counts. + """ + msgs.info('Applying a non-linearity correction to the counts.') + # Check the input + if counts.shape != ampimage.shape: + msgs.error('Counts and amplifier image have different shapes.') + _nonlinearity_coeffs = np.asarray(nonlinearity_coeffs) + # Setup the output array + corr_counts = counts.copy() + unqamp = np.unique(ampimage) + for uu in range(unqamp.size): + thisamp = unqamp[uu] + indx = (ampimage == thisamp) + corr_counts[indx] = counts[indx] * (1. + _nonlinearity_coeffs[thisamp]*counts[indx]) + # Apply the correction + return corr_counts diff --git a/pypeit/core/pydl.py b/pypeit/core/pydl.py index b7916dfbd4..97c093ccea 100644 --- a/pypeit/core/pydl.py +++ b/pypeit/core/pydl.py @@ -55,7 +55,7 @@ def djs_maskinterp1(yval, mask, xval=None, const=False): if igood[ngood-1] != ny-1: ynew[igood[ngood-1]+1:ny] = ynew[igood[ngood-1]] else: - ii = xval.argsort() + ii = xval.argsort(kind='stable') ibad = (mask[ii] != 0).nonzero()[0] igood = (mask[ii] == 0).nonzero()[0] ynew[ii[ibad]] = np.interp(xval[ii[ibad]], xval[ii[igood]], @@ -882,7 +882,7 @@ def djs_reject(data, model, outmask=None, inmask=None, # Test if too many points rejected in this group. # if np.sum(badness[jj] != 0) > maxrej1[iloop]: - isort = badness[jj].argsort() + isort = badness[jj].argsort(kind='stable') # # Make the following points good again. # @@ -1656,7 +1656,7 @@ def spherematch(ra1, dec1, ra2, dec2, matchlength, chunksize=None, omatch1 = np.array(match1) omatch2 = np.array(match2) odistance12 = np.array(distance12) - s = odistance12.argsort() + s = odistance12.argsort(kind='stable') # # Retain only desired matches # diff --git a/pypeit/core/skysub.py b/pypeit/core/skysub.py index 91acc0f2bc..c826c444a9 100644 --- a/pypeit/core/skysub.py +++ b/pypeit/core/skysub.py @@ -136,15 +136,18 @@ def global_skysub(image, ivar, tilts, thismask, slit_left, slit_righ, inmask=Non msgs.error("Type of inmask should be bool and is of type: {:}".format(inmask.dtype)) # Sky pixels for fitting - gpm = thismask & (ivar > 0.0) & inmask & np.logical_not(edgmask) & np.isfinite(image) & np.isfinite(ivar) + gpm = thismask & (ivar > 0.0) & inmask & np.logical_not(edgmask) \ + & np.isfinite(image) & np.isfinite(ivar) bad_pixel_frac = np.sum(thismask & np.logical_not(gpm))/np.sum(thismask) if bad_pixel_frac > max_mask_frac: - msgs.warn('This slit/order has {:5.3f}% of the pixels masked, which exceeds the threshold of {:f}%. '.format(100.0*bad_pixel_frac, 100.0*max_mask_frac) - + msgs.newline() + 'There is likely a problem with this slit. Giving up on global sky-subtraction.') + msgs.warn(f'This slit/order has {100.0*bad_pixel_frac:.3f}% of the pixels masked, which ' + f'exceeds the threshold of {100.0*max_mask_frac:.3f}%.' + + msgs.newline() + 'There is likely a problem with this slit. Giving up on ' + 'global sky-subtraction.') return np.zeros(np.sum(thismask)) # Sub arrays - isrt = np.argsort(piximg[thismask]) + isrt = np.argsort(piximg[thismask], kind='stable') pix = piximg[thismask][isrt] sky = image[thismask][isrt] sky_ivar = ivar[thismask][isrt] @@ -245,7 +248,6 @@ def global_skysub(image, ivar, tilts, thismask, slit_left, slit_righ, inmask=Non return ythis - def skyoptimal(piximg, data, ivar, oprof, sigrej=3.0, npoly=1, spatial_img=None, fullbkpt=None): """ Utility routine used by local_skysub_extract that performs the joint b-spline fit for sky-background @@ -283,7 +285,7 @@ def skyoptimal(piximg, data, ivar, oprof, sigrej=3.0, npoly=1, spatial_img=None, fullbkpt : `numpy.ndarray`_, optional A 1d float array containing the breakpoints to be used for the B-spline fit. The breakpoints are arranged in the spectral - direction, i.e. along the directino of the piximg independent + direction, i.e. along the direction of the piximg independent variable. Returns @@ -299,7 +301,7 @@ def skyoptimal(piximg, data, ivar, oprof, sigrej=3.0, npoly=1, spatial_img=None, whether a pixel is good (True) or was masked (False). """ - sortpix = piximg.argsort() + sortpix = piximg.argsort(kind='stable') nx = data.size nc = oprof.shape[0] @@ -325,7 +327,7 @@ def skyoptimal(piximg, data, ivar, oprof, sigrej=3.0, npoly=1, spatial_img=None, indx, = np.where(ivar[sortpix] > 0.0) ngood = indx.size good = sortpix[indx] - good = good[piximg[good].argsort()] + good = good[piximg[good].argsort(kind='stable')] relative, = np.where(relative_mask[good]) gpm = np.zeros(piximg.shape, dtype=bool) @@ -425,7 +427,7 @@ def optimal_bkpts(bkpts_optimal, bsp_min, piximg, sampmask, samp_frac=0.80, """ pix = piximg[sampmask] - isrt = pix.argsort() + isrt = pix.argsort(kind='stable') pix = pix[isrt] piximg_min = pix.min() piximg_max = pix.max() @@ -540,7 +542,7 @@ def optimal_bkpts(bkpts_optimal, bsp_min, piximg, sampmask, samp_frac=0.80, def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, slit_left, slit_righ, sobjs, ingpm=None, bkg_redux_global_sky=None, - fwhmimg=None, spat_pix=None, adderr=0.01, bsp=0.6, + fwhmimg=None, flatimg=None, spat_pix=None, adderr=0.01, bsp=0.6, trim_edg=(3,3), std=False, prof_nsigma=None, niter=4, extract_good_frac=0.005, sigrej=3.5, bkpts_optimal=True, debug_bkpts=False, force_gauss=False, sn_gauss=4.0, model_full_slit=False, @@ -586,9 +588,12 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, Global sky model produced by global_skysub without the background subtraction. If sciimg is an A-B image, then this is the global sky modeled from the A image, while `global_sky` is the global modeled from the A-B image. This is None if the sciimg is not an A-B image. - fwhmimg : `numpy.ndarray`_, None, optional: + fwhmimg : `numpy.ndarray`_, None, optional Floating-point image containing the modeled spectral FWHM (in pixels) at every pixel location. Must have the same shape as ``sciimg``, :math:`(N_{\rm spec}, N_{\rm spat})`. + flatimg : `numpy.ndarray`_, None, optional + Image containing the model of the flat field. If None, the + blaze function will not be calculated. spat_pix: `numpy.ndarray`_, optional Image containing the spatial location of pixels. If not input, it will be computed from ``spat_img = @@ -700,7 +705,8 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, A scale factor, :math:`s`, that *has already been applied* to the provided science image. It accounts for the number of frames contributing to the provided counts, and the relative throughput factors that can be measured - from flat-field frames. For example, if the image has been flat-field + from flat-field frames plus a scaling factor applied if the counts of each frame are + scaled to the mean counts of all frames. For example, if the image has been flat-field corrected, this is the inverse of the flat-field counts. If None, set to 1. If a single float, assumed to be constant across the full image. If an array, the shape must match ``base_var``. The variance will be 0 @@ -858,12 +864,12 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, objmask = ((spat_img >= (trace - 2.0 * sobjs[iobj].BOX_RADIUS)) & (spat_img <= (trace + 2.0 * sobjs[iobj].BOX_RADIUS))) # Boxcar extract.extract_boxcar(sciimg-skyimage, modelivar, (outmask & objmask), waveimg, - skyimage, sobjs[iobj], fwhmimg=fwhmimg, base_var=base_var, + skyimage, sobjs[iobj], fwhmimg=fwhmimg, flatimg=flatimg, base_var=base_var, count_scale=count_scale, noise_floor=adderr) # Optimal extract.extract_optimal(sciimg-skyimage, modelivar, (outmask & objmask), waveimg, skyimage, thismask, last_profile, sobjs[iobj], - fwhmimg=fwhmimg, base_var=base_var, count_scale=count_scale, + fwhmimg=fwhmimg, flatimg=flatimg, base_var=base_var, count_scale=count_scale, noise_floor=adderr) # If the extraction is bad do not update if sobjs[iobj].OPT_MASK is not None: @@ -911,7 +917,7 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, isub, = np.where(localmask.flatten()) #sortpix = (piximg.flat[isub]).argsort() obj_profiles_flat = obj_profiles.reshape(nspec * nspat, objwork) - skymask = outmask & np.invert(edgmask) + skymask = outmask & np.logical_not(edgmask) sky_bmodel, obj_bmodel, outmask_opt = skyoptimal( piximg.flat[isub], sciimg.flat[isub], (modelivar * skymask).flat[isub], obj_profiles_flat[isub, :], spatial_img=spatial_img.flat[isub], @@ -970,10 +976,12 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, else: msgs.warn('ERROR: Bspline sky subtraction failed after 4 iterations of bkpt spacing') msgs.warn(' Moving on......') - obj_profiles = np.zeros_like(obj_profiles) + # obj_profiles = np.zeros_like(obj_profiles) isub, = np.where(localmask.flatten()) # Just replace with the global sky skyimage.flat[isub] = global_sky.flat[isub] + if iiter == niter: + msgs.warn('WARNING: LOCAL SKY SUBTRACTION NOT PERFORMED') outmask_extract = outmask if use_2dmodel_mask else inmask @@ -994,12 +1002,12 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, objmask = ((spat_img >= (trace - 2.0 * sobjs[iobj].BOX_RADIUS)) & (spat_img <= (trace + 2.0 * sobjs[iobj].BOX_RADIUS))) extract.extract_optimal(sciimg-skyimage, modelivar * thismask, (outmask_extract & objmask), waveimg, extract_sky, thismask, this_profile, sobjs[iobj], - fwhmimg=fwhmimg, base_var=base_var, count_scale=count_scale, + fwhmimg=fwhmimg, flatimg=flatimg, base_var=base_var, count_scale=count_scale, noise_floor=adderr) # Boxcar extract.extract_boxcar(sciimg-skyimage, modelivar*thismask, (outmask_extract & objmask), waveimg, extract_sky, sobjs[iobj], - fwhmimg=fwhmimg, base_var=base_var, + fwhmimg=fwhmimg, flatimg=flatimg, base_var=base_var, count_scale=count_scale, noise_floor=adderr) sobjs[iobj].min_spat = min_spat sobjs[iobj].max_spat = max_spat @@ -1051,7 +1059,7 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, global_sky, left, right, slitmask, sobjs, spat_pix=None, bkg_redux_global_sky=None, - fit_fwhm=False, + fit_fwhm=False, fwhmimg=None, flatimg=None, min_snr=2.0, bsp=0.6, trim_edg=(3,3), std=False, prof_nsigma=None, use_2dmodel_mask=True, niter=4, sigrej=3.5, bkpts_optimal=True, @@ -1059,7 +1067,7 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, model_noise=True, debug_bkpts=False, show_profile=False, show_resids=False, show_fwhm=False, adderr=0.01, base_var=None, count_scale=None, no_local_sky:bool=False): - """ + r""" Perform local sky subtraction, profile fitting, and optimal extraction slit by slit. Objects are sky/subtracted extracted in order of the highest average (across all orders) S/N ratio object first, and then for a given @@ -1082,7 +1090,7 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, Parameters ---------- sciimg : `numpy.ndarray`_ - science image, usually with a global sky subtracted. + Science image, usually with a global sky subtracted. shape = (nspec, nspat) sciivar : `numpy.ndarray`_ inverse variance of science image. @@ -1124,6 +1132,12 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, fit_fwhm: bool, optional if True, perform a fit to the FWHM of the object profiles to use for non-detected sources + fwhmimg : `numpy.ndarray`_, None, optional + Floating-point image containing the modeled spectral FWHM (in pixels) at every pixel location. + Must have the same shape as ``sciimg``, :math:`(N_{\rm spec}, N_{\rm spat})`. + flatimg : `numpy.ndarray`_, None, optional + Image containing the model of the flat field. If None, the + blaze function will not be calculated. min_snr: float, optional FILL IN bsp : float, default = 0.6 @@ -1167,9 +1181,9 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, fullbkpt = bset.breakpoints force_gauss : bool, default = False - If True, a Gaussian profile will always be assumed for the - optimal extraction using the FWHM determined from object finding (or provided by the user) for the spatial - profile. + If True, a Gaussian profile will always be assumed for the optimal + extraction using the FWHM determined from object finding (or provided by + the user) for the spatial profile. sn_gauss : int or float, default = 4.0 The signal to noise threshold above which optimal extraction with non-parametric b-spline fits to the objects spatial @@ -1224,7 +1238,8 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, A scale factor that *has already been applied* to the provided science image. It accounts for the number of frames contributing to the provided counts, and the relative throughput factors that can be measured - from flat-field frames. For example, if the image has been flat-field corrected, + from flat-field frames plus a scaling factor applied if the counts of each frame are + scaled to the mean counts of all frames. For example, if the image has been flat-field corrected, this is the inverse of the flat-field counts. If None, set to 1. If a single float, assumed to be constant across the full image. If an array, the shape must match ``base_var``. The variance will be 0 wherever this @@ -1316,11 +1331,11 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, # Compute the average SNR and find the brightest object snr_bar = np.mean(order_snr,axis=0) - srt_obj = snr_bar.argsort()[::-1] + srt_obj = snr_bar.argsort(kind='stable')[::-1] ibright = srt_obj[0] # index of the brightest object # Now extract the orders in descending order of S/N for the brightest object - srt_order_snr = order_snr[:,ibright].argsort()[::-1] + srt_order_snr = order_snr[:,ibright].argsort(kind='stable')[::-1] fwhm_here = np.zeros(norders) fwhm_was_fit = np.zeros(norders,dtype=bool) @@ -1410,17 +1425,18 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, # Local sky subtraction and extraction skymodel[thismask], _this_bkg_redux_skymodel, objmodel[thismask], ivarmodel[thismask], extractmask[thismask] \ = local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, - left[:,iord], right[:,iord], sobjs[thisobj], - bkg_redux_global_sky=bkg_redux_global_sky, - spat_pix=spat_pix, ingpm=inmask, std=std, bsp=bsp, - trim_edg=trim_edg, prof_nsigma=prof_nsigma, niter=niter, - sigrej=sigrej, no_local_sky= no_local_sky, - use_2dmodel_mask=use_2dmodel_mask, - bkpts_optimal=bkpts_optimal, force_gauss=force_gauss, - sn_gauss=sn_gauss, model_full_slit=model_full_slit, - model_noise=model_noise, debug_bkpts=debug_bkpts, - show_resids=show_resids, show_profile=show_profile, - adderr=adderr, base_var=base_var, count_scale=count_scale) + left[:,iord], right[:,iord], sobjs[thisobj], + bkg_redux_global_sky=bkg_redux_global_sky, + fwhmimg=fwhmimg, flatimg=flatimg, + spat_pix=spat_pix, ingpm=inmask, std=std, bsp=bsp, + trim_edg=trim_edg, prof_nsigma=prof_nsigma, niter=niter, + sigrej=sigrej, no_local_sky=no_local_sky, + use_2dmodel_mask=use_2dmodel_mask, + bkpts_optimal=bkpts_optimal, force_gauss=force_gauss, + sn_gauss=sn_gauss, model_full_slit=model_full_slit, + model_noise=model_noise, debug_bkpts=debug_bkpts, + show_resids=show_resids, show_profile=show_profile, + adderr=adderr, base_var=base_var, count_scale=count_scale) if bkg_redux_skymodel is not None: bkg_redux_skymodel[thismask] = _this_bkg_redux_skymodel # update the FWHM fitting vector for the brighest object diff --git a/pypeit/core/slitdesign_matching.py b/pypeit/core/slitdesign_matching.py index eda3db0682..f12ce01b45 100644 --- a/pypeit/core/slitdesign_matching.py +++ b/pypeit/core/slitdesign_matching.py @@ -74,7 +74,7 @@ def best_offset(x_det, x_model, step=1, xlag_range=None): for j in range(xlag.size): x_det_lag = x_det+xlag[j] join = np.ma.concatenate([x_det_lag, x_model_trim]) - sind = np.argsort(join) + sind = np.argsort(join, kind='stable') nj = sind.size w1 = np.where(sind < x_det.size) @@ -90,7 +90,7 @@ def best_offset(x_det, x_model, step=1, xlag_range=None): offs = np.amin(np.absolute(x_det_lag[:, None] - x_model_trim[None, :]), axis=1) # use only nbest best matches - soffs = np.argsort(np.abs(offs)) + soffs = np.argsort(np.abs(offs), kind='stable') nbest2 = nbest if nbest 2.0: @@ -334,6 +337,7 @@ def conv_telluric(tell_model, dloglam, res): conv_model = scipy.signal.convolve(tell_model,g,mode='same') return conv_model + def shift_telluric(tell_model, loglam, dloglam, shift, stretch): """ Routine to apply a shift to the telluric model. Note that the shift can be sub-pixel, i.e this routine interpolates. @@ -364,6 +368,7 @@ def shift_telluric(tell_model, loglam, dloglam, shift, stretch): tell_model_shift = np.interp(loglam_shift, loglam, tell_model) return tell_model_shift + def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): """ Evaluate the telluric model. @@ -769,6 +774,7 @@ def general_spec_reader(specfile, ret_flam=False, chk_version=False, ret_order_s raise ValueError("This is an ugly hack until the DataContainer bug is fixed") head = sobjs.header wave, counts, counts_ivar, counts_gpm = unpack_orders(sobjs, ret_flam=ret_flam) + wave_grid_mid = None # Made a change to the if statement to account for unpack_orders now squeezing returned arrays #if (head['PYPELINE'] !='Echelle') and (wave.shape[1]>1) if (head['PYPELINE'] !='Echelle') and (wave.ndim>1): @@ -2410,18 +2416,12 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode self.disp = disp or debug self.sensfunc = sensfunc self.debug = debug - self.log10_blaze_func_in_arr = None # 2) Reshape all spectra to be (nspec, norders) - if log10_blaze_function is not None: + self.wave_in_arr, self.flux_in_arr, self.ivar_in_arr, self.mask_in_arr, self.log10_blaze_func_in_arr, \ + self.nspec_in, self.norders = utils.spec_atleast_2d( + wave, flux, ivar, gpm, log10_blaze_function=log10_blaze_function) - self.wave_in_arr, self.flux_in_arr, self.ivar_in_arr, self.mask_in_arr, self.log10_blaze_func_in_arr, \ - self.nspec_in, self.norders = utils.spec_atleast_2d( - wave, flux, ivar, gpm, log10_blaze_function=log10_blaze_function) - else: - self.wave_in_arr, self.flux_in_arr, self.ivar_in_arr, self.mask_in_arr, _, \ - self.nspec_in, self.norders = utils.spec_atleast_2d( - wave, flux, ivar, gpm) # 3) Read the telluric grid and initalize associated parameters wv_gpm = self.wave_in_arr > 1.0 if self.teltype == 'pca': @@ -2803,6 +2803,6 @@ def sort_telluric(self): tell_med[iord] = np.mean(np.exp(-np.sinh(tell_model_mean))) # Perform fits in order of telluric strength - return tell_med.argsort() + return tell_med.argsort(kind='stable') diff --git a/pypeit/core/trace.py b/pypeit/core/trace.py index 350518ecef..225363ed09 100644 --- a/pypeit/core/trace.py +++ b/pypeit/core/trace.py @@ -1647,7 +1647,41 @@ def predicted_center_difference(lower_spat, spat, width_fit, gap_fit): return np.absolute(spat - test_spat) -def extrapolate_orders(cen, width_fit, gap_fit, min_spat, max_spat, tol=0.01): +def extrapolate_prev_order(cen, width_fit, gap_fit, tol=0.01): + """ + Predict the location of the order to the spatial left (lower spatial pixel + numbers) of the order center provided. + + This is largely a support function for :func:`extrapolate_orders`. + + Args: + cen (:obj:`float`): + Center of the known order. + width_fit (:class:`~pypeit.core.fitting.PypeItFit`): + Model of the order width as a function of the order center. + gap_fit (:class:`~pypeit.core.fitting.PypeItFit`): + Model of the order gap *after* each order as a function of the order + center. + tol (:obj:`float`, optional): + Tolerance used when optimizing the order locations predicted toward + lower spatial pixels. + + Returns: + :obj:`float`: Predicted center of the previous order. + """ + # Guess the position of the previous order + w = width_fit.eval(cen) + guess = np.array([cen - w - gap_fit.eval(cen)]) + # Set the bounds based on this guess and the expected order width + bounds = optimize.Bounds(lb=guess - w/2, ub=guess + w/2) + # Optimize the spatial position + res = optimize.minimize(predicted_center_difference, guess, + args=(cen, width_fit, gap_fit), + method='trust-constr', jac='2-point', bounds=bounds, tol=tol) + return res.x[0] + + +def extrapolate_orders(cen, width_fit, gap_fit, min_spat, max_spat, tol=0.01, bracket=False): """ Predict the locations of additional orders by extrapolation. @@ -1676,6 +1710,9 @@ def extrapolate_orders(cen, width_fit, gap_fit, min_spat, max_spat, tol=0.01): tol (:obj:`float`, optional): Tolerance used when optimizing the order locations predicted toward lower spatial pixels. + bracket (:obj:`bool`, optional): + Bracket the added orders with one additional order on either side. + This can be useful for dealing with predicted overlap. Returns: :obj:`tuple`: Two arrays with orders centers (1) below the first and (2) @@ -1686,16 +1723,7 @@ def extrapolate_orders(cen, width_fit, gap_fit, min_spat, max_spat, tol=0.01): # Extrapolate toward lower spatial positions lower_spat = [cen[0]] while lower_spat[-1] > min_spat: - # Guess the position of the previous order - l = width_fit.eval(lower_spat[-1]) - guess = np.array([lower_spat[-1] - l - gap_fit.eval(lower_spat[-1])]) - # Set the bounds based on this guess and the expected order width - bounds = optimize.Bounds(lb=guess - l/2, ub=guess + l/2) - # Optimize the spatial position - res = optimize.minimize(predicted_center_difference, guess, - args=(lower_spat[-1], width_fit, gap_fit), - method='trust-constr', jac='2-point', bounds=bounds, tol=tol) - lower_spat += [res.x[0]] + lower_spat += [extrapolate_prev_order(lower_spat[-1], width_fit, gap_fit, tol=tol)] # Extrapolate toward larger spatial positions upper_spat = [cen[-1]] @@ -1703,8 +1731,12 @@ def extrapolate_orders(cen, width_fit, gap_fit, min_spat, max_spat, tol=0.01): upper_spat += [upper_spat[-1] + width_fit.eval(upper_spat[-1]) + gap_fit.eval(upper_spat[-1])] - # Return arrays after removing the first and last spatial position (which - # are either repeats of values in `cen` or outside the spatial range) + # Return arrays after removing the first spatial position because it is a + # repeat of the input. If not bracketting, also remove the last point + # because it will be outside the minimum or maximum spatial position (set by + # min_spat, max_spat). + if bracket: + return np.array(lower_spat[-1:0:-1]), np.array(upper_spat[1:]) return np.array(lower_spat[-2:0:-1]), np.array(upper_spat[1:-1]) diff --git a/pypeit/core/wavecal/autoid.py b/pypeit/core/wavecal/autoid.py index 4fcebb1add..116405ddfd 100644 --- a/pypeit/core/wavecal/autoid.py +++ b/pypeit/core/wavecal/autoid.py @@ -1111,8 +1111,8 @@ def full_template(spec, lamps, par, ok_mask, det, binspectral, nsnippet=2, slit_ lines_wav = template_dict['lines_wav'] lines_fit_ord = template_dict['lines_fit_ord'] - temp_wv = temp_wv_og - temp_spec = temp_spec_og + temp_wv = np.copy(temp_wv_og) + temp_spec = np.copy(temp_spec_og) # Deal with binning (not yet tested) if binspectral != temp_bin: @@ -1145,12 +1145,12 @@ def full_template(spec, lamps, par, ok_mask, det, binspectral, nsnippet=2, slit_ obs_spec_i = spec[:,slit] # get FWHM for this slit fwhm = set_fwhm(par, measured_fwhm=measured_fwhms[slit], verbose=True) - + # Find the shift ncomb = temp_spec.size # Remove the continuum before adding the padding to obs_spec_i _, _, _, _, obs_spec_cont_sub = wvutils.arc_lines_from_spec(obs_spec_i) - _, _, _, _, templ_spec_cont_sub = wvutils.arc_lines_from_spec(temp_spec) + _, _, _, _, templ_spec_cont_sub = wvutils.arc_lines_from_spec(temp_spec.reshape(-1)) # Pad pad_spec = np.zeros_like(temp_spec) nspec = len(obs_spec_i) @@ -2339,7 +2339,7 @@ def cross_match(self, good_fit, detections): wvc_gd_jfh[cntr] = wave_soln[self._npix//2] dsp_gd_jfh[cntr]= np.median(wave_soln - np.roll(wave_soln,1)) cntr += 1 - srt = np.argsort(wvc_gd_jfh) + srt = np.argsort(wvc_gd_jfh, kind='stable') sort_idx = idx_gd[srt] sort_wvc = wvc_gd[srt] sort_dsp = dsp_gd[srt] @@ -3189,7 +3189,7 @@ def finalize_fit(self, detections): if self._outroot is not None: # Write IDs out_dict = dict(pix=use_tcent, IDs=self._all_patt_dict[str(slit)]['IDs']) - jdict = ltu.jsonify(out_dict) + jdict = utils.jsonify(out_dict) ltu.savejson(self._outroot + slittxt + '.json', jdict, easy_to_read=True, overwrite=True) msgs.info("Wrote: {:s}".format(self._outroot + slittxt + '.json')) diff --git a/pypeit/core/wavecal/spectrographs/templ_p200_dbsp.py b/pypeit/core/wavecal/spectrographs/templ_p200_dbsp.py index 21902aee15..90b842f216 100644 --- a/pypeit/core/wavecal/spectrographs/templ_p200_dbsp.py +++ b/pypeit/core/wavecal/spectrographs/templ_p200_dbsp.py @@ -41,6 +41,22 @@ def p200_dbsp_red_1200_9400_d55(overwrite=False): # DBSPr 1200/9400 D55 templates.build_template([wfile0], slits, lcut, binspec, outroot, lowredux=False, ifiles=ifiles, normalize=True, overwrite=overwrite) +# ############################## +def p200_dbsp_red_1200_7100_d55(overwrite=False): # DBSPr 1200/7100 D55 + + binspec = 1 + outroot = 'p200_dbsp_red_1200_7100_d55_6680.fits' + # + ifiles = [0] + slits = [55] # Be careful with the order.. + lcut = None + wfile0 = os.path.join(templates.template_path, 'P200_DBSP', + 'R1200_7100_D55', 'wvcalib_6680.fits') + + # + templates.build_template([wfile0], slits, lcut, binspec, outroot, + lowredux=False, ifiles=ifiles, normalize=True, overwrite=overwrite) + # ############################## def p200_dbsp_red_600_10000_d55(overwrite=False): # DBSPr 600/10000 D55 @@ -144,4 +160,5 @@ def p200_dbsp_blue_1200_5000_d68(overwrite=False): # DBSPb 1200/5000 D68 if __name__ == '__main__': # p200_dbsp_red_316_7500_d55(overwrite=True) # p200_dbsp_red_600_10000_d55(overwrite=True) - p200_dbsp_red_1200_7100_d68(overwrite=True) + # p200_dbsp_red_1200_7100_d68(overwrite=True) + p200_dbsp_red_1200_7100_d55(overwrite=True) diff --git a/pypeit/core/wavecal/wvutils.py b/pypeit/core/wavecal/wvutils.py index 89c992700c..d5b44af579 100644 --- a/pypeit/core/wavecal/wvutils.py +++ b/pypeit/core/wavecal/wvutils.py @@ -136,9 +136,13 @@ def get_sampling(waves, pix_per_R=3.0): wave_diff_flat += np.diff(wave_good).tolist() dloglam_flat += np.diff(np.log10(wave_good)).tolist() - + # Compute the median wavelength spacing dwave = np.median(wave_diff_flat) dloglam = np.median(dloglam_flat) + # Check that this won't introduce a divide by zero + if dloglam == 0.0: + msgs.error('The wavelength sampling has zero spacing in log wavelength. This is not supported.') + # Compute a guess of the resolution resln_guess = 1.0 / (pix_per_R* dloglam * np.log(10.0)) pix_per_sigma = 1.0 / resln_guess / (dloglam * np.log(10.0)) / (2.0 * np.sqrt(2.0 * np.log(2))) return dwave, dloglam, resln_guess, pix_per_sigma @@ -281,6 +285,9 @@ def get_wave_grid(waves=None, gpms=None, wave_method='linear', iref=0, wave_grid wave_grid = np.power(10.0,newloglam) elif wave_method == 'iref': # Use the iref index wavelength array + msgs.info(f'iref for the list is set to {iref}') + msgs.info(f'The shape of the list is: {np.shape(waves)}') + msgs.info(f'shape of the first wave_grid in the list is: {np.shape(waves[iref])}') wave_tmp = waves[iref] wave_grid = wave_tmp[wave_tmp > 1.0] if spec_samp_fact != 1: # adjust sampling via internal interpolation diff --git a/pypeit/data/arc_lines/reid_arxiv/aat_uhrf_3875.fits b/pypeit/data/arc_lines/reid_arxiv/aat_uhrf_3875.fits new file mode 100644 index 0000000000..4e2009a902 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/aat_uhrf_3875.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/keck_kcwi_BH3.fits b/pypeit/data/arc_lines/reid_arxiv/keck_kcwi_BH3.fits new file mode 100644 index 0000000000..2e0e8f32b8 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/keck_kcwi_BH3.fits differ diff --git a/pypeit/data/arc_lines/reid_arxiv/p200_dbsp_red_1200_7100_d55_6680.fits b/pypeit/data/arc_lines/reid_arxiv/p200_dbsp_red_1200_7100_d55_6680.fits new file mode 100644 index 0000000000..475918859a Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/p200_dbsp_red_1200_7100_d55_6680.fits differ diff --git a/pypeit/data/pixelflats/PYPEIT_LRISb_pixflat_B400_2x2_15apr2015_specflip.fits.gz b/pypeit/data/pixelflats/PYPEIT_LRISb_pixflat_B400_2x2_15apr2015_specflip.fits.gz new file mode 100644 index 0000000000..5fd3c06e3d Binary files /dev/null and b/pypeit/data/pixelflats/PYPEIT_LRISb_pixflat_B400_2x2_15apr2015_specflip.fits.gz differ diff --git a/pypeit/data/pixelflats/PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz b/pypeit/data/pixelflats/PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz new file mode 100644 index 0000000000..f22c724280 Binary files /dev/null and b/pypeit/data/pixelflats/PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz differ diff --git a/pypeit/data/pixelflats/README b/pypeit/data/pixelflats/README new file mode 100644 index 0000000000..3617e4096c --- /dev/null +++ b/pypeit/data/pixelflats/README @@ -0,0 +1,11 @@ +Directory includes custom pixel flats to be used for Flat Fielding. + +The general naming structure is: +'pixelflat_{spec_name}{dispname}{dichroic}_{binning}_{date}.fits.gz' + +spec_name: spectrograph name (e.g. keck_lris_blue) +dispname: metadata dispname (if part of the spectrograph configuration keys) +dichroic: metadata dichroic (if part of the spectrograph configuration keys) +binning: spectral x spatial binning +date: date of the observations + diff --git a/pypeit/data/pixelflats/pixelflat_keck_hires_RED_1x2_20160330.fits.gz b/pypeit/data/pixelflats/pixelflat_keck_hires_RED_1x2_20160330.fits.gz new file mode 100644 index 0000000000..e5bb58b7b6 Binary files /dev/null and b/pypeit/data/pixelflats/pixelflat_keck_hires_RED_1x2_20160330.fits.gz differ diff --git a/pypeit/data/pixelflats/pixelflat_keck_hires_RED_1x3_20170223.fits.gz b/pypeit/data/pixelflats/pixelflat_keck_hires_RED_1x3_20170223.fits.gz new file mode 100644 index 0000000000..999703ae24 Binary files /dev/null and b/pypeit/data/pixelflats/pixelflat_keck_hires_RED_1x3_20170223.fits.gz differ diff --git a/pypeit/data/pixelflats/pixelflat_keck_hires_RED_2x2_20170614.fits.gz b/pypeit/data/pixelflats/pixelflat_keck_hires_RED_2x2_20170614.fits.gz new file mode 100644 index 0000000000..e38d2d80f2 Binary files /dev/null and b/pypeit/data/pixelflats/pixelflat_keck_hires_RED_2x2_20170614.fits.gz differ diff --git a/pypeit/data/pixelflats/pixelflat_keck_lris_blue_600_4000_d560_2x2_20210411.fits.gz b/pypeit/data/pixelflats/pixelflat_keck_lris_blue_600_4000_d560_2x2_20210411.fits.gz new file mode 100644 index 0000000000..9eba3afdf1 Binary files /dev/null and b/pypeit/data/pixelflats/pixelflat_keck_lris_blue_600_4000_d560_2x2_20210411.fits.gz differ diff --git a/pypeit/data/standards/blackbody/blackbody_info.txt b/pypeit/data/standards/blackbody/blackbody_info.txt index d0cc2e4237..060f9e49b3 100644 --- a/pypeit/data/standards/blackbody/blackbody_info.txt +++ b/pypeit/data/standards/blackbody/blackbody_info.txt @@ -7,20 +7,20 @@ # NOTE: The flux is generated at runtime, so there are no # "Files". The first column is just a placeholder. File Name RA_2000 DEC_2000 g_MAG TYPE T_K a_x10m23 -J0027m0017.fits J0027m0017 00:27:39.497 -00:17:41.93 18.90 DB 10662 0.435 -J0048p0017.fits J0048p0017 00:48:30.324 +00:17:52.80 18.14 DB 10639 0.876 -J0146m0051.fits J0146m0051 01:46:18.898 -00:51:50.51 18.17 DB 11770 0.672 -J0229m0041.fits J0229m0041 02:29:36.715 -00:41:13.63 19.07 DB 8901 0.640 -J0832p3709.fits J0832p3709 08:32:26.568 +37:09:55.48 18.98 DB 7952 1.137 -J0837p5427.fits J0837p5427 08:37:36.557 +54:27:58.64 18.73 DB 7533 1.703 -J1004p1215.fits J1004p1215 10:04:49.541 +12:15:59.65 19.18 DB 9773 0.440 -J1045p0157.fits J1045p0157 10:45:23.866 +01:57:21.96 19.09 DB 8956 0.666 -J1117p4059.fits J1117p4059 11:17:20.801 +40:59:54.67 18.08 DB 10992 0.844 -J1147p1713.fits J1147p1713 11:47:22.608 +17:13:25.21 18.65 DB 9962 0.669 -J1245p4238.fits J1245p4238 12:45:35.626 +42:38:24.58 17.14 DB 9912 2.817 -J1255p1924.fits J1255p1924 12:55:07.082 +19:24:59.00 18.53 DB 8882 1.157 -J1343p2706.fits J1343p2706 13:43:05.302 +27:06:23.98 18.93 DB 10678 0.427 -J1417p4941.fits J1417p4941 14:17:24.329 +49:41:27.85 17.25 DB 10461 2.192 -J1518p0028.fits J1518p0028 15:18:59.717 +00:28:39.58 19.44 DB 9072 0.458 -J1617p1813.fits J1617p1813 16:17:04.078 +18:13:11.96 18.78 DB 8568 0.996 -J2302m0030.fits J2302m0030 23:02:40.032 -00:30:21.60 17.80 DB 10478 1.241 +J0027m0017.fits J0027m0017 00:27:39.497 -00:17:41.93 18.90 DC 10662 0.435 +J0048p0017.fits J0048p0017 00:48:30.324 +00:17:52.80 18.14 DC 10639 0.876 +J0146m0051.fits J0146m0051 01:46:18.898 -00:51:50.51 18.17 DC 11770 0.672 +J0229m0041.fits J0229m0041 02:29:36.715 -00:41:13.63 19.07 DC 8901 0.640 +J0832p3709.fits J0832p3709 08:32:26.568 +37:09:55.48 18.98 DC 7952 1.137 +J0837p5427.fits J0837p5427 08:37:36.557 +54:27:58.64 18.73 DC 7533 1.703 +J1004p1215.fits J1004p1215 10:04:49.541 +12:15:59.65 19.18 DC 9773 0.440 +J1045p0157.fits J1045p0157 10:45:23.866 +01:57:21.96 19.09 DC 8956 0.666 +J1117p4059.fits J1117p4059 11:17:20.801 +40:59:54.67 18.08 DC 10992 0.844 +J1147p1713.fits J1147p1713 11:47:22.608 +17:13:25.21 18.65 DC 9962 0.669 +J1245p4238.fits J1245p4238 12:45:35.626 +42:38:24.58 17.14 DC 9912 2.817 +J1255p1924.fits J1255p1924 12:55:07.082 +19:24:59.00 18.53 DC 8882 1.157 +J1343p2706.fits J1343p2706 13:43:05.302 +27:06:23.98 18.93 DC 10678 0.427 +J1417p4941.fits J1417p4941 14:17:24.329 +49:41:27.85 17.25 DC 10461 2.192 +J1518p0028.fits J1518p0028 15:18:59.717 +00:28:39.58 19.44 DC 9072 0.458 +J1617p1813.fits J1617p1813 16:17:04.078 +18:13:11.96 18.78 DC 8568 0.996 +J2302m0030.fits J2302m0030 23:02:40.032 -00:30:21.60 17.80 DC 10478 1.241 diff --git a/pypeit/display/display.py b/pypeit/display/display.py index ab8b29155e..9cb087bf59 100644 --- a/pypeit/display/display.py +++ b/pypeit/display/display.py @@ -486,6 +486,7 @@ def show_trace(viewer, ch, trace, trc_name=None, maskdef_extr=None, manual_extr= ntrace = trace.shape[1] _maskdef_extr = ntrace*[False] if maskdef_extr is None else maskdef_extr _manual_extr = ntrace*[False] if manual_extr is None else manual_extr + _trc_name = ntrace*[''] if trc_name is None else trc_name # Show if yval is None: @@ -508,8 +509,8 @@ def show_trace(viewer, ch, trace, trc_name=None, maskdef_extr=None, manual_extr= # Text ohf = len(trace[:,i])//2 # Do it - canvas_list += [dict(type='text',args=(float(y[ohf,i]), float(trace[ohf,i]), str(trc_name[i])) if rotate - else (float(trace[ohf,i]), float(y[ohf,i]), str(trc_name[i])), + canvas_list += [dict(type='text',args=(float(y[ohf,i]), float(trace[ohf,i]), str(_trc_name[i])) if rotate + else (float(trace[ohf,i]), float(y[ohf,i]), str(_trc_name[i])), kwargs=dict(color=_color, fontsize=17., rot_deg=90.))] canvas.add('constructedcanvas', canvas_list) diff --git a/pypeit/edgetrace.py b/pypeit/edgetrace.py index c7e4891139..be6d75d9fe 100644 --- a/pypeit/edgetrace.py +++ b/pypeit/edgetrace.py @@ -877,11 +877,21 @@ def auto_trace(self, bpm=None, debug=False, show_stages=False): if not self.is_empty and self.par['add_missed_orders']: # Refine the order traces self.order_refine(debug=debug) - # Check that the edges are still sinked (not overkill if orders are - # missed) - self.success = self.sync() - if not self.success: - return + # Check that the edges are still synced + if not self.is_synced: + msgs.error('Traces are no longer synced after adding in missed orders.') + +# KBW: Keep this code around for a while. It is the old code that +# resynced the edges just after adding in new orders. Nominally, this +# shouldn't be necessary, but the comment suggests this may be +# necessary if orders are missed. We should keep this around until +# we're sure it's not needed. +# # Check that the edges are still sinked (not overkill if orders are +# # missed) +# self.success = self.sync(debug=True) +# if not self.success: +# return + if show_stages: self.show(title='After adding in missing orders') @@ -2612,10 +2622,19 @@ def check_synced(self, rebuild_pca=False): # ' changes along the dispersion direction.') # self.remove_traces(dl_flag, rebuild_pca=_rebuild_pca) - # Get the slits that have been flagged as abnormally short. This should - # be the same as the definition above, it's just redone here to ensure - # `short` is defined when `length_rtol` is None. - short = self.fully_masked_traces(flag='ABNORMALSLIT_SHORT') + # Try to detect overlap between adjacent slits by finding abnormally + # short slits. + # - Find abnormally short slits that *do not* include inserted edges; + # i.e., these must be *detected* edges, not inserted ones. + # - *Both* edges in the fit must be flagged because of this + # requirement that the trace not be inserted. This means that we + # set mode='neither' when running synced_selection. I also set + # assume_synced=True: the traces should be synced if the code has + # made it this far. Any flags that would indicate otherwise will + # have been set by this function. + short = self.fully_masked_traces(flag='ABNORMALSLIT_SHORT', + exclude=self.bitmask.insert_flags) + short = self.synced_selection(short, mode='neither', assume_synced=True) if self.par['overlap'] and np.any(short): msgs.info('Assuming slits flagged as abnormally short are actually due to ' 'overlapping slit edges.') @@ -2819,7 +2838,7 @@ def synced_selection(self, indx, mode='ignore', assume_synced=False): if not assume_synced and not self.is_synced: msgs.error('To synchronize the trace selection, it is expected that the traces have ' - 'been left-right synchronized. Either run sync() to sychronize, ignore ' + 'been left-right synchronized. Either run sync() to sychronize or ignore ' 'the synchronization (which may raise an exception) by setting ' 'assume_synced=True.') if mode == 'both': @@ -4090,7 +4109,7 @@ def sync(self, rebuild_pca=True, debug=False): if debug: msgs.info('Show instance includes inserted traces but before checking the sync.') - self.show(flag='any') + self.show(title='includes inserted traces before checking the sync', flag='any') # Check the full synchronized list and log completion of the # method @@ -4100,6 +4119,7 @@ def sync(self, rebuild_pca=True, debug=False): i += 1 if i == maxiter: msgs.error('Fatal left-right trace de-synchronization error.') + if self.log is not None: self.log += [inspect.stack()[0][3]] return True @@ -4271,7 +4291,7 @@ def insert_traces(self, side, trace_cen, loc=None, mode='user', resort=True, nud if loc.size != ntrace: msgs.error('Number of sides does not match the number of insertion locations.') - msgs.info('Inserting {0} new traces.'.format(ntrace)) + msgs.info(f'Inserting {ntrace} new traces.') # Nudge the traces if nudge: @@ -4563,7 +4583,7 @@ def maskdesign_matching(self, debug=False): # Append the missing indices and re-sort all ind_b = np.append(ind_b, needind_b) sortind_b = np.argsort(utils.index_of_x_eq_y(self.slitmask.slitid[sortindx], - self.slitmask.slitid[ind_b], strict=True)) + self.slitmask.slitid[ind_b], strict=True), kind='stable') ind_b = ind_b[sortind_b] for i in range(bot_edge_pred[needind_b].size): # check if the trace that will be added is off detector @@ -4596,7 +4616,7 @@ def maskdesign_matching(self, debug=False): # Append the missing indices and re-sort all ind_t = np.append(ind_t, needind_t) sortind_t = np.argsort(utils.index_of_x_eq_y(self.slitmask.slitid[sortindx], - self.slitmask.slitid[ind_t], strict=True)) + self.slitmask.slitid[ind_t], strict=True), kind='stable') ind_t = ind_t[sortind_t] for i in range(top_edge_pred[needind_t].size): # check if the trace that will be added is off detector @@ -4855,13 +4875,17 @@ def order_refine(self, debug=False): add_left, add_right = self.order_refine_fixed_format(reference_row, debug=debug) rmtraces = None else: + # TODO: `bracket` is hard-coded! Currently I expect we always want + # to set bracket=True, but we should plan to revisit this and maybe + # expose as a user parameter. + bracket = True add_left, add_right, rmtraces \ - = self.order_refine_free_format(reference_row, debug=debug) + = self.order_refine_free_format(reference_row, bracket=bracket, debug=debug) if add_left is None or add_right is None: msgs.info('No additional orders found to add') return - + if rmtraces is not None: self.remove_traces(rmtraces, rebuild_pca=True) @@ -4915,12 +4939,58 @@ def order_refine_fixed_format(self, reference_row, debug=False): return add_left_edges, add_right_edges - # NOTE: combined_order_tol is effectively hard-coded. - def order_refine_free_format(self, reference_row, combined_order_tol=1.8, debug=False): + # NOTE: combined_order_tol is effectively hard-coded; i.e., the current code + # always uses the default when calling this function. + def order_refine_free_format(self, reference_row, combined_order_tol=1.8, bracket=True, + debug=False): """ Refine the order locations for "free-format" Echelles. - Traces must be synced before reaching here. + Traces must be synced before calling this function. + + The procedure is as follows: + + - The function selects the good traces and calculates the width of + each order and the gap between each order and fits them with + Legendre polynomials (using the polynomial orders set by the + ``order_width_poly`` and ``order_gap_poly`` parameters); 5-sigma + outliers are removed from the fit. + + - Based on this fit, the code adds missed orders, both interspersed + with detected orders and extrapolated over the full spatial range + of the detector/mosaic. The spatial extent over which this + prediction is performed is set by ``order_spat_range`` and can be + limited by any resulting overlap in the prediction, as set by + ``max_overlap``. + + - Any detected "orders" that are actually the adjoining of one or + more orders are flagged for rejection. + + Args: + reference_row (:obj:`int`): + The index of the spectral pixel (row) in the set of left and + right traces at which to predict the positions of the missed + orders. Nominally, this is the reference row used for the + construction of the trace PCA. + combined_order_tol (:obj:`float`, optional): + For orders that are very nearly overlapping, the automated edge + tracing can often miss the right and left edges of two adjacent + orders. This leads to the detected edges of two adjacent orders + being combined into a single order. This value sets the maximum + ratio of the width of any given detected order to the polynomial + fit to the order width as a function of spatial position on the + detector. + bracket (:obj:`bool`, optional): + Bracket the added orders with one additional order on either side. + This can be useful for dealing with predicted overlap. + debug (:obj:`bool`, optional): + Run in debug mode. + + Returns: + :obj:`tuple`: Three `numpy.ndarray`_ objects that provide (1,2) the + left and right edges of orders to be added to the set of edge traces + and (3) a boolean array indicating which of the existing traces + should be removed. """ # Select the left/right traces # TODO: This is pulled from get_slits. Maybe want a function for this. @@ -4928,7 +4998,8 @@ def order_refine_free_format(self, reference_row, combined_order_tol=1.8, debug= # Save the list of good left edges in case we need to remove any left_gpm = gpm & self.is_left left = self.edge_fit[:,left_gpm] - right = self.edge_fit[:,gpm & self.is_right] + right_gpm = gpm & self.is_right + right = self.edge_fit[:,right_gpm] # Use the trace locations at the middle of the spectral shape of the # detector/mosaic @@ -4941,45 +5012,59 @@ def order_refine_free_format(self, reference_row, combined_order_tol=1.8, debug= gap = left[1:] - right[:-1] # Create the polynomial models. - # TODO: - # - Expose the rejection parameters to the user? - # - Be more strict with upper rejection, to preferentially ignore - # measurements biased by missing orders and/or combined orders? width_fit = fitting.robust_fit(cen, width, self.par['order_width_poly'], - function='legendre', lower=3., upper=3., maxiter=5, - sticky=True) + function='legendre', lower=self.par['order_fitrej'], + upper=self.par['order_fitrej'], maxiter=5, sticky=True) # Connection of center to gap uses the gap spatially *after* the order. gap_fit = fitting.robust_fit(cen[:-1], gap, self.par['order_gap_poly'], - function='legendre', lower=3., upper=3., maxiter=5, - sticky=True) - - # Ideally, measured widths/gaps should be regected for one of the + function='legendre', lower=self.par['order_fitrej'], + upper=self.par['order_fitrej'], maxiter=5, sticky=True) + + # Ideally, measured widths/gaps should be rejected for one of the # following reasons: # - The width is too large because gaps were missed (i.e. multiple # orders were combined) # - The width is too small because order overlap was detected and # removed. # - The gap is too large because orders were missed - # This finds cases where multiple orders have been combined - combined_orders = width / width_fit.eval(cen) > combined_order_tol + + # In the case when the user does not reject "outliers", we still reject + # orders that we expected to be cases where multiple orders have been + # combined + bad_order = width / width_fit.eval(cen) > combined_order_tol + if self.par['order_outlier'] is not None: + # Exclude "outliers" + resid = np.absolute(width_fit.yval - width_fit.eval(width_fit.xval)) + bad_order |= (resid/width_fit.calc_fit_rms() > self.par['order_outlier']) + # TODO: The gaps for HIRES can have *very* large residuals. Using + # the gaps to identify outliers would remove many orders that + # probably shouldn't be removed. +# resid = np.absolute(gap_fit.yval - gap_fit.eval(gap_fit.xval)) +# bad_order[:-1] |= (resid/gap_fit.calc_fit_rms() > self.par['order_outlier']) + # And sets flags used to remove them, in favor of replacing them with # the predicted locations of the individual orders. rmtraces = np.zeros(left_gpm.size, dtype=bool) - rmtraces[np.where(left_gpm)[0][combined_orders]] = True + rmtraces[np.where(left_gpm)[0][bad_order]] = True rmtraces = self.synced_selection(rmtraces, mode='both') # Interpolate any missing orders # TODO: Expose tolerances to the user? - individual_orders = np.logical_not(combined_orders) + good_order = np.logical_not(bad_order) order_cen, order_missing \ - = trace.find_missing_orders(cen[individual_orders], width_fit, gap_fit) - - # Extrapolate orders + = trace.find_missing_orders(cen[good_order], width_fit, gap_fit) + if np.sum(order_missing) > order_missing.size // 2: + msgs.warn('Found more missing orders than detected orders. Check the order ' + 'refinement QA file! The code will continue, but you likely need to adjust ' + 'your edge-tracing parameters.') + + # Extrapolate orders; this includes one additional order to either side + # of the spatial extent set by rng. rng = [0., float(self.nspat)] if self.par['order_spat_range'] is None \ else self.par['order_spat_range'] lower_order_cen, upper_order_cen \ - = trace.extrapolate_orders(cen[individual_orders], width_fit, gap_fit, - rng[0], rng[1]) + = trace.extrapolate_orders(cen[good_order], width_fit, gap_fit, + rng[0], rng[1], bracket=bracket) # Combine the results order_cen = np.concatenate((lower_order_cen, order_cen, upper_order_cen)) @@ -4996,123 +5081,328 @@ def order_refine_free_format(self, reference_row, combined_order_tol=1.8, debug= # TODO: Making this directory should probably be done elsewhere if ofile is not None and not ofile.parent.is_dir(): ofile.parent.mkdir(parents=True) - self.order_refine_free_format_qa(cen, combined_orders, width, gap, width_fit, gap_fit, - order_cen, order_missing, ofile=ofile) + self.order_refine_free_format_qa(cen, bad_order, width, gap, width_fit, gap_fit, + order_cen, order_missing, bracket=bracket, ofile=ofile) # Return the coordinates for the left and right edges to add add_width = width_fit.eval(order_cen[order_missing]) - add_left, add_right = order_cen[order_missing] - add_width / 2, order_cen[order_missing] + add_width / 2 + add_left = order_cen[order_missing] - add_width / 2 + add_right = order_cen[order_missing] + add_width / 2 # Join the added edges with the existing ones - _left = np.append(add_left, left[individual_orders]) + _left = np.append(add_left, left[good_order]) # Create a sorting vector srt = np.argsort(_left) # Create a vector that will reverse the sorting isrt = np.argsort(srt) # Join and sort the right edges - _right = np.append(add_right, right[individual_orders])[srt] + _right = np.append(add_right, right[good_order])[srt] # Sort the left edges _left = _left[srt] + # NOTE: Although I haven't tested this, I think this approach works best + # under the assumption that the overlap *decreases* from small pixel + # numbers to large pixel numbers. This should be true if the pypeit + # convention is maintained with blue orders toward small pixel values + # and red orders at large pixel values. + # Deal with overlapping orders among the ones to be added. The edges # are adjusted equally on both sides to avoid changing the order center # and exclude the overlap regions from the reduction. - if np.any(_left[1:] - _right[:-1] < 0): - # Loop sequentially so that each pair is updated as the loop progresses - for i in range(1, _left.size): - # *Negative* of the gap; i.e., positives values means there's - # overlap - ngap = _right[i-1] - _left[i] - if ngap > 0: - # Adjust both order edges to avoid the overlap region but - # keep the same center coordinate - _left[i-1] += ngap - _right[i-1] -= ngap - _left[i] += ngap - _right[i] -= ngap + if np.all(_left[1:] - _right[:-1] > 0): + if bracket: + add_left, add_right = self._handle_bracketing_orders(add_left, add_right) + # There is no overlap, so just return the orders to add + return add_left, add_right, rmtraces + + # Used to remove orders that have too much overlap + nord = _left.size + ok_overlap = np.ones(nord, dtype=bool) + + # Loop sequentially so that each pair is updated as the loop progresses + for i in range(1, nord): + # *Negative* of the gap; i.e., positives values means there's + # overlap + ngap = _right[i-1] - _left[i] + if ngap > 0: + if self.par['max_overlap'] is not None: + ok_overlap[i-1] = 2*ngap/(_right[i-1] - _left[i-1]) < self.par['max_overlap'] + ok_overlap[i] = 2*ngap/(_right[i] - _left[i]) < self.par['max_overlap'] + # Adjust both order edges to avoid the overlap region but + # keep the same center coordinate + _left[i-1] += ngap + _right[i-1] -= ngap + _left[i] += ngap + _right[i] -= ngap + + # For any *existing* traces that were adjusted because of the overlap, + # this applies the adjustment to the `edge_fit` data. + # NOTE: This only adjusts the "fit" locations (edge_fit), *not* the + # measured centroid locations (edge_cen). This should not cause + # problems because, e.g., the `get_slits` function uses `edge_fit`. + nadd = add_left.size + left_indx = np.where(left_gpm)[0][good_order] + offset = _left[isrt][nadd:] - left + self.edge_fit[:,left_indx] += offset[None,:] + right_indx = np.where(right_gpm)[0][good_order] + offset = _right[isrt][nadd:] - right + self.edge_fit[:,right_indx] += offset[None,:] # Get the adjusted traces to add. Note this currently does *not* change # the original traces - return _left[isrt][:add_left.size], _right[isrt][:add_right.size], rmtraces - - def order_refine_free_format_qa(self, cen, combined_orders, width, gap, width_fit, gap_fit, - order_cen, order_missing, ofile=None): + ok_overlap = ok_overlap[isrt][:nadd] + add_left = _left[isrt][:nadd][ok_overlap] + add_right = _right[isrt][:nadd][ok_overlap] + + if bracket: + add_left, add_right = self._handle_bracketing_orders(add_left, add_right) + return add_left, add_right, rmtraces + + @staticmethod + def _handle_bracketing_orders(add_left, add_right): """ - QA plot for order placements + Utility function to remove added orders that bracket the left and right + edge of the detector, used to handle overlap. + + Args: + add_left (`numpy.ndarray`_): + List of left edges to add + add_right (`numpy.ndarray`_): + List of right edges to add + + Returns: + :obj:`tuple`: The two `numpy.ndarray`_ objects after removing the + bracketing orders. """ + nadd = add_left.size + if nadd < 2: + # TODO: The code should not get here! If it does, we need to + # figure out why and fix it. + msgs.error('CODING ERROR: Order bracketing failed!') + if nadd == 2: + return None, None + return add_left[1:-1], add_right[1:-1] + + def order_refine_free_format_qa(self, cen, bad_order, width, gap, width_fit, gap_fit, + order_cen, order_missing, bracket=False, ofile=None): + """ + Create the QA plot for order modeling. + + Args: + cen (`numpy.ndarray`_): + Spatial centers of the detected orders. + bad_order (`numpy.ndarray`_): + Boolean array selecting "orders" that have been flagged as + outliers. + width (`numpy.ndarray`_): + Measured order spatial widths in pixels. + gap (`numpy.ndarray`_): + Measured order gaps in pixels. + width_fit (:class:`~pypeit.core.fitting.PypeItFit`): + Model of the order width as a function of the order center. + gap_fit (:class:`~pypeit.core.fitting.PypeItFit`): + Model of the order gap *after* each order as a function of the order + center. + order_cen (`numpy.ndarray`_): + Spatial centers of all "individual" orders. + order_missing (`numpy.ndarray`_): + Boolean array selecting "individual" orders that were not traced + by the automated tracing and flagged as missing. See + :func:`~pypeit.core.trace.find_missing_orders` and + :func:`~pypeit.core.trace.extrapolate_orders`. + bracket (:obj:`bool`, optional): + Flag that missing orders have been bracketed by additional + orders in an attempt to deal with overlap regions. + ofile (:obj:`str`, `Path`_, optional): + Path for the QA figure file. If None, the plot is shown in a + matplotlib window. + """ + # Setup + w_resid = width - width_fit.eval(cen) + w_rms = width_fit.calc_fit_rms() + med_wr = np.median(w_resid) + mad_wr = np.median(np.absolute(w_resid - med_wr)) + + w_out = bad_order & width_fit.gpm.astype(bool) + w_rej = np.logical_not(bad_order) & np.logical_not(width_fit.gpm) + w_outrej = bad_order & np.logical_not(width_fit.gpm) + w_good = np.logical_not(w_out | w_rej | w_outrej) + + g_cen = cen[:-1] + g_bad_order = bad_order[:-1] + g_resid = gap - gap_fit.eval(g_cen) + g_rms = gap_fit.calc_fit_rms() + med_gr = np.median(g_resid) + mad_gr = np.median(np.absolute(g_resid - med_gr)) + + g_out = g_bad_order & gap_fit.gpm.astype(bool) + g_rej = np.logical_not(g_bad_order) & np.logical_not(gap_fit.gpm) + g_outrej = g_bad_order & np.logical_not(gap_fit.gpm) + g_good = np.logical_not(g_out | g_rej | g_outrej) + + # Set the spatial limits based on the extent of the order centers and/or + # the detector spatial extent + sx = min(0, np.amin(order_cen)) + ex = max(self.nspat, np.amax(order_cen)) + buf = 1.1 + xlim = [(sx * (1 + buf) + ex * (1 - buf))/2, (sx * (1 - buf) + ex * (1 + buf))/2] + + # Set the residual plot limits based on the median and median absolute + # deviation + width_lim = np.array([med_wr - 20*mad_wr, med_wr + 20*mad_wr]) + gap_lim = np.array([med_gr - 20*mad_gr, med_gr + 20*mad_gr]) + + # Sample the width and gap models over the full width of the plots + mod_cen = np.linspace(*xlim, 100) + mod_width = width_fit.eval(mod_cen) + mod_gap = gap_fit.eval(mod_cen) + + # Create the plot w,h = plt.figaspect(1) fig = plt.figure(figsize=(1.5*w,1.5*h)) - ax = fig.add_axes([0.15, 0.35, 0.8, 0.6]) + # Plot the data and each fit + ax = fig.add_axes([0.10, 0.35, 0.8, 0.6]) ax.minorticks_on() ax.tick_params(which='both', direction='in', top=True, right=True) ax.grid(True, which='major', color='0.7', zorder=0, linestyle='-') - ax.set_xlim([0, self.nspat]) + ax.set_xlim(xlim) ax.xaxis.set_major_formatter(ticker.NullFormatter()) - # TODO: Do something similar for the gaps? - if np.any(combined_orders): - individual_orders = np.logical_not(combined_orders) - ax.scatter(cen[combined_orders], width[combined_orders], - marker='^', color='C1', s=80, lw=0, label='combined orders flag', - zorder=3) - ax.scatter(cen[individual_orders], width[individual_orders], - marker='.', color='C0', s=50, lw=0, label='measured widths', - zorder=3) - else: - ax.scatter(cen, width, - marker='.', color='C0', s=50, lw=0, label='measured widths', - zorder=3) + # Set the plot title + title = 'Order prediction model' + if bracket: + title += ' (bracketed)' + ax.text(0.5, 1.02, title, ha='center', va='center', transform=ax.transAxes) + + # Plot the detector bounds + ax.axvline(0, color='k', ls='--', lw=2) + ax.axvline(self.nspat, color='k', ls='--', lw=2) + + # Models + ax.plot(mod_cen, mod_width, color='C0', alpha=0.3, lw=3, zorder=3) + ax.plot(mod_cen, mod_gap, color='C2', alpha=0.3, lw=3, zorder=3) + + # Measurements included in the fit + ax.scatter(cen[w_good], width[w_good], + marker='.', color='C0', s=50, lw=0, label='fitted widths', zorder=4) + if np.any(w_rej): + # Rejected but not considered an outlier + ax.scatter(cen[w_rej], width[w_rej], + marker='x', color='C1', s=50, lw=1, label='rej widths', zorder=4) + if np.any(w_out): + # Outlier but not rejected + ax.scatter(cen[w_out], width[w_out], + marker='^', facecolor='none', edgecolor='C1', s=50, lw=1, + label='outlier widths', zorder=4) + if np.any(w_outrej): + # Both outlier and rejected + ax.scatter(cen[w_outrej], width[w_outrej], + marker='^', facecolor='C1', s=50, lw=1, label='rej,outlier widths', zorder=4) + # Orders to add ax.scatter(order_cen[order_missing], width_fit.eval(order_cen[order_missing]), - marker='x', color='C0', s=80, lw=1, label='missing widths', zorder=3) - ax.plot(order_cen, width_fit.eval(order_cen), color='C0', alpha=0.3, lw=3, zorder=2) - ax.scatter(cen[:-1], gap, marker='.', color='C2', s=50, lw=0, label='measured gaps', - zorder=3) + marker='s', facecolor='none', edgecolor='C0', s=80, lw=1, + label='missing widths', zorder=3) + + # Same as above but for gaps + ax.scatter(g_cen[g_good], gap[g_good], + marker='.', color='C2', s=50, lw=0, label='fitted gaps', zorder=4) + if np.any(g_rej): + ax.scatter(g_cen[g_rej], gap[g_rej], + marker='x', color='C4', s=50, lw=1, label='rej gaps', zorder=4) + if np.any(g_out): + ax.scatter(g_cen[g_out], gap[g_out], + marker='^', facecolor='none', edgecolor='C4', s=50, lw=1, + label='outlier gaps', zorder=4) + if np.any(g_outrej): + ax.scatter(g_cen[g_outrej], gap[g_outrej], + marker='^', facecolor='C4', s=50, lw=1, label='rej,outlier gaps', zorder=4) ax.scatter(order_cen[order_missing], gap_fit.eval(order_cen[order_missing]), - marker='x', color='C2', s=80, lw=1, label='missing gaps', zorder=3) - ax.plot(order_cen, gap_fit.eval(order_cen), color='C2', alpha=0.3, lw=3, zorder=2) + marker='s', facecolor='none', edgecolor='C2', s=80, lw=1, + label='missing gaps', zorder=3) + + # Add the y label and legend ax.set_ylabel('Order Width/Gap [pix]') ax.legend() - width_resid = width - width_fit.eval(cen) - med_wr = np.median(width_resid) - mad_wr = np.median(np.absolute(width_resid - med_wr)) - width_lim = [med_wr - 10*mad_wr, med_wr + 10*mad_wr] - - gap_resid = gap - gap_fit.eval(cen[:-1]) - med_gr = np.median(gap_resid) - mad_gr = np.median(np.absolute(gap_resid - med_gr)) - gap_lim = [med_gr - 10*mad_gr, med_gr + 10*mad_gr] - - ax = fig.add_axes([0.15, 0.25, 0.8, 0.1]) + # Plot the width residuals + ax = fig.add_axes([0.10, 0.25, 0.8, 0.1]) ax.minorticks_on() - ax.tick_params(which='both', direction='in', top=True, right=True) - ax.grid(True, which='major', color='0.7', zorder=0, linestyle='-') - ax.set_xlim([0, self.nspat]) + ax.tick_params(which='both', direction='in', top=True, right=False) + ax.set_xlim(xlim) ax.set_ylim(width_lim) ax.xaxis.set_major_formatter(ticker.NullFormatter()) - if np.any(combined_orders): - ax.scatter(cen[combined_orders], width_resid[combined_orders], - marker='^', color='C1', s=80, lw=0, zorder=3) - ax.scatter(cen[individual_orders], width_resid[individual_orders], - marker='.', color='C0', s=50, lw=0, zorder=3) - else: - ax.scatter(cen, width_resid, marker='.', color='C0', s=50, lw=0, zorder=3) + + # Plot the detector bounds + ax.axvline(0, color='k', ls='--', lw=2) + ax.axvline(self.nspat, color='k', ls='--', lw=2) + + # Model is at 0 residual ax.axhline(0, color='C0', alpha=0.3, lw=3, zorder=2) + # Measurements included in the fit + ax.scatter(cen[w_good], w_resid[w_good], marker='.', color='C0', s=50, lw=0, zorder=4) + # Rejected but not considered an outlier + ax.scatter(cen[w_rej], w_resid[w_rej], marker='x', color='C1', s=50, lw=1, zorder=4) + # Outlier but not rejected + ax.scatter(cen[w_out], w_resid[w_out], + marker='^', facecolor='none', edgecolor='C1', s=50, lw=1, zorder=4) + # Both outlier and rejected + ax.scatter(cen[w_outrej], w_resid[w_outrej], + marker='^', facecolor='C1', s=50, lw=1, zorder=4) + + # Add the label ax.set_ylabel(r'$\Delta$Width') - ax = fig.add_axes([0.15, 0.15, 0.8, 0.1]) + # Add a right axis that gives the residuals normalized by the rms; use + # this to set the grid. + axt = ax.twinx() + axt.minorticks_on() + axt.tick_params(which='both', direction='in') + axt.grid(True, which='major', color='0.7', zorder=0, linestyle='-') + axt.set_xlim(xlim) + axt.set_ylim(width_lim / w_rms) + axt.set_ylabel(r'$\Delta$/RMS') + + # Plot the gap residuals + ax = fig.add_axes([0.10, 0.15, 0.8, 0.1]) ax.minorticks_on() - ax.tick_params(which='both', direction='in', top=True, right=True) - ax.grid(True, which='major', color='0.7', zorder=0, linestyle='-') - ax.set_xlim([0, self.nspat]) + ax.tick_params(which='both', direction='in', top=True, right=False) + ax.set_xlim(xlim) ax.set_ylim(gap_lim) - ax.scatter(cen[:-1], gap_resid, marker='.', color='C2', s=50, lw=0, zorder=3) + + # Plot the detector bounds + ax.axvline(0, color='k', ls='--', lw=2) + ax.axvline(self.nspat, color='k', ls='--', lw=2) + + # Model is at 0 residual ax.axhline(0, color='C2', alpha=0.3, lw=3, zorder=2) + # Measurements included in the fit + ax.scatter(g_cen[g_good], g_resid[g_good], + marker='.', color='C2', s=50, lw=0, zorder=4) + # Rejected but not considered an outlier + ax.scatter(g_cen[g_rej], g_resid[g_rej], + marker='x', color='C4', s=50, lw=1, zorder=4) + # Outlier but not rejected + ax.scatter(g_cen[g_out], g_resid[g_out], + marker='^', facecolor='none', edgecolor='C4', s=50, lw=1, zorder=4) + # Both outlier and rejected + ax.scatter(g_cen[g_outrej], g_resid[g_outrej], + marker='^', facecolor='C4', s=50, lw=1, zorder=4) + + # Add the axis labels ax.set_ylabel(r'$\Delta$Gap') - ax.set_xlabel('Spatial pixel') + # Add a right axis that gives the residuals normalized by the rms; use + # this to set the grid. + axt = ax.twinx() + axt.minorticks_on() + axt.tick_params(which='both', direction='in') + axt.grid(True, which='major', color='0.7', zorder=0, linestyle='-') + axt.set_xlim(xlim) + axt.set_ylim(gap_lim / g_rms) + axt.set_ylabel(r'$\Delta$/RMS') + if ofile is None: plt.show() else: diff --git a/pypeit/extraction.py b/pypeit/extraction.py index 99f36f6432..0a818b1c22 100644 --- a/pypeit/extraction.py +++ b/pypeit/extraction.py @@ -64,7 +64,7 @@ class Extract: @classmethod def get_instance(cls, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_sky=None, bkg_redux_global_sky=None, waveTilts=None, tilts=None, wv_calib=None, waveimg=None, - bkg_redux=False, return_negative=False, std_redux=False, show=False, basename=None): + flatimg=None, bkg_redux=False, return_negative=False, std_redux=False, show=False, basename=None): """ Instantiate the Extract subclass appropriate for the provided spectrograph. @@ -101,6 +101,9 @@ def get_instance(cls, sciImg, slits, sobjs_obj, spectrograph, par, objtype, glob This is the waveCalib object which is optional, but either wv_calib or waveimg must be provided. waveimg (`numpy.ndarray`_, optional): Wave image. Either a wave image or wv_calib object (above) must be provided + flatimg (`numpy.ndarray`_, optional): + Flat image. This is optional, but if provided, it is used to extract the + normalized blaze profile. Same shape as ``sciImg``. bkg_redux (:obj:`bool`, optional): If True, the sciImg has been subtracted by a background image (e.g. standard treatment in the IR) @@ -128,12 +131,12 @@ def get_instance(cls, sciImg, slits, sobjs_obj, spectrograph, par, objtype, glob if c.__name__ == (spectrograph.pypeline + 'Extract'))( sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_sky=global_sky, bkg_redux_global_sky=bkg_redux_global_sky, waveTilts=waveTilts, tilts=tilts, - wv_calib=wv_calib, waveimg=waveimg, bkg_redux=bkg_redux, return_negative=return_negative, + wv_calib=wv_calib, waveimg=waveimg, flatimg=flatimg, bkg_redux=bkg_redux, return_negative=return_negative, std_redux=std_redux, show=show, basename=basename) def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_sky=None, bkg_redux_global_sky=None, waveTilts=None, tilts=None, wv_calib=None, waveimg=None, - bkg_redux=False, return_negative=False, std_redux=False, show=False, + flatimg=None, bkg_redux=False, return_negative=False, std_redux=False, show=False, basename=None): # Setup the parameters sets for this object. NOTE: This uses objtype, not frametype! @@ -146,6 +149,7 @@ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_ self.par = par self.global_sky = global_sky if global_sky is not None else np.zeros_like(sciImg.image) self.bkg_redux_global_sky = bkg_redux_global_sky + self.flatimg = flatimg self.basename = basename # Parse @@ -776,7 +780,7 @@ def local_skysub_extract(self, global_sky, sobjs, bkg_redux_global_sky=None, self.slits_right[:, slit_idx], self.sobjs[thisobj], ingpm=ingpm, bkg_redux_global_sky=bkg_redux_global_sky, - fwhmimg=self.fwhmimg, spat_pix=spat_pix, + fwhmimg=self.fwhmimg, flatimg=self.flatimg, spat_pix=spat_pix, model_full_slit=model_full_slit, sigrej=sigrej, model_noise=model_noise, std=self.std_redux, bsp=bsp, @@ -888,6 +892,7 @@ def local_skysub_extract(self, global_sky, sobjs, bkg_redux_global_sky=None, self.slits_right[:, gdorders], self.slitmask, sobjs, bkg_redux_global_sky=bkg_redux_global_sky, spat_pix=spat_pix, + fwhmimg=self.fwhmimg, flatimg=self.flatimg, std=self.std_redux, fit_fwhm=fit_fwhm, min_snr=min_snr, bsp=bsp, sigrej=sigrej, force_gauss=force_gauss, sn_gauss=sn_gauss, diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index 120a800bcc..44ba731508 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -15,7 +15,6 @@ from pypeit import specobjs from pypeit import msgs, utils -from pypeit import flatfield from pypeit.display import display from pypeit.core import skysub, qa, parse, flat, flexure from pypeit.core import procimg @@ -253,28 +252,29 @@ def create_skymask(self, sobjs_obj): subtraction. True = usable for sky subtraction, False = should be masked when sky subtracting. """ - # Masking options - boxcar_rad_pix = None - + # Instantiate the mask skymask = np.ones_like(self.sciImg.image, dtype=bool) - gdslits = np.where(np.invert(self.reduce_bpm))[0] - if sobjs_obj.nobj > 0: - for slit_idx in gdslits: - slit_spat = self.slits.spat_id[slit_idx] - qa_title ="Generating skymask for slit # {:d}".format(slit_spat) - msgs.info(qa_title) - thismask = self.slitmask == slit_spat - this_sobjs = sobjs_obj.SLITID == slit_spat - # Boxcar mask? - if self.par['reduce']['skysub']['mask_by_boxcar']: - boxcar_rad_pix = self.par['reduce']['extraction']['boxcar_radius'] / \ - self.get_platescale(slitord_id=self.slits.slitord_id[slit_idx]) - # Do it - skymask[thismask] = findobj_skymask.create_skymask(sobjs_obj[this_sobjs], thismask, - self.slits_left[:,slit_idx], - self.slits_right[:,slit_idx], - box_rad_pix=boxcar_rad_pix, - trim_edg=self.par['reduce']['findobj']['find_trim_edge']) + if sobjs_obj.nobj == 0: + # No objects found, so entire image contains sky + return skymask + + # Build the mask for each slit + boxcar_rad_pix = None + gdslits = np.where(np.logical_not(self.reduce_bpm))[0] + for slit_idx in gdslits: + slit_spat = self.slits.spat_id[slit_idx] + msgs.info(f'Generating skymask for slit # {slit_spat}') + thismask = self.slitmask == slit_spat + this_sobjs = sobjs_obj.SLITID == slit_spat + # Boxcar mask? + if self.par['reduce']['skysub']['mask_by_boxcar']: + boxcar_rad_pix = self.par['reduce']['extraction']['boxcar_radius'] / \ + self.get_platescale(slitord_id=self.slits.slitord_id[slit_idx]) + # Do it + skymask[thismask] = findobj_skymask.create_skymask( + sobjs_obj[this_sobjs], thismask, self.slits_left[:,slit_idx], + self.slits_right[:,slit_idx], box_rad_pix=boxcar_rad_pix, + trim_edg=self.par['reduce']['findobj']['find_trim_edge']) # Return return skymask @@ -325,8 +325,13 @@ def run(self, std_trace=None, show_peaks=False, show_skysub_fit=False): Parameters ---------- - std_trace : `numpy.ndarray`_, optional - Trace of the standard star + std_trace : `astropy.table.Table`_, optional + Table with the trace of the standard star on the input detector, + which is used as a crutch for tracing. For MultiSlit reduction, + the table has a single column: `TRACE_SPAT`. + For Echelle reduction, the table has two columns: `ECH_ORDER` and `TRACE_SPAT`. + The shape of each row must be (nspec,). For SlicerIFU reduction, std_trace is None. + If None, the slit boundaries are used as the crutch. show_peaks : :obj:`bool`, optional Show peaks in find_objects methods show_skysub_fit : :obj:`bool`, optional @@ -399,11 +404,13 @@ def find_objects(self, image, ivar, std_trace=None, Image to search for objects from. This floating-point image has shape (nspec, nspat) where the first dimension (nspec) is spectral, and second dimension (nspat) is spatial. - std_trace : `numpy.ndarray`_, optional - This is a one dimensional float array with shape = (nspec,) containing the standard star - trace which is used as a crutch for tracing. If the no - standard star is provided the code uses the the slit - boundaries as the crutch. + std_trace : `astropy.table.Table`_, optional + Table with the trace of the standard star on the input detector, + which is used as a crutch for tracing. For MultiSlit reduction, + the table has a single column: `TRACE_SPAT`. + For Echelle reduction, the table has two columns: `ECH_ORDER` and `TRACE_SPAT`. + The shape of each row must be (nspec,). For SlicerIFU reduction, std_trace is None. + If None, the slit boundaries are used as the crutch. show_peaks : :obj:`bool`, optional Generate QA showing peaks identified by object finding show_fits : :obj:`bool`, optional @@ -592,6 +599,7 @@ def global_skysub(self, skymask=None, bkg_redux_sciimg=None, pos_mask=not self.bkg_redux and not objs_not_masked, max_mask_frac=self.par['reduce']['skysub']['max_mask_frac'], show_fit=show_fit) + # Mask if something went wrong if np.sum(global_sky[thismask]) == 0.: msgs.warn("Bad fit to sky. Rejecting slit: {:d}".format(slit_spat)) @@ -722,11 +730,12 @@ def find_objects_pypeline(self, image, ivar, std_trace=None, Image to search for objects from. This floating-point image has shape (nspec, nspat) where the first dimension (nspec) is spectral, and second dimension (nspat) is spatial. - std_trace : `numpy.ndarray`_, optional - This is a one dimensional float array with shape = (nspec,) containing the standard star - trace which is used as a crutch for tracing. If the no - standard star is provided the code uses the the slit - boundaries as the crutch. + std_trace : `astropy.table.Table`_, optional + Table with the trace of the standard star on the input detector, + which is used as a crutch for tracing. For MultiSlit reduction, + the table has a single column: `TRACE_SPAT`. + The shape of each row must be (nspec,). If None, + the slit boundaries are used as the crutch. manual_extract_dict : :obj:`dict`, optional Dict guiding the manual extraction show_peaks : :obj:`bool`, optional @@ -790,13 +799,17 @@ def find_objects_pypeline(self, image, ivar, std_trace=None, maxnumber = self.par['reduce']['findobj']['maxnumber_std'] if self.std_redux \ else self.par['reduce']['findobj']['maxnumber_sci'] + # standard star + std_in = std_trace[0]['TRACE_SPAT'] if std_trace is not None else None + + # Find objects sobjs_slit = \ findobj_skymask.objs_in_slit(image, ivar, thismask, self.slits_left[:,slit_idx], self.slits_right[:,slit_idx], inmask=inmask, ncoeff=self.par['reduce']['findobj']['trace_npoly'], - std_trace=std_trace, + std_trace=std_in, snr_thresh=snr_thresh, hand_extract_dict=manual_extract_dict, specobj_dict=specobj_dict, show_peaks=show_peaks, @@ -872,11 +885,12 @@ def find_objects_pypeline(self, image, ivar, std_trace=None, Image to search for objects from. This floating-point image has shape (nspec, nspat) where the first dimension (nspec) is spectral, and second dimension (nspat) is spatial. - std_trace : `numpy.ndarray`_, optional - This is a one dimensional float array with shape = (nspec,) containing the standard star - trace which is used as a crutch for tracing. If the no - standard star is provided the code uses the the slit - boundaries as the crutch. + std_trace : `astropy.table.Table`_, optional + Table with the trace of the standard star on the input detector, + which is used as a crutch for tracing. For Echelle reduction, + the table has two columns: `ECH_ORDER` and `TRACE_SPAT`. + The shape of each row must be (nspec,). If None, + the slit boundaries are used as the crutch. manual_extract_dict : :obj:`dict`, optional Dict guiding the manual extraction show_peaks : :obj:`bool`, optional @@ -972,22 +986,6 @@ class SlicerIFUFindObjects(MultiSlitFindObjects): def __init__(self, sciImg, slits, spectrograph, par, objtype, **kwargs): super().__init__(sciImg, slits, spectrograph, par, objtype, **kwargs) - def initialize_slits(self, slits, initial=True): - """ - Gather all the :class:`~pypeit.slittrace.SlitTraceSet` attributes that - we'll use here in :class:`FindObjects`. Identical to the parent but the - slits are not trimmed. - - Args: - slits (:class:`~pypeit.slittrace.SlitTraceSet`): - SlitTraceSet object containing the slit boundaries that will be - initialized. - initial (:obj:`bool`, optional): - Use the initial definition of the slits. If False, - tweaked slits are used. - """ - super().initialize_slits(slits, initial=True) - def global_skysub(self, skymask=None, bkg_redux_sciimg=None, update_crmask=True, previous_sky=None, show_fit=False, show=False, show_objs=False, objs_not_masked=False, reinit_bpm: bool = True): @@ -1074,7 +1072,7 @@ def joint_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), # Use the FWHM map determined from the arc lines to convert the science frame # to have the same effective spectral resolution. - fwhm_map = self.wv_calib.build_fwhmimg(self.tilts, self.slits, initial=True, spat_flexure=self.spat_flexure_shift) + fwhm_map = self.wv_calib.build_fwhmimg(self.tilts, self.slits, spat_flexure=self.spat_flexure_shift) thismask = thismask & (fwhm_map != 0.0) # Need to include S/N for deconvolution sciimg = skysub.convolve_skymodel(self.sciImg.image, fwhm_map, thismask) @@ -1083,8 +1081,8 @@ def joint_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), model_ivar = self.sciImg.ivar sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] # Prepare the slitmasks for the relative spectral illumination - slitmask = self.slits.slit_img(pad=0, initial=True, flexure=self.spat_flexure_shift) - slitmask_trim = self.slits.slit_img(pad=-3, initial=True, flexure=self.spat_flexure_shift) + slitmask = self.slits.slit_img(pad=0, flexure=self.spat_flexure_shift) + slitmask_trim = self.slits.slit_img(pad=-3, flexure=self.spat_flexure_shift) for nn in range(numiter): msgs.info("Performing iterative joint sky subtraction - ITERATION {0:d}/{1:d}".format(nn+1, numiter)) # TODO trim_edg is in the parset so it should be passed in here via trim_edg=tuple(self.par['reduce']['trim_edge']), diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index e73f8462ae..e97b39825e 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -5,11 +5,15 @@ .. include:: ../include/links.rst """ +from pathlib import Path +from copy import deepcopy import inspect import numpy as np from scipy import interpolate, ndimage +from astropy.io import fits + from matplotlib import pyplot as plt from matplotlib import gridspec @@ -22,13 +26,21 @@ from pypeit import datamodel from pypeit import calibframe +from pypeit import edgetrace +from pypeit import io from pypeit.display import display +from pypeit.images import buildimage from pypeit.core import qa from pypeit.core import flat from pypeit.core import tracewave from pypeit.core import basis from pypeit.core import fitting +from pypeit.core import parse +from pypeit.core.mosaic import build_image_mosaic +from pypeit.spectrographs.util import load_spectrograph from pypeit import slittrace +from pypeit import dataPaths +from pypeit import cache class FlatImages(calibframe.CalibFrame): @@ -483,10 +495,14 @@ class FlatField: corrections. If None, the default parameters are used. slits (:class:`~pypeit.slittrace.SlitTraceSet`): The current slit traces. - wavetilts (:class:`~pypeit.wavetilts.WaveTilts`): - The current wavelength tilt traces; see - wv_calib (:class:`~pypeit.wavecalib.WaveCalib`): - Wavelength calibration object + wavetilts (:class:`~pypeit.wavetilts.WaveTilts`, optional): + The current fit to the wavelength tilts. I can be None, + for example, if slitless is True. + wv_calib (:class:`~pypeit.wavecalib.WaveCalib`, optional): + Wavelength calibration object. It can be None, for example, if + slitless is True. + slitless (bool, optional): + True if the input rawflatimg is a slitless flat. Default is False. spat_illum_only (bool, optional): Only perform the spatial illumination calculation, and ignore the 2D bspline fit. This should only be set to true if you @@ -510,8 +526,8 @@ class FlatField: Image of the relative spectral illumination for a multislit spectrograph """ - def __init__(self, rawflatimg, spectrograph, flatpar, slits, wavetilts, wv_calib, - spat_illum_only=False, qa_path=None, calib_key=None): + def __init__(self, rawflatimg, spectrograph, flatpar, slits, wavetilts=None, wv_calib=None, + slitless=False, spat_illum_only=False, qa_path=None, calib_key=None): # Defaults self.spectrograph = spectrograph @@ -528,7 +544,8 @@ def __init__(self, rawflatimg, spectrograph, flatpar, slits, wavetilts, wv_calib self.wv_calib = wv_calib # Worth a check - self.wavetilts.is_synced(self.slits) + if self.wavetilts is not None and not slitless: + self.wavetilts.is_synced(self.slits) # Attributes unique to this Object self.rawflatimg = rawflatimg # Un-normalized pixel flat as a PypeItImage @@ -540,6 +557,13 @@ def __init__(self, rawflatimg, spectrograph, flatpar, slits, wavetilts, wv_calib self.spat_illum_only = spat_illum_only self.spec_illum = None # Relative spectral illumination image self.waveimg = None + self.slitless = slitless # is this a slitless flat? + + # get waveimg here if available + if self.wavetilts is None or self.wv_calib is None: + msgs.warn("Wavelength calib or tilts are not available. Wavelength image not generated.") + else: + self.build_waveimg() # this set self.waveimg # Completed steps self.steps = [] @@ -578,13 +602,20 @@ def run(self, doqa=False, debug=False, show=False): :class:`FlatImages`: Container with the results of the flat-field analysis. """ + + # check if self.wavetilts is available. It can be None if the flat is slitless, but it's needed otherwise + if self.wavetilts is None and not self.slitless: + msgs.warn("Wavelength tilts are not available. Cannot generate this flat image.") + return None + # Fit it # NOTE: Tilts do not change and self.slits is updated internally. if not self.flatpar['fit_2d_det_response']: # This spectrograph does not have a structure correction # implemented. Ignore detector structure. self.fit(spat_illum_only=self.spat_illum_only, doqa=doqa, debug=debug) - else: # Iterate on the pixelflat if required by the spectrograph + elif self.waveimg is not None: + # Iterate on the pixelflat if required by the spectrograph # User has requested a structure correction. # Note: This will only be performed if it is coded for each individual spectrograph. # Make a copy of the original flat @@ -671,11 +702,14 @@ def build_waveimg(self): Generate an image of the wavelength of each pixel. """ msgs.info("Generating wavelength image") - flex = self.wavetilts.spat_flexure - slitmask = self.slits.slit_img(initial=True, flexure=flex) - tilts = self.wavetilts.fit2tiltimg(slitmask, flexure=flex) - # Save to class attribute for inclusion in the Flat calibration frame - self.waveimg = self.wv_calib.build_waveimg(tilts, self.slits, spat_flexure=flex) + if self.wavetilts is None or self.wv_calib is None: + msgs.error("Wavelength calib or tilts are not available. Cannot generate wavelength image.") + else: + flex = self.wavetilts.spat_flexure + slitmask = self.slits.slit_img(initial=True, flexure=flex) + tilts = self.wavetilts.fit2tiltimg(slitmask, flexure=flex) + # Save to class attribute for inclusion in the Flat calibration frame + self.waveimg = self.wv_calib.build_waveimg(tilts, self.slits, spat_flexure=flex) def show(self, wcs_match=True): """ @@ -803,6 +837,7 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): # Set parameters (for convenience; spec_samp_fine = self.flatpar['spec_samp_fine'] spec_samp_coarse = self.flatpar['spec_samp_coarse'] + tweak_method = self.flatpar['tweak_method'] tweak_slits = self.flatpar['tweak_slits'] tweak_slits_thresh = self.flatpar['tweak_slits_thresh'] tweak_slits_maxfrac = self.flatpar['tweak_slits_maxfrac'] @@ -815,9 +850,6 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): npoly = self.flatpar['twod_fit_npoly'] saturated_slits = self.flatpar['saturated_slits'] - # Build wavelength image -- not always used, but for convenience done here - if self.waveimg is None: self.build_waveimg() - # Setup images nspec, nspat = self.rawflatimg.image.shape rawflat = self.rawflatimg.image @@ -832,14 +864,11 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): ivar_log = gpm_log.astype(float)/0.5**2 # Get the non-linear count level - # TODO: This is currently hacked to deal with Mosaics - try: + if self.rawflatimg.is_mosaic: + # if this is a mosaic we take the maximum value among all the detectors + nonlinear_counts = np.max([rawdets.nonlinear_counts() for rawdets in self.rawflatimg.detector.detectors]) + else: nonlinear_counts = self.rawflatimg.detector.nonlinear_counts() - except: - nonlinear_counts = 1e10 - # Other setup -# nonlinear_counts = self.spectrograph.nonlinear_counts(self.rawflatimg.detector) -# nonlinear_counts = self.rawflatimg.detector.nonlinear_counts() # TODO -- JFH -- CONFIRM THIS SHOULD BE ON INIT # It does need to be *all* of the slits @@ -961,10 +990,13 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): # TODO: Put this stuff in a self.spectral_fit method? # Create the tilts image for this slit - # TODO -- JFH Confirm the sign of this shift is correct! - _flexure = 0. if self.wavetilts.spat_flexure is None else self.wavetilts.spat_flexure - tilts = tracewave.fit2tilts(rawflat.shape, self.wavetilts['coeffs'][:,:,slit_idx], - self.wavetilts['func2d'], spat_shift=-1*_flexure) + if self.slitless: + tilts = np.tile(np.arange(rawflat.shape[0]) / rawflat.shape[0], (rawflat.shape[1], 1)).T + else: + # TODO -- JFH Confirm the sign of this shift is correct! + _flexure = 0. if self.wavetilts.spat_flexure is None else self.wavetilts.spat_flexure + tilts = tracewave.fit2tilts(rawflat.shape, self.wavetilts['coeffs'][:,:,slit_idx], + self.wavetilts['func2d'], spat_shift=-1*_flexure) # Convert the tilt image to an image with the spectral pixel index spec_coo = tilts * (nspec-1) @@ -1025,7 +1057,7 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): if sticky: # Add rejected pixels to gpm - gpm[spec_gpm] = (spec_gpm_fit & spec_gpm_data)[np.argsort(spec_srt)] + gpm[spec_gpm] = (spec_gpm_fit & spec_gpm_data)[np.argsort(spec_srt, kind='stable')] # Construct the model of the flat-field spectral shape # including padding on either side of the slit. @@ -1089,9 +1121,10 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): # TODO: Will this break if left_thresh, left_shift, self.slits.left_tweak[:,slit_idx], right_thresh, \ right_shift, self.slits.right_tweak[:,slit_idx] \ - = flat.tweak_slit_edges(self.slits.left_init[:,slit_idx], + = self.tweak_slit_edges(self.slits.left_init[:,slit_idx], self.slits.right_init[:,slit_idx], spat_coo_data, spat_flat_data, + method=tweak_method, thresh=tweak_slits_thresh, maxfrac=tweak_slits_maxfrac, debug=debug) # TODO: Because the padding doesn't consider adjacent @@ -1102,14 +1135,15 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): # Update the onslit mask _slitid_img = self.slits.slit_img(slitidx=slit_idx, initial=False) onslit_tweak = _slitid_img == slit_spat - spat_coo_tweak = self.slits.spatial_coordinate_image(slitidx=slit_idx, - slitid_img=_slitid_img) + # Note, we need to get the full image with the coordinates similar to spat_coo_init, otherwise, the + # tweaked locations are biased. + spat_coo_tweak = self.slits.spatial_coordinate_image(slitidx=slit_idx, full=True, slitid_img=_slitid_img) # Construct the empirical illumination profile # TODO This is extremely inefficient, because we only need to re-fit the illumflat, but # spatial_fit does both the reconstruction of the illumination function and the bspline fitting. # Only the b-spline fitting needs be reddone with the new tweaked spatial coordinates, so that would - # save a ton of runtime. It is not a trivial change becauase the coords are sorted, etc. + # save a ton of runtime. It is not a trivial change because the coords are sorted, etc. exit_status, spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit, \ spat_flat_fit, spat_flat_data_raw = self.spatial_fit( norm_spec, spat_coo_tweak, median_slit_widths[slit_idx], spat_gpm, gpm, debug=False) @@ -1160,7 +1194,7 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): spat_model[onslit_padded] = spat_bspl.value(spat_coo_final[onslit_padded])[0] specspat_illum = np.fmax(spec_model, 1.0) * spat_model norm_spatspec = rawflat / specspat_illum - self.spatial_fit_finecorr(norm_spatspec, onslit_tweak, slit_idx, slit_spat, gpm, doqa=doqa) + spat_illum_fine = self.spatial_fit_finecorr(norm_spatspec, onslit_tweak, slit_idx, slit_spat, gpm, doqa=doqa)[onslit_tweak] # ---------------------------------------------------------- # Construct the illumination profile with the tweaked edges @@ -1270,8 +1304,8 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): 'flat-field corrections included in model of slit {0}!'.format(slit_spat)) self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADFLATCALIB') else: - twod_model[twod_gpm] = twod_flat_fit[np.argsort(twod_srt)] - twod_gpm_out[twod_gpm] = twod_gpm_fit[np.argsort(twod_srt)] + twod_model[twod_gpm] = twod_flat_fit[np.argsort(twod_srt, kind='stable')] + twod_gpm_out[twod_gpm] = twod_gpm_fit[np.argsort(twod_srt, kind='stable')] # Construct the full flat-field model @@ -1280,19 +1314,35 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): * np.fmax(self.msillumflat[onslit_tweak], 0.05) \ * np.fmax(spec_model[onslit_tweak], 1.0) + # Check for infinities and NaNs in the flat-field model + winfnan = np.where(np.logical_not(np.isfinite(self.flat_model[onslit_tweak]))) + if winfnan[0].size != 0: + msgs.warn('There are {0:d} pixels with non-finite values in the flat-field model ' + 'for slit {1:d}!'.format(winfnan[0].size, slit_spat) + msgs.newline() + + 'These model pixel values will be set to the raw pixel value.') + self.flat_model[np.where(onslit_tweak)[0][winfnan]] = rawflat[np.where(onslit_tweak)[0][winfnan]] + # Check for unrealistically high or low values of the model + whilo = np.where((self.flat_model[onslit_tweak] >= nonlinear_counts) | + (self.flat_model[onslit_tweak] <= 0.0)) + if whilo[0].size != 0: + msgs.warn('There are {0:d} pixels with unrealistically high or low values in the flat-field model ' + 'for slit {1:d}!'.format(whilo[0].size, slit_spat) + msgs.newline() + + 'These model pixel values will be set to the raw pixel value.') + self.flat_model[np.where(onslit_tweak)[0][whilo]] = rawflat[np.where(onslit_tweak)[0][whilo]] + # Construct the pixel flat #trimmed_slitid_img_anew = self.slits.slit_img(pad=-trim, slitidx=slit_idx) #onslit_trimmed_anew = trimmed_slitid_img_anew == slit_spat - self.mspixelflat[onslit_tweak] = rawflat[onslit_tweak]/self.flat_model[onslit_tweak] + self.mspixelflat[onslit_tweak] = rawflat[onslit_tweak] * utils.inverse(self.flat_model[onslit_tweak]) # TODO: Add some code here to treat the edges and places where fits # go bad? # Minimum wavelength? - if self.flatpar['pixelflat_min_wave'] is not None: + if self.flatpar['pixelflat_min_wave'] is not None and self.waveimg is not None: bad_wv = self.waveimg[onslit_tweak] < self.flatpar['pixelflat_min_wave'] self.mspixelflat[np.where(onslit_tweak)[0][bad_wv]] = 1. # Maximum wavelength? - if self.flatpar['pixelflat_max_wave'] is not None: + if self.flatpar['pixelflat_max_wave'] is not None and self.waveimg is not None: bad_wv = self.waveimg[onslit_tweak] > self.flatpar['pixelflat_max_wave'] self.mspixelflat[np.where(onslit_tweak)[0][bad_wv]] = 1. @@ -1375,7 +1425,8 @@ def spatial_fit(self, norm_spec, spat_coo, median_slit_width, spat_gpm, gpm, deb return exit_status, spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit, \ spat_flat_fit, spat_flat_data_raw - def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, slit_trim=3, doqa=False): + def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, + slit_trim=3, tolerance=0.1, doqa=False): """ Generate a relative scaling image for a slicer IFU. All slits are scaled relative to a reference slit, specified in @@ -1399,9 +1450,22 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s Trim the slit edges by this number of pixels during the fitting. Note that the fit will be evaluated on the pixels indicated by onslit_tweak. A positive number trims the slit edges, a negative number pads the slit edges. + tolerance : float, optional + Tolerance for the relative scaling of the slits. A value of 0.1 means that the + relative scaling of the slits must be within 10% of unity. Any data outside of + this tolerance will be masked. doqa : :obj:`bool`, optional: Save the QA? + + Returns + ------- + illumflat_finecorr: `numpy.ndarray`_ + An image (same shape as normed) containing the fine correction to the spatial illumination profile """ + # check id self.waveimg is available + if self.waveimg is None: + msgs.warn("Cannot perform the fine correction to the spatial illumination without the wavelength image.") + return # TODO :: Include fit_order in the parset?? fit_order = np.array([3, 6]) slit_txt = self.slits.slitord_txt @@ -1413,7 +1477,8 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s onslit_tweak_trim = self.slits.slit_img(pad=-slit_trim, slitidx=slit_idx, initial=False) == slit_spat # Setup slitimg = (slit_spat + 1) * onslit_tweak.astype(int) - 1 # Need to +1 and -1 so that slitimg=-1 when off the slit - left, right, msk = self.slits.select_edges(initial=True, flexure=self.wavetilts.spat_flexure) + + left, right, msk = self.slits.select_edges(flexure=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0) this_left = left[:, slit_idx] this_right = right[:, slit_idx] slitlen = int(np.median(this_right - this_left)) @@ -1422,9 +1487,8 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s this_slit = np.where(onslit_tweak & self.rawflatimg.select_flag(invert=True) & (self.waveimg!=0.0)) this_wave = self.waveimg[this_slit] xpos_img = self.slits.spatial_coordinate_image(slitidx=slit_idx, - initial=True, slitid_img=slitimg, - flexure_shift=self.wavetilts.spat_flexure) + flexure_shift=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0) # Generate the trimmed versions for fitting this_slit_trim = np.where(onslit_tweak_trim & self.rawflatimg.select_flag(invert=True)) this_wave_trim = self.waveimg[this_slit_trim] @@ -1442,6 +1506,10 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s if xfrac * slitlen < slit_trim: xfrac = slit_trim/slitlen gpmfit[np.where((xpos_fit < xfrac) | (xpos_fit > 1-xfrac))] = False + # If the data deviate too much from unity, mask them. We're only interested in a + # relative correction that's less than ~10% from unity. + gpmfit[np.where((normed[this_slit_trim] < 1-tolerance) | (normed[this_slit_trim] > 1 + tolerance))] = False + # Perform the full fit fullfit = fitting.robust_fit(xpos_fit, normed[this_slit_trim], fit_order, x2=ypos_fit, in_gpm=gpmfit, function='legendre2d', upper=2, lower=2, maxdev=1.0, minx=0.0, maxx=1.0, minx2=0.0, maxx2=1.0) @@ -1452,7 +1520,10 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s illumflat_finecorr[this_slit] = fullfit.eval(xpos, ypos) else: msgs.warn(f"Fine correction to the spatial illumination failed for {slit_txt} {slit_ordid}") - return + return illumflat_finecorr + + # If corrections exceed the tolerance, then clip them to the level of the tolerance + illumflat_finecorr = np.clip(illumflat_finecorr, 1-tolerance, 1+tolerance) # Prepare QA if doqa: @@ -1465,7 +1536,7 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s normed[np.logical_not(onslit_tweak)] = 1 # For the QA, make everything off the slit equal to 1 spatillum_finecorr_qa(normed, illumflat_finecorr, this_left, this_right, ypos_fit, this_slit_trim, outfile=outfile, title=title, half_slen=slitlen//2) - return + return illumflat_finecorr def extract_structure(self, rawflat_orig, slit_trim=3): """ @@ -1489,6 +1560,11 @@ def extract_structure(self, rawflat_orig, slit_trim=3): divided by the spectral and spatial illumination profile fits). """ msgs.info("Extracting flatfield structure") + + # check if the waveimg is available + if self.waveimg is None: + msgs.error("Cannot perform the extraction of the flatfield structure without the wavelength image.") + # Build the mask and make a temporary instance of FlatImages bpmflats = self.build_mask() # Initialise bad splines (for when the fit goes wrong) @@ -1512,7 +1588,7 @@ def extract_structure(self, rawflat_orig, slit_trim=3): scale_model = illum_profile_spectral(rawflat, self.waveimg, self.slits, slit_illum_ref_idx=self.flatpar['slit_illum_ref_idx'], model=None, gpmask=gpm, skymask=None, trim=self.flatpar['slit_trim'], - flexure=self.wavetilts.spat_flexure, + flexure=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0, smooth_npix=self.flatpar['slit_illum_smooth_npix']) # Trim the edges by a few pixels to avoid edge effects onslits_trim = gpm & (self.slits.slit_img(pad=-slit_trim, initial=False) != -1) @@ -1558,8 +1634,10 @@ def spectral_illumination(self, gpm=None, debug=False): An image containing the appropriate scaling """ msgs.info("Deriving spectral illumination profile") - # Generate a wavelength image - if self.waveimg is None: self.build_waveimg() + # check if the waveimg is available + if self.waveimg is None: + msgs.warn("Cannot perform the spectral illumination without the wavelength image.") + return None msgs.info('Performing a joint fit to the flat-field response') # Grab some parameters trim = self.flatpar['slit_trim'] @@ -1573,8 +1651,281 @@ def spectral_illumination(self, gpm=None, debug=False): return illum_profile_spectral(rawflat, self.waveimg, self.slits, slit_illum_ref_idx=self.flatpar['slit_illum_ref_idx'], model=None, gpmask=gpm, skymask=None, trim=trim, - flexure=self.wavetilts.spat_flexure, - smooth_npix=self.flatpar['slit_illum_smooth_npix']) + flexure=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0, + smooth_npix=self.flatpar['slit_illum_smooth_npix'], + debug=debug) + + def tweak_slit_edges(self, left, right, spat_coo, norm_flat, method='threshold', thresh=0.93, + maxfrac=0.1, debug=False): + r""" + Tweak the slit edges based on the normalized slit illumination profile. + + Args: + left (`numpy.ndarray`_): + Array with the left slit edge for a single slit. Shape is + :math:`(N_{\rm spec},)`. + right (`numpy.ndarray`_): + Array with the right slit edge for a single slit. Shape + is :math:`(N_{\rm spec},)`. + spat_coo (`numpy.ndarray`_): + Spatial pixel coordinates in fractions of the slit width + at each spectral row for the provided normalized flat + data. Coordinates are relative to the left edge (with the + left edge at 0.). Shape is :math:`(N_{\rm flat},)`. + Function assumes the coordinate array is sorted. + norm_flat (`numpy.ndarray`_) + Normalized flat data that provide the slit illumination + profile. Shape is :math:`(N_{\rm flat},)`. + method (:obj:`str`, optional): + Method to use for tweaking the slit edges. Options are: + + - ``'threshold'``: Use the threshold to set the slit edge + and then shift it to the left or right based on the + illumination profile. + + - ``'gradient'``: Use the gradient of the illumination + profile to set the slit edge and then shift it to the left + or right based on the illumination profile. + + thresh (:obj:`float`, optional): + Threshold of the normalized flat profile at which to + place the two slit edges. + maxfrac (:obj:`float`, optional): + The maximum fraction of the slit width that the slit edge + can be adjusted by this algorithm. If ``maxfrac = 0.1``, + this means the maximum change in the slit width (either + narrowing or broadening) is 20% (i.e., 10% for either + edge). + debug (:obj:`bool`, optional): + Show flow interrupting plots that show illumination + profile in the case of a failure and the placement of the + tweaked edge for each side of the slit regardless. + + Returns: + tuple: Returns six objects: + + - The threshold used to set the left edge + - The fraction of the slit that the left edge is shifted to the + right + - The adjusted left edge + - The threshold used to set the right edge + - The fraction of the slit that the right edge is shifted to the + left + - The adjusted right edge + + """ + # TODO :: Since this is just a wrapper, and not really "core", maybe it should be moved to pypeit.flatfield? + # Tweak the edges via the specified method + if method == "threshold": + return flat.tweak_slit_edges_threshold(left, right, spat_coo, norm_flat, + thresh=thresh, maxfrac=maxfrac, debug=debug) + elif method == "gradient": + return flat.tweak_slit_edges_gradient(left, right, spat_coo, norm_flat, maxfrac=maxfrac, debug=debug) + else: + msgs.error("Method for tweaking slit edges not recognized: {0}".format(method)) + + +class SlitlessFlat: + """ + Class to generate a slitless pixel flat-field calibration image. + + Args: + fitstbl (:class:`~pypeit.metadata.PypeItMetaData`): + The class holding the metadata for all the frames. + slitless_rows (`numpy.ndarray`_): + Boolean array selecting the rows in the fitstbl that + correspond to the slitless frames. + spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): + The spectrograph object. + par (:class:`~pypeit.par.pypeitpar.CalibrationsPar`): + Parameter set defining optional parameters of PypeIt's algorithms + for Calibrations + qa_path (`Path`_): + Path for the QA diagnostics. + + """ + + def __init__(self, fitstbl, slitless_rows, spectrograph, par, qa_path=None): + + self.fitstbl = fitstbl + # Boolean array selecting the rows in the fitstbl that correspond to the slitless frames. + self.slitless_rows = slitless_rows + self.spectrograph = spectrograph + self.par = par + self.qa_path = qa_path + + def slitless_pixflat_fname(self): + """ + Generate the name of the slitless pixel flat file. + + Returns: + :obj:`str`: The name of the slitless pixel flat + + """ + if len(self.slitless_rows) == 0: + msgs.error('No slitless_pixflat frames found. Cannot generate the slitless pixel flat file name.') + + # generate the slitless pixel flat file name + spec_name = self.fitstbl.spectrograph.name + date = self.fitstbl.construct_obstime(self.slitless_rows[0]).iso.split(' ')[0].replace('-', '') if \ + self.fitstbl[self.slitless_rows][0]['mjd'] is not None else '00000000' + # setup info to add to the filename + dispname = '' if 'dispname' not in self.spectrograph.configuration_keys() else \ + f"_{self.fitstbl[self.slitless_rows[0]]['dispname'].replace('/', '_').replace(' ', '_').replace('(', '').replace(')', '').replace(':', '_').replace('+', '_')}" + dichroic = '' if 'dichroic' not in self.spectrograph.configuration_keys() else \ + f"_d{self.fitstbl[self.slitless_rows[0]]['dichroic']}" + binning = self.fitstbl[self.slitless_rows[0]]['binning'].replace(',', 'x') + # file name + return f'pixelflat_{spec_name}{dispname}{dichroic}_{binning}_{date}.fits' + + def make_slitless_pixflat(self, msbias=None, msdark=None, calib_dir=None, write_qa=False, show=False): + """ + Generate and save to disc a slitless pixel flat-field calibration images. + The pixel flat file will have one extension per detector, even in the case of a mosaic. + Contrary to the regular calibration flow, the slitless pixel flat is created for all detectors + of the current spectrograph at once, and not only the one for the current detector. + Since the slitless pixel flat images are saved to disc, this approach helps with the I/O + This is a method is used in `~pypeit.calibrations.get_flats()`. + + Note: par['flatfield']['pixelflat_file'] is updated in this method. + + Args: + msbias (:class:`~pypeit.images.buildimage.BiasImage`, optional): + Bias image for bias subtraction; passed to + :func:`~pypeit.images.buildimage.buildimage_fromlist()` + msdark (:class:`~pypeit.images.buildimage.DarkImage`, optional): + Dark-current image; passed to + :func:`~pypeit.images.buildimage.buildimage_fromlist()` + calib_dir (`Path`_): + Path for the processed calibration files. + write_qa (:obj:`bool`, optional): + Write QA plots to disk? + show (:obj:`bool`, optional): + Show the diagnostic plots? + + Returns: + :obj:`str`: The name of the slitless pixel flat file that was generated. + + """ + + # First thing first, check if the user has provided slitless_pixflat frames + if len(self.slitless_rows) == 0: + # return unchanged self.par['flatfield']['pixelflat_file'] + return self.par['flatfield']['pixelflat_file'] + + # all detectors of this spectrograph + _detectors = np.array(self.spectrograph.select_detectors()) + + # Check if a user-provided slitless pixelflat already exists for the current detectors + if self.par['flatfield']['pixelflat_file'] is not None: + _pixel_flat_file = dataPaths.pixelflat.get_file_path(self.par['flatfield']['pixelflat_file'], + return_none=True) + + if _pixel_flat_file is not None: + # get detector names + detnames = np.array([self.spectrograph.get_det_name(_det) for _det in _detectors]) + # open the file + with io.fits_open(_pixel_flat_file) as hdu: + # list of available detectors in the pixel flat file + file_detnames = [h.name.split('-')[0] for h in hdu] + # check if the current detnames are in the list + in_file = np.array([d in file_detnames for d in detnames]) + # if all detectors are in the file, return + if np.all(in_file): + msgs.info(f"Both slitless_pixflat frames and user-defined file found. " + f"The user-defined file will be used: {self.par['flatfield']['pixelflat_file']}") + # return unchanged self.par['flatfield']['pixelflat_file'] + return self.par['flatfield']['pixelflat_file'] + else: + # get the detectors that are not in the file + _detectors = _detectors[np.logical_not(in_file)] + detnames = detnames[np.logical_not(in_file)] + msgs.info(f'Both slitless_pixflat frames and user-defined file found, but the ' + f'following detectors are not in the file: {detnames}. Using the ' + f'slitless_pixflat frames to generate the missing detectors.') + + # make the slitless pixel flat + pixflat_norm_list = [] + detname_list = [] + for _det in _detectors: + # Parse the raw slitless pixelflat frames. Note that this is spectrograph dependent. + # If the method does not exist in the specific spectrograph class, nothing will happen + this_raw_idx = self.spectrograph.parse_raw_files(self.fitstbl[self.slitless_rows], det=_det, + ftype='slitless_pixflat') + if len(this_raw_idx) == 0: + msgs.warn(f'No raw slitless_pixflat frames found for {self.spectrograph.get_det_name(_det)}. ' + f'Continuing...') + continue + this_raw_files = self.fitstbl.frame_paths(self.slitless_rows[this_raw_idx]) + msgs.info(f'Creating slitless pixel-flat calibration frame ' + f'for {self.spectrograph.get_det_name(_det)} using files: ') + for f in this_raw_files: + msgs.prindent(f'{Path(f).name}') + + # Reset the BPM + msbpm = self.spectrograph.bpm(this_raw_files[0], _det, msbias=msbias if self.par['bpm_usebias'] else None) + + # trace image + traceimg = buildimage.buildimage_fromlist(self.spectrograph, _det, self.par['traceframe'], + [this_raw_files[0]], dark=msdark, bias=msbias, bpm=msbpm) + # slit edges + # we need to change some parameters for the slit edge tracing + edges_par = deepcopy(self.par['slitedges']) + # lower the threshold for edge detection + edges_par['edge_thresh'] = 50. + # this is used for longslit (i.e., no pca) + edges_par['sync_predict'] = 'nearest' + # remove spurious edges by setting a large minimum slit gap (20% of the detector size + platescale = parse.parse_binning(traceimg.detector.binning)[1] * traceimg.detector['platescale'] + edges_par['minimum_slit_gap'] = 0.2 * traceimg.image.shape[1] * platescale + # if no slits are found the bound_detector parameter add 2 traces at the detector edges + edges_par['bound_detector'] = True + # set the buffer to 0 + edges_par['det_buffer'] = 0 + _spectrograph = deepcopy(self.spectrograph) + # need to treat this as a MultiSlit spectrograph (no echelle parameters used) + _spectrograph.pypeline = 'MultiSlit' + edges = edgetrace.EdgeTraceSet(traceimg, _spectrograph, edges_par, auto=True) + slits = edges.get_slits() + if show: + edges.show(title='Slitless flat edge tracing') + # + # flat image + slitless_pixel_flat = buildimage.buildimage_fromlist(self.spectrograph, _det, self.par['slitless_pixflatframe'], + this_raw_files, dark=msdark, bias=msbias, bpm=msbpm) + + # increase saturation threshold (some hires slitless flats are very bright) + slitless_pixel_flat.detector.saturation *= 1.5 + # Initialise the pixel flat + flatpar = deepcopy(self.par['flatfield']) + # do not tweak the slits + flatpar['tweak_slits'] = False + flatpar['slit_illum_finecorr'] = False + pixelFlatField = FlatField(slitless_pixel_flat, self.spectrograph, flatpar, slits, wavetilts=None, + wv_calib=None, slitless=True, qa_path=self.qa_path) + + # Generate + pixelflatImages = pixelFlatField.run(doqa=write_qa, show=show) + pixflat_norm_list.append(pixelflatImages.pixelflat_norm) + detname_list.append(self.spectrograph.get_det_name(_det)) + + if len(detname_list) > 0: + # get the pixel flat file name + if self.par['flatfield']['pixelflat_file'] is not None and _pixel_flat_file is not None: + fname = self.par['flatfield']['pixelflat_file'] + else: + fname = self.slitless_pixflat_fname() + # file will be saved in the reduction directory, but also cached in the data/pixelflats folder + # therefore we update self.par['flatfield']['pixelflat_file'] to the new file, + # so that it can be used for the rest of the reduction and for the other files in the same run + self.par['flatfield']['pixelflat_file'] = fname + + # Save the result + write_pixflat_to_fits(pixflat_norm_list, detname_list, self.spectrograph.name, + calib_dir.parent if calib_dir is not None else Path('.').absolute(), + fname, to_cache=True) + + return self.par['flatfield']['pixelflat_file'] def spatillum_finecorr_qa(normed, finecorr, left, right, ypos, cut, outfile=None, title=None, half_slen=50): @@ -1801,7 +2152,7 @@ def show_flats(image_list, wcs_match=True, slits=None, waveimg=None): # TODO :: This could possibly be moved to core.flat def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_npix=None, polydeg=None, - model=None, gpmask=None, skymask=None, trim=3, flexure=None, maxiter=5): + model=None, gpmask=None, skymask=None, trim=3, flexure=None, maxiter=5, debug=False): """ Determine the relative spectral illumination of all slits. Currently only used for image slicer IFUs. @@ -1834,6 +2185,8 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ Spatial flexure maxiter : :obj:`int` Maximum number of iterations to perform + debug : :obj:`bool` + Show the results of the relative spectral illumination correction Returns ------- @@ -1850,19 +2203,19 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ gpm = gpmask if (gpmask is not None) else np.ones_like(rawimg, dtype=bool) modelimg = model if (model is not None) else rawimg.copy() # Setup the slits - slitid_img_init = slits.slit_img(pad=0, initial=True, flexure=flexure) - slitid_img_trim = slits.slit_img(pad=-trim, initial=True, flexure=flexure) + slitid_img = slits.slit_img(pad=0, flexure=flexure) + slitid_img_trim = slits.slit_img(pad=-trim, flexure=flexure) scaleImg = np.ones_like(rawimg) modelimg_copy = modelimg.copy() # Obtain the minimum and maximum wavelength of all slits mnmx_wv = np.zeros((slits.nslits, 2)) for slit_idx, slit_spat in enumerate(slits.spat_id): - onslit_init = (slitid_img_init == slit_spat) + onslit_init = (slitid_img == slit_spat) mnmx_wv[slit_idx, 0] = np.min(waveimg[onslit_init]) mnmx_wv[slit_idx, 1] = np.max(waveimg[onslit_init]) wavecen = np.mean(mnmx_wv, axis=1) # Sort the central wavelengths by those that are closest to the reference slit - wvsrt = np.argsort(np.abs(wavecen - wavecen[slit_illum_ref_idx])) + wvsrt = np.argsort(np.abs(wavecen - wavecen[slit_illum_ref_idx]), kind='stable') # Prepare wavelength array for all spectra dwav = np.max((mnmx_wv[:, 1] - mnmx_wv[:, 0])/slits.nspec) @@ -1892,7 +2245,7 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ for ss in range(1, slits.spat_id.size): # Calculate the region of overlap onslit_b = (slitid_img_trim == slits.spat_id[wvsrt[ss]]) - onslit_b_init = (slitid_img_init == slits.spat_id[wvsrt[ss]]) + onslit_b_init = (slitid_img == slits.spat_id[wvsrt[ss]]) onslit_b_olap = onslit_b & gpm & (waveimg >= mnmx_wv[wvsrt[ss], 0]) & (waveimg <= mnmx_wv[wvsrt[ss], 1]) & skymask_now hist, edge = np.histogram(waveimg[onslit_b_olap], bins=wavebins, weights=modelimg_copy[onslit_b_olap]) cntr, edge = np.histogram(waveimg[onslit_b_olap], bins=wavebins) @@ -1943,7 +2296,6 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ modelimg_copy /= relscl_model if max(abs(1/minv), abs(maxv)) < 1.005: # Relative accuracy of 0.5% is sufficient break - debug = False if debug: embed() ricp = rawimg.copy() @@ -1958,12 +2310,24 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ scale_ref = histScl * norm plt.subplot(211) plt.plot(wave_ref, this_spec) + plt.xlim([3600, 4500]) plt.subplot(212) plt.plot(wave_ref, scale_ref) + plt.xlim([3600, 4500]) plt.subplot(211) plt.plot(wave_ref, spec_ref, 'k--') + plt.xlim([3600, 4500]) + plt.show() + # Plot the relative scales of each slit + scales_med, scales_avg = np.zeros(slits.spat_id.size), np.zeros(slits.spat_id.size) + for ss in range(slits.spat_id.size): + onslit_ref_trim = (slitid_img_trim == slits.spat_id[ss]) & gpm & skymask_now & (waveimg>3628) & (waveimg<4510) + scales_med[ss] = np.median(ricp[onslit_ref_trim]/scaleImg[onslit_ref_trim]) + scales_avg[ss] = np.mean(ricp[onslit_ref_trim]/scaleImg[onslit_ref_trim]) + plt.plot(slits.spat_id, scales_med, 'bo-', label='Median') + plt.plot(slits.spat_id, scales_avg, 'ro-', label='Mean') + plt.legend() plt.show() - return scaleImg @@ -2010,4 +2374,203 @@ def merge(init_cls, merge_cls): return FlatImages(**dd) +def write_pixflat_to_fits(pixflat_norm_list, detname_list, spec_name, outdir, pixelflat_name, to_cache=True): + """ + Write the pixel-to-pixel flat-field images to a FITS file. + The FITS file will have an extension for each detector (never a mosaic). + The `load_pixflat()` method read this file and transform it into a mosaic if needed. + This image is generally used as a user-provided pixel flat-field image and ingested + in the reduction using the `pixelflat_file` parameter in the PypeIt file. + + Args: + pixflat_norm_list (:obj:`list`): + List of 2D `numpy.ndarray`_ arrays containing the pixel-to-pixel flat-field images. + detname_list (:obj:`list`): + List of detector names. + spec_name (:obj:`str`): + Name of the spectrograph. + outdir (:obj:`pathlib.Path`): + Path to the output directory. + pixelflat_name (:obj:`str`): + Name of the output file to be written. + to_cache (:obj:`bool`, optional): + If True, the file will be written to the cache directory pypeit/data/pixflats. + + """ + + msgs.info("Writing the pixel-to-pixel flat-field images to a FITS file.") + + # Check that the number of detectors matches the number of pixelflat_norm arrays + if len(pixflat_norm_list) != len(detname_list): + msgs.error("The number of detectors does not match the number of pixelflat_norm arrays. " + "The pixelflat file cannot be written.") + + # local output (reduction directory) + pixelflat_file = outdir / pixelflat_name + + # Check if the file already exists + old_hdus = [] + old_detnames = [] + old_hdr = None + if pixelflat_file.exists(): + msgs.warn("The pixelflat file already exists. It will be overwritten/updated.") + old_hdus = fits.open(pixelflat_file) + old_detnames = [h.name.split('-')[0] for h in old_hdus] # this has also 'PRIMARY' + old_hdr = old_hdus[0].header + + # load spectrograph + spec = load_spectrograph(spec_name) + + # Create the new HDUList + _hdr = io.initialize_header(hdr=old_hdr) + prihdu = fits.PrimaryHDU(header=_hdr) + prihdu.header['CALIBTYP'] = (FlatImages.calib_type, 'PypeIt: Calibration frame type') + new_hdus = [prihdu] + + extnum = 1 + for d in spec.select_detectors(): + detname = spec.get_det_name(d) + extname = f'{detname}-PIXELFLAT_NORM' + # update or add the detectors that we want to save + if detname in detname_list: + det_idx = detname_list.index(detname) + pixflat_norm = pixflat_norm_list[det_idx] + hdu = fits.ImageHDU(data=pixflat_norm, name=extname) + prihdu.header[f'EXT{extnum:04d}'] = hdu.name + new_hdus.append(hdu) + # keep the old detectors that were not updated + elif detname in old_detnames: + old_det_idx = old_detnames.index(detname) + hdu = old_hdus[old_det_idx] + prihdu.header[f'EXT{extnum:04d}'] = hdu.name + new_hdus.append(hdu) + extnum += 1 + + # Write the new HDUList + new_hdulist = fits.HDUList(new_hdus) + # Check if the directory exists + if not pixelflat_file.parent.is_dir(): + pixelflat_file.parent.mkdir(parents=True) + new_hdulist.writeto(pixelflat_file, overwrite=True) + msgs.info(f'A slitless Pixel Flat file for detectors {detname_list} has been saved to {msgs.newline()}' + f'{pixelflat_file}') + + # common msg + add_msgs = f"add the following to your PypeIt Reduction File:{msgs.newline()}" \ + f" [calibrations]{msgs.newline()}" \ + f" [[flatfield]]{msgs.newline()}" \ + f" pixelflat_file = {pixelflat_name}{msgs.newline()}{msgs.newline()}{msgs.newline()}" \ + f"Please consider sharing your Pixel Flat file with the PypeIt Developers.{msgs.newline()}" \ + + + if to_cache: + # NOTE that the file saved in the cache is gzipped, while the one saved in the outdir is not + # This prevents `dataPaths.pixelflat.get_file_path()` from returning the file saved in the outdir + cache.write_file_to_cache(pixelflat_file, pixelflat_name+'.gz', f"pixelflats") + msgs.info(f"The slitless Pixel Flat file has also been saved to the PypeIt cache directory {msgs.newline()}" + f"{str(dataPaths.pixelflat)} {msgs.newline()}" + f"It will be automatically used in this run. " + f"If you want to use this file in future runs, {add_msgs}") + else: + msgs.info(f"To use this file, move it to the PypeIt data directory {msgs.newline()}" + f"{str(dataPaths.pixelflat)} {msgs.newline()} and {add_msgs}") + + +def load_pixflat(pixel_flat_file, spectrograph, det, flatimages, calib_dir=None, chk_version=False): + """ + Load a pixel flat from a file and add it to the flatimages object. + The pixel flat file has one detector per extension, even in the case of a mosaic. + Therefore, if this is a mosaic reduction, this script will construct a pixel flat + mosaic. The Edges file needs to exist in the Calibration Folder, since the mosaic + parameters are pulled from it. + This is used in `~pypeit.calibrations.get_flats()`. + + Args: + pixel_flat_file (:obj:`str`): + Name of the pixel flat file. + spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): + The spectrograph object. + det (:obj:`int`, :obj:`tuple`): + The single detector or set of detectors in a mosaic to process. + flatimages (:class:`~pypeit.flatfield.FlatImages`): + The flat field images object. + calib_dir (:obj:`str`, optional): + The path to the calibration directory. + chk_version (:obj:`bool`, optional): + Check the version of the file. + + Returns: + :class:`~pypeit.flatfield.FlatImages`: The flat images object with the pixel flat added. + + """ + # Check if the pixel flat file exists + if pixel_flat_file is None: + msgs.error('No pixel flat file defined. Cannot load the pixel flat!') + + # get the path + _pixel_flat_file = dataPaths.pixelflat.get_file_path(pixel_flat_file, return_none=True) + if _pixel_flat_file is None: + msgs.error(f'Cannot load the pixel flat file, {pixel_flat_file}. It is not a direct path, ' + f'a cached file, or a file that can be downloaded from a PypeIt repository.') + + # If this is a mosaic, we need to construct the pixel flat mosaic + if isinstance(det, tuple): + # We need to grab mosaic info from another existing calibration frame. + # We use EdgeTraceSet image to get `tform` and `msc_ord`. Check if EdgeTraceSet file exists. + edges_file = Path(edgetrace.EdgeTraceSet.construct_file_name(flatimages.calib_key, + calib_dir=calib_dir)).absolute() + if not edges_file.exists(): + msgs.error('Edges file not found in the Calibrations folder. ' + 'It is needed to grab the mosaic parameters to load and mosaic the input pixel flat!') + + # Load detector info from EdgeTraceSet file + traceimg = edgetrace.EdgeTraceSet.from_file(edges_file, chk_version=chk_version).traceimg + det_info = traceimg.detector + # check that the mosaic parameters are defined + if not np.all(np.in1d(['tform', 'msc_ord'], list(det_info.keys()))) or \ + det_info.tform is None or det_info.msc_ord is None: + msgs.error('Mosaic parameters are not defined in the Edges frame. Cannot load the pixel flat!') + + # read the file + with io.fits_open(_pixel_flat_file) as hdu: + # list of available detectors in the pixel flat file + file_dets = [int(h.name.split('-')[0].split('DET')[1]) for h in hdu[1:]] + # check if all detectors required for the mosaic are in the list + if not np.all(np.in1d(list(det), file_dets)): + msgs.error(f'Not all detectors in the mosaic are in the pixel flat file: ' + f'{pixel_flat_file}. Cannot load the pixel flat!') + + # get the pixel flat images of only the detectors in the mosaic + pixflat_images = np.concatenate([hdu[f'DET{d:02d}-PIXELFLAT_NORM'].data[None,:,:] for d in det]) + # construct the pixel flat mosaic + pixflat_msc, _,_,_ = build_image_mosaic(pixflat_images, det_info.tform, order=det_info.msc_ord) + # check that the mosaic has the correct shape + if pixflat_msc.shape != traceimg.image.shape: + msgs.error('The constructed pixel flat mosaic does not have the correct shape. ' + 'Cannot load this pixel flat as a mosaic!') + msgs.info(f'Using pixelflat file: {pixel_flat_file} ' + f'for {spectrograph.get_det_name(det)}.') + nrm_image = FlatImages(pixelflat_norm=pixflat_msc) + + # If this is not a mosaic, we can simply read the pixel flat for the current detector + else: + # current detector name + detname = spectrograph.get_det_name(det) + # read the file + with io.fits_open(_pixel_flat_file) as hdu: + # list of available detectors in the pixel flat file + file_detnames = [h.name.split('-')[0] for h in hdu] # this list has also the 'PRIMARY' extension + # check if the current detector is in the list + if detname in file_detnames: + # get the index of the current detector + idx = file_detnames.index(detname) + # get the pixel flat image + msgs.info(f'Using pixelflat file: {pixel_flat_file} for {detname}.') + nrm_image = FlatImages(pixelflat_norm=hdu[idx].data) + else: + msgs.error(f'{detname} not found in the pixel flat file: ' + f'{pixel_flat_file}. Cannot load the pixel flat!') + nrm_image = None + + return merge(flatimages, nrm_image) diff --git a/pypeit/images/buildimage.py b/pypeit/images/buildimage.py index b9ecfa9339..b09581ed56 100644 --- a/pypeit/images/buildimage.py +++ b/pypeit/images/buildimage.py @@ -10,11 +10,13 @@ from pypeit import msgs from pypeit.par import pypeitpar +from pypeit.images import rawimage from pypeit.images import combineimage from pypeit.images import pypeitimage from pypeit.core.framematch import valid_frametype + class ArcImage(pypeitimage.PypeItCalibrationImage): """ Simple DataContainer for the Arc Image @@ -157,10 +159,13 @@ def construct_file_name(cls, calib_key, calib_dir=None, basename=None): def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm=None, dark=None, - scattlight=None, flatimages=None, maxiters=5, ignore_saturation=True, slits=None, - mosaic=None, calib_dir=None, setup=None, calib_id=None): + scattlight=None, flatimages=None, maxiters=5, ignore_saturation=True, + slits=None, mosaic=None, calib_dir=None, setup=None, calib_id=None): """ - Perform basic image processing on a list of images and combine the results. + Perform basic image processing on a list of images and combine the results. All + core processing steps for each image are handled by :class:`~pypeit.images.rawimage.RawImage` and + image combination is handled by :class:`~pypeit.images.combineimage.CombineImage`. + This function can be used to process both single images, lists of images, and detector mosaics. .. warning:: @@ -247,15 +252,19 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm= if mosaic is None: mosaic = isinstance(det, tuple) and frame_par['frametype'] not in ['bias', 'dark'] - # Do it - combineImage = combineimage.CombineImage(spectrograph, det, frame_par['process'], file_list) - pypeitImage = combineImage.run(bias=bias, bpm=bpm, dark=dark, flatimages=flatimages, scattlight=scattlight, - sigma_clip=frame_par['process']['clip'], - sigrej=frame_par['process']['comb_sigrej'], - maxiters=maxiters, ignore_saturation=ignore_saturation, - slits=slits, combine_method=frame_par['process']['combine'], - mosaic=mosaic) + rawImage_list = [] + # Loop on the files + for ifile in file_list: + # Load raw image + rawImage = rawimage.RawImage(ifile, spectrograph, det) + # Process + rawImage_list.append(rawImage.process( + frame_par['process'], scattlight=scattlight, bias=bias, + bpm=bpm, dark=dark, flatimages=flatimages, slits=slits, mosaic=mosaic)) + # Do it + combineImage = combineimage.CombineImage(rawImage_list, frame_par['process']) + pypeitImage = combineImage.run(maxiters=maxiters, ignore_saturation=ignore_saturation) # Return class type, if returning any of the frame_image_classes cls = frame_image_classes[frame_par['frametype']] \ if frame_par['frametype'] in frame_image_classes.keys() else None diff --git a/pypeit/images/combineimage.py b/pypeit/images/combineimage.py index ffb0b20247..31980b00b5 100644 --- a/pypeit/images/combineimage.py +++ b/pypeit/images/combineimage.py @@ -4,8 +4,6 @@ .. include:: ../include/links.rst """ -import os - from IPython import embed import numpy as np @@ -16,54 +14,43 @@ from pypeit.par import pypeitpar from pypeit import utils from pypeit.images import pypeitimage -from pypeit.images import rawimage from pypeit.images import imagebitmask + class CombineImage: """ - Process and combine detector images. - - All core processing steps for each image are handled by - :class:`~pypeit.images.rawimage.RawImage`. This class can be used to - process both single images, lists of images, and detector mosaics. + Process and combine detector images. Args: - spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): - Spectrograph used to take the data. - det (:obj:`int`, :obj:`tuple`): - The 1-indexed detector number(s) to process. If a tuple, it must - include detectors viable as a mosaic for the provided spectrograph; - see :func:`~pypeit.spectrographs.spectrograph.Spectrograph.allowed_mosaics`. + rawImages (:obj:`list`, :class:`~pypeit.images.pypeitimage.PypeItImage`): + Either a single :class:`~pypeit.images.pypeitimage.PypeItImage` + object or a list of one or more of these objects to be combined into + an image. par (:class:`~pypeit.par.pypeitpar.ProcessImagesPar`): Parameters that dictate the processing of the images. - files (:obj:`str`, array-like): - A set of one or more images to process and combine. Attributes: - spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): - Spectrograph used to take the data. det (:obj:`int`, :obj:`tuple`): The 1-indexed detector number(s) to process. par (:class:`~pypeit.par.pypeitpar.ProcessImagesPar`): Parameters that dictate the processing of the images. - files (:obj:`list`): - A set of one or more images to process and combine. - + rawImages (:obj:`list`): + A list of one or more :class:`~pypeit.images.rawimage.RawImage` objects + to be combined. """ - def __init__(self, spectrograph, det, par, files): - self.spectrograph = spectrograph - self.det = det + def __init__(self, rawImages, par): if not isinstance(par, pypeitpar.ProcessImagesPar): msgs.error('Provided ParSet for must be type ProcessImagesPar.') + self.rawImages = list(rawImages) if hasattr(rawImages, '__len__') else [rawImages] self.par = par # This musts be named this way as it is frequently a child - self.files = list(files) if hasattr(files, '__len__') else [files] - # NOTE: nfiles is a property method. Defining files above must come + + # NOTE: nimgs is a property method. Defining rawImages above must come # before this check! - if self.nfiles == 0: + if self.nimgs == 0: msgs.error('CombineImage requires a list of files to instantiate') - def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=False, sigma_clip=True, - bpm=None, sigrej=None, maxiters=5, slits=None, dark=None, combine_method='mean', mosaic=False): + + def run(self, ignore_saturation=False, maxiters=5): r""" Process and combine all images. @@ -75,7 +62,7 @@ def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=Fal file and returns the result. If there are multiple files, all the files are processed and the - processed images are combined based on the ``combine_method``, where the + processed images are combined based on the ``par['combine']``, where the options are: - 'mean': If ``sigma_clip`` is True, this is a sigma-clipped mean; @@ -133,129 +120,72 @@ def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=Fal in images of the same shape. Args: - bias (:class:`~pypeit.images.buildimage.BiasImage`, optional): - Bias image for bias subtraction; passed directly to - :func:`~pypeit.images.rawimage.RawImage.process` for all images. - scattlight (:class:`~pypeit.scattlight.ScatteredLight`, optional): - Scattered light model to be used to determine scattered light. - flatimages (:class:`~pypeit.flatfield.FlatImages`, optional): - Flat-field images for flat fielding; passed directly to - :func:`~pypeit.images.rawimage.RawImage.process` for all images. ignore_saturation (:obj:`bool`, optional): If True, turn off the saturation flag in the individual images before stacking. This avoids having such values set to 0, which for certain images (e.g. flat calibrations) can have unintended consequences. - sigma_clip (:obj:`bool`, optional): - When ``combine_method='mean'``, perform a sigma-clip the data; - see :func:`~pypeit.core.combine.weighted_combine`. - bpm (`numpy.ndarray`_, optional): - Bad pixel mask; passed directly to - :func:`~pypeit.images.rawimage.RawImage.process` for all images. - sigrej (:obj:`float`, optional): - When ``combine_method='mean'``, this sets the sigma-rejection - thresholds used when sigma-clipping the image combination. - Ignored if ``sigma_clip`` is False. If None and ``sigma_clip`` - is True, the thresholds are determined automatically based on - the number of images provided; see - :func:`~pypeit.core.combine.weighted_combine``. maxiters (:obj:`int`, optional): - When ``combine_method='mean'``) and sigma-clipping + When ``par['combine']='mean'``) and sigma-clipping (``sigma_clip`` is True), this sets the maximum number of rejection iterations. If None, rejection iterations continue until no more data are rejected; see :func:`~pypeit.core.combine.weighted_combine``. - slits (:class:`~pypeit.slittrace.SlitTraceSet`, optional): - Slit edge trace locations; passed directly to - :func:`~pypeit.images.rawimage.RawImage.process` for all images. - dark (:class:`~pypeit.images.buildimage.DarkImage`, optional): - Dark-current image; passed directly to - :func:`~pypeit.images.rawimage.RawImage.process` for all images. - combine_method (:obj:`str`, optional): - Method used to combine images. Must be ``'mean'`` or - ``'median'``; see above. - mosaic (:obj:`bool`, optional): - If multiple detectors are being processes, mosaic them into a - single image. See - :func:`~pypeit.images.rawimage.RawImage.process`. Returns: :class:`~pypeit.images.pypeitimage.PypeItImage`: The combination of all the processed images. """ + + # Check the input (i.e., bomb out *before* it does any processing) - if self.nfiles == 0: + if self.nimgs == 0: msgs.error('Object contains no files to process!') - if self.nfiles > 1 and combine_method not in ['mean', 'median']: - msgs.error(f'Unknown image combination method, {combine_method}. Must be ' + if self.nimgs > 1 and self.par['combine'] not in ['mean', 'median']: + msgs.error(f'Unknown image combination method, {self.par["combine"]}. Must be ' '"mean" or "median".') - + file_list = [] # Loop on the files - for kk, ifile in enumerate(self.files): - # Load raw image - rawImage = rawimage.RawImage(ifile, self.spectrograph, self.det) - # Process - pypeitImage = rawImage.process(self.par, scattlight=scattlight, bias=bias, bpm=bpm, dark=dark, - flatimages=flatimages, slits=slits, mosaic=mosaic) - - if self.nfiles == 1: + for kk, rawImage in enumerate(self.rawImages): + if self.nimgs == 1: # Only 1 file, so we're done - pypeitImage.files = self.files - return pypeitImage + rawImage.files = [rawImage.filename] + return rawImage elif kk == 0: # Allocate arrays to collect data for each frame - shape = (self.nfiles,) + pypeitImage.shape + shape = (self.nimgs,) + rawImage.shape img_stack = np.zeros(shape, dtype=float) scl_stack = np.ones(shape, dtype=float) rn2img_stack = np.zeros(shape, dtype=float) basev_stack = np.zeros(shape, dtype=float) gpm_stack = np.zeros(shape, dtype=bool) - lampstat = [None]*self.nfiles - exptime = np.zeros(self.nfiles, dtype=float) + exptime = np.zeros(self.nimgs, dtype=float) - # Save the lamp status - # TODO: As far as I can tell, this is the *only* place rawheadlist - # is used. Is there a way we can get this from fitstbl instead? - lampstat[kk] = self.spectrograph.get_lamps_status(pypeitImage.rawheadlist) # Save the exposure time to check if it's consistent for all images. - exptime[kk] = pypeitImage.exptime + exptime[kk] = rawImage.exptime # Processed image - img_stack[kk] = pypeitImage.image + img_stack[kk] = rawImage.image # Get the count scaling - if pypeitImage.img_scale is not None: - scl_stack[kk] = pypeitImage.img_scale + if rawImage.img_scale is not None: + scl_stack[kk] = rawImage.img_scale # Read noise squared image - if pypeitImage.rn2img is not None: - rn2img_stack[kk] = pypeitImage.rn2img * scl_stack[kk]**2 + if rawImage.rn2img is not None: + rn2img_stack[kk] = rawImage.rn2img * scl_stack[kk]**2 # Processing variance image - if pypeitImage.base_var is not None: - basev_stack[kk] = pypeitImage.base_var * scl_stack[kk]**2 + if rawImage.base_var is not None: + basev_stack[kk] = rawImage.base_var * scl_stack[kk]**2 # Final mask for this image # TODO: This seems kludgy to me. Why not just pass ignore_saturation # to process_one and ignore the saturation when the mask is actually # built, rather than untoggling the bit here? if ignore_saturation: # Important for calibrations as we don't want replacement by 0 - pypeitImage.update_mask('SATURATION', action='turn_off') + rawImage.update_mask('SATURATION', action='turn_off') # Get a simple boolean good-pixel mask for all the unmasked pixels - gpm_stack[kk] = pypeitImage.select_flag(invert=True) - - # Check that the lamps being combined are all the same: - if not lampstat[1:] == lampstat[:-1]: - msgs.warn("The following files contain different lamp status") - # Get the longest strings - maxlen = max([len("Filename")]+[len(os.path.split(x)[1]) for x in self.files]) - maxlmp = max([len("Lamp status")]+[len(x) for x in lampstat]) - strout = "{0:" + str(maxlen) + "} {1:s}" - # Print the messages - print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp) - print(msgs.indent() + strout.format("Filename", "Lamp status")) - print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp) - for ff, file in enumerate(self.files): - print(msgs.indent() - + strout.format(os.path.split(file)[1], " ".join(lampstat[ff].split("_")))) - print(msgs.indent() + '-'*maxlen + " " + '-'*maxlmp) - - # Do a similar check for exptime + gpm_stack[kk] = rawImage.select_flag(invert=True) + file_list.append(rawImage.filename) + + # Check that all exposure times are consistent + # TODO: JFH suggests that we move this to calibrations.check_calibrations if np.any(np.absolute(np.diff(exptime)) > 0): # TODO: This should likely throw an error instead! msgs.warn('Exposure time is not consistent for all images being combined! ' @@ -264,22 +194,49 @@ def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=Fal else: comb_texp = exptime[0] + # scale the images to their mean, if requested, before combining + if self.par['scale_to_mean']: + msgs.info("Scaling images to have the same mean before combining") + # calculate the mean of the images + [mean_img], _, mean_gpm, _ = combine.weighted_combine(np.ones(self.nimgs, dtype=float)/self.nimgs, + [img_stack], + [rn2img_stack], + # var_list is added because it is + # required by the function but not used + gpm_stack, sigma_clip=self.par['clip'], + sigma_clip_stack=img_stack, + sigrej=self.par['comb_sigrej'], maxiters=maxiters) + + # scale factor + # TODO: Chose the median over the whole frame to avoid outliers. Is this the right choice? + _mscale = np.nanmedian(mean_img[None, mean_gpm]/img_stack[:, mean_gpm], axis=1) + # reshape the scale factor + mscale = _mscale[:, None, None] + # scale the images + img_stack *= mscale + # scale the scales + scl_stack *= mscale + + # scale the variances + rn2img_stack *= mscale**2 + basev_stack *= mscale**2 + # Coadd them - if combine_method == 'mean': - weights = np.ones(self.nfiles, dtype=float)/self.nfiles + if self.par['combine'] == 'mean': + weights = np.ones(self.nimgs, dtype=float)/self.nimgs img_list_out, var_list_out, gpm, nframes \ = combine.weighted_combine(weights, [img_stack, scl_stack], # images to stack [rn2img_stack, basev_stack], # variances to stack - gpm_stack, sigma_clip=sigma_clip, + gpm_stack, sigma_clip=self.par['clip'], sigma_clip_stack=img_stack, # clipping based on img - sigrej=sigrej, maxiters=maxiters) + sigrej=self.par['comb_sigrej'], maxiters=maxiters) comb_img, comb_scl = img_list_out comb_rn2, comb_basev = var_list_out # Divide by the number of images that contributed to each pixel comb_scl[gpm] /= nframes[gpm] - elif combine_method == 'median': + elif self.par['combine'] == 'median': bpm_stack = np.logical_not(gpm_stack) nframes = np.sum(gpm_stack, axis=0) gpm = nframes > 0 @@ -310,26 +267,26 @@ def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=Fal # Build the combined image comb = pypeitimage.PypeItImage(image=comb_img, ivar=utils.inverse(comb_var), nimg=nframes, - amp_img=pypeitImage.amp_img, det_img=pypeitImage.det_img, + amp_img=rawImage.amp_img, det_img=rawImage.det_img, rn2img=comb_rn2, base_var=comb_basev, img_scale=comb_scl, # NOTE: This *must* be a boolean. bpm=np.logical_not(gpm), # NOTE: The detector is needed here so # that we can get the dark current later. - detector=pypeitImage.detector, - PYP_SPEC=self.spectrograph.name, + detector=rawImage.detector, + PYP_SPEC=rawImage.PYP_SPEC, units='e-' if self.par['apply_gain'] else 'ADU', exptime=comb_texp, noise_floor=self.par['noise_floor'], shot_noise=self.par['shot_noise']) # Internals # TODO: Do we need these? - comb.files = self.files - comb.rawheadlist = pypeitImage.rawheadlist - comb.process_steps = pypeitImage.process_steps + comb.files = file_list + comb.rawheadlist = rawImage.rawheadlist + comb.process_steps = rawImage.process_steps # Build the base level mask - comb.build_mask(saturation='default', mincounts='default') + comb.build_mask(saturation='default' if not ignore_saturation else None, mincounts='default') # Flag all pixels with no contributions from any of the stacked images. comb.update_mask('STCKMASK', indx=np.logical_not(gpm)) @@ -338,10 +295,10 @@ def run(self, bias=None, scattlight=None, flatimages=None, ignore_saturation=Fal return comb @property - def nfiles(self): + def nimgs(self): """ The number of files in :attr:`files`. """ - return len(self.files) if isinstance(self.files, (np.ndarray, list)) else 0 + return len(self.rawImages) if isinstance(self.rawImages, (np.ndarray, list)) else 0 diff --git a/pypeit/images/mosaic.py b/pypeit/images/mosaic.py index 9bf4d95355..a8b6e87899 100644 --- a/pypeit/images/mosaic.py +++ b/pypeit/images/mosaic.py @@ -30,7 +30,7 @@ class Mosaic(datamodel.DataContainer): """ # Set the version of this class - version = '1.0.0' + version = '1.0.1' # WARNING: `binning` and `platescale` have the same names as datamodel # components in pypeit.images.detector_container.DetectorContainer. This is @@ -53,14 +53,15 @@ class Mosaic(datamodel.DataContainer): 'tform': dict(otype=np.ndarray, atype=float, descr='The full transformation matrix for each detector used to ' 'construct the mosaic.'), - 'msc_order': dict(otype=int, descr='Order of the interpolation used to construct the mosaic.')} + 'msc_ord': dict(otype=int, + descr='Order of the interpolation used to construct the mosaic.')} name_prefix = 'MSC' """ Prefix for the name of the mosaic. """ - def __init__(self, id, detectors, shape, shift, rot, tform, msc_order): + def __init__(self, id, detectors, shape, shift, rot, tform, msc_ord): args, _, _, values = inspect.getargvalues(inspect.currentframe()) d = dict([(k,values[k]) for k in args[1:]]) @@ -107,8 +108,8 @@ def _bundle(self): tbl['rot'] = self.rot if self.tform is not None: tbl['tform'] = self.tform - if self.msc_order is not None: - tbl.meta['msc_order'] = self.msc_order + if self.msc_ord is not None: + tbl.meta['msc_ord'] = self.msc_ord if self.id is not None: tbl.meta['id'] = self.id if self.shape is not None: @@ -159,10 +160,18 @@ def _parse(cls, hdu, hdu_prefix=None, **kwargs): hdr = fits.Header() hdr['DMODCLS'] = DetectorContainer.__name__ hdr['DMODVER'] = _hdu.header['DETMODV'] - d['detectors'] = np.array([DetectorContainer.from_hdu( - fits.BinTableHDU(data=table.Table(tbl[i]), - name='DETECTOR', header=hdr)) - for i in range(ndet)]) + d['detectors'] = [] + for i in range(ndet): + _hdu = fits.BinTableHDU(data=table.Table(tbl[i]), name='DETECTOR', header=hdr) + # NOTE: I'm using _parse() to ensure that I keep the result of the + # version and type checking. + _d, vp, tp, ph = DetectorContainer._parse(_hdu) + if not vp: + msgs.warn('Detector datamodel version is incorrect. May cause a fault.') + version_passed &= vp + d['detectors'] += [DetectorContainer.from_dict(d=_d) if tp else None] + type_passed &= tp + d['detectors'] = np.array(d['detectors'], dtype=object) return d, version_passed, type_passed, parsed_hdus @@ -213,5 +222,7 @@ def copy(self): """ Return a (deep) copy of the object. """ - return Mosaic(id=self.id, detectors=np.array([det.copy() for det in self.detectors]), shape=self.shape, shift=self.shift.copy(), rot=self.rot.copy(), tform=self.tform.copy(), msc_order=self.msc_order) + return Mosaic(id=self.id, detectors=np.array([det.copy() for det in self.detectors]), + shape=self.shape, shift=self.shift.copy(), rot=self.rot.copy(), + tform=self.tform.copy(), msc_ord=self.msc_ord) diff --git a/pypeit/images/pypeitimage.py b/pypeit/images/pypeitimage.py index d8a6edcb8a..dc8e07b591 100644 --- a/pypeit/images/pypeitimage.py +++ b/pypeit/images/pypeitimage.py @@ -91,6 +91,9 @@ class PypeItImage(datamodel.DataContainer): :class:`~pypeit.images.rawimage.RawImage.process`. """ + # TODO These docs are confusing. The __init__ method needs to be documented just as it is for + # every other class that we have written in PypeIt, i.e. the arguments all need to be documented. They are not + # documented here and instead we have the odd Args documentation above. version = '1.3.0' """Datamodel version number""" @@ -127,7 +130,8 @@ class PypeItImage(datamodel.DataContainer): 'shot_noise': dict(otype=bool, descr='Shot-noise included in variance'), 'spat_flexure': dict(otype=float, descr='Shift, in spatial pixels, between this image ' - 'and SlitTrace')} + 'and SlitTrace'), + 'filename': dict(otype=str, descr='Filename for the image'),} """Data model components.""" internals = ['process_steps', 'files', 'rawheadlist'] @@ -160,10 +164,11 @@ def from_pypeitimage(cls, pypeitImage): # Done return self + # TODO This init method needs proper docs, which includes every optional argument. See my comment above. def __init__(self, image, ivar=None, nimg=None, amp_img=None, det_img=None, rn2img=None, base_var=None, img_scale=None, fullmask=None, detector=None, spat_flexure=None, - PYP_SPEC=None, units=None, exptime=None, noise_floor=None, shot_noise=None, - bpm=None, crmask=None, usermask=None, clean_mask=False): + filename=None, PYP_SPEC=None, units=None, exptime=None, noise_floor=None, + shot_noise=None, bpm=None, crmask=None, usermask=None, clean_mask=False): if image is None: msgs.error('Must provide an image when instantiating PypeItImage.') diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index e4c5fd9fef..8389cc0ac9 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -177,6 +177,7 @@ def __init__(self, ifile, spectrograph, det): self.steps = dict(apply_gain=False, subtract_pattern=False, subtract_overscan=False, + correct_nonlinear=False, subtract_continuum=False, subtract_scattlight=False, trim=False, @@ -304,6 +305,26 @@ def build_ivar(self): noise_floor=self.par['noise_floor']) return utils.inverse(var) + def correct_nonlinear(self): + """ + Apply a non-linear correction to the image. + + This is a simple wrapper for :func:`~pypeit.core.procimg.nonlinear_counts`. + """ + step = inspect.stack()[0][3] + if self.steps[step]: + # Already applied + msgs.warn('Non-linear correction was already applied.') + return + + inim = self.image.copy() + for ii in range(self.nimg): + # Correct the image for non-linearity. Note that the variance image is not changed here. + self.image[ii, ...] = procimg.nonlinear_counts(self.image[ii, ...], self.datasec_img[ii, ...]-1, + self.par['correct_nonlinear']) + + self.steps[step] = True + def estimate_readnoise(self): r""" Estimate the readnoise (in electrons) based on the overscan regions of @@ -613,7 +634,12 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl self.subtract_bias(bias) # TODO: Checking for count (well-depth) saturation should be done here. - # TODO :: Non-linearity correction should be done here. + + # - Perform a non-linearity correction. This is done before the + # flat-field and dark correction because the flat-field modifies + # the counts. + if self.par['correct_nonlinear'] is not None: + self.correct_nonlinear() # - Create the dark current image(s). The dark-current image *always* # includes the tabulated dark current and the call below ensures @@ -676,7 +702,8 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl exptime=self.exptime, noise_floor=self.par['noise_floor'], shot_noise=self.par['shot_noise'], - bpm=_bpm.astype(bool)) + bpm=_bpm.astype(bool), + filename=self.filename) pypeitImage.rawheadlist = self.headarr pypeitImage.process_steps = [key for key in self.steps.keys() if self.steps[key]] @@ -1192,7 +1219,7 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): f" {tmp[13]}, {tmp[14]}, {tmp[15]}]) # Polynomial terms (coefficients of spec**index)\n" print(strprint) pad = msscattlight.pad // spatbin - offslitmask = slits.slit_img(pad=pad, initial=True, flexure=None) == -1 + offslitmask = slits.slit_img(pad=pad, flexure=None) == -1 from matplotlib import pyplot as plt _frame = self.image[ii, ...] vmin, vmax = 0, np.max(scatt_img) @@ -1217,7 +1244,7 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): elif self.par["scattlight"]["method"] == "frame": # Calculate a model specific for this frame pad = msscattlight.pad // spatbin - offslitmask = slits.slit_img(pad=pad, initial=True, flexure=None) == -1 + offslitmask = slits.slit_img(pad=pad, flexure=None) == -1 # Get starting parameters for the scattered light model x0, bounds = self.spectrograph.scattered_light_archive(binning, dispname) # Perform a fit to the scattered light @@ -1241,11 +1268,11 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): # Check if a fine correction to the scattered light should be applied if do_finecorr: pad = self.par['scattlight']['finecorr_pad'] // spatbin - offslitmask = slits.slit_img(pad=pad, initial=True, flexure=None) == -1 + offslitmask = slits.slit_img(pad=pad, flexure=None) == -1 # Check if the user wishes to mask some inter-slit regions if self.par['scattlight']['finecorr_mask'] is not None: # Get the central trace of each slit - left, right, _ = slits.select_edges(initial=True, flexure=None) + left, right, _ = slits.select_edges(flexure=None) centrace = 0.5*(left+right) # Now mask user-defined inter-slit regions offslitmask = scattlight.mask_slit_regions(offslitmask, centrace, @@ -1331,7 +1358,7 @@ def build_mosaic(self): # Transform the image data to the mosaic frame. This call determines # the shape of the mosaic image and adjusts the relative transforms to # the absolute mosaic frame. - self.image, _, _img_npix, _tforms = build_image_mosaic(self.image, self.mosaic.tform, order=self.mosaic.msc_order) + self.image, _, _img_npix, _tforms = build_image_mosaic(self.image, self.mosaic.tform, order=self.mosaic.msc_ord) shape = self.image.shape # Maintain dimensionality self.image = np.expand_dims(self.image, 0) @@ -1342,7 +1369,7 @@ def build_mosaic(self): # Transform the BPM and maintain its type bpm_type = self.bpm.dtype - self._bpm = build_image_mosaic(self.bpm.astype(float), _tforms, mosaic_shape=shape, order=self.mosaic.msc_order)[0] + self._bpm = build_image_mosaic(self.bpm.astype(float), _tforms, mosaic_shape=shape, order=self.mosaic.msc_ord)[0] # Include pixels that have no contribution from the original image in # the bad pixel mask of the mosaic. self._bpm[_img_npix < 1] = 1 @@ -1357,29 +1384,29 @@ def build_mosaic(self): # Get the pixels associated with each amplifier self.datasec_img = build_image_mosaic(self.datasec_img.astype(float), _tforms, - mosaic_shape=shape, order=self.mosaic.msc_order)[0] + mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.datasec_img = np.expand_dims(np.round(self.datasec_img).astype(int), 0) # Get the pixels associated with each detector self.det_img = build_image_mosaic(self.det_img.astype(float), _tforms, - mosaic_shape=shape, order=self.mosaic.msc_order)[0] + mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.det_img = np.expand_dims(np.round(self.det_img).astype(int), 0) # Transform all the variance arrays, as necessary if self.rn2img is not None: - self.rn2img = build_image_mosaic(self.rn2img, _tforms, mosaic_shape=shape, order=self.mosaic.msc_order)[0] + self.rn2img = build_image_mosaic(self.rn2img, _tforms, mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.rn2img = np.expand_dims(self.rn2img, 0) if self.dark is not None: - self.dark = build_image_mosaic(self.dark, _tforms, mosaic_shape=shape, order=self.mosaic.msc_order)[0] + self.dark = build_image_mosaic(self.dark, _tforms, mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.dark = np.expand_dims(self.dark, 0) if self.dark_var is not None: - self.dark_var = build_image_mosaic(self.dark_var, _tforms, mosaic_shape=shape, order=self.mosaic.msc_order)[0] + self.dark_var = build_image_mosaic(self.dark_var, _tforms, mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.dark_var = np.expand_dims(self.dark_var, 0) if self.proc_var is not None: - self.proc_var = build_image_mosaic(self.proc_var, _tforms, mosaic_shape=shape, order=self.mosaic.msc_order)[0] + self.proc_var = build_image_mosaic(self.proc_var, _tforms, mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.proc_var = np.expand_dims(self.proc_var, 0) if self.base_var is not None: - self.base_var = build_image_mosaic(self.base_var, _tforms, mosaic_shape=shape, order=self.mosaic.msc_order)[0] + self.base_var = build_image_mosaic(self.base_var, _tforms, mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.base_var = np.expand_dims(self.base_var, 0) # TODO: Mosaicing means that many of the internals are no longer diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index cfcefba6f4..52a41f7c11 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -735,6 +735,16 @@ class SensFile(InputFile): datablock_required = False setup_required = False + +class ExtractFile(InputFile): + """Child class for the Extraction input file + """ + data_block = 'extract' # Defines naming of data block + flavor = 'Extract' # Defines naming of file + setup_required = False + datablock_required = False + + class FluxFile(InputFile): """Child class for the Fluxing input file """ @@ -895,6 +905,18 @@ def options(self): Parse the options associated with a cube block. Here is a description of the available options: + - ``sensfunc``: The name of an a sensitivity function file that is used + for the flux calibration. The file provided here should be generated by + (or of the same format as the output of) the command :ref:`pypeit_sensfunc`. + This parameter can also be set for all frames + with the default command: + + .. code-block:: ini + + [reduce] + [[cube]] + sensfile = sens.fits + - ``scale_corr``: The name of an alternative spec2d file that is used for the relative spectral scale correction. This parameter can also be set for all frames with the default command: @@ -905,6 +927,16 @@ def options(self): [[cube]] scale_corr = spec2d_alternative.fits + - ``grating_corr``: The name of a Flat calibrations file that is used + for the grating tilt correction. This parameter can also be set for all frames + with the default command: + + .. code-block:: ini + + [reduce] + [[cube]] + grating_corr = Flat_A_0_DET01.fits + - ``skysub_frame``: The name of an alternative spec2d file that is used for the sky subtraction. This parameter can also be set for all frames with the default command: @@ -915,23 +947,47 @@ def options(self): [[cube]] skysub_frame = spec2d_alternative.fits + - ``ra_offset``: The RA offset to apply to the WCS of the cube. + + - ``dec_offset``: The DEC offset to apply to the WCS of the cube. + + Returns ------- opts: dict Dictionary containing cube options. """ # Define the list of allowed parameters - opts = dict(scale_corr=None, skysub_frame=None, ra_offset=None, dec_offset=None) + opts = dict(sensfile=None, scale_corr=None, grating_corr=None, skysub_frame=None, + ra_offset=None, dec_offset=None) + + # Get the sensfunc files + sensfile = self.path_and_files('sensfile', skip_blank=False, check_exists=False) + if sensfile is None: + opts['sensfile'] = None + elif len(sensfile) == 1 and len(self.filenames) > 1: + opts['sensfile'] = sensfile*len(self.filenames) + elif len(sensfile) != 0: + opts['sensfile'] = sensfile # Get the scale correction files scale_corr = self.path_and_files('scale_corr', skip_blank=False, check_exists=False) if scale_corr is None: opts['scale_corr'] = [None]*len(self.filenames) elif len(scale_corr) == 1 and len(self.filenames) > 1: - opts['scale_corr'] = scale_corr.lower()*len(self.filenames) + opts['scale_corr'] = scale_corr*len(self.filenames) elif len(scale_corr) != 0: opts['scale_corr'] = scale_corr + # Get the grating correction files + grating_corr = self.path_and_files('grating_corr', skip_blank=False, check_exists=False) + if grating_corr is None: + opts['grating_corr'] = [None]*len(self.filenames) + elif len(grating_corr) == 1 and len(self.filenames) > 1: + msgs.error("You cannot specify a single grating correction file for multiple input files.") + elif len(grating_corr) != 0: + opts['grating_corr'] = grating_corr + # Get the skysub files skysub_frame = self.path_and_files('skysub_frame', skip_blank=False, check_exists=False) if skysub_frame is None: @@ -1093,6 +1149,5 @@ def grab_rawfiles(file_of_files:str=None, list_of_files:list=None, raw_paths:lis return [str(p / f) for p in _raw_paths for f in list_of_files if (p / f).exists()] # Find all files that have the correct extension - return np.concatenate([files_from_extension(str(p), extension=extension) - for p in _raw_paths]).tolist() + return files_from_extension(_raw_paths, extension=extension) diff --git a/pypeit/io.py b/pypeit/io.py index 2e0c13b992..1d230ca5eb 100644 --- a/pypeit/io.py +++ b/pypeit/io.py @@ -851,40 +851,48 @@ def create_symlink(filename, symlink_dir, relative_symlink=False, overwrite=Fals os.symlink(olink_src, olink_dest) -def files_from_extension(raw_path, - extension:str='fits'): +def files_from_extension(raw_path, extension='.fits'): """ - Grab the list of files with a given extension + Find files from one or more paths with one or more extensions. - Args: - raw_path (str or list): - Path(s) to raw files, which may or may not include the prefix of the - files to search for. - - For a string input, for example, this can be the directory - ``'/path/to/files/'`` or the directory plus the file prefix - ``'/path/to/files/prefix'``, which yeilds the search strings - ``'/path/to/files/*fits'`` or ``'/path/to/files/prefix*fits'``, - respectively. - - For a list input, this can use wildcards for multiple directories. + This is a recursive function. If ``raw_path`` is a list, the function is + called for every item in the list and the results are concatenated. - extension (str, optional): - File extension to search on. + Args: + raw_path (:obj:`str`, `Path`_, :obj:`list`): + One or more paths to search for files, which may or may not include + the prefix of the files to search for. For string input, this can + be the directory ``'/path/to/files/'`` or the directory plus the + file prefix ``'/path/to/files/prefix'``, which yeilds the search + strings ``'/path/to/files/*fits'`` or + ``'/path/to/files/prefix*fits'``, respectively. For a list input, + this can use wildcards for multiple directories. + + extension (:obj:`str`, :obj:`list`, optional): + One or more file extensions to search on. Returns: - list: List of raw data filenames (sorted) with full path - """ - if isinstance(raw_path, str): - # Grab the list of files - dfname = os.path.join(raw_path, f'*{extension}*') \ - if os.path.isdir(raw_path) else f'{raw_path}*{extension}*' - return sorted(glob.glob(dfname)) + :obj:`list`: List of `Path`_ objects with the full path to the set of + unique raw data filenames that match the provided criteria search + strings. + """ + if isinstance(raw_path, (str, Path)): + _raw_path = Path(raw_path).absolute() + if _raw_path.is_dir(): + prefix = '' + else: + _raw_path, prefix = _raw_path.parent, _raw_path.name + if not _raw_path.is_dir(): + msgs.error(f'{_raw_path} does not exist!') + ext = [extension] if isinstance(extension, str) else extension + files = numpy.concatenate([sorted(_raw_path.glob(f'{prefix}*{e}')) for e in ext]) + return numpy.unique(files).tolist() if isinstance(raw_path, list): - return numpy.concatenate([files_from_extension(p, extension=extension) for p in raw_path]).tolist() + files = numpy.concatenate([files_from_extension(p, extension=extension) for p in raw_path]) + return numpy.unique(files).tolist() - msgs.error(f"Incorrect type {type(raw_path)} for raw_path (must be str or list)") + msgs.error(f"Incorrect type {type(raw_path)} for raw_path; must be str, Path, or list.") diff --git a/pypeit/metadata.py b/pypeit/metadata.py index 3fc1d1b17d..f72a7c20c9 100644 --- a/pypeit/metadata.py +++ b/pypeit/metadata.py @@ -20,7 +20,6 @@ from pypeit import msgs from pypeit import inputfiles from pypeit.core import framematch -from pypeit.core import flux_calib from pypeit.core import parse from pypeit.core import meta from pypeit.io import dict_to_lines @@ -747,7 +746,7 @@ def unique_configurations(self, force=False, copy=False, rm_none=False): ignore_frames, ignore_indx = self.ignore_frames() # Find the indices of the frames not to ignore indx = np.arange(len(self.table)) - indx = indx[np.logical_not(np.in1d(indx, ignore_indx))] + indx = indx[np.logical_not(np.isin(indx, ignore_indx))] if len(indx) == 0: msgs.error('No frames to use to define configurations!') @@ -1033,18 +1032,28 @@ def _set_calib_group_bits(self): Set the calibration group bit based on the string values of the 'calib' column. """ - # NOTE: This is a hack to ensure the type of the *elements* of the calib - # column are all strings, but that the type of the column remains as - # "object". I'm calling this a hack because doing this is easier than + # Ensure that the type of the *elements* of the calib column are all + # strings, but that the type of the column remains as "object". + # NOTE: This is effectively a hack because doing this is easier than # trying to track down everywhere calib is changed to values that may or # may not be integers instead of strings. self['calib'] = np.array([str(c) for c in self['calib']], dtype=object) + # Collect and expand any lists # group_names = np.unique(np.concatenate( # [s.split(',') for s in self['calib'] if s not in ['all', 'None']])) - # DP changed to below because np.concatenate does not accept an empty list, - # which is the case when calib is None for all frames. This should avoid the code to crash - group_names = np.unique(sum([s.split(',') for s in self['calib'] if s not in ['all', 'None']], [])) + # NOTE: The above doesn't always work because np.concatenate does not + # accept an empty list, which is the case when calib is None or 'all' + # for all frames. + group_names = np.unique(sum([s.split(',') for s in self['calib'] + if s not in ['all', 'None']], [])) + + # If all the calibration groups are set to None or 'all', group_names + # can be an empty list. But we need to identify at least one + # calibration group, so I insert a mock value. + if group_names.size == 0: + group_names = np.array(['0'], dtype=object) + # Expand any ranges keep_group = np.ones(group_names.size, dtype=bool) added_groups = [] @@ -1053,6 +1062,7 @@ def _set_calib_group_bits(self): # Parse the range keep_group[i] = False added_groups += [str(n) for n in parse.str2list(name)] + # Combine and find the unique *integer* identifiers group_names = np.unique(np.asarray(added_groups + (group_names[keep_group]).tolist()).astype(int)) @@ -1438,28 +1448,10 @@ def get_frame_types(self, flag_unknown=False, user=None, merge=True): indx = self.spectrograph.check_frame_type(ftype, self.table, exprng=exprng) # Turn on the relevant bits type_bits[indx] = self.type_bitmask.turn_on(type_bits[indx], flag=ftype) - - # Find the nearest standard star to each science frame - # TODO: Should this be 'standard' or 'science' or both? - if 'ra' not in self.keys() or 'dec' not in self.keys(): - msgs.warn('Cannot associate standard with science frames without sky coordinates.') - else: - # TODO: Do we want to do this here? - indx = self.type_bitmask.flagged(type_bits, flag='standard') - for b, f, ra, dec in zip(type_bits[indx], self['filename'][indx], self['ra'][indx], - self['dec'][indx]): - if ra == 'None' or dec == 'None': - msgs.warn('RA and DEC must not be None for file:' + msgs.newline() + f) - msgs.warn('The above file could be a twilight flat frame that was' - + msgs.newline() + 'missed by the automatic identification.') - b = self.type_bitmask.turn_off(b, flag='standard') - continue - # If an object exists within 20 arcmins of a listed standard, - # then it is probably a standard star - foundstd = flux_calib.find_standard_file(ra, dec, check=True) - b = self.type_bitmask.turn_off(b, flag='science' if foundstd else 'standard') - + # Vet assigned frame types (this can be spectrograph dependent) + self.spectrograph.vet_assigned_ftypes(type_bits, self) + # Find the files without any types indx = np.logical_not(self.type_bitmask.flagged(type_bits)) if np.any(indx): diff --git a/pypeit/par/parset.py b/pypeit/par/parset.py index a2f8202153..10b660f415 100644 --- a/pypeit/par/parset.py +++ b/pypeit/par/parset.py @@ -345,7 +345,7 @@ def _data_table_string(data_table, delimeter='print'): return '\n'.join(row_string)+'\n' @staticmethod - def _data_string(data, use_repr=True, verbatim=False): + def _data_string(data, use_repr=False, verbatim=False): """ Convert a single datum into a string diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index d9f42ec6dd..a19680b156 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -73,6 +73,7 @@ def __init__(self, existing_par=None, foo=None): from pypeit.par import util from pypeit.core.framematch import FrameTypeBitMask from pypeit import msgs +from pypeit import dataPaths def tuple_force(par): @@ -209,6 +210,7 @@ def __init__(self, trim=None, apply_gain=None, orient=None, overscan_method=None, overscan_par=None, combine=None, satpix=None, mask_cr=None, clip=None, + scale_to_mean=None, #cr_sigrej=None, n_lohi=None, #replace=None, lamaxiter=None, grow=None, @@ -216,7 +218,7 @@ def __init__(self, trim=None, apply_gain=None, orient=None, # calib_setup_and_bit=None, rmcompact=None, sigclip=None, sigfrac=None, objlim=None, use_biasimage=None, use_overscan=None, use_darkimage=None, - dark_expscale=None, + dark_expscale=None, correct_nonlinear=None, empirical_rn=None, shot_noise=None, noise_floor=None, use_pixelflat=None, use_illumflat=None, use_specillum=None, use_pattern=None, subtract_scattlight=None, scattlight=None, subtract_continuum=None, @@ -273,6 +275,15 @@ def __init__(self, trim=None, apply_gain=None, orient=None, 'for \'savgol\', set overscan_par = order, window size ; ' \ 'for \'median\', set overscan_par = None or omit the keyword.' + defaults['correct_nonlinear'] = None + dtypes['correct_nonlinear'] = list + descr['correct_nonlinear'] = 'Correct for non-linear response of the detector. If None, ' \ + 'no correction is performed. If a list, then the list should be ' \ + 'the non-linear correction parameter (alpha), where the functional ' \ + 'form is given by Ct = Cm (1 + alpha x Cm), with Ct and Cm the true ' \ + 'and measured counts. This parameter is usually ' \ + 'hard-coded for a given spectrograph, and should otherwise be left as None.' \ + defaults['use_darkimage'] = False dtypes['use_darkimage'] = bool descr['use_darkimage'] = 'Subtract off a dark image. If True, one or more darks must ' \ @@ -363,6 +374,10 @@ def __init__(self, trim=None, apply_gain=None, orient=None, dtypes['clip'] = bool descr['clip'] = 'Perform sigma clipping when combining. Only used with combine=mean' + defaults['scale_to_mean'] = False + dtypes['scale_to_mean'] = bool + descr['scale_to_mean'] = 'If True, scale the input images to have the same mean before combining.' + defaults['comb_sigrej'] = None dtypes['comb_sigrej'] = float descr['comb_sigrej'] = 'Sigma-clipping level for when clip=True; ' \ @@ -443,14 +458,14 @@ def __init__(self, trim=None, apply_gain=None, orient=None, @classmethod def from_dict(cls, cfg): k = np.array([*cfg.keys()]) - parkeys = ['trim', 'apply_gain', 'orient', 'use_biasimage', 'subtract_continuum', 'subtract_scattlight', - 'scattlight', 'use_pattern', 'use_overscan', 'overscan_method', 'overscan_par', - 'use_darkimage', 'dark_expscale', 'spat_flexure_correct', 'spat_flexure_maxlag', 'use_illumflat', - 'use_specillum', 'empirical_rn', 'shot_noise', 'noise_floor', 'use_pixelflat', 'combine', - 'satpix', #'calib_setup_and_bit', - 'n_lohi', 'mask_cr', - 'lamaxiter', 'grow', 'clip', 'comb_sigrej', 'rmcompact', 'sigclip', - 'sigfrac', 'objlim'] + parkeys = ['trim', 'apply_gain', 'orient', 'use_biasimage', 'subtract_continuum', + 'subtract_scattlight', 'scattlight', 'use_pattern', 'use_overscan', + 'overscan_method', 'overscan_par', 'use_darkimage', 'dark_expscale', + 'spat_flexure_correct', 'spat_flexure_maxlag', 'use_illumflat', 'use_specillum', + 'empirical_rn', 'shot_noise', 'noise_floor', 'use_pixelflat', 'combine', + 'scale_to_mean', 'correct_nonlinear', 'satpix', #'calib_setup_and_bit', + 'n_lohi', 'mask_cr', 'lamaxiter', 'grow', 'clip', 'comb_sigrej', 'rmcompact', + 'sigclip', 'sigfrac', 'objlim'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): @@ -577,7 +592,7 @@ class FlatFieldPar(ParSet): see :ref:`parameters`. """ def __init__(self, method=None, pixelflat_file=None, spec_samp_fine=None, - spec_samp_coarse=None, spat_samp=None, tweak_slits=None, tweak_slits_thresh=None, + spec_samp_coarse=None, spat_samp=None, tweak_slits=None, tweak_method=None, tweak_slits_thresh=None, tweak_slits_maxfrac=None, rej_sticky=None, slit_trim=None, slit_illum_pad=None, illum_iter=None, illum_rej=None, twod_fit_npoly=None, saturated_slits=None, slit_illum_relative=None, slit_illum_ref_idx=None, slit_illum_smooth_npix=None, @@ -643,6 +658,17 @@ def __init__(self, method=None, pixelflat_file=None, spec_samp_fine=None, descr['tweak_slits'] = 'Use the illumination flat field to tweak the slit edges. ' \ 'This will work even if illumflatten is set to False ' + defaults['tweak_method'] = 'threshold' + options['tweak_method'] = FlatFieldPar.valid_tweak_methods() + dtypes['tweak_method'] = str + descr['tweak_method'] = 'Method used to tweak the slit edges (when "tweak_slits" is set to True). ' \ + 'Options include: {0:s}. '.format(', '.join(options['tweak_method'])) + \ + 'The "threshold" method determines when the left and right slit edges ' \ + 'fall below a threshold relative to the peak illumination. ' \ + 'The "gradient" method determines where the gradient is the highest at ' \ + 'the left and right slit edges. This method performs better when there is ' \ + 'systematic vignetting in the spatial direction. ' \ + defaults['tweak_slits_thresh'] = 0.93 dtypes['tweak_slits_thresh'] = float descr['tweak_slits_thresh'] = 'If tweak_slits is True, this sets the illumination function threshold used to ' \ @@ -760,7 +786,7 @@ def from_dict(cls, cfg): k = np.array([*cfg.keys()]) parkeys = ['method', 'pixelflat_file', 'spec_samp_fine', 'spec_samp_coarse', 'spat_samp', 'pixelflat_min_wave', 'pixelflat_max_wave', - 'tweak_slits', 'tweak_slits_thresh', 'tweak_slits_maxfrac', + 'tweak_slits', 'tweak_method', 'tweak_slits_thresh', 'tweak_slits_maxfrac', 'rej_sticky', 'slit_trim', 'slit_illum_pad', 'slit_illum_relative', 'illum_iter', 'illum_rej', 'twod_fit_npoly', 'saturated_slits', 'slit_illum_ref_idx', 'slit_illum_smooth_npix', 'slit_illum_finecorr', 'fit_2d_det_response'] @@ -781,6 +807,13 @@ def valid_methods(): """ return ['bspline', 'skip'] # [ 'PolyScan', 'bspline' ]. Same here. Not sure what PolyScan is + @staticmethod + def valid_tweak_methods(): + """ + Return the valid options for tweaking slits. + """ + return ['threshold', 'gradient'] + @staticmethod def valid_saturated_slits_methods(): """ @@ -806,9 +839,13 @@ def validate(self): return # Check the frame exists - if not os.path.isfile(self.data['pixelflat_file']): - raise ValueError('Provided frame file name does not exist: {0}'.format( - self.data['pixelflat_file'])) + # only the file name is provided, so we need to check if the file exists + # in the right place (data/pixelflats) + file_path = dataPaths.pixelflat.get_file_path(self.data['pixelflat_file'], return_none=True) + if file_path is None: + msgs.error( + f'Provided pixelflat file, {self.data["pixelflat_file"]} not found. It is not a direct path, ' + 'a cached file, or a file that can be downloaded from a PypeIt repository.') # Check that if tweak slits is true that illumflatten is alwo true # TODO -- We don't need this set, do we?? See the desc of tweak_slits above @@ -1428,7 +1465,7 @@ class Coadd2DPar(ParSet): see :ref:`parameters`. """ def __init__(self, only_slits=None, exclude_slits=None, offsets=None, spat_toler=None, weights=None, user_obj=None, - use_slits4wvgrid=None, manual=None, wave_method=None): + use_slits4wvgrid=None, manual=None, wave_method=None, spec_samp_fact=None, spat_samp_fact=None): # Grab the parameter names and values from the function # arguments @@ -1517,6 +1554,23 @@ def __init__(self, only_slits=None, exclude_slits=None, offsets=None, spat_toler "* 'log10' -- Grid is uniform in log10(wave). This is the same as velocity." \ "* 'linear' -- Grid is uniform in wavelength" \ + + defaults['spec_samp_fact'] = 1.0 + dtypes['spec_samp_fact'] = float + descr['spec_samp_fact'] = "Make the wavelength grid sampling finer (``spec_samp_fact`` less than 1.0)" \ + "or coarser (``spec_samp_fact`` greater than 1.0) by this sampling factor." \ + "This multiples the 'native' spectral pixel size by ``spec_samp_fact``," \ + "i.e. the units of ``spec_samp_fact`` are pixels." + + + defaults['spat_samp_fact'] = 1.0 + dtypes['spat_samp_fact'] = float + descr['spat_samp_fact'] = "Make the spatial sampling finer (``spat_samp_fact`` less" \ + "than 1.0) or coarser (``spat_samp_fact`` greather than 1.0) by" \ + "this sampling factor. This basically multiples the 'native'" \ + "spatial pixel size by ``spat_samp_fact``, i.e. the units of" \ + "``spat_samp_fact`` are pixels." + # Instantiate the parameter set super(Coadd2DPar, self).__init__(list(pars.keys()), @@ -1532,7 +1586,7 @@ def __init__(self, only_slits=None, exclude_slits=None, offsets=None, spat_toler def from_dict(cls, cfg): k = np.array([*cfg.keys()]) parkeys = ['only_slits', 'exclude_slits', 'offsets', 'spat_toler', 'weights', 'user_obj', 'use_slits4wvgrid', - 'manual', 'wave_method'] + 'manual', 'wave_method', 'spec_samp_fact', 'spat_samp_fact'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): @@ -1565,9 +1619,9 @@ class CubePar(ParSet): """ def __init__(self, slit_spec=None, weight_method=None, align=None, combine=None, output_filename=None, - standard_cube=None, reference_image=None, save_whitelight=None, whitelight_range=None, method=None, + sensfile=None, reference_image=None, save_whitelight=None, whitelight_range=None, method=None, ra_min=None, ra_max=None, dec_min=None, dec_max=None, wave_min=None, wave_max=None, - spatial_delta=None, wave_delta=None, astrometric=None, grating_corr=None, scale_corr=None, + spatial_delta=None, wave_delta=None, astrometric=None, scale_corr=None, skysub_frame=None, spec_subpixel=None, spat_subpixel=None, slice_subpixel=None, correct_dar=None): @@ -1635,11 +1689,11 @@ def __init__(self, slit_spec=None, weight_method=None, align=None, combine=None, 'the combined datacube. If combine=False, the output filenames will be ' \ 'prefixed with ``spec3d_*``' - defaults['standard_cube'] = None - dtypes['standard_cube'] = str - descr['standard_cube'] = 'Filename of a standard star datacube. This cube will be used to correct ' \ - 'the relative scales of the slits, and to flux calibrate the science ' \ - 'datacube.' + defaults['sensfile'] = None + dtypes['sensfile'] = str + descr['sensfile'] = 'Filename of a sensitivity function to use to flux calibrate your datacube. ' \ + 'The sensitivity function file will also be used to correct the relative scales ' \ + 'of the slits.' defaults['reference_image'] = None dtypes['reference_image'] = str @@ -1753,13 +1807,6 @@ def __init__(self, slit_spec=None, weight_method=None, align=None, combine=None, dtypes['astrometric'] = bool descr['astrometric'] = 'If true, an astrometric correction will be applied using the alignment frames.' - defaults['grating_corr'] = True - dtypes['grating_corr'] = bool - descr['grating_corr'] = 'This option performs a small correction for the relative blaze function of all ' \ - 'input frames that have (even slightly) different grating angles, or if you are ' \ - 'flux calibrating your science data with a standard star that was observed with ' \ - 'a slightly different setup.' - defaults['scale_corr'] = None dtypes['scale_corr'] = str descr['scale_corr'] = 'This option performs a small correction for the relative spectral illumination ' \ @@ -1798,10 +1845,10 @@ def from_dict(cls, cfg): k = np.array([*cfg.keys()]) # Basic keywords - parkeys = ['slit_spec', 'output_filename', 'standard_cube', 'reference_image', 'save_whitelight', + parkeys = ['slit_spec', 'output_filename', 'sensfile', 'reference_image', 'save_whitelight', 'method', 'spec_subpixel', 'spat_subpixel', 'slice_subpixel', 'ra_min', 'ra_max', 'dec_min', 'dec_max', 'wave_min', 'wave_max', 'spatial_delta', 'wave_delta', 'weight_method', 'align', 'combine', - 'astrometric', 'grating_corr', 'scale_corr', 'skysub_frame', 'whitelight_range', 'correct_dar'] + 'astrometric', 'scale_corr', 'skysub_frame', 'whitelight_range', 'correct_dar'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): @@ -1828,7 +1875,6 @@ def validate(self): raise ValueError("'weight_method' must be one of:\n" + ", ".join(allowed_weight_methods)) - class FluxCalibratePar(ParSet): """ A parameter set holding the arguments for how to perform the flux @@ -1920,8 +1966,8 @@ class SensFuncPar(ParSet): For a table with the current keywords, defaults, and descriptions, see :ref:`parameters`. """ - def __init__(self, flatfile=None, extrap_blu=None, extrap_red=None, samp_fact=None, multi_spec_det=None, algorithm=None, UVIS=None, - IR=None, polyorder=None, star_type=None, star_mag=None, star_ra=None, + def __init__(self, use_flat=None, extrap_blu=None, extrap_red=None, samp_fact=None, multi_spec_det=None, algorithm=None, UVIS=None, + IR=None, polyorder=None, star_type=None, star_mag=None, star_ra=None, extr=None, star_dec=None, mask_hydrogen_lines=None, mask_helium_lines=None, hydrogen_mask_wid=None): # Grab the parameter names and values from the function arguments args, _, _, values = inspect.getargvalues(inspect.currentframe()) @@ -1933,11 +1979,14 @@ def __init__(self, flatfile=None, extrap_blu=None, extrap_red=None, samp_fact=No dtypes = OrderedDict.fromkeys(pars.keys()) descr = OrderedDict.fromkeys(pars.keys()) - defaults['flatfile'] = None - dtypes['flatfile'] = str - descr['flatfile'] = 'Flat field file to be used if the sensitivity function model will utilize the blaze ' \ - 'function computed from a flat field file in the Calibrations directory, e.g.' \ - 'Calibrations/Flat_A_0_DET01.fits' + defaults['use_flat'] = False + dtypes['use_flat'] = bool + descr['use_flat'] = 'If True, the flatfield spectrum will be used when computing the sensitivity function.' + + defaults['extr'] = 'OPT' + dtypes['extr'] = str + descr['extr'] = 'Extraction method to use for the sensitivity function. Options are: ' \ + '\'OPT\' (optimal extraction), \'BOX\' (boxcar extraction). Default is \'OPT\'.' defaults['extrap_blu'] = 0.1 dtypes['extrap_blu'] = float @@ -2028,15 +2077,15 @@ def __init__(self, flatfile=None, extrap_blu=None, extrap_red=None, samp_fact=No options=list(options.values()), dtypes=list(dtypes.values()), descr=list(descr.values())) -# self.validate() + self.validate() @classmethod def from_dict(cls, cfg): k = np.array([*cfg.keys()]) # Single element parameters - parkeys = ['flatfile', 'extrap_blu', 'extrap_red', 'samp_fact', 'multi_spec_det', 'algorithm', - 'polyorder', 'star_type', 'star_mag', 'star_ra', 'star_dec', + parkeys = ['use_flat', 'extrap_blu', 'extrap_red', 'samp_fact', 'multi_spec_det', 'algorithm', + 'polyorder', 'star_type', 'star_mag', 'star_ra', 'star_dec', 'extr', 'mask_hydrogen_lines', 'mask_helium_lines', 'hydrogen_mask_wid'] # All parameters, including nested ParSets @@ -2058,6 +2107,14 @@ def from_dict(cls, cfg): return cls(**kwargs) + def validate(self): + """ + Check the parameters are valid for the provided method. + """ + allowed_extractions = ['BOX', 'OPT'] + if self.data['extr'] not in allowed_extractions: + msgs.error("'extr' must be one of:\n" + ", ".join(allowed_extractions)) + @staticmethod def valid_algorithms(): """ @@ -2921,15 +2978,16 @@ def __init__(self, reference=None, method=None, echelle=None, ech_nspec_coeff=No defaults['fwhm'] = 4. dtypes['fwhm'] = [int, float] descr['fwhm'] = 'Spectral sampling of the arc lines. This is the FWHM of an arcline in ' \ - 'binned pixels of the input arc image' + 'binned pixels of the input arc image. Note that this is used also in the ' \ + 'wave tilts calibration.' defaults['fwhm_fromlines'] = True dtypes['fwhm_fromlines'] = bool descr['fwhm_fromlines'] = 'Estimate spectral resolution in each slit using the arc lines. '\ - 'If True, the estimated FWHM will override ``fwhm`` only in '\ + 'If True, the estimated FWHM will override ``fwhm`` in '\ 'the determination of the wavelength solution (including the ' \ 'calculation of the threshold for the solution RMS, see ' \ - '``rms_thresh_frac_fwhm``), but not for the wave tilts calibration.' \ + '``rms_thresh_frac_fwhm``), and ALSO for the wave tilts calibration.' \ defaults['fwhm_spat_order'] = 0 dtypes['fwhm_spat_order'] = int @@ -3208,9 +3266,10 @@ def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enha minimum_slit_dlength=None, dlength_range=None, minimum_slit_length=None, minimum_slit_length_sci=None, length_range=None, minimum_slit_gap=None, clip=None, order_match=None, order_offset=None, add_missed_orders=None, order_width_poly=None, - order_gap_poly=None, order_spat_range=None, overlap=None, use_maskdesign=None, - maskdesign_maxsep=None, maskdesign_step=None, maskdesign_sigrej=None, pad=None, - add_slits=None, add_predict=None, rm_slits=None, maskdesign_filename=None): + order_gap_poly=None, order_fitrej=None, order_outlier=None, order_spat_range=None, + overlap=None, max_overlap=None, use_maskdesign=None, maskdesign_maxsep=None, + maskdesign_step=None, maskdesign_sigrej=None, pad=None, add_slits=None, + add_predict=None, rm_slits=None, maskdesign_filename=None): # Grab the parameter names and values from the function # arguments @@ -3594,11 +3653,12 @@ def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enha descr['add_missed_orders'] = 'For any Echelle spectrograph (fixed-format or otherwise), ' \ 'attempt to add orders that have been missed by the ' \ 'automated edge tracing algorithm. For *fixed-format* ' \ - 'Echelles, this is based on the expected positions on ' \ + 'echelles, this is based on the expected positions on ' \ 'on the detector. Otherwise, the detected orders are ' \ - 'modeled and roughly used to predict the locations of ' \ - 'missed orders; see additional parameters ' \ - '``order_width_poly``, ``order_gap_poly``, and ' \ + 'modeled and used to predict the locations of missed ' \ + 'orders; see additional parameters ' \ + '``order_width_poly``, ``order_gap_poly``, ' \ + '``order_fitrej``, ``order_outlier``, and ' \ '``order_spat_range``.' defaults['order_width_poly'] = 2 @@ -3613,6 +3673,24 @@ def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enha 'gap between orders as a function of the order spatial ' \ 'position. See ``add_missed_orders``.' + defaults['order_fitrej'] = 3. + dtypes['order_fitrej'] = [int, float] + descr['order_fitrej'] = 'When fitting the width of and gap beteween echelle orders with ' \ + 'Legendre polynomials, this is the sigma-clipping threshold ' \ + 'when excluding data from the fit. See ``add_missed_orders``.' + + defaults['order_outlier'] = None + dtypes['order_outlier'] = [int, float] + descr['order_outlier'] = 'When fitting the width of echelle orders with Legendre ' \ + 'polynomials, this is the sigma-clipping threshold used to ' \ + 'identify outliers. Orders clipped by this threshold are ' \ + '*removed* from further consideration, whereas orders clipped ' \ + 'by ``order_fitrej`` are excluded from the polynomial fit ' \ + 'but are not removed. Note this is *only applied to the order ' \ + 'widths*, not the order gaps. If None, no "outliers" are ' \ + 'identified/removed. Should be larger or equal to ' \ + '``order_fitrej``.' + dtypes['order_spat_range'] = list descr['order_spat_range'] = 'The spatial range of the detector/mosaic over which to ' \ 'predict order locations. If None, the full ' \ @@ -3627,6 +3705,19 @@ def __init__(self, filt_iter=None, sobel_mode=None, edge_thresh=None, sobel_enha 'the code attempts to convert the short slits into slit gaps. This ' \ 'is particularly useful for blue orders in Keck-HIRES data.' + defaults['max_overlap'] = None + dtypes['max_overlap'] = float + descr['max_overlap'] = 'When adding missing echelle orders based on where existing ' \ + 'orders are found, the prediction can yield overlapping orders. ' \ + 'The edges of these orders are adjusted to eliminate the ' \ + 'overlap, and orders can be added up over the spatial range of ' \ + 'the detector set by ``order_spate_range``. If this value is ' \ + 'None, orders are added regardless of how much they overlap. ' \ + 'If not None, this defines the maximum fraction of an order ' \ + 'spatial width that can overlap with other orders. For example, ' \ + 'if ``max_overlap=0.5``, any order that overlaps its neighboring ' \ + 'orders by more than 50% will not be added as a missing order.' + defaults['use_maskdesign'] = False dtypes['use_maskdesign'] = bool descr['use_maskdesign'] = 'Use slit-mask designs to identify slits.' @@ -3732,9 +3823,10 @@ def from_dict(cls, cfg): 'bound_detector', 'minimum_slit_dlength', 'dlength_range', 'minimum_slit_length', 'minimum_slit_length_sci', 'length_range', 'minimum_slit_gap', 'clip', 'order_match', 'order_offset', 'add_missed_orders', 'order_width_poly', - 'order_gap_poly', 'order_spat_range', 'overlap', 'use_maskdesign', - 'maskdesign_maxsep', 'maskdesign_step', 'maskdesign_sigrej', - 'maskdesign_filename', 'pad', 'add_slits', 'add_predict', 'rm_slits'] + 'order_gap_poly', 'order_fitrej', 'order_outlier', 'order_spat_range','overlap', + 'max_overlap', 'use_maskdesign', 'maskdesign_maxsep', 'maskdesign_step', + 'maskdesign_sigrej', 'maskdesign_filename', 'pad', 'add_slits', 'add_predict', + 'rm_slits'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): @@ -3772,6 +3864,10 @@ def validate(self): if not self['auto_pca'] and self['sync_predict'] == 'pca': warnings.warn('sync_predict cannot be pca if auto_pca is False. Setting to nearest.') self['sync_predict'] = 'nearest' + if self['max_overlap'] is not None and (self['max_overlap'] < 0 or self['max_overlap'] > 1): + msgs.error('If defined, max_overlap must be in the range [0,1].') + if self['order_outlier'] is not None and self['order_outlier'] < self['order_fitrej']: + msgs.warn('Order outlier threshold should not be less than the rejection threshold.') class WaveTiltsPar(ParSet): @@ -4052,7 +4148,8 @@ def __init__(self, trace_npoly=None, snr_thresh=None, find_trim_edge=None, find_maxdev=None, find_extrap_npoly=None, maxnumber_sci=None, maxnumber_std=None, find_fwhm=None, ech_find_max_snr=None, ech_find_min_snr=None, ech_find_nabove_min_snr=None, skip_second_find=None, skip_final_global=None, - skip_skysub=None, find_negative=None, find_min_max=None, std_spec1d=None, fof_link = None): + skip_skysub=None, find_negative=None, find_min_max=None, std_spec1d=None, + use_std_trace=None, fof_link = None): # Grab the parameter names and values from the function # arguments args, _, _, values = inspect.getargvalues(inspect.currentframe()) @@ -4175,14 +4272,21 @@ def __init__(self, trace_npoly=None, snr_thresh=None, find_trim_edge=None, 'detector. It only used for object finding. This parameter is helpful if your object only ' \ 'has emission lines or at high redshift and the trace only shows in part of the detector.' + defaults['use_std_trace'] = True + dtypes['use_std_trace'] = bool + descr['use_std_trace'] = 'If True, the trace of the standard star spectrum is used as a crutch for ' \ + 'tracing the object spectra. This is useful when a direct trace is not possible ' \ + '(i.e., faint sources). Note that a standard star exposure must be included in your ' \ + 'pypeit file, or the ``std_spec1d`` parameter must be set for this to work. ' + + defaults['std_spec1d'] = None dtypes['std_spec1d'] = str - descr['std_spec1d'] = 'A PypeIt spec1d file of a previously reduced standard star. The ' \ - 'trace of the standard star spectrum is used as a crutch for ' \ - 'tracing the object spectra, when a direct trace is not possible ' \ - '(i.e., faint sources). If provided, this overrides use of any ' \ - 'standards included in your pypeit file; the standard exposures ' \ - 'will still be reduced.' + descr['std_spec1d'] = 'A PypeIt spec1d file of a previously reduced standard star. ' \ + 'This can be used to trace the object spectra, but the ``use_std_trace`` ' \ + 'parameter must be set to True. If provided, this overrides use of ' \ + 'any standards included in your pypeit file; the standard exposures ' \ + 'will still be reduced.' # Instantiate the parameter set super(FindObjPar, self).__init__(list(pars.keys()), @@ -4203,7 +4307,7 @@ def from_dict(cls, cfg): 'find_maxdev', 'find_fwhm', 'ech_find_max_snr', 'ech_find_min_snr', 'ech_find_nabove_min_snr', 'skip_second_find', 'skip_final_global', 'skip_skysub', 'find_negative', 'find_min_max', - 'std_spec1d', 'fof_link'] + 'std_spec1d', 'use_std_trace', 'fof_link'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): @@ -4215,9 +4319,11 @@ def from_dict(cls, cfg): return cls(**kwargs) def validate(self): - if self.data['std_spec1d'] is not None \ - and not Path(self.data['std_spec1d']).absolute().exists(): - msgs.error(f'{self.data["std_spec1d"]} does not exist!') + if self.data['std_spec1d'] is not None: + if not self.data['use_std_trace']: + msgs.error('If you provide a standard star spectrum for tracing, you must set use_std_trace=True.') + elif not Path(self.data['std_spec1d']).absolute().exists(): + msgs.error(f'{self.data["std_spec1d"]} does not exist!') class SkySubPar(ParSet): @@ -4453,7 +4559,7 @@ class CalibrationsPar(ParSet): def __init__(self, calib_dir=None, bpm_usebias=None, biasframe=None, darkframe=None, arcframe=None, tiltframe=None, pixelflatframe=None, pinholeframe=None, alignframe=None, alignment=None, traceframe=None, illumflatframe=None, - lampoffflatsframe=None, scattlightframe=None, skyframe=None, standardframe=None, + lampoffflatsframe=None, slitless_pixflatframe=None, scattlightframe=None, skyframe=None, standardframe=None, scattlight_pad=None, flatfield=None, wavelengths=None, slitedges=None, tilts=None, raise_chk_error=None): @@ -4539,6 +4645,15 @@ def __init__(self, calib_dir=None, bpm_usebias=None, biasframe=None, darkframe=N dtypes['lampoffflatsframe'] = [ ParSet, dict ] descr['lampoffflatsframe'] = 'The frames and combination rules for the lamp off flats' + defaults['slitless_pixflatframe'] = FrameGroupPar(frametype='slitless_pixflat', + process=ProcessImagesPar(satpix='nothing', + use_pixelflat=False, + use_illumflat=False, + use_specillum=False, + combine='median')) + dtypes['slitless_pixflatframe'] = [ ParSet, dict ] + descr['slitless_pixflatframe'] = 'The frames and combination rules for the slitless pixel flat' + defaults['pinholeframe'] = FrameGroupPar(frametype='pinhole') dtypes['pinholeframe'] = [ ParSet, dict ] descr['pinholeframe'] = 'The frames and combination rules for the pinholes' @@ -4630,7 +4745,7 @@ def from_dict(cls, cfg): parkeys = [ 'calib_dir', 'bpm_usebias', 'raise_chk_error'] allkeys = parkeys + ['biasframe', 'darkframe', 'arcframe', 'tiltframe', 'pixelflatframe', - 'illumflatframe', 'lampoffflatsframe', 'scattlightframe', + 'illumflatframe', 'lampoffflatsframe', 'slitless_pixflatframe', 'scattlightframe', 'pinholeframe', 'alignframe', 'alignment', 'traceframe', 'standardframe', 'skyframe', 'scattlight_pad', 'flatfield', 'wavelengths', 'slitedges', 'tilts'] badkeys = np.array([pk not in allkeys for pk in k]) @@ -4656,6 +4771,8 @@ def from_dict(cls, cfg): kwargs[pk] = FrameGroupPar.from_dict('illumflat', cfg[pk]) if pk in k else None pk = 'lampoffflatsframe' kwargs[pk] = FrameGroupPar.from_dict('lampoffflats', cfg[pk]) if pk in k else None + pk = 'slitless_pixflatframe' + kwargs[pk] = FrameGroupPar.from_dict('slitless_pixflat', cfg[pk]) if pk in k else None pk = 'pinholeframe' kwargs[pk] = FrameGroupPar.from_dict('pinhole', cfg[pk]) if pk in k else None pk = 'scattlightframe' @@ -5230,7 +5347,7 @@ def valid_telescopes(): """ Return the valid telescopes. """ - return [ 'GEMINI-N','GEMINI-S', 'KECK', 'SHANE', 'WHT', 'APF', 'TNG', 'VLT', 'MAGELLAN', 'LBT', 'MMT', + return ['AAT', 'GEMINI-N','GEMINI-S', 'KECK', 'SHANE', 'WHT', 'APF', 'TNG', 'VLT', 'MAGELLAN', 'LBT', 'MMT', 'KPNO', 'NOT', 'P200', 'BOK', 'GTC', 'SOAR', 'NTT', 'LDT', 'JWST', 'HILTNER'] def validate(self): diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index 863bd8e597..432bfe76f5 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -245,7 +245,9 @@ def get_std_outfile(self, standard_frames): # isolate where the name of the standard-star spec1d file is defined. std_outfile = self.par['reduce']['findobj']['std_spec1d'] if std_outfile is not None: - if not Path(std_outfile).absolute().exists(): + if not self.par['reduce']['findobj']['use_std_trace']: + msgs.error('If you provide a standard star spectrum for tracing, you must set use_std_trace=True') + elif not Path(std_outfile).absolute().exists(): msgs.error(f'Provided standard spec1d file does not exist: {std_outfile}') return std_outfile @@ -254,7 +256,8 @@ def get_std_outfile(self, standard_frames): # standard associated with a given science frame. Below, I # just use the first standard - std_frame = None if len(standard_frames) == 0 else standard_frames[0] + std_frame = None if (len(standard_frames) == 0 or not self.par['reduce']['findobj']['use_std_trace']) \ + else standard_frames[0] # Prepare to load up standard? if std_frame is not None: std_outfile = self.spec_output_file(std_frame) \ @@ -705,8 +708,8 @@ def calib_one(self, frames, det): msgs.info(f'Building/loading calibrations for detector {det}') # Instantiate Calibrations class - user_slits=slittrace.merge_user_slit(self.par['rdx']['slitspatnum'], - self.par['rdx']['maskIDs']) + user_slits = slittrace.merge_user_slit(self.par['rdx']['slitspatnum'], + self.par['rdx']['maskIDs']) caliBrate = calibrations.Calibrations.get_instance( self.fitstbl, self.par['calibrations'], self.spectrograph, self.calibrations_path, qadir=self.qa_path, @@ -760,7 +763,8 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): # Is this a standard star? self.std_redux = self.objtype == 'standard' - frame_par = self.par['calibrations']['standardframe'] if self.std_redux else self.par['scienceframe'] + frame_par = self.par['calibrations']['standardframe'] \ + if self.std_redux else self.par['scienceframe'] # Get the standard trace if need be if self.std_redux is False and std_outfile is not None: @@ -789,6 +793,9 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): bkg_redux_sciimg = sciImg # Build the background image bg_file_list = self.fitstbl.frame_paths(bg_frames) + # TODO I think we should create a separate self.par['bkgframe'] parameter set to hold the image + # processing parameters for the background frames. This would allow the user to specify different + # parameters for the background frames than for the science frames. bgimg = buildimage.buildimage_fromlist(self.spectrograph, det, frame_par, bg_file_list, bpm=self.caliBrate.msbpm, bias=self.caliBrate.msbias, @@ -1022,6 +1029,10 @@ def extract_one(self, frames, det, sciImg, bkg_redux_sciimg, objFind, initial_sk if not self.par['reduce']['extraction']['skip_extraction']: msgs.info(f"Extraction begins for {self.basename} on det={det}") + # set the flatimg, if it exists + flatimg = None if self.caliBrate.flatimages is None else self.caliBrate.flatimages.pixelflat_model + if flatimg is None: + msgs.warn("No flat image was found. A spectrum of the flatfield will not be extracted!") # Instantiate Reduce object # Required for pipeline specific object # At instantiation, the fullmask in self.sciImg is modified @@ -1029,7 +1040,7 @@ def extract_one(self, frames, det, sciImg, bkg_redux_sciimg, objFind, initial_sk self.exTract = extraction.Extract.get_instance( sciImg, slits, sobjs_obj, self.spectrograph, self.par, self.objtype, global_sky=final_global_sky, bkg_redux_global_sky=bkg_redux_global_sky, - waveTilts=self.caliBrate.wavetilts, wv_calib=self.caliBrate.wv_calib, + waveTilts=self.caliBrate.wavetilts, wv_calib=self.caliBrate.wv_calib, flatimg=flatimg, bkg_redux=self.bkg_redux, return_negative=self.par['reduce']['extraction']['return_negative'], std_redux=self.std_redux, basename=self.basename, show=self.show) # Perform the extraction diff --git a/pypeit/pypeitdata.py b/pypeit/pypeitdata.py index 5cfad9b8cc..779aeb4241 100644 --- a/pypeit/pypeitdata.py +++ b/pypeit/pypeitdata.py @@ -224,7 +224,7 @@ def _parse_format(f): return _f.suffix.replace('.','').lower() def get_file_path(self, data_file, force_update=False, to_pkg=None, return_format=False, - quiet=False): + return_none=False, quiet=False): """ Return the path to a file. @@ -266,6 +266,9 @@ def get_file_path(self, data_file, force_update=False, to_pkg=None, return_forma If True, the returned object is a :obj:`tuple` that includes the file path and its format (e.g., ``'fits'``). If False, only the file path is returned. + return_none (:obj:`bool`, optional): + If True, return None if the file does not exist. If False, an + error is raised if the file does not exist. quiet (:obj:`bool`, optional): Suppress messages @@ -300,7 +303,10 @@ def get_file_path(self, data_file, force_update=False, to_pkg=None, return_forma # if the file exists in the cache and force_update is False. subdir = str(self.path.relative_to(self.data)) _cached_file = cache.fetch_remote_file(data_file, subdir, remote_host=self.host, - force_update=force_update) + force_update=force_update, return_none=return_none) + if _cached_file is None: + msgs.warn(f'File {data_file} not found in the cache.') + return None # If we've made it this far, the file is being pulled from the cache. if to_pkg is None: @@ -359,6 +365,8 @@ class PypeItDataPaths: 'skisim': {'path': 'skisim', 'host': 'github'}, 'filters': {'path': 'filters', 'host': None}, 'sensfunc': {'path': 'sensfuncs', 'host': 'github'}, + # Pixel Flats + 'pixelflat': {'path': 'pixelflats', 'host': 'github'}, # Other 'sky_spec': {'path': 'sky_spec', 'host': None}, 'static_calibs': {'path': 'static_calibs', 'host': None}, diff --git a/pypeit/pypeitsetup.py b/pypeit/pypeitsetup.py index 74a82159dc..20b95b8c03 100644 --- a/pypeit/pypeitsetup.py +++ b/pypeit/pypeitsetup.py @@ -163,30 +163,40 @@ def from_pypeit_file(cls, filename): cfg_lines=pypeItFile.cfg_lines, pypeit_file=filename) - # TODO: Make the default here match the default used by - # io.files_from_extension? @classmethod - def from_file_root(cls, root, spectrograph, extension='.fits'): + def from_file_root(cls, root, spectrograph, extension=None): """ Instantiate the :class:`~pypeit.pypeitsetup.PypeItSetup` object by providing a file root. Args: - root (:obj:`str`): - String used to find the raw files; see + root (:obj:`str`, `Path`_, :obj:`list`): + One or more paths within which to search for files; see :func:`~pypeit.io.files_from_extension`. - spectrograph (:obj:`str`): + spectrograph (:obj:`str`, :class:`~pypeit.spectrographs.spectrograph.Spectrograph`): The PypeIt name of the spectrograph used to take the observations. This should be one of the available options in :attr:`~pypeit.spectrographs.available_spectrographs`. - extension (:obj:`str`, optional): + extension (:obj:`str`, :obj:`list`, optional): The extension common to all the fits files to reduce; see - :func:`~pypeit.io.files_from_extension`. - + :func:`~pypeit.io.files_from_extension`. If None, uses the + ``allowed_extensions`` of the spectrograph class. Otherwise, + this *must* be a subset of the allowed extensions for the + selected spectrograph. + Returns: :class:`PypeitSetup`: The instance of the class. """ - return cls.from_rawfiles(io.files_from_extension(root, extension=extension), spectrograph) + # NOTE: This works if `spectrograph` is either a string or a + # Spectrograph object + spec = load_spectrograph(spectrograph).__class__ + files = spec.find_raw_files(root, extension=extension) + nfiles = len(files) + if nfiles == 0: + msgs.error(f'Unable to find any raw files for {spec.name} in {root}!') + else: + msgs.info(f'Found {nfiles} {spec.name} raw files.') + return cls.from_rawfiles(files, spectrograph) @classmethod def from_rawfiles(cls, data_files:list, spectrograph:str, frametype=None): diff --git a/pypeit/sampling.py b/pypeit/sampling.py index 9db62b90e3..71718dbac5 100644 --- a/pypeit/sampling.py +++ b/pypeit/sampling.py @@ -509,7 +509,7 @@ def _resample_linear(self, v, quad=False): # Combine the input coordinates and the output borders combinedX = numpy.append(self.outborders, self.x) - srt = numpy.argsort(combinedX) + srt = numpy.argsort(combinedX, kind='stable') combinedX = combinedX[srt] # Get the indices where the data should be reduced diff --git a/pypeit/scattlight.py b/pypeit/scattlight.py index a6f149a6d4..675da0d07b 100644 --- a/pypeit/scattlight.py +++ b/pypeit/scattlight.py @@ -122,7 +122,7 @@ def show(self, image=None, slits=None, mask=False, wcs_match=True): wcs_match : :obj:`bool`, optional Use a reference image for the WCS and match all image in other channels to it. """ - offslitmask = slits.slit_img(pad=0, initial=True, flexure=None) == -1 if mask else 1 + offslitmask = slits.slit_img(pad=0, flexure=None) == -1 if mask else 1 # Prepare the frames _data = self.scattlight_raw if image is None else image diff --git a/pypeit/scripts/__init__.py b/pypeit/scripts/__init__.py index 4ab31620bb..9f4bef02c6 100644 --- a/pypeit/scripts/__init__.py +++ b/pypeit/scripts/__init__.py @@ -9,6 +9,7 @@ from pypeit.scripts import chk_alignments from pypeit.scripts import chk_edges from pypeit.scripts import chk_flats +from pypeit.scripts import chk_flexure from pypeit.scripts import chk_tilts from pypeit.scripts import chk_for_calibs from pypeit.scripts import chk_noise_1dspec @@ -21,6 +22,7 @@ from pypeit.scripts import collate_1d from pypeit.scripts import compare_sky from pypeit.scripts import edge_inspector +from pypeit.scripts import extract_datacube from pypeit.scripts import flux_calib from pypeit.scripts import flux_setup from pypeit.scripts import identify @@ -50,6 +52,7 @@ from pypeit.scripts import trace_edges from pypeit.scripts import view_fits from pypeit.scripts import compile_wvarxiv +from pypeit.scripts import show_pixflat # Build the list of script classes @@ -64,5 +67,3 @@ def script_classes(): return dict([ (n,c) for n,c in zip(scr_n[srt],scr_c[srt])]) pypeit_scripts = list(script_classes().keys()) - - diff --git a/pypeit/scripts/cache_github_data.py b/pypeit/scripts/cache_github_data.py index 6e49169949..b7ec524722 100644 --- a/pypeit/scripts/cache_github_data.py +++ b/pypeit/scripts/cache_github_data.py @@ -130,7 +130,7 @@ def main(args): files = np.array(contents[path])[to_download[path]] if len(files) == 0: continue - data_path = PypeItDataPath(path) + data_path = PypeItDataPath(path, remote_host="github") # NOTE: I'm using POSIX path here because I'm unsure what will # happen on Windows if the file is in a subdirectory. root = pathlib.PurePosixPath(f'pypeit/data/{path}') diff --git a/pypeit/scripts/chk_flexure.py b/pypeit/scripts/chk_flexure.py new file mode 100644 index 0000000000..74faf77b01 --- /dev/null +++ b/pypeit/scripts/chk_flexure.py @@ -0,0 +1,66 @@ +""" +This script displays the flexure (spatial or spectral) applied to the science data. + +.. include common links, assuming primary doc root is up one directory +.. include:: ../include/links.rst +""" + +from pypeit.scripts import scriptbase + + +class ChkFlexure(scriptbase.ScriptBase): + + @classmethod + def get_parser(cls, width=None): + parser = super().get_parser(description='Print QA on flexure to the screen', + width=width) + + parser.add_argument('input_file', type=str, nargs='+', + help='One or more PypeIt spec2d or spec1d file') + inp = parser.add_mutually_exclusive_group(required=True) + inp.add_argument('--spec', default=False, action='store_true', + help='Check the spectral flexure') + inp.add_argument('--spat', default=False, action='store_true', + help='Check the spatial flexure') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') + return parser + + @staticmethod + def main(args): + + from IPython import embed + from astropy.io import fits + from pypeit import msgs + from pypeit import specobjs + from pypeit import spec2dobj + + chk_version = not args.try_old + flexure_type = 'spat' if args.spat else 'spec' + + # Loop over the input files + for in_file in args.input_file: + + msgs.info(f'Checking fluxure for file: {in_file}') + + # What kind of file are we?? + hdul = fits.open(in_file) + head0 = hdul[0].header + + if 'PYP_CLS' in head0.keys() and head0['PYP_CLS'].strip() == 'AllSpec2DObj': + # load the spec2d file + allspec2D = spec2dobj.AllSpec2DObj.from_fits(in_file, chk_version=chk_version) + allspec2D.flexure_diagnostics(flexure_type=flexure_type) + elif 'DMODCLS' in head0.keys() and head0['DMODCLS'].strip() == 'SpecObjs': + if flexure_type == 'spat': + msgs.error("Spat flexure not available in the spec1d file, try with a " + "spec2d file") + # load the spec1d file + sobjs = specobjs.SpecObjs.from_fitsfile(in_file, chk_version=chk_version) + sobjs.flexure_diagnostics() + else: + msgs.error("Bad file type input!") + + # space between files for clarity + print('') + diff --git a/pypeit/scripts/chk_for_calibs.py b/pypeit/scripts/chk_for_calibs.py index ed3000d992..9269529fa7 100644 --- a/pypeit/scripts/chk_for_calibs.py +++ b/pypeit/scripts/chk_for_calibs.py @@ -21,8 +21,11 @@ def get_parser(cls, width=None): parser.add_argument('-s', '--spectrograph', default=None, type=str, help='A valid spectrograph identifier: {0}'.format( ', '.join(available_spectrographs))) - parser.add_argument('-e', '--extension', default='.fits', - help='File extension; compression indicators (e.g. .gz) not required.') + parser.add_argument('-e', '--extension', default=None, + help='File extension to use. Must include the period (e.g., ".fits") ' + 'and it must be one of the allowed extensions for this ' + 'spectrograph. If None, root directory will be searched for ' + 'all files with any of the allowed extensions.') parser.add_argument('--save_setups', default=False, action='store_true', help='If not toggled, remove setup_files/ folder and its files.') return parser diff --git a/pypeit/scripts/chk_tilts.py b/pypeit/scripts/chk_tilts.py index 95f1d2ded2..baddabc85f 100644 --- a/pypeit/scripts/chk_tilts.py +++ b/pypeit/scripts/chk_tilts.py @@ -32,14 +32,18 @@ def get_parser(cls, width=None): @staticmethod def main(args): + from pathlib import Path from pypeit import wavetilts chk_version = not args.try_old + # tilts file path + file = Path(args.file).absolute() + # Load - tilts = wavetilts.WaveTilts.from_file(args.file, chk_version=chk_version) + tilts = wavetilts.WaveTilts.from_file(file, chk_version=chk_version) tilts.show(in_ginga=np.logical_not(args.mpl), show_traces=args.show_traces, - chk_version=chk_version) + calib_dir=file.parent, chk_version=chk_version) diff --git a/pypeit/scripts/coadd_1dspec.py b/pypeit/scripts/coadd_1dspec.py index 7763f9b905..98306e1126 100644 --- a/pypeit/scripts/coadd_1dspec.py +++ b/pypeit/scripts/coadd_1dspec.py @@ -22,69 +22,6 @@ from pypeit.spectrographs.util import load_spectrograph -## TODO: This is basically the exact same code as read_fluxfile in the fluxing -## script. Consolidate them? Make this a standard method in parse or io. -#def read_coaddfile(ifile): -# """ -# Read a ``PypeIt`` coadd1d file, akin to a standard PypeIt file. -# -# The top is a config block that sets ParSet parameters. The name of the -# spectrograph is required. -# -# Args: -# ifile (:obj:`str`): -# Name of the coadd file -# -# Returns: -# :obj:`tuple`: Three objects are returned: a :obj:`list` with the -# configuration entries used to modify the relevant -# :class:`~pypeit.par.parset.ParSet` parameters, a :obj:`list` the names -# of spec1d files to be coadded, and a :obj:`list` with the object IDs -# aligned with each of the spec1d files. -# """ -# # Read in the pypeit reduction file -# msgs.info('Loading the coadd1d file') -# lines = inputfiles.read_pypeit_file_lines(ifile) -# is_config = np.ones(len(lines), dtype=bool) -# -# -# # Parse the fluxing block -# spec1dfiles = [] -# objids_in = [] -# s, e = inputfiles.InputFile.find_block(lines, 'coadd1d') -# if s >= 0 and e < 0: -# msgs.error("Missing 'coadd1d end' in {0}".format(ifile)) -# elif (s < 0) or (s==e): -# msgs.error("Missing coadd1d read or [coadd1d] block in in {0}. Check the input format for the .coadd1d file".format(ifile)) -# else: -# for ctr, line in enumerate(lines[s:e]): -# prs = line.split(' ') -# spec1dfiles.append(prs[0]) -# if ctr == 0 and len(prs) != 2: -# msgs.error('Invalid format for .coadd1d file.' + msgs.newline() + -# 'You must have specify a spec1dfile and objid on the first line of the coadd1d block') -# if len(prs) > 1: -# objids_in.append(prs[1]) -# is_config[s-1:e+1] = False -# -# # Chck the sizes of the inputs -# nspec = len(spec1dfiles) -# if len(objids_in) == 1: -# objids = nspec*objids_in -# elif len(objids_in) == nspec: -# objids = objids_in -# else: -# msgs.error('Invalid format for .flux file.' + msgs.newline() + -# 'You must specify a single objid on the first line of the coadd1d block,' + msgs.newline() + -# 'or specify am objid for every spec1dfile in the coadd1d block.' + msgs.newline() + -# 'Run pypeit_coadd_1dspec --help for information on the format') -# # Construct config to get spectrograph -# cfg_lines = list(lines[is_config]) -# -# # Return -# return cfg_lines, spec1dfiles, objids - - def build_coadd_file_name(spec1dfiles, spectrograph): """Build the output file name for coadding. The filename convention is coadd1d___.fits or @@ -116,6 +53,7 @@ def build_coadd_file_name(spec1dfiles, spectrograph): path = os.path.dirname(os.path.abspath(spec1dfiles[0])) return os.path.join(path, f'coadd1d_{target}_{instrument_name}_{date_portion}.fits') + class CoAdd1DSpec(scriptbase.ScriptBase): @classmethod diff --git a/pypeit/scripts/coadd_2dspec.py b/pypeit/scripts/coadd_2dspec.py index 355c6dd72b..2fb0c1038c 100644 --- a/pypeit/scripts/coadd_2dspec.py +++ b/pypeit/scripts/coadd_2dspec.py @@ -32,18 +32,6 @@ def get_parser(cls, width=None): parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename coadd_2dspec_YYYYMMDD-HHMM.log') - - # TODO: Make spec_samp_fact and spat_samp_fact parameters in CoAdd2DPar, - # and then move these to setup_coadd2d.py - parser.add_argument('--spec_samp_fact', default=1.0, type=float, - help="Make the wavelength grid finer (spec_samp_fact < 1.0) or " - "coarser (spec_samp_fact > 1.0) by this sampling factor, i.e. " - "units of spec_samp_fact are pixels.") - parser.add_argument('--spat_samp_fact', default=1.0, type=float, - help="Make the spatial grid finer (spat_samp_fact < 1.0) or coarser " - "(spat_samp_fact > 1.0) by this sampling factor, i.e. units of " - "spat_samp_fact are pixels.") - #parser.add_argument("--wave_method", type=str, default=None, # help="Wavelength method for wavelength grid. If not set, code will " # "use linear for Multislit and log10 for Echelle") @@ -177,8 +165,8 @@ def main(args): weights=par['coadd2d']['weights'], only_slits=this_only_slits, exclude_slits=this_exclude_slits, - spec_samp_fact=args.spec_samp_fact, - spat_samp_fact=args.spat_samp_fact, + spec_samp_fact=par['coadd2d']['spec_samp_fact'], + spat_samp_fact=par['coadd2d']['spat_samp_fact'], bkg_redux=bkg_redux, find_negative=find_negative, debug_offsets=args.debug_offsets, debug=args.debug) diff --git a/pypeit/scripts/coadd_datacube.py b/pypeit/scripts/coadd_datacube.py index 2d4fbe4d71..d3c9c0aaa7 100644 --- a/pypeit/scripts/coadd_datacube.py +++ b/pypeit/scripts/coadd_datacube.py @@ -5,13 +5,6 @@ .. include common links, assuming primary doc root is up one directory .. include:: ../include/links.rst """ -import time -from pypeit import msgs -from pypeit import par -from pypeit import inputfiles -from pypeit import utils -from pypeit.coadd3d import CoAdd3D -from pypeit.spectrographs.util import load_spectrograph from pypeit.scripts import scriptbase from IPython import embed @@ -32,6 +25,15 @@ def get_parser(cls, width=None): @staticmethod def main(args): + import time + + from pypeit import msgs + from pypeit import par + from pypeit import inputfiles + from pypeit import utils + from pypeit.coadd3d import CoAdd3D + from pypeit.spectrographs.util import load_spectrograph + # Set the verbosity, and create a logfile if verbosity == 2 msgs.set_logfile_and_verbosity('coadd_datacube', args.verbosity) @@ -58,13 +60,15 @@ def main(args): dec_offsets = coadd3dfile.options['dec_offset'] skysub_frame = coadd3dfile.options['skysub_frame'] scale_corr = coadd3dfile.options['scale_corr'] + sensfile = coadd3dfile.options['sensfile'] + grating_corr = coadd3dfile.options['grating_corr'] # Instantiate CoAdd3d tstart = time.time() - coadd = CoAdd3D.get_instance(coadd3dfile.filenames, parset, skysub_frame=skysub_frame, - scale_corr=scale_corr, ra_offsets=ra_offsets, - dec_offsets=dec_offsets, spectrograph=spectrograph, - det=args.det, overwrite=args.overwrite) + coadd = CoAdd3D.get_instance(coadd3dfile.filenames, parset, skysub_frame=skysub_frame, sensfile=sensfile, + scale_corr=scale_corr, grating_corr=grating_corr, + ra_offsets=ra_offsets, dec_offsets=dec_offsets, + spectrograph=spectrograph, det=args.det, overwrite=args.overwrite) # Coadd the files coadd.run() diff --git a/pypeit/scripts/extract_datacube.py b/pypeit/scripts/extract_datacube.py new file mode 100644 index 0000000000..f819c4aad6 --- /dev/null +++ b/pypeit/scripts/extract_datacube.py @@ -0,0 +1,79 @@ +""" +This script allows the user to read a spec3D FITS file (DataCube) +from IFU instruments, and extract a 1D spectrum of the brightest +object. This script is primarily used to extract a spectrum of a +point source from a DataCube, and save it as a spec1d file. A +common usage is to extract a spectrum of a standard star from a +DataCube, and use it to flux calibrate the science DataCubes. + +.. include common links, assuming primary doc root is up one directory +.. include:: ../include/links.rst +""" +from pypeit.scripts import scriptbase + + +class ExtractDataCube(scriptbase.ScriptBase): + + @classmethod + def get_parser(cls, width=None): + parser = super().get_parser(description='Read in a datacube, extract a spectrum of a point source,' + 'and save it as a spec1d file.', width=width) + parser.add_argument('file', type = str, default=None, help='spec3d.fits DataCube file') + parser.add_argument("-e", "--ext_file", type=str, + help='Configuration file with extraction parameters') + parser.add_argument("-s", "--save", type=str, + help='Output spec1d filename') + parser.add_argument('-o', '--overwrite', default=False, action='store_true', + help='Overwrite any existing files/directories') + parser.add_argument('-b', '--boxcar_radius', type=float, default=None, + help='Radius of the circular boxcar (in arcseconds) to use for the extraction.') + parser.add_argument('-v', '--verbosity', type=int, default=1, + help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' + 'Level 2 writes a log with filename extract_datacube_YYYYMMDD-HHMM.log') + return parser + + @staticmethod + def main(args): + import time + + from pypeit import msgs + from pypeit import par + from pypeit import inputfiles + from pypeit import utils + from pypeit.spectrographs.util import load_spectrograph + from pypeit.coadd3d import DataCube + + # Set the verbosity, and create a logfile if verbosity == 2 + msgs.set_logfile_and_verbosity('extract_datacube', args.verbosity) + + # Check that a file has been provided + if args.file is None: + msgs.error('You must input a spec3d (i.e. PypeIt DataCube) fits file') + extcube = DataCube.from_file(args.file) + spectrograph = load_spectrograph(extcube.PYP_SPEC) + + if args.ext_file is None: + parset = spectrograph.default_pypeit_par() + else: + # Read in the relevant information from the .extract file + ext3dfile = inputfiles.ExtractFile.from_file(args.ext_file) + + # Parameters + spectrograph_def_par = spectrograph.default_pypeit_par() + parset = par.PypeItPar.from_cfg_lines(cfg_lines=spectrograph_def_par.to_config(), + merge_with=(ext3dfile.cfg_lines,)) + + # Set the boxcar radius + boxcar_radius = args.boxcar_radius + + # Set the output name + outname = None if args.save is None else args.save + + # Load the DataCube + tstart = time.time() + + # Extract the spectrum + extcube.extract_spec(parset['reduce'], outname=outname, boxcar_radius=boxcar_radius, overwrite=args.overwrite) + + # Report the extraction time + msgs.info(utils.get_time_string(time.time()-tstart)) diff --git a/pypeit/scripts/identify.py b/pypeit/scripts/identify.py index 610241dc81..f5d123ed62 100644 --- a/pypeit/scripts/identify.py +++ b/pypeit/scripts/identify.py @@ -67,7 +67,7 @@ def main(args): from pypeit import slittrace from pypeit.images.buildimage import ArcImage from pypeit.core.wavecal import autoid - from linetools.utils import jsonify + from pypeit.utils import jsonify chk_version = not args.try_old @@ -158,7 +158,7 @@ def main(args): else: slits_inds = np.array(list(slits.strip('[]').split(",")), dtype=int) fits_dicts = [] - specdata = [] + specdata_multi = [] wv_fits_arr = [] lines_pix_arr = [] lines_wav_arr = [] @@ -180,12 +180,14 @@ def main(args): arccen, arc_maskslit = wavecal.extract_arcs(slitIDs=[slit_val]) - # Launch the identify window - # TODO -- REMOVE THIS HACK - try: + # Get the non-linear count level + if msarc.is_mosaic: + # if this is a mosaic we take the maximum value among all the detectors + nonlinear_counts = np.max([rawdets.nonlinear_counts() for rawdets in msarc.detector.detectors]) + else: nonlinear_counts = msarc.detector.nonlinear_counts() - except AttributeError: - nonlinear_counts = None + + # Launch the identify window arcfitter = Identify.initialise(arccen, lamps, slits, slit=int(slit_val), par=par, wv_calib_all=wv_calib_slit, wavelim=[args.wmin, args.wmax], nonlinear_counts=nonlinear_counts, @@ -201,7 +203,7 @@ def main(args): return arcfitter, msarc final_fit = arcfitter.get_results() fits_dicts.append(arcfitter._fitdict) - specdata.append(arccen[:,slit_val]) + specdata_multi.append(arccen[:,slit_val]) # Build here to avoid circular import # Note: This needs to be duplicated in test_scripts.py # Wavecalib (wanted when dealing with multiple detectors, eg. GMOS) @@ -266,15 +268,21 @@ def main(args): if args.new_sol: wv_calib.copy_calib_internals(msarc) + # convert specdata into an array, since it's currently a list + specdata_multi = np.array(specdata_multi) + # If we just want the normal one-trace output else: arccen, arc_maskslit = wavecal.extract_arcs(slitIDs=[int(args.slits)]) - # Launch the identify window - # TODO -- REMOVE THIS HACK - try: + + # Get the non-linear count level + if msarc.is_mosaic: + # if this is a mosaic we take the maximum value among all the detectors + nonlinear_counts = np.max([rawdets.nonlinear_counts() for rawdets in msarc.detector.detectors]) + else: nonlinear_counts = msarc.detector.nonlinear_counts() - except AttributeError: - nonlinear_counts = None + + # Launch the identify window arcfitter = Identify.initialise(arccen, lamps, slits, slit=int(args.slits), par=par, wv_calib_all=wv_calib, wavelim=[args.wmin, args.wmax], nonlinear_counts=nonlinear_counts, @@ -303,8 +311,7 @@ def main(args): waveCalib = None fits_dicts = None - specdata = None - slits = None + specdata_multi = None lines_pix_arr = None lines_wav_arr = None lines_fit_ord = None @@ -318,7 +325,7 @@ def main(args): rmstol=args.rmstol, force_save=args.force_save, multi = args.multi, fits_dicts = fits_dicts, - specdata = np.array(specdata), + specdata_multi = specdata_multi, slits = slits, lines_pix_arr = lines_pix_arr, lines_wav_arr = lines_wav_arr, diff --git a/pypeit/scripts/multislit_flexure.py b/pypeit/scripts/multislit_flexure.py index 98e836196b..1691361687 100644 --- a/pypeit/scripts/multislit_flexure.py +++ b/pypeit/scripts/multislit_flexure.py @@ -18,55 +18,6 @@ from pypeit.scripts import scriptbase -#def read_flexfile(ifile): -# """ -# Read a ``PypeIt`` flexure file, akin to a standard ``PypeIt`` file. -# -# The top is a config block that sets ParSet parameters. -# -# Args: -# ifile (:obj:`str`): -# Name of the flexure file -# -# Returns: -# :obj:`tuple`: Two objects are returned: a :obj:`list` with the -# configuration entries used to modify the relevant -# :class:`~pypeit.par.parset.ParSet` parameters and a :obj:`list` with the -# names of spec1d files to be flexure corrected. -# """ -# # Read in the pypeit reduction file -# msgs.info('Loading the flexure file') -# lines = inputfiles.read_pypeit_file_lines(ifile) -# is_config = np.ones(len(lines), dtype=bool) -# -# # Parse the fluxing block -# spec1dfiles = [] -# objids_in = [] -# s, e = inputfiles.InputFile.find_block(lines, 'flexure') -# if s >= 0 and e < 0: -# msgs.error("Missing 'flexure end' in {0}".format(ifile)) -# elif (s < 0) or (s == e): -# msgs.error( -# "Missing flexure read block in {0}. Check the input format for the .flex file".format(ifile)) -# else: -# for ctr, line in enumerate(lines[s:e]): -# prs = line.split(' ') -# spec1dfiles.append(prs[0]) -# if len(prs) > 1: -# msgs.error('Invalid format for .flex file.' + msgs.newline() + -# 'You must specify only spec1dfiles in the block ') -# is_config[s-1:e+1] = False -# -# # Chck the sizes of the inputs -# nspec = len(spec1dfiles) -# -# # Construct config to get spectrograph -# cfg_lines = list(lines[is_config]) -# -# # Return -# return cfg_lines, spec1dfiles - - # TODO: Maybe not a good idea to name this script the same as the # flexure.MultiSlitFlexure class, but it is technically okay... class MultiSlitFlexure(scriptbase.ScriptBase): diff --git a/pypeit/scripts/obslog.py b/pypeit/scripts/obslog.py index 693b670a0c..6e1e7a7657 100644 --- a/pypeit/scripts/obslog.py +++ b/pypeit/scripts/obslog.py @@ -65,8 +65,11 @@ def get_parser(cls, width=None): parser.add_argument('-s', '--sort', default='mjd', type=str, help='Metadata keyword (pypeit-specific) to use to sort the output ' 'table.') - parser.add_argument('-e', '--extension', default='.fits', - help='File extension; compression indicators (e.g. .gz) not required.') + parser.add_argument('-e', '--extension', default=None, + help='File extension to use. Must include the period (e.g., ".fits") ' + 'and it must be one of the allowed extensions for this ' + 'spectrograph. If None, root directory will be searched for ' + 'all files with any of the allowed extensions.') parser.add_argument('-d', '--output_path', default='current working directory', help='Path to top-level output directory.') parser.add_argument('-o', '--overwrite', default=False, action='store_true', @@ -115,8 +118,7 @@ def main(args): f'argument.') # Generate the metadata table - ps = PypeItSetup.from_file_root(args.root, args.spec, - extension=args.extension) + ps = PypeItSetup.from_file_root(args.root, args.spec, extension=args.extension) ps.run(setup_only=True, # This allows for bad headers groupings=args.groupings, clean_config=args.bad_frames) diff --git a/pypeit/scripts/ql.py b/pypeit/scripts/ql.py index fbfcebcff0..f3c27e95f5 100644 --- a/pypeit/scripts/ql.py +++ b/pypeit/scripts/ql.py @@ -714,36 +714,40 @@ def get_parser(cls, width=None): help='If standard star observations are automatically detected, ' 'ignore those frames. Otherwise, they are included with the ' 'reduction of the science frames.') - parser.add_argument('--skip_display', dest='show', default=True, action='store_false', - help='Run the quicklook without displaying any results.') + parser.add_argument('--skip_display', default=False, action='store_true', + help='Run the quicklook without displaying any results. The default skip_display=False will show the results.') + parser.add_argument('--removetrace', default=False, action='store_true', + help='When the image is shown, do not overplot traces in the skysub, sky_resid, and resid ' + 'channels') # TODO: Add fluxing option? # Coadding options parser.add_argument('--coadd2d', default=False, action='store_true', help='Perform default 2D coadding.') - # TODO: Consolidate slitspatnum and only_slits! - parser.add_argument('--only_slits', type=str, nargs='+', - help='If coadding, only coadd this space-separated set of slits. If ' - 'not provided, all slits are coadded.') + parser.add_argument('--spec_samp_fact', default=1.0, type=float, + help='If coadding, adjust the wavelength grid sampling by this ' + 'factor. For a finer grid, set value to <1.0; for coarser ' + 'sampling, set value to >1.0).') + parser.add_argument('--spat_samp_fact', default=1.0, type=float, + help='If coadding, adjust the spatial grid sampling by this ' + 'factor. For a finer grid, set value to <1.0; for coarser ' + 'sampling, set value to >1.0).') parser.add_argument('--offsets', type=str, default=None, help='If coadding, spatial offsets to apply to each image; see the ' '[coadd2d][offsets] parameter. Options are restricted here to ' 'either maskdef_offsets or auto. If not specified, the ' - '(spectrograph-specific) default is used.') + '(spectrograph-specific) default is used.') parser.add_argument('--weights', type=str, default=None, help='If coadding, weights used to coadd images; see the ' '[coadd2d][weights] parameter. Options are restricted here to ' 'either uniform or auto. If not specified, the ' '(spectrograph-specific) default is used.') - parser.add_argument('--spec_samp_fact', default=1.0, type=float, - help='If coadding, adjust the wavelength grid sampling by this ' - 'factor. For a finer grid, set value to <1.0; for coarser ' - 'sampling, set value to >1.0).') - parser.add_argument('--spat_samp_fact', default=1.0, type=float, - help='If coadding, adjust the spatial grid sampling by this ' - 'factor. For a finer grid, set value to <1.0; for coarser ' - 'sampling, set value to >1.0).') + # TODO: Consolidate slitspatnum and only_slits! + parser.add_argument('--only_slits', type=str, nargs='+', + help='If coadding, only coadd this space-separated set of slits. If ' + 'not provided, all slits are coadded.') + parser.add_argument('--try_old', default=False, action='store_true', help='Attempt to load old datamodel versions. A crash may ensue..') @@ -989,6 +993,10 @@ def main(args): command_line_args += ['--offsets', args.offsets] if args.weights is not None: command_line_args += ['--weights', args.weights] + if args.spec_samp_fact != 1.0: + command_line_args += ['--spec_samp_fact', str(args.spec_samp_fact)] + if args.spat_samp_fact != 1.0: + command_line_args += ['--spat_samp_fact', str(args.spat_samp_fact)] SetupCoAdd2D.main(SetupCoAdd2D.parse_args(command_line_args)) # Find all the coadd2d scripts @@ -1002,9 +1010,7 @@ def main(args): # Run the coadding coadd2dFile = inputfiles.Coadd2DFile.from_file(coadd_file) - CoAdd2DSpec.main(CoAdd2DSpec.parse_args([str(coadd_file), - '--spec_samp_fact', str(args.spec_samp_fact), - '--spat_samp_fact', str(args.spat_samp_fact)])) + CoAdd2DSpec.main(CoAdd2DSpec.parse_args([str(coadd_file)])) # Get the output file name spectrograph, par, _ = coadd2dFile.get_pypeitpar() @@ -1017,9 +1023,12 @@ def main(args): frame = pypeIt.fitstbl.find_frames('science', index=True)[0] spec2d_file = pypeIt.spec_output_file(frame, twod=True) - if args.show: + if not args.skip_display: # TODO: Need to parse detector here? - Show2DSpec.main(Show2DSpec.parse_args([spec2d_file])) + show2d_spec_args = [spec2d_file] + if args.removetrace: + show2d_spec_args += ['--removetrace'] + Show2DSpec.main(Show2DSpec.parse_args(show2d_spec_args)) # TODO: # - Print a statement that allows users to copy-paste the correct diff --git a/pypeit/scripts/sensfunc.py b/pypeit/scripts/sensfunc.py index b687dab2ca..6a8a1d58ae 100644 --- a/pypeit/scripts/sensfunc.py +++ b/pypeit/scripts/sensfunc.py @@ -21,6 +21,14 @@ def get_parser(cls, width=None): parser.add_argument("spec1dfile", type=str, help='spec1d file for the standard that will be used to compute ' 'the sensitivity function') + parser.add_argument("--extr", type=str, default=None, choices=['OPT', 'BOX'], + help="R|Override the default extraction method used for computing the sensitivity " + "function. Note that it is not possible to set --extr and " + "simultaneously use a .sens file with the --sens_file option. If " + "you are using a .sens file, set the algorithm there via:\n\n" + "F| [sensfunc]\n" + "F| extr = BOX\n" + "\nThe extraction options are: OPT or BOX") parser.add_argument("--algorithm", type=str, default=None, choices=['UVIS', 'IR'], help="R|Override the default algorithm for computing the sensitivity " "function. Note that it is not possible to set --algorithm and " @@ -58,16 +66,16 @@ def get_parser(cls, width=None): 'provided but with .fits trimmed off if it is in the filename.') parser.add_argument("-s", "--sens_file", type=str, help='Configuration file with sensitivity function parameters') - parser.add_argument("-f", "--flatfile", type=str, - help="R|Use the flat file for computing the sensitivity " - "function. Note that it is not possible to set --flatfile and " - "simultaneously use a .sens file with the --sens_file option. If " - "you are using a .sens file, set the flatfile there via e.g.:\n\n" + parser.add_argument("-f", "--use_flat", default=False, action="store_true", + help="R|Use the extracted spectrum of the flatfield calibration to estimate the blaze " + "function when generating the sensitivity function. This is helpful to account for " + "small scale undulations in the sensitivity function. The spec1dfile must contain the " + "extracted flatfield response in order to use this option. This spectrum is extracted " + "by default, unless you did not compute a pixelflat frame. Note that it is not " + "possible to set --use_flat and simultaneously use a .sens file with the --sens_file " + "option. If you are using a .sens file, set the use_flat flag with the argument:\n\n" "F| [sensfunc]\n" - "F| flatfile = Calibrations/Flat_A_0_DET01.fits\n\n" - "Where Flat_A_0_DET01.fits is the flat file in your " - "Calibrations directory\n") - + "F| use_flat = True") parser.add_argument("--debug", default=False, action="store_true", help="show debug plots?") parser.add_argument("--par_outfile", default='sensfunc.par', @@ -102,13 +110,14 @@ def main(args): " [sensfunc]\n" " algorithm = IR\n" "\n") - if args.flatfile is not None and args.sens_file is not None: - msgs.error("It is not possible to set --flatfile and simultaneously use a .sens " + + if args.use_flat and args.sens_file is not None: + msgs.error("It is not possible to set --use_flat and simultaneously use a .sens " "file via the --sens_file option. If you are using a .sens file set the " - "flatfile there via:\n" + "use_flat flag in your .sens file using the argument:\n" "\n" " [sensfunc]\n" - " flatfile = Calibrations/Flat_A_0_DET01.fits'\n" + " use_flat = True\n" "\n") if args.multi is not None and args.sens_file is not None: @@ -120,6 +129,15 @@ def main(args): " multi_spec_det = 3,7\n" "\n") + if args.extr is not None and args.sens_file is not None: + msgs.error("It is not possible to set --extr and simultaneously use a .sens file via " + "the --sens_file option. If you are using a .sens file set the extraction " + "method there via:\n" + "\n" + " [sensfunc]\n" + " extr = BOX\n" + "\n") + # Determine the spectrograph and generate the primary FITS header with io.fits_open(args.spec1dfile) as hdul: @@ -154,10 +172,10 @@ def main(args): if args.algorithm is not None: par['sensfunc']['algorithm'] = args.algorithm - # If flatfile was provided override defaults. Note this does undo .sens + # If use_flat was flagged in the input, set use_flat to True. Note this does undo .sens # file since they cannot both be passed - if args.flatfile is not None: - par['sensfunc']['flatfile'] = args.flatfile + if args.use_flat: + par['sensfunc']['use_flat'] = True # If multi was set override defaults. Note this does undo .sens file # since they cannot both be passed @@ -166,6 +184,11 @@ def main(args): multi_spec_det = [int(item) for item in args.multi.split(',')] par['sensfunc']['multi_spec_det'] = multi_spec_det + # If extr was provided override defaults. Note this does undo .sens + # file since they cannot both be passed + if args.extr is not None: + par['sensfunc']['extr'] = args.extr + # TODO Add parsing of detectors here. If detectors passed from the # command line, overwrite the parset values read in from the .sens file diff --git a/pypeit/scripts/setup.py b/pypeit/scripts/setup.py index 9de645baa3..f417caf172 100644 --- a/pypeit/scripts/setup.py +++ b/pypeit/scripts/setup.py @@ -28,8 +28,11 @@ def get_parser(cls, width=None): 'directory (e.g., /data/Kast) or the search string up through ' 'the wildcard (.e.g, /data/Kast/b). Use the --extension option ' 'to set the types of files to search for.') - parser.add_argument('-e', '--extension', default='.fits', - help='File extension; compression indicators (e.g. .gz) not required.') + parser.add_argument('-e', '--extension', default=None, + help='File extension to use. Must include the period (e.g., ".fits") ' + 'and it must be one of the allowed extensions for this ' + 'spectrograph. If None, root directory will be searched for ' + 'all files with any of the allowed extensions.') parser.add_argument('-d', '--output_path', default='current working directory', help='Path to top-level output directory.') parser.add_argument('-o', '--overwrite', default=False, action='store_true', diff --git a/pypeit/scripts/setup_coadd2d.py b/pypeit/scripts/setup_coadd2d.py index 428c347da0..b19214c06d 100644 --- a/pypeit/scripts/setup_coadd2d.py +++ b/pypeit/scripts/setup_coadd2d.py @@ -70,6 +70,14 @@ def get_parser(cls, width=None): 'either uniform or auto. If not specified, the ' '(spectrograph-specific) default is used. Other options exist ' 'but must be entered by directly editing the coadd2d file.') + parser.add_argument('--spec_samp_fact', default=1.0, type=float, + help="Make the wavelength grid finer (spec_samp_fact < 1.0) or " + "coarser (spec_samp_fact > 1.0) by this sampling factor, i.e. " + "units of spec_samp_fact are pixels.") + parser.add_argument('--spat_samp_fact', default=1.0, type=float, + help="Make the spatial grid finer (spat_samp_fact < 1.0) or coarser " + "(spat_samp_fact > 1.0) by this sampling factor, i.e. units of " + "spat_samp_fact are pixels.") return parser @@ -178,7 +186,12 @@ def main(args): if args.weights is None else args.weights cfg['coadd2d']['spat_toler'] = par['coadd2d']['spat_toler'] \ if args.spat_toler is None else args.spat_toler - + cfg['coadd2d']['spec_samp_fact'] = par['coadd2d']['spec_samp_fact'] \ + if args.spec_samp_fact is None else args.spec_samp_fact + cfg['coadd2d']['spat_samp_fact'] = par['coadd2d']['spat_samp_fact'] \ + if args.spat_samp_fact is None else args.spat_samp_fact + + # TODO JFH Why are exclude_slits and only_slits set here when they are parameters in the parset? # Build the default parameters cfg = CoAdd2D.default_par(spec_name, inp_cfg=cfg, det=args.det, only_slits=args.only_slits, exclude_slits=args.exclude_slits) diff --git a/pypeit/scripts/setup_gui.py b/pypeit/scripts/setup_gui.py index d864eb0372..b8925be7e6 100644 --- a/pypeit/scripts/setup_gui.py +++ b/pypeit/scripts/setup_gui.py @@ -27,8 +27,11 @@ def get_parser(cls, width=None): 'the wildcard (.e.g, /data/Kast/b). Use the --extension option ' 'to set the types of files to search for. Default is the ' 'current working directory.') - parser.add_argument('-e', '--extension', default='.fits', - help='File extension; compression indicators (e.g. .gz) not required.') + parser.add_argument('-e', '--extension', default=None, + help='File extension to use. Must include the period (e.g., ".fits") ' + 'and it must be one of the allowed extensions for this ' + 'spectrograph. If None, root directory will be searched for ' + 'all files with any of the allowed extensions.') parser.add_argument('-l', '--logfile', type=str, default=None, help="Write the PypeIt logs to the given file. If the file exists it will be renamed.") parser.add_argument('-v', '--verbosity', type=int, default=2, diff --git a/pypeit/scripts/show_2dspec.py b/pypeit/scripts/show_2dspec.py index 6c54be71f0..fd16e874fd 100644 --- a/pypeit/scripts/show_2dspec.py +++ b/pypeit/scripts/show_2dspec.py @@ -41,12 +41,17 @@ def show_trace(sobjs, det, viewer, ch): det (:obj:`str`): The string identifier for the detector or mosaic used to select the extractions to show. - viewer (?): - ch (?): + viewer (:class:`~ginga.misc.Bunch.Bunch`): + Ginga viewer object. + ch (:obj:`str`): + The name of the channel in the Ginga viewer to plot the traces. """ + if sobjs is None: return in_det = np.where(sobjs.DET == det)[0] + if len(in_det) == 0: + return trace_list = [] trc_name_list = [] maskdef_extr_list = [] @@ -64,8 +69,11 @@ def show_trace(sobjs, det, viewer, ch): maskdef_extr_list.append(maskdef_extr_flag is True) manual_extr_list.append(manual_extr_flag is True) - display.show_trace(viewer, ch, np.swapaxes(trace_list, 1,0), np.array(trc_name_list), - maskdef_extr=np.array(maskdef_extr_list), manual_extr=np.array(manual_extr_list)) + if len(trace_list) > 0: + display.show_trace(viewer, ch, np.swapaxes(trace_list, 1,0), np.array(trc_name_list), + maskdef_extr=np.array(maskdef_extr_list), manual_extr=np.array(manual_extr_list)) + else: + msgs.warn('spec1d file found, but no objects were extracted for this detector.') class Show2DSpec(scriptbase.ScriptBase): @@ -79,11 +87,12 @@ def get_parser(cls, width=None): parser.add_argument('file', type=str, default=None, help='Path to a PypeIt spec2d file') parser.add_argument('--list', default=False, action='store_true', help='List the extensions only?') - parser.add_argument('--det', default='1', type=str, + parser.add_argument('--det', default=None, type=str, help='Detector name or number. If a number, the name is constructed ' 'assuming the reduction is for a single detector. If a string, ' 'it must match the name of the detector object (e.g., DET01 for ' - 'a detector, MSC01 for a mosaic).') + 'a detector, MSC01 for a mosaic). If not set, the first available detector' + 'in the spec2d file will be shown') parser.add_argument('--spat_id', type=int, default=None, help='Restrict plotting to this slit (PypeIt ID notation)') parser.add_argument('--maskID', type=int, default=None, @@ -134,12 +143,16 @@ def main(args): msgs.set_logfile_and_verbosity('show_2dspec', args.verbosity) # Parse the detector name - try: - det = int(args.det) - except: - detname = args.det - else: - detname = DetectorContainer.get_name(det) + if args.det is None: + allspec2d_det = fits.getval(args.file,'HIERARCH ALLSPEC2D_DETS') + detname = allspec2d_det.split(',')[0] + else: + try: + det = int(args.det) + except: + detname = args.det + else: + detname = DetectorContainer.get_name(det) # Find the set of channels to show show_channels = [0,1,2,3] if args.channels is None \ @@ -248,6 +261,9 @@ def main(args): waveimg = spec2DObj.waveimg img_gpm = spec2DObj.select_flag(invert=True) + if not np.any(img_gpm): + msgs.warn('The full science image is masked!') + model_gpm = img_gpm.copy() if args.ignore_extract_mask: model_gpm |= spec2DObj.select_flag(flag='EXTRACT') @@ -306,6 +322,7 @@ def main(args): cut_min = mean - 1.0 * sigma cut_max = mean + 4.0 * sigma chname_sci = args.prefix+f'sciimg-{detname}' + # Clear all channels at the beginning viewer, ch_sci = display.show_image(sciimg, chname=chname_sci, waveimg=waveimg, clear=_clear, cuts=(cut_min, cut_max)) diff --git a/pypeit/scripts/show_pixflat.py b/pypeit/scripts/show_pixflat.py new file mode 100644 index 0000000000..0240837691 --- /dev/null +++ b/pypeit/scripts/show_pixflat.py @@ -0,0 +1,62 @@ +""" +Show on a ginga window the archived pixel flat field image + +.. include common links, assuming primary doc root is up one directory +.. include:: ../include/links.rst +""" + +from pypeit.scripts import scriptbase +from IPython import embed + + +class ShowPixFlat(scriptbase.ScriptBase): + + @classmethod + def get_parser(cls, width=None): + parser = super().get_parser(description='Show an archived Pixel Flat image in a ginga window.', + width=width) + parser.add_argument("file", type=str, help="Pixel Flat filename, e.g. pixelflat_keck_lris_blue.fits.gz") + parser.add_argument('--det', default=None, type=int, nargs='+', + help='Detector(s) to show. If more than one, list the detectors as, e.g. --det 1 2 ' + 'to show detectors 1 and 2. If not provided, all detectors will be shown.') + return parser + + @staticmethod + def main(args): + import numpy as np + from pypeit import msgs + from pypeit import io + from pypeit.display import display + from pypeit import dataPaths + + # check if the file exists + file_path = dataPaths.pixelflat.get_file_path(args.file, return_none=True) + if file_path is None: + msgs.error(f'Provided pixelflat file, {args.file} not found. It is not a direct path, ' + f'a cached file, or a file that can be downloaded from a PypeIt repository.') + + # Load the image + with io.fits_open(file_path) as hdu: + # get all the available detectors in the file + file_dets = [int(h.name.split('-')[0].split('DET')[1]) for h in hdu[1:]] + # if detectors are provided, check if they are in the file + if args.det is not None: + in_file = np.isin(args.det, file_dets) + # if none of the provided detectors are in the file, raise an error + if not np.any(in_file): + msgs.error(f"Provided detector(s) not found in the file. Available detectors are {file_dets}") + # if some of the provided detectors are not in the file, warn the user + elif np.any(np.logical_not(in_file)): + det_not_in_file = np.array(args.det)[np.logical_not(in_file)] + msgs.warn(f"Detector(s) {det_not_in_file} not found in the file. Available detectors are {file_dets}") + + # show the image + display.connect_to_ginga(raise_err=True, allow_new=True) + for h in hdu[1:]: + det = int(h.name.split('-')[0].split('DET')[1]) + if args.det is not None and det not in args.det: + continue + display.show_image(h.data, chname=h.name, cuts=(0.9, 1.1), clear=False, wcs_match=True) + + + diff --git a/pypeit/scripts/trace_edges.py b/pypeit/scripts/trace_edges.py index f4827f4168..79be6ec845 100644 --- a/pypeit/scripts/trace_edges.py +++ b/pypeit/scripts/trace_edges.py @@ -175,7 +175,7 @@ def main(args): debug=args.debug, show_stages=args.show, qa_path=qa_path) - print('Tracing for detector {0} finished in {1} s.'.format(det, time.perf_counter()-t)) + msgs.info(f'Tracing for detector {det} finished in { time.perf_counter()-t:.1f} s.') # Write the two calibration frames edges.to_file() edges.get_slits().to_file() diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index 9494888054..e32c0d94ce 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -78,6 +78,7 @@ class SensFunc(datamodel.DataContainer): 'pypeline': dict(otype=str, descr='PypeIt pipeline reduction path'), 'spec1df': dict(otype=str, descr='PypeIt spec1D file used to for sensitivity function'), + 'extr': dict(otype=str, descr='Extraction method used for the standard star (OPT or BOX)'), 'std_name': dict(otype=str, descr='Type of standard source'), 'std_cal': dict(otype=str, descr='File name (or shorthand) with the standard flux data'), @@ -213,6 +214,7 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False, # Input and Output files self.spec1df = spec1dfile + self.extr = par['extr'] self.sensfile = sensfile self.par = par self.chk_version = chk_version @@ -252,13 +254,9 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False, msgs.error(f'There is a problem with your standard star spec1d file: {self.spec1df}') # Unpack standard - wave, counts, counts_ivar, counts_mask, trace_spec, trace_spat, self.meta_spec, header \ - = self.sobjs_std.unpack_object(ret_flam=False) - - # Compute the blaze function - # TODO Make the blaze function optional - log10_blaze_function = self.compute_blaze(wave, trace_spec, trace_spat, par['flatfile']) \ - if par['flatfile'] is not None else None + wave, counts, counts_ivar, counts_mask, log10_blaze_function, self.meta_spec, header \ + = self.sobjs_std.unpack_object(ret_flam=False, log10blaze=True, extract_blaze=par['use_flat'], + extract_type=self.extr, remove_missing=True) # Perform any instrument tweaks wave_twk, counts_twk, counts_ivar_twk, counts_mask_twk, log10_blaze_function_twk = \ @@ -279,65 +277,6 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False, star_mag=self.par['star_mag'], ra=star_ra, dec=star_dec) - def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, - min_blaze_value=1e-3, debug=False): - """ - Compute the blaze function from a flat field image. - - Args: - wave (`numpy.ndarray`_): - Wavelength array. Shape = (nspec, norddet) - trace_spec (`numpy.ndarray`_): - Spectral pixels for the trace of the spectrum. Shape = (nspec, norddet) - trace_spat (`numpy.ndarray`_): - Spatial pixels for the trace of the spectrum. Shape = (nspec, norddet) - flatfile (:obj:`str`): - Filename for the flat field calibration image - box_radius (:obj:`float`, optional): - Radius of the boxcar extraction region used to extract the blaze function in pixels - min_blaze_value (:obj:`float`, optional): - Minimum value of the blaze function. Values below this are clipped and set to this value. Default=1e-3 - debug (:obj:`bool`, optional): - Show plots useful for debugging. Default=False - - Returns: - `numpy.ndarray`_: The log10 blaze function. Shape = (nspec, norddet) - if norddet > 1, else shape = (nspec,) - """ - flatImages = flatfield.FlatImages.from_file(flatfile, chk_version=self.chk_version) - - pixelflat_raw = flatImages.pixelflat_raw - pixelflat_norm = flatImages.pixelflat_norm - pixelflat_proc, flat_bpm = flat.flatfield(pixelflat_raw, pixelflat_norm) - - flux_box = moment1d(pixelflat_proc * np.logical_not(flat_bpm), trace_spat, 2 * box_radius, row=trace_spec)[0] - - pixtot = moment1d(pixelflat_proc*0 + 1.0, trace_spat, 2 * box_radius, row=trace_spec)[0] - pixmsk = moment1d(flat_bpm, trace_spat, 2 * box_radius, row=trace_spec)[0] - - mask_box = (pixmsk != pixtot) & np.isfinite(wave) & (wave > 0.0) - - # TODO This is ugly and redundant with spec_atleast_2d, but the order of operations compels me to do it this way - blaze_function = (np.clip(flux_box*mask_box, 1e-3, 1e9)).reshape(-1,1) \ - if flux_box.ndim == 1 else flux_box*mask_box - wave_debug = wave.reshape(-1,1) if wave.ndim == 1 else wave - log10_blaze_function = np.zeros_like(blaze_function) - norddet = log10_blaze_function.shape[1] - for iorddet in range(norddet): - blaze_function_smooth = utils.fast_running_median(blaze_function[:, iorddet], 5) - blaze_function_norm = blaze_function_smooth/blaze_function_smooth.max() - log10_blaze_function[:, iorddet] = np.log10(np.clip(blaze_function_norm, min_blaze_value, None)) - if debug: - plt.plot(wave_debug[:, iorddet], log10_blaze_function[:,iorddet]) - if debug: - plt.show() - - - # TODO It would probably better to just return an array of shape (nspec, norddet) even if norddet = 1, i.e. - # to get rid of this .squeeze() - return log10_blaze_function.squeeze() - - def _bundle(self): """ Bundle the object for writing using @@ -470,8 +409,8 @@ def flux_std(self): # TODO assign this to the data model # Unpack the fluxed standard - _wave, _flam, _flam_ivar, _flam_mask, _, _, _, _ \ - = self.sobjs_std.unpack_object(ret_flam=True) + _wave, _flam, _flam_ivar, _flam_mask, _blaze, _, _ \ + = self.sobjs_std.unpack_object(ret_flam=True, extract_type=self.extr) # Reshape to 2d arrays wave, flam, flam_ivar, flam_mask, _, _, _ \ = utils.spec_atleast_2d(_wave, _flam, _flam_ivar, _flam_mask) @@ -481,7 +420,6 @@ def flux_std(self): self.sens['SENS_FLUXED_STD_FLAM_IVAR'] = flam_ivar.T self.sens['SENS_FLUXED_STD_MASK'] = flam_mask.T - def eval_zeropoint(self, wave, iorddet): """ Dummy method, overloaded by subclasses @@ -803,11 +741,13 @@ def write_QA(self): wave_gpm = self.sens['SENS_FLUXED_STD_WAVE'][iorddet] > 1.0 model_flux_sav[iorddet][wave_gpm] = model_interp_func(self.sens['SENS_FLUXED_STD_WAVE'][iorddet][wave_gpm]) + self.sens['SENS_STD_MODEL_FLAM'] = model_flux_sav + @classmethod - def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True, chk_version=True): + def sensfunc_weights(cls, sensfile, waves, ech_order_vec=None, debug=False, extrap_sens=True, chk_version=True): """ Get the weights based on the sensfunc @@ -817,8 +757,12 @@ def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True, chk_ve waves (`numpy.ndarray`_): wavelength grid for your output weights. Shape is (nspec, norders, nexp) or (nspec, norders). + ech_order_vec (`numpy.ndarray`_, optional): + Vector of echelle orders. Only used for echelle data. debug (bool): default=False show the weights QA + extrap_sens (bool): default=True + Extrapolate the sensitivity function chk_version (:obj:`bool`, optional): When reading in existing files written by PypeIt, perform strict version checking to ensure a valid file. If False, the code @@ -833,6 +777,10 @@ def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True, chk_ve if waves.ndim == 2: nspec, norder = waves.shape + if ech_order_vec.size != norder: + msgs.warn('The number of orders in the wave grid does not match the ' + 'number of orders in the unpacked sobjs. Echelle order vector not used.') + ech_order_vec = None nexp = 1 waves_stack = np.reshape(waves, (nspec, norder, 1)) elif waves.ndim == 3: @@ -845,16 +793,29 @@ def sensfunc_weights(cls, sensfile, waves, debug=False, extrap_sens=True, chk_ve else: msgs.error('Unrecognized dimensionality for waves') - weights_stack = np.zeros_like(waves_stack) + weights_stack = np.ones_like(waves_stack) - if norder != sens.zeropoint.shape[1]: + if norder != sens.zeropoint.shape[1] and ech_order_vec is None: msgs.error('The number of orders in {:} does not agree with your data. Wrong sensfile?'.format(sensfile)) - - for iord in range(norder): + elif norder != sens.zeropoint.shape[1] and ech_order_vec is not None: + msgs.warn('The number of orders in {:} does not match the number of orders in the data. ' + 'Using only the matching orders.'.format(sensfile)) + + # array of order to loop through + orders = np.arange(norder) if ech_order_vec is None else ech_order_vec + for iord,this_ord in enumerate(orders): + if ech_order_vec is None: + isens = iord + # find the index of the sensfunc for this order + elif ech_order_vec is not None and np.any(sens.sens['ECH_ORDERS'].value == this_ord): + isens = np.where(sens.sens['ECH_ORDERS'].value == this_ord)[0][0] + else: + # if the order is not in the sensfunc file, skip it + continue for iexp in range(nexp): sensfunc_iord = flux_calib.get_sensfunc_factor(waves_stack[:,iord,iexp], - sens.wave[:,iord], - sens.zeropoint[:,iord], 1.0, + sens.wave[:,isens], + sens.zeropoint[:,isens], 1.0, extrap_sens=extrap_sens) weights_stack[:,iord,iexp] = utils.inverse(sensfunc_iord) diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 93b4b7df32..e1367629f3 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -68,7 +68,7 @@ def exclude_for_reducing(self): def exclude_for_flexure(self): # Ignore these flags when performing a flexure calculation # Currently they are *all* of the flags.. - return ['SHORTSLIT', 'USERIGNORE', 'BADWVCALIB', 'BADTILTCALIB', 'BADALIGNCALIB', + return ['SHORTSLIT', 'BOXSLIT', 'USERIGNORE', 'BADWVCALIB', 'BADTILTCALIB', 'BADALIGNCALIB', 'SKIPFLATCALIB', 'BADFLATCALIB', 'BADSKYSUB', 'BADEXTRACT'] @@ -479,7 +479,7 @@ def get_slitlengths(self, initial=False, median=False): slitlen = right - left return np.median(slitlen, axis=1) if median else slitlen - def get_radec_image(self, wcs, alignSplines, tilts, slit_compute=None, slice_offset=None, initial=True, flexure=None): + def get_radec_image(self, wcs, alignSplines, tilts, slit_compute=None, slice_offset=None, initial=False, flexure=None): """Generate an RA and DEC image for every pixel in the frame NOTE: This function is currently only used for SlicerIFU reductions. diff --git a/pypeit/spec2dobj.py b/pypeit/spec2dobj.py index 6aaae67396..5e693e44c0 100644 --- a/pypeit/spec2dobj.py +++ b/pypeit/spec2dobj.py @@ -728,3 +728,45 @@ def __repr__(self): txt += ') >' return txt + def flexure_diagnostics(self, flexure_type='spat'): + """ + Print and return the spectral or spatial flexure of a spec2d file. + + Args: + flexure_type (:obj:`str`, optional): + Type of flexure to check. Options are 'spec' or 'spat'. Default + is 'spec'. + + Returns: + :obj:`dict`: Dictionary with the flexure values for each detector. If + flexure_type is 'spec', the spectral flexure is stored in an astropy table. + If flexure_type is 'spat', the spatial flexure is stored in a float. + + """ + if flexure_type not in ['spat', 'spec']: + msgs.error(f'flexure_type must be spat or spec, not {flexure_type}') + return_flex = {} + # Loop on Detectors + for det in self.detectors: + print('') + print('=' * 50 + f'{det:^7}' + '=' * 51) + # get and print the spectral flexure + if flexure_type == 'spec': + spec_flex = self[det].sci_spec_flexure + spec_flex.rename_column('sci_spec_flexure', 'global_spec_shift') + if np.all(spec_flex['global_spec_shift'] != None): + spec_flex['global_spec_shift'].format = '0.3f' + # print the table + spec_flex.pprint_all() + # return the table + return_flex[det] = spec_flex + # get and print the spatial flexure + if flexure_type == 'spat': + spat_flex = self[det].sci_spat_flexure + # print the value + print(f'Spat shift: {spat_flex}') + # return the value + return_flex[det] = spat_flex + + return return_flex + diff --git a/pypeit/specobj.py b/pypeit/specobj.py index 97ce35aa99..eb28627b06 100644 --- a/pypeit/specobj.py +++ b/pypeit/specobj.py @@ -55,7 +55,7 @@ class SpecObj(datamodel.DataContainer): Running index for the order. """ - version = '1.1.10' + version = '1.1.11' """ Current datamodel version number. """ @@ -87,6 +87,8 @@ class SpecObj(datamodel.DataContainer): 'OPT_COUNTS_NIVAR': dict(otype=np.ndarray, atype=float, descr='Optimally extracted noise variance, sky+read ' 'noise only (counts^2)'), + 'OPT_FLAT': dict(otype=np.ndarray, atype=float, + descr='Optimally extracted flatfield spectrum, normalised to the peak value.'), 'OPT_MASK': dict(otype=np.ndarray, atype=np.bool_, descr='Mask for optimally extracted flux. True=good'), 'OPT_FWHM': dict(otype=np.ndarray, atype=float, @@ -120,6 +122,8 @@ class SpecObj(datamodel.DataContainer): 'BOX_COUNTS_NIVAR': dict(otype=np.ndarray, atype=float, descr='Boxcar extracted noise variance, sky+read noise ' 'only (counts^2)'), + 'BOX_FLAT': dict(otype=np.ndarray, atype=float, + descr='Boxcar extracted flatfield spectrum, normalized to the peak value.'), 'BOX_MASK': dict(otype=np.ndarray, atype=np.bool_, descr='Mask for boxcar extracted flux. True=good'), 'BOX_FWHM': dict(otype=np.ndarray, atype=float, @@ -251,11 +255,20 @@ def __init__(self, PYPELINE, DET, OBJTYPE='unknown', @classmethod def from_arrays(cls, PYPELINE:str, wave:np.ndarray, counts:np.ndarray, ivar:np.ndarray, - mode='OPT', DET='DET01', SLITID=0, **kwargs): + flat=None, mode='OPT', DET='DET01', SLITID=0, **kwargs): # Instantiate slf = cls(PYPELINE, DET, SLITID=SLITID) + # Check the type of the flat field if it's not None + if flat is not None: + if not isinstance(flat, np.ndarray): + msgs.error('Flat must be a numpy array') + if flat.shape != counts.shape: + msgs.error('Flat and counts must have the same shape') # Add in arrays - for item, attr in zip([wave, counts, ivar], ['_WAVE', '_COUNTS', '_COUNTS_IVAR']): + for item, attr in zip([wave, counts, ivar, flat], ['_WAVE', '_COUNTS', '_COUNTS_IVAR', '_FLAT']): + # Check if any of the arrays are None. If so, skip + if item is None: + continue setattr(slf, mode+attr, item.astype(float)) # Mask. Watch out for places where ivar is infinite due to a divide by 0 slf[mode+'_MASK'] = (slf[mode+'_COUNTS_IVAR'] > 0.) & np.isfinite(slf[mode+'_COUNTS_IVAR']) @@ -485,7 +498,6 @@ def update_flex_shift(self, shift, flex_type='local'): # Now update the total flexure self.FLEX_SHIFT_TOTAL += shift - # TODO This should be a wrapper calling a core algorithm. def apply_flux_calib(self, wave_zp, zeropoint, exptime, tellmodel=None, extinct_correct=False, airmass=None, longitude=None, latitude=None, extinctfilepar=None, extrap_sens=False): @@ -502,8 +514,8 @@ def apply_flux_calib(self, wave_zp, zeropoint, exptime, tellmodel=None, extinct_ exptime (float): Exposure time tellmodel (?): - Telluric correction - extinct_correct (?): + Telluric correction. Note: This is deprecated and will be removed in a future version. + extinct_correct (bool, optional): If True, extinction correct airmass (float, optional): Airmass diff --git a/pypeit/specobjs.py b/pypeit/specobjs.py index 62cea7d46c..f992cf7cd1 100644 --- a/pypeit/specobjs.py +++ b/pypeit/specobjs.py @@ -113,7 +113,7 @@ def from_fitsfile(cls, fits_file, det=None, chk_version=True): # from_hdu method, and the name of the HDU must have a known format # (e.g., 'DET01-DETECTOR'). _det = hdu.name.split('-')[0] - detector_hdus[_det] = dmodcls.from_hdu(hdu) + detector_hdus[_det] = dmodcls.from_hdu(hdu, chk_version=chk_version) # Now the objects for hdu in hdul[1:]: @@ -188,7 +188,8 @@ def nobj(self): """ return len(self.specobjs) - def unpack_object(self, ret_flam=False, extract_type='OPT'): + def unpack_object(self, ret_flam=False, log10blaze=False, min_blaze_value=1e-3, extract_type='OPT', + extract_blaze=False, remove_missing=False): """ Utility function to unpack the sobjs for one object and return various numpy arrays describing the spectrum and meta @@ -196,8 +197,19 @@ def unpack_object(self, ret_flam=False, extract_type='OPT'): the relevant indices for the object. Args: - ret_flam (:obj:`bool`, optional): - If True return the FLAM, otherwise return COUNTS. + ret_flam (:obj:`bool`, optional): + If True return the FLAM, otherwise return COUNTS. + log10blaze (:obj:`bool`, optional): + If True return the log10 of the blaze function. + min_blaze_value (:obj:`float`, optional): + Minimum value of the blaze function to consider as good. + extract_type (:obj:`str`, optional): + Extraction type to use. Default is 'OPT'. + extract_blaze (:obj:`bool`, optional): + If True, extract the blaze function. Default is False. + remove_missing (:obj:`bool`, optional): + If True, remove any missing data (i.e. where the flux is None). + Default is False. Returns: tuple: Returns the following where all numpy arrays @@ -210,6 +222,7 @@ def unpack_object(self, ret_flam=False, extract_type='OPT'): Flambda or counts) - flux_gpm (`numpy.ndarray`_): Good pixel mask. True=Good + - blaze (`numpy.ndarray`_, None): Blaze function - meta_spec (dict:) Dictionary containing meta data. The keys are defined by spectrograph.parse_spec_header() @@ -217,26 +230,40 @@ def unpack_object(self, ret_flam=False, extract_type='OPT'): spec1d file """ # Prep - norddet = self.nobj flux_attr = 'FLAM' if ret_flam else 'COUNTS' flux_key = '{}_{}'.format(extract_type, flux_attr) wave_key = '{}_WAVE'.format(extract_type) - if getattr(self, flux_key)[0] is None: - msgs.error("Flux not available for {}. Try the other ".format(flux_key)) + blaze_key = '{}_FLAT'.format(extract_type) + + # Check for missing data + none_flux = [f is None for f in getattr(self, flux_key)] + if np.any(none_flux): + other = 'OPT' if extract_type == 'BOX' else 'BOX' + msg = f"{extract_type} extracted flux is not available for all slits/orders. " \ + f"Consider trying the {other} extraction." + if not remove_missing: + msgs.error(msg) + else: + msg += f"{msgs.newline()}-- The missing data will be removed --" + msgs.warn(msg) + # Remove missing data + r_indx = np.where(none_flux)[0] + self.remove_sobj(r_indx) + # + norddet = self.nobj nspec = getattr(self, flux_key)[0].size # Allocate arrays and unpack spectrum wave = np.zeros((nspec, norddet)) flux = np.zeros((nspec, norddet)) flux_ivar = np.zeros((nspec, norddet)) flux_gpm = np.zeros((nspec, norddet), dtype=bool) - trace_spec = np.zeros((nspec, norddet)) - trace_spat = np.zeros((nspec, norddet)) + if extract_blaze: + blaze = np.zeros((nspec, norddet), dtype=float) detector = [None]*norddet ech_orders = np.zeros(norddet, dtype=int) - # TODO make the extraction that is desired OPT vs BOX an optional input variable. for iorddet in range(norddet): wave[:, iorddet] = getattr(self, wave_key)[iorddet] flux_gpm[:, iorddet] = getattr(self, '{}_MASK'.format(extract_type))[iorddet] @@ -244,9 +271,20 @@ def unpack_object(self, ret_flam=False, extract_type='OPT'): if self[0].PYPELINE == 'Echelle': ech_orders[iorddet] = self[iorddet].ECH_ORDER flux[:, iorddet] = getattr(self, flux_key)[iorddet] - flux_ivar[:, iorddet] = getattr(self, flux_key+'_IVAR')[iorddet] #OPT_FLAM_IVAR - trace_spat[:, iorddet] = self[iorddet].TRACE_SPAT - trace_spec[:, iorddet] = self[iorddet].trace_spec + flux_ivar[:, iorddet] = getattr(self, flux_key+'_IVAR')[iorddet] + if extract_blaze: + blaze[:, iorddet] = getattr(self, blaze_key)[iorddet] + + # Log10 blaze + if extract_blaze: + blaze_function = np.copy(blaze) + if log10blaze: + for iorddet in range(norddet): + blaze_function_smooth = utils.fast_running_median(blaze[:, iorddet], 5) + blaze_function_norm = blaze_function_smooth / blaze_function_smooth.max() + blaze_function[:, iorddet] = np.log10(np.clip(blaze_function_norm, min_blaze_value, None)) + else: + blaze_function = None # Populate meta data spectrograph = load_spectrograph(self.header['PYP_SPEC']) @@ -261,11 +299,12 @@ def unpack_object(self, ret_flam=False, extract_type='OPT'): # Return if self[0].PYPELINE in ['MultiSlit', 'SlicerIFU'] and self.nobj == 1: meta_spec['ECH_ORDERS'] = None + blaze_ret = blaze_function.reshape(nspec) if extract_blaze else None return wave.reshape(nspec), flux.reshape(nspec), flux_ivar.reshape(nspec), \ - flux_gpm.reshape(nspec), trace_spec.reshape(nspec), trace_spat.reshape(nspec), meta_spec, self.header + flux_gpm.reshape(nspec), blaze_ret, meta_spec, self.header else: meta_spec['ECH_ORDERS'] = ech_orders - return wave, flux, flux_ivar, flux_gpm, trace_spec, trace_spat, meta_spec, self.header + return wave, flux, flux_ivar, flux_gpm, blaze_function, meta_spec, self.header def get_std(self, multi_spec_det=None): """ @@ -719,16 +758,23 @@ def write_to_fits(self, subheader, outfile, overwrite=True, update_det=None, Args: subheader (:obj:`dict`): + Dictionary with header keywords and values to be added to the + primary header of the output file. outfile (str): + Name of the output file overwrite (bool, optional): + Overwrite the output file if it exists? + update_det (int or list, optional): + If provided, do not clobber the existing file but only update + the indicated detectors. Useful for re-running on a subset of detectors slitspatnum (:obj:`str` or :obj:`list`, optional): Restricted set of slits for reduction. If provided, do not clobber the existing file but only update the indicated slits. Useful for re-running on a subset of slits - update_det (int or list, optional): - If provided, do not clobber the existing file but only update - the indicated detectors. Useful for re-running on a subset of detectors - + history (:obj:`str`, optional): + String to be added to the header HISTORY keyword. + debug (:obj:`bool`, optional): + If True, run in debug mode. """ if os.path.isfile(outfile) and not overwrite: msgs.warn(f'{outfile} exits. Set overwrite=True to overwrite it.') @@ -766,6 +812,15 @@ def write_to_fits(self, subheader, outfile, overwrite=True, update_det=None, header[key.upper()] = line else: header[key.upper()] = subheader[key] + # Also store the datetime in ISOT format + if key.upper() == 'MJD': + if isinstance(subheader[key], (list, tuple)): + mjdval = subheader[key][0] + elif isinstance(subheader[key], float): + mjdval = subheader[key] + else: + raise ValueError('Header card must be a float or a FITS header tuple') + header['DATETIME'] = (Time(mjdval, format='mjd').isot, "Date and time of the observation in ISOT format") # Add calibration associations to Header if self.calibs is not None: for key, val in self.calibs.items(): @@ -1002,6 +1057,30 @@ def get_extraction_groups(self, model_full_slit=False) -> List[List[int]]: return groups + def flexure_diagnostics(self): + """ + Print and return the spectral flexure of a spec1d file. + + Returns: + :obj:`astropy.table.Table`: Table with the spectral flexure. + """ + spec_flex = Table() + spec_flex['NAME'] = self.NAME + spec_flex['global_spec_shift'] = self.FLEX_SHIFT_GLOBAL + if np.all(spec_flex['global_spec_shift'] != None): + spec_flex['global_spec_shift'].format = '0.3f' + spec_flex['local_spec_shift'] = self.FLEX_SHIFT_LOCAL + if np.all(spec_flex['local_spec_shift'] != None): + spec_flex['local_spec_shift'].format = '0.3f' + spec_flex['total_spec_shift'] = self.FLEX_SHIFT_TOTAL + if np.all(spec_flex['total_spec_shift'] != None): + spec_flex['total_spec_shift'].format = '0.3f' + # print the table + spec_flex.pprint_all() + # return the table + return spec_flex + + #TODO Should this be a classmethod on specobjs?? def get_std_trace(detname, std_outfile, chk_version=True): """ @@ -1014,9 +1093,11 @@ def get_std_trace(detname, std_outfile, chk_version=True): Filename with the standard star spec1d file. Can be None. Returns: - `numpy.ndarray`_: Trace of the standard star on input detector. Will - be None if ``std_outfile`` is None, or if the selected detector/mosaic - is not available in the provided spec1d file. + `astropy.table.Table`_: Table with the trace of the standard star on the input detector. + If this is a MultiSlit reduction, the table will have a single column: `TRACE_SPAT`. + If this is an Echelle reduction, the table will have two columns: `ECH_ORDER` and `TRACE_SPAT`. + Will be None if ``std_outfile`` is None, or if the selected detector/mosaic + is not available in the provided spec1d file, or for SlicerIFU reductions. """ sobjs = SpecObjs.from_fitsfile(std_outfile, chk_version=chk_version) @@ -1033,20 +1114,24 @@ def get_std_trace(detname, std_outfile, chk_version=True): # No standard extracted on this detector?? if sobjs_std is None: return None - std_trace = sobjs_std.TRACE_SPAT + + # create table that contains the trace of the standard + std_tab = Table() # flatten the array if this multislit if 'MultiSlit' in pypeline: - std_trace = std_trace.flatten() + std_tab['TRACE_SPAT'] = sobjs_std.TRACE_SPAT elif 'Echelle' in pypeline: - std_trace = std_trace.T + std_tab['ECH_ORDER'] = sobjs_std.ECH_ORDER + std_tab['TRACE_SPAT'] = sobjs_std.TRACE_SPAT elif 'SlicerIFU' in pypeline: - std_trace = None + std_tab = None else: msgs.error('Unrecognized pypeline') else: - std_trace = None + std_tab = None + + return std_tab - return std_trace def lst_to_array(lst, mask=None): """ @@ -1055,7 +1140,7 @@ def lst_to_array(lst, mask=None): Allows for a list of Quantity objects Args: - lst : list + lst (:obj:`list`): Should be number or Quantities mask (`numpy.ndarray`_, optional): Boolean array used to limit to a subset of the list. True=good @@ -1063,10 +1148,25 @@ def lst_to_array(lst, mask=None): Returns: `numpy.ndarray`_, `astropy.units.Quantity`_: Converted list """ - if mask is None: - mask = np.array([True]*len(lst)) + _mask = np.ones(len(lst), dtype=bool) if mask is None else mask + + # Return a Quantity array if isinstance(lst[0], units.Quantity): - return units.Quantity(lst)[mask] - else: - return np.array(lst)[mask] + return units.Quantity(lst)[_mask] + + # If all the elements of lst have the same type, np.array(lst)[mask] will work + if len(set(map(type, lst))) == 1: + return np.array(lst)[_mask] + + # Otherwise, we have to set the array type to object + return np.array(lst, dtype=object)[_mask] + + # NOTE: The dtype="object" is needed for the case where one element of lst + # is not a list but None. For example, if trying to unpack SpecObjs OPT fluxes + # and for one slit/order the OPT extraction failed (but not the BOX extraction), + # OPT_COUNTS is None for that slit/order, and lst would be something like + # [array, array, array, None, array], which makes np.array to fail and give the error + # "ValueError: setting an array element with a sequence. The requested array has an + # inhomogeneous shape after 1 dimensions..." + diff --git a/pypeit/spectrographs/__init__.py b/pypeit/spectrographs/__init__.py index 3377156931..06efedfb57 100644 --- a/pypeit/spectrographs/__init__.py +++ b/pypeit/spectrographs/__init__.py @@ -3,11 +3,12 @@ # The import of all the spectrograph modules here is what enables the dynamic # compiling of all the available spectrographs below -from pypeit.spectrographs import gtc_osiris +from pypeit.spectrographs import aat_uhrf from pypeit.spectrographs import bok_bc from pypeit.spectrographs import gemini_flamingos from pypeit.spectrographs import gemini_gmos from pypeit.spectrographs import gemini_gnirs +from pypeit.spectrographs import gtc_osiris from pypeit.spectrographs import keck_esi from pypeit.spectrographs import keck_deimos from pypeit.spectrographs import keck_hires diff --git a/pypeit/spectrographs/aat_uhrf.py b/pypeit/spectrographs/aat_uhrf.py new file mode 100644 index 0000000000..ea71c1ed10 --- /dev/null +++ b/pypeit/spectrographs/aat_uhrf.py @@ -0,0 +1,257 @@ +""" +Module for Shane/Kast specific methods. + +.. include:: ../include/links.rst +""" +import os + +from IPython import embed + +import numpy as np + +from astropy.time import Time + +from pypeit import msgs +from pypeit import telescopes +from pypeit.core import framematch +from pypeit.spectrographs import spectrograph +from pypeit.images import detector_container +from pypeit import data + + +class AATUHRFSpectrograph(spectrograph.Spectrograph): + """ + Child to handle AAT/UHRF specific code + """ + ndet = 1 + telescope = telescopes.AATTelescopePar() + url = 'https://aat.anu.edu.au/science/instruments/decomissioned/uhrf/overview' + ql_supported = False + name = 'aat_uhrf' + camera = 'UHRF' + supported = True + header_name = 'uhrf' + allowed_extensions = [".FTS"] + + def get_detector_par(self, det, hdu=None): + """ + Return metadata for the selected detector. + + Args: + det (:obj:`int`): + 1-indexed detector number. + hdu (`astropy.io.fits.HDUList`_, optional): + The open fits file with the raw image of interest. If not + provided, frame-dependent parameters are set to a default. + + Returns: + :class:`~pypeit.images.detector_container.DetectorContainer`: + Object with the detector metadata. + """ + # Retrieve the binning + binning = '1,1' if hdu is None else self.compound_meta(self.get_headarr(hdu), "binning") + dsec = 1 + 1024//int(binning.split(',')[0]) + # Detector 1 + detector_dict = dict( + binning=binning, + det=1, + dataext=0, + specaxis=0, + specflip=False, + spatflip=False, + platescale=0.05, # Not sure about this value + saturation=65535., + mincounts=-1e10, + nonlinear=0.76, + numamplifiers=1, + gain=np.asarray([1.0]), # Not sure about this value + ronoise=np.asarray([0.0]), # Determine the read noise from the overscan region + xgap=0., + ygap=0., + ysize=1., + darkcurr=0.0, # e-/pixel/hour + # These are rows, columns on the raw frame, 1-indexed + datasec=np.asarray(['[:, 1:{:d}]'.format(dsec)]), + oscansec=np.asarray(['[:, {:d}:]'.format(dsec+1)]) + ) + return detector_container.DetectorContainer(**detector_dict) + + @classmethod + def default_pypeit_par(cls): + """ + Return the default parameters to use for this instrument. + + Returns: + :class:`~pypeit.par.pypeitpar.PypeItPar`: Parameters required by + all of PypeIt methods. + """ + par = super().default_pypeit_par() + + # Ignore PCA + par['calibrations']['slitedges']['sync_predict'] = 'nearest' + # Bound the detector with slit edges if no edges are found + par['calibrations']['slitedges']['bound_detector'] = True + + # Never correct for flexure - the sky is subdominant compared to the object and basically never detected. + par['flexure']['spec_method'] = 'skip' + + # Sky subtraction parameters - this instrument has no sky lines, but we still use the sky subtraction + # routine to subtract scattered light. + par['reduce']['skysub']['no_poly'] = True + par['reduce']['skysub']['bspline_spacing'] = 3.0 + par['reduce']['skysub']['user_regions'] = ':10,75:' # This is about right for most setups tested so far + par['scienceframe']['process']['sigclip'] = 10.0 + + # Set some parameters for the calibrations + # par['calibrations']['wavelengths']['reid_arxiv'] = 'None' + par['calibrations']['wavelengths']['lamps'] = ['ThAr'] + par['calibrations']['wavelengths']['n_final'] = 3 + par['calibrations']['tilts']['spat_order'] = 4 + par['calibrations']['tilts']['spec_order'] = 1 + + # Set the default exposure time ranges for the frame typing + # Trace frames should be the same as arc frames - it will force a bound detector and this + # allows the scattered light to be subtracted. A pixel-to-pixel sensitivity correction is + # not needed for this instrument, since it's a small slicer that projects the target onto + # multiple pixels. This instrument observes bright objects only, so sky subtraction is not + # important, but the sky subtraction routine is used to subtract scattered light, instead. + par['calibrations']['arcframe']['exprng'] = [None, 60.0] + par['calibrations']['tiltframe']['exprng'] = [None, 60.0] + par['calibrations']['traceframe']['exprng'] = [None, 60.0] + par['scienceframe']['exprng'] = [61, None] + + return par + + def init_meta(self): + """ + Define how metadata are derived from the spectrograph files. + + That is, this associates the PypeIt-specific metadata keywords + with the instrument-specific header cards using :attr:`meta`. + """ + self.meta = {} + # Required (core) + self.meta['ra'] = dict(ext=0, card='MEANRA') + self.meta['dec'] = dict(ext=0, card='MEANDEC') + self.meta['target'] = dict(ext=0, card='OBJECT') + # dispname is arm specific (blue/red) + self.meta['decker'] = dict(ext=0, card='WINDOW') + self.meta['dispname'] = dict(ext=0, card='WINDOW') + self.meta['binning'] = dict(ext=0, card=None, compound=True) + self.meta['mjd'] = dict(ext=0, card=None, compound=True) + self.meta['exptime'] = dict(ext=0, card='TOTALEXP') + self.meta['airmass'] = dict(ext=0, card=None, compound=True) + # Additional ones, generally for configuration determination or time + # self.meta['dichroic'] = dict(ext=0, card='BSPLIT_N') + # self.meta['instrument'] = dict(ext=0, card='VERSION') + + def compound_meta(self, headarr, meta_key): + """ + Methods to generate metadata requiring interpretation of the header + data, instead of simply reading the value of a header card. + + Args: + headarr (:obj:`list`): + List of `astropy.io.fits.Header`_ objects. + meta_key (:obj:`str`): + Metadata keyword to construct. + + Returns: + object: Metadata value read from the header(s). + """ + if meta_key == 'mjd': + date = headarr[0]['UTDATE'].replace(":","-") + time = headarr[0]['UTSTART'] + ttime = Time(f'{date}T{time}', format='isot') + return ttime.mjd + elif meta_key == 'binning': + binspat = int(np.ceil(1024/headarr[0]['NAXIS1'])) + binspec = int(np.ceil(1024/headarr[0]['NAXIS2'])) + return f'{binspat},{binspec}' + elif meta_key == 'airmass': + # Calculate the zenith distance + zendist = 0.5*(headarr[0]['ZDSTART']+headarr[0]['ZDEND']) + # Return the airmass based on the zenith distance + return 1./np.cos(np.deg2rad(zendist)) + msgs.error("Not ready for this compound meta") + + def configuration_keys(self): + """ + Return the metadata keys that define a unique instrument + configuration. + + This list is used by :class:`~pypeit.metadata.PypeItMetaData` to + identify the unique configurations among the list of frames read + for a given reduction. + + Returns: + :obj:`list`: List of keywords of data pulled from file headers + and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` + object. + """ + # decker is not included because arcs are often taken with a 0.5" slit + return ['dispname'] + + def check_frame_type(self, ftype, fitstbl, exprng=None): + """ + Check for frames of the provided type. + + Args: + ftype (:obj:`str`): + Type of frame to check. Must be a valid frame type; see + frame-type :ref:`frame_type_defs`. + fitstbl (`astropy.table.Table`_): + The table with the metadata for one or more frames to check. + exprng (:obj:`list`, optional): + Range in the allowed exposure time for a frame of type + ``ftype``. See + :func:`pypeit.core.framematch.check_frame_exptime`. + + Returns: + `numpy.ndarray`_: Boolean array with the flags selecting the + exposures in ``fitstbl`` that are ``ftype`` type frames. + """ + good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) + if ftype in ['science']: + return good_exp + if ftype in ['standard']: + return np.zeros(len(fitstbl), dtype=bool) + if ftype == 'bias': + return np.zeros(len(fitstbl), dtype=bool) + if ftype in ['pixelflat', 'trace', 'illumflat']: + # Flats and trace frames are typed together + return np.zeros(len(fitstbl), dtype=bool) + if ftype in ['pinhole', 'dark']: + # Don't type pinhole or dark frames + return np.zeros(len(fitstbl), dtype=bool) + if ftype in ['arc', 'tilt']: + return good_exp + + msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) + return np.zeros(len(fitstbl), dtype=bool) + + def config_specific_par(self, scifile, inp_par=None): + """ + Modify the PypeIt parameters to hard-wired values used for + specific instrument configurations. + + Args: + scifile (:obj:`str`): + File to use when determining the configuration and how + to adjust the input parameters. + inp_par (:class:`~pypeit.par.parset.ParSet`, optional): + Parameter set used for the full run of PypeIt. If None, + use :func:`default_pypeit_par`. + + Returns: + :class:`~pypeit.par.parset.ParSet`: The PypeIt parameter set + adjusted for configuration specific parameter values. + """ + par = super().config_specific_par(scifile, inp_par=inp_par) + + if par['calibrations']['wavelengths']['reid_arxiv'] is None: + msgs.warn("Wavelength setup not supported!" + msgs.newline() + msgs.newline() + + "Please perform your own wavelength calibration, and provide the path+filename using:" + msgs.newline() + + msgs.pypeitpar_text(['calibrations', 'wavelengths', 'reid_arxiv = '])) + # Return + return par diff --git a/pypeit/spectrographs/gemini_gmos.py b/pypeit/spectrographs/gemini_gmos.py index e00506920e..269527a897 100644 --- a/pypeit/spectrographs/gemini_gmos.py +++ b/pypeit/spectrographs/gemini_gmos.py @@ -108,6 +108,7 @@ class GeminiGMOSSpectrograph(spectrograph.Spectrograph): """ ndet = 3 url = 'http://www.gemini.edu/instrumentation/gmos' + allowed_extensions = ['.fits', '.fits.bz2', '.fits.gz'] def __init__(self): super().__init__() @@ -460,7 +461,7 @@ def get_rawimage(self, raw_file, det): return detectors[0], array[0], hdu, exptime, rawdatasec_img[0], oscansec_img[0] return mosaic, array, hdu, exptime, rawdatasec_img, oscansec_img - def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): + def get_mosaic_par(self, mosaic, hdu=None, msc_ord=0): """ Return the hard-coded parameters needed to construct detector mosaics from unbinned images. @@ -481,7 +482,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): default. BEWARE: If ``hdu`` is not provided, the binning is assumed to be `1,1`, which will cause faults if applied to binned images! - msc_order (:obj:`int`, optional): + msc_ord (:obj:`int`, optional): Order of the interpolation used to construct the mosaic. Returns: @@ -535,7 +536,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): msc_tfm[i] = build_image_mosaic_transform(shape, msc_sft[i], msc_rot[i], tuple(reversed(binning))) return Mosaic(mosaic_id, detectors, shape, np.array(msc_sft), np.array(msc_rot), - np.array(msc_tfm), msc_order) + np.array(msc_tfm), msc_ord) @property def allowed_mosaics(self): @@ -864,7 +865,7 @@ def get_detector_par(self, det, hdu=None): # Return return detector_container.DetectorContainer(**detectors[det-1]) - def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): + def get_mosaic_par(self, mosaic, hdu=None, msc_ord=0): """ Return the hard-coded parameters needed to construct detector mosaics from unbinned images. @@ -885,7 +886,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): default. BEWARE: If ``hdu`` is not provided, the binning is assumed to be `1,1`, which will cause faults if applied to binned images! - msc_order (:obj:`int`, optional): + msc_ord (:obj:`int`, optional): Order of the interpolation used to construct the mosaic. Returns: @@ -907,7 +908,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): else: self.detid = 'BI5-36-4k-2,BI11-33-4k-1,BI12-34-4k-1' - return super().get_mosaic_par(mosaic, hdu=hdu, msc_order=msc_order) + return super().get_mosaic_par(mosaic, hdu=hdu, msc_ord=msc_ord) @classmethod def default_pypeit_par(cls): @@ -1027,6 +1028,8 @@ def config_specific_par(self, scifile, inp_par=None): elif self.get_meta_value(scifile, 'dispname')[0:4] == 'B600': par['calibrations']['wavelengths']['reid_arxiv'] = 'gemini_gmos_south_ham_b600_compiled.fits' par['calibrations']['wavelengths']['method'] = 'reidentify' + elif self.get_meta_value(scifile, 'dispname')[0:4] == 'B480': + par['calibrations']['wavelengths']['reid_arxiv'] = 'gemini_gmos_north_ham_b480.fits' # The bad amp needs a larger follow_span for slit edge tracing obs_epoch = time.Time(self.get_meta_value(scifile, 'mjd'), format='mjd').jyear @@ -1045,6 +1048,7 @@ class GeminiGMOSNSpectrograph(GeminiGMOSSpectrograph): telescope = telescopes.GeminiNTelescopePar() camera = 'GMOS-N' header_name = 'GMOS-N' + allowed_extensions = ['.fits', '.fits.bz2', '.fits.gz'] class GeminiGMOSNHamSpectrograph(GeminiGMOSNSpectrograph): @@ -1142,7 +1146,7 @@ def get_detector_par(self, det, hdu=None): # Return return detector_container.DetectorContainer(**detectors[det-1]) - def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): + def get_mosaic_par(self, mosaic, hdu=None, msc_ord=0): """ Return the hard-coded parameters needed to construct detector mosaics from unbinned images. @@ -1163,7 +1167,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): default. BEWARE: If ``hdu`` is not provided, the binning is assumed to be `1,1`, which will cause faults if applied to binned images! - msc_order (:obj:`int`, optional): + msc_ord (:obj:`int`, optional): Order of the interpolation used to construct the mosaic. Returns: @@ -1174,7 +1178,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): # Detector ID (it is used to identify the correct mosaic geometry) self.detid = 'BI13-20-4k-1,BI12-09-4k-2,BI13-18-4k-2' - return super().get_mosaic_par(mosaic, hdu=hdu, msc_order=msc_order) + return super().get_mosaic_par(mosaic, hdu=hdu, msc_ord=msc_ord) def config_specific_par(self, scifile, inp_par=None): """ @@ -1410,7 +1414,7 @@ def get_detector_par(self, det, hdu=None): # Return return detector_container.DetectorContainer(**detectors[det-1]) - def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): + def get_mosaic_par(self, mosaic, hdu=None, msc_ord=0): """ Return the hard-coded parameters needed to construct detector mosaics from unbinned images. @@ -1431,7 +1435,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): default. BEWARE: If ``hdu`` is not provided, the binning is assumed to be `1,1`, which will cause faults if applied to binned images! - msc_order (:obj:`int`, optional): + msc_ord (:obj:`int`, optional): Order of the interpolation used to construct the mosaic. Returns: @@ -1443,7 +1447,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): # TODO: Check this is correct self.detid = 'e2v 10031-23-05,10031-01-03,10031-18-04' - return super().get_mosaic_par(mosaic, hdu=hdu, msc_order=msc_order) + return super().get_mosaic_par(mosaic, hdu=hdu, msc_ord=msc_ord) def config_specific_par(self, scifile, inp_par=None): """ diff --git a/pypeit/spectrographs/gemini_gnirs.py b/pypeit/spectrographs/gemini_gnirs.py index 148464d7d1..52e346661b 100644 --- a/pypeit/spectrographs/gemini_gnirs.py +++ b/pypeit/spectrographs/gemini_gnirs.py @@ -24,6 +24,7 @@ class GeminiGNIRSSpectrograph(spectrograph.Spectrograph): url = 'https://www.gemini.edu/instrumentation/gnirs' header_name = 'GNIRS' telescope = telescopes.GeminiNTelescopePar() + allowed_extensions = ['.fits', '.fits.bz2'] def __init__(self): super().__init__() @@ -612,11 +613,13 @@ def default_pypeit_par(cls): # Don't do 1D extraction for 3D data - it's meaningless because the DAR correction must be performed on the 3D data. par['reduce']['extraction']['skip_extraction'] = True # Because extraction occurs before the DAR correction, don't extract - #par['calibrations']['flatfield']['tweak_slits'] = False # Do not tweak the slit edges (we want to use the full slit) + # Tweak the slit edges using the gradient method for SlicerIFU + par['calibrations']['flatfield']['tweak_slits'] = True # Tweak the slit edges + par['calibrations']['flatfield']['tweak_method'] = 'gradient' # The gradient method is better for SlicerIFU. par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.0 # Make sure the full slit is used (i.e. when the illumination fraction is > 0.5) par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.0 # Make sure the full slit is used (i.e. no padding) par['calibrations']['flatfield']['slit_trim'] = 2 # Trim the slit edges - par['calibrations']['slitedges']['pad'] = 2 # Need to pad out the tilts for the astrometric transform when creating a datacube. + par['calibrations']['slitedges']['pad'] = 0 # Do not pad the slits - this ensures that the tweak_edges method=gradient guarantees that the edges are defined at the maximum gradient. # Decrease the wave tilts order, given the shorter slits of the IFU par['calibrations']['tilts']['spat_order'] = 1 @@ -624,7 +627,6 @@ def default_pypeit_par(cls): # Make sure that this is reduced as a slit (as opposed to fiber) spectrograph par['reduce']['cube']['slit_spec'] = True - par['reduce']['cube']['grating_corr'] = False par['reduce']['cube']['combine'] = False # Make separate spec3d files from the input spec2d files # Sky subtraction parameters @@ -717,7 +719,7 @@ def get_wcs(self, hdr, slits, platescale, wave0, dwv, spatial_scale=None): pxscl = spatial_scale / 3600.0 # 3600 is to convert arcsec to degrees # Get the typical slit length (this changes by ~0.3% over all slits, so a constant is fine for now) - slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) + slitlength = int(np.round(np.median(slits.get_slitlengths(median=True)))) # Get RA/DEC raval = self.get_meta_value([hdr], 'ra') diff --git a/pypeit/spectrographs/gtc_osiris.py b/pypeit/spectrographs/gtc_osiris.py index f622ad99de..78bc81fa21 100644 --- a/pypeit/spectrographs/gtc_osiris.py +++ b/pypeit/spectrographs/gtc_osiris.py @@ -471,6 +471,13 @@ def default_pypeit_par(cls): par['calibrations']['tilts']['spat_order'] = 1 par['calibrations']['tilts']['spec_order'] = 1 + # Tweak the slit edges using the gradient method for SlicerIFU + par['calibrations']['slitedges']['pad'] = 0 # Do not pad the slits - this ensures that the tweak_edges method=gradient guarantees that the edges are defined at the maximum gradient. + par['calibrations']['flatfield']['tweak_slits'] = True # Tweak the slit edges + par['calibrations']['flatfield']['tweak_method'] = 'gradient' # The gradient method is better for SlicerIFU. + par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.0 # Make sure the full slit is used (i.e. when the illumination fraction is > 0.5) + par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.0 # Make sure the full slit is used (i.e. no padding) + # Make sure that this is reduced as a slit (as opposed to fiber) spectrograph par['reduce']['cube']['slit_spec'] = True par['reduce']['cube']['combine'] = False # Make separate spec3d files from the input spec2d files @@ -527,7 +534,7 @@ def get_wcs(self, hdr, slits, platescale, wave0, dwv, spatial_scale=None): pxscl = spatial_scale / 3600.0 # 3600 is to convert arcsec to degrees # Get the typical slit length (this changes by ~0.3% over all slits, so a constant is fine for now) - slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) + slitlength = int(np.round(np.median(slits.get_slitlengths(median=True)))) # Get RA/DEC raval = self.get_meta_value([hdr], 'ra') diff --git a/pypeit/spectrographs/jwst_nirspec.py b/pypeit/spectrographs/jwst_nirspec.py index 1a5fb586e8..80a8abb2f3 100644 --- a/pypeit/spectrographs/jwst_nirspec.py +++ b/pypeit/spectrographs/jwst_nirspec.py @@ -254,7 +254,7 @@ def allowed_mosaics(self): ``PypeIt``. """ return [(1,2)] - + def get_rawimage(self, raw_file, det): """ Read raw images and generate a few other bits and pieces diff --git a/pypeit/spectrographs/keck_deimos.py b/pypeit/spectrographs/keck_deimos.py index b7208fbb5f..5549b388c2 100644 --- a/pypeit/spectrographs/keck_deimos.py +++ b/pypeit/spectrographs/keck_deimos.py @@ -766,7 +766,6 @@ def get_rawimage(self, raw_file, det): mosaic = None if nimg == 1 else self.get_mosaic_par(det, hdu=hdu) detectors = [self.get_detector_par(det, hdu=hdu)] if nimg == 1 else mosaic.detectors - # TODO check that that read noise and gain are the same for this amplifier mode?? if hdu[0].header['AMPMODE'] not in ['SINGLE:B', 'SINGLE:A']: msgs.error('PypeIt can only reduce images with AMPMODE == SINGLE:B or AMPMODE == SINGLE:A.') if hdu[0].header['MOSMODE'] != 'Spectral': @@ -825,7 +824,7 @@ def get_rawimage(self, raw_file, det): return detectors[0], image[0], hdu, exptime, rawdatasec_img[0], oscansec_img[0] return mosaic, image, hdu, exptime, rawdatasec_img, oscansec_img - def get_mosaic_par(self, mosaic, hdu=None, msc_order=5): + def get_mosaic_par(self, mosaic, hdu=None, msc_ord=5): """ Return the hard-coded parameters needed to construct detector mosaics from unbinned images. @@ -846,7 +845,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=5): default. BEWARE: If ``hdu`` is not provided, the binning is assumed to be `1,1`, which will cause faults if applied to binned images! - msc_order (:obj:`int`, optional): + msc_ord (:obj:`int`, optional): Order of the interpolation used to construct the mosaic. Returns: @@ -893,7 +892,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=5): msc_tfm[i] = build_image_mosaic_transform(shape, msc_sft[i], msc_rot[i], binning) return Mosaic(mosaic_id, detectors, shape, np.array(msc_sft), np.array(msc_rot), - np.array(msc_tfm), msc_order) + np.array(msc_tfm), msc_ord) @property def allowed_mosaics(self): @@ -1965,19 +1964,22 @@ def load_wmko_std_spectrum(fits_file:str, outfile=None, pad = False, split=True) # Generate SpecObj if not split: sobj1 = specobj.SpecObj.from_arrays('MultiSlit', idl_vac.value[0:npix], - idl_spec['COUNTS'].data[0:npix], - 1./(idl_spec['COUNTS'].data[0:npix]), - DET='MSC03') + idl_spec['COUNTS'].data[0:npix], + 1./(idl_spec['COUNTS'].data[0:npix]), + np.ones(idl_spec['COUNTS'].data[0:npix].size), + DET='MSC03') else: sobj1 = specobj.SpecObj.from_arrays('MultiSlit', idl_vac.value[0:npix], - idl_spec['COUNTS'].data[0:npix], - 1./(idl_spec['COUNTS'].data[0:npix]), - DET='DET03') + idl_spec['COUNTS'].data[0:npix], + 1./(idl_spec['COUNTS'].data[0:npix]), + np.ones(idl_spec['COUNTS'].data[0:npix].size), + DET='DET03') sobj2 = specobj.SpecObj.from_arrays('MultiSlit', idl_vac.value[npix:], - idl_spec['COUNTS'].data[npix:], - 1./(idl_spec['COUNTS'].data[npix:]), - DET='DET07') + idl_spec['COUNTS'].data[npix:], + 1./(idl_spec['COUNTS'].data[npix:]), + np.ones(idl_spec['COUNTS'].data[npix:].size), + DET='DET07') # SpecObjs sobjs = specobjs.SpecObjs() diff --git a/pypeit/spectrographs/keck_hires.py b/pypeit/spectrographs/keck_hires.py index ba4329f0cc..7ba80d00be 100644 --- a/pypeit/spectrographs/keck_hires.py +++ b/pypeit/spectrographs/keck_hires.py @@ -62,9 +62,11 @@ class KECKHIRESSpectrograph(spectrograph.Spectrograph): ech_fixed_format = False supported = False # TODO before support = True - # 1. Implement flat fielding - # 2. Test on several different setups - # 3. Implement PCA extrapolation into the blue + # 1. Implement flat fielding - DONE + # 2. Test on several different setups - DONE + # 3. Implement PCA extrapolation into the blue + + comment = 'Post detector upgrade (~ August 2004). See :doc:`keck_hires`' # TODO: Place holder parameter set taken from X-shooter VIS for now. @@ -99,6 +101,9 @@ def default_pypeit_par(cls): par['calibrations']['standardframe']['exprng'] = [1, 600] par['scienceframe']['exprng'] = [601, None] + # Set default processing for slitless_pixflat + par['calibrations']['slitless_pixflatframe']['process']['scale_to_mean'] = True + # Slit tracing par['calibrations']['slitedges']['edge_thresh'] = 8.0 par['calibrations']['slitedges']['fit_order'] = 8 @@ -154,7 +159,9 @@ def default_pypeit_par(cls): par['reduce']['extraction']['model_full_slit'] = True # Mask 3 edges pixels since the slit is short, insted of default (5,5) par['reduce']['findobj']['find_trim_edge'] = [3, 3] - # Continnum order for determining thresholds + # number of objects + par['reduce']['findobj']['maxnumber_sci'] = 2 # Assume that there is max two object in each order. + par['reduce']['findobj']['maxnumber_std'] = 1 # Assume that there is only one object in each order. # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' @@ -219,22 +226,21 @@ def init_meta(self): # Required (core) self.meta['ra'] = dict(ext=0, card='RA', required_ftypes=['science', 'standard']) self.meta['dec'] = dict(ext=0, card='DEC', required_ftypes=['science', 'standard']) - self.meta['target'] = dict(ext=0, card='OBJECT') + self.meta['target'] = dict(ext=0, card='TARGNAME') self.meta['decker'] = dict(ext=0, card='DECKNAME') self.meta['binning'] = dict(card=None, compound=True) self.meta['mjd'] = dict(card=None, compound=True) # This may depend on the old/new detector self.meta['exptime'] = dict(ext=0, card='ELAPTIME') self.meta['airmass'] = dict(ext=0, card='AIRMASS') - #self.meta['dispname'] = dict(ext=0, card='ECHNAME') + # Extras for config and frametyping self.meta['hatch'] = dict(ext=0, card='HATOPEN') self.meta['dispname'] = dict(ext=0, card='XDISPERS') self.meta['filter1'] = dict(ext=0, card='FIL1NAME') self.meta['echangle'] = dict(ext=0, card='ECHANGL', rtol=1e-3, atol=1e-2) self.meta['xdangle'] = dict(ext=0, card='XDANGL', rtol=1e-2) -# self.meta['idname'] = dict(ext=0, card='IMAGETYP') - # NOTE: This is the native keyword. IMAGETYP is from KOA. + self.meta['object'] = dict(ext=0, card='OBJECT') self.meta['idname'] = dict(card=None, compound=True) self.meta['frameno'] = dict(ext=0, card='FRAMENO') self.meta['instrument'] = dict(ext=0, card='INSTRUME') @@ -255,7 +261,6 @@ def compound_meta(self, headarr, meta_key): object: Metadata value read from the header(s). """ if meta_key == 'binning': - # TODO JFH Is this correct or should it be flipped? binspatial, binspec = parse.parse_binning(headarr[0]['BINNING']) binning = parse.binning2string(binspec, binspatial) return binning @@ -275,22 +280,28 @@ def compound_meta(self, headarr, meta_key): return 'off' elif meta_key == 'idname': - if not headarr[0].get('LAMPCAT1') and not headarr[0].get('LAMPCAT2') and \ + xcovopen = headarr[0].get('XCOVOPEN') + collcoveropen = (headarr[0].get('XDISPERS') == 'RED' and headarr[0].get('RCCVOPEN')) or \ + (headarr[0].get('XDISPERS') == 'UV' and headarr[0].get('BCCVOPEN')) + + if xcovopen and collcoveropen and \ + not headarr[0].get('LAMPCAT1') and not headarr[0].get('LAMPCAT2') and \ not headarr[0].get('LAMPQTZ2') and not (headarr[0].get('LAMPNAME') == 'quartz1'): if headarr[0].get('HATOPEN') and headarr[0].get('AUTOSHUT'): return 'Object' elif not headarr[0].get('HATOPEN'): return 'Bias' if not headarr[0].get('AUTOSHUT') else 'Dark' - elif headarr[0].get('AUTOSHUT') and (headarr[0].get('LAMPCAT1') or headarr[0].get('LAMPCAT2')): - if (headarr[0].get('XDISPERS') == 'RED' and not headarr[0].get('RCCVOPEN')) or \ - (headarr[0].get('XDISPERS') == 'UV' and not headarr[0].get('BCCVOPEN')): + elif xcovopen and collcoveropen and \ + headarr[0].get('AUTOSHUT') and (headarr[0].get('LAMPCAT1') or headarr[0].get('LAMPCAT2')): + return 'Line' + elif collcoveropen and \ + headarr[0].get('AUTOSHUT') and \ + (headarr[0].get('LAMPQTZ2') or (headarr[0].get('LAMPNAME') == 'quartz1')) and \ + not headarr[0].get('HATOPEN'): + if not xcovopen: return 'slitlessFlat' else: - return 'Line' - elif headarr[0].get('AUTOSHUT') and \ - (headarr[0].get('LAMPQTZ2') or (headarr[0].get('LAMPNAME') == 'quartz1')) \ - and not headarr[0].get('HATOPEN'): - return 'IntFlat' + return 'IntFlat' else: msgs.error("Not ready for this compound meta") @@ -309,7 +320,26 @@ def configuration_keys(self): and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` object. """ - return ['decker', 'dispname', 'filter1', 'echangle', 'xdangle', 'binning'] + return ['dispname', 'decker', 'filter1', 'echangle', 'xdangle', 'binning'] + + def config_independent_frames(self): + """ + Define frame types that are independent of the fully defined + instrument configuration. + + Bias and dark frames are considered independent of a configuration, + but the DATE-OBS keyword is used to assign each to the most-relevant + configuration frame group. See + :func:`~pypeit.metadata.PypeItMetaData.set_configurations`. + + Returns: + :obj:`dict`: Dictionary where the keys are the frame types that + are configuration independent and the values are the metadata + keywords that can be used to assign the frames to a configuration + group. + """ + return {'bias': ['dispname', 'binning'], 'dark': ['dispname', 'binning'], + 'slitless_pixflat': ['dispname', 'binning']} def raw_header_cards(self): """ @@ -342,9 +372,6 @@ def pypeit_file_keys(self): """ return super().pypeit_file_keys() + ['hatch', 'lampstat01', 'frameno'] - - - def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. @@ -367,14 +394,14 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) # TODO: Allow for 'sky' frame type, for now include sky in # 'science' category - if ftype == 'science': - return good_exp & (fitstbl['idname'] == 'Object') - if ftype == 'standard': + if ftype in ['science', 'standard']: return good_exp & (fitstbl['idname'] == 'Object') if ftype == 'bias': return good_exp & (fitstbl['idname'] == 'Bias') if ftype == 'dark': return good_exp & (fitstbl['idname'] == 'Dark') + if ftype == 'slitless_pixflat': + return good_exp & (fitstbl['idname'] == 'slitlessFlat') if ftype in ['illumflat', 'pixelflat', 'trace']: # Flats and trace frames are typed together return good_exp & (fitstbl['idname'] == 'IntFlat') @@ -385,6 +412,117 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool) + def vet_assigned_ftypes(self, type_bits, fitstbl): + """ + + NOTE: this function should only be called when running pypeit_setup, + in order to not overwrite any user-provided frame types. + + This method checks the assigned frame types for consistency. + For frames that are assigned both the science and standard types, + this method chooses the one that is most likely, by checking if the + frames are within 10 arcmin of a listed standard star. + + In addition, for this instrument, if a frame is assigned both a + pixelflat and slitless_pixflat type, the pixelflat type is removed. + NOTE: if the same frame is assigned to multiple configurations, this + method will remove the pixelflat type for all configurations, i.e., + it is not possible to use slitless_pixflat type for one calibration group + and pixelflat for another. + + Args: + type_bits (`numpy.ndarray`_): + Array with the frame types assigned to each frame. + fitstbl (:class:`~pypeit.metadata.PypeItMetaData`): + The class holding the metadata for all the frames. + + Returns: + `numpy.ndarray`_: The updated frame types. + + """ + type_bits = super().vet_assigned_ftypes(type_bits, fitstbl) + + # If both pixelflat and slitless_pixflat are assigned to the same frame, remove pixelflat + + # where slitless_pixflat is assigned + slitless_idx = fitstbl.type_bitmask.flagged(type_bits, flag='slitless_pixflat') + # where pixelflat is assigned + pixelflat_idx = fitstbl.type_bitmask.flagged(type_bits, flag='pixelflat') + + # find configurations where both pixelflat and slitless_pixflat are assigned + pixflat_match = np.zeros(len(fitstbl), dtype=bool) + + for f, frame in enumerate(fitstbl): + if pixelflat_idx[f]: + match_config_values = [] + for slitless in fitstbl[slitless_idx]: + match_config_values.append(np.all([frame[c] == slitless[c] + for c in self.config_independent_frames()['slitless_pixflat']])) + pixflat_match[f] = np.any(match_config_values) + + # remove pixelflat from the type_bits + type_bits[pixflat_match] = fitstbl.type_bitmask.turn_off(type_bits[pixflat_match], 'pixelflat') + + return type_bits + + def parse_raw_files(self, fitstbl, det=1, ftype=None): + """ + Parse the list of raw files with given frame type and detector. + This is spectrograph-specific, and it is not defined for all + spectrographs. + Since different slitless_pixflat frames are usually taken for + each of the three detectors, this method parses the slitless_pixflat + frames and returns the correct one for the requested detector. + + Args: + fitstbl (`astropy.table.Table`_): + Table with metadata of the raw files to parse. + det (:obj:`int`, optional): + 1-indexed detector number to parse. + ftype (:obj:`str`, optional): + Frame type to parse. If None, no frames are parsed + and the indices of all frames are returned. + + Returns: + `numpy.ndarray`_: The indices of the raw files in the fitstbl that are parsed. + + """ + + if ftype == 'slitless_pixflat': + # Check for the required info + if len(fitstbl) == 0: + msgs.warn('Fitstbl provided is emtpy. No parsing done.') + # return empty array + return np.array([], dtype=int) + elif det is None: + msgs.warn('Detector number must be provided to parse slitless_pixflat frames. No parsing done.') + # return index array of length of fitstbl + return np.arange(len(fitstbl)) + + # how many unique xdangle values are there? + # If they are 3, then we have a different slitless flat file per detector + xdangles = np.unique(np.int32(fitstbl['xdangle'].value)) + if len(xdangles) == 3: + sort_xdagles = np.argsort(xdangles) + # xdagles: -5 for red(det=3), -4 for green (det=2), -3 for blue (det=1) dets + # select the corresponding files for the requested detector + if det == 1: + # blue detector + return np.where(np.int32(fitstbl['xdangle'].value) == -3)[0] + elif det == 2: + # green detector + return np.where(np.int32(fitstbl['xdangle'].value) == -4)[0] + elif det == 3: + # red detector + return np.where(np.int32(fitstbl['xdangle'].value) == -5)[0] + else: + msgs.warn('The provided list of slitless_pixflat frames does not have exactly 3 unique XDANGLE values. ' + 'Pypeit cannot determine which slitless_pixflat frame corresponds to the requested detector. ' + 'All frames will be used.') + return np.arange(len(fitstbl)) + + else: + return super().parse_raw_files(fitstbl, det=det, ftype=ftype) def get_rawimage(self, raw_file, det, spectrim=20): """ @@ -500,7 +638,7 @@ def get_rawimage(self, raw_file, det, spectrim=20): return mosaic, image, hdu, exptime, rawdatasec_img, oscansec_img - def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): + def get_mosaic_par(self, mosaic, hdu=None, msc_ord=0): """ Return the hard-coded parameters needed to construct detector mosaics from unbinned images. @@ -521,7 +659,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): default. BEWARE: If ``hdu`` is not provided, the binning is assumed to be `1,1`, which will cause faults if applied to binned images! - msc_order (:obj:`int`, optional): + msc_ord (:obj:`int`, optional): Order of the interpolation used to construct the mosaic. Returns: @@ -572,7 +710,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): msc_tfm[i] = build_image_mosaic_transform(shape, msc_sft[i], msc_rot[i], tuple(reversed(binning))) return Mosaic(mosaic_id, detectors, shape, np.array(msc_sft), np.array(msc_rot), - np.array(msc_tfm), msc_order) + np.array(msc_tfm), msc_ord) @property diff --git a/pypeit/spectrographs/keck_kcwi.py b/pypeit/spectrographs/keck_kcwi.py index 5743eaa6eb..218937bd66 100644 --- a/pypeit/spectrographs/keck_kcwi.py +++ b/pypeit/spectrographs/keck_kcwi.py @@ -134,6 +134,8 @@ def config_specific_par(self, scifile, inp_par=None): par['calibrations']['wavelengths']['lamps'] = ['FeI', 'ArI', 'ArII'] if self.get_meta_value(headarr, 'dispname') == 'BH2': par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_kcwi_BH2.fits' + elif self.get_meta_value(headarr, 'dispname') == 'BH3': + par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_kcwi_BH3.fits' elif self.get_meta_value(headarr, 'dispname') == 'BM': par['calibrations']['wavelengths']['reid_arxiv'] = 'keck_kcwi_BM.fits' elif self.get_meta_value(headarr, 'dispname') == 'BL': @@ -283,6 +285,34 @@ def default_pypeit_par(cls): # Set the number of alignments in the align frames par['calibrations']['alignment']['locations'] = [0.1, 0.3, 0.5, 0.7, 0.9] # TODO:: Check this - is this accurate enough? + # Correct the illumflat for pixel-to-pixel sensitivity variations + par['calibrations']['illumflatframe']['process']['use_pixelflat'] = True + + # Make sure the overscan is subtracted from the dark + par['calibrations']['darkframe']['process']['use_overscan'] = True + + # Set the slit edge parameters + par['calibrations']['slitedges']['fit_order'] = 4 + par['calibrations']['slitedges']['pad'] = 0 # Do not pad the slits - this ensures that the tweak_edges method=gradient guarantees that the edges are defined at the maximum gradient. + par['calibrations']['slitedges']['edge_thresh'] = 5 # 5 works well with a range of setups tested by RJC (mostly 1x1 binning) + + # KCWI has non-uniform spectral resolution across the field-of-view + par['calibrations']['wavelengths']['fwhm_spec_order'] = 1 + par['calibrations']['wavelengths']['fwhm_spat_order'] = 2 + + # Alter the method used to combine pixel flats + par['calibrations']['pixelflatframe']['process']['combine'] = 'median' + par['calibrations']['flatfield']['spec_samp_coarse'] = 20.0 + par['calibrations']['flatfield']['tweak_slits'] = True # Tweak the slit edges + par['calibrations']['flatfield']['tweak_method'] = 'gradient' # The gradient method is better for SlicerIFU. + par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.0 # Make sure the full slit is used (i.e. when the illumination fraction is > 0.5) + par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.0 # Make sure the full slit is used (i.e. no padding) + par['calibrations']['flatfield']['slit_trim'] = 3 # Trim the slit edges + # Relative illumination correction + par['calibrations']['flatfield']['slit_illum_relative'] = True # Calculate the relative slit illumination + par['calibrations']['flatfield']['slit_illum_ref_idx'] = 14 # The reference index - this should probably be the same for the science frame + par['calibrations']['flatfield']['slit_illum_smooth_npix'] = 10 # Sufficiently small value so less structure in relative weights + # LACosmics parameters par['scienceframe']['process']['sigclip'] = 4.0 par['scienceframe']['process']['objlim'] = 1.5 @@ -607,7 +637,7 @@ def get_wcs(self, hdr, slits, platescale, wave0, dwv, spatial_scale=None): pxscl = spatial_scale / 3600.0 # 3600 is to convert arcsec to degrees # Get the typical slit length (this changes by ~0.3% over all slits, so a constant is fine for now) - slitlength = int(np.round(np.median(slits.get_slitlengths(initial=True, median=True)))) + slitlength = int(np.round(np.median(slits.get_slitlengths(median=True)))) # Get RA/DEC ra = self.compound_meta([hdr], 'ra') @@ -797,7 +827,7 @@ class KeckKCWISpectrograph(KeckKCWIKCRMSpectrograph): camera = 'KCWI' url = 'https://www2.keck.hawaii.edu/inst/kcwi/' header_name = 'KCWI' - comment = 'Supported setups: BL, BM, BH2; see :doc:`keck_kcwi`' + comment = 'Supported setups: BL, BM, BH2, BH3; see :doc:`keck_kcwi`' def get_detector_par(self, det, hdu=None): """ @@ -832,7 +862,7 @@ def get_detector_par(self, det, hdu=None): # Some properties of the image binning = self.compound_meta(self.get_headarr(hdu), "binning") numamps = hdu[0].header['NVIDINP'] - specflip = True if hdu[0].header['AMPID1'] == 2 else False + specflip = False if hdu[0].header['AMPMODE'] == 'ALL' else True gainmul, gainarr = hdu[0].header['GAINMUL'], np.zeros(numamps) ronarr = np.zeros(numamps) # Set this to zero (determine the readout noise from the overscan regions) # dsecarr = np.array(['']*numamps) @@ -920,39 +950,26 @@ def default_pypeit_par(cls): par['calibrations']['scattlight_pad'] = 6 # This is the unbinned number of pixels to pad par['calibrations']['pixelflatframe']['process']['subtract_scattlight'] = True par['calibrations']['illumflatframe']['process']['subtract_scattlight'] = True - par['scienceframe']['process']['subtract_scattlight'] = True + par['scienceframe']['process']['subtract_scattlight'] = False par['scienceframe']['process']['scattlight']['finecorr_method'] = 'median' par['scienceframe']['process']['scattlight']['finecorr_pad'] = 4 # This is the unbinned number of pixels to pad par['scienceframe']['process']['scattlight']['finecorr_order'] = 2 # par['scienceframe']['process']['scattlight']['finecorr_mask'] = 12 # Mask the middle inter-slit region. It contains a strange scattered light feature that doesn't appear to affect any other inter-slit regions - # Correct the illumflat for pixel-to-pixel sensitivity variations - par['calibrations']['illumflatframe']['process']['use_pixelflat'] = True - - # Make sure the overscan is subtracted from the dark - par['calibrations']['darkframe']['process']['use_overscan'] = True - - # Set the slit edge parameters - par['calibrations']['slitedges']['fit_order'] = 4 - par['calibrations']['slitedges']['pad'] = 2 # Need to pad out the tilts for the astrometric transform when creating a datacube. - par['calibrations']['slitedges']['edge_thresh'] = 5 # 5 works well with a range of setups tested by RJC (mostly 1x1 binning) - - # KCWI has non-uniform spectral resolution across the field-of-view - par['calibrations']['wavelengths']['fwhm_spec_order'] = 1 - par['calibrations']['wavelengths']['fwhm_spat_order'] = 2 + # Correct for non-linear behaviour in the detector response + nonlin_array = [-1.4E-7, -1.4E-7, -1.2E-7, -1.8E-7] # AMPID=0,1,2,3 respectively + par['calibrations']['arcframe']['process']['correct_nonlinear'] = nonlin_array + par['calibrations']['tiltframe']['process']['correct_nonlinear'] = nonlin_array + par['calibrations']['pixelflatframe']['process']['correct_nonlinear'] = nonlin_array + par['calibrations']['illumflatframe']['process']['correct_nonlinear'] = nonlin_array + par['calibrations']['standardframe']['process']['correct_nonlinear'] = nonlin_array + par['scienceframe']['process']['correct_nonlinear'] = nonlin_array # Alter the method used to combine pixel flats - par['calibrations']['pixelflatframe']['process']['combine'] = 'median' - par['calibrations']['flatfield']['spec_samp_coarse'] = 20.0 + par['calibrations']['pixelflatframe']['process']['combine'] = 'mean' par['calibrations']['flatfield']['spat_samp'] = 1.0 # This should give 1% accuracy in the spatial illumination correction for 2x2 binning, and <0.5% accuracy for 1x1 binning - #par['calibrations']['flatfield']['tweak_slits'] = False # Do not tweak the slit edges (we want to use the full slit) - par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.0 # Make sure the full slit is used (i.e. when the illumination fraction is > 0.5) - par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.0 # Make sure the full slit is used (i.e. no padding) - par['calibrations']['flatfield']['slit_trim'] = 3 # Trim the slit edges - # Relative illumination correction - par['calibrations']['flatfield']['slit_illum_relative'] = True # Calculate the relative slit illumination - par['calibrations']['flatfield']['slit_illum_ref_idx'] = 14 # The reference index - this should probably be the same for the science frame - par['calibrations']['flatfield']['slit_illum_smooth_npix'] = 5 # Sufficiently small value so less structure in relative weights + + # Need to fit sinusoidal sensitivity pattern, and include in the relative pixel response par['calibrations']['flatfield']['fit_2d_det_response'] = True # Include the 2D detector response in the pixelflat. return par @@ -1342,33 +1359,8 @@ def default_pypeit_par(cls): par['calibrations']['standardframe']['process']['use_pattern'] = False par['scienceframe']['process']['use_pattern'] = False - # Correct the illumflat for pixel-to-pixel sensitivity variations - par['calibrations']['illumflatframe']['process']['use_pixelflat'] = True - - # Make sure the overscan is subtracted from the dark - par['calibrations']['darkframe']['process']['use_overscan'] = True - - # Set the slit edge parameters - par['calibrations']['slitedges']['fit_order'] = 4 - par['calibrations']['slitedges']['pad'] = 2 # Need to pad out the tilts for the astrometric transform when creating a datacube. - par['calibrations']['slitedges']['edge_thresh'] = 5 # 5 works well with a range of setups tested by RJC (mostly 1x1 binning) - - # KCWI has non-uniform spectral resolution across the field-of-view - par['calibrations']['wavelengths']['fwhm_spec_order'] = 1 - par['calibrations']['wavelengths']['fwhm_spat_order'] = 2 - - # Alter the method used to combine pixel flats - par['calibrations']['pixelflatframe']['process']['combine'] = 'median' - par['calibrations']['flatfield']['spec_samp_coarse'] = 20.0 - #par['calibrations']['flatfield']['tweak_slits'] = False # Do not tweak the slit edges (we want to use the full slit) - par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.0 # Make sure the full slit is used (i.e. when the illumination fraction is > 0.5) - par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.0 # Make sure the full slit is used (i.e. no padding) - par['calibrations']['flatfield']['slit_trim'] = 3 # Trim the slit edges - # Relative illumination correction - par['calibrations']['flatfield']['slit_illum_relative'] = True # Calculate the relative slit illumination - par['calibrations']['flatfield']['slit_illum_ref_idx'] = 14 # The reference index - this should probably be the same for the science frame - par['calibrations']['flatfield']['slit_illum_smooth_npix'] = 5 # Sufficiently small value so less structure in relative weights - par['calibrations']['flatfield']['fit_2d_det_response'] = True # Include the 2D detector response in the pixelflat. + # This is probably not necessary for KCRM + par['calibrations']['flatfield']['fit_2d_det_response'] = False # Include the 2D detector response in the pixelflat. # Sky subtraction parameters par['reduce']['skysub']['bspline_spacing'] = 0.4 diff --git a/pypeit/spectrographs/keck_lris.py b/pypeit/spectrographs/keck_lris.py index ffc42afb87..8c807d052a 100644 --- a/pypeit/spectrographs/keck_lris.py +++ b/pypeit/spectrographs/keck_lris.py @@ -22,6 +22,7 @@ from pypeit import io from pypeit.core import parse from pypeit.core import framematch +from pypeit.core import flux_calib from pypeit.spectrographs import spectrograph from pypeit.spectrographs import slitmask from pypeit.images import detector_container @@ -80,14 +81,21 @@ def default_pypeit_par(cls): par['calibrations']['wavelengths']['n_first'] = 3 par['calibrations']['wavelengths']['n_final'] = 5 # Set the default exposure time ranges for the frame typing - par['calibrations']['biasframe']['exprng'] = [None, 0.001] + par['calibrations']['biasframe']['exprng'] = [None, 1] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames - par['calibrations']['pixelflatframe']['exprng'] = [None, 60] - par['calibrations']['traceframe']['exprng'] = [None, 60] - par['calibrations']['illumflatframe']['exprng'] = [None, 60] + par['calibrations']['pixelflatframe']['exprng'] = [0, 60] + par['calibrations']['traceframe']['exprng'] = [0, 60] + par['calibrations']['illumflatframe']['exprng'] = [0, 60] + par['calibrations']['slitless_pixflatframe']['exprng'] = [0, 60] par['calibrations']['standardframe']['exprng'] = [1, 61] + # Set default processing for slitless_pixflat + par['calibrations']['slitless_pixflatframe']['process']['scale_to_mean'] = True + + # Turn off flat illumination fine correction (it's more often bad than good) + par['calibrations']['flatfield']['slit_illum_finecorr'] = False + # Flexure # Always correct for spectral flexure, starting with default parameters par['flexure']['spec_method'] = 'boxcar' @@ -153,8 +161,8 @@ def init_meta(self): """ self.meta = {} # Required (core) - self.meta['ra'] = dict(ext=0, card='RA') - self.meta['dec'] = dict(ext=0, card='DEC') + self.meta['ra'] = dict(card=None, compound=True) + self.meta['dec'] = dict(card=None, compound=True) self.meta['target'] = dict(ext=0, card='TARGNAME') self.meta['decker'] = dict(ext=0, card='SLITNAME') self.meta['binning'] = dict(card=None, compound=True) @@ -174,6 +182,7 @@ def init_meta(self): # Extras for pypeit file self.meta['dateobs'] = dict(card=None, compound=True) self.meta['amp'] = dict(ext=0, card='NUMAMPS') + self.meta['object'] = dict(ext=0, card='OBJECT') # Lamps # similar approach to DEIMOS @@ -193,7 +202,20 @@ def compound_meta(self, headarr, meta_key): Returns: object: Metadata value read from the header(s). """ - if meta_key == 'binning': + # LRIS sometime misses RA and/or Dec in the header. When this happens, set them to 0 + if meta_key == 'ra': + if headarr[0].get('RA') is None: + msgs.warn('Keyword RA not found in header. Setting to 0') + return '00:00:00.00' + else: + return headarr[0]['RA'] + elif meta_key == 'dec': + if headarr[0].get('DEC') is None: + msgs.warn('Keyword DEC not found in header. Setting to 0') + return '+00:00:00.0' + else: + return headarr[0]['DEC'] + elif meta_key == 'binning': binspatial, binspec = parse.parse_binning(headarr[0]['BINNING']) binning = parse.binning2string(binspec, binspatial) return binning @@ -292,7 +314,8 @@ def config_independent_frames(self): keywords that can be used to assign the frames to a configuration group. """ - return {'bias': ['amp', 'binning', 'dateobs'], 'dark': ['amp', 'binning', 'dateobs']} + return {'bias': ['amp', 'binning', 'dateobs'], 'dark': ['amp', 'binning', 'dateobs'], + 'slitless_pixflat': ['amp', 'binning', 'dateobs', 'dispname', 'dichroic']} def pypeit_file_keys(self): """ @@ -324,20 +347,49 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): `numpy.ndarray`_: Boolean array with the flags selecting the exposures in ``fitstbl`` that are ``ftype`` type frames. """ - good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) + # good exposures + good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) & (fitstbl['decker'] != 'GOH_LRIS') + # no images no_img = np.array([d not in ['Mirror', 'mirror', 'clear'] for d in fitstbl['dispname']]) + + # Check frame type if ftype == 'science': return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'open') & no_img if ftype == 'standard': - return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'open') & no_img + std = np.zeros(len(fitstbl), dtype=bool) + if 'ra' in fitstbl.keys() and 'dec' in fitstbl.keys(): + std = np.array([flux_calib.find_standard_file(ra, dec, toler=10.*units.arcmin, check=True) + for ra, dec in zip(fitstbl['ra'], fitstbl['dec'])]) + return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'open') & no_img & std if ftype == 'bias': return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'closed') + if ftype == 'slitless_pixflat': + # these are sky flats, like science but without the slitmask + return (good_exp & self.lamps(fitstbl, 'off') & + (fitstbl['hatch'] == 'open') & no_img & (fitstbl['decker'] == 'direct')) if ftype in ['pixelflat', 'trace', 'illumflat']: # Allow for dome or internal good_dome = self.lamps(fitstbl, 'dome') & (fitstbl['hatch'] == 'open') good_internal = self.lamps(fitstbl, 'internal') & (fitstbl['hatch'] == 'closed') + # attempt at identifying sky flats (not robust, but better than nothing) + # they are basically science frames, so we look for "sky" words in the header + is_sky = self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'open') + # look for specific words in the target or object header keywords + words_to_search = ['sky', 'blank', 'twilight', 'twiflat', 'twi flat'] + for i, row in enumerate(fitstbl): + in_target = False + if row['target'] is not None: + if np.any([w in row['target'].lower() for w in words_to_search]): + in_target = True + in_object = False + if row['object'] is not None: + if np.any([w in row['object'].lower() for w in words_to_search]): + in_object = True + is_sky[i] = in_target or in_object + # put together the sky flats requirement + sky_flat = is_sky & (fitstbl['decker'] != 'direct') # Flats and trace frames are typed together - return good_exp & (good_dome + good_internal) & no_img + return good_exp & (good_dome + good_internal + sky_flat) & no_img if ftype in ['pinhole', 'dark']: # Don't type pinhole or dark frames return np.zeros(len(fitstbl), dtype=bool) @@ -346,6 +398,59 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool) + + def vet_assigned_ftypes(self, type_bits, fitstbl): + """ + + NOTE: this function should only be called when running pypeit_setup, + in order to not overwrite any user-provided frame types. + + This method checks the assigned frame types for consistency. + For frames that are assigned both the science and standard types, + this method chooses the one that is most likely, by checking if the + frames are within 10 arcmin of a listed standard star. + + In addition, for this instrument, if a frame is assigned both a + pixelflat and slitless_pixflat type, the pixelflat type is removed. + NOTE: if the same frame is assigned to multiple configurations, this + method will remove the pixelflat type for all configurations, i.e., + it is not possible to use slitless_pixflat type for one calibration group + and pixelflat for another. + + Args: + type_bits (`numpy.ndarray`_): + Array with the frame types assigned to each frame. + fitstbl (:class:`~pypeit.metadata.PypeItMetaData`): + The class holding the metadata for all the frames. + + Returns: + `numpy.ndarray`_: The updated frame types. + + """ + type_bits = super().vet_assigned_ftypes(type_bits, fitstbl) + + # If both pixelflat and slitless_pixflat are assigned to the same frame, remove pixelflat + + # where slitless_pixflat is assigned + slitless_idx = fitstbl.type_bitmask.flagged(type_bits, flag='slitless_pixflat') + # where pixelflat is assigned + pixelflat_idx = fitstbl.type_bitmask.flagged(type_bits, flag='pixelflat') + + # find configurations where both pixelflat and slitless_pixflat are assigned + pixflat_match = np.zeros(len(fitstbl), dtype=bool) + + for f, frame in enumerate(fitstbl): + if pixelflat_idx[f]: + match_config_values = [] + for slitless in fitstbl[slitless_idx]: + match_config_values.append(np.all([frame[c] == slitless[c] + for c in self.config_independent_frames()['slitless_pixflat']])) + pixflat_match[f] = np.any(match_config_values) + + # remove pixelflat from the type_bits + type_bits[pixflat_match] = fitstbl.type_bitmask.turn_off(type_bits[pixflat_match], 'pixelflat') + + return type_bits def lamps(self, fitstbl, status): """ @@ -853,6 +958,8 @@ def default_pypeit_par(cls): par['calibrations']['traceframe']['exprng'] = [None, 300] par['calibrations']['illumflatframe']['exprng'] = [None, 300] + par['calibrations']['standardframe']['exprng'] = [1, 901] + return par def config_specific_par(self, scifile, inp_par=None): @@ -1927,38 +2034,48 @@ def lris_read_amp(inp, ext): return data, predata, postdata, x1, y1 -def convert_lowredux_pixelflat(infil, outfil): +def convert_lowredux_pixelflat(infil, outfil, specflip=False, separate_extensions=False): """ Convert LowRedux pixelflat to PYPIT format Returns ------- """ # Read - hdu = io.fits_open(infil) - data = hdu[0].data + hdu0 = io.fits_open(infil) + data = hdu0[0].data # prihdu = fits.PrimaryHDU() hdus = [prihdu] - prihdu.header['FRAMETYP'] = 'pixelflat' + prihdu.header['CALIBTYP'] = ('Flat', 'PypeIt: Calibration frame type') # Detector 1 - img1 = data[:,:data.shape[1]//2] + if separate_extensions: + img1 = hdu0['DET1'].data + else: + img1 = data[:, :data.shape[1] // 2] + if specflip: + img1 = np.flip(img1, axis=0) hdu = fits.ImageHDU(img1) - hdu.name = 'DET1' - prihdu.header['EXT0001'] = 'DET1-pixelflat' + hdu.name = 'DET01-PIXELFLAT_NORM' + prihdu.header['EXT0001'] = hdu.name hdus.append(hdu) # Detector 2 - img2 = data[:,data.shape[1]//2:] + if separate_extensions: + img2 = hdu0['DET2'].data + else: + img2 = data[:, data.shape[1] // 2:] + if specflip: + img2 = np.flip(img2, axis=0) hdu = fits.ImageHDU(img2) - hdu.name = 'DET2' - prihdu.header['EXT0002'] = 'DET2-pixelflat' + hdu.name = 'DET02-PIXELFLAT_NORM' + prihdu.header['EXT0002'] = hdu.name hdus.append(hdu) # Finish hdulist = fits.HDUList(hdus) - hdulist.writeto(outfil, clobber=True) + hdulist.writeto(outfil, overwrite=True) print('Wrote {:s}'.format(outfil)) diff --git a/pypeit/spectrographs/keck_mosfire.py b/pypeit/spectrographs/keck_mosfire.py index e874baea83..44ffc91f1d 100644 --- a/pypeit/spectrographs/keck_mosfire.py +++ b/pypeit/spectrographs/keck_mosfire.py @@ -335,7 +335,7 @@ def compound_meta(self, headarr, meta_key): FLATSPEC = headarr[0].get('FLATSPEC') PWSTATA7 = headarr[0].get('PWSTATA7') PWSTATA8 = headarr[0].get('PWSTATA8') - if FLATSPEC == 0 and PWSTATA7 == 0 and PWSTATA8 == 0: + if FLATSPEC == 0 and PWSTATA7 == 0 and PWSTATA8 == 0 and headarr[0].get('FILTER') != 'Dark': if 'Flat' in headarr[0].get('OBJECT'): return 'flatlampoff' else: diff --git a/pypeit/spectrographs/keck_nires.py b/pypeit/spectrographs/keck_nires.py index 761fcb7253..b410a10d25 100644 --- a/pypeit/spectrographs/keck_nires.py +++ b/pypeit/spectrographs/keck_nires.py @@ -121,7 +121,7 @@ def default_pypeit_par(cls): #par['reduce']['findobj']['ech_find_nabove_min_snr'] = 1 # Require detection in a single order since given only 5 orders and slitlosses for NIRES, often # things are only detected in the K-band? Decided not to make this the default. - + par['reduce']['findobj']['maxnumber_std'] = 1 # Assume that there is only one object in each order. # Flexure par['flexure']['spec_method'] = 'skip' diff --git a/pypeit/spectrographs/ldt_deveny.py b/pypeit/spectrographs/ldt_deveny.py index 6405862874..dae7d1d0ed 100644 --- a/pypeit/spectrographs/ldt_deveny.py +++ b/pypeit/spectrographs/ldt_deveny.py @@ -566,79 +566,17 @@ def config_specific_par(self, scifile, inp_par=None): def get_rawimage(self, raw_file, det): """ - Read raw images and generate a few other bits and pieces - that are key for image processing. + Read raw spectrograph image files and return data and relevant metadata + needed for image processing. For LDT/DeVeny, the LOIS control system automatically adjusts the - ``DATASEC`` and ``OSCANSEC`` regions if the CCD is used in a binning other - than 1x1. The :meth:`~pypeit.spectrographs.spectrograph.Spectrograph.get_rawimage` - method in the base class assumes these sections are fixed and adjusts - them based on the binning -- an incorrect assumption for this instrument. - - This method is a stripped-down version of the base class method and - additionally does *NOT* send the binning to :func:`~pypeit.core.parse.sec2slice`. - - Parameters - ---------- - raw_file : :obj:`str` - File to read - det : :obj:`int` - 1-indexed detector to read - - Returns - ------- - detector_par : :class:`~pypeit.images.detector_container.DetectorContainer` - Detector metadata parameters. - raw_img : `numpy.ndarray`_ - Raw image for this detector. - hdu : `astropy.io.fits.HDUList`_ - Opened fits file - exptime : :obj:`float` - Exposure time *in seconds*. - rawdatasec_img : `numpy.ndarray`_ - Data (Science) section of the detector as provided by setting the - (1-indexed) number of the amplifier used to read each detector - pixel. Pixels unassociated with any amplifier are set to 0. - oscansec_img : `numpy.ndarray`_ - Overscan section of the detector as provided by setting the - (1-indexed) number of the amplifier used to read each detector - pixel. Pixels unassociated with any amplifier are set to 0. + ``DATASEC`` and ``OSCANSEC`` regions if the CCD is used in a binning + other than 1x1. This is a simple wrapper for + :func:`pypeit.spectrographs.spectrograph.Spectrograph.get_rawimage` that + sets ``sec_includes_binning`` to True. See the base-class function for + the detailed descriptions of the input parameters and returned objects. """ - # Open - hdu = io.fits_open(raw_file) - - # Grab the DetectorContainer and extract the raw image - detector = self.get_detector_par(det, hdu=hdu) - raw_img = hdu[detector['dataext']].data.astype(float) - - # Exposure time (used by RawImage) from the header - headarr = self.get_headarr(hdu) - exptime = self.get_meta_value(headarr, 'exptime') - - for section in ['datasec', 'oscansec']: - # Get the data section from Detector - image_sections = detector[section] - - # Initialize the image (0 means no amplifier) - pix_img = np.zeros(raw_img.shape, dtype=int) - for i in range(detector['numamplifiers']): - - if image_sections is not None: - # Convert the (FITS) data section from a string to a slice - # DO NOT send the binning (default: None) - datasec = parse.sec2slice(image_sections[i], one_indexed=True, - include_end=True, require_dim=2) - # Assign the amplifier - pix_img[datasec] = i+1 - - # Finish - if section == 'datasec': - rawdatasec_img = pix_img.copy() - else: - oscansec_img = pix_img.copy() - - # Return - return detector, raw_img, hdu, exptime, rawdatasec_img, oscansec_img + return super().get_rawimage(raw_file, det, sec_includes_binning=True) def calc_pattern_freq(self, frame, rawdatasec_img, oscansec_img, hdu): """ diff --git a/pypeit/spectrographs/mdm_modspec.py b/pypeit/spectrographs/mdm_modspec.py index 68b1dbe2c6..25f0f78077 100644 --- a/pypeit/spectrographs/mdm_modspec.py +++ b/pypeit/spectrographs/mdm_modspec.py @@ -31,6 +31,8 @@ class MDMModspecEchelleSpectrograph(spectrograph.Spectrograph): supported = True comment = 'MDM Modspec spectrometer; Only 1200l/mm disperser (so far)' + allowed_extensions = ['.fit'] + def get_detector_par(self, det, hdu=None): """ Return metadata for the selected detector. diff --git a/pypeit/spectrographs/p200_dbsp.py b/pypeit/spectrographs/p200_dbsp.py index 0ea3032236..19cd3b8e3f 100644 --- a/pypeit/spectrographs/p200_dbsp.py +++ b/pypeit/spectrographs/p200_dbsp.py @@ -169,6 +169,20 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool) + def get_rawimage(self, raw_file, det): + """ + Read raw spectrograph image files and return data and relevant metadata + needed for image processing. + + For P200/DBSP, the ``DATASEC`` and ``OSCANSEC`` regions are read + directly from the file header and are automatically adjusted to account + for the on-chip binning. This is a simple wrapper for + :func:`pypeit.spectrographs.spectrograph.Spectrograph.get_rawimage` that + sets ``sec_includes_binning`` to True. See the base-class function for + the detailed descriptions of the input parameters and returned objects. + """ + return super().get_rawimage(raw_file, det, sec_includes_binning=True) + class P200DBSPBlueSpectrograph(P200DBSPSpectrograph): """ @@ -575,6 +589,9 @@ def config_specific_par(self, scifile, inp_par=None): 'D68': { 7600: 'p200_dbsp_red_1200_7100_d68.fits', 8200: 'p200_dbsp_red_1200_7100_d68.fits' + }, + 'D55': { + 6680: 'p200_dbsp_red_1200_7100_d55_6680.fits' } }, '1200/9400': { diff --git a/pypeit/spectrographs/spectrograph.py b/pypeit/spectrographs/spectrograph.py index 240580c134..bb0bd0b4ff 100644 --- a/pypeit/spectrographs/spectrograph.py +++ b/pypeit/spectrographs/spectrograph.py @@ -33,12 +33,14 @@ import numpy as np from astropy.io import fits +from astropy import units from pypeit import msgs from pypeit import io from pypeit.core import parse from pypeit.core import procimg from pypeit.core import meta +from pypeit.core import flux_calib from pypeit.par import pypeitpar from pypeit.images.detector_container import DetectorContainer from pypeit.images.mosaic import Mosaic @@ -145,7 +147,7 @@ class Spectrograph: Metadata model that is generic to all spectrographs. """ - allowed_extensions = None + allowed_extensions = ['.fits', '.fits.gz'] """ Defines the allowed extensions for the input fits files. """ @@ -269,6 +271,44 @@ def ql_par(): ) ) ) + + @classmethod + def find_raw_files(cls, root, extension=None): + """ + Find raw observations for this spectrograph in the provided directory. + + This is a wrapper for :func:`~pypeit.io.files_from_extension` that + handles the restrictions of the file extensions specific to this + spectrograph. + + Args: + root (:obj:`str`, `Path`_, :obj:`list`): + One or more paths to search for files, which may or may not include + the prefix of the files to search for. For string input, this can + be the directory ``'/path/to/files/'`` or the directory plus the + file prefix ``'/path/to/files/prefix'``, which yeilds the search + strings ``'/path/to/files/*fits'`` or + ``'/path/to/files/prefix*fits'``, respectively. For a list input, + this can use wildcards for multiple directories. + extension (:obj:`str`, :obj:`list`, optional): + One or more file extensions to search on. If None, uses + :attr:`allowed_extensions`. Otherwise, this *must* be a subset + of the allowed extensions for the selected spectrograph. + + Returns: + :obj:`list`: List of `Path`_ objects with the full path to the set of + unique raw data filenames that match the provided criteria search + strings. + """ + if extension is None: + _ext = cls.allowed_extensions + else: + _ext = [extension] if isinstance(extension, str) else extension + _ext = [e for e in _ext if e in cls.allowed_extensions] + if len(_ext) == 0: + msgs.error(f'{extension} is not or does not include allowed extensions for ' + f'{cls.name}; choose from {cls.allowed_extensions}.') + return io.files_from_extension(root, extension=_ext) def _check_extensions(self, filename): """ @@ -280,9 +320,9 @@ def _check_extensions(self, filename): """ if self.allowed_extensions is not None: _filename = Path(filename).absolute() - if _filename.suffix not in self.allowed_extensions: + if not any([_filename.name.endswith(ext) for ext in self.allowed_extensions]): msgs.error(f'The input file ({_filename.name}) does not have a recognized ' - f'extension ({_filename.suffix}). The allowed extensions for ' + f'extension. The allowed extensions for ' f'{self.name} include {",".join(self.allowed_extensions)}.') def _check_telescope(self): @@ -627,6 +667,28 @@ def list_detectors(self, mosaic=False): dets = self.allowed_mosaics if mosaic else range(1,self.ndet+1) return np.array([self.get_det_name(det) for det in dets]) + def parse_raw_files(self, fitstbl, det=1, ftype=None): + """ + Parse the list of raw files with given frame type and detector. + This is spectrograph-specific, and it is not defined for all + spectrographs. Therefore, this generic method + returns the indices of all the files in the input table. + + Args: + fitstbl (`astropy.table.Table`_): + Table with metadata of the raw files to parse. + det (:obj:`int`, optional): + 1-indexed detector number to parse. + ftype (:obj:`str`, optional): + Frame type to parse. If None, no frames are parsed + and the indices of all frames are returned. + + Returns: + `numpy.ndarray`_: The indices of the raw files in the fitstbl that are parsed. + + """ + + return np.arange(len(fitstbl)) def get_lamps(self, fitstbl): """ @@ -1101,10 +1163,10 @@ def validate_det(self, det): msgs.error(f'Provided det must have type tuple or integer, not {type(det)}.') return 1, (det,) - def get_rawimage(self, raw_file, det): + def get_rawimage(self, raw_file, det, sec_includes_binning=False): """ - Read raw images and generate a few other bits and pieces that are key - for image processing. + Read raw spectrograph image files and return data and relevant metadata + needed for image processing. .. warning:: @@ -1119,6 +1181,14 @@ def get_rawimage(self, raw_file, det): 1-indexed detector(s) to read. An image mosaic is selected using a :obj:`tuple` with the detectors in the mosaic, which must be one of the allowed mosaics returned by :func:`allowed_mosaics`. + sec_includes_binning : :obj:`bool`, optional + Some instruments use hard-coded image-section strings to define the + data and overscan regions, which are then automatically adjusted by + the on-chip binning read from the header. Others read the data and + overscan sections directly from the header. If these sections + *include* the on-chip binning automatically when the image is + written, this flag should be set to true so that this reader returns + the correct image sections. Returns ------- @@ -1145,7 +1215,8 @@ def get_rawimage(self, raw_file, det): """ # Check extension and then open self._check_extensions(raw_file) - hdu = io.fits_open(raw_file, ignore_missing_end=True, output_verify = 'ignore', ignore_blank=True) + hdu = io.fits_open(raw_file, ignore_missing_end=True, output_verify='ignore', + ignore_blank=True) # Validate the entered (list of) detector(s) nimg, _det = self.validate_det(det) @@ -1163,15 +1234,26 @@ def get_rawimage(self, raw_file, det): # NOTE: This *must* be (converted to) seconds. exptime = self.get_meta_value(headarr, 'exptime') - # Rawdatasec, oscansec images - binning = self.get_meta_value(headarr, 'binning') - # NOTE: This means that `specaxis` must be the same for all detectors in - # a mosaic - if detectors[0]['specaxis'] == 1: - binning_raw = (',').join(binning.split(',')[::-1]) + # Binning + if sec_includes_binning: + # The section in the header includes the binning, so set it to None + # here. + binning_raw = None else: - binning_raw = binning + binning = self.get_meta_value(headarr, 'binning') + # NOTE: This means that `specaxis` must be the same for all detectors in + # a mosaic + if detectors[0]['specaxis'] == 1: + binning_raw = (',').join(binning.split(',')[::-1]) + else: + binning_raw = binning + + # Always assume normal FITS header formatting + one_indexed = True + include_last = True + require_dim = 2 + # Read the image(s) raw_img = [None]*nimg rawdatasec_img = [None]*nimg oscansec_img = [None]*nimg @@ -1192,28 +1274,18 @@ def get_rawimage(self, raw_file, det): for section in ['datasec', 'oscansec']: - # Get the data section - # Try using the image sections as header keywords - # TODO -- Deal with user windowing of the CCD (e.g. Kast red) - # Code like the following maybe useful - #hdr = hdu[detector[det - 1]['dataext']].header - #image_sections = [hdr[key] for key in detector[det - 1][section]] - # Grab from Detector + # Get the data sections from the detector object (see get_detector_par above) + # TODO: Add ability to incude user windowing (e.g., Kast Red) image_sections = detectors[i][section] - #if not isinstance(image_sections, list): - # image_sections = [image_sections] - # Always assume normal FITS header formatting - one_indexed = True - include_last = True # Initialize the image (0 means no amplifier) pix_img = np.zeros(raw_img[i].shape, dtype=int) for j in range(detectors[i]['numamplifiers']): if image_sections is not None: # and image_sections[i] is not None: - # Convert the data section from a string to a slice + # Convert the (FITS) data section from a string to a slice datasec = parse.sec2slice(image_sections[j], one_indexed=one_indexed, - include_end=include_last, require_dim=2, + include_end=include_last, require_dim=require_dim, binning=binning_raw) # Assign the amplifier pix_img[datasec] = j+1 @@ -1596,6 +1668,73 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): """ raise NotImplementedError('Frame typing not defined for {0}.'.format(self.name)) + def vet_assigned_ftypes(self, type_bits, fitstbl): + """ + NOTE: this function should only be called when running pypeit_setup, + in order to not overwrite any user-provided frame types. + + This method checks the assigned frame types for consistency. + For frames that are assigned both the science and standard types, + this method chooses the one that is most likely, by checking if the + frames are within 10 arcmin of a listed standard star. + + In addition, this method can perform other checks on the assigned frame types + that are spectrograph-specific. + + Args: + type_bits (`numpy.ndarray`_): + Array with the frame types assigned to each frame. + fitstbl (:class:`~pypeit.metadata.PypeItMetaData`): + The class holding the metadata for all the frames. + + Returns: + `numpy.ndarray`_: The updated frame types. + + """ + # For frames that are assigned both science and standard types, choose the one that is most likely + # find frames that are assigned both science and standard star types + indx = fitstbl.type_bitmask.flagged(type_bits, flag='standard') & \ + fitstbl.type_bitmask.flagged(type_bits, flag='science') + if np.any(indx): + msgs.warn('Some frames are assigned both science and standard types. Choosing the most likely type.') + if 'ra' not in fitstbl.keys() or 'dec' not in fitstbl.keys(): + msgs.warn('Sky coordinates are not available. Standard stars cannot be identified.') + # turn off the standard flag for all frames + type_bits[indx] = fitstbl.type_bitmask.turn_off(type_bits[indx], flag='standard') + return type_bits + # check if any coordinates are None + none_coords = indx & ((fitstbl['ra'] == 'None') | (fitstbl['dec'] == 'None') | + np.isnan(fitstbl['ra']) | np.isnan(fitstbl['dec'])) + if np.any(none_coords): + msgs.warn('The following frames have None coordinates. ' + 'They could be a twilight flat frame that was missed by the automatic identification') + [msgs.prindent(f) for f in fitstbl['filename'][none_coords]] + # turn off the standard star flag for these frames + type_bits[none_coords] = fitstbl.type_bitmask.turn_off(type_bits[none_coords], flag='standard') + + # If the frame is within 10 arcmin of a listed standard star, then it is probably a standard star + # Find the nearest standard star to each frame that is assigned both science and standard types + # deal with possible None coordinates + is_std = np.array([], dtype=bool) + for ra, dec in zip(fitstbl['ra'], fitstbl['dec']): + if ra == 'None' or dec == 'None' or np.isnan(ra) or np.isnan(dec): + is_std = np.append(is_std, False) + else: + is_std = np.append(is_std, flux_calib.find_standard_file(ra, dec, toler=10.*units.arcmin, check=True)) + + foundstd = indx & is_std + # turn off the science flag for frames that are found to be standard stars and + # turn off the standard flag for frames that are not + if np.any(foundstd): + type_bits[foundstd] = fitstbl.type_bitmask.turn_off(type_bits[foundstd], flag='science') + type_bits[np.logical_not(foundstd)] = \ + fitstbl.type_bitmask.turn_off(type_bits[np.logical_not(foundstd)], flag='standard') + else: + # if no standard stars are found, turn off the standard flag for all frames + type_bits[indx] = fitstbl.type_bitmask.turn_off(type_bits[indx], flag='standard') + + return type_bits + def idname(self, ftype): """ Return the ``idname`` for the selected frame type for this @@ -1792,13 +1931,11 @@ def spec1d_match_spectra(self, sobjs): """ msgs.error(f'Method to match slits across detectors not defined for {self.name}') - def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table, log10_blaze_function=None): """ - This routine is for performing instrument/disperser specific tweaks to standard stars so that sensitivity function fits will be well behaved. For example, masking second order light. For instruments that don't - require such tweaks it will just return the inputs, but for isntruments that do this function is overloaded + require such tweaks it will just return the inputs, but for instruments that do this function is overloaded with a method that performs the tweaks. Parameters diff --git a/pypeit/spectrographs/vlt_xshooter.py b/pypeit/spectrographs/vlt_xshooter.py index b12ac92cbb..9160f834e3 100644 --- a/pypeit/spectrographs/vlt_xshooter.py +++ b/pypeit/spectrographs/vlt_xshooter.py @@ -222,6 +222,7 @@ def get_detector_par(self, det, hdu=None): :class:`~pypeit.images.detector_container.DetectorContainer`: Object with the detector metadata. """ + # Detector 1 detector_dict = dict( binning = '1,1', # No binning in near-IR @@ -230,21 +231,15 @@ def get_detector_par(self, det, hdu=None): specaxis = 1, specflip = False, spatflip = False, - platescale = 0.197, # average between order 11 & 30, see manual - darkcurr = 0.0, # e-/pixel/hour + platescale = 0.245, # average across all orders, see manual + darkcurr = 72.0, # e-/pixel/hour saturation = 2.0e5, # I think saturation may never be a problem here since there are many DITs nonlinear = 0.86, mincounts = -1e10, numamplifiers = 1, - gain = np.atleast_1d(2.12), # - ronoise = np.atleast_1d(8.0), # ?? more precise value? #TODO the read noise is exposure time dependent and should be grabbed from header + gain = np.atleast_1d(2.29), + ronoise = np.atleast_1d(8.0), # Read noise depends on exposure time (see XShooter manual) but this is the typical value. datasec = np.atleast_1d('[4:2044,4:]'), # These are all unbinned pixels - # EMA: No real overscan for XSHOOTER-NIR: - # See Table 6 in http://www.eso.org/sci/facilities/paranal/instruments/xshooter/doc/VLT-MAN-ESO-14650-4942_P103v1.pdf - # The overscan region below contains only zeros - # ToDo should we just set it as empty? - # JXP says yes - #oscansec = np.atleast_1d('[4:2044,1:3]'), # These are all unbinned pixels. ) return detector_container.DetectorContainer(**detector_dict) @@ -263,19 +258,12 @@ def default_pypeit_par(cls): turn_off = dict(use_illumflat=False, use_biasimage=False, use_overscan=False, use_darkimage=False) par.reset_all_processimages_par(**turn_off) - # Require dark images to be subtracted from the flat images used for - # tracing, pixelflats, and illumflats - # par['calibrations']['traceframe']['process']['use_darkimage'] = True - # par['calibrations']['pixelflatframe']['process']['use_darkimage'] = True - # par['calibrations']['illumflatframe']['process']['use_darkimage'] = True - # TODO: `mask_cr` now defaults to True for darks. Should this be turned off? # Is this needed below? par['scienceframe']['process']['sigclip'] = 20.0 par['scienceframe']['process']['satpix'] = 'nothing' # TODO tune up LA COSMICS parameters here for X-shooter as tellurics are being excessively masked - # Adjustments to slit and tilts for NIR par['calibrations']['slitedges']['edge_thresh'] = 50. par['calibrations']['slitedges']['fit_order'] = 8 @@ -304,16 +292,17 @@ def default_pypeit_par(cls): par['calibrations']['wavelengths']['reid_arxiv'] = 'vlt_xshooter_nir.fits' par['calibrations']['wavelengths']['cc_thresh'] = 0.50 par['calibrations']['wavelengths']['cc_local_thresh'] = 0.50 -# par['calibrations']['wavelengths']['ech_fix_format'] = True # Echelle parameters par['calibrations']['wavelengths']['echelle'] = True par['calibrations']['wavelengths']['ech_nspec_coeff'] = 5 par['calibrations']['wavelengths']['ech_norder_coeff'] = 5 par['calibrations']['wavelengths']['ech_sigrej'] = 3.0 par['calibrations']['wavelengths']['qa_log'] = False + # Measured FWHM is correct, but resulting wavelength solution is poor. + # This should be explored further, but for now, turning off fwhm_fromlines helps. + par['calibrations']['wavelengths']['fwhm_fromlines'] = False # Flats - #par['calibrations']['standardframe']['process']['illumflatten'] = False par['calibrations']['flatfield']['tweak_slits_thresh'] = 0.90 par['calibrations']['flatfield']['tweak_slits_maxfrac'] = 0.10 @@ -324,23 +313,10 @@ def default_pypeit_par(cls): par['reduce']['skysub']['bspline_spacing'] = 0.8 par['reduce']['skysub']['global_sky_std'] = False # Do not perform global sky subtraction for standard stars par['reduce']['extraction']['model_full_slit'] = True # local sky subtraction operates on entire slit - par['reduce']['findobj']['trace_npoly'] = 8 + par['reduce']['findobj']['trace_npoly'] = 10 par['reduce']['findobj']['maxnumber_sci'] = 2 # Assume that there is only one object on the slit. par['reduce']['findobj']['maxnumber_std'] = 1 # Assume that there is only one object on the slit. - - # The settings below enable X-shooter dark subtraction from the traceframe and pixelflatframe, but enforce - # that this bias won't be subtracted from other images. It is a hack for now, because eventually we want to - # perform this operation with the dark frame class, and we want to attach individual sets of darks to specific - # images. - #par['calibrations']['biasframe']['useframe'] = 'bias' - #par['calibrations']['traceframe']['process']['bias'] = 'force' - #par['calibrations']['pixelflatframe']['process']['bias'] = 'force' - #par['calibrations']['arcframe']['process']['bias'] = 'skip' - #par['calibrations']['tiltframe']['process']['bias'] = 'skip' - #par['calibrations']['standardframe']['process']['bias'] = 'skip' - #par['scienceframe']['process']['bias'] = 'skip' - # Sensitivity function parameters par['sensfunc']['algorithm'] = 'IR' par['sensfunc']['polyorder'] = 8 @@ -437,7 +413,7 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): | (fitstbl['target'] == 'LAMP,FLAT')) & good_flat_seq) - if ftype in ['dark']: + if ftype in ['lampoffflats']: # Lamp off flats are taken second (even exposure number) return good_exp & (((fitstbl['target'] == 'LAMP,DFLAT') | (fitstbl['target'] == 'LAMP,QFLAT') @@ -488,32 +464,6 @@ def bpm(self, filename, det, shape=None, msbias=None): vlt_sc = dataPaths.static_calibs / 'vlt_xshoooter' bpm_loc = np.loadtxt(vlt_sc.get_file_path('BP_MAP_RP_NIR.dat'), usecols=(0,1)) bpm_img[bpm_loc[:,0].astype(int),bpm_loc[:,1].astype(int)] = 1. -# try : -# bpm_loc = np.loadtxt(vlt_sc.get_file_path('BP_MAP_RP_NIR.dat'), usecols=(0,1)) -# except IOError : -# # TODO: Do we need this anymore? Both the *.dat and *.fits.gz -# # files are present in the repo. -# msgs.warn('BP_MAP_RP_NIR.dat not present in the static database') -# bpm_fits = io.fits_open(vlt_sc.get_file_path('BP_MAP_RP_NIR.fits.gz')) -# # ToDo: this depends on datasec, biassec, specflip, and specaxis -# # and should become able to adapt to these parameters. -# # Flipping and shifting BPM to match the PypeIt format -# y_shift = -2 -# x_shift = 18 -# bpm_data = np.flipud(bpm_fits[0].data) -# y_len = len(bpm_data[:,0]) -# x_len = len(bpm_data[0,:]) -# bpm_data_pypeit = np.full( ((y_len+abs(y_shift)),(x_len+abs(x_shift))) , 0) -# bpm_data_pypeit[:-abs(y_shift),:-abs(x_shift)] = bpm_data_pypeit[:-abs(y_shift),:-abs(x_shift)] + bpm_data -# bpm_data_pypeit = np.roll(bpm_data_pypeit,-y_shift,axis=0) -# bpm_data_pypeit = np.roll(bpm_data_pypeit,x_shift,axis=1) -# filt_bpm = bpm_data_pypeit[1:y_len,1:x_len]>100. -# y_bpm, x_bpm = np.where(filt_bpm) -# bpm_loc = np.array([y_bpm,x_bpm]).T -# # NOTE: This directly access the path, but we shouldn't be doing that... -# np.savetxt(vlt_sc.path / 'BP_MAP_RP_NIR.dat', bpm_loc, fmt=['%d','%d']) -# finally : -# bpm_img[bpm_loc[:,0].astype(int),bpm_loc[:,1].astype(int)] = 1. return bpm_img @@ -548,8 +498,8 @@ def spec_min_max(self): Return the minimum and maximum spectral pixel expected for the spectral range of each order. """ - spec_max = np.asarray([1467,1502,1540, 1580,1620,1665,1720, 1770,1825,1895, 1966, 2000,2000,2000,2000,2000]) - spec_min = np.asarray([420 ,390 , 370, 345, 315, 285, 248, 210, 165, 115, 63, 10, 0, 0, 0, 0]) + spec_max = np.asarray([1477,1513,1547, 1588,1628,1682,1733,1795,1855,1930,2005,2040,2040,2040,2040,2040]) + spec_min = np.asarray([420 ,390 , 370, 345, 315, 285, 248, 210, 165, 115, 58, 5, 0, 0, 0, 0]) return np.vstack((spec_min, spec_max)) def order_platescale(self, order_vec, binning=None): @@ -569,12 +519,9 @@ def order_platescale(self, order_vec, binning=None): `numpy.ndarray`_: An array with the platescale for each order provided by ``order``. """ - # TODO: Either assume a linear trend or measure this - # X-shooter manual says, but gives no exact numbers per order. - # NIR: 52.4 pixels (0.210"/pix) at order 11 to 59.9 pixels (0.184"/pix) at order 26. - - # Right now I just assume a simple linear trend - plate_scale = 0.184 + (order_vec - 26)*(0.184-0.210)/(26 - 11) + # TODO: Figure out the order-dependence of the updated plate scale + # From the X-Shooter P113 manual, average over all orders. No order-dependent values given. + plate_scale = 0.245*np.ones_like(order_vec) return plate_scale @property @@ -624,6 +571,11 @@ def get_detector_par(self, det, hdu=None): # Binning # TODO: Could this be detector dependent?? binning = '1,1' if hdu is None else self.get_meta_value(self.get_headarr(hdu), 'binning') + + # Grab the gain and read noise from the header. + # If hdu not present, use typical defaults + gain = None if hdu is None else np.atleast_1d(hdu[0].header['HIERARCH ESO DET OUT1 CONAD']) + ronoise = None if hdu is None else np.atleast_1d(hdu[0].header['HIERARCH ESO DET OUT1 RON']) # Detector 1 detector_dict = dict( @@ -633,17 +585,17 @@ def get_detector_par(self, det, hdu=None): specaxis = 0, specflip = False, spatflip = False, - platescale = 0.16, # average from order 17 and order 30, see manual + platescale = 0.154, # average from order 17 and order 30, see manual darkcurr = 0.0, # e-/pixel/hour saturation = 65535., nonlinear = 0.86, mincounts = -1e10, numamplifiers = 1, - gain = np.atleast_1d(0.595), # FITS format is flipped: PrimaryHDU (2106, 4000) w/respect to Python - ronoise = np.atleast_1d(3.1), # raw unbinned images are (4000,2106) (spec, spat) - datasec=np.atleast_1d('[:,11:2058]'), # pre and oscan are in the spatial direction - oscansec=np.atleast_1d('[:,2059:2106]'), - ) + gain = gain, + ronoise = ronoise, + datasec=np.atleast_1d('[:,11:2058]'), # FITS format is flipped: PrimaryHDU (2106, 4000) w/respect to Python + oscansec=np.atleast_1d('[:,2059:2106]'), # raw unbinned images are (4000,2106) (spec, spat) + ) # pre and oscan are in the spatial direction return detector_container.DetectorContainer(**detector_dict) @classmethod @@ -808,15 +760,12 @@ def order_platescale(self, order_vec, binning=None): `numpy.ndarray`_: An array with the platescale for each order provided by ``order``. """ - # VIS has no binning, but for an instrument with binning we would do this binspectral, binspatial = parse.parse_binning(binning) - - # ToDO Either assume a linear trend or measure this - # X-shooter manual says, but gives no exact numbers per order. - # VIS: 65.9 pixels (0.167"/pix) at order 17 to 72.0 pixels (0.153"/pix) at order 30. - - # Right now I just assume a simple linear trend - plate_scale = 0.153 + (order_vec - 30)*(0.153-0.167)/(30 - 17) + + # TODO: Figure out the order-dependence of the updated plate scale + # From the X-Shooter P113 manual, average over all orders. No order-dependent values given. + plate_scale = 0.154*np.ones_like(order_vec) + return plate_scale*binspatial @property @@ -915,6 +864,10 @@ def get_detector_par(self, det, hdu=None): """ # Binning binning = '1,1' if hdu is None else self.get_meta_value(self.get_headarr(hdu), 'binning') + + # Grab the gain and read noise from the header. + gain = None if hdu is None else np.atleast_1d(hdu[0].header['HIERARCH ESO DET OUT1 CONAD']) + ronoise = None if hdu is None else np.atleast_1d(hdu[0].header['HIERARCH ESO DET OUT1 RON']) # Detector 1 detector_dict = dict( @@ -924,14 +877,14 @@ def get_detector_par(self, det, hdu=None): specaxis = 0, specflip = True, spatflip = True, - platescale = 0.161, # average from order 14 and order 24, see manual + platescale = 0.164, # average from order 14 and order 24, see manual darkcurr = 0.0, # e-/pixel/hour saturation = 65000., nonlinear = 0.86, mincounts = -1e10, numamplifiers = 1, - gain = np.atleast_1d(1.61), - ronoise = np.atleast_1d(2.60), + gain = gain, + ronoise = ronoise, datasec = np.atleast_1d('[:,49:2096]'), # '[49:2000,1:2999]', oscansec = np.atleast_1d('[:,1:48]'), # '[1:48, 1:2999]', ) @@ -1116,15 +1069,11 @@ def order_platescale(self, order_vec, binning = None): """ binspectral, binspatial = parse.parse_binning(binning) - # ToDO Either assume a linear trend or measure this - # X-shooter manual says, but gives no exact numbers per order. - # UVB: 65.9 pixels (0.167“/pix) at order 14 to 70.8 pixels (0.155”/pix) at order 24 - - # Assume a simple linear trend - plate_scale = 0.155 + (order_vec - 24)*(0.155-0.167)/(24 - 14) + # TODO: Figure out the order-dependence of the updated plate scale + # From the X-Shooter P113 manual, average over all orders. No order-dependent values given. + plate_scale = 0.164*np.ones_like(order_vec) - # Right now I just took the average - return np.full(self.norders, 0.161)*binspatial + return plate_scale*binspatial def bpm(self, filename, det, shape=None, msbias=None): """ diff --git a/pypeit/spectrographs/wht_isis.py b/pypeit/spectrographs/wht_isis.py index e107264ac3..7464e459b8 100644 --- a/pypeit/spectrographs/wht_isis.py +++ b/pypeit/spectrographs/wht_isis.py @@ -125,6 +125,7 @@ class WHTISISBlueSpectrograph(WHTISISSpectrograph): name = 'wht_isis_blue' camera = 'ISISb' comment = 'Blue camera' + allowed_extensions = ['.fit', '.fit.gz'] def get_detector_par(self, det, hdu=None): """ diff --git a/pypeit/telescopes.py b/pypeit/telescopes.py index e8bc8c3d3a..32e9cbff80 100644 --- a/pypeit/telescopes.py +++ b/pypeit/telescopes.py @@ -10,6 +10,18 @@ #TODO: Remove 'Par' from class name? + +class AATTelescopePar(TelescopePar): + def __init__(self): + loc = EarthLocation.of_site('Siding Spring Observatory') + super(AATTelescopePar, self).__init__(name='AAT', + longitude=loc.lon.to(units.deg).value, + latitude=loc.lat.to(units.deg).value, + elevation=loc.height.to(units.m).value, + diameter=3.9, + eff_aperture=12.0) + + class GTCTelescopePar(TelescopePar): def __init__(self): loc = EarthLocation.of_site('Roque de los Muchachos') diff --git a/pypeit/tests/test_arc.py b/pypeit/tests/test_arc.py index fef470534e..3cde322d37 100644 --- a/pypeit/tests/test_arc.py +++ b/pypeit/tests/test_arc.py @@ -2,10 +2,12 @@ Module to run tests on ararclines """ import pytest +import numpy as np from pypeit.core import arc from pypeit import io + def test_detect_lines(): # Using Paranal night sky as an 'arc' arx_sky = io.load_sky_spectrum('paranal_sky.fits') diff --git a/pypeit/tests/test_calibrations.py b/pypeit/tests/test_calibrations.py index 3705674b59..52ac1958ac 100644 --- a/pypeit/tests/test_calibrations.py +++ b/pypeit/tests/test_calibrations.py @@ -1,13 +1,10 @@ -""" -Module to run tests on FlatField class -Requires files in Development suite and an Environmental variable -""" from pathlib import Path -import os import yaml import pytest import shutil +from IPython import embed + import numpy as np from pypeit import dataPaths @@ -16,9 +13,8 @@ from pypeit.images import buildimage from pypeit.par import pypeitpar from pypeit.spectrographs.util import load_spectrograph -from IPython import embed -from pypeit.tests.tstutils import dummy_fitstbl, data_output_path +from pypeit.tests.tstutils import data_output_path @pytest.fixture def fitstbl(): diff --git a/pypeit/tests/test_fluxspec.py b/pypeit/tests/test_fluxspec.py index fc0a73da34..a51b3aed00 100644 --- a/pypeit/tests/test_fluxspec.py +++ b/pypeit/tests/test_fluxspec.py @@ -181,7 +181,8 @@ def extinction_correction_tester(algorithm): wave = np.linspace(4000, 6000) counts = np.ones_like(wave) ivar = np.ones_like(wave) - sobj = specobj.SpecObj.from_arrays('MultiSlit', wave, counts, ivar) + flat = np.ones_like(wave) + sobj = specobj.SpecObj.from_arrays('MultiSlit', wave, counts, ivar, flat) sobjs = specobjs.SpecObjs([sobj]) # choice of PYP_SPEC, DISPNAME and EXPTIME are unimportant here diff --git a/pypeit/tests/test_io.py b/pypeit/tests/test_io.py index 95d03abb14..2a6f226297 100644 --- a/pypeit/tests/test_io.py +++ b/pypeit/tests/test_io.py @@ -48,5 +48,5 @@ def test_grab_rawfiles(): _raw_files = inputfiles.grab_rawfiles(raw_paths=[str(root)], extension='.fits.gz') assert len(_raw_files) == 9, 'Found the wrong number of files' - assert all([str(root / f) in _raw_files for f in tbl['filename']]), 'Missing expected files' + assert all([root / f in _raw_files for f in tbl['filename']]), 'Missing expected files' diff --git a/pypeit/tests/test_metadata.py b/pypeit/tests/test_metadata.py index 048191cc3f..467daa44bf 100644 --- a/pypeit/tests/test_metadata.py +++ b/pypeit/tests/test_metadata.py @@ -30,7 +30,7 @@ def test_read_combid(): # Generate the pypeit file with the comb_id droot = tstutils.data_output_path('b') pargs = Setup.parse_args(['-r', droot, '-s', 'shane_kast_blue', '-c', 'all', '-b', - '--extension', 'fits.gz', '--output_path', f'{config_dir.parent}']) + '--output_path', f'{config_dir.parent}']) Setup.main(pargs) pypeit_file = config_dir / 'shane_kast_blue_A.pypeit' diff --git a/pypeit/tests/test_mosaic.py b/pypeit/tests/test_mosaic.py new file mode 100644 index 0000000000..9079be9925 --- /dev/null +++ b/pypeit/tests/test_mosaic.py @@ -0,0 +1,41 @@ +from pathlib import Path +from IPython import embed + +import pytest + +from astropy.io import fits + +from pypeit.pypmsgs import PypeItDataModelError +from pypeit.tests.tstutils import data_output_path +from pypeit.images.mosaic import Mosaic +from pypeit.spectrographs.util import load_spectrograph + + +def test_io(): + # Create the mosaic + spec = load_spectrograph('keck_deimos') + mpar = spec.get_mosaic_par((1,5)) + + # Write it + ofile = data_output_path('tmp_mosaic.fits') + mpar.to_file(ofile, overwrite=True) + + # Try to read it + _mpar = Mosaic.from_file(ofile) + + # Change the version + _ofile = data_output_path('tmp_mosaic_wrongver.fits') + with fits.open(ofile) as hdu: + hdu['MOSAIC'].header['DMODVER'] = '1.0.0' + hdu.writeto(_ofile, overwrite=True) + + # Reading should fail because version is checked by default + with pytest.raises(PypeItDataModelError): + _mpar = Mosaic.from_file(_ofile) + + # Should not fail because skipping the version check + _mpar = Mosaic.from_file(_ofile, chk_version=False) + + # Remove files + Path(ofile).unlink() + Path(_ofile).unlink() diff --git a/pypeit/tests/test_setups.py b/pypeit/tests/test_setups.py index e4356968e1..5759072024 100644 --- a/pypeit/tests/test_setups.py +++ b/pypeit/tests/test_setups.py @@ -63,12 +63,12 @@ def test_run_setup(): droot = tstutils.data_output_path('b') odir = Path(tstutils.data_output_path('')).absolute() / 'shane_kast_blue_A' pargs = Setup.parse_args(['-r', droot, '-s', 'shane_kast_blue', '-c', 'all', - '--extension', 'fits.gz', '--output_path', f'{odir.parent}']) + '--output_path', f'{odir.parent}']) Setup.main(pargs) # Fails because name of spectrograph is wrong pargs2 = Setup.parse_args(['-r', droot, '-s', 'shane_kast_blu', '-c', 'all', - '--extension', 'fits.gz', '--output_path', f'{odir.parent}']) + '--output_path', f'{odir.parent}']) with pytest.raises(ValueError): Setup.main(pargs2) diff --git a/pypeit/tests/test_specobj.py b/pypeit/tests/test_specobj.py index 5e647648df..6ec2da6cfe 100644 --- a/pypeit/tests/test_specobj.py +++ b/pypeit/tests/test_specobj.py @@ -101,7 +101,8 @@ def test_from_arrays(): wave = np.linspace(5000., 6000, 1000) flux = np.ones_like(wave) ivar = 0.1*np.ones_like(wave) - sobj = specobj.SpecObj.from_arrays('MultiSlit', wave, flux, ivar) + flat = np.ones_like(wave) + sobj = specobj.SpecObj.from_arrays('MultiSlit', wave, flux, ivar, flat) assert sobj.OPT_WAVE[0] == 5000. diff --git a/pypeit/utils.py b/pypeit/utils.py index fbc6550c15..a7b0f6ace1 100644 --- a/pypeit/utils.py +++ b/pypeit/utils.py @@ -646,7 +646,7 @@ def growth_lim(a, lim, fac=1.0, midpoint=None, default=[0., 1.]): end -= 1 # Set the full range and multiply it by the provided factor - srt = np.ma.argsort(_a) + srt = np.ma.argsort(_a, kind='stable') Da = (_a[srt[end]] - _a[srt[start]]) * fac # Set the midpoint @@ -887,7 +887,7 @@ def index_of_x_eq_y(x, y, strict=False): """ if y.ndim != 1 or y.ndim != 1: raise ValueError('Arrays must be 1D.') - srt = np.argsort(x) + srt = np.argsort(x, kind='stable') indx = np.searchsorted(x[srt], y) x2y = np.take(srt, indx, mode='clip') if strict and not np.array_equal(x[x2y], y): @@ -1380,6 +1380,38 @@ def find_nearest(array, values): return idxs +def linear_interpolate(x1, y1, x2, y2, x): + r""" + Interplate or extrapolate between two points. + + Given a line defined two points, :math:`(x_1,y_1)` and + :math:`(x_2,y_2)`, return the :math:`y` value of a new point on + the line at coordinate :math:`x`. + + This function is meant for speed. No type checking is performed and + the only check is that the two provided ordinate coordinates are not + numerically identical. By definition, the function will extrapolate + without any warning. + + Args: + x1 (:obj:`float`): + First abscissa position + y1 (:obj:`float`): + First ordinate position + x2 (:obj:`float`): + Second abscissa position + y3 (:obj:`float`): + Second ordinate position + x (:obj:`float`): + Abcissa for new value + + Returns: + :obj:`float`: Interpolated/extrapolated value of ordinate at + :math:`x`. + """ + return y1 if np.isclose(x1,x2) else y1 + (x-x1)*(y2-y1)/(x2-x1) + + def replace_bad(frame, bpm): """ Find all bad pixels, and replace the bad pixels with the nearest good pixel @@ -1442,7 +1474,7 @@ def yamlify(obj, debug=False): obj = bool(obj) # elif isinstance(obj, bytes): # obj = obj.decode('utf-8') - elif isinstance(obj, (np.string_, str)): + elif isinstance(obj, (np.str_, str)): obj = str(obj) # Worry about colons! if ':' in obj: @@ -1484,6 +1516,64 @@ def yamlify(obj, debug=False): print(type(obj)) return obj +def jsonify(obj, debug=False): + """ Recursively process an object so it can be serialised in json + format. Taken from linetools. + + WARNING - the input object may be modified if it's a dictionary or + list! + + Parameters + ---------- + obj : any object + debug : bool, optional + + Returns + ------- + obj - the same obj is json_friendly format (arrays turned to + lists, np.int64 converted to int, np.float64 to float, and so on). + + """ + if isinstance(obj, np.float64): + obj = float(obj) + elif isinstance(obj, np.float32): + obj = float(obj) + elif isinstance(obj, np.int32): + obj = int(obj) + elif isinstance(obj, np.int64): + obj = int(obj) + elif isinstance(obj, np.int16): + obj = int(obj) + elif isinstance(obj, np.bool_): + obj = bool(obj) + elif isinstance(obj, np.str_): + obj = str(obj) + elif isinstance(obj, units.Quantity): + if obj.size == 1: + obj = dict(value=obj.value, unit=obj.unit.to_string()) + else: + obj = dict(value=obj.value.tolist(), unit=obj.unit.to_string()) + elif isinstance(obj, np.ndarray): # Must come after Quantity + obj = obj.tolist() + elif isinstance(obj, dict): + for key, value in obj.items(): + obj[key] = jsonify(value, debug=debug) + elif isinstance(obj, list): + for i,item in enumerate(obj): + obj[i] = jsonify(item, debug=debug) + elif isinstance(obj, tuple): + obj = list(obj) + for i,item in enumerate(obj): + obj[i] = jsonify(item, debug=debug) + obj = tuple(obj) + elif isinstance(obj, units.Unit): + obj = obj.name + elif obj is units.dimensionless_unscaled: + obj = 'dimensionless_unit' + + if debug: + print(type(obj)) + return obj def add_sub_dict(d, key): """ diff --git a/pypeit/wavecalib.py b/pypeit/wavecalib.py index db1b5b8aa0..969b6b3079 100644 --- a/pypeit/wavecalib.py +++ b/pypeit/wavecalib.py @@ -10,7 +10,7 @@ import numpy as np from matplotlib import pyplot as plt -from linetools.utils import jsonify +from pypeit.utils import jsonify from astropy.table import Table from astropy.io import fits @@ -217,7 +217,8 @@ def from_hdu(cls, hdu, chk_version=True, **kwargs): # Parse all the WAVE2DFIT extensions # TODO: It would be good to have the WAVE2DFIT extensions follow the # same naming convention as the WAVEFIT extensions... - wave2d_fits = [fitting.PypeItFit.from_hdu(hdu[e], chk_version=chk_version) + wave2d_fits = [fitting.PypeItFit() if len(hdu[e].data) == 0 + else fitting.PypeItFit.from_hdu(hdu[e], chk_version=chk_version) for e in ext if 'WAVE2DFIT' in e] if len(wave2d_fits) > 0: d['wv_fit2d'] = np.asarray(wave2d_fits) @@ -281,9 +282,14 @@ def build_fwhmimg(self, tilts, slits, initial=False, spat_flexure=None): # Generate the slit mask and slit edges - pad slitmask by 1 for edge effects slitmask = slits.slit_img(pad=1, initial=initial, flexure=spat_flexure) slits_left, slits_right, _ = slits.select_edges(initial=initial, flexure=spat_flexure) + # need to exclude slits that are masked (are bad) + bad_slits = slits.bitmask.flagged(slits.mask, and_not=slits.bitmask.exclude_for_reducing) + ok_spat_ids = slits.spat_id[np.logical_not(bad_slits)] # Build a map of the spectral FWHM fwhmimg = np.zeros(tilts.shape) for sl, spat_id in enumerate(slits.spat_id): + if spat_id not in ok_spat_ids: + continue this_mask = slitmask == spat_id spec, spat = np.where(this_mask) spat_loc = (spat - slits_left[spec, sl]) / (slits_right[spec, sl] - slits_left[spec, sl]) @@ -327,13 +333,11 @@ def build_waveimg(self, tilts, slits, spat_flexure=None, spec_flexure=None): spec_flex /= (slits.nspec - 1) # Setup - #ok_slits = slits.mask == 0 -# bpm = slits.mask.astype(bool) -# bpm &= np.logical_not(slits.bitmask.flagged(slits.mask, flag=slits.bitmask.exclude_for_reducing)) bpm = slits.bitmask.flagged(slits.mask, and_not=slits.bitmask.exclude_for_reducing) ok_slits = np.logical_not(bpm) # image = np.zeros_like(tilts) + # Grab slit_img slitmask = slits.slit_img(flexure=spat_flexure, exclude_flag=slits.bitmask.exclude_for_reducing) # Separate detectors for the 2D solutions? @@ -341,9 +345,7 @@ def build_waveimg(self, tilts, slits, spat_flexure=None, spec_flexure=None): # Error checking if self.det_img is None: msgs.error("This WaveCalib object was not generated with ech_separate_2d=True") - # Grab slit_img - slit_img = slits.slit_img() - + # Unpack some 2-d fit parameters if this is echelle for islit in np.where(ok_slits)[0]: slit_spat = slits.spat_id[islit] @@ -353,9 +355,7 @@ def build_waveimg(self, tilts, slits, spat_flexure=None, spec_flexure=None): if self.par['echelle'] and self.par['ech_2dfit']: # evaluate solution -- if self.par['ech_separate_2d']: - ordr_det = slits.det_of_slit( - slit_spat, self.det_img, - slit_img=slit_img) + ordr_det = slits.det_of_slit(slit_spat, self.det_img, slit_img=slitmask) # There are ways for this to go sour.. # if the seperate solutions are not aligned with the detectors # or if one reruns with a different number of detectors @@ -525,11 +525,11 @@ def __init__(self, msarc, slits, spectrograph, par, lamps, self.arccen = None # central arc spectrum # Get the non-linear count level - # TODO: This is currently hacked to deal with Mosaics - try: + if self.msarc.is_mosaic: + # if this is a mosaic we take the maximum value among all the detectors + self.nonlinear_counts = np.max([rawdets.nonlinear_counts() for rawdets in self.msarc.detector.detectors]) + else: self.nonlinear_counts = self.msarc.detector.nonlinear_counts() - except: - self.nonlinear_counts = 1e10 # -------------------------------------------------------------- # TODO: Build another base class that does these things for both diff --git a/pypeit/wavetilts.py b/pypeit/wavetilts.py index cf9977744a..e0949b9d99 100644 --- a/pypeit/wavetilts.py +++ b/pypeit/wavetilts.py @@ -25,6 +25,7 @@ from pypeit.display import display from pypeit.core import arc from pypeit.core import tracewave +from pypeit.core.wavecal import autoid from pypeit.images import buildimage @@ -174,7 +175,7 @@ def spatid_to_zero(self, spat_id): return np.where(mtch)[0][0] def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False, - chk_version=True): + calib_dir=None, chk_version=True): """ Show in ginga or mpl Tiltimg with the tilts traced and fitted overlaid @@ -188,6 +189,9 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False, If True, show the image in ginga. Otherwise, use matplotlib. show_traces (bool, optional): If True, show the traces of the tilts on the image. + calib_dir (`Path`_): + Path to the calibration directory. If None, the path is taken from the + WaveTilts object. chk_version (:obj:`bool`, optional): When reading in existing files written by PypeIt, perform strict version checking to ensure a valid file. If False, the code @@ -195,7 +199,14 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False, failures. User beware! """ # get tilt_img_dict - cal_file = Path(self.calib_dir).absolute() / self.tiltimg_filename + _calib_dir = self.calib_dir + if calib_dir is not None and calib_dir.exists(): + _calib_dir = calib_dir + msgs.info(f'Searching for other calibration files in {str(_calib_dir)}') + else: + msgs.info(f'Searching for other calibration files in the default directory {str(_calib_dir)}') + + cal_file = Path(_calib_dir).absolute() / self.tiltimg_filename if cal_file.exists(): tilt_img_dict = buildimage.TiltImage.from_file(cal_file, chk_version=chk_version) else: @@ -203,7 +214,7 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False, # get slits slitmask = None - cal_file = Path(self.calib_dir).absolute() / self.slits_filename + cal_file = Path(_calib_dir).absolute() / self.slits_filename if cal_file.exists(): slits = slittrace.SlitTraceSet.from_file(cal_file, chk_version=chk_version) _slitmask = slits.slit_img(initial=True, flexure=self.spat_flexure) @@ -220,7 +231,7 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False, # get waveimg same_size = (slits.nspec, slits.nspat) == tilt_img_dict.image.shape if waveimg is None and slits is not None and same_size and in_ginga: - wv_calib_name = wavecalib.WaveCalib.construct_file_name(self.calib_key, calib_dir=self.calib_dir) + wv_calib_name = wavecalib.WaveCalib.construct_file_name(self.calib_key, calib_dir=_calib_dir) if Path(wv_calib_name).absolute().exists(): wv_calib = wavecalib.WaveCalib.from_file(wv_calib_name, chk_version=chk_version) tilts = self.fit2tiltimg(slitmask, flexure=self.spat_flexure) @@ -268,24 +279,32 @@ class BuildWaveTilts: slits (:class:`~pypeit.slittrace.SlitTraceSet`): Slit edges spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): - Spectrograph object + The `Spectrograph` instance that sets the instrument used. Used to set + :attr:`spectrograph`. par (:class:`~pypeit.par.pypeitpar.WaveTiltsPar` or None): - The parameters used to fuss with the tilts + The parameters used for the tilt calibration. + Uses ``['calibrations']['tilts']``. wavepar (:class:`~pypeit.par.pypeitpar.WavelengthSolutionPar` or None): - The parameters used for the wavelength solution - det (int): Detector index + The parameters used for the wavelength solution. + Uses ``['calibrations']['wavelengths']``. + det (int): + Detector index qa_path (:obj:`str`, optional): Directory for QA output. spat_flexure (float, optional): If input, the slitmask and slit edges are shifted prior to tilt analysis. + measured_fwhms (`numpy.ndarray`_, optional): + FWHM of the arc lines measured during wavecalib. If provided, this + will be used for arc/sky line detection. + Attributes: spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): - ?? + The `Spectrograph` instance that sets the instrument used. steps (list): - ?? + List of the processing steps performed mask (`numpy.ndarray`_): boolean array; True = Ignore this slit all_trcdict (list): @@ -304,7 +323,7 @@ class BuildWaveTilts: # TODO This needs to be modified to take an inmask def __init__(self, mstilt, slits, spectrograph, par, wavepar, det=1, qa_path=None, - spat_flexure=None): + spat_flexure=None, measured_fwhms=None): # TODO: Perform type checking self.spectrograph = spectrograph @@ -316,13 +335,14 @@ def __init__(self, mstilt, slits, spectrograph, par, wavepar, det=1, qa_path=Non self.det = det self.qa_path = qa_path self.spat_flexure = spat_flexure + self.measured_fwhms = measured_fwhms if measured_fwhms is not None else np.array([None] * slits.nslits) # Get the non-linear count level - # TODO: This is currently hacked to deal with Mosaics - try: + if self.mstilt.is_mosaic: + # if this is a mosaic we take the maximum value among all the detectors + self.nonlinear_counts = np.max([rawdets.nonlinear_counts() for rawdets in self.mstilt.detector.detectors]) + else: self.nonlinear_counts = self.mstilt.detector.nonlinear_counts() - except: - self.nonlinear_counts = 1e10 # Set the slitmask and slit boundary related attributes that the # code needs for execution. This also deals with arcimages that @@ -385,23 +405,26 @@ def extract_arcs(self): return arccen, arccen_bpm - def find_lines(self, arcspec, slit_cen, slit_idx, bpm=None, debug=False): + def find_lines(self, arcspec, slit_cen, slit_idx, fwhm, bpm=None, debug=False): """ Find the lines for tracing Wrapper to tracewave.tilts_find_lines() Args: - arcspec (): - ?? - slit_cen (): - ?? + arcspec (`numpy.ndarray`_): + 1D spectrum to be searched for significant detections. + slit_cen (`numpy.ndarray`_): + Trace down the center of the slit. Must match the shape of arcspec. slit_idx (int): - Slit index, zero-based + Slit index, zero-based. + fwhm (float): + FWHM of the arc lines. bpm (`numpy.ndarray`_, optional): - ?? + Bad-pixel mask for input spectrum. If None, all pixels considered good. + If passed, must match the shape of arcspec. debug (bool, optional): - ?? + Show a QA plot for the line detection. Returns: tuple: 2 objectcs @@ -423,7 +446,7 @@ def find_lines(self, arcspec, slit_cen, slit_idx, bpm=None, debug=False): sig_neigh=self.par['sig_neigh'], nfwhm_neigh=self.par['nfwhm_neigh'], only_these_lines=only_these_lines, - fwhm=self.wavepar['fwhm'], + fwhm=fwhm, nonlinear_counts=self.nonlinear_counts, bpm=bpm, debug_peaks=False, debug_lines=debug) @@ -478,7 +501,7 @@ def fit_tilts(self, trc_tilt_dict, thismask, slit_cen, spat_order, spec_order, s self.steps.append(inspect.stack()[0][3]) return self.all_fit_dict[slit_idx]['coeff2'] - def trace_tilts(self, arcimg, lines_spec, lines_spat, thismask, slit_cen, + def trace_tilts(self, arcimg, lines_spec, lines_spat, thismask, slit_cen, fwhm, debug_pca=False, show_tracefits=False): """ Trace the tilts @@ -500,6 +523,12 @@ def trace_tilts(self, arcimg, lines_spec, lines_spat, thismask, slit_cen, is (nspec, nspat) with dtype=bool. slit_cen (:obj:`int`): Integer index indicating the slit in question. + fwhm (:obj:`float`): + FWHM of the arc lines. + debug_pca (:obj:`bool`, optional): + Show the PCA modeling QA plots. + show_tracefits (:obj:`bool`, optional): + Show the trace fits. Returns: dict: Dictionary containing information on the traced tilts required @@ -507,7 +536,7 @@ def trace_tilts(self, arcimg, lines_spec, lines_spat, thismask, slit_cen, """ trace_dict = tracewave.trace_tilts(arcimg, lines_spec, lines_spat, thismask, slit_cen, - inmask=self.gpm, fwhm=self.wavepar['fwhm'], + inmask=self.gpm, fwhm=fwhm, spat_order=self.par['spat_order'], maxdev_tracefit=self.par['maxdev_tracefit'], sigrej_trace=self.par['sigrej_trace'], @@ -561,11 +590,13 @@ def model_arc_continuum(self, debug=False): for i in range(nslits): if self.tilt_bpm[i]: continue + # get FWHM for this slit + fwhm = autoid.set_fwhm(self.wavepar, measured_fwhm=self.measured_fwhms[i]) # TODO: What to do with the following iter_continuum parameters?: # sigthresh, sigrej, niter_cont, cont_samp, cont_frac_fwhm arc_continuum[:,i], arc_fitmask[:,i] \ = arc.iter_continuum(self.arccen[:,i], gpm=np.invert(self.arccen_bpm[:,i]), - fwhm=self.wavepar['fwhm']) + fwhm=fwhm) # TODO: Original version. Please leave it for now. # arc_fitmask[:,i], coeff \ # = utils.robust_polyfit_djs(spec, self.arccen[:,i], self.par['cont_order'], @@ -677,6 +708,7 @@ def run(self, doqa=True, debug=False, show=False): # Subtract arc continuum _mstilt = self.mstilt.image.copy() if self.par['rm_continuum']: + msgs.info('Subtracting the continuum') continuum = self.model_arc_continuum(debug=debug) _mstilt -= continuum if debug: @@ -713,11 +745,13 @@ def run(self, doqa=True, debug=False, show=False): self.slits.mask[slit_idx] = self.slits.bitmask.turn_on(self.slits.mask[slit_idx], 'BADTILTCALIB') continue msgs.info(f'Computing tilts for slit/order {self.slits.slitord_id[slit_idx]} ({slit_idx+1}/{self.slits.nslits})') + # Get the arc FWHM for this slit + fwhm = autoid.set_fwhm(self.wavepar, measured_fwhm=self.measured_fwhms[slit_idx], verbose=True) # Identify lines for tracing tilts msgs.info('Finding lines for tilt analysis') self.lines_spec, self.lines_spat \ = self.find_lines(self.arccen[:,slit_idx], self.slitcen[:,slit_idx], - slit_idx, + slit_idx, fwhm, bpm=self.arccen_bpm[:,slit_idx], debug=debug) if self.lines_spec is None: @@ -733,7 +767,7 @@ def run(self, doqa=True, debug=False, show=False): # each line. msgs.info('Trace the tilts') self.trace_dict = self.trace_tilts(_mstilt, self.lines_spec, self.lines_spat, - thismask, self.slitcen[:, slit_idx]) + thismask, self.slitcen[:, slit_idx], fwhm) # IF there are < 2 usable arc lines for tilt tracing, PCA fit does not work and the reduction crushes # TODO investigate why some slits have <2 usable arc lines if np.sum(self.trace_dict['use_tilt']) < 2: diff --git a/setup.cfg b/setup.cfg index 1c0ac31359..294a1e1615 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,6 @@ classifiers = Natural Language :: English Operating System :: OS Independent Programming Language :: Python - Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 Topic :: Documentation :: Sphinx @@ -29,28 +28,28 @@ classifiers = zip_safe = False use_2to3=False packages = find: -python_requires = >=3.10,<3.13 +python_requires = >=3.11,<3.13 setup_requires = setuptools_scm include_package_data = True install_requires = - numpy>=1.23,<2.0.0 + numpy>=1.24 astropy>=6.0 - extension-helpers>=0.1 - packaging>=0.19 - scipy>=1.7 + extension-helpers>=1.0 + packaging>=22.0 + scipy>=1.9 matplotlib>=3.7 - PyYAML>=5.1 + PyYAML>=6.0 PyERFA>=2.0.0 fast-histogram>=0.11 configobj>=5.0.6 - scikit-learn>=1.0 - IPython>=7.10.0 + scikit-learn>=1.2 + IPython>=8.0.0 ginga>=5.1.0 - linetools>=0.3.1 - qtpy>=2.0.1 + linetools>=0.3.2 + qtpy>=2.2.0 pygithub bottleneck - pyqt6<=6.7.0 + pyqt6 scripts = bin/pypeit_c_enabled bin/pypeit_chk_plugins @@ -64,12 +63,12 @@ scripts = [options.extras_require] scikit-image = - scikit-image + scikit-image>=0.23 specutils = specutils>=1.13 test = pygit2 - pytest>=6.0.0 + pytest>=7.0.0 pytest-astropy tox pytest-cov @@ -84,12 +83,12 @@ devsuite = pytest-qt dev = # scikit-image - scikit-image + scikit-image>=0.23 # specutils specutils>=1.13 # test pygit2 - pytest>=6.0.0 + pytest>=7.0.0 pytest-astropy tox pytest-cov @@ -110,6 +109,7 @@ console_scripts = pypeit_arxiv_solution = pypeit.scripts.arxiv_solution:ArxivSolution.entry_point pypeit_cache_github_data = pypeit.scripts.cache_github_data:CacheGithubData.entry_point pypeit_clean_cache = pypeit.scripts.clean_cache:CleanCache.entry_point + pypeit_chk_flexure = pypeit.scripts.chk_flexure:ChkFlexure.entry_point pypeit_chk_for_calibs = pypeit.scripts.chk_for_calibs:ChkForCalibs.entry_point pypeit_chk_noise_1dspec = pypeit.scripts.chk_noise_1dspec:ChkNoise1D.entry_point pypeit_chk_noise_2dspec = pypeit.scripts.chk_noise_2dspec:ChkNoise2D.entry_point @@ -119,6 +119,7 @@ console_scripts = pypeit_coadd_datacube = pypeit.scripts.coadd_datacube:CoAddDataCube.entry_point pypeit_collate_1d = pypeit.scripts.collate_1d:Collate1D.entry_point pypeit_edge_inspector = pypeit.scripts.edge_inspector:EdgeInspector.entry_point + pypeit_extract_datacube = pypeit.scripts.extract_datacube:ExtractDataCube.entry_point pypeit_flux_calib = pypeit.scripts.flux_calib:FluxCalib.entry_point pypeit_flux_setup = pypeit.scripts.flux_setup:FluxSetup.entry_point pypeit_install_extinctfile = pypeit.scripts.install_extinctfile:InstallExtinctfile.entry_point @@ -168,6 +169,7 @@ console_scripts = pypeit_skysub_regions = pypeit.scripts.skysub_regions:SkySubRegions.entry_point pypeit_view_fits = pypeit.scripts.view_fits:ViewFits.entry_point pypeit_setup_gui = pypeit.scripts.setup_gui:SetupGUI.entry_point + pypeit_show_pixflat = pypeit.scripts.show_pixflat:ShowPixFlat.entry_point ginga.rv.plugins = SlitWavelength = pypeit.display:setup_SlitWavelength diff --git a/tox.ini b/tox.ini index 55bbe46e07..c8b7d51210 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,12 @@ [tox] envlist = - {3.10,3.11,3.12}-test{,-alldeps,-shapely,-specutils}{,-cov} - {3.10,3.11,3.12}-test-numpy{123,124,125,126} - {3.10,3.11,3.12}-test-{numpy,astropy,linetools,ginga}dev + {3.11,3.12}-test{,-alldeps,-shapely,-specutils}{,-cov} + {3.11,3.12}-test-numpy{124,125,126,200,201} + {3.11,3.12}-test-{numpy,astropy,linetools,ginga}dev codestyle requires = - setuptools >= 30.3.0 - pip >= 19.3.1 + setuptools >= 65.0 + pip >= 22.0 isolated_build = true [testenv] @@ -36,19 +36,21 @@ description = devdeps: with the latest developer version of key dependencies oldestdeps: with the oldest supported version of key dependencies cov: and test coverage - numpy123: with numpy 1.23.* numpy124: with numpy 1.24.* numpy125: with numpy 1.25.* numpy126: with numpy 1.26.* + numpy200: with numpy 2.0.* + numpy201: with numpy 2.1.* # The following provides some specific pinnings for key packages deps = cov: coverage - numpy123: numpy==1.23.* numpy124: numpy==1.24.* numpy125: numpy==1.25.* numpy126: numpy==1.26.* + numpy200: numpy==2.0.* + numpy201: numpy==2.1.* numpydev: numpy>=0.0.dev0 astropydev: git+https://github.com/astropy/astropy.git#egg=astropy