From 45420e6ebcc85b0f6f243d7ed528dcabea532c13 Mon Sep 17 00:00:00 2001 From: tgalvin Date: Mon, 8 Apr 2024 15:10:20 +0800 Subject: [PATCH 1/8] initial commit of MaskingOptions --- flint/masking.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/flint/masking.py b/flint/masking.py index 132aabfc..57eb2767 100644 --- a/flint/masking.py +++ b/flint/masking.py @@ -4,7 +4,7 @@ from argparse import ArgumentParser from pathlib import Path -from typing import Optional, Tuple +from typing import Optional, Tuple, NamedTuple import numpy as np from astropy.io import fits @@ -24,6 +24,34 @@ from flint.naming import FITSMaskNames, create_fits_mask_names +class MaskingOptions(NamedTuple): + """Contains options for the creation of clean masks from some subject + image. Clipping levels specified are in units of RMS (or sigma). They + are NOT in absolute units. + """ + + base_snr_clip: float = 4 + """A base clipping level to be used should other options not be activated""" + flood_fill: bool = True + """Whether to attempt to flood fill when constructing a mask""" + flood_fill_positive_seed_clip: float = 4.5 + """The clipping level to seed islands that will be grown to lower SNR""" + flood_fill_positive_flood_clip: float = 1.5 + """Clipping level used to grow seeded islands down to""" + suppress_artefacts: bool = True + """Whether to attempt artefacts based on the presence of sigificant negatives""" + suppress_artefacts_negative_seed_clip: Optional[float] = 5 + """The significance level of a negative island for the sidelobe suppresion to be activated. This should be a positive number (the signal map is internally inverted)""" + supress_artefacts_guard_negative_dilation: float = 40 + """The minimum positive signifance pixels should have to be guarded when attempting to suppress artefacts around bright sources""" + grow_low_snr_islands: bool = True + """Whether to attempt to grow a mask to capture islands of low SNR (e.g. diffuse emission)""" + grow_low_snr_clip: float = 1.75 + """The minimum signifance levels of pixels to be to seed low SNR islands for consideration""" + grow_low_snr_island_size: int = 512 + """The number of pixels an island has to be for it to be accepted""" + + def extract_beam_mask_from_mosaic( fits_beam_image_path: Path, fits_mosaic_mask_names: FITSMaskNames ) -> FITSMaskNames: From 00f943c2c18bdd5641dae7bea306aa164d55157f Mon Sep 17 00:00:00 2001 From: tgalvin Date: Mon, 8 Apr 2024 15:14:38 +0800 Subject: [PATCH 2/8] removed butterworth filter - dead code and no good in practise --- flint/masking.py | 125 ++-------------------- flint/prefect/common/imaging.py | 27 ++--- flint/prefect/flows/continuum_pipeline.py | 8 -- 3 files changed, 15 insertions(+), 145 deletions(-) diff --git a/flint/masking.py b/flint/masking.py index 57eb2767..3549620d 100644 --- a/flint/masking.py +++ b/flint/masking.py @@ -4,7 +4,7 @@ from argparse import ArgumentParser from pathlib import Path -from typing import Optional, Tuple, NamedTuple +from typing import Optional, NamedTuple import numpy as np from astropy.io import fits @@ -17,8 +17,6 @@ binary_erosion as scipy_binary_erosion, # Rename to distinguish from skimage ) from scipy.ndimage import label -from skimage.filters import butterworth -from skimage.morphology import binary_erosion from flint.logging import logger from flint.naming import FITSMaskNames, create_fits_mask_names @@ -298,103 +296,6 @@ def reverse_negative_flood_fill( return positive_dilated_mask.astype(np.int32) -def create_snr_mask_wbutter_from_fits( - fits_image_path: Path, - fits_rms_path: Path, - fits_bkg_path: Path, - create_signal_fits: bool = False, - min_snr: float = 5, - connectivity_shape: Tuple[int, int] = (4, 4), - overwrite: bool = True, -) -> FITSMaskNames: - """Create a mask for an input FITS image based on a signal to noise given a corresponding pair of RMS and background FITS images. - - Internally the signal image is computed as something akin to: - > signal = (image - background) / rms - - Before deriving a signal map the image is first smoothed using a butterworth filter, and a - crude rescaling factor is applied based on the ratio of the maximum pixel values before and - after the smoothing is applied. - - This is done in a staged manner to minimise the number of (potentially large) images - held in memory. - - Each of the input images needs to share the same shape. This means that compression - features offered by some tooling (e.g. BANE --compress) can not be used. - - Once the signal map as been computed, all pixels below ``min_snr`` are flagged. The resulting - islands then have a binary erosion applied to contract the resultingn islands. - - Args: - fits_image_path (Path): Path to the FITS file containing an image - fits_rms_path (Path): Path to the FITS file with an RMS image corresponding to ``fits_image_path`` - fits_bkg_path (Path): Path to the FITS file with an baclground image corresponding to ``fits_image_path`` - create_signal_fits (bool, optional): Create an output signal map. Defaults to False. - min_snr (float, optional): Minimum signal-to-noise ratio for the masking to include a pixel. Defaults to 3.5. - connectivity_shape (Tuple[int, int], optional): The connectivity matrix used in the scikit-image binary erosion applied to the mask. Defaults to (4, 4). - overwrite (bool): Passed to `fits.writeto`, and will overwrite files should they exist. Defaults to True. - - Returns: - FITSMaskNames: Container describing the signal and mask FITS image paths. If ``create_signal_path`` is None, then the ``signal_fits`` attribute will be None. - """ - logger.info(f"Creating a mask image with SNR>{min_snr:.2f}") - mask_names = create_fits_mask_names( - fits_image=fits_image_path, include_signal_path=create_signal_fits - ) - - with fits.open(fits_image_path) as fits_image: - fits_header = fits_image[0].header - - image_max = np.nanmax(fits_image[0].data) - image_butter = butterworth( - np.nan_to_num(np.squeeze(fits_image[0].data)), 0.045, high_pass=False - ) - - butter_max = np.nanmax(image_butter) - scale_ratio = image_max / butter_max - logger.info(f"Scaling smoothed image by {scale_ratio:.4f}") - image_butter *= image_max / butter_max - - with fits.open(fits_bkg_path) as fits_bkg: - logger.info("Subtracting background") - signal_data = image_butter - np.squeeze(fits_bkg[0].data) - - with fits.open(fits_rms_path) as fits_rms: - logger.info("Dividing by RMS") - signal_data /= np.squeeze(fits_rms[0].data) - - if create_signal_fits: - logger.info(f"Writing {mask_names.signal_fits}") - fits.writeto( - filename=mask_names.signal_fits, - data=signal_data, - header=fits_header, - overwrite=overwrite, - ) - - # Following the help in wsclean: - # WSClean accepts masks in CASA format and in fits file format. A mask is a - # normal, single polarization image file, where all zero values are interpreted - # as being not masked, and all non-zero values are interpreted as masked. In the - # case of a fits file, the file may either contain a single frequency or it may - # contain a cube of images. - logger.info(f"Clipping using a {min_snr=}") - mask_data = (signal_data > min_snr).astype(np.int32) - - logger.info(f"Applying binary erosion with {connectivity_shape=}") - mask_data = binary_erosion(mask_data, np.ones(connectivity_shape)) - - logger.info(f"Writing {mask_names.mask_fits}") - fits.writeto( - filename=mask_names.mask_fits, - data=mask_data.astype(np.int32), - header=fits_header, - overwrite=overwrite, - ) - - return mask_names - - def create_snr_mask_from_fits( fits_image_path: Path, fits_rms_path: Path, @@ -557,23 +458,13 @@ def cli(): args = parser.parse_args() if args.mode == "snrmask": - if args.user_butterworth: - create_snr_mask_wbutter_from_fits( - fits_image_path=args.image, - fits_rms_path=args.rms, - fits_bkg_path=args.bkg, - create_signal_fits=args.save_signal, - min_snr=args.min_snr, - connectivity_shape=tuple(args.connectivity_shape), - ) - else: - create_snr_mask_from_fits( - fits_image_path=args.image, - fits_rms_path=args.rms, - fits_bkg_path=args.bkg, - create_signal_fits=args.save_signal, - min_snr=args.min_snr, - ) + create_snr_mask_from_fits( + fits_image_path=args.image, + fits_rms_path=args.rms, + fits_bkg_path=args.bkg, + create_signal_fits=args.save_signal, + min_snr=args.min_snr, + ) elif args.mode == "extractmask": extract_beam_mask_from_mosaic( diff --git a/flint/prefect/common/imaging.py b/flint/prefect/common/imaging.py index a5be6b88..2dc91ab5 100644 --- a/flint/prefect/common/imaging.py +++ b/flint/prefect/common/imaging.py @@ -24,7 +24,6 @@ from flint.logging import logger from flint.masking import ( create_snr_mask_from_fits, - create_snr_mask_wbutter_from_fits, extract_beam_mask_from_mosaic, ) from flint.ms import MS, preprocess_askap_ms, rename_column_in_ms, split_by_field @@ -462,7 +461,6 @@ def task_create_image_mask_model( image: Union[LinmosCommand, ImageSet, WSCleanCommand], image_products: AegeanOutputs, min_snr: Optional[float] = 3.5, - with_butterworth: bool = False, ) -> FITSMaskNames: """Create a mask from a linmos image, with the intention of providing it as a clean mask to an appropriate imager. This is derived using a simple signal to noise cut. @@ -471,7 +469,6 @@ def task_create_image_mask_model( linmos_parset (LinmosCommand): Linmos command and associated meta-data image_products (AegeanOutputs): Images of the RMS and BKG min_snr (float, optional): The minimum S/N a pixel should be for it to be included in the clean mask. - with_butterworth (bool, optional): whether to taper the input image with a Butterworth filter before masking. Raises: ValueError: Raised when ``image_products`` are not known @@ -502,23 +499,13 @@ def task_create_image_mask_model( logger.info(f"Using {source_rms=}") logger.info(f"Using {source_bkg=}") - if with_butterworth: - mask_names = create_snr_mask_wbutter_from_fits( - fits_image_path=source_image, - fits_bkg_path=source_bkg, - fits_rms_path=source_rms, - create_signal_fits=True, - min_snr=min_snr, - connectivity_shape=(3, 3), - ) - else: - mask_names = create_snr_mask_from_fits( - fits_image_path=source_image, - fits_bkg_path=source_bkg, - fits_rms_path=source_rms, - create_signal_fits=True, - min_snr=min_snr, - ) + mask_names = create_snr_mask_from_fits( + fits_image_path=source_image, + fits_bkg_path=source_bkg, + fits_rms_path=source_rms, + create_signal_fits=True, + min_snr=min_snr, + ) logger.info(f"Created {mask_names.mask_fits}") diff --git a/flint/prefect/flows/continuum_pipeline.py b/flint/prefect/flows/continuum_pipeline.py index d9505deb..92794f63 100644 --- a/flint/prefect/flows/continuum_pipeline.py +++ b/flint/prefect/flows/continuum_pipeline.py @@ -263,7 +263,6 @@ def process_science_fields( image=wsclean_cmds, image_products=beam_aegean_outputs, min_snr=3.5, - with_butterworth=field_options.use_beam_mask_wbutterworth, ) wsclean_options["auto_mask"] = 1.25 wsclean_options["auto_threshold"] = 1.0 @@ -482,12 +481,6 @@ def get_parser() -> ArgumentParser: default=False, help="Whether to use (or search for solutions with) the preflagger operations applied to the bandpass gain solutions", ) - parser.add_argument( - "--use-smoothed", - action="store_true", - default=False, - help="Whether to use (or search for solutions with) the smoothing operations applied to the bandpass gain solutions", - ) parser.add_argument( "--use-beam-masks", default=False, @@ -537,7 +530,6 @@ def cli() -> None: beam_cutoff=args.beam_cutoff, pb_cutoff=args.pb_cutoff, use_preflagger=args.use_preflagger, - use_smoothed=args.use_smoothed, use_beam_masks=args.use_beam_masks, use_beam_masks_from=args.use_beam_masks_from, use_beam_mask_wbutterworth=args.use_beam_masks_wbutterworth, From b9905bc9a5379880bb133d74e3ff26a1e59eccd8 Mon Sep 17 00:00:00 2001 From: tgalvin Date: Mon, 8 Apr 2024 17:17:31 +0800 Subject: [PATCH 3/8] plugging in MaskingOptions --- flint/masking.py | 50 +++++++++++++++++++-------------- flint/prefect/common/imaging.py | 8 ++++++ tests/test_masking.py | 17 ++++++++++- tests/test_options.py | 6 +--- 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/flint/masking.py b/flint/masking.py index 3549620d..8589310b 100644 --- a/flint/masking.py +++ b/flint/masking.py @@ -2,6 +2,8 @@ thought being towards FITS images. """ +from __future__ import annotations + from argparse import ArgumentParser from pathlib import Path from typing import Optional, NamedTuple @@ -31,7 +33,7 @@ class MaskingOptions(NamedTuple): base_snr_clip: float = 4 """A base clipping level to be used should other options not be activated""" flood_fill: bool = True - """Whether to attempt to flood fill when constructing a mask""" + """Whether to attempt to flood fill when constructing a mask. This should be `True` for `grow_low_snr_islands` and `suppress_artefacts to have an effect. """ flood_fill_positive_seed_clip: float = 4.5 """The clipping level to seed islands that will be grown to lower SNR""" flood_fill_positive_flood_clip: float = 1.5 @@ -40,15 +42,22 @@ class MaskingOptions(NamedTuple): """Whether to attempt artefacts based on the presence of sigificant negatives""" suppress_artefacts_negative_seed_clip: Optional[float] = 5 """The significance level of a negative island for the sidelobe suppresion to be activated. This should be a positive number (the signal map is internally inverted)""" - supress_artefacts_guard_negative_dilation: float = 40 + suppress_artefacts_guard_negative_dilation: float = 40 """The minimum positive signifance pixels should have to be guarded when attempting to suppress artefacts around bright sources""" grow_low_snr_islands: bool = True """Whether to attempt to grow a mask to capture islands of low SNR (e.g. diffuse emission)""" grow_low_snr_clip: float = 1.75 """The minimum signifance levels of pixels to be to seed low SNR islands for consideration""" - grow_low_snr_island_size: int = 512 + grow_low_snr_island_size: int = 768 """The number of pixels an island has to be for it to be accepted""" + def with_options(self, **kwargs) -> MaskingOptions: + """Return a new instance of the MaskingOptions""" + _dict = self._asdict() + _dict.update(**kwargs) + + return MaskingOptions(**_dict) + def extract_beam_mask_from_mosaic( fits_beam_image_path: Path, fits_mosaic_mask_names: FITSMaskNames @@ -191,7 +200,8 @@ def reverse_negative_flood_fill( signal: Optional[np.ndarray] = None, positive_seed_clip: float = 4, positive_flood_clip: float = 2, - negative_seed_clip: Optional[float] = 5, + suppress_artefacts: bool = False, + negative_seed_clip: float = 5, guard_negative_dilation: float = 50, grow_low_snr: Optional[float] = 2, grow_low_island_size: int = 512, @@ -231,6 +241,7 @@ def reverse_negative_flood_fill( signal(Optional[np.ndarray], optional): A signal map. Defaults to None. positive_seed_clip (float, optional): Initial clip of the mask before islands are grown. Defaults to 4. positive_flood_clip (float, optional): Pixels above `positive_seed_clip` are dilated to this threshold. Defaults to 2. + suppress_artefacts (boo, optional): Attempt to suppress regions around presumed artefacts. Defaults to False. negative_seed_clip (Optional[float], optional): Initial clip of negative pixels. This operation is on the inverted signal mask (so this value should be a positive number). If None this second operation is not performed. Defaults to 5. guard_negative_dilation (float, optional): Positive pixels from the computed signal mask will be above this threshold to be protect from the negative island mask dilation. Defaults to 50. grow_low__snr (Optional[float], optional): Attempt to grow islands of contigous pixels above thius low SNR ration. If None this is not performed. Defaults to 2. @@ -271,8 +282,7 @@ def reverse_negative_flood_fill( # - if there are brightish negative islands there is also positive brightish arteefact islands nearby # For this reason the guard mask should be sufficently high to protect the # main source but nuke the fask positive islands - negative_dilated_mask = None - if negative_seed_clip: + if suppress_artefacts: negative_mask = negative_signal > negative_seed_clip negative_dilated_mask = scipy_binary_dilation( input=negative_mask, @@ -300,9 +310,8 @@ def create_snr_mask_from_fits( fits_image_path: Path, fits_rms_path: Path, fits_bkg_path: Path, + masking_options: MaskingOptions, create_signal_fits: bool = False, - min_snr: float = 3.5, - attempt_reverse_nergative_flood_fill: bool = True, overwrite: bool = True, ) -> FITSMaskNames: """Create a mask for an input FITS image based on a signal to noise given a corresponding pair of RMS and background FITS images. @@ -322,15 +331,13 @@ def create_snr_mask_from_fits( fits_image_path (Path): Path to the FITS file containing an image fits_rms_path (Path): Path to the FITS file with an RMS image corresponding to ``fits_image_path`` fits_bkg_path (Path): Path to the FITS file with an baclground image corresponding to ``fits_image_path`` + masking_options (MaskingOptions): Configurables on the masking operation procedure. create_signal_fits (bool, optional): Create an output signal map. Defaults to False. - min_snr (float, optional): Minimum signal-to-noise ratio for the masking to include a pixel. Defaults to 3.5. - attempt_negative_flood_fill (bool): Attempt to filter out negative sidelobes from the bask. See `reverse_negative_flood_fill`. Defaults to True. overwrite (bool): Passed to `fits.writeto`, and will overwrite files should they exist. Defaults to True. Returns: FITSMaskNames: Container describing the signal and mask FITS image paths. If ``create_signal_path`` is None, then the ``signal_fits`` attribute will be None. """ - logger.info(f"Creating a mask image with SNR>{min_snr:.2f}") mask_names = create_fits_mask_names( fits_image=fits_image_path, include_signal_path=create_signal_fits ) @@ -360,20 +367,20 @@ def create_snr_mask_from_fits( # as being not masked, and all non-zero values are interpreted as masked. In the # case of a fits file, the file may either contain a single frequency or it may # contain a cube of images. - if attempt_reverse_nergative_flood_fill: + if masking_options.flood_fill: mask_data = reverse_negative_flood_fill( signal=np.squeeze(signal_data), - positive_seed_clip=4, - positive_flood_clip=1.5, - negative_seed_clip=4, - guard_negative_dilation=30, - grow_low_snr=1.75, - grow_low_island_size=768, + positive_seed_clip=masking_options.flood_fill_positive_seed_clip, + positive_flood_clip=masking_options.flood_fill_positive_flood_clip, + negative_seed_clip=masking_options.suppress_artefacts_negative_seed_clip, + guard_negative_dilation=masking_options.suppress_artefacts_guard_negative_dilation, + grow_low_snr=masking_options.grow_low_snr_clip, + grow_low_island_size=masking_options.grow_low_snr_island_size, ) mask_data = mask_data.reshape(signal_data.shape) else: - logger.info(f"Clipping using a {min_snr=}") - mask_data = (signal_data > min_snr).astype(np.int32) + logger.info(f"Clipping using a {masking_options.base_snr_clip=}") + mask_data = (signal_data > masking_options.base_snr_clip).astype(np.int32) logger.info(f"Writing {mask_names.mask_fits}") fits.writeto( @@ -458,12 +465,13 @@ def cli(): args = parser.parse_args() if args.mode == "snrmask": + masking_options = MaskingOptions(base_snr_clip=args.min_snr) create_snr_mask_from_fits( fits_image_path=args.image, fits_rms_path=args.rms, fits_bkg_path=args.bkg, create_signal_fits=args.save_signal, - min_snr=args.min_snr, + masking_options=masking_options, ) elif args.mode == "extractmask": diff --git a/flint/prefect/common/imaging.py b/flint/prefect/common/imaging.py index 2dc91ab5..5cfa599e 100644 --- a/flint/prefect/common/imaging.py +++ b/flint/prefect/common/imaging.py @@ -25,6 +25,7 @@ from flint.masking import ( create_snr_mask_from_fits, extract_beam_mask_from_mosaic, + MaskingOptions, ) from flint.ms import MS, preprocess_askap_ms, rename_column_in_ms, split_by_field from flint.naming import FITSMaskNames, processed_ms_format @@ -461,6 +462,7 @@ def task_create_image_mask_model( image: Union[LinmosCommand, ImageSet, WSCleanCommand], image_products: AegeanOutputs, min_snr: Optional[float] = 3.5, + update_masking_options: Optional[Dict[str, Any]] = None, ) -> FITSMaskNames: """Create a mask from a linmos image, with the intention of providing it as a clean mask to an appropriate imager. This is derived using a simple signal to noise cut. @@ -469,6 +471,8 @@ def task_create_image_mask_model( linmos_parset (LinmosCommand): Linmos command and associated meta-data image_products (AegeanOutputs): Images of the RMS and BKG min_snr (float, optional): The minimum S/N a pixel should be for it to be included in the clean mask. + update_masking_options (Optional[Dict[str,Any]], optional): Updated options supplied to the default MaskingOptions. Defaults to None. + Raises: ValueError: Raised when ``image_products`` are not known @@ -499,6 +503,10 @@ def task_create_image_mask_model( logger.info(f"Using {source_rms=}") logger.info(f"Using {source_bkg=}") + masking_options = MaskingOptions() + if update_masking_options: + masking_options = masking_options.with_options(**update_masking_options) + mask_names = create_snr_mask_from_fits( fits_image_path=source_image, fits_bkg_path=source_bkg, diff --git a/tests/test_masking.py b/tests/test_masking.py index aa3186dc..d7a3a10b 100644 --- a/tests/test_masking.py +++ b/tests/test_masking.py @@ -4,7 +4,7 @@ import pytest from astropy.io import fits -from flint.masking import create_snr_mask_from_fits +from flint.masking import create_snr_mask_from_fits, MaskingOptions from flint.naming import FITSMaskNames SHAPE = (100, 100) @@ -28,11 +28,24 @@ def fits_dir(tmpdir): return fits_dir +def test_make_masking_options(): + """Just a dump test to make sure the options structure is ok""" + + masking_options = MaskingOptions() + + assert masking_options.base_snr_clip != -1 + + masking_options = masking_options.with_options(base_snr_clip=-1) + assert masking_options.base_snr_clip == -1 + + def test_fits_masking(fits_dir): + masking_options = MaskingOptions(flood_fill=False) names = create_snr_mask_from_fits( fits_image_path=fits_dir / "image.fits", fits_rms_path=fits_dir / "rms.fits", fits_bkg_path=fits_dir / "bkg.fits", + masking_options=masking_options, ) assert isinstance(names, FITSMaskNames) @@ -45,10 +58,12 @@ def test_fits_masking(fits_dir): def test_fits_masking_with_signal(fits_dir): + masking_options = MaskingOptions(flood_fill=False) names = create_snr_mask_from_fits( fits_image_path=fits_dir / "image.fits", fits_rms_path=fits_dir / "rms.fits", fits_bkg_path=fits_dir / "bkg.fits", + masking_options=masking_options, create_signal_fits=True, ) diff --git a/tests/test_options.py b/tests/test_options.py index 03452dd3..b04009d0 100644 --- a/tests/test_options.py +++ b/tests/test_options.py @@ -2,6 +2,7 @@ somewhat tracked, especially when using an argparse object to create it """ + from pathlib import Path from flint.options import FieldOptions @@ -46,12 +47,10 @@ def test_create_field_options(): beam_cutoff=args.beam_cutoff, pb_cutoff=args.pb_cutoff, use_preflagger=args.use_preflagger, - use_smoothed=args.use_smoothed, ) assert isinstance(field_options, FieldOptions) assert field_options.use_preflagger is False - assert field_options.use_smoothed is False assert field_options.zip_ms is True assert field_options.linmos_residuals is True assert field_options.rounds == 2 @@ -75,7 +74,6 @@ def test_create_field_options2(): --aegean-container '/scratch3/gal16b/containers/aegean.sif' --reference-catalogue-directory '/scratch3/gal16b/reference_catalogues/' --use-preflagger - --use-smoothed """.split() ) @@ -96,12 +94,10 @@ def test_create_field_options2(): beam_cutoff=args.beam_cutoff, pb_cutoff=args.pb_cutoff, use_preflagger=args.use_preflagger, - use_smoothed=args.use_smoothed, ) assert isinstance(field_options, FieldOptions) assert field_options.use_preflagger is True - assert field_options.use_smoothed is True assert field_options.zip_ms is False assert field_options.linmos_residuals is False assert field_options.rounds == 2 From e2da83997e4499a42d5463143499aa4ca7c5f26f Mon Sep 17 00:00:00 2001 From: tgalvin Date: Mon, 8 Apr 2024 18:06:54 +0800 Subject: [PATCH 4/8] changed default use_msoothed field_option --- flint/options.py | 4 +--- flint/prefect/flows/continuum_pipeline.py | 7 ------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/flint/options.py b/flint/options.py index 39db2cf6..b6c0d6a8 100644 --- a/flint/options.py +++ b/flint/options.py @@ -90,11 +90,9 @@ class FieldOptions(NamedTuple): """Primary beam attentuation cutoff to use during linmos""" use_preflagger: bool = True """Whether to apply (or search for solutions with) bandpass solutions that have gone through the preflagging operations""" - use_smoothed: bool = True + use_smoothed: bool = False """Whether to apply (or search for solutions with) a bandpass smoothing operation applied""" use_beam_masks: bool = True """Construct beam masks from MFS images to use for the next round of imaging. """ use_beam_masks_from: int = 2 """If `use_beam_masks` is True, start using them from this round of self-calibration""" - use_beam_mask_wbutterworth: bool = False - """If `use_beam_masks` is True, this will specify whether a Butterworth filter is used to smooth the image before the S/N clip is applied""" diff --git a/flint/prefect/flows/continuum_pipeline.py b/flint/prefect/flows/continuum_pipeline.py index 92794f63..a7d38107 100644 --- a/flint/prefect/flows/continuum_pipeline.py +++ b/flint/prefect/flows/continuum_pipeline.py @@ -493,12 +493,6 @@ def get_parser() -> ArgumentParser: type=int, help="If --use-beam-masks is provided, this option specifies from which round of self-calibration the masking operation will be used onwards from. ", ) - parser.add_argument( - "--use-beam-masks-wbutterworth", - default=False, - action="store_true", - help="If --use-beam-masks is provided, this will specify whether a Butterworth filter is first used to smooth an image before applying the S/N cut", - ) return parser @@ -532,7 +526,6 @@ def cli() -> None: use_preflagger=args.use_preflagger, use_beam_masks=args.use_beam_masks, use_beam_masks_from=args.use_beam_masks_from, - use_beam_mask_wbutterworth=args.use_beam_masks_wbutterworth, ) setup_run_process_science_field( From bcc576ab54bbabadbe4eebc618dd4b6fcf50f44b Mon Sep 17 00:00:00 2001 From: tgalvin Date: Mon, 8 Apr 2024 19:10:40 +0800 Subject: [PATCH 5/8] calling the new masking_options argument --- flint/prefect/common/imaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flint/prefect/common/imaging.py b/flint/prefect/common/imaging.py index 5cfa599e..3e9e1364 100644 --- a/flint/prefect/common/imaging.py +++ b/flint/prefect/common/imaging.py @@ -511,8 +511,8 @@ def task_create_image_mask_model( fits_image_path=source_image, fits_bkg_path=source_bkg, fits_rms_path=source_rms, + masking_options=masking_options, create_signal_fits=True, - min_snr=min_snr, ) logger.info(f"Created {mask_names.mask_fits}") From f28720289e221611fd2b09b05a4600fb5eae6d1c Mon Sep 17 00:00:00 2001 From: tgalvin Date: Mon, 8 Apr 2024 20:35:15 +0800 Subject: [PATCH 6/8] added a guard clause --- flint/masking.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flint/masking.py b/flint/masking.py index 8589310b..85104476 100644 --- a/flint/masking.py +++ b/flint/masking.py @@ -276,6 +276,9 @@ def reverse_negative_flood_fill( structure=np.ones((3, 3)), ) + # TODO: This function should be divided up + negative_dilated_mask = None + # Now do the same but on negative islands. The assumption here is that: # - no genuine source of negative sky emission # - negative islands are around bright sources with deconvolution/calibration errors From 0e320e79cc5cf639b118506f8d787be1cf9cb6a1 Mon Sep 17 00:00:00 2001 From: tgalvin Date: Mon, 8 Apr 2024 22:22:42 +0800 Subject: [PATCH 7/8] removed unsused round of self-cal --- flint/configuration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flint/configuration.py b/flint/configuration.py index 69a45fb8..a30c9537 100644 --- a/flint/configuration.py +++ b/flint/configuration.py @@ -36,7 +36,6 @@ def get_selfcal_options_from_yaml(input_yaml: Optional[Path] = None) -> Dict: 2: {"solint": "30s", "calmode": "p", "uvrange": ">235m", "nspw": 4}, 3: {"solint": "60s", "calmode": "ap", "uvrange": ">235m", "nspw": 4}, 4: {"solint": "30s", "calmode": "ap", "uvrange": ">235m", "nspw": 4}, - 5: {"solint": "30s", "calmode": "ap", "uvrange": ">235m", "nspw": 1}, } From c39243583f36e428c3f51c0f01d9e94e19f3c5ed Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 9 Apr 2024 03:59:59 +0000 Subject: [PATCH 8/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- flint/masking.py | 2 +- flint/prefect/common/imaging.py | 2 +- tests/test_masking.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flint/masking.py b/flint/masking.py index 85104476..caea8cf0 100644 --- a/flint/masking.py +++ b/flint/masking.py @@ -6,7 +6,7 @@ from argparse import ArgumentParser from pathlib import Path -from typing import Optional, NamedTuple +from typing import NamedTuple, Optional import numpy as np from astropy.io import fits diff --git a/flint/prefect/common/imaging.py b/flint/prefect/common/imaging.py index 3e9e1364..9a4eedca 100644 --- a/flint/prefect/common/imaging.py +++ b/flint/prefect/common/imaging.py @@ -23,9 +23,9 @@ from flint.imager.wsclean import ImageSet, WSCleanCommand, wsclean_imager from flint.logging import logger from flint.masking import ( + MaskingOptions, create_snr_mask_from_fits, extract_beam_mask_from_mosaic, - MaskingOptions, ) from flint.ms import MS, preprocess_askap_ms, rename_column_in_ms, split_by_field from flint.naming import FITSMaskNames, processed_ms_format diff --git a/tests/test_masking.py b/tests/test_masking.py index d7a3a10b..0582ee7d 100644 --- a/tests/test_masking.py +++ b/tests/test_masking.py @@ -4,7 +4,7 @@ import pytest from astropy.io import fits -from flint.masking import create_snr_mask_from_fits, MaskingOptions +from flint.masking import MaskingOptions, create_snr_mask_from_fits from flint.naming import FITSMaskNames SHAPE = (100, 100)