Merge branch 'master' of https://github.com/AlecThomson/arrakis

CHANGELOG.md

@@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.1.7] - 2024-04-03
+### What's Changed
+
+Updated documentation.
+
+**Full Changelog**: https://github.com/AlecThomson/arrakis/compare/v2.1.6...v2.1.7
+
+## [2.1.6] - 2024-04-01
+### What's Changed
+* Image docs by @AlecThomson in https://github.com/AlecThomson/arrakis/pull/60
+
+
+**Full Changelog**: https://github.com/AlecThomson/arrakis/compare/v2.1.5...v2.1.6
+
 ## [2.1.5] - 2024-03-27
 ### What's Changed
 * [pre-commit.ci] pre-commit autoupdate by @pre-commit-ci in https://github.com/AlecThomson/arrakis/pull/56

arrakis/.default_config.cfg

@@ -0,0 +1,102 @@
+# Arrakis default config
+
+## Note 'None' will be interpreated as a string, not a NoneType
+## If you want to use the default value, just leave the line blank
+
+[Imaging options]
+# temp_dir: None
+# psf_cutoff: None
+robust: -0.5
+nchan: 36
+pols: 'IQU'
+size: 4096
+scale: 2.5
+mgain: 0.8
+niter: 100000
+# nmiter: None
+auto_mask: 3.0
+auto_threshold: 1.0
+local_rms: False
+# local_rms_window: None
+# force_mask_rounds: None
+# gridder: None
+# taper: None
+minuv: 0.0
+# parallel: None
+purge: False
+mpi: False
+multiscale: False
+# multiscale_scale_bias: None
+multiscale_scales: '0,2,4,8,16,32,64,128'
+# absmem: None
+make_residual_cubes: False
+ms_glob_pattern: 'scienceData*_averaged_cal.leakage.ms'
+data_column: 'CORRECTED_DATA'
+no_mf_weighting: False
+skip_fix_ms: False
+hosted-wsclean: 'docker://alecthomson/wsclean:latest'
+# local_wsclean: None
+
+[Pipeline options]
+# config: None
+epoch: 0
+# host: None
+# username: None
+# password: None
+# dask_config: None
+# imager_dask_config: None
+# holofile: None
+yanda: '1.3.0'
+# yanda_image: None
+imager_only: False
+skip_imager: False
+skip_cutout: False
+skip_linmos: False
+skip_cleanup: False
+skip_frion: False
+skip_rmsynth: False
+skip_rmclean: False
+skip_cat: False
+verbose: False
+
+[Cutout options]
+pad: 5
+dryrun: False
+
+[RMsynth options]
+dimension: '1d'
+database: False
+# tt0: None
+# tt1: None
+validate: False
+# limit: None
+own_fit: False
+savePlots: False
+weightType: 'variance'
+fit_function: 'log'
+fitRMSF: False
+# phiMax_radm2: None
+# dPhi_radm2: None
+nSamples: 5
+polyOrd: 3
+noStokesI: False
+showPlots: False
+not_RMSF: False
+rm_verbose: False
+debug: False
+
+[RMclean options]
+cutoff: -3
+maxIter: 10000
+gain: 0.1
+# window: None
+
+[FRion options]
+ionex_server: 'ftp://ftp.aiub.unibe.ch/CODE/'
+ionex_prefix: 'codg'
+# ionex_proxy_server: None
+# ionex_formatter: None
+ionex_predownload: False
+
+[Catalog options]
+# outfile: None

arrakis/cleanup.py

@@ -10,7 +10,7 @@
 from prefect import flow, get_run_logger, task, unmapped
 from tqdm.auto import tqdm
 
-from arrakis.logger import TqdmToLogger, logger
+from arrakis.logger import TqdmToLogger, UltimateHelpFormatter, logger
 from arrakis.utils.pipeline import logo_str
 
 logger.setLevel(logging.INFO)
@@ -61,10 +61,11 @@ def main(
     datadir: Path,
     overwrite: bool = False,
 ) -> None:
-    """Clean up beam images
+    """Clean up beam images flow
 
     Args:
         datadir (Path): Directory with sub dir 'cutouts'
+        overwrite (bool): Overwrite existing tarball
     """
 
     cutdir = datadir / "cutouts"
@@ -117,7 +118,7 @@ def cli():
 
     # Parse the command line options
     parser = argparse.ArgumentParser(
-        description=descStr, formatter_class=argparse.ArgumentDefaultsHelpFormatter
+        description=descStr, formatter_class=UltimateHelpFormatter
     )
     parser.add_argument(
         "outdir",

arrakis/cutout.py

@@ -26,7 +26,7 @@
 from spectral_cube.utils import SpectralCubeWarning
 from tqdm.auto import tqdm
 
-from arrakis.logger import TqdmToLogger, logger
+from arrakis.logger import TqdmToLogger, UltimateHelpFormatter, logger
 from arrakis.utils.database import get_db, test_db
 from arrakis.utils.fitsutils import fix_header
 from arrakis.utils.io import try_mkdir
@@ -397,7 +397,7 @@ def cutout_islands(
     dryrun: bool = True,
     limit: Optional[int] = None,
 ) -> None:
-    """Perform cutouts of RACS islands in parallel.
+    """Flow to cutout islands in parallel.
 
     Args:
         field (str): RACS field name.
@@ -531,7 +531,7 @@ def cutout_parser(parent_parser: bool = False) -> argparse.ArgumentParser:
     cut_parser = argparse.ArgumentParser(
         add_help=not parent_parser,
         description=descStr,
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        formatter_class=UltimateHelpFormatter,
     )
     parser = cut_parser.add_argument_group("cutout arguments")
 

arrakis/frion.py

@@ -17,7 +17,7 @@
 from FRion import correct, predict
 from prefect import flow, task, unmapped
 
-from arrakis.logger import logger
+from arrakis.logger import UltimateHelpFormatter, logger
 from arrakis.utils.database import get_db, get_field_db, test_db
 from arrakis.utils.fitsutils import getfreq
 from arrakis.utils.io import try_mkdir
@@ -202,19 +202,21 @@ def main(
     ionex_predownload: bool = False,
     limit: Optional[int] = None,
 ):
-    """Main script
+    """FRion flow
 
     Args:
         field (str): RACS field name
-        outdir (str): Output directory
+        outdir (Path): Output directory
         host (str): MongoDB host IP address
+        epoch (int): Epoch of observation
         username (str, optional): Mongo username. Defaults to None.
         password (str, optional): Mongo passwrod. Defaults to None.
         database (bool, optional): Update database. Defaults to False.
-        verbose (bool, optional): Verbose output. Defaults to True.
         ionex_server (str, optional): IONEX server. Defaults to "ftp://ftp.aiub.unibe.ch/CODE/".
         ionex_proxy_server (str, optional): Proxy server. Defaults to None.
         ionex_formatter (Union[str, Callable], optional): IONEX formatter. Defaults to "ftp.aiub.unibe.ch".
+        ionex_predownload (bool, optional): Pre-download IONEX files. Defaults to False.
+        limit (int, optional): Limit to number of islands. Defaults to None.
     """
     # Query database for data
     outdir = os.path.abspath(outdir)
@@ -352,7 +354,7 @@ def cli():
 
     # Parse the command line options
     parser = argparse.ArgumentParser(
-        description=descStr, formatter_class=argparse.ArgumentDefaultsHelpFormatter
+        description=descStr, formatter_class=UltimateHelpFormatter
     )
     parser.add_argument(
         "field", metavar="field", type=str, help="RACS field to mosaic - e.g. 2132-50A."

arrakis/imager.py

@@ -27,7 +27,7 @@
 from spython.main import Client as sclient
 from tqdm.auto import tqdm
 
-from arrakis.logger import TqdmToLogger, logger
+from arrakis.logger import TqdmToLogger, UltimateHelpFormatter, logger
 from arrakis.utils.io import parse_env_path
 from arrakis.utils.msutils import (
     beam_from_ms,
@@ -637,6 +637,43 @@ def main(
     skip_fix_ms: bool = False,
     no_mf_weighting: bool = False,
 ):
+    """Arrakis imager flow
+
+    Args:
+        msdir (Path): Path to the directory containing the MS files.
+        out_dir (Path): Path to the directory where the images will be written.
+        temp_dir (Optional[Path], optional): Path for temporary files to be written. Defaults to None.
+        cutoff (Optional[float], optional): WSClean cutoff. Defaults to None.
+        robust (float, optional): WSClean Briggs robust parameter. Defaults to -0.5.
+        pols (str, optional): WSClean polarisations. Defaults to "IQU".
+        nchan (int, optional): WSClean number of output channels. Defaults to 36.
+        size (int, optional): WSClean image size. Defaults to 6074.
+        scale (float, optional): WSClean pixel size (arcseconds). Defaults to 2.5.
+        mgain (float, optional): WSClean mgain. Defaults to 0.8.
+        niter (int, optional): WSClean niter. Defaults to 100_000.
+        auto_mask (float, optional): WSClean automatic masking (in SNR). Defaults to 3.
+        force_mask_rounds (Union[int, None], optional): WSClean force mask rounds (requires modified WSClean). Defaults to None.
+        auto_threshold (float, optional): WSClean auto threshold (in SNR). Defaults to 1.
+        taper (Union[float, None], optional): WSClean taper (in arcsec). Defaults to None.
+        purge (bool, optional): Purge auxillary files after imaging. Defaults to False.
+        minuv (float, optional): WSClean minuv-l. Defaults to 0.0.
+        parallel_deconvolution (Optional[int], optional): WSClean parallel deconvolution. Defaults to None.
+        gridder (Optional[str], optional): WSClean gridder. Defaults to None.
+        nmiter (Optional[int], optional): WSClean nmiter. Defaults to None.
+        local_rms (bool, optional): WSClean local_rms. Defaults to False.
+        local_rms_window (Optional[float], optional): WSClean local_rms_window. Defaults to None.
+        wsclean_path (Path | str, optional): Path or URL for WSClean container. Defaults to "docker://alecthomson/wsclean:latest".
+        multiscale (Optional[bool], optional): WSClean multiscale. Defaults to None.
+        multiscale_scale_bias (Optional[float], optional): WSClean multiscale bias. Defaults to None.
+        multiscale_scales (Optional[str], optional): WSClean scales. Defaults to "0,2,4,8,16,32,64,128".
+        absmem (Optional[float], optional): WSClean absmem usage. Defaults to None.
+        make_residual_cubes (Optional[bool], optional): Make resiudal image cubes. Defaults to False.
+        ms_glob_pattern (str, optional): Globe pattern for MS files. Defaults to "scienceData*_averaged_cal.leakage.ms".
+        data_column (str, optional): Data column to image. Defaults to "CORRECTED_DATA".
+        skip_fix_ms (bool, optional): Apply FixMS. Defaults to False.
+        no_mf_weighting (bool, optional): WSClean no_mf_weighting. Defaults to False.
+    """
+
     simage = get_wsclean(wsclean=wsclean_path)
 
     logger.info(f"Searching {msdir} for MS matching {ms_glob_pattern}.")
@@ -785,7 +822,7 @@ def imager_parser(parent_parser: bool = False) -> argparse.ArgumentParser:
     img_parser = argparse.ArgumentParser(
         add_help=not parent_parser,
         description=descStr,
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        formatter_class=UltimateHelpFormatter,
     )
 
     parser = img_parser.add_argument_group("imaging arguments")

arrakis/init_database.py

@@ -13,7 +13,7 @@
 from pymongo.results import InsertManyResult
 from tqdm import tqdm
 
-from arrakis.logger import TqdmToLogger, logger
+from arrakis.logger import TqdmToLogger, UltimateHelpFormatter, logger
 from arrakis.utils.database import get_beam_inf_db, get_db, get_field_db, test_db
 from arrakis.utils.json import MyEncoder
 from arrakis.utils.meta import yes_or_no
@@ -553,7 +553,7 @@ def cli():
 
     # Parse the command line options
     parser = argparse.ArgumentParser(
-        description=descStr, formatter_class=argparse.ArgumentDefaultsHelpFormatter
+        description=descStr, formatter_class=UltimateHelpFormatter
     )
 
     parser.add_argument(

arrakis/linmos.py

@@ -18,7 +18,7 @@
 from spectral_cube.utils import SpectralCubeWarning
 from spython.main import Client as sclient
 
-from arrakis.logger import logger
+from arrakis.logger import UltimateHelpFormatter, logger
 from arrakis.utils.database import get_db, test_db
 
 warnings.filterwarnings(action="ignore", category=SpectralCubeWarning, append=True)
@@ -284,7 +284,7 @@ def main(
     stokeslist: Optional[List[str]] = None,
     limit: Optional[int] = None,
 ) -> None:
-    """Main script
+    """LINMOS flow
 
     Args:
         field (str): RACS field name.
@@ -296,7 +296,7 @@ def main(
         yanda (str, optional): Yandasoft version. Defaults to "1.3.0".
         yanda_img (Path, optional): Path to a yandasoft singularirt image. If `None`, the container version `yanda` will be downloaded. Defaults to None.
         stokeslist (List[str], optional): Stokes parameters to process. Defaults to None.
-        verbose (bool, optional): Verbose output. Defaults to True.
+        limit (int, optional): Limit the number of islands to process. Defaults to None.
     """
     # Setup singularity image
     image = get_yanda(version=yanda) if yanda_img is None else yanda_img
@@ -383,7 +383,7 @@ def cli():
 
     # Parse the command line options
     parser = argparse.ArgumentParser(
-        description=descStr, formatter_class=argparse.ArgumentDefaultsHelpFormatter
+        description=descStr, formatter_class=UltimateHelpFormatter
     )
 
     parser.add_argument(