diff --git a/flint/configuration.py b/flint/configuration.py index cccfd0cb..7a1a2143 100644 --- a/flint/configuration.py +++ b/flint/configuration.py @@ -4,6 +4,7 @@ throughout the pipeline. """ +import shutil from argparse import ArgumentParser from pathlib import Path from typing import Any, Dict, Optional, Union @@ -13,6 +14,7 @@ from flint.imager.wsclean import WSCleanOptions from flint.logging import logger from flint.masking import MaskingOptions +from flint.naming import add_timestamp_to_path from flint.selfcal.casa import GainCalOptions KNOWN_HEADERS = ("defaults", "initial", "selfcal", "version") @@ -34,6 +36,26 @@ class Strategy(dict): pass +def copy_and_timestamp_strategy_file(output_dir: Path, input_yaml: Path) -> Path: + """Timestamp and copy the input strategy file to an + output directory + + Args: + output_dir (Path): Output directory the file will be copied to + input_yaml (Path): The file to copy + + Returns: + Path: Copied and timestamped file path + """ + stamped_imaging_strategy = ( + output_dir / add_timestamp_to_path(input_path=input_yaml).name + ) + logger.info(f"Copying {input_yaml.absolute()} to {stamped_imaging_strategy}") + shutil.copyfile(input_yaml.absolute(), stamped_imaging_strategy) + + return Path(stamped_imaging_strategy) + + def get_selfcal_options_from_yaml(input_yaml: Optional[Path] = None) -> Dict: """Stub to represent interaction with a configurationf ile @@ -80,6 +102,16 @@ def get_image_options_from_yaml( MULTISCALE_SCALES = (0, 15, 30, 40, 50, 60, 70, 120) IMAGE_SIZE = 7144 + # These werte teh settings I was using when overloading if + # mask was created for the cleaning + + # wsclean_options["auto_mask"] = 1.25 + # wsclean_options["auto_threshold"] = 1.0 + # wsclean_options["force_mask_rounds"] = 13 + # wsclean_options["local_rms"] = False + # wsclean_options["niter"] = 1750000 + # wsclean_options["nmiter"] = 30 + if not self_cal_rounds: return { "size": IMAGE_SIZE, diff --git a/flint/naming.py b/flint/naming.py index 80acd6a8..71a98fb0 100644 --- a/flint/naming.py +++ b/flint/naming.py @@ -3,12 +3,37 @@ """ import re +from datetime import datetime from pathlib import Path from typing import Any, List, NamedTuple, Optional, Union from flint.logging import logger +def add_timestamp_to_path( + input_path: Union[Path, str], timestamp: Optional[datetime] = None +) -> Path: + """Add a timestamp to a input path, where the timestamp is the + current data and time. The time will be added to the name component + before the file suffix. If the name component of the `input_path` + has multiple suffixes than the timestamp will be added before the last. + + Args: + input_path (Union[Path, str]): Path that should have a timestamp added + timestamp: (Optional[datetime], optional): The date-time to add. If None the current time is used. Defaults to None. + Returns: + Path: Updated path with a timestamp in the file name + """ + input_path = Path(input_path) + timestamp = timestamp if timestamp else datetime.now() + + time_str = timestamp.strftime("%Y%m%d-%H%M%S") + new_name = f"{input_path.stem}-{time_str}{input_path.suffix}" + output_path = input_path.with_name(new_name) + + return output_path + + class RawNameComponents(NamedTuple): date: str """Date that the data were taken, of the form YYYY-MM-DD""" diff --git a/flint/prefect/flows/continuum_pipeline.py b/flint/prefect/flows/continuum_pipeline.py index 8af21260..f6ba1b21 100644 --- a/flint/prefect/flows/continuum_pipeline.py +++ b/flint/prefect/flows/continuum_pipeline.py @@ -12,7 +12,11 @@ from prefect import flow, unmapped from flint.calibrate.aocalibrate import find_existing_solutions -from flint.configuration import get_options_from_strategy, load_strategy_yaml +from flint.configuration import ( + copy_and_timestamp_strategy_file, + get_options_from_strategy, + load_strategy_yaml, +) from flint.logging import logger from flint.ms import MS from flint.naming import get_sbid_from_path @@ -72,12 +76,6 @@ def process_science_fields( len(science_mss) == field_options.expected_ms ), f"Expected to find {field_options.expected_ms} in {str(science_path)}, found {len(science_mss)}." - strategy = ( - load_strategy_yaml(input_yaml=field_options.imaging_strategy, verify=True) - if field_options.imaging_strategy - else None - ) - science_folder_name = science_path.name output_split_science_path = ( @@ -93,6 +91,18 @@ def process_science_fields( logger.info(f"Creating {str(output_split_science_path)}") output_split_science_path.mkdir(parents=True) + strategy = ( + load_strategy_yaml( + input_yaml=copy_and_timestamp_strategy_file( + output_dir=output_split_science_path, + input_yaml=field_options.imaging_strategy, + ), + verify=True, + ) + if field_options.imaging_strategy + else None + ) + logger.info(f"{field_options=}") logger.info(f"Found the following raw measurement sets: {science_mss}") @@ -264,12 +274,6 @@ def process_science_fields( image_products=beam_aegean_outputs, min_snr=3.5, ) - # wsclean_options["auto_mask"] = 1.25 - # wsclean_options["auto_threshold"] = 1.0 - # wsclean_options["force_mask_rounds"] = 13 - # wsclean_options["local_rms"] = False - # wsclean_options["niter"] = 1750000 - # wsclean_options["nmiter"] = 30 wsclean_cmds = task_wsclean_imager.map( in_ms=cal_mss, diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 2781a2df..740a95fb 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1,9 +1,11 @@ +import filecmp from pathlib import Path import pytest from flint.configuration import ( Strategy, + copy_and_timestamp_strategy_file, create_default_yaml, get_image_options_from_yaml, get_options_from_strategy, @@ -37,6 +39,17 @@ def strategy(tmpdir): return strat +def test_copy_and_timestamp(tmpdir): + # a single function toe rename and copy a file because pirates needs to be efficient + example = get_packaged_resource_path( + package="flint", filename="data/tests/test_config.yaml" + ) + copy_path = copy_and_timestamp_strategy_file(output_dir=tmpdir, input_yaml=example) + + assert copy_path != example + assert filecmp.cmp(example, copy_path) + + def test_verify_options_with_class(package_strategy): # ebsure that the errors raised from options passed through # to the input structures correctly raise errors should they diff --git a/tests/test_naming.py b/tests/test_naming.py index 32a4b48b..02333648 100644 --- a/tests/test_naming.py +++ b/tests/test_naming.py @@ -1,11 +1,13 @@ """Some tests related to components around measurement sets.""" +from datetime import datetime from pathlib import Path from flint.naming import ( FITSMaskNames, ProcessedNameComponents, RawNameComponents, + add_timestamp_to_path, create_fits_mask_names, create_ms_name, extract_beam_from_name, @@ -17,6 +19,25 @@ ) +def test_add_timestamp_to_path(): + # make sure adding a timestamp to a file name works + dd = datetime(2024, 4, 12, 10, 30, 50, 243910) + + example_str = "/test/this/is/filename.txt" + stamped_path = add_timestamp_to_path(input_path=example_str, timestamp=dd) + expected = Path("/test/this/is/filename-20240412-103050.txt") + + assert stamped_path == expected + + example_path = Path("/test/this/is/filename.txt") + stamped_path = add_timestamp_to_path(input_path=example_path, timestamp=dd) + + assert stamped_path == expected + + now_path = add_timestamp_to_path(input_path=example_path) + assert now_path != expected + + def test_create_fits_mask_names_no_signal(): fits_image = Path("38960/SB38960.RACS_1418-12.noselfcal_linmos.fits")