From 4bd6d525396d8643a0ed546de477f24ace185615 Mon Sep 17 00:00:00 2001 From: tgalvin Date: Fri, 12 Apr 2024 11:32:31 +0800 Subject: [PATCH 1/9] add_timestamp_to_path / copying strat --- flint/configuration.py | 10 ++++++++ flint/naming.py | 25 ++++++++++++++++++++ flint/prefect/flows/continuum_pipeline.py | 28 ++++++++++++----------- tests/test_naming.py | 21 +++++++++++++++++ 4 files changed, 71 insertions(+), 13 deletions(-) diff --git a/flint/configuration.py b/flint/configuration.py index cccfd0cb..04f7ea55 100644 --- a/flint/configuration.py +++ b/flint/configuration.py @@ -80,6 +80,16 @@ def get_image_options_from_yaml( MULTISCALE_SCALES = (0, 15, 30, 40, 50, 60, 70, 120) IMAGE_SIZE = 7144 + # These werte teh settings I was using when overloading if + # mask was created for the cleaning + + # wsclean_options["auto_mask"] = 1.25 + # wsclean_options["auto_threshold"] = 1.0 + # wsclean_options["force_mask_rounds"] = 13 + # wsclean_options["local_rms"] = False + # wsclean_options["niter"] = 1750000 + # wsclean_options["nmiter"] = 30 + if not self_cal_rounds: return { "size": IMAGE_SIZE, diff --git a/flint/naming.py b/flint/naming.py index 80acd6a8..71a98fb0 100644 --- a/flint/naming.py +++ b/flint/naming.py @@ -3,12 +3,37 @@ """ import re +from datetime import datetime from pathlib import Path from typing import Any, List, NamedTuple, Optional, Union from flint.logging import logger +def add_timestamp_to_path( + input_path: Union[Path, str], timestamp: Optional[datetime] = None +) -> Path: + """Add a timestamp to a input path, where the timestamp is the + current data and time. The time will be added to the name component + before the file suffix. If the name component of the `input_path` + has multiple suffixes than the timestamp will be added before the last. + + Args: + input_path (Union[Path, str]): Path that should have a timestamp added + timestamp: (Optional[datetime], optional): The date-time to add. If None the current time is used. Defaults to None. + Returns: + Path: Updated path with a timestamp in the file name + """ + input_path = Path(input_path) + timestamp = timestamp if timestamp else datetime.now() + + time_str = timestamp.strftime("%Y%m%d-%H%M%S") + new_name = f"{input_path.stem}-{time_str}{input_path.suffix}" + output_path = input_path.with_name(new_name) + + return output_path + + class RawNameComponents(NamedTuple): date: str """Date that the data were taken, of the form YYYY-MM-DD""" diff --git a/flint/prefect/flows/continuum_pipeline.py b/flint/prefect/flows/continuum_pipeline.py index 8af21260..b04c8b0e 100644 --- a/flint/prefect/flows/continuum_pipeline.py +++ b/flint/prefect/flows/continuum_pipeline.py @@ -5,6 +5,7 @@ - run aegean source finding """ +import shutil from argparse import ArgumentParser from pathlib import Path from typing import Union @@ -15,7 +16,7 @@ from flint.configuration import get_options_from_strategy, load_strategy_yaml from flint.logging import logger from flint.ms import MS -from flint.naming import get_sbid_from_path +from flint.naming import get_sbid_from_path, add_timestamp_to_path from flint.options import FieldOptions from flint.prefect.clusters import get_dask_runner from flint.prefect.common.imaging import ( @@ -72,12 +73,6 @@ def process_science_fields( len(science_mss) == field_options.expected_ms ), f"Expected to find {field_options.expected_ms} in {str(science_path)}, found {len(science_mss)}." - strategy = ( - load_strategy_yaml(input_yaml=field_options.imaging_strategy, verify=True) - if field_options.imaging_strategy - else None - ) - science_folder_name = science_path.name output_split_science_path = ( @@ -90,6 +85,19 @@ def process_science_fields( ) raise ValueError("Output science directory already exists. ") + if field_options.imaging_strategy: + stamped_imaging_strategy = ( + output_split_science_path + / add_timestamp_to_path(input_path=field_options.imaging_strategy).name + ) + shutil.copy(field_options.imaging_strategy, stamped_imaging_strategy) + + strategy = ( + load_strategy_yaml(input_yaml=field_options.imaging_strategy, verify=True) + if field_options.imaging_strategy + else None + ) + logger.info(f"Creating {str(output_split_science_path)}") output_split_science_path.mkdir(parents=True) @@ -264,12 +272,6 @@ def process_science_fields( image_products=beam_aegean_outputs, min_snr=3.5, ) - # wsclean_options["auto_mask"] = 1.25 - # wsclean_options["auto_threshold"] = 1.0 - # wsclean_options["force_mask_rounds"] = 13 - # wsclean_options["local_rms"] = False - # wsclean_options["niter"] = 1750000 - # wsclean_options["nmiter"] = 30 wsclean_cmds = task_wsclean_imager.map( in_ms=cal_mss, diff --git a/tests/test_naming.py b/tests/test_naming.py index 32a4b48b..02333648 100644 --- a/tests/test_naming.py +++ b/tests/test_naming.py @@ -1,11 +1,13 @@ """Some tests related to components around measurement sets.""" +from datetime import datetime from pathlib import Path from flint.naming import ( FITSMaskNames, ProcessedNameComponents, RawNameComponents, + add_timestamp_to_path, create_fits_mask_names, create_ms_name, extract_beam_from_name, @@ -17,6 +19,25 @@ ) +def test_add_timestamp_to_path(): + # make sure adding a timestamp to a file name works + dd = datetime(2024, 4, 12, 10, 30, 50, 243910) + + example_str = "/test/this/is/filename.txt" + stamped_path = add_timestamp_to_path(input_path=example_str, timestamp=dd) + expected = Path("/test/this/is/filename-20240412-103050.txt") + + assert stamped_path == expected + + example_path = Path("/test/this/is/filename.txt") + stamped_path = add_timestamp_to_path(input_path=example_path, timestamp=dd) + + assert stamped_path == expected + + now_path = add_timestamp_to_path(input_path=example_path) + assert now_path != expected + + def test_create_fits_mask_names_no_signal(): fits_image = Path("38960/SB38960.RACS_1418-12.noselfcal_linmos.fits") From 0f07521646464ad035cd40180b8ae3a9433e7ec4 Mon Sep 17 00:00:00 2001 From: tgalvin Date: Fri, 12 Apr 2024 13:32:27 +0800 Subject: [PATCH 2/9] copyfile and log message --- flint/prefect/flows/continuum_pipeline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flint/prefect/flows/continuum_pipeline.py b/flint/prefect/flows/continuum_pipeline.py index b04c8b0e..4c199992 100644 --- a/flint/prefect/flows/continuum_pipeline.py +++ b/flint/prefect/flows/continuum_pipeline.py @@ -90,7 +90,10 @@ def process_science_fields( output_split_science_path / add_timestamp_to_path(input_path=field_options.imaging_strategy).name ) - shutil.copy(field_options.imaging_strategy, stamped_imaging_strategy) + logger.info( + f"Copying {field_options.imaging_strategy} to {stamped_imaging_strategy}" + ) + shutil.copyfile(field_options.imaging_strategy, stamped_imaging_strategy) strategy = ( load_strategy_yaml(input_yaml=field_options.imaging_strategy, verify=True) From 80a68efbfd4b4b9c212493bc530cc53bfe083d82 Mon Sep 17 00:00:00 2001 From: tgalvin Date: Fri, 12 Apr 2024 13:35:23 +0800 Subject: [PATCH 3/9] rearanged order --- flint/prefect/flows/continuum_pipeline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flint/prefect/flows/continuum_pipeline.py b/flint/prefect/flows/continuum_pipeline.py index 4c199992..1aa873e1 100644 --- a/flint/prefect/flows/continuum_pipeline.py +++ b/flint/prefect/flows/continuum_pipeline.py @@ -85,6 +85,9 @@ def process_science_fields( ) raise ValueError("Output science directory already exists. ") + logger.info(f"Creating {str(output_split_science_path)}") + output_split_science_path.mkdir(parents=True) + if field_options.imaging_strategy: stamped_imaging_strategy = ( output_split_science_path @@ -101,9 +104,6 @@ def process_science_fields( else None ) - logger.info(f"Creating {str(output_split_science_path)}") - output_split_science_path.mkdir(parents=True) - logger.info(f"{field_options=}") logger.info(f"Found the following raw measurement sets: {science_mss}") From da654992ab2b97a9f9a0b6d860811e55c1077c10 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 05:42:00 +0000 Subject: [PATCH 4/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- flint/prefect/flows/continuum_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flint/prefect/flows/continuum_pipeline.py b/flint/prefect/flows/continuum_pipeline.py index 1aa873e1..3a178bf3 100644 --- a/flint/prefect/flows/continuum_pipeline.py +++ b/flint/prefect/flows/continuum_pipeline.py @@ -16,7 +16,7 @@ from flint.configuration import get_options_from_strategy, load_strategy_yaml from flint.logging import logger from flint.ms import MS -from flint.naming import get_sbid_from_path, add_timestamp_to_path +from flint.naming import add_timestamp_to_path, get_sbid_from_path from flint.options import FieldOptions from flint.prefect.clusters import get_dask_runner from flint.prefect.common.imaging import ( From 3e3fb6cb0624cd3377a0ef003e7d210002f27275 Mon Sep 17 00:00:00 2001 From: tgalvin Date: Fri, 12 Apr 2024 14:32:07 +0800 Subject: [PATCH 5/9] restructure the copy and timestamping --- flint/configuration.py | 22 ++++++++++++++++++ flint/prefect/flows/continuum_pipeline.py | 27 +++++++++++------------ tests/test_configuration.py | 15 +++++++++++++ 3 files changed, 50 insertions(+), 14 deletions(-) diff --git a/flint/configuration.py b/flint/configuration.py index 04f7ea55..a0056d37 100644 --- a/flint/configuration.py +++ b/flint/configuration.py @@ -4,6 +4,7 @@ throughout the pipeline. """ +import shutil from argparse import ArgumentParser from pathlib import Path from typing import Any, Dict, Optional, Union @@ -12,6 +13,7 @@ from flint.imager.wsclean import WSCleanOptions from flint.logging import logger +from flint.naming import add_timestamp_to_path from flint.masking import MaskingOptions from flint.selfcal.casa import GainCalOptions @@ -34,6 +36,26 @@ class Strategy(dict): pass +def copy_and_timestamp_strategy_file(output_dir: Path, input_yaml: Path) -> Path: + """Timestamp and copy the input strategy file to an + output directory + + Args: + output_dir (Path): Output directory the file will be copied to + input_yaml (Path): The file to copy + + Returns: + Path: Copied and timestamped file path + """ + stamped_imaging_strategy = ( + output_dir / add_timestamp_to_path(input_path=input_yaml).name + ) + logger.info(f"Copying {input_yaml} to {stamped_imaging_strategy}") + shutil.copyfile(input_yaml, stamped_imaging_strategy) + + return Path(stamped_imaging_strategy) + + def get_selfcal_options_from_yaml(input_yaml: Optional[Path] = None) -> Dict: """Stub to represent interaction with a configurationf ile diff --git a/flint/prefect/flows/continuum_pipeline.py b/flint/prefect/flows/continuum_pipeline.py index 1aa873e1..e43f73dc 100644 --- a/flint/prefect/flows/continuum_pipeline.py +++ b/flint/prefect/flows/continuum_pipeline.py @@ -5,7 +5,6 @@ - run aegean source finding """ -import shutil from argparse import ArgumentParser from pathlib import Path from typing import Union @@ -13,10 +12,14 @@ from prefect import flow, unmapped from flint.calibrate.aocalibrate import find_existing_solutions -from flint.configuration import get_options_from_strategy, load_strategy_yaml +from flint.configuration import ( + get_options_from_strategy, + load_strategy_yaml, + copy_and_timestamp_strategy_file, +) from flint.logging import logger from flint.ms import MS -from flint.naming import get_sbid_from_path, add_timestamp_to_path +from flint.naming import get_sbid_from_path from flint.options import FieldOptions from flint.prefect.clusters import get_dask_runner from flint.prefect.common.imaging import ( @@ -88,18 +91,14 @@ def process_science_fields( logger.info(f"Creating {str(output_split_science_path)}") output_split_science_path.mkdir(parents=True) - if field_options.imaging_strategy: - stamped_imaging_strategy = ( - output_split_science_path - / add_timestamp_to_path(input_path=field_options.imaging_strategy).name - ) - logger.info( - f"Copying {field_options.imaging_strategy} to {stamped_imaging_strategy}" - ) - shutil.copyfile(field_options.imaging_strategy, stamped_imaging_strategy) - strategy = ( - load_strategy_yaml(input_yaml=field_options.imaging_strategy, verify=True) + load_strategy_yaml( + input_yaml=copy_and_timestamp_strategy_file( + output_dir=output_split_science_path, + input_yaml=field_options.imaging_strategy, + ), + verify=True, + ) if field_options.imaging_strategy else None ) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 2781a2df..60197a98 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1,9 +1,11 @@ +import filecmp from pathlib import Path import pytest from flint.configuration import ( Strategy, + copy_and_timestamp_strategy_file, create_default_yaml, get_image_options_from_yaml, get_options_from_strategy, @@ -37,6 +39,19 @@ def strategy(tmpdir): return strat +def test_copy_and_timestamp(tmpdir): + example = get_packaged_resource_path( + package="flint", filename="data/tests/test_config.yaml" + ) + copy_path = copy_and_timestamp_strategy_file(output_dir=tmpdir, input_yaml=example) + + print(example) + print(copy_path) + + assert copy_path != example + assert filecmp.cmp(example, copy_path) + + def test_verify_options_with_class(package_strategy): # ebsure that the errors raised from options passed through # to the input structures correctly raise errors should they From aa5d463618115095eb0f4a45bfd0db7a91c146c1 Mon Sep 17 00:00:00 2001 From: tgalvin Date: Fri, 12 Apr 2024 14:33:40 +0800 Subject: [PATCH 6/9] removed some prints and added a note --- tests/test_configuration.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 60197a98..740a95fb 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -40,14 +40,12 @@ def strategy(tmpdir): def test_copy_and_timestamp(tmpdir): + # a single function toe rename and copy a file because pirates needs to be efficient example = get_packaged_resource_path( package="flint", filename="data/tests/test_config.yaml" ) copy_path = copy_and_timestamp_strategy_file(output_dir=tmpdir, input_yaml=example) - print(example) - print(copy_path) - assert copy_path != example assert filecmp.cmp(example, copy_path) From e4213076b6f2c4809c0309eb6274ca48225ec22c Mon Sep 17 00:00:00 2001 From: tgalvin Date: Fri, 12 Apr 2024 14:53:37 +0800 Subject: [PATCH 7/9] removed unused import --- flint/prefect/flows/continuum_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flint/prefect/flows/continuum_pipeline.py b/flint/prefect/flows/continuum_pipeline.py index 915f4541..e43f73dc 100644 --- a/flint/prefect/flows/continuum_pipeline.py +++ b/flint/prefect/flows/continuum_pipeline.py @@ -19,7 +19,7 @@ ) from flint.logging import logger from flint.ms import MS -from flint.naming import add_timestamp_to_path, get_sbid_from_path +from flint.naming import get_sbid_from_path from flint.options import FieldOptions from flint.prefect.clusters import get_dask_runner from flint.prefect.common.imaging import ( From 3dc25068bd08890b0ab74ef328f54986d8dbabba Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 06:54:02 +0000 Subject: [PATCH 8/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- flint/configuration.py | 2 +- flint/prefect/flows/continuum_pipeline.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flint/configuration.py b/flint/configuration.py index a0056d37..0fde3f1e 100644 --- a/flint/configuration.py +++ b/flint/configuration.py @@ -13,8 +13,8 @@ from flint.imager.wsclean import WSCleanOptions from flint.logging import logger -from flint.naming import add_timestamp_to_path from flint.masking import MaskingOptions +from flint.naming import add_timestamp_to_path from flint.selfcal.casa import GainCalOptions KNOWN_HEADERS = ("defaults", "initial", "selfcal", "version") diff --git a/flint/prefect/flows/continuum_pipeline.py b/flint/prefect/flows/continuum_pipeline.py index e43f73dc..f6ba1b21 100644 --- a/flint/prefect/flows/continuum_pipeline.py +++ b/flint/prefect/flows/continuum_pipeline.py @@ -13,9 +13,9 @@ from flint.calibrate.aocalibrate import find_existing_solutions from flint.configuration import ( + copy_and_timestamp_strategy_file, get_options_from_strategy, load_strategy_yaml, - copy_and_timestamp_strategy_file, ) from flint.logging import logger from flint.ms import MS From d91221863da8b0eb87e61db959a9f4bd00138c91 Mon Sep 17 00:00:00 2001 From: tgalvin Date: Fri, 12 Apr 2024 14:55:42 +0800 Subject: [PATCH 9/9] absolute path --- flint/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flint/configuration.py b/flint/configuration.py index a0056d37..fb4ce73a 100644 --- a/flint/configuration.py +++ b/flint/configuration.py @@ -50,8 +50,8 @@ def copy_and_timestamp_strategy_file(output_dir: Path, input_yaml: Path) -> Path stamped_imaging_strategy = ( output_dir / add_timestamp_to_path(input_path=input_yaml).name ) - logger.info(f"Copying {input_yaml} to {stamped_imaging_strategy}") - shutil.copyfile(input_yaml, stamped_imaging_strategy) + logger.info(f"Copying {input_yaml.absolute()} to {stamped_imaging_strategy}") + shutil.copyfile(input_yaml.absolute(), stamped_imaging_strategy) return Path(stamped_imaging_strategy)