Merge pull request #83 from tjgalvin/copystrat

Copy strategy file to science directory

flint/configuration.py

@@ -4,6 +4,7 @@
 throughout the pipeline.
 """
 
+import shutil
 from argparse import ArgumentParser
 from pathlib import Path
 from typing import Any, Dict, Optional, Union
@@ -13,6 +14,7 @@
 from flint.imager.wsclean import WSCleanOptions
 from flint.logging import logger
 from flint.masking import MaskingOptions
+from flint.naming import add_timestamp_to_path
 from flint.selfcal.casa import GainCalOptions
 
 KNOWN_HEADERS = ("defaults", "initial", "selfcal", "version")
@@ -34,6 +36,26 @@ class Strategy(dict):
     pass
 
 
+def copy_and_timestamp_strategy_file(output_dir: Path, input_yaml: Path) -> Path:
+    """Timestamp and copy the input strategy file to an
+    output directory
+
+    Args:
+        output_dir (Path): Output directory the file will be copied to
+        input_yaml (Path): The file to copy
+
+    Returns:
+        Path: Copied and timestamped file path
+    """
+    stamped_imaging_strategy = (
+        output_dir / add_timestamp_to_path(input_path=input_yaml).name
+    )
+    logger.info(f"Copying {input_yaml.absolute()} to {stamped_imaging_strategy}")
+    shutil.copyfile(input_yaml.absolute(), stamped_imaging_strategy)
+
+    return Path(stamped_imaging_strategy)
+
+
 def get_selfcal_options_from_yaml(input_yaml: Optional[Path] = None) -> Dict:
     """Stub to represent interaction with a configurationf ile
 
@@ -80,6 +102,16 @@ def get_image_options_from_yaml(
     MULTISCALE_SCALES = (0, 15, 30, 40, 50, 60, 70, 120)
     IMAGE_SIZE = 7144
 
+    # These werte teh settings I was using when overloading if
+    # mask was created for the cleaning
+
+    # wsclean_options["auto_mask"] = 1.25
+    # wsclean_options["auto_threshold"] = 1.0
+    # wsclean_options["force_mask_rounds"] = 13
+    # wsclean_options["local_rms"] = False
+    # wsclean_options["niter"] = 1750000
+    # wsclean_options["nmiter"] = 30
+
     if not self_cal_rounds:
         return {
             "size": IMAGE_SIZE,

flint/naming.py

@@ -3,12 +3,37 @@
 """
 
 import re
+from datetime import datetime
 from pathlib import Path
 from typing import Any, List, NamedTuple, Optional, Union
 
 from flint.logging import logger
 
 
+def add_timestamp_to_path(
+    input_path: Union[Path, str], timestamp: Optional[datetime] = None
+) -> Path:
+    """Add a timestamp to a input path, where the timestamp is the
+    current data and time. The time will be added to the name component
+    before the file suffix. If the name component of the `input_path`
+    has multiple suffixes than the timestamp will be added before the last.
+
+    Args:
+        input_path (Union[Path, str]): Path that should have a timestamp added
+        timestamp: (Optional[datetime], optional): The date-time to add. If None the current time is used. Defaults to None.
+    Returns:
+        Path: Updated path with a timestamp in the file name
+    """
+    input_path = Path(input_path)
+    timestamp = timestamp if timestamp else datetime.now()
+
+    time_str = timestamp.strftime("%Y%m%d-%H%M%S")
+    new_name = f"{input_path.stem}-{time_str}{input_path.suffix}"
+    output_path = input_path.with_name(new_name)
+
+    return output_path
+
+
 class RawNameComponents(NamedTuple):
     date: str
     """Date that the data were taken, of the form YYYY-MM-DD"""

flint/prefect/flows/continuum_pipeline.py

@@ -12,7 +12,11 @@
 from prefect import flow, unmapped
 
 from flint.calibrate.aocalibrate import find_existing_solutions
-from flint.configuration import get_options_from_strategy, load_strategy_yaml
+from flint.configuration import (
+    copy_and_timestamp_strategy_file,
+    get_options_from_strategy,
+    load_strategy_yaml,
+)
 from flint.logging import logger
 from flint.ms import MS
 from flint.naming import get_sbid_from_path
@@ -72,12 +76,6 @@ def process_science_fields(
         len(science_mss) == field_options.expected_ms
     ), f"Expected to find {field_options.expected_ms} in {str(science_path)}, found {len(science_mss)}."
 
-    strategy = (
-        load_strategy_yaml(input_yaml=field_options.imaging_strategy, verify=True)
-        if field_options.imaging_strategy
-        else None
-    )
-
     science_folder_name = science_path.name
 
     output_split_science_path = (
@@ -93,6 +91,18 @@ def process_science_fields(
     logger.info(f"Creating {str(output_split_science_path)}")
     output_split_science_path.mkdir(parents=True)
 
+    strategy = (
+        load_strategy_yaml(
+            input_yaml=copy_and_timestamp_strategy_file(
+                output_dir=output_split_science_path,
+                input_yaml=field_options.imaging_strategy,
+            ),
+            verify=True,
+        )
+        if field_options.imaging_strategy
+        else None
+    )
+
     logger.info(f"{field_options=}")
 
     logger.info(f"Found the following raw measurement sets: {science_mss}")
@@ -264,12 +274,6 @@ def process_science_fields(
                 image_products=beam_aegean_outputs,
                 min_snr=3.5,
             )
-            # wsclean_options["auto_mask"] = 1.25
-            # wsclean_options["auto_threshold"] = 1.0
-            # wsclean_options["force_mask_rounds"] = 13
-            # wsclean_options["local_rms"] = False
-            # wsclean_options["niter"] = 1750000
-            # wsclean_options["nmiter"] = 30
 
         wsclean_cmds = task_wsclean_imager.map(
             in_ms=cal_mss,

tests/test_configuration.py

@@ -1,9 +1,11 @@
+import filecmp
 from pathlib import Path
 
 import pytest
 
 from flint.configuration import (
     Strategy,
+    copy_and_timestamp_strategy_file,
     create_default_yaml,
     get_image_options_from_yaml,
     get_options_from_strategy,
@@ -37,6 +39,17 @@ def strategy(tmpdir):
     return strat
 
 
+def test_copy_and_timestamp(tmpdir):
+    # a single function toe rename and copy a file because pirates needs to be efficient
+    example = get_packaged_resource_path(
+        package="flint", filename="data/tests/test_config.yaml"
+    )
+    copy_path = copy_and_timestamp_strategy_file(output_dir=tmpdir, input_yaml=example)
+
+    assert copy_path != example
+    assert filecmp.cmp(example, copy_path)
+
+
 def test_verify_options_with_class(package_strategy):
     # ebsure that the errors raised from options passed through
     # to the input structures correctly raise errors should they

tests/test_naming.py

@@ -1,11 +1,13 @@
 """Some tests related to components around measurement sets."""
 
+from datetime import datetime
 from pathlib import Path
 
 from flint.naming import (
     FITSMaskNames,
     ProcessedNameComponents,
     RawNameComponents,
+    add_timestamp_to_path,
     create_fits_mask_names,
     create_ms_name,
     extract_beam_from_name,
@@ -17,6 +19,25 @@
 )
 
 
+def test_add_timestamp_to_path():
+    # make sure adding a timestamp to a file name works
+    dd = datetime(2024, 4, 12, 10, 30, 50, 243910)
+
+    example_str = "/test/this/is/filename.txt"
+    stamped_path = add_timestamp_to_path(input_path=example_str, timestamp=dd)
+    expected = Path("/test/this/is/filename-20240412-103050.txt")
+
+    assert stamped_path == expected
+
+    example_path = Path("/test/this/is/filename.txt")
+    stamped_path = add_timestamp_to_path(input_path=example_path, timestamp=dd)
+
+    assert stamped_path == expected
+
+    now_path = add_timestamp_to_path(input_path=example_path)
+    assert now_path != expected
+
+
 def test_create_fits_mask_names_no_signal():
     fits_image = Path("38960/SB38960.RACS_1418-12.noselfcal_linmos.fits")