Merge branch 'main' of https://github.com/tjgalvin/flint

.github/workflows/python-lint-test.yml

@@ -0,0 +1,35 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python application
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.8"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest pre-commit .
+    - name: Lint with pre-commit
+      run: |
+        pre-commit run --all-files
+    - name: Test with pytest
+      run: |
+        pytest

flint/prefect/common/imaging.py

@@ -28,6 +28,7 @@
 from flint.ms import MS, preprocess_askap_ms, split_by_field
 from flint.naming import FITSMaskNames, processed_ms_format
 from flint.options import FieldOptions
+from flint.prefect.common.utils import upload_image_as_artifact
 from flint.selfcal.casa import gaincal_applycal_ms
 from flint.source_finding.aegean import AegeanOutputs, run_bane_and_aegean
 from flint.utils import zip_folder
@@ -542,13 +543,16 @@ def task_extract_beam_mask_image(
 
 @task
 def task_create_validation_plot(
-    aegean_outputs: AegeanOutputs, reference_catalogue_directory: Path
+    aegean_outputs: AegeanOutputs,
+    reference_catalogue_directory: Path,
+    upload_artifact: bool = True,
 ) -> Path:
     """Create a multi-panel figure highlighting the RMS, flux scale and astrometry of a field
 
     Args:
         aegean_outputs (AegeanOutputs): Output aegean products
         reference_catalogue_directory (Path): Directory containing NVSS, SUMSS and ICRS reference catalogues. These catalogues are reconginised internally and have expected names.
+        upload_artifact (bool, optional): If True the validation plot will be uploaded to the prefect service as an artifact. Defaults to True.
 
     Returns:
         Path: Path to the output figure created
@@ -557,9 +561,16 @@ def task_create_validation_plot(
 
     logger.info(f"Will create {output_figure_path=}")
 
-    return create_validation_plot(
+    plot_path = create_validation_plot(
         rms_image_path=aegean_outputs.rms,
         source_catalogue_path=aegean_outputs.comp,
         output_path=output_figure_path,
         reference_catalogue_directory=reference_catalogue_directory,
     )
+
+    if upload_artifact:
+        upload_image_as_artifact(
+            image_path=plot_path, descrition=f"Validation plot {str(plot_path)}"
+        )
+
+    return plot_path

flint/prefect/common/utils.py

@@ -1,13 +1,55 @@
 """Common prefect related utilities that can be used between flows.
 """
-from typing import Any, List, TypeVar
+import base64
+from pathlib import Path
+from typing import Any, List, Optional, TypeVar
+from uuid import UUID
 
 from prefect import task
+from prefect.artifacts import create_markdown_artifact
 
 from flint.logging import logger
 
 T = TypeVar("T")
 
+SUPPORTED_IMAGE_TYPES = ("png",)
+
+
+def upload_image_as_artifact(
+    image_path: Path, description: Optional[str] = None
+) -> UUID:
+    """Create and submit a markdown artifact tracked by prefect for an
+    input image. Currently supporting png formatted images.
+
+    The input image is converted to a base64 encoding, and embedded directly
+    within the markdown string. Therefore, be mindful of the image size as this
+    is tracked in the postgres database.
+
+    Args:
+        image_path (Path): Path to the image to upload
+        description (Optional[str], optional): A description passed to the markdown artifact. Defaults to None.
+
+    Returns:
+        UUID: Generated UUID of the registered artifact
+    """
+    image_type = image_path.suffix.replace(".", "")
+    assert image_path.exists(), f"{image_path} does not exist"
+    assert (
+        image_type in SUPPORTED_IMAGE_TYPES
+    ), f"{image_path} has type {image_type}, and is not supported. Supported types are {SUPPORTED_IMAGE_TYPES}"
+
+    with open(image_path, "rb") as open_image:
+        logger.info(f"Encoding {image_path} in base64")
+        image_base64 = base64.b64encode(open_image.read()).decode()
+
+    logger.info("Creating markdown tag")
+    markdown = f"![{image_path.stem}](data:image/{image_type};base64,{image_base64})"
+
+    logger.info("Registering artifact")
+    image_uuid = create_markdown_artifact(markdown=markdown, description=description)
+
+    return image_uuid
+
 
 @task
 def task_get_attributes(item: Any, attribute: str) -> Any:

flint/prefect/flows/continuum_mask_pipeline.py

@@ -106,7 +106,7 @@ def process_science_fields(
     # other calibration strategies get added
     # Scan the existing bandpass directory for the existing solutions
     calibrate_cmds = find_existing_solutions(
-        bandpass_directory=bandpass_path, use_preflagged=True, use_smoothed=False
+        bandpass_directory=bandpass_path, use_preflagged=True, use_smoothed=True
     )
 
     logger.info(f"Constructed the following {calibrate_cmds=}")