Casacore related version fixes

.github/workflows/python-lint-test.yml

@@ -14,14 +14,17 @@ permissions:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
 
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3.12
         uses: actions/setup-python@v3
         with:
           python-version: "3.12"
+      - name: Install casacore and boost
+        run: |
+          sudo apt install -y build-essential libcfitsio-dev liblapack-dev libboost-python-dev python3-dev wcslib-dev casacore-dev
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip uv

README.md

@@ -9,6 +9,56 @@ Yarrrr-Harrrr fiddley-dee!
 
 <img src="docs/logo.jpeg" alt="Capn' Flint - Credit: DALLE 3" style="width:400px;"/>
 
+## Installation
+
+Provided an appropriate environment installation should be as simple as a
+`pip install`.
+
+However, on some systems there are interactions with `casacore` and building
+`python-casacore` appropriately. Issues have been noted when interacting with
+large measurement sets across components with different `casacore` versions.
+This seems to happen even across container boundaries (i.e. different versions
+in containers might play a role). The exact cause is not at all understood, but
+it appears to be related to the version of `python-casacore`, `numpy` and
+whether pre-built wheels are used.
+
+In practise it might be easier to leverage `conda` to install the appropriate
+`boost` and `casacore` libraries.
+
+A helpful script below may be of use.
+
+```
+
+BRANCH="main" # replace this with appropriate branch or tag
+DIR="flint_${BRANCH}"
+PYVERSION="3.12"
+
+mkdir "${DIR}" || exit
+cd "${DIR}" || exit
+
+
+git clone [email protected]:tjgalvin/flint.git && \
+        cd flint && \
+        git checkout "${BRANCH}"
+
+conda create -y  -n "${DIR}" python="${PYVERSION}" &&  \
+        source /home/$(whoami)/.bashrc && \
+        conda activate "${DIR}" && \
+        conda install -y -c conda-forge boost casacore && \
+        PIP_NO_BINARY="python-casacore" pip install -e .
+```
+
+This may set up an appropriate environment that is compatible with the
+containers currently being used.
+
+### The error
+
+The error looked something like this:
+
+```
+An unhandled exception occurred: FiledesIO::read /path/to/data.ms/table.mf - read returned a bad value
+```
+
 ## About
 
 This `flint` package is trying to get a minimum start-to-finish calibration and
@@ -113,15 +163,17 @@ the time of writing there are six containers for:
 - source peeling: `potatopeel` is a package that uses `wsclean`, `casa` and a
   customisable rule set to peel out troublesome annoying objects. Although it is
   a python installable and importable package, there are potential conflicts
-  with the `casatasks` and `python-casacore` modules that `flint` uses. See
+  with the `python-casacore` modules that `flint` uses. See
   [potatopeel's github repository for more information](https://gitlab.com/Sunmish/potato/-/tree/main)
 - linear mosaicing: The `linmos` task from `yandasoft` is used to perform linear
   mosaicing. Importanting this `linmos` is capable of using the ASKAP primary
   beam responses characterised through holography. `yandasoft` docker images
   [are available from the CSIRO dockerhub page.](https://hub.docker.com/r/csirocass/askapsoft).
 - self-calibration: `casa` is used to perform antenna-based self-calibration.
   Specifically the tasks `gaincal`, `applysolutions`, `cvel` and `mstransform`
-  are used throughout this process.
+  are used throughout this process. Careful selection of an appropriate CASA
+  version should be made to keep the `casacore` library in compatible state with
+  other components. Try the `docker://alecthomson/casa:ks9-5.8.0` image.
 
 ## Configuration based settings
 

flint/flagging.py

@@ -152,6 +152,8 @@ def nan_zero_extreme_flag_ms(
             logger.info(f"Setting {np.sum(flags)} DATA items to NaN.")
             tab.putcol(data_column, data)
 
+        tab.close()
+
     return ms
 
 

flint/ms.py

@@ -535,6 +535,7 @@ def rename_column_in_ms(
 
         logger.info(f"Renaming {original_column_name} to {new_column_name}")
         tab.renamecol(oldname=original_column_name, newname=new_column_name)
+        tab.flush()
 
     if update_tracked_column:
         ms = ms.with_options(column=new_column_name)
@@ -604,6 +605,8 @@ def preprocess_askap_ms(
             logger.info(f"Renaming {data_column} to {instrument_column}.")
             tab.renamecol(data_column, instrument_column)
 
+        tab.flush()
+
     logger.info("Correcting the field table. ")
     fix_ms_dir(ms=str(ms.path))
 
@@ -676,6 +679,7 @@ def copy_and_preprocess_casda_askap_ms(
             data_column in column_names and instrument_column not in column_names
         ), f"{ms.path} column names failed. {data_column=} {instrument_column=} {column_names=}"
         tab.renamecol(data_column, instrument_column)
+        tab.flush()
 
     logger.info("Correcting directions. ")
     fix_ms_dir(ms=str(ms.path))
@@ -768,6 +772,8 @@ def rename_ms_and_columns_for_selfcal(
             logger.info(f"Renaming {corrected_data} to DATA")
             tab.renamecol(corrected_data, data)
 
+        tab.flush()
+
     # This is a safe guard against my bad handling of the above / mutineers
     # There could be interplay with these columns when potato peel is used
     # as some MSs will have CORRECYED_DATA and others may not.

flint/selfcal/casa.py

@@ -221,6 +221,8 @@ def copy_and_clean_ms_casagain(
                 logger.info("Renaming CORRECTED_DATA to DATA. ")
                 tab.renamecol("CORRECTED_DATA", "DATA")
 
+            tab.flush()
+
     # Note that the out_ms_path needs to be set, even if the data  column is initially DATA.
     # Since casa expects DATA, we will force the column to be DATA with the expectation that
     # previous pirates in the lines above have dealt with the renaming.

pyproject.toml

@@ -16,10 +16,11 @@ include = [
 ]
 
 [tool.poetry.dependencies]
-python = "3.12"
+python = "^3.11"
 astropy = "*"
-numpy = "*"
-python-casacore = "*"
+numpy = "^1.26.0"
+python-casacore = "3.5.2" # newer version caused MS errors (maybe related to pre-built wheel)
+pydantic = "2.8.2" # version 2.9.0 caused prefect model validation error
 scipy = "*"
 spython = "^0.3.1"
 fixms = ">=0.2.6"