[WIP] recovered from dockerfile after Gacrux crash

Dockerfile

@@ -0,0 +1,42 @@
+ARG NVIDIA_VERSION=11.4.3
+
+FROM nvidia/cuda:${NVIDIA_VERSION}-devel-ubuntu20.04
+
+ENV DEBIAN_FRONTEND="noninteractive"
+RUN apt-get update -y && \
+    apt-get -y install \
+            tzdata \
+            build-essential \
+            pkg-config \
+            cmake \
+            curl \
+            git \
+            lcov \
+            fontconfig \
+            libfreetype-dev \
+            libexpat1-dev \
+            libcfitsio-dev \
+            libhdf5-dev \
+            clang \
+            libfontconfig-dev \
+            && apt-get clean all \
+            && rm -rf /var/lib/apt/lists/*
+
+ARG RUST_VERSION=1.72
+ARG TARGET_CPU=x86-64
+# example: 70
+ARG CUDA_COMPUTE
+
+# Get Rust
+RUN mkdir -m755 /opt/rust /opt/cargo
+ENV RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/cargo PATH=/opt/cargo/bin:$PATH
+# set minimal rust version here to use a newer stable version
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain=$RUST_VERSION
+
+ADD . /mwa_hyperdrive
+WORKDIR /mwa_hyperdrive
+ENV CXX=/usr/bin/g++
+ENV CARGO_BUILD_RUSTFLAGS="-C target-cpu=${TARGET_CPU}"
+RUN [ -z "$CUDA_COMPUTE" ] || export HYPERDRIVE_CUDA_COMPUTE=${CUDA_COMPUTE}; \
+    cargo install --path . --no-default-features --features=cuda,plotting --locked \
+    && cargo clean

src/cli/common/beam/mod.rs

@@ -118,8 +118,15 @@ impl BeamArgs {
                 trace!("Attempting to use delays:");
                 match &dipole_delays {
                     Delays::Full(d) => {
-                        for row in d.outer_iter() {
-                            trace!("{row}");
+                        let mut last_row = None;
+                        for (i, row) in d.outer_iter().enumerate() {
+                            if let Some(last_row) = last_row {
+                                if row == last_row {
+                                    continue
+                                }
+                            }
+                            trace!("{i:03} {row}");
+                            last_row = Some(row);
                         }
                     }
                     Delays::Partial(d) => trace!("{d:?}"),
@@ -178,8 +185,15 @@ impl BeamArgs {
                 };
                 if let Some(dipole_gains) = dipole_gains.as_ref() {
                     trace!("Attempting to use dipole gains:");
-                    for row in dipole_gains.outer_iter() {
-                        trace!("{row}");
+                    let mut last_row = None;
+                    for (i, row) in dipole_gains.outer_iter().enumerate() {
+                        if let Some(last_row) = last_row {
+                            if row == last_row {
+                                continue
+                            }
+                        }
+                        trace!("{i:03} {row}");
+                        last_row = Some(row);
                     }
 
                     // Currently, the only way to have dipole gains other than

src/cli/common/input_vis/mod.rs

@@ -832,8 +832,9 @@ impl InputVisArgs {
                 r
             }
         };
+        let first_obs_ts = obs_context.timestamps.first();
         time_printer
-            .push_line(format!("First obs timestamp: {}", obs_context.timestamps.first()).into());
+            .push_line(format!("First obs timestamp: {} | {:.2}", first_obs_ts, first_obs_ts.to_gpst_seconds()).into());
         time_printer.push_block(vec![
             format!(
                 "Available timesteps: {}",
@@ -852,26 +853,34 @@ impl InputVisArgs {
         )
         .into()];
         match timesteps_to_use.as_slice() {
-            [t] => block.push(
-                format!(
-                    "Only timestamp (GPS): {:.2}",
-                    obs_context.timestamps[*t].to_gpst_seconds()
+            [t] => {
+                let only_use_ts = obs_context.timestamps[*t];
+                block.push(
+                    format!(
+                        "Only timestamp:      {} ({:+.2}s)",
+                        only_use_ts,
+                        only_use_ts.to_gpst_seconds() - first_obs_ts.to_gpst_seconds(),
+                    )
+                    .into(),
                 )
-                .into(),
-            ),
+            },
 
             [f, .., l] => {
+                let first_use_ts = obs_context.timestamps[*f];
                 block.push(
                     format!(
-                        "First timestamp (GPS): {:.2}",
-                        obs_context.timestamps[*f].to_gpst_seconds()
+                        "First timestamp:     {} ({:+.2}s)",
+                        first_use_ts,
+                        first_use_ts.to_gpst_seconds() - first_obs_ts.to_gpst_seconds(),
                     )
                     .into(),
                 );
+                let last_use_ts = obs_context.timestamps[*l];
                 block.push(
                     format!(
-                        "Last timestamp  (GPS): {:.2}",
-                        obs_context.timestamps[*l].to_gpst_seconds()
+                        "Last timestamp :     {} ({:+.2}s)",
+                        last_use_ts,
+                        last_use_ts.to_gpst_seconds() - first_obs_ts.to_gpst_seconds(),
                     )
                     .into(),
                 );

src/cli/peel/mod.rs

@@ -28,6 +28,7 @@ use crate::{
     params::{ModellingParams, PeelParams},
     unit_parsing::{parse_wavelength, WavelengthUnit, WAVELENGTH_FORMATS},
     HyperdriveError,
+    math::div_ceil,
 };
 
 const DEFAULT_OUTPUT_PEEL_FILENAME: &str = "hyperdrive_peeled.uvfits";
@@ -481,8 +482,8 @@ impl PeelArgs {
         let n_low_freqs = low_res_spw.get_all_freqs().len();
         let n_input_freqs = input_vis_params.spw.get_all_freqs().len();
         assert_eq!(
-            n_low_freqs * low_res_freq_average_factor.get(),
-            n_input_freqs,
+            n_low_freqs,
+            div_ceil(n_input_freqs, low_res_freq_average_factor.get()),
             "low chans (flagged+unflagged) {} * low_res_freq_average_factor {} != input chans (flagged+unflagged) {}. also iono_freq_average_factor {}",
             n_low_freqs,
             low_res_freq_average_factor.get(),
@@ -687,12 +688,15 @@ impl PeelArgs {
                 block.push(
                     format!(
                         "- low: {} kHz (averaging {}x)",
-                        low_res_spw.freq_res / 1e3 * low_res_freq_average_factor.get() as f64,
+                        low_res_spw.freq_res / 1e3,
                         low_res_freq_average_factor
                     )
                     .into(),
                 );
             }
+            block.push(
+                format!("- {num_passes} passes of {num_loops} loops at {convergence:.2} convergence").into()
+            );
             peel_printer.push_block(block);
         }
 

src/gpu/peel.cu

@@ -467,11 +467,12 @@ __global__ void reduce_freqs(
         // warning: gain is actually needed for model real to line up with vis real.
         iono_consts->gain *= 1. + convergence * (s_vm / s_mm - 1);
         // printf(
-        //     "iono: a %+6.4e b %+6.4e a_uu %+6.4e a_uv %+6.4e a_vv %+6.4e A_u %+6.4e A_v %+6.4e denom %+6.4e s_vm %+6.4e s_mm %+6.4e s_vm/s_mm %+6.4e \n",
+        //     "iono: a %+6.4e b %+6.4e a_uu %+6.4e a_uv %+6.4e a_vv %+6.4e A_u %+6.4e A_v %+6.4e denom %+6.4e s_vm %+6.4e s_mm %+6.4e s_vm/s_mm %+6.4e nf %d \n",
         //     iono_consts->alpha, iono_consts->beta,
         //     a_uu, a_uv, a_vv, A_u, A_v, denom,
         //     // iono_consts->gain,
-        //     s_vm, s_mm, s_vm / s_mm
+        //     s_vm, s_mm, s_vm / s_mm,
+        //     num_freqs
         // );
     }
 }
@@ -820,9 +821,10 @@ extern "C" const char *iono_loop(const JonesF32 *d_vis_residual, const float *d_
         // Sane?
         gpuMemcpy(iono_consts, d_iono_consts, sizeof(IonoConsts), gpuMemcpyDeviceToHost);
         if (iono_consts->gain < 0.0) {
-            iono_consts->alpha = 0.0;
-            iono_consts->beta = 0.0;
-            iono_consts->gain = 1.0;
+            // let's handle this in cpu world. This gets overwritten by the gpuMemcpy at the end of this function anyway
+            // iono_consts->alpha = 0.0;
+            // iono_consts->beta = 0.0;
+            // iono_consts->gain = 1.0;
             break;
         }
         // printf("iter %d: %.4e %.4e %.4e\n", iteration, iono_consts->alpha, iono_consts->beta, iono_consts->gain);

src/hyperdrive-peel.code-workspace

@@ -0,0 +1,20 @@
+{
+	"folders": [
+		{
+			"path": ".."
+		},
+		{
+			"path": "../../../../../data/dev/1226233968"
+		},
+		{
+			"path": "../../../../../data/dev/1090701368"
+		},
+		{
+			"path": "../../../../../data/dev/1094090416"
+		},
+		{
+			"path": "../../../../../data/dev/1096811152"
+		}
+	],
+	"settings": {}
+}

src/io/read/uvfits/mod.rs

@@ -1096,6 +1096,11 @@ impl VisRead for UvfitsReader {
             weights_fb,
             tile_baseline_flags,
         });
+        debug!(
+            "[uvfits read_crosses] npols {:?} fppol {:?}",
+            self.metadata.pols.num_pols(),
+            self.metadata.num_floats_per_pol
+        );
         match (
             self.metadata.pols.num_pols(),
             self.metadata.num_floats_per_pol,

src/math/mod.rs

@@ -42,7 +42,14 @@ pub(crate) fn average_epoch<I: IntoIterator<Item = Epoch>>(es: I) -> Epoch {
     Epoch::from_gpst_seconds(average).round(10.milliseconds())
 }
 
+// TODO (dev): a.div_ceil(b) would be better, but it's nightly:
+// https://doc.rust-lang.org/std/primitive.i32.html#method.div_ceil
+pub(crate) fn div_ceil(a: usize, b: usize) -> usize {
+    (a + b - 1) / b
+}
+
 /// Information on flagged tiles, baselines and maps to and from array indices.
+#[derive(Debug)]
 pub struct TileBaselineFlags {
     /// Map between a pair of tile numbers and its unflagged *cross-correlation*
     /// baseline index. e.g. If tiles 0 and 2 are flagged, (1, 3) maps to 0

src/params/input_vis.rs

@@ -65,6 +65,26 @@ pub(crate) struct InputVisParams {
     pub(crate) dut1: Duration,
 }
 
+// derive debug for InputVisParams
+impl std::fmt::Debug for InputVisParams {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("InputVisParams")
+            .field("vis_reader.get_input_data_type()", &self.vis_reader.get_input_data_type())
+            .field("solutions.is_some()", &self.solutions.is_some())
+            .field("timeblocks.timestamps", &self.timeblocks.iter().map(|t| t.timestamps.clone()).collect::<Vec<_>>())
+            .field("time_res.to_seconds()", &self.time_res.to_seconds())
+            .field("spw.chanblocks.len()", &self.spw.chanblocks.len())
+            .field("spw.chans_per_chanblock", &self.spw.chans_per_chanblock)
+            .field("spw.first_freq", &self.spw.first_freq)
+            .field("spw.flagged_chan_indices", &self.spw.flagged_chan_indices)
+            .field("tile_baseline_flags.flagged_tiles", &self.tile_baseline_flags.flagged_tiles)
+            .field("using_autos", &self.using_autos)
+            .field("ignore_weights", &self.ignore_weights)
+            .field("dut1.to_seconds()", &self.dut1.to_seconds())
+            .finish()
+    }
+}
+
 impl InputVisParams {
     pub(crate) fn get_obs_context(&self) -> &ObsContext {
         self.vis_reader.get_obs_context()
@@ -103,6 +123,8 @@ impl InputVisParams {
             assert_eq!(auto_weights_fb.dim(), avg_auto_vis_shape);
         }
 
+        debug!("[read_timeblock] {:?}", self);
+
         let averaging = timeblock.timestamps.len() > 1 || self.spw.chans_per_chanblock.get() > 1;
 
         if averaging {