CoDICE: Production preparation

imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml

@@ -1,7 +1,7 @@
 # <=== Useful Variables ===>
 int_fillval: &int_fillval -9223372036854775808
 uint32_fillval: &uint32_fillval 4294967295
-real_fillval: &real_fillval -1.0e+31
+real_fillval: &real_fillval -1.0E+31
 
 min_int: &min_int -9223372036854775808
 min_epoch: &min_epoch -315575942816000000
@@ -98,7 +98,7 @@ spin_sector_pairs:
 priority:
     CATDESC: Priority Level
     FIELDNAM: Priority Level
-    FILLVAL: -1
+    FILLVAL: *uint32_fillval
     FORMAT: I2
     LABLAXIS: " "
     SCALETYP: linear
@@ -172,8 +172,9 @@ energy_species_label:
     VAR_TYPE: metadata
 
 # <=== Dataset Variable Attributes ===>
-acquisition_time_per_step:
+acquisition_time_per_esa_step:
     CATDESC: Acquisition time for each step of energy
+    DEPEND_0: epoch
     DEPEND_1: esa_step
     FIELDNAM: Acquisition Time
     FILLVAL: *real_fillval
@@ -187,6 +188,7 @@ acquisition_time_per_step:
 
 half_spin_per_esa_step:
     CATDESC: Half spin number for each step of energy
+    DEPEND_0: epoch
     DEPEND_1: esa_step
     FIELDNAM: Half Spin Number
     FILLVAL: 255
@@ -315,8 +317,8 @@ sw_bias_gain_mode:
 counters_base: &counters_base
     DEPEND_0: epoch
     DISPLAY_TYPE: time_series
-    FILLVAL: *uint32_fillval
-    FORMAT: I7
+    FILLVAL: *real_fillval
+    FORMAT: F32.9
     SCALETYP: linear
     UNITS: counts
     VALIDMIN: 0
@@ -410,8 +412,8 @@ hi-species-attrs:
     DEPEND_1: energy_{species}
     DISPLAY_TYPE: time_series
     FIELDNAM: Species {species}
-    FILLVAL: *uint32_fillval
-    FORMAT: I7
+    FILLVAL: *real_fillval
+    FORMAT: F32.9
     LABL_PTR_1: energy_{species}_label
     SCALETYP: linear
     UNITS: counts
@@ -425,13 +427,13 @@ hi-species-unc-attrs:
     DEPEND_1: energy_{species}
     DISPLAY_TYPE: time_series
     FIELDNAM: Species {species}
-    FILLVAL: *uint32_fillval
-    FORMAT: I7
+    FILLVAL: *real_fillval
+    FORMAT: F32.9
     LABL_PTR_1: energy_{species}_label
     SCALETYP: linear
     UNITS: counts
     VALIDMAX: *max_uint32
-    VALIDMIN: 0
+    VALIDMIN: 0.0
     VAR_TYPE: data
 
 hi-energy-attrs:
@@ -468,11 +470,11 @@ hi-energy-delta-attrs:
 hi_priorities_attrs: &hi_priorities_default
     DEPEND_0: epoch
     DISPLAY_TYPE: time_series
-    FILLVAL: *uint32_fillval
-    FORMAT: I5
+    FILLVAL: *real_fillval
+    FORMAT: F32.9
     LABLAXIS: "events"
     UNITS: events
-    VALIDMAX: *max_uint32
+    VALIDMAX: *real_fillval
     VALIDMIN: 0
     VAR_TYPE: data
 
@@ -653,8 +655,8 @@ lo_counters_singles:
     DEPEND_3: inst_az
     DISPLAY_TYPE: time_series
     FIELDNAM: Rates - Single (APD)
-    FILLVAL: *uint32_fillval
-    FORMAT: I7
+    FILLVAL: *real_fillval
+    FORMAT: F32.9
     LABL_PTR_1: esa_step_label
     LABL_PTR_2: spin_sector_pairs_label
     LABL_PTR_3: inst_az_label
@@ -673,8 +675,8 @@ lo-angular-attrs:
     DEPEND_3: inst_az
     DISPLAY_TYPE: time_series
     FIELDNAM: "SW - {species}"
-    FILLVAL: *int_fillval
-    FORMAT: I7
+    FILLVAL: *real_fillval
+    FORMAT: F32.9
     LABL_PTR_1: esa_step_label
     LABL_PTR_2: spin_sector_label
     LABL_PTR_3: inst_az_label
@@ -691,8 +693,8 @@ lo-angular-unc-attrs:
     DEPEND_3: inst_az
     DISPLAY_TYPE: time_series
     FIELDNAM: "NSW - {species}"
-    FILLVAL: *int_fillval
-    FORMAT: I7
+    FILLVAL: *real_fillval
+    FORMAT: F19
     LABL_PTR_1: esa_step_label
     LABL_PTR_2: spin_sector_label
     LABL_PTR_3: inst_az_label
@@ -707,8 +709,8 @@ lo_priority_base: &lo_priority_base
     DEPEND_1: esa_step
     DEPEND_2: spin_sector
     DISPLAY_TYPE: time_series
-    FILLVAL: *uint32_fillval
-    FORMAT: I7
+    FILLVAL: *real_fillval
+    FORMAT: F32.9
     LABL_PTR_1: esa_step_label
     LABL_PTR_2: spin_sector_label
     SCALETYP: linear
@@ -762,8 +764,8 @@ lo-species-attrs:
     DEPEND_2: spin_sector
     DISPLAY_TYPE: time_series
     FIELDNAM: "{direction} - {species}"
-    FILLVAL: *int_fillval
-    FORMAT: I7
+    FILLVAL: *real_fillval
+    FORMAT: F32.9
     LABL_PTR_1: esa_step_label
     LABL_PTR_2: spin_sector_label
     UNITS: counts
@@ -778,8 +780,8 @@ lo-pui-species-attrs:
     DEPEND_2: spin_sector
     DISPLAY_TYPE: time_series
     FIELDNAM: "{direction} - {species}"
-    FILLVAL: *int_fillval
-    FORMAT: I7
+    FILLVAL: *real_fillval
+    FORMAT: F32.9
     LABL_PTR_1: esa_step_label
     LABL_PTR_2: spin_sector_label
     UNITS: counts

imap_processing/codice/codice_l1a.py

@@ -7,7 +7,6 @@
 
 from imap_processing import imap_module_directory
 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
-from imap_processing.codice.codice_l1a_de import l1a_direct_event
 from imap_processing.codice.codice_l1a_hi_counters_aggregated import (
     l1a_hi_counters_aggregated,
 )
@@ -87,11 +86,15 @@ def process_l1a(  # noqa: PLR0912
             logger.info("Processing Hi Sectored Species Counts")
             datasets.append(l1a_hi_sectored(datasets_by_apid[apid], lut_file))
         elif apid == CODICEAPID.COD_HI_PHA:
-            logger.info("Processing Direct Events for Hi")
-            datasets.append(l1a_direct_event(datasets_by_apid[apid], apid=apid))
+            logger.info("Skip processing Direct Events for Hi")
+            continue
+            # TODO: undo this in coming DE segmented work
+            # datasets.append(l1a_direct_event(datasets_by_apid[apid], apid=apid))
         elif apid == CODICEAPID.COD_LO_PHA:
-            logger.info("Processing Direct Events for Lo")
-            datasets.append(l1a_direct_event(datasets_by_apid[apid], apid=apid))
+            logger.info("Skip processing Direct Events for Lo")
+            continue
+            # TODO: undo this in coming DE segmented work
+            # datasets.append(l1a_direct_event(datasets_by_apid[apid], apid=apid))
         elif apid in [
             CODICEAPID.COD_LO_SW_PRIORITY_COUNTS,
             CODICEAPID.COD_LO_NSW_PRIORITY_COUNTS,

imap_processing/codice/codice_l1a_hi_counters_aggregated.py

@@ -101,6 +101,8 @@ def l1a_hi_counters_aggregated(
     counters_data = np.array(decompressed_data, dtype=np.uint32).reshape(
         -1, num_variables
     )
+    # Convert counters data to float
+    counters_data = counters_data.astype(np.float64)
 
     # ========= Get Epoch Time Data ===========
     # Epoch center time and delta

imap_processing/codice/codice_l1a_hi_counters_singles.py

@@ -96,7 +96,8 @@ def l1a_hi_counters_singles(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.
     counters_data = np.array(decompressed_data, dtype=np.uint32).reshape(
         -1, len(variable_names), inst_az
     )
-
+    # Convert counters data to float
+    counters_data = counters_data.astype(np.float64)
     # ========= Get Epoch Time Data ===========
     # Epoch center time and delta
     epoch_center, deltas = get_codice_epoch_time(

imap_processing/codice/codice_l1a_hi_omni.py

@@ -237,6 +237,8 @@ def l1a_hi_omni(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
         species_attrs = apply_replacements_to_attrs(
             species_attrs, {"species": species_name}
         )
+        # Convert to float
+        species_data = species_data.astype(np.float64)
         l1a_dataset[species_name] = xr.DataArray(
             species_data,
             dims=("epoch", f"energy_{species_name}"),

imap_processing/codice/codice_l1a_hi_priority.py

@@ -109,7 +109,7 @@ def l1a_hi_priority(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
     species_data = np.array(decompressed_data, dtype=np.uint32).reshape(
         num_packets, collapse_shape[1]
     )
-
+    species_data = species_data.astype(np.float64)
     # ========== Create CDF Dataset with Metadata ===========
     cdf_attrs = ImapCdfAttributes()
     cdf_attrs.add_instrument_global_attrs("codice")

imap_processing/codice/codice_l1a_hi_sectored.py

@@ -237,6 +237,7 @@ def l1a_hi_sectored(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
         species_attrs = apply_replacements_to_attrs(
             species_attrs, {"species": species_name}
         )
+        species_data = species_data.astype(np.float64)
         # Add DEPEND_2, DEPEND_3
         species_attrs["DEPEND_2"] = "spin_sector"
         species_attrs["LABL_PTR_2"] = "spin_sector_label"

imap_processing/codice/codice_l1a_lo_angular.py

@@ -67,28 +67,28 @@ def _despin_species_data(
     # index_to_position gets the position from collapse table. Eg.
     #   [1, 2, 3, 23, 24] for SW angular
     angular_position = index_to_position(sci_lut_data, 0, view_tab_obj.collapse_table)
-    orientation_a = pixel_orientation == "A"
-    orientation_b = pixel_orientation == "B"
+    orientation_a_indices = np.where(pixel_orientation == "A")[0]
+    orientation_b_indices = np.where(pixel_orientation == "B")[0]
 
     # Despin data based on orientation and angular position
     for pos_idx, position in enumerate(angular_position):
         if position <= 12:
             # Case 1: position 0-12, orientation A, append to first half
-            despun_data[:, :, orientation_a, :12, pos_idx] = species_data[
-                :, :, orientation_a, :, pos_idx
+            despun_data[:, :, orientation_a_indices, :12, pos_idx] = species_data[
+                :, :, orientation_a_indices, :, pos_idx
             ]
             # Case 2: position 13-24, orientation B, append to second half
-            despun_data[:, :, orientation_b, 12:, pos_idx] = species_data[
-                :, :, orientation_b, :, pos_idx
+            despun_data[:, :, orientation_b_indices, 12:, pos_idx] = species_data[
+                :, :, orientation_b_indices, :, pos_idx
             ]
         else:
             # Case 3: position 13-24, orientation A, append to second half
-            despun_data[:, :, orientation_a, 12:, pos_idx] = species_data[
-                :, :, orientation_a, :, pos_idx
+            despun_data[:, :, orientation_a_indices, 12:, pos_idx] = species_data[
+                :, :, orientation_a_indices, :, pos_idx
             ]
             # Case 4: position 0-12, orientation B, append to first half
-            despun_data[:, :, orientation_b, :12, pos_idx] = species_data[
-                :, :, orientation_b, :, pos_idx
+            despun_data[:, :, orientation_b_indices, :12, pos_idx] = species_data[
+                :, :, orientation_b_indices, :, pos_idx
             ]
 
     return despun_data
@@ -198,6 +198,35 @@ def l1a_lo_angular(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
     ]
     voltage_data = sci_lut_data["esa_sweep_tab"][f"{esa_table_number}"]
 
+    half_spin_per_esa_step = sci_lut_data["lo_stepping_tab"]["row_number"].get("data")
+    # TODO: Handle epoch dependent acquisition time and half spin per esa step
+    #   For now, just tile the same array for all epochs.
+    #   Eventually we may have data from a day where the LUT changed. If this is the
+    #  case, we need to split the data by epoch and assign different acquisition times
+    half_spin_per_esa_step = np.tile(
+        np.asarray(half_spin_per_esa_step),
+        (len(unpacked_dataset["acq_start_seconds"]), 1),
+    )
+    # Get acquisition time per esa step
+    acquisition_time_per_step = calculate_acq_time_per_step(
+        sci_lut_data["lo_stepping_tab"]
+    )
+    acquisition_time_per_step = np.tile(
+        np.asarray(acquisition_time_per_step),
+        (len(unpacked_dataset["acq_start_seconds"]), 1),
+    )
+    # For every energy after nso_half_spin, set data to fill values
+    nso_half_spin = unpacked_dataset["nso_half_spin"].values
+    nso_mask = half_spin_per_esa_step > nso_half_spin[:, np.newaxis]
+    species_mask = nso_mask[:, np.newaxis, :, np.newaxis, np.newaxis]
+    species_mask = np.broadcast_to(species_mask, species_data.shape)
+    species_data = species_data.astype(np.float64)
+    species_data[species_mask] = np.nan
+    # Set half_spin_per_esa_step to 63 which is the fill value
+    # half_spin_per_esa_step[nso_mask] = 63
+    # # Set acquisition time per esa step to nan where nso_mask is True
+    # acquisition_time_per_step[nso_mask] = np.nan
+
     # ========= Get Epoch Time Data ===========
     # Epoch center time and delta
     epoch_center, deltas = get_codice_epoch_time(
@@ -239,8 +268,11 @@ def l1a_lo_angular(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
                 attrs=cdf_attrs.get_variable_attributes("esa_step", check_schema=False),
             ),
             "half_spin_per_esa_step": xr.DataArray(
-                sci_lut_data["lo_stepping_tab"]["row_number"].get("data"),
-                dims=("esa_step",),
+                half_spin_per_esa_step,
+                dims=(
+                    "epoch",
+                    "esa_step",
+                ),
                 attrs=cdf_attrs.get_variable_attributes(
                     "half_spin_per_esa_step", check_schema=False
                 ),
@@ -309,11 +341,11 @@ def l1a_lo_angular(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
         dims=("epoch",),
         attrs=cdf_attrs.get_variable_attributes("data_quality"),
     )
-    l1a_dataset["acquisition_time_per_step"] = xr.DataArray(
-        calculate_acq_time_per_step(sci_lut_data["lo_stepping_tab"]),
-        dims=("esa_step",),
+    l1a_dataset["acquisition_time_per_esa_step"] = xr.DataArray(
+        acquisition_time_per_step,
+        dims=("epoch", "esa_step"),
         attrs=cdf_attrs.get_variable_attributes(
-            "acquisition_time_per_step", check_schema=False
+            "acquisition_time_per_esa_step", check_schema=False
         ),
     )