Skip to content

Fix esa_step dimension mismatch in CoDICE-Lo processing #2574

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Copilot wants to merge 5 commits into
base: dev
Choose a base branch
from
Draft

Fix esa_step dimension mismatch in CoDICE-Lo processing #2574

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ac73779
Initial plan
Copilot Jan 13, 2026
4d282b8
Fix esa_step dimension mismatch by using actual voltage data length
Copilot Jan 13, 2026
12215a5
Fix esa_step dimension mismatch in lo_angular and lo_priority modules
Copilot Jan 13, 2026
d6bca64
Complete esa_step dimension fix for all lo_* modules
Copilot Jan 13, 2026
f3f85bf
Use dynamic lookup table size instead of hardcoded 128 for energy val…
Copilot Jan 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 14 additions & 12 deletions imap_processing/codice/codice_l1a_lo_angular.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,12 +175,22 @@ def l1a_lo_angular(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
sci_lut_data, view_tab_obj.sensor, view_tab_obj.collapse_table
)

# ========== Get Voltage Data from LUT ===========
# Use plan id and plan step to get voltage data's table_number in ESA sweep table.
# Voltage data length varies by configuration (e.g., 104 or 128 steps)
esa_table_number = sci_lut_data["plan_tab"][f"({plan_id}, {plan_step})"][
"lo_stepping"
]
voltage_data = sci_lut_data["esa_sweep_tab"][f"{esa_table_number}"]

# Determine actual number of ESA steps from the voltage data
esa_steps = len(voltage_data)

# Reshape decompressed data to:
# (num_packets, num_species, esa_steps, 12, 5)
# 24 includes despinning spin sector. Then at later steps,
# we handle despinning.
num_packets = len(binary_data_list)
esa_steps = constants.NUM_ESA_STEPS
num_species = len(species_names)
species_data = np.array(decompressed_data, dtype=np.uint32).reshape(
num_packets, num_species, esa_steps, *collapsed_shape
Expand All @@ -190,14 +200,6 @@ def l1a_lo_angular(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
# ----------------
species_data = _despin_species_data(species_data, sci_lut_data, view_tab_obj)

# ========== Get Voltage Data from LUT ===========
# Use plan id and plan step to get voltage data's table_number in ESA sweep table.
# Voltage data is (128,)
esa_table_number = sci_lut_data["plan_tab"][f"({plan_id}, {plan_step})"][
"lo_stepping"
]
voltage_data = sci_lut_data["esa_sweep_tab"][f"{esa_table_number}"]

# ========= Get Epoch Time Data ===========
# Epoch center time and delta
epoch_center, deltas = get_codice_epoch_time(
Expand Down Expand Up @@ -234,12 +236,12 @@ def l1a_lo_angular(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
),
),
"esa_step": xr.DataArray(
np.arange(128),
np.arange(esa_steps),
dims=("esa_step",),
attrs=cdf_attrs.get_variable_attributes("esa_step", check_schema=False),
),
"esa_step_label": xr.DataArray(
np.arange(128).astype(str),
np.arange(esa_steps).astype(str),
dims=("esa_step",),
attrs=cdf_attrs.get_variable_attributes(
"esa_step_label", check_schema=False
Expand Down Expand Up @@ -303,7 +305,7 @@ def l1a_lo_angular(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
attrs=cdf_attrs.get_variable_attributes("data_quality"),
)
l1a_dataset["acquisition_time_per_step"] = xr.DataArray(
calculate_acq_time_per_step(sci_lut_data["lo_stepping_tab"]),
calculate_acq_time_per_step(sci_lut_data["lo_stepping_tab"])[:esa_steps],
dims=("esa_step",),
attrs=cdf_attrs.get_variable_attributes(
"acquisition_time_per_step", check_schema=False
Expand Down
8 changes: 5 additions & 3 deletions imap_processing/codice/codice_l1a_lo_counters_aggregated.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,14 @@ def l1a_lo_counters_aggregated(

# ========== Get Voltage Data from LUT ===========
# Use plan id and plan step to get voltage data's table_number in ESA sweep table.
# Voltage data is (128,)
# Voltage data length varies by configuration (e.g., 104 or 128 steps)
esa_table_number = sci_lut_data["plan_tab"][f"({plan_id}, {plan_step})"][
"lo_stepping"
]
voltage_data = sci_lut_data["esa_sweep_tab"][f"{esa_table_number}"]

# Determine actual number of ESA steps from the voltage data
esa_step = len(voltage_data)

# ========= Decompress and Reshape Data ===========
logical_source_id = "imap_codice_l1a_lo-counters-aggregated"
Expand All @@ -92,7 +95,6 @@ def l1a_lo_counters_aggregated(
# Dimensions
num_variables = len(non_reserved_keys)
spin_sector_pairs = non_reserved_variables["tcr"]
esa_step = constants.NUM_ESA_STEPS
# Decompress data using byte count information from decommed data
binary_data_list = unpacked_dataset["data"].values
byte_count_list = unpacked_dataset["byte_count"].values
Expand Down Expand Up @@ -201,7 +203,7 @@ def l1a_lo_counters_aggregated(
attrs=cdf_attrs.get_variable_attributes("data_quality"),
)
l1a_dataset["acquisition_time_per_step"] = xr.DataArray(
calculate_acq_time_per_step(sci_lut_data["lo_stepping_tab"]),
calculate_acq_time_per_step(sci_lut_data["lo_stepping_tab"])[:esa_step],
dims=("esa_step",),
attrs=cdf_attrs.get_variable_attributes(
"acquisition_time_per_step", check_schema=False
Expand Down
12 changes: 7 additions & 5 deletions imap_processing/codice/codice_l1a_lo_priority.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,11 +70,14 @@ def l1a_lo_priority(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:

# ========== Get Voltage Data from LUT ===========
# Use plan id and plan step to get voltage data's table_number in ESA sweep table.
# Voltage data is (128,)
# Voltage data length varies by configuration (e.g., 104 or 128 steps)
esa_table_number = sci_lut_data["plan_tab"][f"({plan_id}, {plan_step})"][
"lo_stepping"
]
voltage_data = sci_lut_data["esa_sweep_tab"][f"{esa_table_number}"]

# Determine actual number of ESA steps from the voltage data
esa_steps = len(voltage_data)

# ========= Get Epoch Time Data ===========
# Epoch center time and delta
Expand Down Expand Up @@ -122,7 +125,6 @@ def l1a_lo_priority(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
# Reshape decompressed data to in below for loop:
# (num_packets, num_species, esa_steps, collapse_shape[0](spin_sector))
num_species = len(species_names)
esa_steps = constants.NUM_ESA_STEPS
collapse_shape = get_collapse_pattern_shape(
sci_lut_data,
view_tab_obj.sensor,
Expand Down Expand Up @@ -160,12 +162,12 @@ def l1a_lo_priority(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
),
),
"esa_step": xr.DataArray(
np.arange(128),
np.arange(esa_steps),
dims=("esa_step",),
attrs=cdf_attrs.get_variable_attributes("esa_step", check_schema=False),
),
"esa_step_label": xr.DataArray(
np.arange(128).astype(str),
np.arange(esa_steps).astype(str),
dims=("esa_step",),
attrs=cdf_attrs.get_variable_attributes(
"esa_step_label", check_schema=False
Expand Down Expand Up @@ -217,7 +219,7 @@ def l1a_lo_priority(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
attrs=cdf_attrs.get_variable_attributes("data_quality"),
)
l1a_dataset["acquisition_time_per_step"] = xr.DataArray(
calculate_acq_time_per_step(sci_lut_data["lo_stepping_tab"]),
calculate_acq_time_per_step(sci_lut_data["lo_stepping_tab"])[:esa_steps],
dims=("esa_step",),
attrs=cdf_attrs.get_variable_attributes(
"acquisition_time_per_step", check_schema=False
Expand Down
26 changes: 14 additions & 12 deletions imap_processing/codice/codice_l1a_lo_species.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,24 +111,26 @@ def l1a_lo_species(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
sci_lut_data, view_tab_obj.sensor, view_tab_obj.collapse_table
)

# ========== Get Voltage Data from LUT ===========
# Use plan id and plan step to get voltage data's table_number in ESA sweep table.
# Voltage data length varies by configuration (e.g., 104 or 128 steps)
esa_table_number = sci_lut_data["plan_tab"][f"({plan_id}, {plan_step})"][
"lo_stepping"
]
voltage_data = sci_lut_data["esa_sweep_tab"][f"{esa_table_number}"]

# Determine actual number of ESA steps from the voltage data
esa_steps = len(voltage_data)

# Reshape decompressed data to:
# (num_packets, num_species, esa_steps, *collapsed_shape)
# where collapsed_shape is usually (1,) for Lo species.
num_packets = len(binary_data_list)
num_species = len(species_names)
esa_steps = constants.NUM_ESA_STEPS
species_data = np.array(decompressed_data, dtype=np.uint32).reshape(
num_packets, num_species, esa_steps, *collapsed_shape
)

# ========== Get Voltage Data from LUT ===========
# Use plan id and plan step to get voltage data's table_number in ESA sweep table.
# Voltage data is (128,)
esa_table_number = sci_lut_data["plan_tab"][f"({plan_id}, {plan_step})"][
"lo_stepping"
]
voltage_data = sci_lut_data["esa_sweep_tab"][f"{esa_table_number}"]

# ========= Get Epoch Time Data ===========
# Epoch center time and delta
epoch_center, deltas = get_codice_epoch_time(
Expand Down Expand Up @@ -165,12 +167,12 @@ def l1a_lo_species(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
),
),
"esa_step": xr.DataArray(
np.arange(128),
np.arange(esa_steps),
dims=("esa_step",),
attrs=cdf_attrs.get_variable_attributes("esa_step", check_schema=False),
),
"esa_step_label": xr.DataArray(
np.arange(128).astype(str),
np.arange(esa_steps).astype(str),
dims=("esa_step",),
attrs=cdf_attrs.get_variable_attributes(
"esa_step_label", check_schema=False
Expand Down Expand Up @@ -222,7 +224,7 @@ def l1a_lo_species(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset:
attrs=cdf_attrs.get_variable_attributes("data_quality"),
)
l1a_dataset["acquisition_time_per_step"] = xr.DataArray(
calculate_acq_time_per_step(sci_lut_data["lo_stepping_tab"]),
calculate_acq_time_per_step(sci_lut_data["lo_stepping_tab"])[:esa_steps],
dims=("esa_step",),
attrs=cdf_attrs.get_variable_attributes(
"acquisition_time_per_step", check_schema=False
Expand Down
17 changes: 10 additions & 7 deletions imap_processing/codice/codice_l2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,9 +312,12 @@ def compute_geometric_factors(
val: key for key, vals in HALF_SPIN_LUT.items() for val in vals
}

# Create a list of half_spin values corresponding to ESA steps (0 to 127)
# Determine the actual number of ESA steps from the dataset
num_esa_steps = len(dataset.esa_step) if "esa_step" in dataset.dims else 128

# Create a list of half_spin values corresponding to ESA steps (0 to num_esa_steps-1)
half_spin_values = np.array(
[esa_step_to_half_spin_map[step] for step in range(128)]
[esa_step_to_half_spin_map[step] for step in range(num_esa_steps)]
)
# Expand dimensions to compare each rgfo_half_spin value against
# all half_spin_values
Expand All @@ -324,11 +327,11 @@ def compute_geometric_factors(
# false (full mode)
modes = half_spin_values > rgfo_half_spin

# Get the geometric factors based on the modes
# Get the geometric factors based on the modes, slicing to match actual esa_steps
gf = np.where(
modes[:, :, np.newaxis], # Shape (epoch, esa_step, 1)
geometric_factor_lookup["reduced"], # Shape (1, esa_step, 24) - reduced mode
geometric_factor_lookup["full"], # Shape (1, esa_step, 24) - full mode
geometric_factor_lookup["reduced"][:num_esa_steps, :], # Shape (num_esa_steps, 24) - reduced mode
geometric_factor_lookup["full"][:num_esa_steps, :], # Shape (num_esa_steps, 24) - full mode
) # Shape: (epoch, esa_step, inst_az)

return xr.DataArray(gf, dims=("epoch", "esa_step", "inst_az"))
Expand Down Expand Up @@ -1114,8 +1117,8 @@ def process_lo_direct_events(dependencies: ProcessingInputCollection) -> xr.Data
energy_steps = l2_dataset["energy_step"].values.flatten()
# Create output array
kev = np.full(energy_steps.shape, np.nan, dtype=np.float64)
# Get only valid energy_steps between 0 and 128
valid_mask = (energy_steps >= 0) & (energy_steps < 128)
# Get only valid energy_steps (valid indices into the esa_kev lookup table)
valid_mask = (energy_steps >= 0) & (energy_steps < len(esa_kev))
kev[valid_mask] = esa_kev[energy_steps[valid_mask]]
# Reshape back to original shape
l2_dataset["energy_per_charge"] = (
Expand Down